Skip to content

Commit

Permalink
init commit
Browse files Browse the repository at this point in the history
  • Loading branch information
DimitrisStaratzis committed Aug 21, 2024
1 parent e51e6cd commit 51ae9a9
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 1 deletion.
38 changes: 38 additions & 0 deletions test/src/unit-cppapi-consolidation.cc
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,44 @@ TEST_CASE(
remove_array(array_name);
}

TEST_CASE(
"C++ API: Test consolidation with wrong fragment list",
"[cppapi][consolidation][dstara]") {
std::string array_name = "cppapi_consolidation";
remove_array(array_name);

create_array(array_name);
write_array(array_name, {1, 2}, {1, 2});
write_array(array_name, {3, 3}, {3});
CHECK(tiledb::test::num_fragments(array_name) == 2);

read_array(array_name, {1, 3}, {1, 2, 3});

Context ctx;
Config config;
config.set("sm.consolidation.buffer_size", "1000");

FragmentInfo fragment_info(ctx, array_name);
fragment_info.load();
std::string fragment_name1 = fragment_info.fragment_uri(0);
std::string fragment_name2 = fragment_info.fragment_uri(1);
std::string short_fragment_name1 =
fragment_name1.substr(fragment_name1.find_last_of('/') + 1);
std::string short_fragment_name2 =
fragment_name2.substr(fragment_name2.find_last_of('/') + 1);

const char* fragment_uris[2] = {
short_fragment_name1.c_str(), short_fragment_name2.c_str()};

REQUIRE_NOTHROW(
Array::consolidate(ctx, array_name, fragment_uris, 2, &config));
CHECK(tiledb::test::num_fragments(array_name) == 3);

read_array(array_name, {1, 3}, {1, 2, 3});

remove_array(array_name);
}

TEST_CASE(
"C++ API: Test consolidation with timestamp and max domain",
"[cppapi][consolidation][timestamp][maxdomain]") {
Expand Down
1 change: 0 additions & 1 deletion tiledb/sm/c_api/tiledb.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1773,7 +1773,6 @@ int32_t tiledb_array_consolidate_fragments(
const char** fragment_uris,
const uint64_t num_fragments,
tiledb_config_t* config) {
// Sanity checks

// Convert the list of fragments to a vector
std::vector<std::string> uris;
Expand Down
8 changes: 8 additions & 0 deletions tiledb/sm/consolidator/fragment_consolidator.cc
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,14 @@ Status FragmentConsolidator::consolidate_fragments(
std::to_string(fragment_uris.size()) + " required fragments.");
}

// In case we have a dense Array check that the fragments can be consolidated without data loss
if (array_for_reads->array_schema_latest().array_type() == ArrayType::DENSE) {
// pseudocode
// search every other fragment in this array
// if any of them overlaps in ranges AND its timestamp range falls between the range of the fragments to consolidate
// throw descriptive error
}

FragmentConsolidationWorkspace cw(consolidator_memory_tracker_);

// Consolidate the selected fragments
Expand Down
1 change: 1 addition & 0 deletions tiledb/sm/consolidator/fragment_consolidator.h
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
#include "tiledb/sm/array/array.h"
#include "tiledb/sm/consolidator/consolidator.h"
#include "tiledb/sm/misc/types.h"
#include "tiledb/sm/enums/array_type.h"
#include "tiledb/sm/storage_manager/context_resources.h"
#include "tiledb/sm/storage_manager/storage_manager_declaration.h"

Expand Down

0 comments on commit 51ae9a9

Please sign in to comment.