Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove StorageManager from stats dumps. #5297

Merged
merged 2 commits into from
Sep 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions test/src/test-cppapi-aggregates.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1156,8 +1156,7 @@ void CppAggregatesFx<T>::validate_data_var(

uint64_t get_stat(std::string name, std::string& stats) {
// Parse num_tiles_read from the stats.
std::string to_find =
"\"Context.StorageManager.Query.Reader." + name + "\": ";
std::string to_find = "\"Context.Query.Reader." + name + "\": ";
auto start_pos = stats.find(to_find);

if (start_pos != std::string::npos) {
Expand Down
14 changes: 6 additions & 8 deletions test/src/unit-capi-serialized_queries.cc
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,7 @@ struct SerializationFx {
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto loop_num =
counters->find("Context.StorageManager.Query.Reader.loop_num");
auto loop_num = counters->find("Context.Query.Reader.loop_num");
REQUIRE((loop_num != counters->end()));
REQUIRE(loop_num->second > 0);
}
Expand All @@ -112,8 +111,7 @@ struct SerializationFx {
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto loop_num =
counters->find("Context.StorageManager.Query.Writer.attr_num");
auto loop_num = counters->find("Context.Query.Writer.attr_num");
REQUIRE((loop_num != counters->end()));
REQUIRE(loop_num->second > 0);
}
Expand All @@ -125,11 +123,11 @@ struct SerializationFx {
// Note: if these checks fail, use Stats::dump(stdout) to validate counters
CHECK(
stats.find(
"\"Context.StorageManager.subSubarray.add_range_dim_0\": " +
"\"Context.subSubarray.add_range_dim_0\": " +
std::to_string(dim0_expected)) != std::string::npos);
CHECK(
stats.find(
"\"Context.StorageManager.subSubarray.add_range_dim_1\": " +
"\"Context.subSubarray.add_range_dim_1\": " +
std::to_string(dim1_expected)) != std::string::npos);
Stats::disable();
}
Expand All @@ -139,8 +137,8 @@ struct SerializationFx {
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto dowork_num = counters->find(
"Context.StorageManager.Query.Deletes.dowork.timer_count");
auto dowork_num =
counters->find("Context.Query.Deletes.dowork.timer_count");
REQUIRE((dowork_num != counters->end()));
REQUIRE(dowork_num->second > 0);
}
Expand Down
16 changes: 4 additions & 12 deletions test/src/unit-cppapi-array.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1697,14 +1697,10 @@ TEST_CASE(
// cpp_unit_array/__schema
// cpp_unit_array/__meta
// cpp_unit_array/__fragment_meta
CHECK(
stats.find("\"Context.StorageManager.VFS.read_ops_num\": 5") !=
std::string::npos);
CHECK(stats.find("\"Context.VFS.read_ops_num\": 5") != std::string::npos);

// Expect file_size on the fragment.
CHECK(
stats.find("\"Context.StorageManager.VFS.file_size_num\": 1") !=
std::string::npos);
CHECK(stats.find("\"Context.VFS.file_size_num\": 1") != std::string::npos);
}

TEST_CASE(
Expand Down Expand Up @@ -1757,14 +1753,10 @@ TEST_CASE(
// cpp_unit_array/__schema
// cpp_unit_array/__meta
// cpp_unit_array/__fragment_meta
CHECK(
stats.find("\"Context.StorageManager.VFS.read_ops_num\": 5") !=
std::string::npos);
CHECK(stats.find("\"Context.VFS.read_ops_num\": 5") != std::string::npos);

// Expect file_size on the fragment.
CHECK(
stats.find("\"Context.StorageManager.VFS.file_size_num\": 1") !=
std::string::npos);
CHECK(stats.find("\"Context.VFS.file_size_num\": 1") != std::string::npos);
}

TEST_CASE(
Expand Down
33 changes: 11 additions & 22 deletions test/src/unit-cppapi-consolidation-with-timestamps.cc
Original file line number Diff line number Diff line change
Expand Up @@ -652,8 +652,7 @@ TEST_CASE_METHOD(

// Make sure there was an internal loop on the reader.
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.internal_loop_num\": 2") !=
stats.find("\"Context.Query.Reader.internal_loop_num\": 2") !=
std::string::npos);

remove_sparse_array();
Expand Down Expand Up @@ -705,8 +704,7 @@ TEST_CASE_METHOD(

// Make sure there was an internal loop on the reader.
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.internal_loop_num\": 2") !=
stats.find("\"Context.Query.Reader.internal_loop_num\": 2") !=
std::string::npos);

remove_sparse_array();
Expand Down Expand Up @@ -1476,51 +1474,45 @@ TEST_CASE_METHOD(
// partially overlaps with the first 1:3 consolidated fragment.
read_sparse(a, dim1, dim2, stats, layout, 2);
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.num_tiles_read\": 4") !=
stats.find("\"Context.Query.Reader.num_tiles_read\": 4") !=
std::string::npos);

// Same but skip timestamps: 4 - 1 = 3, since 0:4 array fully overlaps with
// the first 1:3 consolidated fragment, so no need to check timestamps for
// cells.
read_sparse(a, dim1, dim2, stats, layout, 4);
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.num_tiles_read\": 3") !=
stats.find("\"Context.Query.Reader.num_tiles_read\": 3") !=
std::string::npos);

// Read 2 tiles for dim1, dim2, attr a1 and 1 for timestamps: 3 * 2 + 1 = 7
// since 0:6 array fully overlaps with the first 1:3, but only partially.
// overlaps with the second 5:7 consolidated fragment.
read_sparse(a, dim1, dim2, stats, layout, 6);
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.num_tiles_read\": 7") !=
stats.find("\"Context.Query.Reader.num_tiles_read\": 7") !=
std::string::npos);

// Read 2 tiles for dim1, dim2, attr a1 and skip timestamps: 3 * 2, since
// 0:8 array fully overlaps with both consolidateded fragments 1:3 and 5:7.
read_sparse(a, dim1, dim2, stats, layout, 8);
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.num_tiles_read\": 6") !=
stats.find("\"Context.Query.Reader.num_tiles_read\": 6") !=
std::string::npos);

// Read 2 tiles for dim1, dim2, attr a1 and timestamps: 4 * 2, since 3:5
// array partially overlaps with both consolidateded fragments 1:3 and 5:7.
reopen_sparse(a, dim1, dim2, stats, layout, 3, 5);
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.num_tiles_read\": 8") !=
stats.find("\"Context.Query.Reader.num_tiles_read\": 8") !=
std::string::npos);
}

SECTION("User requested timestamps") {
std::vector<uint64_t> timestamps(16);
read_sparse(a, dim1, dim2, stats, layout, 4, &timestamps);
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.num_tiles_read\": 4") !=
stats.find("\"Context.Query.Reader.num_tiles_read\": 4") !=
std::string::npos);
}

Expand All @@ -1538,8 +1530,7 @@ TEST_CASE_METHOD(
consolidate_sparse(vacuum);
read_sparse(a, dim1, dim2, stats, layout, 4);
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.num_tiles_read\": 4") !=
stats.find("\"Context.Query.Reader.num_tiles_read\": 4") !=
std::string::npos);
}

Expand All @@ -1557,17 +1548,15 @@ TEST_CASE_METHOD(
// have any.
read_sparse(a, dim1, dim2, stats, layout, 2);
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.num_tiles_read\": 3") !=
stats.find("\"Context.Query.Reader.num_tiles_read\": 3") !=
std::string::npos);

// Request timestamps - expect no timestamps getting read since fragment
// doesn't have any.
std::vector<uint64_t> timestamps(16);
read_sparse(a, dim1, dim2, stats, layout, 4, &timestamps);
CHECK(
stats.find(
"\"Context.StorageManager.Query.Reader.num_tiles_read\": 6") !=
stats.find("\"Context.Query.Reader.num_tiles_read\": 6") !=
std::string::npos);
}

Expand Down
9 changes: 3 additions & 6 deletions test/src/unit-dense-reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -531,8 +531,7 @@ void CDenseFx::read(
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto loop_num =
counters->find("Context.StorageManager.Query.Reader.internal_loop_num");
auto loop_num = counters->find("Context.Query.Reader.internal_loop_num");
CHECK(2 == loop_num->second);
}

Expand Down Expand Up @@ -660,8 +659,7 @@ void CDenseFx::read_strings(
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto loop_num =
counters->find("Context.StorageManager.Query.Reader.internal_loop_num");
auto loop_num = counters->find("Context.Query.Reader.internal_loop_num");
CHECK(2 == loop_num->second);
}

Expand Down Expand Up @@ -840,8 +838,7 @@ void CDenseFx::read_fixed_strings(
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto loop_num =
counters->find("Context.StorageManager.Query.Reader.internal_loop_num");
auto loop_num = counters->find("Context.Query.Reader.internal_loop_num");
CHECK(expected_num_loops == loop_num->second);
}

Expand Down
3 changes: 1 addition & 2 deletions test/src/unit-ordered-dim-label-reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1078,8 +1078,7 @@ TEST_CASE_METHOD(
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto loop_num =
counters->find("Context.StorageManager.Query.Reader.loop_num");
auto loop_num = counters->find("Context.Query.Reader.loop_num");
CHECK(2 == loop_num->second);
}

Expand Down
3 changes: 1 addition & 2 deletions test/src/unit-sparse-global-order-reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -823,8 +823,7 @@ TEST_CASE_METHOD(
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto loop_num =
counters->find("Context.StorageManager.Query.Reader.internal_loop_num");
auto loop_num = counters->find("Context.Query.Reader.internal_loop_num");
CHECK(5 == loop_num->second);

// Check incomplete query status.
Expand Down
6 changes: 2 additions & 4 deletions test/src/unit-sparse-unordered-with-dups-reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1059,8 +1059,7 @@ TEST_CASE_METHOD(
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto loop_num =
counters->find("Context.StorageManager.Query.Reader.internal_loop_num");
auto loop_num = counters->find("Context.Query.Reader.internal_loop_num");

if (one_frag) {
CHECK(1 == loop_num->second);
Expand Down Expand Up @@ -1174,8 +1173,7 @@ TEST_CASE_METHOD(
REQUIRE(stats != nullptr);
auto counters = stats->counters();
REQUIRE(counters != nullptr);
auto loop_num =
counters->find("Context.StorageManager.Query.Reader.internal_loop_num");
auto loop_num = counters->find("Context.Query.Reader.internal_loop_num");
CHECK(uint64_t(num_frags * 3) == loop_num->second);

// Check query status.
Expand Down
2 changes: 1 addition & 1 deletion tiledb/sm/stats/global_stats.h
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@
* As mentioned above, the Stats objects used under a Context form a tree.
* Each Stats object mentains a list of children Stats and a pointer to the
* parent Stats object.
* The Stats object created by ContextResources(named "Context.StorageManager")
* The Stats object created by ContextResources(named "Context")
* is the only Stats constructed in a standalone fashion using the Stats
* constructor, all the other objects under this root Stats are created via
* the Stats::create_child API.
Expand Down
5 changes: 1 addition & 4 deletions tiledb/sm/storage_manager/context.cc
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,7 @@ Context::Context(const Config& config)
logger_,
get_compute_thread_count(config),
get_io_thread_count(config),
// TODO: Remove `.StorageManager` from statistic names
// We're sticking with `Context.StorageManager` here because
// it is part of the public facing API.
"Context.StorageManager")
"Context")
, storage_manager_{resources_, logger_, config} {
/*
* Logger class is not yet C.41-compliant
Expand Down
Loading