Skip to content

Commit

Permalink
Adding concurrent search versions of query count/time metrics
Browse files Browse the repository at this point in the history 
Signed-off-by: Jay Deng <jayd0104@gmail.com>
  • Loading branch information
@jed326
jed326 authored and Jay Deng committed Aug 29, 2023
1 parent 8cfde6c commit 969ee5f
Show file tree
Hide file tree
Showing 8 changed files with 205 additions and 7 deletions.
95 changes: 93 additions & 2 deletions rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,105 @@
"Help":
- skip:
version: " - 2.9.99"
reason: concurrent search stats were added in 2.10.0
features: node_selector
- do:
cat.shards:
help: true
node_selector:
version: "2.10.0 - "

- match:
$body: |
/^ index .+ \n
shard .+ \n
prirep .+ \n
state .+ \n
docs .+ \n
store .+ \n
ip .+ \n
id .+ \n
node .+ \n
sync_id .+ \n
unassigned.reason .+ \n
unassigned.at .+ \n
unassigned.for .+ \n
unassigned.details .+ \n
recoverysource.type .+ \n
completion.size .+ \n
fielddata.memory_size .+ \n
fielddata.evictions .+ \n
query_cache.memory_size .+ \n
query_cache.evictions .+ \n
flush.total .+ \n
flush.total_time .+ \n
get.current .+ \n
get.time .+ \n
get.total .+ \n
get.exists_time .+ \n
get.exists_total .+ \n
get.missing_time .+ \n
get.missing_total .+ \n
indexing.delete_current .+ \n
indexing.delete_time .+ \n
indexing.delete_total .+ \n
indexing.index_current .+ \n
indexing.index_time .+ \n
indexing.index_total .+ \n
indexing.index_failed .+ \n
merges.current .+ \n
merges.current_docs .+ \n
merges.current_size .+ \n
merges.total .+ \n
merges.total_docs .+ \n
merges.total_size .+ \n
merges.total_time .+ \n
refresh.total .+ \n
refresh.time .+ \n
refresh.external_total .+ \n
refresh.external_time .+ \n
refresh.listeners .+ \n
search.fetch_current .+ \n
search.fetch_time .+ \n
search.fetch_total .+ \n
search.open_contexts .+ \n
search.query_current .+ \n
search.query_time .+ \n
search.query_total .+ \n
search.concurrent_query_current .+ \n
search.concurrent_query_time .+ \n
search.concurrent_query_total .+ \n
search.scroll_current .+ \n
search.scroll_time .+ \n
search.scroll_total .+ \n
search.point_in_time_current .+ \n
search.point_in_time_time .+ \n
search.point_in_time_total .+ \n
segments.count .+ \n
segments.memory .+ \n
segments.index_writer_memory .+ \n
segments.version_map_memory .+ \n
segments.fixed_bitset_memory .+ \n
seq_no.max .+ \n
seq_no.local_checkpoint .+ \n
seq_no.global_checkpoint .+ \n
warmer.current .+ \n
warmer.total .+ \n
warmer.total_time .+ \n
path.data .+ \n
path.state .+ \n
$/
---
"Help":
- skip:
version: " - 2.3.99"
version: "2.3.99 - 2.10.0"
reason: point in time stats were added in 2.4.0
features: node_selector
- do:
cat.shards:
help: true
node_selector:
version: "2.4.0 - "
version: "2.4.0 - 2.9.99"

- match:
$body: |
Expand Down
48 changes: 48 additions & 0 deletions server/src/main/java/org/opensearch/index/search/stats/SearchStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ public static class Stats implements Writeable, ToXContentFragment {
private long queryTimeInMillis;
private long queryCurrent;

private long concurrentQueryCount;
private long concurrentQueryTimeInMillis;
private long concurrentQueryCurrent;

private long fetchCount;
private long fetchTimeInMillis;
private long fetchCurrent;
Expand All @@ -91,6 +95,9 @@ public Stats(
long queryCount,
long queryTimeInMillis,
long queryCurrent,
long concurrentQueryCount,
long concurrentQueryTimeInMillis,
long concurrentQueryCurrent,
long fetchCount,
long fetchTimeInMillis,
long fetchCurrent,
Expand All @@ -108,6 +115,10 @@ public Stats(
this.queryTimeInMillis = queryTimeInMillis;
this.queryCurrent = queryCurrent;

this.concurrentQueryCount = concurrentQueryCount;
this.concurrentQueryTimeInMillis = concurrentQueryTimeInMillis;
this.concurrentQueryCurrent = concurrentQueryCurrent;

this.fetchCount = fetchCount;
this.fetchTimeInMillis = fetchTimeInMillis;
this.fetchCurrent = fetchCurrent;
Expand All @@ -130,6 +141,12 @@ private Stats(StreamInput in) throws IOException {
queryTimeInMillis = in.readVLong();
queryCurrent = in.readVLong();

if (in.getVersion().onOrAfter(Version.V_2_10_0)) {
concurrentQueryCount = in.readVLong();
concurrentQueryTimeInMillis = in.readVLong();
concurrentQueryCurrent = in.readVLong();
}

fetchCount = in.readVLong();
fetchTimeInMillis = in.readVLong();
fetchCurrent = in.readVLong();
Expand All @@ -154,6 +171,10 @@ public void add(Stats stats) {
queryTimeInMillis += stats.queryTimeInMillis;
queryCurrent += stats.queryCurrent;

concurrentQueryCount += stats.concurrentQueryCount;
concurrentQueryTimeInMillis += stats.concurrentQueryTimeInMillis;
concurrentQueryCurrent += stats.concurrentQueryCurrent;

fetchCount += stats.fetchCount;
fetchTimeInMillis += stats.fetchTimeInMillis;
fetchCurrent += stats.fetchCurrent;
Expand All @@ -175,6 +196,9 @@ public void addForClosingShard(Stats stats) {
queryCount += stats.queryCount;
queryTimeInMillis += stats.queryTimeInMillis;

concurrentQueryCount += stats.concurrentQueryCount;
concurrentQueryTimeInMillis += stats.concurrentQueryTimeInMillis;

fetchCount += stats.fetchCount;
fetchTimeInMillis += stats.fetchTimeInMillis;

Expand Down Expand Up @@ -207,6 +231,22 @@ public long getQueryCurrent() {
return queryCurrent;
}

public long getConcurrentQueryCount() {
return concurrentQueryCount;
}

public TimeValue getConcurrentQueryTime() {
return new TimeValue(concurrentQueryTimeInMillis);
}

public long getConcurrentQueryTimeInMillis() {
return concurrentQueryTimeInMillis;
}

public long getConcurrentQueryCurrent() {
return concurrentQueryCurrent;
}

public long getFetchCount() {
return fetchCount;
}
Expand Down Expand Up @@ -306,6 +346,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.humanReadableField(Fields.QUERY_TIME_IN_MILLIS, Fields.QUERY_TIME, getQueryTime());
builder.field(Fields.QUERY_CURRENT, queryCurrent);

builder.field(Fields.CONCURRENT_QUERY_TOTAL, concurrentQueryCount);
builder.humanReadableField(Fields.CONCURRENT_QUERY_TIME_IN_MILLIS, Fields.CONCURRENT_QUERY_TIME, getConcurrentQueryTime());
builder.field(Fields.CONCURRENT_QUERY_CURRENT, concurrentQueryCurrent);

builder.field(Fields.FETCH_TOTAL, fetchCount);
builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime());
builder.field(Fields.FETCH_CURRENT, fetchCurrent);
Expand Down Expand Up @@ -430,6 +474,10 @@ static final class Fields {
static final String QUERY_TIME = "query_time";
static final String QUERY_TIME_IN_MILLIS = "query_time_in_millis";
static final String QUERY_CURRENT = "query_current";
static final String CONCURRENT_QUERY_TOTAL = "concurrent_query_total";
static final String CONCURRENT_QUERY_TIME = "concurrent_query_time";
static final String CONCURRENT_QUERY_TIME_IN_MILLIS = "concurrent_query_time_in_millis";
static final String CONCURRENT_QUERY_CURRENT = "concurrent_query_current";
static final String FETCH_TOTAL = "fetch_total";
static final String FETCH_TIME = "fetch_time";
static final String FETCH_TIME_IN_MILLIS = "fetch_time_in_millis";
Expand Down
17 changes: 17 additions & 0 deletions server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ public void onPreQueryPhase(SearchContext searchContext) {
statsHolder.suggestCurrent.inc();
} else {
statsHolder.queryCurrent.inc();
if (searchContext.shouldUseConcurrentSearch()) {
statsHolder.concurrentQueryCurrent.inc();
}
}
});
}
Expand All @@ -104,6 +107,10 @@ public void onFailedQueryPhase(SearchContext searchContext) {
} else {
statsHolder.queryCurrent.dec();
assert statsHolder.queryCurrent.count() >= 0;
if (searchContext.shouldUseConcurrentSearch()) {
statsHolder.concurrentQueryCurrent.dec();
assert statsHolder.concurrentQueryCurrent.count() >= 0;
}
}
});
}
Expand All @@ -119,6 +126,11 @@ public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
statsHolder.queryMetric.inc(tookInNanos);
statsHolder.queryCurrent.dec();
assert statsHolder.queryCurrent.count() >= 0;
if (searchContext.shouldUseConcurrentSearch()) {
statsHolder.concurrentQueryMetric.inc(tookInNanos);
statsHolder.concurrentQueryCurrent.dec();
assert statsHolder.concurrentQueryCurrent.count() >= 0;
}
}
});
}
Expand Down Expand Up @@ -206,6 +218,7 @@ public void onFreePitContext(ReaderContext readerContext) {
*/
static final class StatsHolder {
final MeanMetric queryMetric = new MeanMetric();
final MeanMetric concurrentQueryMetric = new MeanMetric();
final MeanMetric fetchMetric = new MeanMetric();
/* We store scroll statistics in microseconds because with nanoseconds we run the risk of overflowing the total stats if there are
* many scrolls. For example, on a system with 2^24 scrolls that have been executed, each executing for 2^10 seconds, then using
Expand All @@ -218,6 +231,7 @@ static final class StatsHolder {
final MeanMetric pitMetric = new MeanMetric();
final MeanMetric suggestMetric = new MeanMetric();
final CounterMetric queryCurrent = new CounterMetric();
final CounterMetric concurrentQueryCurrent = new CounterMetric();
final CounterMetric fetchCurrent = new CounterMetric();
final CounterMetric scrollCurrent = new CounterMetric();
final CounterMetric pitCurrent = new CounterMetric();
Expand All @@ -228,6 +242,9 @@ SearchStats.Stats stats() {
queryMetric.count(),
TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()),
queryCurrent.count(),
concurrentQueryMetric.count(),
TimeUnit.NANOSECONDS.toMillis(concurrentQueryMetric.sum()),
concurrentQueryCurrent.count(),
fetchMetric.count(),
TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()),
fetchCurrent.count(),
Expand Down
27 changes: 27 additions & 0 deletions server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,24 @@ protected Table getTableWithHeader(final RestRequest request) {
"sibling:pri;alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops"
);
table.addCell("pri.search.query_total", "default:false;text-align:right;desc:total query phase ops");
table.addCell(
"search.concurrent_query_current",
"sibling:pri;alias:sqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops"
);
table.addCell("pri.search.concurrent_query_current", "default:false;text-align:right;desc:current concurrent query phase ops");

table.addCell(
"search.concurrent_query_time",
"sibling:pri;alias:sqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase"
);
table.addCell("pri.search.concurrent_query_time", "default:false;text-align:right;desc:time spent in concurrent query phase");

table.addCell(
"search.concurrent_query_total",
"sibling:pri;alias:sqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total query phase ops"
);
table.addCell("pri.search.concurrent_query_total", "default:false;text-align:right;desc:total query phase ops");


table.addCell(
"search.scroll_current",
Expand Down Expand Up @@ -890,6 +908,15 @@ Table buildTable(
table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount());
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount());

table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent());
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent());

table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime());
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime());

table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount());
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount());

table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent());
table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent());

Expand Down
6 changes: 6 additions & 0 deletions server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,9 @@ protected Table getTableWithHeader(final RestRequest request) {
table.addCell("search.query_current", "alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops");
table.addCell("search.query_time", "alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase");
table.addCell("search.query_total", "alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops");
table.addCell("search.concurrent_query_current", "alias:sqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops");
table.addCell("search.concurrent_query_time", "alias:sqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase");
table.addCell("search.concurrent_query_total", "alias:sqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total concurrent query phase ops");
table.addCell("search.scroll_current", "alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts");
table.addCell(
"search.scroll_time",
Expand Down Expand Up @@ -529,6 +532,9 @@ Table buildTable(
table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCurrent());
table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryTime());
table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCount());
table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryCurrent());
table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryTime());
table.addCell(searchStats == null ? null : searchStats.getTotal().getConcurrentQueryCount());
table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCurrent());
table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollTime());
table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCount());
Expand Down
6 changes: 6 additions & 0 deletions server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,9 @@ protected Table getTableWithHeader(final RestRequest request) {
table.addCell("search.query_current", "alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops");
table.addCell("search.query_time", "alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase");
table.addCell("search.query_total", "alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops");
table.addCell("search.concurrent_query_current", "alias:sqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops");
table.addCell("search.concurrent_query_time", "alias:sqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase");
table.addCell("search.concurrent_query_total", "alias:sqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total concurrent query phase ops");
table.addCell("search.scroll_current", "alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts");
table.addCell(
"search.scroll_time",
Expand Down Expand Up @@ -399,6 +402,9 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getQueryCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getConcurrentQueryCount()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCurrent()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollTime()));
table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getScrollCount()));
Expand Down
Loading

0 comments on commit 969ee5f

Please sign in to comment.