From c14385f344519923506e46bd1460d376aa83b361 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 2 Aug 2024 11:40:43 -0700 Subject: [PATCH 01/29] Query grouping framework and group by query similarity Signed-off-by: Siddhant Deshmukh --- .../plugin/insights/QueryInsightsPlugin.java | 1 + .../core/listener/QueryInsightsListener.java | 25 +- .../core/service/QueryGroupingService.java | 200 +++++ .../core/service/QueryInsightsService.java | 48 +- .../insights/core/service/QueryShape.java | 34 + .../core/service/TopQueriesService.java | 25 +- .../SearchQueryAggregationCategorizer.java | 5 +- .../categorizer/SearchQueryCategorizer.java | 9 +- .../SearchQueryCategorizingVisitor.java | 7 +- .../categorizer/SearchQueryCounters.java | 15 +- .../insights/rules/model/Attribute.java | 8 +- .../insights/rules/model/DimensionType.java | 20 + .../insights/rules/model/GroupingType.java | 62 ++ .../insights/rules/model/Measurement.java | 162 ++++ .../insights/rules/model/MetricType.java | 24 +- .../rules/model/SearchQueryRecord.java | 54 +- .../settings/QueryInsightsSettings.java | 15 +- .../insights/QueryInsightsPluginTests.java | 1 + .../insights/QueryInsightsTestUtils.java | 69 +- .../service/QueryGroupingServiceTests.java | 753 ++++++++++++++++++ .../service/QueryInsightsServiceTests.java | 87 ++ .../core/service/TopQueriesServiceTests.java | 36 + .../top_queries/TopQueriesResponseTests.java | 2 +- 23 files changed, 1608 insertions(+), 54 deletions(-) create mode 100644 src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java create mode 100644 src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java create mode 100644 src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java create mode 100644 src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java create mode 100644 src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java create mode 100644 src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java diff --git a/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java b/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java index 78ab22e..60a3a9a 100644 --- a/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java +++ b/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java @@ -130,6 +130,7 @@ public List> getSettings() { QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE, QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE, QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS, + QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY, QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING ); } diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index 00bf997..9ed8508 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -12,6 +12,7 @@ import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNEnabledSetting; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNSizeSetting; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNWindowSizeSetting; +import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY; import java.util.Collections; import java.util.HashMap; @@ -31,7 +32,9 @@ import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.plugin.insights.core.service.QueryInsightsService; +import org.opensearch.plugin.insights.core.service.QueryShape; import org.opensearch.plugin.insights.rules.model.Attribute; +import org.opensearch.plugin.insights.rules.model.Measurement; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; import org.opensearch.tasks.Task; @@ -104,6 +107,13 @@ public QueryInsightsListener( clusterService.getClusterSettings() .addSettingsUpdateConsumer(SEARCH_QUERY_METRICS_ENABLED_SETTING, v -> setSearchQueryMetricsEnabled(v)); setSearchQueryMetricsEnabled(clusterService.getClusterSettings().get(SEARCH_QUERY_METRICS_ENABLED_SETTING)); + + clusterService.getClusterSettings() + .addSettingsUpdateConsumer( + TOP_N_QUERIES_GROUP_BY, + v -> this.queryInsightsService.validateAndSetGrouping(v) + ); + this.queryInsightsService.validateAndSetGrouping(clusterService.getClusterSettings().get(TOP_N_QUERIES_GROUP_BY)); } /** @@ -191,25 +201,31 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final final SearchRequest request = context.getRequest(); try { - Map measurements = new HashMap<>(); + Map measurements = new HashMap<>(); if (shouldCollect(MetricType.LATENCY)) { measurements.put( MetricType.LATENCY, - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos()) + new Measurement(MetricType.LATENCY, + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos())) ); } if (shouldCollect(MetricType.CPU)) { measurements.put( MetricType.CPU, - tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getCpuTimeInNanos()).mapToLong(Long::longValue).sum() + new Measurement(MetricType.CPU, + tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getCpuTimeInNanos()).mapToLong(Long::longValue).sum()) ); } if (shouldCollect(MetricType.MEMORY)) { measurements.put( MetricType.MEMORY, - tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getMemoryInBytes()).mapToLong(Long::longValue).sum() + new Measurement(MetricType.MEMORY, + tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getMemoryInBytes()).mapToLong(Long::longValue).sum()) ); } + // TODO: Use David's QueryShape library to get the query shape hashcode + final QueryShape queryShape = new QueryShape(request.source()); + Map attributes = new HashMap<>(); attributes.put(Attribute.SEARCH_TYPE, request.searchType().toString().toLowerCase(Locale.ROOT)); attributes.put(Attribute.SOURCE, request.source()); @@ -217,6 +233,7 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final attributes.put(Attribute.INDICES, request.indices()); attributes.put(Attribute.PHASE_LATENCY_MAP, searchRequestContext.phaseTookMap()); attributes.put(Attribute.TASK_RESOURCE_USAGES, tasksResourceUsages); + attributes.put(Attribute.QUERY_HASHCODE, queryShape.hashCode()); Map labels = new HashMap<>(); // Retrieve user provided label if exists diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java new file mode 100644 index 0000000..243feaa --- /dev/null +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java @@ -0,0 +1,200 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.insights.core.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.collect.Tuple; +import org.opensearch.plugin.insights.rules.model.Attribute; +import org.opensearch.plugin.insights.rules.model.DimensionType; +import org.opensearch.plugin.insights.rules.model.GroupingType; +import org.opensearch.plugin.insights.rules.model.MetricType; +import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.PriorityQueue; + +/** + * Handles grouping of search queries based on the GroupingType for the MetricType + */ +public class QueryGroupingService { + + private static final Logger log = LogManager.getLogger(QueryGroupingService.class); + private GroupingType groupingType; + private MetricType metricType; + + private DimensionType dimensionType; + private Map> groupIdToAggSearchQueryRecord; + private PriorityQueue minHeapTopQueriesStore; + private PriorityQueue maxHeapQueryStore; + + private int topNSize; + + public QueryGroupingService(MetricType metricType, GroupingType groupingType, DimensionType dimensionType, PriorityQueue topQueriesStore, int topNSize) { + this.groupingType = groupingType; + this.metricType = metricType; + this.dimensionType = dimensionType; + this.groupIdToAggSearchQueryRecord = new HashMap<>(); + this.minHeapTopQueriesStore = topQueriesStore; + this.maxHeapQueryStore = new PriorityQueue<>((a, b) -> SearchQueryRecord.compare(b, a, metricType)); + + this.topNSize = topNSize; + } + + /** + * Add query to the group based on the GroupType setting. + * The grouping of metrics will be stored within the searchQueryRecord. + * @param searchQueryRecord record + * @return return the search query record that represents the group + */ + public SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord) { + if (groupingType == GroupingType.NONE) { + throw new IllegalArgumentException("Do not use addQueryToGroup when GroupingType is None"); + } + SearchQueryRecord aggregateSearchQueryRecord; + String groupId = getGroupingId(searchQueryRecord); + + // New group + if (!groupIdToAggSearchQueryRecord.containsKey(groupId)) { + aggregateSearchQueryRecord = searchQueryRecord; + aggregateSearchQueryRecord.setGroupingId(groupId); + aggregateSearchQueryRecord.setMeasurementDimension(metricType, dimensionType); + addToMinPQ(aggregateSearchQueryRecord, groupId); + } + // Old group + else { + aggregateSearchQueryRecord = groupIdToAggSearchQueryRecord.get(groupId).v1(); + boolean isPresentInMinPQ = groupIdToAggSearchQueryRecord.get(groupId).v2(); + if (isPresentInMinPQ) { + updateToMinPQ(searchQueryRecord, aggregateSearchQueryRecord, groupId); + } else { + updateToMaxPQ(searchQueryRecord, aggregateSearchQueryRecord, groupId); + } + } + return aggregateSearchQueryRecord; + } + + private void addToMinPQ(SearchQueryRecord searchQueryRecord, String groupId) { + minHeapTopQueriesStore.add(searchQueryRecord); + groupIdToAggSearchQueryRecord.put(groupId, new Tuple<>(searchQueryRecord, true)); + + while (minHeapTopQueriesStore.size() > topNSize) { + SearchQueryRecord recordMovedFromMinToMax = minHeapTopQueriesStore.poll(); + maxHeapQueryStore.add(recordMovedFromMinToMax); + groupIdToAggSearchQueryRecord.put(recordMovedFromMinToMax.getGroupingId(), new Tuple<>(recordMovedFromMinToMax, false)); + } + } + + private void updateToMaxPQ(SearchQueryRecord searchQueryRecord, SearchQueryRecord aggregateSearchQueryRecord, String groupId) { + maxHeapQueryStore.remove(aggregateSearchQueryRecord); + Number measurementToAdd = searchQueryRecord.getMeasurement(metricType); + aggregateSearchQueryRecord.addMeasurement(metricType, measurementToAdd); + + addToMinPQ(aggregateSearchQueryRecord, groupId); + + } + + private void updateToMinPQ(SearchQueryRecord searchQueryRecord, SearchQueryRecord aggregateSearchQueryRecord, String groupId) { + minHeapTopQueriesStore.remove(aggregateSearchQueryRecord); + Number measurementToAdd = searchQueryRecord.getMeasurement(metricType); + aggregateSearchQueryRecord.addMeasurement(metricType, measurementToAdd); + + if (maxHeapQueryStore.size() > 0) { + addToMaxPQ(aggregateSearchQueryRecord, groupId); + } else { + addToMinPQ(aggregateSearchQueryRecord, groupId); + } + } + + private void addToMaxPQ(SearchQueryRecord aggregateSearchQueryRecord, String groupId) { + maxHeapQueryStore.add(aggregateSearchQueryRecord); + groupIdToAggSearchQueryRecord.put(groupId, new Tuple<>(aggregateSearchQueryRecord, false)); + + while (minHeapTopQueriesStore.size() < topNSize && !maxHeapQueryStore.isEmpty()) { + SearchQueryRecord recordMovedFromMaxToMin = maxHeapQueryStore.poll(); + minHeapTopQueriesStore.add(recordMovedFromMaxToMin); + groupIdToAggSearchQueryRecord.put(recordMovedFromMaxToMin.getGroupingId(), new Tuple<>(recordMovedFromMaxToMin, true)); + } + } + + /** + * Drain the internal grouping. Needs to be performed after every window. + */ + public void drain() { + log.debug("Number of groups for the current window is " + numberOfGroups()); + groupIdToAggSearchQueryRecord.clear(); + maxHeapQueryStore.clear(); + minHeapTopQueriesStore.clear(); + } + + /** + * Gives the number of groups as part of the current grouping. + * @return number of groups + */ + int numberOfGroups() { + return groupIdToAggSearchQueryRecord.size(); + } + + /** + * Gives the number of groups that are part of the top groups + * @return number of top groups + */ + int numberOfTopGroups() { + return minHeapTopQueriesStore.size(); + } + + /** + * Set Grouping Type + * @param newGroupingType grouping type + */ + public void setGroupingType(GroupingType newGroupingType) { + if (this.groupingType != newGroupingType) { + this.groupingType = newGroupingType; + drain(); + } + } + + public GroupingType getGroupingType() { + return groupingType; + } + + /** + * Get groupingId. This should be query hashcode for SIMILARITY grouping and user_id for USER_ID grouping. + * @param searchQueryRecord record + * @return Grouping Id + */ + private String getGroupingId(SearchQueryRecord searchQueryRecord) { + switch (groupingType) { + case SIMILARITY: + return searchQueryRecord.getAttributes().get(Attribute.QUERY_HASHCODE).toString(); + case NONE: + throw new IllegalArgumentException("Should not try to group queries if grouping type is NONE"); + default: + throw new IllegalArgumentException("The following grouping type is not supported : " + groupingType); + } + } + + /** + * Update Top N size + * @param newSize new size + */ + public void updateTopNSize(int newSize) { + this.topNSize = newSize; + } +} + + + + + + + + diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java index 75dea9e..c6b9dcf 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java @@ -9,6 +9,7 @@ package org.opensearch.plugin.insights.core.service; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getExporterSettings; +import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_GROUPING_TYPE; import java.io.IOException; import java.util.ArrayList; @@ -27,6 +28,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory; import org.opensearch.plugin.insights.core.service.categorizer.SearchQueryCategorizer; +import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; import org.opensearch.plugin.insights.settings.QueryInsightsSettings; @@ -73,6 +75,11 @@ public class QueryInsightsService extends AbstractLifecycleComponent { */ final QueryInsightsExporterFactory queryInsightsExporterFactory; + /** + * Flags for enabling insight data grouping for different metric types + */ + private GroupingType groupingType; + private volatile boolean searchQueryMetricsEnabled; private SearchQueryCategorizer searchQueryCategorizer; @@ -112,16 +119,17 @@ public QueryInsightsService( this.searchQueryCategorizer = SearchQueryCategorizer.getInstance(metricsRegistry); this.enableSearchQueryMetricsFeature(false); + this.groupingType = DEFAULT_GROUPING_TYPE; } /** * Ingest the query data into in-memory stores * * @param record the record to ingest - * @return SearchQueryRecord + * @return true/false */ public boolean addRecord(final SearchQueryRecord record) { - boolean shouldAdd = searchQueryMetricsEnabled; + boolean shouldAdd = isSearchQueryMetricsFeatureEnabled() || isGroupingEnabled(); if (!shouldAdd) { for (Map.Entry entry : topQueriesServices.entrySet()) { if (!enableCollect.get(entry.getKey())) { @@ -185,6 +193,33 @@ public void enableCollection(final MetricType metricType, final boolean enable) this.topQueriesServices.get(metricType).setEnabled(enable); } + /** + * Set grouping based on the given GroupingType setting for the given metric type + * + * @param groupingTypeSetting grouping type setting + */ + public void validateAndSetGrouping(final String groupingTypeSetting) { + GroupingType newGroupingType = GroupingType.getGroupingTypeFromSettingAndValidate(groupingTypeSetting); + GroupingType oldGroupingType = groupingType; + + if (oldGroupingType != newGroupingType) { + groupingType = newGroupingType; + + for (MetricType metricType : MetricType.allMetricTypes()) { + this.topQueriesServices.get(metricType).setGrouping(newGroupingType); + } + } + } + + /** + * Get the grouping type based on the metricType + * @return GroupingType + */ + + public GroupingType getGrouping() { + return groupingType; + } + /** * Get if the Query Insights data collection is enabled for a MetricType * @@ -226,9 +261,18 @@ public boolean isSearchQueryMetricsFeatureEnabled() { return this.searchQueryMetricsEnabled; } + /** + * Is grouping feature enabled and grouping not NONE + * @return boolean + */ + public boolean isGroupingEnabled() { + return this.groupingType != GroupingType.NONE; + } + /** * Enable/Disable search query metrics feature. * @param enable enable/disable search query metrics feature + * Stops query insights service if no features enabled */ public void enableSearchQueryMetricsFeature(boolean enable) { searchQueryMetricsEnabled = enable; diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java new file mode 100644 index 0000000..5ff9398 --- /dev/null +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.insights.core.service; + +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.search.aggregations.AggregatorFactories; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.sort.SortBuilder; + +import java.util.List; +import java.util.Objects; + +public class QueryShape { + QueryBuilder query; + AggregatorFactories.Builder aggregations; + List> sorts; + + public QueryShape(SearchSourceBuilder source) { + query = source.query(); + aggregations = source.aggregations(); + sorts = source.sorts(); + } + + @Override + public int hashCode() { + return Objects.hash(query, aggregations, sorts); + } +} diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index b79e4e7..13f916a 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -93,6 +93,8 @@ public class TopQueriesService { */ private QueryInsightsExporter exporter; + private QueryGroupingService queryGroupingService; + TopQueriesService( final MetricType metricType, final ThreadPool threadPool, @@ -109,6 +111,7 @@ public class TopQueriesService { topQueriesStore = new PriorityQueue<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot = new AtomicReference<>(new ArrayList<>()); topQueriesHistorySnapshot = new AtomicReference<>(new ArrayList<>()); + queryGroupingService = new QueryGroupingService(metricType, QueryInsightsSettings.DEFAULT_GROUPING_TYPE, DimensionType.AVERAGE, topQueriesStore, topNSize); } /** @@ -118,6 +121,7 @@ public class TopQueriesService { */ public void setTopNSize(final int topNSize) { this.topNSize = topNSize; + this.queryGroupingService.updateTopNSize(topNSize); } /** @@ -169,6 +173,10 @@ public void setWindowSize(final TimeValue windowSize) { this.windowStart = -1L; } + public void setGrouping(final GroupingType groupingType) { + queryGroupingService.setGroupingType(groupingType); + } + /** * Validate if the window size is valid, based on internal constrains. * @@ -306,10 +314,16 @@ void consumeRecords(final List records) { } private void addToTopNStore(final List records) { - topQueriesStore.addAll(records); - // remove top elements for fix sizing priority queue - while (topQueriesStore.size() > topNSize) { - topQueriesStore.poll(); + if (queryGroupingService.getGroupingType() != GroupingType.NONE) { + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + } else { + topQueriesStore.addAll(records); + // remove top elements for fix sizing priority queue + while (topQueriesStore.size() > topNSize) { + topQueriesStore.poll(); + } } } @@ -329,6 +343,9 @@ private void rotateWindowIfNecessary(final long newWindowStart) { } topQueriesHistorySnapshot.set(history); topQueriesStore.clear(); + if (queryGroupingService.getGroupingType() != GroupingType.NONE) { + queryGroupingService.drain(); + } topQueriesCurrentSnapshot.set(new ArrayList<>()); windowStart = newWindowStart; // export to the configured sink diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryAggregationCategorizer.java b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryAggregationCategorizer.java index 7ed861f..534d067 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryAggregationCategorizer.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryAggregationCategorizer.java @@ -10,6 +10,7 @@ import java.util.Collection; import java.util.Map; +import org.opensearch.plugin.insights.rules.model.Measurement; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.PipelineAggregationBuilder; @@ -39,14 +40,14 @@ public SearchQueryAggregationCategorizer(SearchQueryCounters searchQueryCounters */ public void incrementSearchQueryAggregationCounters( Collection aggregatorFactories, - Map measurements + Map measurements ) { for (AggregationBuilder aggregationBuilder : aggregatorFactories) { incrementCountersRecursively(aggregationBuilder, measurements); } } - private void incrementCountersRecursively(AggregationBuilder aggregationBuilder, Map measurements) { + private void incrementCountersRecursively(AggregationBuilder aggregationBuilder, Map measurements) { // Increment counters for the current aggregation String aggregationType = aggregationBuilder.getType(); searchQueryCounters.incrementAggCounter(1, Tags.create().addTag(AGGREGATION_TYPE_TAG, aggregationType), measurements); diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCategorizer.java b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCategorizer.java index af89a37..df3cb7b 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCategorizer.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCategorizer.java @@ -15,6 +15,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilderVisitor; import org.opensearch.plugin.insights.rules.model.Attribute; +import org.opensearch.plugin.insights.rules.model.Measurement; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; import org.opensearch.search.aggregations.AggregatorFactories; @@ -81,7 +82,7 @@ public void consumeRecords(List records) { */ public void categorize(SearchQueryRecord record) { SearchSourceBuilder source = (SearchSourceBuilder) record.getAttributes().get(Attribute.SOURCE); - Map measurements = record.getMeasurements(); + Map measurements = record.getMeasurements(); incrementQueryTypeCounters(source.query(), measurements); incrementQueryAggregationCounters(source.aggregations(), measurements); @@ -93,7 +94,7 @@ public void categorize(SearchQueryRecord record) { } } - private void incrementQuerySortCounters(List> sorts, Map measurements) { + private void incrementQuerySortCounters(List> sorts, Map measurements) { if (sorts != null && sorts.size() > 0) { for (SortBuilder sortBuilder : sorts) { String sortOrder = sortBuilder.order().toString(); @@ -102,7 +103,7 @@ private void incrementQuerySortCounters(List> sorts, Map measurements) { + private void incrementQueryAggregationCounters(AggregatorFactories.Builder aggregations, Map measurements) { if (aggregations == null) { return; } @@ -110,7 +111,7 @@ private void incrementQueryAggregationCounters(AggregatorFactories.Builder aggre searchQueryAggregationCategorizer.incrementSearchQueryAggregationCounters(aggregations.getAggregatorFactories(), measurements); } - private void incrementQueryTypeCounters(QueryBuilder topLevelQueryBuilder, Map measurements) { + private void incrementQueryTypeCounters(QueryBuilder topLevelQueryBuilder, Map measurements) { if (topLevelQueryBuilder == null) { return; } diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCategorizingVisitor.java b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCategorizingVisitor.java index f6addca..98c9bf6 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCategorizingVisitor.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCategorizingVisitor.java @@ -12,6 +12,7 @@ import org.apache.lucene.search.BooleanClause; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilderVisitor; +import org.opensearch.plugin.insights.rules.model.Measurement; import org.opensearch.plugin.insights.rules.model.MetricType; /** @@ -21,13 +22,13 @@ final class SearchQueryCategorizingVisitor implements QueryBuilderVisitor { private final int level; private final SearchQueryCounters searchQueryCounters; - private final Map measurements; + private final Map measurements; - public SearchQueryCategorizingVisitor(SearchQueryCounters searchQueryCounters, Map measurements) { + public SearchQueryCategorizingVisitor(SearchQueryCounters searchQueryCounters, Map measurements) { this(searchQueryCounters, 0, measurements); } - private SearchQueryCategorizingVisitor(SearchQueryCounters counters, int level, Map measurements) { + private SearchQueryCategorizingVisitor(SearchQueryCounters counters, int level, Map measurements) { this.searchQueryCounters = counters; this.level = level; this.measurements = measurements; diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCounters.java b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCounters.java index cb89022..21f4aae 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCounters.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/SearchQueryCounters.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.plugin.insights.rules.model.Measurement; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.telemetry.metrics.Counter; import org.opensearch.telemetry.metrics.Histogram; @@ -108,7 +109,7 @@ public SearchQueryCounters(MetricsRegistry metricsRegistry) { * @param level level of query builder, 0 being highest level * @param measurements metrics measurements */ - public void incrementCounter(QueryBuilder queryBuilder, int level, Map measurements) { + public void incrementCounter(QueryBuilder queryBuilder, int level, Map measurements) { String uniqueQueryCounterName = queryBuilder.getName(); Counter counter = nameToQueryTypeCounters.computeIfAbsent(uniqueQueryCounterName, k -> createQueryCounter(k)); @@ -122,7 +123,7 @@ public void incrementCounter(QueryBuilder queryBuilder, int level, Map measurements) { + public void incrementAggCounter(double value, Tags tags, Map measurements) { aggCounter.add(value, tags); incrementAllHistograms(tags, measurements); } @@ -133,15 +134,15 @@ public void incrementAggCounter(double value, Tags tags, Map * @param tags tags * @param measurements metrics measurements */ - public void incrementSortCounter(double value, Tags tags, Map measurements) { + public void incrementSortCounter(double value, Tags tags, Map measurements) { sortCounter.add(value, tags); incrementAllHistograms(tags, measurements); } - private void incrementAllHistograms(Tags tags, Map measurements) { - queryTypeLatencyHistogram.record(measurements.get(MetricType.LATENCY).doubleValue(), tags); - queryTypeCpuHistogram.record(measurements.get(MetricType.CPU).doubleValue(), tags); - queryTypeMemoryHistogram.record(measurements.get(MetricType.MEMORY).doubleValue(), tags); + private void incrementAllHistograms(Tags tags, Map measurements) { + queryTypeLatencyHistogram.record(measurements.get(MetricType.LATENCY).getMeasurement().doubleValue(), tags); + queryTypeCpuHistogram.record(measurements.get(MetricType.CPU).getMeasurement().doubleValue(), tags); + queryTypeMemoryHistogram.record(measurements.get(MetricType.MEMORY).getMeasurement().doubleValue(), tags); } /** diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java b/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java index c8b14fd..5ebb98f 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java @@ -9,8 +9,10 @@ package org.opensearch.plugin.insights.rules.model; import java.io.IOException; +import java.io.InputStream; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -56,7 +58,11 @@ public enum Attribute { /** * Custom search request labels */ - LABELS; + LABELS, + /** + * Unique hashcode used to group similar queries + */ + QUERY_HASHCODE; /** * Read an Attribute from a StreamInput diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java new file mode 100644 index 0000000..fa3a2fb --- /dev/null +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.insights.rules.model; + +/** + * Dimension type for a measurement. Default is NONE and average is used for grouping Top N queries by similarity. + */ +public enum DimensionType { + NONE, + AVERAGE, + SUM; + + public static DimensionType DEFUALT_DIMENSION_TYPE = NONE; +} diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java new file mode 100644 index 0000000..0790de0 --- /dev/null +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.insights.rules.model; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Grouping type to group the Top N queries + */ +public enum GroupingType { + NONE("none"), + SIMILARITY("similarity"), + USER_ID("userid"); + + private final String stringValue; + + GroupingType(String stringValue) { + this.stringValue = stringValue; + } + + public String getValue() { + return stringValue; + } + + /** + * Get all valid GroupingTypes + * + * @return A set contains all valid GroupingTypes + */ + public static Set allGroupingTypes() { + return Arrays.stream(values()).collect(Collectors.toSet()); + } + + /** + * Get grouping type from setting string and validate + * @param settingValue value + * @return GroupingType + */ + public static GroupingType getGroupingTypeFromSettingAndValidate(String settingValue) { + try { + return GroupingType.valueOf(settingValue.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Invalid exporter type [%s], type should be one of %s", + settingValue, + allGroupingTypes() + ) + ); + } + } +} diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java new file mode 100644 index 0000000..01348e1 --- /dev/null +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.insights.rules.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +/** + * Measurement that is stored in the SearchQueryRecord. Measurement can be of a specific DimensionType and MetricType + */ +public class Measurement implements ToXContentObject, Writeable { + private static int DEFAULT_COUNT = 1; + private DimensionType dimensionType; + private MetricType metricType; + private Number number; + private int count; + + /** + * Constructor + * @param metricType metricType + * @param number number + * @param count count + * @param dimensionType dimensionType + */ + public Measurement(MetricType metricType, Number number, int count, DimensionType dimensionType) { + this.metricType = metricType; + this.number = number; + this.count = count; + this.dimensionType = dimensionType; + } + + /** + * Constructor + * @param metricType metricType + * @param number number + * @param dimensionType dimensionType + */ + public Measurement(MetricType metricType, Number number, DimensionType dimensionType) { + this(metricType, number, DEFAULT_COUNT, dimensionType); + } + + /** + * Constructor + * @param metricType metricType + * @param number number + */ + public Measurement(MetricType metricType, Number number) { + this(metricType, number, DEFAULT_COUNT, DimensionType.DEFUALT_DIMENSION_TYPE); + } + + /** + * Add measurement number to the current number based on the dimension type + * @param toAdd number to add + */ + public void addMeasurement(Number toAdd) { + switch (dimensionType) { + case NONE: + dimensionType = DimensionType.SUM; + setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); + break; + case SUM: + setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); + break; + case AVERAGE: + count+=1; + setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); + break; + default: + throw new IllegalArgumentException("The following dimension type is not supported : " + dimensionType); + } + } + + /** + * Get measurement number based on the dimension type + * @return measurement number + */ + public Number getMeasurement() { + switch (dimensionType) { + case NONE: + case SUM: + return number; + case AVERAGE: + return MetricType.getAverageMeasurement(number, count, metricType); + default: + throw new IllegalArgumentException("Dimension Type should be set for measurement."); + } + } + + /** + * Set measurement + * @param measurement measurement number + */ + public void setMeasurement(Number measurement) { + number = measurement; + } + + /** + * Set dimension type + * @param dimensionType dimension type + */ + public void setDimensionType(DimensionType dimensionType) { + this.dimensionType = dimensionType; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field("metricType", metricType.toString()); + builder.field("number", number); + builder.field("count", count); + builder.field("dimensionType", dimensionType.toString()); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(metricType.toString()); + out.writeGenericValue(metricType.parseValue(number)); + out.writeInt(count); + out.writeString(dimensionType.toString()); + } + + public static Measurement readFromStream(StreamInput in) throws IOException { + MetricType metricType = MetricType.valueOf(in.readString().toUpperCase(Locale.ROOT)); + Number number = metricType.parseValue(in.readGenericValue()); + int count = in.readInt(); + DimensionType dimensionType = DimensionType.valueOf(in.readString()); + return new Measurement(metricType, number, count, dimensionType); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Measurement that = (Measurement) o; + return count == that.count && + metricType == that.metricType && + Objects.equals(number, that.number) && + dimensionType == that.dimensionType; + } + + @Override + public int hashCode() { + return Objects.hash(metricType, number, count, dimensionType); + } +} diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java index 52b8331..2be7d51 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java @@ -97,13 +97,35 @@ public int compare(final Number a, final Number b) { return -1; } + public static Number addMeasurements(Number a, Number b, MetricType metricType) { + switch (metricType) { + case LATENCY: + case MEMORY: + case CPU: + return a.longValue() + b.longValue(); + default: + throw new IllegalArgumentException("Unsupported metric type: " + metricType); + } + } + + public static Number getAverageMeasurement(Number total, int count, MetricType metricType) { + switch (metricType) { + case LATENCY: + case MEMORY: + case CPU: + return total.longValue()/count; + default: + throw new IllegalArgumentException("Unsupported metric type: " + metricType); + } + } + /** * Parse a value with the correct type based on MetricType * * @param o the generic object to parse * @return {@link Number} */ - Number parseValue(final Object o) { + public Number parseValue(final Object o) { switch (this) { case LATENCY: case CPU: diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java index 7283bd1..0712ec1 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java @@ -9,7 +9,7 @@ package org.opensearch.plugin.insights.rules.model; import java.io.IOException; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import org.opensearch.core.common.Strings; @@ -27,8 +27,9 @@ */ public class SearchQueryRecord implements ToXContentObject, Writeable { private final long timestamp; - private final Map measurements; + private final Map measurements; private final Map attributes; + private String groupingId; /** * Constructor of SearchQueryRecord @@ -39,10 +40,11 @@ public class SearchQueryRecord implements ToXContentObject, Writeable { */ public SearchQueryRecord(final StreamInput in) throws IOException, ClassCastException { this.timestamp = in.readLong(); - measurements = new HashMap<>(); - in.readMap(MetricType::readFromStream, StreamInput::readGenericValue) - .forEach(((metricType, o) -> measurements.put(metricType, metricType.parseValue(o)))); + measurements = new LinkedHashMap<>(); + in.readOrderedMap(MetricType::readFromStream, Measurement::readFromStream) + .forEach(((metricType, measurement) -> measurements.put(metricType, measurement))); this.attributes = Attribute.readAttributeMap(in); + this.groupingId = null; } /** @@ -52,7 +54,7 @@ public SearchQueryRecord(final StreamInput in) throws IOException, ClassCastExce * @param measurements A list of Measurement associated with this query * @param attributes A list of Attributes associated with this query */ - public SearchQueryRecord(final long timestamp, Map measurements, final Map attributes) { + public SearchQueryRecord(final long timestamp, Map measurements, final Map attributes) { if (measurements == null) { throw new IllegalArgumentException("Measurements cannot be null"); } @@ -77,7 +79,7 @@ public long getTimestamp() { * @return the measurement object, or null if not found */ public Number getMeasurement(final MetricType name) { - return measurements.get(name); + return measurements.get(name).getMeasurement(); } /** @@ -85,7 +87,26 @@ public Number getMeasurement(final MetricType name) { * * @return a map of measurement names to measurement objects */ - public Map getMeasurements() { + + /** + * Add measurement to SearchQueryRecord. Applicable when we are grouping multiple queries based on GroupingType. + * @param name the name of the measurement + * @param measurement The measurement we want to add to the current measurement. + */ + public void addMeasurement(final MetricType name, Number measurement) { + measurements.get(name).addMeasurement(measurement); + } + + /** + * Set the dimension type for measurement + * @param name the name of the measurement + * @param dimensionType Dimension type to set + */ + public void setMeasurementDimension(final MetricType name, DimensionType dimensionType) { + measurements.get(name).setDimensionType(dimensionType); + } + + public Map getMeasurements() { return measurements; } @@ -115,9 +136,12 @@ public XContentBuilder toXContent(final XContentBuilder builder, final ToXConten for (Map.Entry entry : attributes.entrySet()) { builder.field(entry.getKey().toString(), entry.getValue()); } - for (Map.Entry entry : measurements.entrySet()) { - builder.field(entry.getKey().toString(), entry.getValue()); + builder.startObject("measurements"); + for (Map.Entry entry : measurements.entrySet()) { + builder.field(entry.getKey().toString()); // MetricType as field name + entry.getValue().toXContent(builder, params); // Serialize Measurement object } + builder.endObject(); return builder.endObject(); } @@ -130,7 +154,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final ToXConten @Override public void writeTo(final StreamOutput out) throws IOException { out.writeLong(timestamp); - out.writeMap(measurements, (stream, metricType) -> MetricType.writeTo(out, metricType), StreamOutput::writeGenericValue); + out.writeMap(measurements, (stream, metricType) -> MetricType.writeTo(out, metricType), (stream, measurement) -> measurement.writeTo(out)); out.writeMap( attributes, (stream, attribute) -> Attribute.writeTo(out, attribute), @@ -181,4 +205,12 @@ public int hashCode() { public String toString() { return Strings.toString(MediaTypeRegistry.JSON, this); } + + public void setGroupingId(String groupingId) { + this.groupingId = groupingId; + } + + public String getGroupingId() { + return this.groupingId; + } } diff --git a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java index cb9b39d..46dd078 100644 --- a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java +++ b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java @@ -16,6 +16,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.plugin.insights.core.exporter.SinkType; +import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; /** @@ -33,7 +34,7 @@ public class QueryInsightsSettings { /** * Max number of requests for the consumer to collect at one time */ - public static final int QUERY_RECORD_QUEUE_CAPACITY = 1000; + public static final int QUERY_RECORD_QUEUE_CAPACITY = 100000; /** * Time interval for record queue consumer to run */ @@ -69,6 +70,8 @@ public class QueryInsightsSettings { */ public static final String PLUGINS_BASE_URI = "/_insights"; + public static final GroupingType DEFAULT_GROUPING_TYPE = GroupingType.NONE; + /** * Settings for Top Queries * @@ -112,6 +115,16 @@ public class QueryInsightsSettings { Setting.Property.Dynamic ); + /** + * Define the group_by option for Top N queries to group queries. + */ + public static final Setting TOP_N_QUERIES_GROUP_BY = Setting.simpleString( + TOP_N_QUERIES_SETTING_PREFIX + ".group_by", + DEFAULT_GROUPING_TYPE.getValue(), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** * Boolean setting for enabling top queries by cpu. */ diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java index 7a921e4..0060c55 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java @@ -75,6 +75,7 @@ public void testGetSettings() { QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE, QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE, QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS, + QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY, QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING ), queryInsightsPlugin.getSettings() diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java index 6b2d8c6..0c4cc2d 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java @@ -25,9 +25,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeSet; import org.opensearch.action.search.SearchType; @@ -40,6 +42,8 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.plugin.insights.rules.action.top_queries.TopQueries; import org.opensearch.plugin.insights.rules.model.Attribute; +import org.opensearch.plugin.insights.rules.model.DimensionType; +import org.opensearch.plugin.insights.rules.model.Measurement; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; import org.opensearch.plugin.insights.settings.QueryCategorizationSettings; @@ -57,41 +61,57 @@ public QueryInsightsTestUtils() {} * @return List of records */ public static List generateQueryInsightRecords(int count) { - return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0); + return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, DimensionType.DEFUALT_DIMENSION_TYPE); } /** - * Returns list of randomly generated search query records. + * Returns list of randomly generated search query records with specific searchSourceBuilder * @param count number of records * @param searchSourceBuilder source * @return List of records */ public static List generateQueryInsightRecords(int count, SearchSourceBuilder searchSourceBuilder) { - List records = generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0); + List records = generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, DimensionType.DEFUALT_DIMENSION_TYPE); for (SearchQueryRecord record : records) { record.getAttributes().put(Attribute.SOURCE, searchSourceBuilder); } return records; } + /** + * Returns list of randomly generated search query records with specific dimension type for measurements + * @param count number of records + * @param dimensionType source + * @return List of records + */ + public static List generateQueryInsightRecords(int count, DimensionType dimensionType) { + return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, dimensionType); + } + /** * Creates a List of random Query Insight Records for testing purpose */ public static List generateQueryInsightRecords(int lower, int upper, long startTimeStamp, long interval) { + return generateQueryInsightRecords(lower, upper, startTimeStamp, interval, DimensionType.NONE); + } + + /** + * Creates a List of random Query Insight Records for testing purpose with dimenstion type specified + */ + public static List generateQueryInsightRecords(int lower, int upper, long startTimeStamp, long interval, DimensionType dimensionType) { List records = new ArrayList<>(); int countOfRecords = randomIntBetween(lower, upper); long timestamp = startTimeStamp; for (int i = 0; i < countOfRecords; ++i) { - Map measurements = Map.of( - MetricType.LATENCY, - randomLongBetween(1000, 10000), - MetricType.CPU, - randomLongBetween(1000, 10000), - MetricType.MEMORY, - randomLongBetween(1000, 10000) - ); + long latencyValue = randomLongBetween(1000, 10000); // Replace with actual method to generate a random long + long cpuValue = randomLongBetween(1000, 10000); + long memoryValue = randomLongBetween(1000, 10000); + Map measurements = new LinkedHashMap<>(); + measurements.put(MetricType.LATENCY, new Measurement(MetricType.LATENCY, latencyValue, dimensionType)); + measurements.put(MetricType.CPU, new Measurement(MetricType.CPU, cpuValue, dimensionType)); + measurements.put(MetricType.MEMORY, new Measurement(MetricType.MEMORY, memoryValue, dimensionType)); - Map phaseLatencyMap = new HashMap<>(); + Map phaseLatencyMap = new LinkedHashMap<>(); int countOfPhases = randomIntBetween(2, 5); for (int j = 0; j < countOfPhases; ++j) { phaseLatencyMap.put(randomAlphaOfLengthBetween(5, 10), randomLong()); @@ -106,6 +126,7 @@ public static List generateQueryInsightRecords(int lower, int attributes.put(Attribute.TOTAL_SHARDS, randomIntBetween(1, 100)); attributes.put(Attribute.INDICES, randomArray(1, 3, Object[]::new, () -> randomAlphaOfLengthBetween(5, 10))); attributes.put(Attribute.PHASE_LATENCY_MAP, phaseLatencyMap); + attributes.put(Attribute.QUERY_HASHCODE, Objects.hashCode(i)); attributes.put( Attribute.TASK_RESOURCE_USAGES, List.of( @@ -132,6 +153,27 @@ public static List generateQueryInsightRecords(int lower, int return records; } + public static List generateQueryInsightsRecordsWithMeasurement(int count, MetricType metricType, Number measurement) { + List records = generateQueryInsightRecords(count); + + for (SearchQueryRecord record : records) { + record.getMeasurements().get(metricType).setMeasurement(measurement); + } + return records; + } + + public static void populateSameQueryHashcodes(List searchQueryRecords) { + for (SearchQueryRecord record : searchQueryRecords) { + record.getAttributes().put(Attribute.QUERY_HASHCODE, 1); + } + } + + public static void populateHashcode(List searchQueryRecords, int hash) { + for (SearchQueryRecord record : searchQueryRecords) { + record.getAttributes().put(Attribute.QUERY_HASHCODE, hash); + } + } + public static TopQueries createRandomTopQueries() { DiscoveryNode node = new DiscoveryNode( "node_for_top_queries_test", @@ -161,7 +203,7 @@ public static TopQueries createFixedTopQueries() { public static SearchQueryRecord createFixedSearchQueryRecord() { long timestamp = 1706574180000L; - Map measurements = Map.of(MetricType.LATENCY, 1L); + Map measurements = Map.of(MetricType.LATENCY, new Measurement(MetricType.LATENCY, 1L)); Map phaseLatencyMap = new HashMap<>(); Map attributes = new HashMap<>(); @@ -246,6 +288,7 @@ public static void registerAllQueryInsightsSettings(ClusterSettings clusterSetti clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE); clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE); clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS); + clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY); clusterSettings.registerSetting(QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING); } } diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java new file mode 100644 index 0000000..278adc5 --- /dev/null +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java @@ -0,0 +1,753 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.insights.core.service; + +import org.junit.Before; +import org.opensearch.plugin.insights.QueryInsightsTestUtils; +import org.opensearch.plugin.insights.rules.model.Attribute; +import org.opensearch.plugin.insights.rules.model.DimensionType; +import org.opensearch.plugin.insights.rules.model.GroupingType; +import org.opensearch.plugin.insights.rules.model.Measurement; +import org.opensearch.plugin.insights.rules.model.MetricType; +import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.stream.Stream; + +/** + * Unit Tests for {@link QueryGroupingService}. + */ +public class QueryGroupingServiceTests extends OpenSearchTestCase { + private QueryGroupingService queryGroupingService; + private PriorityQueue topQueriesStore = new PriorityQueue<>(100, (a, b) -> SearchQueryRecord.compare(a, b, MetricType.LATENCY)); + + @Before + public void setup() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.DEFUALT_DIMENSION_TYPE, topQueriesStore, 10); + } + + public void testWithAllDifferentHashcodes() { + int numOfRecords = 10; + final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); + SearchQueryRecord groupedRecord; + Set hashcodeSet = new HashSet<>(); + for (SearchQueryRecord record : records) { + groupedRecord = queryGroupingService.addQueryToGroup(record); + int hashcode = (int) groupedRecord.getAttributes().get(Attribute.QUERY_HASHCODE); + hashcodeSet.add(hashcode); + } + assertEquals(numOfRecords, hashcodeSet.size()); + } + + public void testWithAllSameHashcodes() { + int numOfRecords = 10; + final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + SearchQueryRecord groupedRecord; + Set hashcodeSet = new HashSet<>(); + for (SearchQueryRecord record : records) { + groupedRecord = queryGroupingService.addQueryToGroup(record); + int hashcode = (int) groupedRecord.getAttributes().get(Attribute.QUERY_HASHCODE); + hashcodeSet.add(hashcode); + } + assertEquals(1, hashcodeSet.size()); + } + + public void testDrain() { + int numOfRecords = 10; + final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + int groupsBeforeDrain = queryGroupingService.numberOfGroups(); + queryGroupingService.drain(); + int groupsAfterDrain = queryGroupingService.numberOfGroups(); + + assertEquals(numOfRecords, groupsBeforeDrain); + assertEquals(0, groupsAfterDrain); + } + + public void testChangeTopNSize() { + int numOfRecords = 15; + final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); + + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + + assertEquals(10, queryGroupingService.numberOfTopGroups()); // Initially expects top 10 groups + + queryGroupingService.updateTopNSize(5); + queryGroupingService.drain(); // Clear previous state + + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + + assertEquals(5, queryGroupingService.numberOfTopGroups()); // After update, expects top 5 groups + } + + public void testEmptyPriorityQueues() { + int groupsBeforeDrain = queryGroupingService.numberOfGroups(); + assertEquals(0, groupsBeforeDrain); + + queryGroupingService.drain(); + int groupsAfterDrain = queryGroupingService.numberOfGroups(); + assertEquals(0, groupsAfterDrain); // No groups should be present after draining + } + + public void testAddRemoveFromMaxHeap() { + int numOfRecords = 15; + final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); + + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + + assertTrue(queryGroupingService.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap + + queryGroupingService.updateTopNSize(5); // Change size to 5 + queryGroupingService.drain(); // Clear previous state + + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + + assertEquals(5, queryGroupingService.numberOfTopGroups()); // Should be exactly 5 in the min heap + } + + public void testInvalidGroupingType() { + QueryGroupingService invalidGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.NONE, DimensionType.DEFUALT_DIMENSION_TYPE, topQueriesStore, 10); + SearchQueryRecord record = QueryInsightsTestUtils.generateQueryInsightRecords(1).get(0); + expectThrows(IllegalArgumentException.class, () -> invalidGroupingService.addQueryToGroup(record)); + } + + public void testLargeNumberOfRecords() { + int numOfRecords = 1000; + final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); + + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + + assertTrue(queryGroupingService.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap + } + + public void testChangeGroupingType() { + int numOfRecords = 10; + final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); + + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + + int groupsBeforeChange = queryGroupingService.numberOfGroups(); + assertTrue(groupsBeforeChange > 0); + + queryGroupingService.setGroupingType(GroupingType.NONE); // Changing to NONE should clear groups + + int groupsAfterChange = queryGroupingService.numberOfGroups(); + assertEquals(0, groupsAfterChange); // Expect no groups after changing to NONE + } + + public void testDrainWithMultipleGroupingTypes() { + int numOfRecords = 20; + final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); + + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + + int groupsBeforeDrain = queryGroupingService.numberOfGroups(); + assertTrue(groupsBeforeDrain > 0); + + queryGroupingService.setGroupingType(GroupingType.SIMILARITY); + queryGroupingService.drain(); + + int groupsAfterDrain = queryGroupingService.numberOfGroups(); + assertEquals(0, groupsAfterDrain); // After drain, groups should be cleared + } + + public void testVaryingTopNSize() { + int numOfRecords = 30; + final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); + + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + + queryGroupingService.updateTopNSize(15); + queryGroupingService.drain(); // Clear previous state + + for (SearchQueryRecord record : records) { + queryGroupingService.addQueryToGroup(record); + } + + assertEquals(15, queryGroupingService.numberOfTopGroups()); // Should reflect the updated top N size + } + + public void testAddMeasurementSumDimensionLatency() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.SUM, topQueriesStore, 10); + int numOfRecords = 10; + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + SearchQueryRecord aggregatedRecord = null; + + Number expectedSum = 0; + for (SearchQueryRecord record : records) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); + } + + assertEquals(expectedSum, aggregatedRecord.getMeasurement(MetricType.LATENCY)); + } + + public void testAddMeasurementAverageDimensionLatency() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 10); + int numOfRecords = 10; + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + SearchQueryRecord aggregatedRecord = null; + + Number expectedSum = 0; + int expectedCount = 0; + for (SearchQueryRecord record : records) { + expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + expectedCount +=1; + } + + long expectedAverage = (long) expectedSum / expectedCount; + assertEquals(expectedAverage, aggregatedRecord.getMeasurement(MetricType.LATENCY)); + } + + public void testAddMeasurementNoneDimensionLatency() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.NONE, topQueriesStore, 10); + int numOfRecords = 10; + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + SearchQueryRecord aggregatedRecord = null; + + Number expectedSum = 0; + for (SearchQueryRecord record : records) { + expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + + assertEquals(expectedSum, aggregatedRecord.getMeasurement(MetricType.LATENCY)); + } + + public void testAddMeasurementSumDimensionCpu() { + queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, DimensionType.SUM, topQueriesStore, 10); + int numOfRecords = 10; + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + SearchQueryRecord aggregatedRecord = null; + + Number expectedSum = 0; + for (SearchQueryRecord record : records) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); + } + + assertEquals(expectedSum, aggregatedRecord.getMeasurement(MetricType.CPU)); + } + + public void testAddMeasurementAverageDimensionCpu() { + queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 10); + int numOfRecords = 10; + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + SearchQueryRecord aggregatedRecord = null; + + Number expectedSum = 0; + int expectedCount = 0; + for (SearchQueryRecord record : records) { + expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + expectedCount +=1; + } + + long expectedAverage = (long) expectedSum / expectedCount; + assertEquals(expectedAverage, aggregatedRecord.getMeasurement(MetricType.CPU)); + } + + public void testAddMeasurementNoneDimensionCpu() { + queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, DimensionType.NONE, topQueriesStore, 10); + int numOfRecords = 10; + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + SearchQueryRecord aggregatedRecord = null; + + Number expectedSum = 0; + for (SearchQueryRecord record : records) { + expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + + assertEquals(expectedSum, aggregatedRecord.getMeasurement(MetricType.CPU)); + } + + public void testNoneGroupingTypeIllegalArgumentException() { + queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.NONE, DimensionType.NONE, topQueriesStore, 10); + int numOfRecords = 10; + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + SearchQueryRecord aggregatedRecord = null; + + Number expectedSum = 0; + for (SearchQueryRecord record : records) { + expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); + assertThrows(IllegalArgumentException.class, () -> { + queryGroupingService.addQueryToGroup(record); + }); + } + } + + // 1. New query group not existing added to MIN + public void testNewGroupAddedToMin() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 2; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } + + // 2. New query group not existing added to MIN and overflows to MAX + public void testNewGroupOverflowsMinToMax() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 2; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } + + // 3. New query group not existing added to MIN and causes other group to overflow to MAX + public void testNewGroupCausesOtherGroupOverflowMinToMax() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 2; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1200); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } + + // 4. Existing query group update to MIN increases average + public void testExistingGroupUpdateToMinIncreaseAverage() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 1; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1200); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + // Average will decrease when this record added and record should be moved out of Top N + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1300); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records4, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3, records4); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } + + // 5. Existing query group update to MIN decrease average - stay in MIN + public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 1; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 600); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + // Average will decrease when this record added and record should be moved out of Top N + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 700); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records4, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3, records4); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(900L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } + + // 6. Existing query group update to MIN decrease average - overflows to MAX + public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 1; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1199); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + // Average will decrease when this record added and record should be moved out of Top N + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records4, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3, records4); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } + + // 7. Existing query group update to MAX increases average - stay in MAX + public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 1; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 920); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records4, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3, records4); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } + + // 8. Existing query group update to MAX increases average - promote to MIN + public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 1; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records4, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3, records4); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } + + // 9. Existing query group update to MAX decrease average + public void testExistingGroupUpdateToMaxDecreaseAverage() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 1; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 800); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records4, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3, records4); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } + + public void testSwitchGroupingTypeToUserId() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 1; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 800); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records4, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3, records4); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + + queryGroupingService.setGroupingType(GroupingType.USER_ID); + assertEquals(0, queryGroupingService.numberOfTopGroups()); + } + + public void testSwitchGroupingTypeToNone() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 1; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 800); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records4, 3); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3, records4); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + + queryGroupingService.setGroupingType(GroupingType.NONE); + assertEquals(0, queryGroupingService.numberOfTopGroups()); + + assertThrows(IllegalArgumentException.class, () -> { + queryGroupingService.addQueryToGroup(allRecords.get(0).get(0)); + }); + } + + public void testMultipleQueryGroupsUpdates() { + queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + int numOfRecords = 1; + + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records2, 2); + + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records3, 3); + + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 800); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records4, 3); + + List records5 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 400); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records5, 2); + + List records6 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1200); + // Set all records to have the same hashcode for aggregation + QueryInsightsTestUtils.populateHashcode(records6, 1); + + SearchQueryRecord aggregatedRecord = null; + List> allRecords = Arrays.asList(records1, records2, records3, records4, records5, records6); + + for (List recordList : allRecords) { + for (SearchQueryRecord record : recordList) { + aggregatedRecord = queryGroupingService.addQueryToGroup(record); + } + } + + assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(850L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + } +} diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java index 56f3203..a174c77 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java @@ -16,12 +16,14 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.plugin.insights.QueryInsightsTestUtils; +import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; import org.opensearch.plugin.insights.settings.QueryInsightsSettings; import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; +import java.util.List; /** * Unit Tests for {@link QueryInsightsService}. @@ -87,4 +89,89 @@ public void testSearchQueryMetricsEnabled() { assertNotNull(queryInsightsService.getSearchQueryCategorizer()); } + + public void testFeaturesEnableDisable() { + // Test case 1: All metric type collection disabled and search query metrics disabled, enable search query metrics + queryInsightsServiceSpy.enableCollection(MetricType.LATENCY, false); + queryInsightsServiceSpy.enableCollection(MetricType.CPU, false); + queryInsightsServiceSpy.enableCollection(MetricType.MEMORY, false); + queryInsightsServiceSpy.setSearchQueryMetricsEnabled(false); + + queryInsightsServiceSpy.setSearchQueryMetricsEnabled(true); + verify(queryInsightsServiceSpy).checkAndRestartQueryInsights(); + + // Test case 2: All metric type collection disabled and search query metrics enabled, disable search query metrics + queryInsightsServiceSpy.enableCollection(MetricType.LATENCY, false); + queryInsightsServiceSpy.enableCollection(MetricType.CPU, false); + queryInsightsServiceSpy.enableCollection(MetricType.MEMORY, false); + queryInsightsServiceSpy.setSearchQueryMetricsEnabled(true); + + queryInsightsServiceSpy.setSearchQueryMetricsEnabled(false); + verify(queryInsightsServiceSpy).checkAndStopQueryInsights(); + } + + public void testAddRecordGroupBySimilarityWithDifferentGroups() { + + int numberOfRecordsRequired = 10; + List records = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numberOfRecordsRequired, MetricType.LATENCY, 5); + + queryInsightsService.validateAndSetGrouping(GroupingType.SIMILARITY.getValue()); + assertEquals(queryInsightsService.getGrouping(), GroupingType.SIMILARITY); + + for (int i = 0; i < numberOfRecordsRequired; i++) { + assertTrue(queryInsightsService.addRecord(records.get(i))); + } + // exceed capacity but handoff to grouping + assertTrue(queryInsightsService.addRecord(records.get(numberOfRecordsRequired-1))); + + queryInsightsService.drainRecords(); + + assertEquals( + QueryInsightsSettings.DEFAULT_TOP_N_SIZE, + queryInsightsService.getTopQueriesService(MetricType.LATENCY).getTopQueriesRecords(false).size() + ); + } + + public void testAddRecordGroupBySimilarityWithOneGroup() { + int numberOfRecordsRequired = 10; + List records = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numberOfRecordsRequired, MetricType.LATENCY, 5); + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + + queryInsightsService.validateAndSetGrouping(GroupingType.SIMILARITY.getValue()); + assertEquals(queryInsightsService.getGrouping(), GroupingType.SIMILARITY); + + for (int i = 0; i < numberOfRecordsRequired; i++) { + assertTrue(queryInsightsService.addRecord(records.get(i))); + } + // exceed capacity but handoff to grouping service + assertTrue(queryInsightsService.addRecord(records.get(numberOfRecordsRequired-1))); + + queryInsightsService.drainRecords(); + assertEquals( + 1, + queryInsightsService.getTopQueriesService(MetricType.LATENCY).getTopQueriesRecords(false).size() + ); + } + + public void testAddRecordGroupBySimilarityWithTwoGroups() { + List records1 = QueryInsightsTestUtils.generateQueryInsightRecords(2, 2, System.currentTimeMillis(), 0); + QueryInsightsTestUtils.populateHashcode(records1, 1); + + List records2 = QueryInsightsTestUtils.generateQueryInsightRecords(2, 2, System.currentTimeMillis(), 0); + QueryInsightsTestUtils.populateHashcode(records2, 2); + + queryInsightsService.validateAndSetGrouping(GroupingType.SIMILARITY.getValue()); + assertEquals(queryInsightsService.getGrouping(), GroupingType.SIMILARITY); + + for (int i = 0; i < 2; i++) { + assertTrue(queryInsightsService.addRecord(records1.get(i))); + assertTrue(queryInsightsService.addRecord(records2.get(i))); + } + + queryInsightsService.drainRecords(); + assertEquals( + 2, + queryInsightsService.getTopQueriesService(MetricType.LATENCY).getTopQueriesRecords(false).size() + ); + } } diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java index ca75560..d69fe53 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/TopQueriesServiceTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.plugin.insights.QueryInsightsTestUtils; import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory; +import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; import org.opensearch.plugin.insights.settings.QueryInsightsSettings; @@ -108,4 +109,39 @@ private static void runUntilTimeoutOrFinish(DeterministicTaskQueue deterministic } } } + + public void testRollingWindowsWithSameGroup() { + topQueriesService.setGrouping(GroupingType.SIMILARITY); + List records; + // Create 5 records at Now - 10 minutes to make sure they belong to the last window + records = QueryInsightsTestUtils.generateQueryInsightRecords(5, 5, System.currentTimeMillis() - 1000 * 60 * 10, 0); + topQueriesService.setWindowSize(TimeValue.timeValueMinutes(10)); + topQueriesService.consumeRecords(records); + assertEquals(0, topQueriesService.getTopQueriesRecords(true).size()); + + // Create 10 records at now + 1 minute, to make sure they belong to the current window + records = QueryInsightsTestUtils.generateQueryInsightRecords(10, 10, System.currentTimeMillis() + 1000 * 60, 0); + topQueriesService.setWindowSize(TimeValue.timeValueMinutes(10)); + topQueriesService.consumeRecords(records); + assertEquals(10, topQueriesService.getTopQueriesRecords(true).size()); + } + + public void testRollingWindowsWithDifferentGroup() { + topQueriesService.setGrouping(GroupingType.SIMILARITY); + List records; + // Create 5 records at Now - 10 minutes to make sure they belong to the last window + records = QueryInsightsTestUtils.generateQueryInsightRecords(5, 5, System.currentTimeMillis() - 1000 * 60 * 10, 0); + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + + topQueriesService.setWindowSize(TimeValue.timeValueMinutes(10)); + topQueriesService.consumeRecords(records); + assertEquals(0, topQueriesService.getTopQueriesRecords(true).size()); + + // Create 10 records at now + 1 minute, to make sure they belong to the current window + records = QueryInsightsTestUtils.generateQueryInsightRecords(10, 10, System.currentTimeMillis() + 1000 * 60, 0); + QueryInsightsTestUtils.populateSameQueryHashcodes(records); + topQueriesService.setWindowSize(TimeValue.timeValueMinutes(10)); + topQueriesService.consumeRecords(records); + assertEquals(1, topQueriesService.getTopQueriesRecords(true).size()); + } } diff --git a/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java b/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java index 08c9c8a..2f75052 100644 --- a/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java +++ b/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java @@ -41,7 +41,7 @@ public void testSerialize() throws Exception { public void testToXContent() throws IOException { char[] expectedXcontent = - "{\"top_queries\":[{\"timestamp\":1706574180000,\"node_id\":\"node_for_top_queries_test\",\"search_type\":\"query_then_fetch\",\"latency\":1}]}" + "{\"top_queries\":[{\"timestamp\":1706574180000,\"node_id\":\"node_for_top_queries_test\",\"search_type\":\"query_then_fetch\",\"measurements\":{\"latency\":{\"metricType\":\"latency\",\"number\":1,\"count\":1,\"dimensionType\":\"NONE\"}}}]}" .toCharArray(); TopQueries topQueries = QueryInsightsTestUtils.createFixedTopQueries(); ClusterName clusterName = new ClusterName("test-cluster"); From 0a923b867d112aae0111ac977479918a72f3b926 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 2 Aug 2024 11:49:46 -0700 Subject: [PATCH 02/29] Spotless apply Signed-off-by: Siddhant Deshmukh --- .../core/listener/QueryInsightsListener.java | 25 +- .../core/service/QueryGroupingService.java | 24 +- .../insights/core/service/QueryShape.java | 5 +- .../core/service/TopQueriesService.java | 8 +- .../insights/rules/model/Attribute.java | 2 - .../insights/rules/model/DimensionType.java | 6 +- .../insights/rules/model/GroupingType.java | 7 +- .../insights/rules/model/Measurement.java | 18 +- .../insights/rules/model/MetricType.java | 2 +- .../rules/model/SearchQueryRecord.java | 6 +- .../insights/QueryInsightsTestUtils.java | 22 +- .../service/QueryGroupingServiceTests.java | 450 ++++++++++++++---- .../service/QueryInsightsServiceTests.java | 28 +- 13 files changed, 454 insertions(+), 149 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index 9ed8508..b2b44f5 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -9,10 +9,10 @@ package org.opensearch.plugin.insights.core.listener; import static org.opensearch.plugin.insights.settings.QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING; +import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNEnabledSetting; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNSizeSetting; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNWindowSizeSetting; -import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY; import java.util.Collections; import java.util.HashMap; @@ -109,10 +109,7 @@ public QueryInsightsListener( setSearchQueryMetricsEnabled(clusterService.getClusterSettings().get(SEARCH_QUERY_METRICS_ENABLED_SETTING)); clusterService.getClusterSettings() - .addSettingsUpdateConsumer( - TOP_N_QUERIES_GROUP_BY, - v -> this.queryInsightsService.validateAndSetGrouping(v) - ); + .addSettingsUpdateConsumer(TOP_N_QUERIES_GROUP_BY, v -> this.queryInsightsService.validateAndSetGrouping(v)); this.queryInsightsService.validateAndSetGrouping(clusterService.getClusterSettings().get(TOP_N_QUERIES_GROUP_BY)); } @@ -205,22 +202,28 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final if (shouldCollect(MetricType.LATENCY)) { measurements.put( MetricType.LATENCY, - new Measurement(MetricType.LATENCY, - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos())) + new Measurement( + MetricType.LATENCY, + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos()) + ) ); } if (shouldCollect(MetricType.CPU)) { measurements.put( MetricType.CPU, - new Measurement(MetricType.CPU, - tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getCpuTimeInNanos()).mapToLong(Long::longValue).sum()) + new Measurement( + MetricType.CPU, + tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getCpuTimeInNanos()).mapToLong(Long::longValue).sum() + ) ); } if (shouldCollect(MetricType.MEMORY)) { measurements.put( MetricType.MEMORY, - new Measurement(MetricType.MEMORY, - tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getMemoryInBytes()).mapToLong(Long::longValue).sum()) + new Measurement( + MetricType.MEMORY, + tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getMemoryInBytes()).mapToLong(Long::longValue).sum() + ) ); } // TODO: Use David's QueryShape library to get the query shape hashcode diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java index 243feaa..0be051a 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java @@ -8,6 +8,9 @@ package org.opensearch.plugin.insights.core.service; +import java.util.HashMap; +import java.util.Map; +import java.util.PriorityQueue; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.collect.Tuple; @@ -17,11 +20,6 @@ import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.PriorityQueue; - /** * Handles grouping of search queries based on the GroupingType for the MetricType */ @@ -38,7 +36,13 @@ public class QueryGroupingService { private int topNSize; - public QueryGroupingService(MetricType metricType, GroupingType groupingType, DimensionType dimensionType, PriorityQueue topQueriesStore, int topNSize) { + public QueryGroupingService( + MetricType metricType, + GroupingType groupingType, + DimensionType dimensionType, + PriorityQueue topQueriesStore, + int topNSize + ) { this.groupingType = groupingType; this.metricType = metricType; this.dimensionType = dimensionType; @@ -190,11 +194,3 @@ public void updateTopNSize(int newSize) { this.topNSize = newSize; } } - - - - - - - - diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java index 5ff9398..856d0aa 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java @@ -8,14 +8,13 @@ package org.opensearch.plugin.insights.core.service; +import java.util.List; +import java.util.Objects; import org.opensearch.index.query.QueryBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.SortBuilder; -import java.util.List; -import java.util.Objects; - public class QueryShape { QueryBuilder query; AggregatorFactories.Builder aggregations; diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index 13f916a..d9a11a4 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -111,7 +111,13 @@ public class TopQueriesService { topQueriesStore = new PriorityQueue<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot = new AtomicReference<>(new ArrayList<>()); topQueriesHistorySnapshot = new AtomicReference<>(new ArrayList<>()); - queryGroupingService = new QueryGroupingService(metricType, QueryInsightsSettings.DEFAULT_GROUPING_TYPE, DimensionType.AVERAGE, topQueriesStore, topNSize); + queryGroupingService = new QueryGroupingService( + metricType, + QueryInsightsSettings.DEFAULT_GROUPING_TYPE, + DimensionType.AVERAGE, + topQueriesStore, + topNSize + ); } /** diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java b/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java index 5ebb98f..8409796 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/Attribute.java @@ -9,10 +9,8 @@ package org.opensearch.plugin.insights.rules.model; import java.io.IOException; -import java.io.InputStream; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java index fa3a2fb..67923a8 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java @@ -12,9 +12,9 @@ * Dimension type for a measurement. Default is NONE and average is used for grouping Top N queries by similarity. */ public enum DimensionType { - NONE, - AVERAGE, - SUM; + NONE, + AVERAGE, + SUM; public static DimensionType DEFUALT_DIMENSION_TYPE = NONE; } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java index 0790de0..46d2c6e 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java @@ -50,12 +50,7 @@ public static GroupingType getGroupingTypeFromSettingAndValidate(String settingV return GroupingType.valueOf(settingValue.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Invalid exporter type [%s], type should be one of %s", - settingValue, - allGroupingTypes() - ) + String.format(Locale.ROOT, "Invalid exporter type [%s], type should be one of %s", settingValue, allGroupingTypes()) ); } } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java index 01348e1..26b545a 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java @@ -8,17 +8,15 @@ package org.opensearch.plugin.insights.rules.model; +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; - -import java.io.IOException; -import java.util.Locale; -import java.util.Objects; /** * Measurement that is stored in the SearchQueryRecord. Measurement can be of a specific DimensionType and MetricType @@ -77,7 +75,7 @@ public void addMeasurement(Number toAdd) { setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); break; case AVERAGE: - count+=1; + count += 1; setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); break; default: @@ -149,10 +147,10 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Measurement that = (Measurement) o; - return count == that.count && - metricType == that.metricType && - Objects.equals(number, that.number) && - dimensionType == that.dimensionType; + return count == that.count + && metricType == that.metricType + && Objects.equals(number, that.number) + && dimensionType == that.dimensionType; } @Override diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java index 2be7d51..1808e76 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java @@ -113,7 +113,7 @@ public static Number getAverageMeasurement(Number total, int count, MetricType m case LATENCY: case MEMORY: case CPU: - return total.longValue()/count; + return total.longValue() / count; default: throw new IllegalArgumentException("Unsupported metric type: " + metricType); } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java index 0712ec1..e5da548 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java @@ -154,7 +154,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final ToXConten @Override public void writeTo(final StreamOutput out) throws IOException { out.writeLong(timestamp); - out.writeMap(measurements, (stream, metricType) -> MetricType.writeTo(out, metricType), (stream, measurement) -> measurement.writeTo(out)); + out.writeMap( + measurements, + (stream, metricType) -> MetricType.writeTo(out, metricType), + (stream, measurement) -> measurement.writeTo(out) + ); out.writeMap( attributes, (stream, attribute) -> Attribute.writeTo(out, attribute), diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java index 0c4cc2d..c9f7db2 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java @@ -71,7 +71,13 @@ public static List generateQueryInsightRecords(int count) { * @return List of records */ public static List generateQueryInsightRecords(int count, SearchSourceBuilder searchSourceBuilder) { - List records = generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, DimensionType.DEFUALT_DIMENSION_TYPE); + List records = generateQueryInsightRecords( + count, + count, + System.currentTimeMillis(), + 0, + DimensionType.DEFUALT_DIMENSION_TYPE + ); for (SearchQueryRecord record : records) { record.getAttributes().put(Attribute.SOURCE, searchSourceBuilder); } @@ -98,7 +104,13 @@ public static List generateQueryInsightRecords(int lower, int /** * Creates a List of random Query Insight Records for testing purpose with dimenstion type specified */ - public static List generateQueryInsightRecords(int lower, int upper, long startTimeStamp, long interval, DimensionType dimensionType) { + public static List generateQueryInsightRecords( + int lower, + int upper, + long startTimeStamp, + long interval, + DimensionType dimensionType + ) { List records = new ArrayList<>(); int countOfRecords = randomIntBetween(lower, upper); long timestamp = startTimeStamp; @@ -153,7 +165,11 @@ public static List generateQueryInsightRecords(int lower, int return records; } - public static List generateQueryInsightsRecordsWithMeasurement(int count, MetricType metricType, Number measurement) { + public static List generateQueryInsightsRecordsWithMeasurement( + int count, + MetricType metricType, + Number measurement + ) { List records = generateQueryInsightRecords(count); for (SearchQueryRecord record : records) { diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java index 278adc5..393f1d2 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java @@ -8,33 +8,39 @@ package org.opensearch.plugin.insights.core.service; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.PriorityQueue; +import java.util.Set; import org.junit.Before; import org.opensearch.plugin.insights.QueryInsightsTestUtils; import org.opensearch.plugin.insights.rules.model.Attribute; import org.opensearch.plugin.insights.rules.model.DimensionType; import org.opensearch.plugin.insights.rules.model.GroupingType; -import org.opensearch.plugin.insights.rules.model.Measurement; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; import org.opensearch.test.OpenSearchTestCase; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.PriorityQueue; -import java.util.Set; -import java.util.stream.Stream; - /** * Unit Tests for {@link QueryGroupingService}. */ public class QueryGroupingServiceTests extends OpenSearchTestCase { private QueryGroupingService queryGroupingService; - private PriorityQueue topQueriesStore = new PriorityQueue<>(100, (a, b) -> SearchQueryRecord.compare(a, b, MetricType.LATENCY)); + private PriorityQueue topQueriesStore = new PriorityQueue<>( + 100, + (a, b) -> SearchQueryRecord.compare(a, b, MetricType.LATENCY) + ); @Before public void setup() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.DEFUALT_DIMENSION_TYPE, topQueriesStore, 10); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.DEFUALT_DIMENSION_TYPE, + topQueriesStore, + 10 + ); } public void testWithAllDifferentHashcodes() { @@ -128,7 +134,13 @@ public void testAddRemoveFromMaxHeap() { } public void testInvalidGroupingType() { - QueryGroupingService invalidGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.NONE, DimensionType.DEFUALT_DIMENSION_TYPE, topQueriesStore, 10); + QueryGroupingService invalidGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.NONE, + DimensionType.DEFUALT_DIMENSION_TYPE, + topQueriesStore, + 10 + ); SearchQueryRecord record = QueryInsightsTestUtils.generateQueryInsightRecords(1).get(0); expectThrows(IllegalArgumentException.class, () -> invalidGroupingService.addQueryToGroup(record)); } @@ -198,7 +210,13 @@ public void testVaryingTopNSize() { } public void testAddMeasurementSumDimensionLatency() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.SUM, topQueriesStore, 10); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.SUM, + topQueriesStore, + 10 + ); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); @@ -216,7 +234,13 @@ public void testAddMeasurementSumDimensionLatency() { } public void testAddMeasurementAverageDimensionLatency() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 10); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 10 + ); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); @@ -229,7 +253,7 @@ public void testAddMeasurementAverageDimensionLatency() { for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); aggregatedRecord = queryGroupingService.addQueryToGroup(record); - expectedCount +=1; + expectedCount += 1; } long expectedAverage = (long) expectedSum / expectedCount; @@ -237,7 +261,13 @@ public void testAddMeasurementAverageDimensionLatency() { } public void testAddMeasurementNoneDimensionLatency() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.NONE, topQueriesStore, 10); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.NONE, + topQueriesStore, + 10 + ); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); @@ -273,7 +303,13 @@ public void testAddMeasurementSumDimensionCpu() { } public void testAddMeasurementAverageDimensionCpu() { - queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 10); + queryGroupingService = new QueryGroupingService( + MetricType.CPU, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 10 + ); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); @@ -286,7 +322,7 @@ public void testAddMeasurementAverageDimensionCpu() { for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); aggregatedRecord = queryGroupingService.addQueryToGroup(record); - expectedCount +=1; + expectedCount += 1; } long expectedAverage = (long) expectedSum / expectedCount; @@ -323,22 +359,34 @@ public void testNoneGroupingTypeIllegalArgumentException() { Number expectedSum = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); - assertThrows(IllegalArgumentException.class, () -> { - queryGroupingService.addQueryToGroup(record); - }); + assertThrows(IllegalArgumentException.class, () -> { queryGroupingService.addQueryToGroup(record); }); } } // 1. New query group not existing added to MIN public void testNewGroupAddedToMin() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 2; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1000 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1100 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); @@ -358,18 +406,36 @@ public void testNewGroupAddedToMin() { // 2. New query group not existing added to MIN and overflows to MAX public void testNewGroupOverflowsMinToMax() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 2; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1000 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1100 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 900 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); @@ -389,18 +455,36 @@ public void testNewGroupOverflowsMinToMax() { // 3. New query group not existing added to MIN and causes other group to overflow to MAX public void testNewGroupCausesOtherGroupOverflowMinToMax() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 2; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1000 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1100 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1200); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1200 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); @@ -420,23 +504,45 @@ public void testNewGroupCausesOtherGroupOverflowMinToMax() { // 4. Existing query group update to MIN increases average public void testExistingGroupUpdateToMinIncreaseAverage() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 1; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1000 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1200); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1200 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1100 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); // Average will decrease when this record added and record should be moved out of Top N - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1300); + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1300 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records4, 3); @@ -456,23 +562,45 @@ public void testExistingGroupUpdateToMinIncreaseAverage() { // 5. Existing query group update to MIN decrease average - stay in MIN public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 1; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1000 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 600); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 600 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1100 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); // Average will decrease when this record added and record should be moved out of Top N - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 700); + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 700 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records4, 3); @@ -492,23 +620,45 @@ public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { // 6. Existing query group update to MIN decrease average - overflows to MAX public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 1; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1000 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1100 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1199); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1199 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); // Average will decrease when this record added and record should be moved out of Top N - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1); + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records4, 3); @@ -528,22 +678,44 @@ public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { // 7. Existing query group update to MAX increases average - stay in MAX public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 1; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 950 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 975 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 900 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 920); + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 920 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records4, 3); @@ -563,22 +735,44 @@ public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { // 8. Existing query group update to MAX increases average - promote to MIN public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 1; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 950 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 975 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 900 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1100); + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1100 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records4, 3); @@ -598,22 +792,44 @@ public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { // 9. Existing query group update to MAX decrease average public void testExistingGroupUpdateToMaxDecreaseAverage() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 1; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 950 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 975 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 900 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 800); + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 800 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records4, 3); @@ -632,22 +848,44 @@ public void testExistingGroupUpdateToMaxDecreaseAverage() { } public void testSwitchGroupingTypeToUserId() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 1; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 950 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 975 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 900 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 800); + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 800 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records4, 3); @@ -669,22 +907,44 @@ public void testSwitchGroupingTypeToUserId() { } public void testSwitchGroupingTypeToNone() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 1; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 950); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 950 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 975); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 975 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 900 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 800); + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 800 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records4, 3); @@ -704,36 +964,64 @@ public void testSwitchGroupingTypeToNone() { queryGroupingService.setGroupingType(GroupingType.NONE); assertEquals(0, queryGroupingService.numberOfTopGroups()); - assertThrows(IllegalArgumentException.class, () -> { - queryGroupingService.addQueryToGroup(allRecords.get(0).get(0)); - }); + assertThrows(IllegalArgumentException.class, () -> { queryGroupingService.addQueryToGroup(allRecords.get(0).get(0)); }); } public void testMultipleQueryGroupsUpdates() { - queryGroupingService = new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, DimensionType.AVERAGE, topQueriesStore, 2); + queryGroupingService = new QueryGroupingService( + MetricType.LATENCY, + GroupingType.SIMILARITY, + DimensionType.AVERAGE, + topQueriesStore, + 2 + ); int numOfRecords = 1; - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1000 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1000); + List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1000 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 900); + List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 900 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records3, 3); - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 800); + List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 800 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records4, 3); - List records5 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 400); + List records5 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 400 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records5, 2); - List records6 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numOfRecords, MetricType.LATENCY, 1200); + List records6 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numOfRecords, + MetricType.LATENCY, + 1200 + ); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateHashcode(records6, 1); diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java index a174c77..9565bd1 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import java.util.List; import org.junit.Before; import org.opensearch.client.Client; import org.opensearch.common.settings.ClusterSettings; @@ -23,7 +24,6 @@ import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; -import java.util.List; /** * Unit Tests for {@link QueryInsightsService}. @@ -113,7 +113,11 @@ public void testFeaturesEnableDisable() { public void testAddRecordGroupBySimilarityWithDifferentGroups() { int numberOfRecordsRequired = 10; - List records = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numberOfRecordsRequired, MetricType.LATENCY, 5); + List records = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numberOfRecordsRequired, + MetricType.LATENCY, + 5 + ); queryInsightsService.validateAndSetGrouping(GroupingType.SIMILARITY.getValue()); assertEquals(queryInsightsService.getGrouping(), GroupingType.SIMILARITY); @@ -122,7 +126,7 @@ public void testAddRecordGroupBySimilarityWithDifferentGroups() { assertTrue(queryInsightsService.addRecord(records.get(i))); } // exceed capacity but handoff to grouping - assertTrue(queryInsightsService.addRecord(records.get(numberOfRecordsRequired-1))); + assertTrue(queryInsightsService.addRecord(records.get(numberOfRecordsRequired - 1))); queryInsightsService.drainRecords(); @@ -134,7 +138,11 @@ public void testAddRecordGroupBySimilarityWithDifferentGroups() { public void testAddRecordGroupBySimilarityWithOneGroup() { int numberOfRecordsRequired = 10; - List records = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement(numberOfRecordsRequired, MetricType.LATENCY, 5); + List records = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( + numberOfRecordsRequired, + MetricType.LATENCY, + 5 + ); QueryInsightsTestUtils.populateSameQueryHashcodes(records); queryInsightsService.validateAndSetGrouping(GroupingType.SIMILARITY.getValue()); @@ -144,13 +152,10 @@ public void testAddRecordGroupBySimilarityWithOneGroup() { assertTrue(queryInsightsService.addRecord(records.get(i))); } // exceed capacity but handoff to grouping service - assertTrue(queryInsightsService.addRecord(records.get(numberOfRecordsRequired-1))); + assertTrue(queryInsightsService.addRecord(records.get(numberOfRecordsRequired - 1))); queryInsightsService.drainRecords(); - assertEquals( - 1, - queryInsightsService.getTopQueriesService(MetricType.LATENCY).getTopQueriesRecords(false).size() - ); + assertEquals(1, queryInsightsService.getTopQueriesService(MetricType.LATENCY).getTopQueriesRecords(false).size()); } public void testAddRecordGroupBySimilarityWithTwoGroups() { @@ -169,9 +174,6 @@ public void testAddRecordGroupBySimilarityWithTwoGroups() { } queryInsightsService.drainRecords(); - assertEquals( - 2, - queryInsightsService.getTopQueriesService(MetricType.LATENCY).getTopQueriesRecords(false).size() - ); + assertEquals(2, queryInsightsService.getTopQueriesService(MetricType.LATENCY).getTopQueriesRecords(false).size()); } } From f56df30f5ac413a053eef52809d12678b2883e42 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 2 Aug 2024 11:54:03 -0700 Subject: [PATCH 03/29] Build fix Signed-off-by: Siddhant Deshmukh --- .../plugin/insights/core/service/TopQueriesService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index d9a11a4..af1180f 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -35,6 +35,8 @@ import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporter; import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory; import org.opensearch.plugin.insights.core.exporter.SinkType; +import org.opensearch.plugin.insights.rules.model.DimensionType; +import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; import org.opensearch.plugin.insights.settings.QueryInsightsSettings; From 0995a3a589deae5209cd671c207bb1d9e2b91bdf Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 2 Aug 2024 13:33:06 -0700 Subject: [PATCH 04/29] Properly configure settings update consumer Signed-off-by: Siddhant Deshmukh --- .../core/listener/QueryInsightsListener.java | 9 +++++++-- .../core/service/QueryInsightsService.java | 15 +++++++++++---- .../plugin/insights/rules/model/GroupingType.java | 2 +- .../core/service/QueryInsightsServiceTests.java | 6 +++--- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index b2b44f5..4467c00 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -109,8 +109,13 @@ public QueryInsightsListener( setSearchQueryMetricsEnabled(clusterService.getClusterSettings().get(SEARCH_QUERY_METRICS_ENABLED_SETTING)); clusterService.getClusterSettings() - .addSettingsUpdateConsumer(TOP_N_QUERIES_GROUP_BY, v -> this.queryInsightsService.validateAndSetGrouping(v)); - this.queryInsightsService.validateAndSetGrouping(clusterService.getClusterSettings().get(TOP_N_QUERIES_GROUP_BY)); + .addSettingsUpdateConsumer( + TOP_N_QUERIES_GROUP_BY, + v -> this.queryInsightsService.setGrouping(v), + v -> this.queryInsightsService.validateGrouping(v) + ); + this.queryInsightsService.validateGrouping(clusterService.getClusterSettings().get(TOP_N_QUERIES_GROUP_BY)); + this.queryInsightsService.setGrouping(clusterService.getClusterSettings().get(TOP_N_QUERIES_GROUP_BY)); } /** diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java index c6b9dcf..d82aba5 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java @@ -194,11 +194,18 @@ public void enableCollection(final MetricType metricType, final boolean enable) } /** - * Set grouping based on the given GroupingType setting for the given metric type - * - * @param groupingTypeSetting grouping type setting + * Validate grouping + * @param groupingTypeSetting grouping + */ + public void validateGrouping(final String groupingTypeSetting) { + GroupingType.getGroupingTypeFromSettingAndValidate(groupingTypeSetting); + } + + /** + * Set grouping + * @param groupingTypeSetting grouping */ - public void validateAndSetGrouping(final String groupingTypeSetting) { + public void setGrouping(final String groupingTypeSetting) { GroupingType newGroupingType = GroupingType.getGroupingTypeFromSettingAndValidate(groupingTypeSetting); GroupingType oldGroupingType = groupingType; diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java index 46d2c6e..9ddabc5 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java @@ -50,7 +50,7 @@ public static GroupingType getGroupingTypeFromSettingAndValidate(String settingV return GroupingType.valueOf(settingValue.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( - String.format(Locale.ROOT, "Invalid exporter type [%s], type should be one of %s", settingValue, allGroupingTypes()) + String.format(Locale.ROOT, "Invalid grouping type [%s], type should be one of %s", settingValue, allGroupingTypes()) ); } } diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java index 9565bd1..ce4d4eb 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java @@ -119,7 +119,7 @@ public void testAddRecordGroupBySimilarityWithDifferentGroups() { 5 ); - queryInsightsService.validateAndSetGrouping(GroupingType.SIMILARITY.getValue()); + queryInsightsService.setGrouping(GroupingType.SIMILARITY.getValue()); assertEquals(queryInsightsService.getGrouping(), GroupingType.SIMILARITY); for (int i = 0; i < numberOfRecordsRequired; i++) { @@ -145,7 +145,7 @@ public void testAddRecordGroupBySimilarityWithOneGroup() { ); QueryInsightsTestUtils.populateSameQueryHashcodes(records); - queryInsightsService.validateAndSetGrouping(GroupingType.SIMILARITY.getValue()); + queryInsightsService.setGrouping(GroupingType.SIMILARITY.getValue()); assertEquals(queryInsightsService.getGrouping(), GroupingType.SIMILARITY); for (int i = 0; i < numberOfRecordsRequired; i++) { @@ -165,7 +165,7 @@ public void testAddRecordGroupBySimilarityWithTwoGroups() { List records2 = QueryInsightsTestUtils.generateQueryInsightRecords(2, 2, System.currentTimeMillis(), 0); QueryInsightsTestUtils.populateHashcode(records2, 2); - queryInsightsService.validateAndSetGrouping(GroupingType.SIMILARITY.getValue()); + queryInsightsService.setGrouping(GroupingType.SIMILARITY.getValue()); assertEquals(queryInsightsService.getGrouping(), GroupingType.SIMILARITY); for (int i = 0; i < 2; i++) { From af90f5c20992cd293a0e4c73f32bf2a7884a589c Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Mon, 5 Aug 2024 13:11:00 -0700 Subject: [PATCH 05/29] Address review comments Signed-off-by: Siddhant Deshmukh --- .../core/service/QueryGroupingService.java | 75 +++++++++-- .../core/service/QueryInsightsService.java | 4 +- .../core/service/TopQueriesService.java | 4 +- ...imensionType.java => AggregationType.java} | 6 +- .../insights/rules/model/GroupingType.java | 3 +- .../insights/rules/model/Measurement.java | 52 +++---- .../rules/model/SearchQueryRecord.java | 11 +- .../settings/QueryInsightsSettings.java | 2 +- .../insights/QueryInsightsTestUtils.java | 24 ++-- .../service/QueryGroupingServiceTests.java | 127 +++++------------- .../top_queries/TopQueriesResponseTests.java | 2 +- 11 files changed, 149 insertions(+), 161 deletions(-) rename src/main/java/org/opensearch/plugin/insights/rules/model/{DimensionType.java => AggregationType.java} (58%) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java index 0be051a..a61cc29 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java @@ -14,38 +14,71 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.collect.Tuple; +import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.Attribute; -import org.opensearch.plugin.insights.rules.model.DimensionType; import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; /** * Handles grouping of search queries based on the GroupingType for the MetricType + * Following algorithm : */ public class QueryGroupingService { + /** + * Logger + */ private static final Logger log = LogManager.getLogger(QueryGroupingService.class); + /** + * Grouping type for the current grouping service + */ private GroupingType groupingType; + /** + * Metric type for the current grouping service + */ private MetricType metricType; - private DimensionType dimensionType; + /** + * Aggregation type for the current grouping service + */ + private AggregationType aggregationType; + /** + * Map storing groupingId to Tuple containing Aggregate search query record and boolean. + * SearchQueryRecord: Aggregate search query record to store the aggregate of a metric type based on the aggregation type.. + * Example: Average latency. This query record will be used to store the average latency for multiple query records + * in this case. + * boolean: True if the aggregate record is in the Top N queries priority query (min heap) and False if the aggregate + * record is in the Max Heap + */ private Map> groupIdToAggSearchQueryRecord; + /** + * Min heap to keep track of the Top N query groups and is passed from TopQueriesService as the topQueriesStore + */ private PriorityQueue minHeapTopQueriesStore; + /** + * The Max heap is an overflow data structure used to manage records that exceed the capacity of the Min heap. + * It stores all records not included in the Top N query results. When the aggregate measurement for one of these + * records is updated and it now qualifies as part of the Top N, the record is moved from the Max heap to the Min heap, + * and the records are rearranged accordingly. + */ private PriorityQueue maxHeapQueryStore; + /** + * Top N size based on the configuration set + */ private int topNSize; public QueryGroupingService( MetricType metricType, GroupingType groupingType, - DimensionType dimensionType, + AggregationType aggregationType, PriorityQueue topQueriesStore, int topNSize ) { this.groupingType = groupingType; this.metricType = metricType; - this.dimensionType = dimensionType; + this.aggregationType = aggregationType; this.groupIdToAggSearchQueryRecord = new HashMap<>(); this.minHeapTopQueriesStore = topQueriesStore; this.maxHeapQueryStore = new PriorityQueue<>((a, b) -> SearchQueryRecord.compare(b, a, metricType)); @@ -66,14 +99,25 @@ public SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord) { SearchQueryRecord aggregateSearchQueryRecord; String groupId = getGroupingId(searchQueryRecord); - // New group + // New group added to the grouping service + // Add to min PQ and overflow records to max PQ (if the number of records in the min PQ exceeds the configured size N) if (!groupIdToAggSearchQueryRecord.containsKey(groupId)) { aggregateSearchQueryRecord = searchQueryRecord; aggregateSearchQueryRecord.setGroupingId(groupId); - aggregateSearchQueryRecord.setMeasurementDimension(metricType, dimensionType); - addToMinPQ(aggregateSearchQueryRecord, groupId); + aggregateSearchQueryRecord.setMeasurementAggregation(metricType, aggregationType); + addToMinPQOverflowToMaxPQ(aggregateSearchQueryRecord, groupId); } - // Old group + // Existing group being updated to the grouping service + // 1. If present in min PQ + // - remove the record from the min PQ + // - update the aggregate record (aggregate measurement could increase or decrease) + // - If max PQ contains elements, add to max PQ and promote any records to min PQ + // - If max PQ is empty, add to min PQ and overflow any records to max PQ + // 2. If present in max PQ + // - remove the record from the max PQ + // - update the aggregate record (aggregate measurement could increase or decrease) + // - If min PQ is full, add to min PQ and overflow any records to max PQ + // - else, add to max PQ and promote any records to min PQ else { aggregateSearchQueryRecord = groupIdToAggSearchQueryRecord.get(groupId).v1(); boolean isPresentInMinPQ = groupIdToAggSearchQueryRecord.get(groupId).v2(); @@ -86,7 +130,7 @@ public SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord) { return aggregateSearchQueryRecord; } - private void addToMinPQ(SearchQueryRecord searchQueryRecord, String groupId) { + private void addToMinPQOverflowToMaxPQ(SearchQueryRecord searchQueryRecord, String groupId) { minHeapTopQueriesStore.add(searchQueryRecord); groupIdToAggSearchQueryRecord.put(groupId, new Tuple<>(searchQueryRecord, true)); @@ -102,8 +146,11 @@ private void updateToMaxPQ(SearchQueryRecord searchQueryRecord, SearchQueryRecor Number measurementToAdd = searchQueryRecord.getMeasurement(metricType); aggregateSearchQueryRecord.addMeasurement(metricType, measurementToAdd); - addToMinPQ(aggregateSearchQueryRecord, groupId); - + if (minHeapTopQueriesStore.size() >= topNSize) { + addToMinPQOverflowToMaxPQ(aggregateSearchQueryRecord, groupId); + } else { + addToMaxPQPromoteToMinPQ(aggregateSearchQueryRecord, groupId); + } } private void updateToMinPQ(SearchQueryRecord searchQueryRecord, SearchQueryRecord aggregateSearchQueryRecord, String groupId) { @@ -112,13 +159,13 @@ private void updateToMinPQ(SearchQueryRecord searchQueryRecord, SearchQueryRecor aggregateSearchQueryRecord.addMeasurement(metricType, measurementToAdd); if (maxHeapQueryStore.size() > 0) { - addToMaxPQ(aggregateSearchQueryRecord, groupId); + addToMaxPQPromoteToMinPQ(aggregateSearchQueryRecord, groupId); } else { - addToMinPQ(aggregateSearchQueryRecord, groupId); + addToMinPQOverflowToMaxPQ(aggregateSearchQueryRecord, groupId); } } - private void addToMaxPQ(SearchQueryRecord aggregateSearchQueryRecord, String groupId) { + private void addToMaxPQPromoteToMinPQ(SearchQueryRecord aggregateSearchQueryRecord, String groupId) { maxHeapQueryStore.add(aggregateSearchQueryRecord); groupIdToAggSearchQueryRecord.put(groupId, new Tuple<>(aggregateSearchQueryRecord, false)); diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java index d82aba5..7082298 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java @@ -194,8 +194,8 @@ public void enableCollection(final MetricType metricType, final boolean enable) } /** - * Validate grouping - * @param groupingTypeSetting grouping + * Validate grouping given grouping type setting + * @param groupingTypeSetting grouping setting */ public void validateGrouping(final String groupingTypeSetting) { GroupingType.getGroupingTypeFromSettingAndValidate(groupingTypeSetting); diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index af1180f..d48ae43 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -35,7 +35,7 @@ import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporter; import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory; import org.opensearch.plugin.insights.core.exporter.SinkType; -import org.opensearch.plugin.insights.rules.model.DimensionType; +import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; @@ -116,7 +116,7 @@ public class TopQueriesService { queryGroupingService = new QueryGroupingService( metricType, QueryInsightsSettings.DEFAULT_GROUPING_TYPE, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, topNSize ); diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/AggregationType.java similarity index 58% rename from src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java rename to src/main/java/org/opensearch/plugin/insights/rules/model/AggregationType.java index 67923a8..c65e33b 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/DimensionType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/AggregationType.java @@ -9,12 +9,12 @@ package org.opensearch.plugin.insights.rules.model; /** - * Dimension type for a measurement. Default is NONE and average is used for grouping Top N queries by similarity. + * Aggregation type for a measurement. Default is NONE and average is used for grouping Top N queries by similarity. */ -public enum DimensionType { +public enum AggregationType { NONE, AVERAGE, SUM; - public static DimensionType DEFUALT_DIMENSION_TYPE = NONE; + public static AggregationType DEFUALT_AGGREGATION_TYPE = NONE; } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java index 9ddabc5..318870a 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/GroupingType.java @@ -18,8 +18,7 @@ */ public enum GroupingType { NONE("none"), - SIMILARITY("similarity"), - USER_ID("userid"); + SIMILARITY("similarity"); private final String stringValue; diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java index 26b545a..ffccedd 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java @@ -19,11 +19,11 @@ import org.opensearch.core.xcontent.XContentBuilder; /** - * Measurement that is stored in the SearchQueryRecord. Measurement can be of a specific DimensionType and MetricType + * Measurement that is stored in the SearchQueryRecord. Measurement can be of a specific AggregationType and MetricType */ public class Measurement implements ToXContentObject, Writeable { private static int DEFAULT_COUNT = 1; - private DimensionType dimensionType; + private AggregationType aggregationType; private MetricType metricType; private Number number; private int count; @@ -33,23 +33,23 @@ public class Measurement implements ToXContentObject, Writeable { * @param metricType metricType * @param number number * @param count count - * @param dimensionType dimensionType + * @param aggregationType aggregationType */ - public Measurement(MetricType metricType, Number number, int count, DimensionType dimensionType) { + public Measurement(MetricType metricType, Number number, int count, AggregationType aggregationType) { this.metricType = metricType; this.number = number; this.count = count; - this.dimensionType = dimensionType; + this.aggregationType = aggregationType; } /** * Constructor * @param metricType metricType * @param number number - * @param dimensionType dimensionType + * @param aggregationType aggregationType */ - public Measurement(MetricType metricType, Number number, DimensionType dimensionType) { - this(metricType, number, DEFAULT_COUNT, dimensionType); + public Measurement(MetricType metricType, Number number, AggregationType aggregationType) { + this(metricType, number, DEFAULT_COUNT, aggregationType); } /** @@ -58,17 +58,17 @@ public Measurement(MetricType metricType, Number number, DimensionType dimension * @param number number */ public Measurement(MetricType metricType, Number number) { - this(metricType, number, DEFAULT_COUNT, DimensionType.DEFUALT_DIMENSION_TYPE); + this(metricType, number, DEFAULT_COUNT, AggregationType.DEFUALT_AGGREGATION_TYPE); } /** - * Add measurement number to the current number based on the dimension type + * Add measurement number to the current number based on the aggregationType * @param toAdd number to add */ public void addMeasurement(Number toAdd) { - switch (dimensionType) { + switch (aggregationType) { case NONE: - dimensionType = DimensionType.SUM; + aggregationType = AggregationType.SUM; setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); break; case SUM: @@ -79,23 +79,23 @@ public void addMeasurement(Number toAdd) { setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); break; default: - throw new IllegalArgumentException("The following dimension type is not supported : " + dimensionType); + throw new IllegalArgumentException("The following aggregation type is not supported : " + aggregationType); } } /** - * Get measurement number based on the dimension type + * Get measurement number based on the aggragation type * @return measurement number */ public Number getMeasurement() { - switch (dimensionType) { + switch (aggregationType) { case NONE: case SUM: return number; case AVERAGE: return MetricType.getAverageMeasurement(number, count, metricType); default: - throw new IllegalArgumentException("Dimension Type should be set for measurement."); + throw new IllegalArgumentException("Aggregation Type should be set for measurement."); } } @@ -108,11 +108,11 @@ public void setMeasurement(Number measurement) { } /** - * Set dimension type - * @param dimensionType dimension type + * Set aggregation type + * @param aggregationType aggregation type */ - public void setDimensionType(DimensionType dimensionType) { - this.dimensionType = dimensionType; + public void setAggregationType(AggregationType aggregationType) { + this.aggregationType = aggregationType; } @Override @@ -121,7 +121,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.field("metricType", metricType.toString()); builder.field("number", number); builder.field("count", count); - builder.field("dimensionType", dimensionType.toString()); + builder.field("aggregationType", aggregationType.toString()); builder.endObject(); return builder; } @@ -131,15 +131,15 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(metricType.toString()); out.writeGenericValue(metricType.parseValue(number)); out.writeInt(count); - out.writeString(dimensionType.toString()); + out.writeString(aggregationType.toString()); } public static Measurement readFromStream(StreamInput in) throws IOException { MetricType metricType = MetricType.valueOf(in.readString().toUpperCase(Locale.ROOT)); Number number = metricType.parseValue(in.readGenericValue()); int count = in.readInt(); - DimensionType dimensionType = DimensionType.valueOf(in.readString()); - return new Measurement(metricType, number, count, dimensionType); + AggregationType aggregationType = AggregationType.valueOf(in.readString()); + return new Measurement(metricType, number, count, aggregationType); } @Override @@ -150,11 +150,11 @@ public boolean equals(Object o) { return count == that.count && metricType == that.metricType && Objects.equals(number, that.number) - && dimensionType == that.dimensionType; + && aggregationType == that.aggregationType; } @Override public int hashCode() { - return Objects.hash(metricType, number, count, dimensionType); + return Objects.hash(metricType, number, count, aggregationType); } } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java index e5da548..8246689 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java @@ -26,6 +26,7 @@ * which contains extensive information related to a search query. */ public class SearchQueryRecord implements ToXContentObject, Writeable { + public static final String MEASUREMENTS = "measurements"; private final long timestamp; private final Map measurements; private final Map attributes; @@ -98,12 +99,12 @@ public void addMeasurement(final MetricType name, Number measurement) { } /** - * Set the dimension type for measurement + * Set the aggregation type for measurement * @param name the name of the measurement - * @param dimensionType Dimension type to set + * @param aggregationType Aggregation type to set */ - public void setMeasurementDimension(final MetricType name, DimensionType dimensionType) { - measurements.get(name).setDimensionType(dimensionType); + public void setMeasurementAggregation(final MetricType name, AggregationType aggregationType) { + measurements.get(name).setAggregationType(aggregationType); } public Map getMeasurements() { @@ -136,7 +137,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final ToXConten for (Map.Entry entry : attributes.entrySet()) { builder.field(entry.getKey().toString(), entry.getValue()); } - builder.startObject("measurements"); + builder.startObject(MEASUREMENTS); for (Map.Entry entry : measurements.entrySet()) { builder.field(entry.getKey().toString()); // MetricType as field name entry.getValue().toXContent(builder, params); // Serialize Measurement object diff --git a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java index 46dd078..876c122 100644 --- a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java +++ b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java @@ -34,7 +34,7 @@ public class QueryInsightsSettings { /** * Max number of requests for the consumer to collect at one time */ - public static final int QUERY_RECORD_QUEUE_CAPACITY = 100000; + public static final int QUERY_RECORD_QUEUE_CAPACITY = 1000; /** * Time interval for record queue consumer to run */ diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java index c9f7db2..6cae81e 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java @@ -41,8 +41,8 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.plugin.insights.rules.action.top_queries.TopQueries; +import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.Attribute; -import org.opensearch.plugin.insights.rules.model.DimensionType; import org.opensearch.plugin.insights.rules.model.Measurement; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; @@ -61,7 +61,7 @@ public QueryInsightsTestUtils() {} * @return List of records */ public static List generateQueryInsightRecords(int count) { - return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, DimensionType.DEFUALT_DIMENSION_TYPE); + return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, AggregationType.DEFUALT_AGGREGATION_TYPE); } /** @@ -76,7 +76,7 @@ public static List generateQueryInsightRecords(int count, Sea count, System.currentTimeMillis(), 0, - DimensionType.DEFUALT_DIMENSION_TYPE + AggregationType.DEFUALT_AGGREGATION_TYPE ); for (SearchQueryRecord record : records) { record.getAttributes().put(Attribute.SOURCE, searchSourceBuilder); @@ -85,20 +85,20 @@ public static List generateQueryInsightRecords(int count, Sea } /** - * Returns list of randomly generated search query records with specific dimension type for measurements + * Returns list of randomly generated search query records with specific aggregation type for measurements * @param count number of records - * @param dimensionType source + * @param aggregationType source * @return List of records */ - public static List generateQueryInsightRecords(int count, DimensionType dimensionType) { - return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, dimensionType); + public static List generateQueryInsightRecords(int count, AggregationType aggregationType) { + return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, aggregationType); } /** * Creates a List of random Query Insight Records for testing purpose */ public static List generateQueryInsightRecords(int lower, int upper, long startTimeStamp, long interval) { - return generateQueryInsightRecords(lower, upper, startTimeStamp, interval, DimensionType.NONE); + return generateQueryInsightRecords(lower, upper, startTimeStamp, interval, AggregationType.NONE); } /** @@ -109,7 +109,7 @@ public static List generateQueryInsightRecords( int upper, long startTimeStamp, long interval, - DimensionType dimensionType + AggregationType aggregationType ) { List records = new ArrayList<>(); int countOfRecords = randomIntBetween(lower, upper); @@ -119,9 +119,9 @@ public static List generateQueryInsightRecords( long cpuValue = randomLongBetween(1000, 10000); long memoryValue = randomLongBetween(1000, 10000); Map measurements = new LinkedHashMap<>(); - measurements.put(MetricType.LATENCY, new Measurement(MetricType.LATENCY, latencyValue, dimensionType)); - measurements.put(MetricType.CPU, new Measurement(MetricType.CPU, cpuValue, dimensionType)); - measurements.put(MetricType.MEMORY, new Measurement(MetricType.MEMORY, memoryValue, dimensionType)); + measurements.put(MetricType.LATENCY, new Measurement(MetricType.LATENCY, latencyValue, aggregationType)); + measurements.put(MetricType.CPU, new Measurement(MetricType.CPU, cpuValue, aggregationType)); + measurements.put(MetricType.MEMORY, new Measurement(MetricType.MEMORY, memoryValue, aggregationType)); Map phaseLatencyMap = new LinkedHashMap<>(); int countOfPhases = randomIntBetween(2, 5); diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java index 393f1d2..34ea56e 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java @@ -15,8 +15,8 @@ import java.util.Set; import org.junit.Before; import org.opensearch.plugin.insights.QueryInsightsTestUtils; +import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.Attribute; -import org.opensearch.plugin.insights.rules.model.DimensionType; import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; @@ -37,7 +37,7 @@ public void setup() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.DEFUALT_DIMENSION_TYPE, + AggregationType.DEFUALT_AGGREGATION_TYPE, topQueriesStore, 10 ); @@ -137,7 +137,7 @@ public void testInvalidGroupingType() { QueryGroupingService invalidGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.NONE, - DimensionType.DEFUALT_DIMENSION_TYPE, + AggregationType.DEFUALT_AGGREGATION_TYPE, topQueriesStore, 10 ); @@ -209,16 +209,16 @@ public void testVaryingTopNSize() { assertEquals(15, queryGroupingService.numberOfTopGroups()); // Should reflect the updated top N size } - public void testAddMeasurementSumDimensionLatency() { + public void testAddMeasurementSumAggregationLatency() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.SUM, + AggregationType.SUM, topQueriesStore, 10 ); int numOfRecords = 10; - List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateSameQueryHashcodes(records); @@ -233,16 +233,16 @@ public void testAddMeasurementSumDimensionLatency() { assertEquals(expectedSum, aggregatedRecord.getMeasurement(MetricType.LATENCY)); } - public void testAddMeasurementAverageDimensionLatency() { + public void testAddMeasurementAverageAggregationLatency() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 10 ); int numOfRecords = 10; - List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateSameQueryHashcodes(records); @@ -260,16 +260,16 @@ public void testAddMeasurementAverageDimensionLatency() { assertEquals(expectedAverage, aggregatedRecord.getMeasurement(MetricType.LATENCY)); } - public void testAddMeasurementNoneDimensionLatency() { + public void testAddMeasurementNoneAggregationLatency() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.NONE, + AggregationType.NONE, topQueriesStore, 10 ); int numOfRecords = 10; - List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateSameQueryHashcodes(records); @@ -284,10 +284,10 @@ public void testAddMeasurementNoneDimensionLatency() { assertEquals(expectedSum, aggregatedRecord.getMeasurement(MetricType.LATENCY)); } - public void testAddMeasurementSumDimensionCpu() { - queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, DimensionType.SUM, topQueriesStore, 10); + public void testAddMeasurementSumAggregationCpu() { + queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.SUM, topQueriesStore, 10); int numOfRecords = 10; - List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateSameQueryHashcodes(records); @@ -302,16 +302,16 @@ public void testAddMeasurementSumDimensionCpu() { assertEquals(expectedSum, aggregatedRecord.getMeasurement(MetricType.CPU)); } - public void testAddMeasurementAverageDimensionCpu() { + public void testAddMeasurementAverageAggregationCpu() { queryGroupingService = new QueryGroupingService( MetricType.CPU, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 10 ); int numOfRecords = 10; - List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateSameQueryHashcodes(records); @@ -329,10 +329,10 @@ public void testAddMeasurementAverageDimensionCpu() { assertEquals(expectedAverage, aggregatedRecord.getMeasurement(MetricType.CPU)); } - public void testAddMeasurementNoneDimensionCpu() { - queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, DimensionType.NONE, topQueriesStore, 10); + public void testAddMeasurementNoneAggregationCpu() { + queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.NONE, topQueriesStore, 10); int numOfRecords = 10; - List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateSameQueryHashcodes(records); @@ -348,9 +348,9 @@ public void testAddMeasurementNoneDimensionCpu() { } public void testNoneGroupingTypeIllegalArgumentException() { - queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.NONE, DimensionType.NONE, topQueriesStore, 10); + queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.NONE, AggregationType.NONE, topQueriesStore, 10); int numOfRecords = 10; - List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, DimensionType.NONE); + List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateSameQueryHashcodes(records); @@ -368,7 +368,7 @@ public void testNewGroupAddedToMin() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -409,7 +409,7 @@ public void testNewGroupOverflowsMinToMax() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -458,7 +458,7 @@ public void testNewGroupCausesOtherGroupOverflowMinToMax() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -507,7 +507,7 @@ public void testExistingGroupUpdateToMinIncreaseAverage() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -565,7 +565,7 @@ public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -623,7 +623,7 @@ public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -681,7 +681,7 @@ public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -738,7 +738,7 @@ public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -795,7 +795,7 @@ public void testExistingGroupUpdateToMaxDecreaseAverage() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -847,70 +847,11 @@ public void testExistingGroupUpdateToMaxDecreaseAverage() { assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - public void testSwitchGroupingTypeToUserId() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - DimensionType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 1; - - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 950 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); - - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 975 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); - - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 900 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); - - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 800 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records4, 3); - - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3, records4); - - for (List recordList : allRecords) { - for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); - } - } - - assertEquals(2, queryGroupingService.numberOfTopGroups()); - assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); - assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); - - queryGroupingService.setGroupingType(GroupingType.USER_ID); - assertEquals(0, queryGroupingService.numberOfTopGroups()); - } - public void testSwitchGroupingTypeToNone() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); @@ -971,7 +912,7 @@ public void testMultipleQueryGroupsUpdates() { queryGroupingService = new QueryGroupingService( MetricType.LATENCY, GroupingType.SIMILARITY, - DimensionType.AVERAGE, + AggregationType.AVERAGE, topQueriesStore, 2 ); diff --git a/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java b/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java index 2f75052..01748c8 100644 --- a/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java +++ b/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java @@ -41,7 +41,7 @@ public void testSerialize() throws Exception { public void testToXContent() throws IOException { char[] expectedXcontent = - "{\"top_queries\":[{\"timestamp\":1706574180000,\"node_id\":\"node_for_top_queries_test\",\"search_type\":\"query_then_fetch\",\"measurements\":{\"latency\":{\"metricType\":\"latency\",\"number\":1,\"count\":1,\"dimensionType\":\"NONE\"}}}]}" + "{\"top_queries\":[{\"timestamp\":1706574180000,\"node_id\":\"node_for_top_queries_test\",\"search_type\":\"query_then_fetch\",\"measurements\":{\"latency\":{\"metricType\":\"latency\",\"number\":1,\"count\":1,\"aggregationType\":\"NONE\"}}}]}" .toCharArray(); TopQueries topQueries = QueryInsightsTestUtils.createFixedTopQueries(); ClusterName clusterName = new ClusterName("test-cluster"); From a337ef0a7095fe66d6dd9db6a82d83d0d955b8eb Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Mon, 5 Aug 2024 15:21:06 -0700 Subject: [PATCH 06/29] Refactor unit tests Signed-off-by: Siddhant Deshmukh --- .../insights/QueryInsightsTestUtils.java | 18 + .../service/QueryGroupingServiceTests.java | 548 ++++-------------- 2 files changed, 122 insertions(+), 444 deletions(-) diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java index 6cae81e..c17b2ef 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java @@ -178,6 +178,24 @@ public static List generateQueryInsightsRecordsWithMeasuremen return records; } + public static List> generateMultipleQueryInsightsRecordsWithMeasurement( + int count, + MetricType metricType, + List measurements + ) { + List> multipleRecordLists = new ArrayList<>(); + + for (int i = 0; i < measurements.size(); i++) { + List records = generateQueryInsightRecords(count); + multipleRecordLists.add(records); + for (SearchQueryRecord record : records) { + record.getMeasurements().get(metricType).setMeasurement(measurements.get(i)); + } + QueryInsightsTestUtils.populateHashcode(records, i); + } + return multipleRecordLists; + } + public static void populateSameQueryHashcodes(List searchQueryRecords) { for (SearchQueryRecord record : searchQueryRecords) { record.getAttributes().put(Attribute.QUERY_HASHCODE, 1); diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java index 34ea56e..a68fa98 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java @@ -8,7 +8,6 @@ package org.opensearch.plugin.insights.core.service; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.PriorityQueue; @@ -34,13 +33,7 @@ public class QueryGroupingServiceTests extends OpenSearchTestCase { @Before public void setup() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.DEFUALT_AGGREGATION_TYPE, - topQueriesStore, - 10 - ); + queryGroupingService = getQueryGroupingService(AggregationType.DEFUALT_AGGREGATION_TYPE, 10); } public void testWithAllDifferentHashcodes() { @@ -210,13 +203,7 @@ public void testVaryingTopNSize() { } public void testAddMeasurementSumAggregationLatency() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.SUM, - topQueriesStore, - 10 - ); + queryGroupingService = getQueryGroupingService(AggregationType.SUM, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -234,13 +221,7 @@ public void testAddMeasurementSumAggregationLatency() { } public void testAddMeasurementAverageAggregationLatency() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 10 - ); + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -261,13 +242,7 @@ public void testAddMeasurementAverageAggregationLatency() { } public void testAddMeasurementNoneAggregationLatency() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.NONE, - topQueriesStore, - 10 - ); + queryGroupingService = getQueryGroupingService(AggregationType.NONE, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -365,37 +340,17 @@ public void testNoneGroupingTypeIllegalArgumentException() { // 1. New query group not existing added to MIN public void testNewGroupAddedToMin() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 2; - - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1000 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 2, MetricType.LATENCY, - 1100 + List.of(1000, 1100) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); - - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2); for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -406,45 +361,17 @@ public void testNewGroupAddedToMin() { // 2. New query group not existing added to MIN and overflows to MAX public void testNewGroupOverflowsMinToMax() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 2; - - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1000 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); - - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1100 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 2, MetricType.LATENCY, - 900 + List.of(1000, 1100, 900) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); - - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3); for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -455,45 +382,17 @@ public void testNewGroupOverflowsMinToMax() { // 3. New query group not existing added to MIN and causes other group to overflow to MAX public void testNewGroupCausesOtherGroupOverflowMinToMax() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 2; + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 2, MetricType.LATENCY, - 1000 + List.of(1000, 1100, 1200) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); - - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1100 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); - - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1200 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); - - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3); for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -504,54 +403,25 @@ public void testNewGroupCausesOtherGroupOverflowMinToMax() { // 4. Existing query group update to MIN increases average public void testExistingGroupUpdateToMinIncreaseAverage() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 1; + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 1000 + List.of(1100, 1200, 1000) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 1200 + List.of(1300) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1100 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); + allRecords1.addAll(allRecords2); - // Average will decrease when this record added and record should be moved out of Top N - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1300 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records4, 3); - - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3, records4); - - for (List recordList : allRecords) { + for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -562,54 +432,25 @@ public void testExistingGroupUpdateToMinIncreaseAverage() { // 5. Existing query group update to MIN decrease average - stay in MIN public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 1; - - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1000 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 600 + List.of(1100, 600, 1000) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 1100 + List.of(700) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); - - // Average will decrease when this record added and record should be moved out of Top N - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 700 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records4, 3); - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3, records4); + allRecords1.addAll(allRecords2); - for (List recordList : allRecords) { + for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -620,54 +461,25 @@ public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { // 6. Existing query group update to MIN decrease average - overflows to MAX public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 1; + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 1000 + List.of(1199, 1100, 1000) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 1100 + List.of(1) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1199 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); + allRecords1.addAll(allRecords2); - // Average will decrease when this record added and record should be moved out of Top N - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records4, 3); - - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3, records4); - - for (List recordList : allRecords) { + for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -678,53 +490,25 @@ public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { // 7. Existing query group update to MAX increases average - stay in MAX public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 1; - - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 950 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 975 + List.of(900, 975, 950) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 900 + List.of(920) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 920 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records4, 3); + allRecords1.addAll(allRecords2); - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3, records4); - - for (List recordList : allRecords) { + for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -735,53 +519,25 @@ public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { // 8. Existing query group update to MAX increases average - promote to MIN public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 1; - - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 950 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); - - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 975 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 900 + List.of(900, 975, 950) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 1100 + List.of(1100) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records4, 3); - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3, records4); + allRecords1.addAll(allRecords2); - for (List recordList : allRecords) { + for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -792,53 +548,25 @@ public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { // 9. Existing query group update to MAX decrease average public void testExistingGroupUpdateToMaxDecreaseAverage() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 1; + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 950 + List.of(900, 975, 950) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 975 + List.of(800) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 900 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); - - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 800 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records4, 3); + allRecords1.addAll(allRecords2); - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3, records4); - - for (List recordList : allRecords) { + for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -848,53 +576,25 @@ public void testExistingGroupUpdateToMaxDecreaseAverage() { } public void testSwitchGroupingTypeToNone() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 1; - - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 950 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); - - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 975 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 900 + List.of(900, 975, 950) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 800 + List.of(800) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records4, 3); - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3, records4); + allRecords1.addAll(allRecords2); - for (List recordList : allRecords) { + for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -905,73 +605,29 @@ public void testSwitchGroupingTypeToNone() { queryGroupingService.setGroupingType(GroupingType.NONE); assertEquals(0, queryGroupingService.numberOfTopGroups()); - assertThrows(IllegalArgumentException.class, () -> { queryGroupingService.addQueryToGroup(allRecords.get(0).get(0)); }); + assertThrows(IllegalArgumentException.class, () -> { queryGroupingService.addQueryToGroup(allRecords1.get(0).get(0)); }); } public void testMultipleQueryGroupsUpdates() { - queryGroupingService = new QueryGroupingService( - MetricType.LATENCY, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 2 - ); - int numOfRecords = 1; - - List records1 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1000 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records1, 1); - - List records2 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 1000 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records2, 2); + queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); - List records3 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 900 + List.of(900, 1000, 1000) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records3, 3); - - List records4 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 800 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records4, 3); - - List records5 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, - MetricType.LATENCY, - 400 - ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records5, 2); - List records6 = QueryInsightsTestUtils.generateQueryInsightsRecordsWithMeasurement( - numOfRecords, + List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, MetricType.LATENCY, - 1200 + List.of(800, 400, 1200) ); - // Set all records to have the same hashcode for aggregation - QueryInsightsTestUtils.populateHashcode(records6, 1); - SearchQueryRecord aggregatedRecord = null; - List> allRecords = Arrays.asList(records1, records2, records3, records4, records5, records6); + allRecords1.addAll(allRecords2); - for (List recordList : allRecords) { + for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + queryGroupingService.addQueryToGroup(record); } } @@ -979,4 +635,8 @@ public void testMultipleQueryGroupsUpdates() { assertEquals(850L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } + + private QueryGroupingService getQueryGroupingService(AggregationType aggregationType, int topNSize) { + return new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, aggregationType, topQueriesStore, topNSize); + } } From 08bb29d2aeb4a8ef1257f7e399a35cb6f9588000 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Tue, 6 Aug 2024 12:52:16 -0700 Subject: [PATCH 07/29] Decouple Measurement and MetricType Signed-off-by: Siddhant Deshmukh --- .../core/listener/QueryInsightsListener.java | 7 +- .../insights/rules/model/Measurement.java | 114 +++++++++++++----- .../insights/rules/model/MetricType.java | 39 ------ .../rules/model/SearchQueryRecord.java | 8 +- .../insights/QueryInsightsTestUtils.java | 8 +- .../service/QueryGroupingServiceTests.java | 38 +++--- .../top_queries/TopQueriesResponseTests.java | 2 +- 7 files changed, 116 insertions(+), 100 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index 4467c00..ae383aa 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -207,17 +207,13 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final if (shouldCollect(MetricType.LATENCY)) { measurements.put( MetricType.LATENCY, - new Measurement( - MetricType.LATENCY, - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos()) - ) + new Measurement(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - searchRequestContext.getAbsoluteStartNanos())) ); } if (shouldCollect(MetricType.CPU)) { measurements.put( MetricType.CPU, new Measurement( - MetricType.CPU, tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getCpuTimeInNanos()).mapToLong(Long::longValue).sum() ) ); @@ -226,7 +222,6 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final measurements.put( MetricType.MEMORY, new Measurement( - MetricType.MEMORY, tasksResourceUsages.stream().map(a -> a.getTaskResourceUsage().getMemoryInBytes()).mapToLong(Long::longValue).sum() ) ); diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java index ffccedd..dba885f 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java @@ -9,7 +9,6 @@ package org.opensearch.plugin.insights.rules.model; import java.io.IOException; -import java.util.Locale; import java.util.Objects; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -19,24 +18,21 @@ import org.opensearch.core.xcontent.XContentBuilder; /** - * Measurement that is stored in the SearchQueryRecord. Measurement can be of a specific AggregationType and MetricType + * Measurement that is stored in the SearchQueryRecord. Measurement can be of a specific AggregationType */ public class Measurement implements ToXContentObject, Writeable { private static int DEFAULT_COUNT = 1; private AggregationType aggregationType; - private MetricType metricType; private Number number; private int count; /** * Constructor - * @param metricType metricType * @param number number * @param count count * @param aggregationType aggregationType */ - public Measurement(MetricType metricType, Number number, int count, AggregationType aggregationType) { - this.metricType = metricType; + public Measurement(Number number, int count, AggregationType aggregationType) { this.number = number; this.count = count; this.aggregationType = aggregationType; @@ -44,21 +40,19 @@ public Measurement(MetricType metricType, Number number, int count, AggregationT /** * Constructor - * @param metricType metricType * @param number number * @param aggregationType aggregationType */ - public Measurement(MetricType metricType, Number number, AggregationType aggregationType) { - this(metricType, number, DEFAULT_COUNT, aggregationType); + public Measurement(Number number, AggregationType aggregationType) { + this(number, DEFAULT_COUNT, aggregationType); } /** * Constructor - * @param metricType metricType * @param number number */ - public Measurement(MetricType metricType, Number number) { - this(metricType, number, DEFAULT_COUNT, AggregationType.DEFUALT_AGGREGATION_TYPE); + public Measurement(Number number) { + this(number, DEFAULT_COUNT, AggregationType.DEFUALT_AGGREGATION_TYPE); } /** @@ -69,20 +63,34 @@ public void addMeasurement(Number toAdd) { switch (aggregationType) { case NONE: aggregationType = AggregationType.SUM; - setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); + setMeasurement(addMeasurementInferType(number, toAdd)); break; case SUM: - setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); + setMeasurement(addMeasurementInferType(number, toAdd)); break; case AVERAGE: count += 1; - setMeasurement(MetricType.addMeasurements(number, toAdd, metricType)); + setMeasurement(addMeasurementInferType(number, toAdd)); break; default: throw new IllegalArgumentException("The following aggregation type is not supported : " + aggregationType); } } + private Number addMeasurementInferType(Number a, Number b) { + if (a instanceof Long && b instanceof Long) { + return a.longValue() + b.longValue(); + } else if (a instanceof Integer && b instanceof Integer) { + return a.intValue() + b.intValue(); + } else if (a instanceof Double && b instanceof Double) { + return a.doubleValue() + b.doubleValue(); + } else if (a instanceof Float && b instanceof Float) { + return a.floatValue() + b.floatValue(); + } else { + throw new IllegalArgumentException("Unsupported number type: " + a.getClass() + " or " + b.getClass()); + } + } + /** * Get measurement number based on the aggragation type * @return measurement number @@ -93,12 +101,36 @@ public Number getMeasurement() { case SUM: return number; case AVERAGE: - return MetricType.getAverageMeasurement(number, count, metricType); + return getAverageMeasurement(number, count); default: throw new IllegalArgumentException("Aggregation Type should be set for measurement."); } } + /** + * Get average measurement number based on the total and count + * @param total total measurement value + * @param count count of measurements + * @return average measurement value + */ + private Number getAverageMeasurement(Number total, int count) { + if (count == 0) { + throw new IllegalArgumentException("Count cannot be zero for average calculation."); + } + + if (total instanceof Long) { + return ((Long) total) / count; + } else if (total instanceof Integer) { + return ((Integer) total) / count; + } else if (total instanceof Double) { + return ((Double) total) / count; + } else if (total instanceof Float) { + return ((Float) total) / count; + } else { + throw new IllegalArgumentException("Unsupported number type: " + total.getClass()); + } + } + /** * Set measurement * @param measurement measurement number @@ -118,7 +150,6 @@ public void setAggregationType(AggregationType aggregationType) { @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); - builder.field("metricType", metricType.toString()); builder.field("number", number); builder.field("count", count); builder.field("aggregationType", aggregationType.toString()); @@ -128,18 +159,50 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(metricType.toString()); - out.writeGenericValue(metricType.parseValue(number)); + writeNumber(out, number); out.writeInt(count); out.writeString(aggregationType.toString()); } + private void writeNumber(StreamOutput out, Number number) throws IOException { + if (number instanceof Long) { + out.writeByte((byte) 0); // Type indicator for Long + out.writeLong((Long) number); + } else if (number instanceof Integer) { + out.writeByte((byte) 1); // Type indicator for Integer + out.writeInt((Integer) number); + } else if (number instanceof Double) { + out.writeByte((byte) 2); // Type indicator for Double + out.writeDouble((Double) number); + } else if (number instanceof Float) { + out.writeByte((byte) 3); // Type indicator for Float + out.writeFloat((Float) number); + } else { + throw new IOException("Unsupported number type: " + number.getClass()); + } + } + + private static Number readNumber(StreamInput in) throws IOException { + byte typeIndicator = in.readByte(); + switch (typeIndicator) { + case 0: + return in.readLong(); + case 1: + return in.readInt(); + case 2: + return in.readDouble(); + case 3: + return in.readFloat(); + default: + throw new IOException("Unsupported number type indicator: " + typeIndicator); + } + } + public static Measurement readFromStream(StreamInput in) throws IOException { - MetricType metricType = MetricType.valueOf(in.readString().toUpperCase(Locale.ROOT)); - Number number = metricType.parseValue(in.readGenericValue()); + Number number = readNumber(in); int count = in.readInt(); AggregationType aggregationType = AggregationType.valueOf(in.readString()); - return new Measurement(metricType, number, count, aggregationType); + return new Measurement(number, count, aggregationType); } @Override @@ -147,14 +210,11 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Measurement that = (Measurement) o; - return count == that.count - && metricType == that.metricType - && Objects.equals(number, that.number) - && aggregationType == that.aggregationType; + return count == that.count && Objects.equals(number, that.number) && aggregationType == that.aggregationType; } @Override public int hashCode() { - return Objects.hash(metricType, number, count, aggregationType); + return Objects.hash(number, count, aggregationType); } } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java index 1808e76..d6717e5 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java @@ -96,43 +96,4 @@ public int compare(final Number a, final Number b) { } return -1; } - - public static Number addMeasurements(Number a, Number b, MetricType metricType) { - switch (metricType) { - case LATENCY: - case MEMORY: - case CPU: - return a.longValue() + b.longValue(); - default: - throw new IllegalArgumentException("Unsupported metric type: " + metricType); - } - } - - public static Number getAverageMeasurement(Number total, int count, MetricType metricType) { - switch (metricType) { - case LATENCY: - case MEMORY: - case CPU: - return total.longValue() / count; - default: - throw new IllegalArgumentException("Unsupported metric type: " + metricType); - } - } - - /** - * Parse a value with the correct type based on MetricType - * - * @param o the generic object to parse - * @return {@link Number} - */ - public Number parseValue(final Object o) { - switch (this) { - case LATENCY: - case CPU: - case MEMORY: - return (Long) o; - default: - return (Number) o; - } - } } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java index 8246689..ede43fc 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java @@ -91,11 +91,11 @@ public Number getMeasurement(final MetricType name) { /** * Add measurement to SearchQueryRecord. Applicable when we are grouping multiple queries based on GroupingType. - * @param name the name of the measurement - * @param measurement The measurement we want to add to the current measurement. + * @param metricType the name of the measurement + * @param numberToAdd The measurement number we want to add to the current measurement. */ - public void addMeasurement(final MetricType name, Number measurement) { - measurements.get(name).addMeasurement(measurement); + public void addMeasurement(final MetricType metricType, Number numberToAdd) { + measurements.get(metricType).addMeasurement(numberToAdd); } /** diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java index c17b2ef..7c3d1d9 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java @@ -119,9 +119,9 @@ public static List generateQueryInsightRecords( long cpuValue = randomLongBetween(1000, 10000); long memoryValue = randomLongBetween(1000, 10000); Map measurements = new LinkedHashMap<>(); - measurements.put(MetricType.LATENCY, new Measurement(MetricType.LATENCY, latencyValue, aggregationType)); - measurements.put(MetricType.CPU, new Measurement(MetricType.CPU, cpuValue, aggregationType)); - measurements.put(MetricType.MEMORY, new Measurement(MetricType.MEMORY, memoryValue, aggregationType)); + measurements.put(MetricType.LATENCY, new Measurement(latencyValue, aggregationType)); + measurements.put(MetricType.CPU, new Measurement(cpuValue, aggregationType)); + measurements.put(MetricType.MEMORY, new Measurement(memoryValue, aggregationType)); Map phaseLatencyMap = new LinkedHashMap<>(); int countOfPhases = randomIntBetween(2, 5); @@ -237,7 +237,7 @@ public static TopQueries createFixedTopQueries() { public static SearchQueryRecord createFixedSearchQueryRecord() { long timestamp = 1706574180000L; - Map measurements = Map.of(MetricType.LATENCY, new Measurement(MetricType.LATENCY, 1L)); + Map measurements = Map.of(MetricType.LATENCY, new Measurement(1L)); Map phaseLatencyMap = new HashMap<>(); Map attributes = new HashMap<>(); diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java index a68fa98..c9623b1 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java @@ -345,7 +345,7 @@ public void testNewGroupAddedToMin() { List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 2, MetricType.LATENCY, - List.of(1000, 1100) + List.of(1000L, 1100L) ); for (List recordList : allRecords) { @@ -366,7 +366,7 @@ public void testNewGroupOverflowsMinToMax() { List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 2, MetricType.LATENCY, - List.of(1000, 1100, 900) + List.of(1000L, 1100L, 900L) ); for (List recordList : allRecords) { @@ -387,7 +387,7 @@ public void testNewGroupCausesOtherGroupOverflowMinToMax() { List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 2, MetricType.LATENCY, - List.of(1000, 1100, 1200) + List.of(1000L, 1100L, 1200L) ); for (List recordList : allRecords) { @@ -408,13 +408,13 @@ public void testExistingGroupUpdateToMinIncreaseAverage() { List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(1100, 1200, 1000) + List.of(1100L, 1200L, 1000L) ); List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(1300) + List.of(1300L) ); allRecords1.addAll(allRecords2); @@ -437,13 +437,13 @@ public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(1100, 600, 1000) + List.of(1100L, 600L, 1000L) ); List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(700) + List.of(700L) ); allRecords1.addAll(allRecords2); @@ -466,13 +466,13 @@ public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(1199, 1100, 1000) + List.of(1199L, 1100L, 1000L) ); List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(1) + List.of(1L) ); allRecords1.addAll(allRecords2); @@ -495,13 +495,13 @@ public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(900, 975, 950) + List.of(900L, 975L, 950L) ); List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(920) + List.of(920L) ); allRecords1.addAll(allRecords2); @@ -524,13 +524,13 @@ public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(900, 975, 950) + List.of(900L, 975L, 950L) ); List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(1100) + List.of(1100L) ); allRecords1.addAll(allRecords2); @@ -553,13 +553,13 @@ public void testExistingGroupUpdateToMaxDecreaseAverage() { List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(900, 975, 950) + List.of(900L, 975L, 950L) ); List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(800) + List.of(800L) ); allRecords1.addAll(allRecords2); @@ -581,13 +581,13 @@ public void testSwitchGroupingTypeToNone() { List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(900, 975, 950) + List.of(900L, 975L, 950L) ); List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(800) + List.of(800L) ); allRecords1.addAll(allRecords2); @@ -614,13 +614,13 @@ public void testMultipleQueryGroupsUpdates() { List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(900, 1000, 1000) + List.of(900L, 1000L, 1000L) ); List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, MetricType.LATENCY, - List.of(800, 400, 1200) + List.of(800L, 400L, 1200L) ); allRecords1.addAll(allRecords2); diff --git a/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java b/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java index 01748c8..dbce9f6 100644 --- a/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java +++ b/src/test/java/org/opensearch/plugin/insights/rules/action/top_queries/TopQueriesResponseTests.java @@ -41,7 +41,7 @@ public void testSerialize() throws Exception { public void testToXContent() throws IOException { char[] expectedXcontent = - "{\"top_queries\":[{\"timestamp\":1706574180000,\"node_id\":\"node_for_top_queries_test\",\"search_type\":\"query_then_fetch\",\"measurements\":{\"latency\":{\"metricType\":\"latency\",\"number\":1,\"count\":1,\"aggregationType\":\"NONE\"}}}]}" + "{\"top_queries\":[{\"timestamp\":1706574180000,\"node_id\":\"node_for_top_queries_test\",\"search_type\":\"query_then_fetch\",\"measurements\":{\"latency\":{\"number\":1,\"count\":1,\"aggregationType\":\"NONE\"}}}]}" .toCharArray(); TopQueries topQueries = QueryInsightsTestUtils.createFixedTopQueries(); ClusterName clusterName = new ClusterName("test-cluster"); From 6b35126dc0655edede2fe71864c10d4519cf7898 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Tue, 6 Aug 2024 13:03:45 -0700 Subject: [PATCH 08/29] Aggregate type NONE will ensure no aggregations computed Signed-off-by: Siddhant Deshmukh --- .../insights/rules/model/Measurement.java | 6 +++--- .../service/QueryGroupingServiceTests.java | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java index dba885f..322fabd 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java @@ -56,14 +56,14 @@ public Measurement(Number number) { } /** - * Add measurement number to the current number based on the aggregationType + * Add measurement number to the current number based on the aggregationType. + * If aggregateType is NONE, replace the number since we are not aggregating in this case. * @param toAdd number to add */ public void addMeasurement(Number toAdd) { switch (aggregationType) { case NONE: - aggregationType = AggregationType.SUM; - setMeasurement(addMeasurementInferType(number, toAdd)); + setMeasurement(toAdd); break; case SUM: setMeasurement(addMeasurementInferType(number, toAdd)); diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java index c9623b1..5a23bc8 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java @@ -248,15 +248,15 @@ public void testAddMeasurementNoneAggregationLatency() { // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateSameQueryHashcodes(records); - SearchQueryRecord aggregatedRecord = null; + SearchQueryRecord lastRecord = null; - Number expectedSum = 0; + Number expectedValue = 0; for (SearchQueryRecord record : records) { - expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + expectedValue = record.getMeasurement(MetricType.LATENCY).longValue(); + lastRecord = queryGroupingService.addQueryToGroup(record); } - assertEquals(expectedSum, aggregatedRecord.getMeasurement(MetricType.LATENCY)); + assertEquals(expectedValue, lastRecord.getMeasurement(MetricType.LATENCY)); } public void testAddMeasurementSumAggregationCpu() { @@ -311,15 +311,15 @@ public void testAddMeasurementNoneAggregationCpu() { // Set all records to have the same hashcode for aggregation QueryInsightsTestUtils.populateSameQueryHashcodes(records); - SearchQueryRecord aggregatedRecord = null; + SearchQueryRecord lastRecord = null; - Number expectedSum = 0; + Number expectedValue = 0; for (SearchQueryRecord record : records) { - expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + expectedValue = record.getMeasurement(MetricType.CPU).longValue(); + lastRecord = queryGroupingService.addQueryToGroup(record); } - assertEquals(expectedSum, aggregatedRecord.getMeasurement(MetricType.CPU)); + assertEquals(expectedValue, lastRecord.getMeasurement(MetricType.CPU)); } public void testNoneGroupingTypeIllegalArgumentException() { From d940428f841e6859b1102a8a7e3a621c66e4b25f Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Mon, 26 Aug 2024 14:16:36 -0700 Subject: [PATCH 09/29] Perform renaming Signed-off-by: Siddhant Deshmukh --- ...GroupingService.java => QueryGrouper.java} | 6 +- .../core/service/TopQueriesService.java | 16 +- .../insights/rules/model/AggregationType.java | 2 +- .../insights/rules/model/Measurement.java | 2 +- .../insights/QueryInsightsTestUtils.java | 4 +- ...rviceTests.java => QueryGrouperTests.java} | 190 +++++++++--------- 6 files changed, 110 insertions(+), 110 deletions(-) rename src/main/java/org/opensearch/plugin/insights/core/service/{QueryGroupingService.java => QueryGrouper.java} (99%) rename src/test/java/org/opensearch/plugin/insights/core/service/{QueryGroupingServiceTests.java => QueryGrouperTests.java} (74%) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java similarity index 99% rename from src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java rename to src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java index a61cc29..c620b4c 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGroupingService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java @@ -24,12 +24,12 @@ * Handles grouping of search queries based on the GroupingType for the MetricType * Following algorithm : */ -public class QueryGroupingService { +public class QueryGrouper { /** * Logger */ - private static final Logger log = LogManager.getLogger(QueryGroupingService.class); + private static final Logger log = LogManager.getLogger(QueryGrouper.class); /** * Grouping type for the current grouping service */ @@ -69,7 +69,7 @@ public class QueryGroupingService { */ private int topNSize; - public QueryGroupingService( + public QueryGrouper( MetricType metricType, GroupingType groupingType, AggregationType aggregationType, diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index d48ae43..f67f113 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -95,7 +95,7 @@ public class TopQueriesService { */ private QueryInsightsExporter exporter; - private QueryGroupingService queryGroupingService; + private QueryGrouper queryGrouper; TopQueriesService( final MetricType metricType, @@ -113,7 +113,7 @@ public class TopQueriesService { topQueriesStore = new PriorityQueue<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot = new AtomicReference<>(new ArrayList<>()); topQueriesHistorySnapshot = new AtomicReference<>(new ArrayList<>()); - queryGroupingService = new QueryGroupingService( + queryGrouper = new QueryGrouper( metricType, QueryInsightsSettings.DEFAULT_GROUPING_TYPE, AggregationType.AVERAGE, @@ -129,7 +129,7 @@ public class TopQueriesService { */ public void setTopNSize(final int topNSize) { this.topNSize = topNSize; - this.queryGroupingService.updateTopNSize(topNSize); + this.queryGrouper.updateTopNSize(topNSize); } /** @@ -182,7 +182,7 @@ public void setWindowSize(final TimeValue windowSize) { } public void setGrouping(final GroupingType groupingType) { - queryGroupingService.setGroupingType(groupingType); + queryGrouper.setGroupingType(groupingType); } /** @@ -322,9 +322,9 @@ void consumeRecords(final List records) { } private void addToTopNStore(final List records) { - if (queryGroupingService.getGroupingType() != GroupingType.NONE) { + if (queryGrouper.getGroupingType() != GroupingType.NONE) { for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } else { topQueriesStore.addAll(records); @@ -351,8 +351,8 @@ private void rotateWindowIfNecessary(final long newWindowStart) { } topQueriesHistorySnapshot.set(history); topQueriesStore.clear(); - if (queryGroupingService.getGroupingType() != GroupingType.NONE) { - queryGroupingService.drain(); + if (queryGrouper.getGroupingType() != GroupingType.NONE) { + queryGrouper.drain(); } topQueriesCurrentSnapshot.set(new ArrayList<>()); windowStart = newWindowStart; diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/AggregationType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/AggregationType.java index c65e33b..b49cc5a 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/AggregationType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/AggregationType.java @@ -16,5 +16,5 @@ public enum AggregationType { AVERAGE, SUM; - public static AggregationType DEFUALT_AGGREGATION_TYPE = NONE; + public static AggregationType DEFAULT_AGGREGATION_TYPE = NONE; } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java index 322fabd..e7e5349 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/Measurement.java @@ -52,7 +52,7 @@ public Measurement(Number number, AggregationType aggregationType) { * @param number number */ public Measurement(Number number) { - this(number, DEFAULT_COUNT, AggregationType.DEFUALT_AGGREGATION_TYPE); + this(number, DEFAULT_COUNT, AggregationType.DEFAULT_AGGREGATION_TYPE); } /** diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java index 7c3d1d9..016bbe3 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java @@ -61,7 +61,7 @@ public QueryInsightsTestUtils() {} * @return List of records */ public static List generateQueryInsightRecords(int count) { - return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, AggregationType.DEFUALT_AGGREGATION_TYPE); + return generateQueryInsightRecords(count, count, System.currentTimeMillis(), 0, AggregationType.DEFAULT_AGGREGATION_TYPE); } /** @@ -76,7 +76,7 @@ public static List generateQueryInsightRecords(int count, Sea count, System.currentTimeMillis(), 0, - AggregationType.DEFUALT_AGGREGATION_TYPE + AggregationType.DEFAULT_AGGREGATION_TYPE ); for (SearchQueryRecord record : records) { record.getAttributes().put(Attribute.SOURCE, searchSourceBuilder); diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java similarity index 74% rename from src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java rename to src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java index 5a23bc8..331c2ca 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGroupingServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java @@ -22,10 +22,10 @@ import org.opensearch.test.OpenSearchTestCase; /** - * Unit Tests for {@link QueryGroupingService}. + * Unit Tests for {@link QueryGrouper}. */ -public class QueryGroupingServiceTests extends OpenSearchTestCase { - private QueryGroupingService queryGroupingService; +public class QueryGrouperTests extends OpenSearchTestCase { + private QueryGrouper queryGrouper; private PriorityQueue topQueriesStore = new PriorityQueue<>( 100, (a, b) -> SearchQueryRecord.compare(a, b, MetricType.LATENCY) @@ -33,7 +33,7 @@ public class QueryGroupingServiceTests extends OpenSearchTestCase { @Before public void setup() { - queryGroupingService = getQueryGroupingService(AggregationType.DEFUALT_AGGREGATION_TYPE, 10); + queryGrouper = getQueryGroupingService(AggregationType.DEFAULT_AGGREGATION_TYPE, 10); } public void testWithAllDifferentHashcodes() { @@ -42,7 +42,7 @@ public void testWithAllDifferentHashcodes() { SearchQueryRecord groupedRecord; Set hashcodeSet = new HashSet<>(); for (SearchQueryRecord record : records) { - groupedRecord = queryGroupingService.addQueryToGroup(record); + groupedRecord = queryGrouper.addQueryToGroup(record); int hashcode = (int) groupedRecord.getAttributes().get(Attribute.QUERY_HASHCODE); hashcodeSet.add(hashcode); } @@ -56,7 +56,7 @@ public void testWithAllSameHashcodes() { SearchQueryRecord groupedRecord; Set hashcodeSet = new HashSet<>(); for (SearchQueryRecord record : records) { - groupedRecord = queryGroupingService.addQueryToGroup(record); + groupedRecord = queryGrouper.addQueryToGroup(record); int hashcode = (int) groupedRecord.getAttributes().get(Attribute.QUERY_HASHCODE); hashcodeSet.add(hashcode); } @@ -67,11 +67,11 @@ public void testDrain() { int numOfRecords = 10; final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - int groupsBeforeDrain = queryGroupingService.numberOfGroups(); - queryGroupingService.drain(); - int groupsAfterDrain = queryGroupingService.numberOfGroups(); + int groupsBeforeDrain = queryGrouper.numberOfGroups(); + queryGrouper.drain(); + int groupsAfterDrain = queryGrouper.numberOfGroups(); assertEquals(numOfRecords, groupsBeforeDrain); assertEquals(0, groupsAfterDrain); @@ -82,27 +82,27 @@ public void testChangeTopNSize() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - assertEquals(10, queryGroupingService.numberOfTopGroups()); // Initially expects top 10 groups + assertEquals(10, queryGrouper.numberOfTopGroups()); // Initially expects top 10 groups - queryGroupingService.updateTopNSize(5); - queryGroupingService.drain(); // Clear previous state + queryGrouper.updateTopNSize(5); + queryGrouper.drain(); // Clear previous state for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - assertEquals(5, queryGroupingService.numberOfTopGroups()); // After update, expects top 5 groups + assertEquals(5, queryGrouper.numberOfTopGroups()); // After update, expects top 5 groups } public void testEmptyPriorityQueues() { - int groupsBeforeDrain = queryGroupingService.numberOfGroups(); + int groupsBeforeDrain = queryGrouper.numberOfGroups(); assertEquals(0, groupsBeforeDrain); - queryGroupingService.drain(); - int groupsAfterDrain = queryGroupingService.numberOfGroups(); + queryGrouper.drain(); + int groupsAfterDrain = queryGrouper.numberOfGroups(); assertEquals(0, groupsAfterDrain); // No groups should be present after draining } @@ -111,26 +111,26 @@ public void testAddRemoveFromMaxHeap() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - assertTrue(queryGroupingService.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap + assertTrue(queryGrouper.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap - queryGroupingService.updateTopNSize(5); // Change size to 5 - queryGroupingService.drain(); // Clear previous state + queryGrouper.updateTopNSize(5); // Change size to 5 + queryGrouper.drain(); // Clear previous state for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - assertEquals(5, queryGroupingService.numberOfTopGroups()); // Should be exactly 5 in the min heap + assertEquals(5, queryGrouper.numberOfTopGroups()); // Should be exactly 5 in the min heap } public void testInvalidGroupingType() { - QueryGroupingService invalidGroupingService = new QueryGroupingService( + QueryGrouper invalidGroupingService = new QueryGrouper( MetricType.LATENCY, GroupingType.NONE, - AggregationType.DEFUALT_AGGREGATION_TYPE, + AggregationType.DEFAULT_AGGREGATION_TYPE, topQueriesStore, 10 ); @@ -143,10 +143,10 @@ public void testLargeNumberOfRecords() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - assertTrue(queryGroupingService.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap + assertTrue(queryGrouper.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap } public void testChangeGroupingType() { @@ -154,15 +154,15 @@ public void testChangeGroupingType() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - int groupsBeforeChange = queryGroupingService.numberOfGroups(); + int groupsBeforeChange = queryGrouper.numberOfGroups(); assertTrue(groupsBeforeChange > 0); - queryGroupingService.setGroupingType(GroupingType.NONE); // Changing to NONE should clear groups + queryGrouper.setGroupingType(GroupingType.NONE); // Changing to NONE should clear groups - int groupsAfterChange = queryGroupingService.numberOfGroups(); + int groupsAfterChange = queryGrouper.numberOfGroups(); assertEquals(0, groupsAfterChange); // Expect no groups after changing to NONE } @@ -171,16 +171,16 @@ public void testDrainWithMultipleGroupingTypes() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - int groupsBeforeDrain = queryGroupingService.numberOfGroups(); + int groupsBeforeDrain = queryGrouper.numberOfGroups(); assertTrue(groupsBeforeDrain > 0); - queryGroupingService.setGroupingType(GroupingType.SIMILARITY); - queryGroupingService.drain(); + queryGrouper.setGroupingType(GroupingType.SIMILARITY); + queryGrouper.drain(); - int groupsAfterDrain = queryGroupingService.numberOfGroups(); + int groupsAfterDrain = queryGrouper.numberOfGroups(); assertEquals(0, groupsAfterDrain); // After drain, groups should be cleared } @@ -189,21 +189,21 @@ public void testVaryingTopNSize() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - queryGroupingService.updateTopNSize(15); - queryGroupingService.drain(); // Clear previous state + queryGrouper.updateTopNSize(15); + queryGrouper.drain(); // Clear previous state for (SearchQueryRecord record : records) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } - assertEquals(15, queryGroupingService.numberOfTopGroups()); // Should reflect the updated top N size + assertEquals(15, queryGrouper.numberOfTopGroups()); // Should reflect the updated top N size } public void testAddMeasurementSumAggregationLatency() { - queryGroupingService = getQueryGroupingService(AggregationType.SUM, 10); + queryGrouper = getQueryGroupingService(AggregationType.SUM, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -213,7 +213,7 @@ public void testAddMeasurementSumAggregationLatency() { Number expectedSum = 0; for (SearchQueryRecord record : records) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + aggregatedRecord = queryGrouper.addQueryToGroup(record); expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); } @@ -221,7 +221,7 @@ public void testAddMeasurementSumAggregationLatency() { } public void testAddMeasurementAverageAggregationLatency() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 10); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -233,7 +233,7 @@ public void testAddMeasurementAverageAggregationLatency() { int expectedCount = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + aggregatedRecord = queryGrouper.addQueryToGroup(record); expectedCount += 1; } @@ -242,7 +242,7 @@ public void testAddMeasurementAverageAggregationLatency() { } public void testAddMeasurementNoneAggregationLatency() { - queryGroupingService = getQueryGroupingService(AggregationType.NONE, 10); + queryGrouper = getQueryGroupingService(AggregationType.NONE, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -253,14 +253,14 @@ public void testAddMeasurementNoneAggregationLatency() { Number expectedValue = 0; for (SearchQueryRecord record : records) { expectedValue = record.getMeasurement(MetricType.LATENCY).longValue(); - lastRecord = queryGroupingService.addQueryToGroup(record); + lastRecord = queryGrouper.addQueryToGroup(record); } assertEquals(expectedValue, lastRecord.getMeasurement(MetricType.LATENCY)); } public void testAddMeasurementSumAggregationCpu() { - queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.SUM, topQueriesStore, 10); + queryGrouper = new QueryGrouper(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.SUM, topQueriesStore, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -270,7 +270,7 @@ public void testAddMeasurementSumAggregationCpu() { Number expectedSum = 0; for (SearchQueryRecord record : records) { - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + aggregatedRecord = queryGrouper.addQueryToGroup(record); expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); } @@ -278,7 +278,7 @@ public void testAddMeasurementSumAggregationCpu() { } public void testAddMeasurementAverageAggregationCpu() { - queryGroupingService = new QueryGroupingService( + queryGrouper = new QueryGrouper( MetricType.CPU, GroupingType.SIMILARITY, AggregationType.AVERAGE, @@ -296,7 +296,7 @@ public void testAddMeasurementAverageAggregationCpu() { int expectedCount = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); - aggregatedRecord = queryGroupingService.addQueryToGroup(record); + aggregatedRecord = queryGrouper.addQueryToGroup(record); expectedCount += 1; } @@ -305,7 +305,7 @@ public void testAddMeasurementAverageAggregationCpu() { } public void testAddMeasurementNoneAggregationCpu() { - queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.NONE, topQueriesStore, 10); + queryGrouper = new QueryGrouper(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.NONE, topQueriesStore, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -316,14 +316,14 @@ public void testAddMeasurementNoneAggregationCpu() { Number expectedValue = 0; for (SearchQueryRecord record : records) { expectedValue = record.getMeasurement(MetricType.CPU).longValue(); - lastRecord = queryGroupingService.addQueryToGroup(record); + lastRecord = queryGrouper.addQueryToGroup(record); } assertEquals(expectedValue, lastRecord.getMeasurement(MetricType.CPU)); } public void testNoneGroupingTypeIllegalArgumentException() { - queryGroupingService = new QueryGroupingService(MetricType.CPU, GroupingType.NONE, AggregationType.NONE, topQueriesStore, 10); + queryGrouper = new QueryGrouper(MetricType.CPU, GroupingType.NONE, AggregationType.NONE, topQueriesStore, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -334,13 +334,13 @@ public void testNoneGroupingTypeIllegalArgumentException() { Number expectedSum = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); - assertThrows(IllegalArgumentException.class, () -> { queryGroupingService.addQueryToGroup(record); }); + assertThrows(IllegalArgumentException.class, () -> { queryGrouper.addQueryToGroup(record); }); } } // 1. New query group not existing added to MIN public void testNewGroupAddedToMin() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 2, @@ -350,18 +350,18 @@ public void testNewGroupAddedToMin() { for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // 2. New query group not existing added to MIN and overflows to MAX public void testNewGroupOverflowsMinToMax() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 2, @@ -371,18 +371,18 @@ public void testNewGroupOverflowsMinToMax() { for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // 3. New query group not existing added to MIN and causes other group to overflow to MAX public void testNewGroupCausesOtherGroupOverflowMinToMax() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 2, @@ -392,18 +392,18 @@ public void testNewGroupCausesOtherGroupOverflowMinToMax() { for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // 4. Existing query group update to MIN increases average public void testExistingGroupUpdateToMinIncreaseAverage() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -421,18 +421,18 @@ public void testExistingGroupUpdateToMinIncreaseAverage() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // 5. Existing query group update to MIN decrease average - stay in MIN public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -450,18 +450,18 @@ public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(900L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // 6. Existing query group update to MIN decrease average - overflows to MAX public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -479,18 +479,18 @@ public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // 7. Existing query group update to MAX increases average - stay in MAX public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -508,18 +508,18 @@ public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // 8. Existing query group update to MAX increases average - promote to MIN public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -537,18 +537,18 @@ public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // 9. Existing query group update to MAX decrease average public void testExistingGroupUpdateToMaxDecreaseAverage() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -566,17 +566,17 @@ public void testExistingGroupUpdateToMaxDecreaseAverage() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } public void testSwitchGroupingTypeToNone() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -594,22 +594,22 @@ public void testSwitchGroupingTypeToNone() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); - queryGroupingService.setGroupingType(GroupingType.NONE); - assertEquals(0, queryGroupingService.numberOfTopGroups()); + queryGrouper.setGroupingType(GroupingType.NONE); + assertEquals(0, queryGrouper.numberOfTopGroups()); - assertThrows(IllegalArgumentException.class, () -> { queryGroupingService.addQueryToGroup(allRecords1.get(0).get(0)); }); + assertThrows(IllegalArgumentException.class, () -> { queryGrouper.addQueryToGroup(allRecords1.get(0).get(0)); }); } public void testMultipleQueryGroupsUpdates() { - queryGroupingService = getQueryGroupingService(AggregationType.AVERAGE, 2); + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -627,16 +627,16 @@ public void testMultipleQueryGroupsUpdates() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGroupingService.addQueryToGroup(record); + queryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGroupingService.numberOfTopGroups()); + assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(850L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - private QueryGroupingService getQueryGroupingService(AggregationType aggregationType, int topNSize) { - return new QueryGroupingService(MetricType.LATENCY, GroupingType.SIMILARITY, aggregationType, topQueriesStore, topNSize); + private QueryGrouper getQueryGroupingService(AggregationType aggregationType, int topNSize) { + return new QueryGrouper(MetricType.LATENCY, GroupingType.SIMILARITY, aggregationType, topQueriesStore, topNSize); } } From cfdf4891bbcade48bf55bc8593aaf7dd1cad04fa Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Mon, 26 Aug 2024 14:21:47 -0700 Subject: [PATCH 10/29] Integrate query shape library with grouping Signed-off-by: Siddhant Deshmukh --- .../core/listener/QueryInsightsListener.java | 7 ++-- .../insights/core/service/QueryShape.java | 33 ------------------- 2 files changed, 4 insertions(+), 36 deletions(-) delete mode 100644 src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index ae383aa..467ef1c 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -28,11 +28,13 @@ import org.opensearch.action.search.SearchRequestOperationsListener; import org.opensearch.action.search.SearchTask; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.hash.MurmurHash3; import org.opensearch.common.inject.Inject; import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.plugin.insights.core.service.QueryInsightsService; import org.opensearch.plugin.insights.core.service.QueryShape; +import org.opensearch.plugin.insights.core.service.categorizer.QueryShapeGenerator; import org.opensearch.plugin.insights.rules.model.Attribute; import org.opensearch.plugin.insights.rules.model.Measurement; import org.opensearch.plugin.insights.rules.model.MetricType; @@ -226,8 +228,7 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final ) ); } - // TODO: Use David's QueryShape library to get the query shape hashcode - final QueryShape queryShape = new QueryShape(request.source()); + MurmurHash3.Hash128 hashcode = QueryShapeGenerator.getShapeHashCode(request.source(), false); Map attributes = new HashMap<>(); attributes.put(Attribute.SEARCH_TYPE, request.searchType().toString().toLowerCase(Locale.ROOT)); @@ -236,7 +237,7 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final attributes.put(Attribute.INDICES, request.indices()); attributes.put(Attribute.PHASE_LATENCY_MAP, searchRequestContext.phaseTookMap()); attributes.put(Attribute.TASK_RESOURCE_USAGES, tasksResourceUsages); - attributes.put(Attribute.QUERY_HASHCODE, queryShape.hashCode()); + attributes.put(Attribute.QUERY_HASHCODE, hashcode.toString()); Map labels = new HashMap<>(); // Retrieve user provided label if exists diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java deleted file mode 100644 index 856d0aa..0000000 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryShape.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.plugin.insights.core.service; - -import java.util.List; -import java.util.Objects; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.search.aggregations.AggregatorFactories; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.sort.SortBuilder; - -public class QueryShape { - QueryBuilder query; - AggregatorFactories.Builder aggregations; - List> sorts; - - public QueryShape(SearchSourceBuilder source) { - query = source.query(); - aggregations = source.aggregations(); - sorts = source.sorts(); - } - - @Override - public int hashCode() { - return Objects.hash(query, aggregations, sorts); - } -} From e3a87f6a00f9707e8f255ee9642e42656b694dde Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Mon, 26 Aug 2024 14:28:21 -0700 Subject: [PATCH 11/29] Spotless Signed-off-by: Siddhant Deshmukh --- .../insights/core/listener/QueryInsightsListener.java | 1 - .../plugin/insights/core/service/QueryGrouperTests.java | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index 467ef1c..f5f8850 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -33,7 +33,6 @@ import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.plugin.insights.core.service.QueryInsightsService; -import org.opensearch.plugin.insights.core.service.QueryShape; import org.opensearch.plugin.insights.core.service.categorizer.QueryShapeGenerator; import org.opensearch.plugin.insights.rules.model.Attribute; import org.opensearch.plugin.insights.rules.model.Measurement; diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java index 331c2ca..6e58f60 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java @@ -278,13 +278,7 @@ public void testAddMeasurementSumAggregationCpu() { } public void testAddMeasurementAverageAggregationCpu() { - queryGrouper = new QueryGrouper( - MetricType.CPU, - GroupingType.SIMILARITY, - AggregationType.AVERAGE, - topQueriesStore, - 10 - ); + queryGrouper = new QueryGrouper(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.AVERAGE, topQueriesStore, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); From edf492d236caa53f29a343cf9b8ba2b22f3bb4fe Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Mon, 26 Aug 2024 15:09:07 -0700 Subject: [PATCH 12/29] Create and consume string hashcode interface Signed-off-by: Siddhant Deshmukh --- .../insights/core/listener/QueryInsightsListener.java | 4 ++-- .../core/service/categorizer/QueryShapeGenerator.java | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index f5f8850..fc1667c 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -227,7 +227,7 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final ) ); } - MurmurHash3.Hash128 hashcode = QueryShapeGenerator.getShapeHashCode(request.source(), false); + String hashcode = QueryShapeGenerator.getShapeHashCodeAsString(request.source(), false); Map attributes = new HashMap<>(); attributes.put(Attribute.SEARCH_TYPE, request.searchType().toString().toLowerCase(Locale.ROOT)); @@ -236,7 +236,7 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final attributes.put(Attribute.INDICES, request.indices()); attributes.put(Attribute.PHASE_LATENCY_MAP, searchRequestContext.phaseTookMap()); attributes.put(Attribute.TASK_RESOURCE_USAGES, tasksResourceUsages); - attributes.put(Attribute.QUERY_HASHCODE, hashcode.toString()); + attributes.put(Attribute.QUERY_HASHCODE, hashcode); Map labels = new HashMap<>(); // Retrieve user provided label if exists diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/QueryShapeGenerator.java b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/QueryShapeGenerator.java index aedc634..5664f3c 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/QueryShapeGenerator.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/categorizer/QueryShapeGenerator.java @@ -90,6 +90,12 @@ public static MurmurHash3.Hash128 getShapeHashCode(SearchSourceBuilder source, B return MurmurHash3.hash128(shapeBytes.bytes, 0, shapeBytes.length, 0, new MurmurHash3.Hash128()); } + public static String getShapeHashCodeAsString(SearchSourceBuilder source, Boolean showFields) { + MurmurHash3.Hash128 hashcode = getShapeHashCode(source, showFields); + String hashAsString = Long.toHexString(hashcode.h1) + Long.toHexString(hashcode.h2); + return hashAsString; + } + /** * Method to build search query shape given a source * @param source search request source From 0c469ac1f2ac5575de035f8dc8df853f76c190a5 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Wed, 28 Aug 2024 11:35:16 -0700 Subject: [PATCH 13/29] Health checks in code Signed-off-by: Siddhant Deshmukh --- .../plugin/insights/core/listener/QueryInsightsListener.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index fc1667c..06b5dab 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -28,7 +28,6 @@ import org.opensearch.action.search.SearchRequestOperationsListener; import org.opensearch.action.search.SearchTask; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.hash.MurmurHash3; import org.opensearch.common.inject.Inject; import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo; import org.opensearch.core.xcontent.ToXContent; From fcd40973f701a88c30dcec7d56ad8596a106aade Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Wed, 28 Aug 2024 13:26:30 -0700 Subject: [PATCH 14/29] Fix tests and spotless apply Signed-off-by: Siddhant Deshmukh --- .../service/QueryInsightsServiceTests.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java index ce4d4eb..e0fb47d 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryInsightsServiceTests.java @@ -90,26 +90,6 @@ public void testSearchQueryMetricsEnabled() { } - public void testFeaturesEnableDisable() { - // Test case 1: All metric type collection disabled and search query metrics disabled, enable search query metrics - queryInsightsServiceSpy.enableCollection(MetricType.LATENCY, false); - queryInsightsServiceSpy.enableCollection(MetricType.CPU, false); - queryInsightsServiceSpy.enableCollection(MetricType.MEMORY, false); - queryInsightsServiceSpy.setSearchQueryMetricsEnabled(false); - - queryInsightsServiceSpy.setSearchQueryMetricsEnabled(true); - verify(queryInsightsServiceSpy).checkAndRestartQueryInsights(); - - // Test case 2: All metric type collection disabled and search query metrics enabled, disable search query metrics - queryInsightsServiceSpy.enableCollection(MetricType.LATENCY, false); - queryInsightsServiceSpy.enableCollection(MetricType.CPU, false); - queryInsightsServiceSpy.enableCollection(MetricType.MEMORY, false); - queryInsightsServiceSpy.setSearchQueryMetricsEnabled(true); - - queryInsightsServiceSpy.setSearchQueryMetricsEnabled(false); - verify(queryInsightsServiceSpy).checkAndStopQueryInsights(); - } - public void testAddRecordGroupBySimilarityWithDifferentGroups() { int numberOfRecordsRequired = 10; From c695463da0dc9354e388267e463352ee609a9571 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Wed, 28 Aug 2024 14:22:41 -0700 Subject: [PATCH 15/29] Minor fixes Signed-off-by: Siddhant Deshmukh --- .../core/service/QueryInsightsService.java | 4 ++-- .../core/service/QueryGrouperTests.java | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java index 7082298..07bf8c0 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java @@ -269,11 +269,11 @@ public boolean isSearchQueryMetricsFeatureEnabled() { } /** - * Is grouping feature enabled and grouping not NONE + * Is grouping feature enabled and TopN feature enabled * @return boolean */ public boolean isGroupingEnabled() { - return this.groupingType != GroupingType.NONE; + return this.groupingType != GroupingType.NONE && isTopNFeatureEnabled(); } /** diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java index 6e58f60..0b6376c 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java @@ -332,7 +332,7 @@ public void testNoneGroupingTypeIllegalArgumentException() { } } - // 1. New query group not existing added to MIN + // New query group not existing added to MIN public void testNewGroupAddedToMin() { queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); @@ -353,7 +353,7 @@ public void testNewGroupAddedToMin() { assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - // 2. New query group not existing added to MIN and overflows to MAX + // New query group not existing added to MIN and overflows to MAX public void testNewGroupOverflowsMinToMax() { queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); @@ -374,7 +374,7 @@ public void testNewGroupOverflowsMinToMax() { assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - // 3. New query group not existing added to MIN and causes other group to overflow to MAX + // New query group not existing added to MIN and causes other group to overflow to MAX public void testNewGroupCausesOtherGroupOverflowMinToMax() { queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); @@ -395,7 +395,7 @@ public void testNewGroupCausesOtherGroupOverflowMinToMax() { assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - // 4. Existing query group update to MIN increases average + // Existing query group update to MIN increases average public void testExistingGroupUpdateToMinIncreaseAverage() { queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); @@ -424,7 +424,7 @@ public void testExistingGroupUpdateToMinIncreaseAverage() { assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - // 5. Existing query group update to MIN decrease average - stay in MIN + // Existing query group update to MIN decrease average - stay in MIN public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); @@ -453,7 +453,7 @@ public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - // 6. Existing query group update to MIN decrease average - overflows to MAX + // Existing query group update to MIN decrease average - overflows to MAX public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); @@ -482,7 +482,7 @@ public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - // 7. Existing query group update to MAX increases average - stay in MAX + // Existing query group update to MAX increases average - stay in MAX public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); @@ -511,7 +511,7 @@ public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - // 8. Existing query group update to MAX increases average - promote to MIN + // Existing query group update to MAX increases average - promote to MIN public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); @@ -540,7 +540,7 @@ public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } - // 9. Existing query group update to MAX decrease average + // Existing query group update to MAX decrease average public void testExistingGroupUpdateToMaxDecreaseAverage() { queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); From a90169a0381284fe6c54be753fec93e06bde8c01 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Thu, 29 Aug 2024 15:18:28 -0700 Subject: [PATCH 16/29] Max groups setting and unit tests Signed-off-by: Siddhant Deshmukh --- .../plugin/insights/QueryInsightsPlugin.java | 1 + .../core/listener/QueryInsightsListener.java | 15 ++++++ .../insights/core/service/QueryGrouper.java | 51 ++++++++++++++++++- .../core/service/QueryInsightsService.java | 22 ++++++++ .../core/service/TopQueriesService.java | 4 ++ .../settings/QueryInsightsSettings.java | 13 +++++ .../insights/QueryInsightsPluginTests.java | 1 + .../insights/QueryInsightsTestUtils.java | 1 + .../core/service/QueryGrouperTests.java | 31 +++++++++++ 9 files changed, 138 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java b/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java index 60a3a9a..4841108 100644 --- a/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java +++ b/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java @@ -131,6 +131,7 @@ public List> getSettings() { QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE, QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS, QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY, + QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS, QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING ); } diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index 06b5dab..b88a305 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -10,6 +10,7 @@ import static org.opensearch.plugin.insights.settings.QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY; +import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNEnabledSetting; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNSizeSetting; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNWindowSizeSetting; @@ -116,6 +117,20 @@ public QueryInsightsListener( ); this.queryInsightsService.validateGrouping(clusterService.getClusterSettings().get(TOP_N_QUERIES_GROUP_BY)); this.queryInsightsService.setGrouping(clusterService.getClusterSettings().get(TOP_N_QUERIES_GROUP_BY)); + + clusterService.getClusterSettings() + .addSettingsUpdateConsumer( + TOP_N_QUERIES_MAX_GROUPS, + v -> this.queryInsightsService.setMaximumGroups(v), + v -> this.queryInsightsService.validateMaximumGroups(v) + ); + this.queryInsightsService.validateMaximumGroups(clusterService.getClusterSettings().get(TOP_N_QUERIES_MAX_GROUPS)); + this.queryInsightsService.setMaximumGroups(clusterService.getClusterSettings().get(TOP_N_QUERIES_MAX_GROUPS)); + + // Settings endpoints set for search query metrics + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(SEARCH_QUERY_METRICS_ENABLED_SETTING, v -> setSearchQueryMetricsEnabled(v)); + setSearchQueryMetricsEnabled(clusterService.getClusterSettings().get(SEARCH_QUERY_METRICS_ENABLED_SETTING)); } /** diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java index c620b4c..0efa791 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java @@ -8,6 +8,8 @@ package org.opensearch.plugin.insights.core.service; +import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS; + import java.util.HashMap; import java.util.Map; import java.util.PriorityQueue; @@ -19,6 +21,7 @@ import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; +import org.opensearch.plugin.insights.settings.QueryInsightsSettings; /** * Handles grouping of search queries based on the GroupingType for the MetricType @@ -69,6 +72,14 @@ public class QueryGrouper { */ private int topNSize; + /** + * To keep track of Top N groups we need to store details of all the groups encountered in the window. + * This value can be arbitrarily large and we need to limit this. + * Following is the maximum number of groups that should be tracked when calculating Top N groups and we have a + * cluster setting to configure. + */ + private int maxGroups; + public QueryGrouper( MetricType metricType, GroupingType groupingType, @@ -84,6 +95,7 @@ public QueryGrouper( this.maxHeapQueryStore = new PriorityQueue<>((a, b) -> SearchQueryRecord.compare(b, a, metricType)); this.topNSize = topNSize; + this.maxGroups = QueryInsightsSettings.DEFAULT_MAX_GROUPS; } /** @@ -102,6 +114,10 @@ public SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord) { // New group added to the grouping service // Add to min PQ and overflow records to max PQ (if the number of records in the min PQ exceeds the configured size N) if (!groupIdToAggSearchQueryRecord.containsKey(groupId)) { + boolean maxGroupsLimitReached = checkMaxGroupsLimitReached(groupId); + if (maxGroupsLimitReached) { + return null; + } aggregateSearchQueryRecord = searchQueryRecord; aggregateSearchQueryRecord.setGroupingId(groupId); aggregateSearchQueryRecord.setMeasurementAggregation(metricType, aggregationType); @@ -176,8 +192,21 @@ private void addToMaxPQPromoteToMinPQ(SearchQueryRecord aggregateSearchQueryReco } } + private boolean checkMaxGroupsLimitReached(String groupId) { + if (maxGroups <= maxHeapQueryStore.size()) { + log.warn( + "Exceeded [{}] setting threshold which is set at {}. Discarding new group with id {}.", + TOP_N_QUERIES_MAX_GROUPS.getKey(), + maxGroups, + groupId + ); + return true; + } + return false; + } + /** - * Drain the internal grouping. Needs to be performed after every window. + * Drain the internal grouping. Needs to be performed after every window or if a setting is changed. */ public void drain() { log.debug("Number of groups for the current window is " + numberOfGroups()); @@ -217,6 +246,26 @@ public GroupingType getGroupingType() { return groupingType; } + /** + * Get maximum number of groups that should be tracked when calculating Top N groups + * @return max number of groups + */ + public int getMaxGroups() { + return maxGroups; + } + + /** + * Set the maximum number of groups that should be tracked when calculating Top N groups. + * If the value changes, reset the state of the query grouper service by draining all internal data. + * @param maxGroups max number of groups + */ + public void setMaxGroups(int maxGroups) { + if (this.maxGroups != maxGroups) { + this.maxGroups = maxGroups; + drain(); + } + } + /** * Get groupingId. This should be query hashcode for SIMILARITY grouping and user_id for USER_ID grouping. * @param searchQueryRecord record diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java index 07bf8c0..bda5809 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java @@ -218,6 +218,28 @@ public void setGrouping(final String groupingTypeSetting) { } } + /** + * Set max number of groups + * @param maxGroups maximum number of groups that should be tracked when calculating Top N groups + */ + public void setMaximumGroups(final int maxGroups) { + for (MetricType metricType : MetricType.allMetricTypes()) { + this.topQueriesServices.get(metricType).setMaxGroups(maxGroups); + } + } + + /** + * Validate max number of groups. Should be between 1 and MAX_GROUPS_LIMIT + * @param maxGroups maximum number of groups that should be tracked when calculating Top N groups + */ + public void validateMaximumGroups(final int maxGroups) { + if (maxGroups < 1 || maxGroups > QueryInsightsSettings.MAX_GROUPS_LIMIT) { + throw new IllegalArgumentException( + "Max groups setting" + " should be between 1 and " + QueryInsightsSettings.MAX_GROUPS_LIMIT + ", was (" + maxGroups + ")" + ); + } + } + /** * Get the grouping type based on the metricType * @return GroupingType diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index f67f113..30531cb 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -185,6 +185,10 @@ public void setGrouping(final GroupingType groupingType) { queryGrouper.setGroupingType(groupingType); } + public void setMaxGroups(final int maxGroups) { + queryGrouper.setMaxGroups(maxGroups); + } + /** * Validate if the window size is valid, based on internal constrains. * diff --git a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java index 876c122..12c4924 100644 --- a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java +++ b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java @@ -71,6 +71,9 @@ public class QueryInsightsSettings { public static final String PLUGINS_BASE_URI = "/_insights"; public static final GroupingType DEFAULT_GROUPING_TYPE = GroupingType.NONE; + public static final int DEFAULT_MAX_GROUPS = 100; + + public static final int MAX_GROUPS_LIMIT = 10000; /** * Settings for Top Queries @@ -125,6 +128,16 @@ public class QueryInsightsSettings { Setting.Property.Dynamic ); + /** + * Define the group_by option for Top N queries to group queries. + */ + public static final Setting TOP_N_QUERIES_MAX_GROUPS = Setting.intSetting( + TOP_N_QUERIES_SETTING_PREFIX + ".max_groups", + DEFAULT_MAX_GROUPS, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** * Boolean setting for enabling top queries by cpu. */ diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java index 0060c55..5799049 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java @@ -76,6 +76,7 @@ public void testGetSettings() { QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE, QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS, QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY, + QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS, QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING ), queryInsightsPlugin.getSettings() diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java index 016bbe3..1a37940 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java @@ -323,6 +323,7 @@ public static void registerAllQueryInsightsSettings(ClusterSettings clusterSetti clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE); clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS); clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY); + clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS); clusterSettings.registerSetting(QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING); } } diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java index 0b6376c..6a042fc 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java @@ -628,6 +628,37 @@ public void testMultipleQueryGroupsUpdates() { assertEquals(2, queryGrouper.numberOfTopGroups()); assertEquals(850L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(3, queryGrouper.numberOfGroups()); + } + + public void testMaxGroupLimitReached() { + queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 1); + + queryGrouper.setMaxGroups(1); + + List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, + MetricType.LATENCY, + List.of(900L, 1000L, 1000L) + ); + + List> allRecords2 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( + 1, + MetricType.LATENCY, + List.of(800L, 400L, 1200L) + ); + + allRecords1.addAll(allRecords2); + + for (List recordList : allRecords1) { + for (SearchQueryRecord record : recordList) { + queryGrouper.addQueryToGroup(record); + } + } + + assertEquals(1, queryGrouper.numberOfTopGroups()); + assertEquals(850L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); + assertEquals(2, queryGrouper.numberOfGroups()); } private QueryGrouper getQueryGroupingService(AggregationType aggregationType, int topNSize) { From ebee6ae96bb7cddfb9f02acc60b6170b1917bdae Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 30 Aug 2024 12:00:00 -0700 Subject: [PATCH 17/29] Address review comments Signed-off-by: Siddhant Deshmukh --- .../core/listener/QueryInsightsListener.java | 5 ++- .../insights/core/service/QueryGrouper.java | 34 +++++++++---------- .../core/service/TopQueriesService.java | 6 ++-- .../core/service/QueryGrouperTests.java | 4 +-- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index b88a305..32a5032 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -241,7 +241,10 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final ) ); } - String hashcode = QueryShapeGenerator.getShapeHashCodeAsString(request.source(), false); + String hashcode = null; + if (queryInsightsService.isSearchQueryMetricsFeatureEnabled()) { + QueryShapeGenerator.getShapeHashCodeAsString(request.source(), false); + } Map attributes = new HashMap<>(); attributes.put(Attribute.SEARCH_TYPE, request.searchType().toString().toLowerCase(Locale.ROOT)); diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java index 0efa791..68d1dbb 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java @@ -13,6 +13,7 @@ import java.util.HashMap; import java.util.Map; import java.util.PriorityQueue; +import java.util.concurrent.PriorityBlockingQueue; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.collect.Tuple; @@ -25,7 +26,7 @@ /** * Handles grouping of search queries based on the GroupingType for the MetricType - * Following algorithm : + * Following algorithm : https://github.com/opensearch-project/OpenSearch/issues/13357#issuecomment-2269706425 */ public class QueryGrouper { @@ -58,7 +59,7 @@ public class QueryGrouper { /** * Min heap to keep track of the Top N query groups and is passed from TopQueriesService as the topQueriesStore */ - private PriorityQueue minHeapTopQueriesStore; + private PriorityBlockingQueue minHeapTopQueriesStore; /** * The Max heap is an overflow data structure used to manage records that exceed the capacity of the Min heap. * It stores all records not included in the Top N query results. When the aggregate measurement for one of these @@ -84,7 +85,7 @@ public QueryGrouper( MetricType metricType, GroupingType groupingType, AggregationType aggregationType, - PriorityQueue topQueriesStore, + PriorityBlockingQueue topQueriesStore, int topNSize ) { this.groupingType = groupingType; @@ -111,8 +112,19 @@ public SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord) { SearchQueryRecord aggregateSearchQueryRecord; String groupId = getGroupingId(searchQueryRecord); - // New group added to the grouping service + // 1) New group added to the grouping service // Add to min PQ and overflow records to max PQ (if the number of records in the min PQ exceeds the configured size N) + // 2) Existing group being updated to the grouping service + // a. If present in min PQ + // - remove the record from the min PQ + // - update the aggregate record (aggregate measurement could increase or decrease) + // - If max PQ contains elements, add to max PQ and promote any records to min PQ + // - If max PQ is empty, add to min PQ and overflow any records to max PQ + // b. If present in max PQ + // - remove the record from the max PQ + // - update the aggregate record (aggregate measurement could increase or decrease) + // - If min PQ is full, add to min PQ and overflow any records to max PQ + // - else, add to max PQ and promote any records to min PQ if (!groupIdToAggSearchQueryRecord.containsKey(groupId)) { boolean maxGroupsLimitReached = checkMaxGroupsLimitReached(groupId); if (maxGroupsLimitReached) { @@ -122,19 +134,7 @@ public SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord) { aggregateSearchQueryRecord.setGroupingId(groupId); aggregateSearchQueryRecord.setMeasurementAggregation(metricType, aggregationType); addToMinPQOverflowToMaxPQ(aggregateSearchQueryRecord, groupId); - } - // Existing group being updated to the grouping service - // 1. If present in min PQ - // - remove the record from the min PQ - // - update the aggregate record (aggregate measurement could increase or decrease) - // - If max PQ contains elements, add to max PQ and promote any records to min PQ - // - If max PQ is empty, add to min PQ and overflow any records to max PQ - // 2. If present in max PQ - // - remove the record from the max PQ - // - update the aggregate record (aggregate measurement could increase or decrease) - // - If min PQ is full, add to min PQ and overflow any records to max PQ - // - else, add to max PQ and promote any records to min PQ - else { + } else { aggregateSearchQueryRecord = groupIdToAggSearchQueryRecord.get(groupId).v1(); boolean isPresentInMinPQ = groupIdToAggSearchQueryRecord.get(groupId).v2(); if (isPresentInMinPQ) { diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index 30531cb..d5fa8fc 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -24,7 +24,7 @@ import java.util.Collection; import java.util.List; import java.util.Locale; -import java.util.PriorityQueue; +import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -68,7 +68,7 @@ public class TopQueriesService { /** * The internal thread-safe store that holds the top n queries insight data */ - private final PriorityQueue topQueriesStore; + private final PriorityBlockingQueue topQueriesStore; /** * The AtomicReference of a snapshot of the current window top queries for getters to consume @@ -110,7 +110,7 @@ public class TopQueriesService { this.windowSize = QueryInsightsSettings.DEFAULT_WINDOW_SIZE; this.windowStart = -1L; this.exporter = null; - topQueriesStore = new PriorityQueue<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); + topQueriesStore = new PriorityBlockingQueue<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot = new AtomicReference<>(new ArrayList<>()); topQueriesHistorySnapshot = new AtomicReference<>(new ArrayList<>()); queryGrouper = new QueryGrouper( diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java index 6a042fc..e2270ad 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java @@ -10,8 +10,8 @@ import java.util.HashSet; import java.util.List; -import java.util.PriorityQueue; import java.util.Set; +import java.util.concurrent.PriorityBlockingQueue; import org.junit.Before; import org.opensearch.plugin.insights.QueryInsightsTestUtils; import org.opensearch.plugin.insights.rules.model.AggregationType; @@ -26,7 +26,7 @@ */ public class QueryGrouperTests extends OpenSearchTestCase { private QueryGrouper queryGrouper; - private PriorityQueue topQueriesStore = new PriorityQueue<>( + private PriorityBlockingQueue topQueriesStore = new PriorityBlockingQueue<>( 100, (a, b) -> SearchQueryRecord.compare(a, b, MetricType.LATENCY) ); From b98050d7fb52ec35c9874ab19e3524c92dbaca0c Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 30 Aug 2024 13:03:37 -0700 Subject: [PATCH 18/29] Address review comments Signed-off-by: Siddhant Deshmukh --- .../core/listener/QueryInsightsListener.java | 6 ++---- .../insights/core/service/QueryGrouper.java | 15 ++++++--------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index 32a5032..a5111ee 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -241,10 +241,8 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final ) ); } - String hashcode = null; - if (queryInsightsService.isSearchQueryMetricsFeatureEnabled()) { - QueryShapeGenerator.getShapeHashCodeAsString(request.source(), false); - } + + String hashcode = QueryShapeGenerator.getShapeHashCodeAsString(request.source(), false); Map attributes = new HashMap<>(); attributes.put(Attribute.SEARCH_TYPE, request.searchType().toString().toLowerCase(Locale.ROOT)); diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java index 68d1dbb..3a8e150 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java @@ -10,9 +10,7 @@ import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS; -import java.util.HashMap; -import java.util.Map; -import java.util.PriorityQueue; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.PriorityBlockingQueue; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -37,7 +35,7 @@ public class QueryGrouper { /** * Grouping type for the current grouping service */ - private GroupingType groupingType; + private volatile GroupingType groupingType; /** * Metric type for the current grouping service */ @@ -55,7 +53,7 @@ public class QueryGrouper { * boolean: True if the aggregate record is in the Top N queries priority query (min heap) and False if the aggregate * record is in the Max Heap */ - private Map> groupIdToAggSearchQueryRecord; + private ConcurrentHashMap> groupIdToAggSearchQueryRecord; /** * Min heap to keep track of the Top N query groups and is passed from TopQueriesService as the topQueriesStore */ @@ -66,7 +64,7 @@ public class QueryGrouper { * records is updated and it now qualifies as part of the Top N, the record is moved from the Max heap to the Min heap, * and the records are rearranged accordingly. */ - private PriorityQueue maxHeapQueryStore; + private PriorityBlockingQueue maxHeapQueryStore; /** * Top N size based on the configuration set @@ -91,12 +89,11 @@ public QueryGrouper( this.groupingType = groupingType; this.metricType = metricType; this.aggregationType = aggregationType; - this.groupIdToAggSearchQueryRecord = new HashMap<>(); + this.groupIdToAggSearchQueryRecord = new ConcurrentHashMap<>(); this.minHeapTopQueriesStore = topQueriesStore; - this.maxHeapQueryStore = new PriorityQueue<>((a, b) -> SearchQueryRecord.compare(b, a, metricType)); - this.topNSize = topNSize; this.maxGroups = QueryInsightsSettings.DEFAULT_MAX_GROUPS; + this.maxHeapQueryStore = new PriorityBlockingQueue<>(maxGroups, (a, b) -> SearchQueryRecord.compare(b, a, metricType)); } /** From cd05fb6c60bcf456771b46089acd10f3bc2a5d9c Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 30 Aug 2024 15:44:33 -0700 Subject: [PATCH 19/29] Create query grouper interface and top query store interface Signed-off-by: Siddhant Deshmukh --- .../core/service/TopQueriesService.java | 39 +-- .../MinMaxHeapQueryGrouper.java} | 127 +++++----- .../core/service/grouper/QueryGrouper.java | 66 ++++++ .../store/PriorityQueueTopQueriesStore.java | 106 +++++++++ .../core/service/store/TopQueriesStore.java | 86 +++++++ .../MinMaxHeapQueryGrouperTests.java} | 224 ++++++++++-------- 6 files changed, 467 insertions(+), 181 deletions(-) rename src/main/java/org/opensearch/plugin/insights/core/service/{QueryGrouper.java => grouper/MinMaxHeapQueryGrouper.java} (93%) create mode 100644 src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java create mode 100644 src/main/java/org/opensearch/plugin/insights/core/service/store/PriorityQueueTopQueriesStore.java create mode 100644 src/main/java/org/opensearch/plugin/insights/core/service/store/TopQueriesStore.java rename src/test/java/org/opensearch/plugin/insights/core/service/{QueryGrouperTests.java => grouper/MinMaxHeapQueryGrouperTests.java} (71%) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index d5fa8fc..0e748a9 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -24,7 +24,6 @@ import java.util.Collection; import java.util.List; import java.util.Locale; -import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -35,6 +34,8 @@ import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporter; import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory; import org.opensearch.plugin.insights.core.exporter.SinkType; +import org.opensearch.plugin.insights.core.service.grouper.MinMaxHeapQueryGrouper; +import org.opensearch.plugin.insights.core.service.store.PriorityQueueTopQueriesStore; import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; @@ -68,7 +69,7 @@ public class TopQueriesService { /** * The internal thread-safe store that holds the top n queries insight data */ - private final PriorityBlockingQueue topQueriesStore; + private final PriorityQueueTopQueriesStore priorityQueueTopQueriesStore; /** * The AtomicReference of a snapshot of the current window top queries for getters to consume @@ -95,7 +96,7 @@ public class TopQueriesService { */ private QueryInsightsExporter exporter; - private QueryGrouper queryGrouper; + private MinMaxHeapQueryGrouper minMaxHeapQueryGrouper; TopQueriesService( final MetricType metricType, @@ -110,14 +111,14 @@ public class TopQueriesService { this.windowSize = QueryInsightsSettings.DEFAULT_WINDOW_SIZE; this.windowStart = -1L; this.exporter = null; - topQueriesStore = new PriorityBlockingQueue<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); + priorityQueueTopQueriesStore = new PriorityQueueTopQueriesStore<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot = new AtomicReference<>(new ArrayList<>()); topQueriesHistorySnapshot = new AtomicReference<>(new ArrayList<>()); - queryGrouper = new QueryGrouper( + minMaxHeapQueryGrouper = new MinMaxHeapQueryGrouper( metricType, QueryInsightsSettings.DEFAULT_GROUPING_TYPE, AggregationType.AVERAGE, - topQueriesStore, + priorityQueueTopQueriesStore, topNSize ); } @@ -129,7 +130,7 @@ public class TopQueriesService { */ public void setTopNSize(final int topNSize) { this.topNSize = topNSize; - this.queryGrouper.updateTopNSize(topNSize); + this.minMaxHeapQueryGrouper.updateTopNSize(topNSize); } /** @@ -182,11 +183,11 @@ public void setWindowSize(final TimeValue windowSize) { } public void setGrouping(final GroupingType groupingType) { - queryGrouper.setGroupingType(groupingType); + minMaxHeapQueryGrouper.setGroupingType(groupingType); } public void setMaxGroups(final int maxGroups) { - queryGrouper.setMaxGroups(maxGroups); + minMaxHeapQueryGrouper.setMaxGroups(maxGroups); } /** @@ -320,21 +321,21 @@ void consumeRecords(final List records) { // add records in current window, if there are any, to the top n store addToTopNStore(recordsInThisWindow); // update the current window snapshot for getters to consume - final List newSnapShot = new ArrayList<>(topQueriesStore); + final List newSnapShot = new ArrayList<>(priorityQueueTopQueriesStore.getSnapshot()); newSnapShot.sort((a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot.set(newSnapShot); } private void addToTopNStore(final List records) { - if (queryGrouper.getGroupingType() != GroupingType.NONE) { + if (minMaxHeapQueryGrouper.getGroupingType() != GroupingType.NONE) { for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } else { - topQueriesStore.addAll(records); + priorityQueueTopQueriesStore.addAll(records); // remove top elements for fix sizing priority queue - while (topQueriesStore.size() > topNSize) { - topQueriesStore.poll(); + while (priorityQueueTopQueriesStore.size() > topNSize) { + priorityQueueTopQueriesStore.poll(); } } } @@ -351,12 +352,12 @@ private void rotateWindowIfNecessary(final long newWindowStart) { final List history = new ArrayList<>(); // rotate the current window to history store only if the data belongs to the last window if (windowStart == newWindowStart - windowSize.getMillis()) { - history.addAll(topQueriesStore); + history.addAll(priorityQueueTopQueriesStore.getSnapshot()); } topQueriesHistorySnapshot.set(history); - topQueriesStore.clear(); - if (queryGrouper.getGroupingType() != GroupingType.NONE) { - queryGrouper.drain(); + priorityQueueTopQueriesStore.clear(); + if (minMaxHeapQueryGrouper.getGroupingType() != GroupingType.NONE) { + minMaxHeapQueryGrouper.drain(); } topQueriesCurrentSnapshot.set(new ArrayList<>()); windowStart = newWindowStart; diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java similarity index 93% rename from src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java rename to src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java index 3a8e150..9f88bcc 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.plugin.insights.core.service; +package org.opensearch.plugin.insights.core.service.grouper; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS; @@ -15,6 +15,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.collect.Tuple; +import org.opensearch.plugin.insights.core.service.store.PriorityQueueTopQueriesStore; import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.Attribute; import org.opensearch.plugin.insights.rules.model.GroupingType; @@ -26,12 +27,12 @@ * Handles grouping of search queries based on the GroupingType for the MetricType * Following algorithm : https://github.com/opensearch-project/OpenSearch/issues/13357#issuecomment-2269706425 */ -public class QueryGrouper { +public class MinMaxHeapQueryGrouper implements QueryGrouper { /** * Logger */ - private static final Logger log = LogManager.getLogger(QueryGrouper.class); + private static final Logger log = LogManager.getLogger(MinMaxHeapQueryGrouper.class); /** * Grouping type for the current grouping service */ @@ -57,7 +58,7 @@ public class QueryGrouper { /** * Min heap to keep track of the Top N query groups and is passed from TopQueriesService as the topQueriesStore */ - private PriorityBlockingQueue minHeapTopQueriesStore; + private PriorityQueueTopQueriesStore minHeapTopQueriesStore; /** * The Max heap is an overflow data structure used to manage records that exceed the capacity of the Min heap. * It stores all records not included in the Top N query results. When the aggregate measurement for one of these @@ -79,11 +80,11 @@ public class QueryGrouper { */ private int maxGroups; - public QueryGrouper( + public MinMaxHeapQueryGrouper( MetricType metricType, GroupingType groupingType, AggregationType aggregationType, - PriorityBlockingQueue topQueriesStore, + PriorityQueueTopQueriesStore topQueriesStore, int topNSize ) { this.groupingType = groupingType; @@ -102,6 +103,7 @@ public QueryGrouper( * @param searchQueryRecord record * @return return the search query record that represents the group */ + @Override public SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord) { if (groupingType == GroupingType.NONE) { throw new IllegalArgumentException("Do not use addQueryToGroup when GroupingType is None"); @@ -143,6 +145,65 @@ public SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord) { return aggregateSearchQueryRecord; } + /** + * Drain the internal grouping. Needs to be performed after every window or if a setting is changed. + */ + @Override + public void drain() { + log.debug("Number of groups for the current window is " + numberOfGroups()); + groupIdToAggSearchQueryRecord.clear(); + maxHeapQueryStore.clear(); + minHeapTopQueriesStore.clear(); + } + + @Override + public PriorityQueueTopQueriesStore getMinHeapTopQueriesStore() { + return this.minHeapTopQueriesStore; + } + + /** + * Set Grouping Type + * @param newGroupingType grouping type + */ + @Override + public void setGroupingType(GroupingType newGroupingType) { + if (this.groupingType != newGroupingType) { + this.groupingType = newGroupingType; + drain(); + } + } + + /** + * Get Grouping Type + * @return grouping type + */ + @Override + public GroupingType getGroupingType() { + return groupingType; + } + + /** + * Set the maximum number of groups that should be tracked when calculating Top N groups. + * If the value changes, reset the state of the query grouper service by draining all internal data. + * @param maxGroups max number of groups + */ + @Override + public void setMaxGroups(int maxGroups) { + if (this.maxGroups != maxGroups) { + this.maxGroups = maxGroups; + drain(); + } + } + + /** + * Update Top N size + * @param newSize new size + */ + @Override + public void updateTopNSize(int newSize) { + this.topNSize = newSize; + } + private void addToMinPQOverflowToMaxPQ(SearchQueryRecord searchQueryRecord, String groupId) { minHeapTopQueriesStore.add(searchQueryRecord); groupIdToAggSearchQueryRecord.put(groupId, new Tuple<>(searchQueryRecord, true)); @@ -202,20 +263,11 @@ private boolean checkMaxGroupsLimitReached(String groupId) { return false; } - /** - * Drain the internal grouping. Needs to be performed after every window or if a setting is changed. - */ - public void drain() { - log.debug("Number of groups for the current window is " + numberOfGroups()); - groupIdToAggSearchQueryRecord.clear(); - maxHeapQueryStore.clear(); - minHeapTopQueriesStore.clear(); - } - /** * Gives the number of groups as part of the current grouping. * @return number of groups */ + int numberOfGroups() { return groupIdToAggSearchQueryRecord.size(); } @@ -228,41 +280,6 @@ int numberOfTopGroups() { return minHeapTopQueriesStore.size(); } - /** - * Set Grouping Type - * @param newGroupingType grouping type - */ - public void setGroupingType(GroupingType newGroupingType) { - if (this.groupingType != newGroupingType) { - this.groupingType = newGroupingType; - drain(); - } - } - - public GroupingType getGroupingType() { - return groupingType; - } - - /** - * Get maximum number of groups that should be tracked when calculating Top N groups - * @return max number of groups - */ - public int getMaxGroups() { - return maxGroups; - } - - /** - * Set the maximum number of groups that should be tracked when calculating Top N groups. - * If the value changes, reset the state of the query grouper service by draining all internal data. - * @param maxGroups max number of groups - */ - public void setMaxGroups(int maxGroups) { - if (this.maxGroups != maxGroups) { - this.maxGroups = maxGroups; - drain(); - } - } - /** * Get groupingId. This should be query hashcode for SIMILARITY grouping and user_id for USER_ID grouping. * @param searchQueryRecord record @@ -278,12 +295,4 @@ private String getGroupingId(SearchQueryRecord searchQueryRecord) { throw new IllegalArgumentException("The following grouping type is not supported : " + groupingType); } } - - /** - * Update Top N size - * @param newSize new size - */ - public void updateTopNSize(int newSize) { - this.topNSize = newSize; - } } diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java new file mode 100644 index 0000000..6716643 --- /dev/null +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.insights.core.service.grouper; + +import org.opensearch.plugin.insights.core.service.store.PriorityQueueTopQueriesStore; +import org.opensearch.plugin.insights.rules.model.GroupingType; +import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; + +/** + * Interface for grouping search queries based on grouping type for the metric type. + */ +public interface QueryGrouper { + + /** + * Add query to the group based on the GroupType setting. + * @param searchQueryRecord record to be added + * @return the aggregate search query record representing the group + */ + SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord); + + /** + * Drain the internal grouping. Needs to be performed after every window or if a setting is changed. + */ + void drain(); + + /** + * Get the min heap queue that holds the top N queries. + * + * @return the PriorityBlockingQueue containing the top N queries + */ + PriorityQueueTopQueriesStore getMinHeapTopQueriesStore(); + + /** + * Set the grouping type for this grouper. + * + * @param groupingType the grouping type to set + */ + void setGroupingType(GroupingType groupingType); + + /** + * Get the current grouping type for this grouper. + * + * @return the current grouping type + */ + GroupingType getGroupingType(); + + /** + * Set the maximum number of groups allowed. + * + * @param maxGroups the maximum number of groups + */ + void setMaxGroups(int maxGroups); + + /** + * Update the top N size for the grouper. + * + * @param topNSize the new top N size + */ + void updateTopNSize(int topNSize); +} diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/store/PriorityQueueTopQueriesStore.java b/src/main/java/org/opensearch/plugin/insights/core/service/store/PriorityQueueTopQueriesStore.java new file mode 100644 index 0000000..258ced1 --- /dev/null +++ b/src/main/java/org/opensearch/plugin/insights/core/service/store/PriorityQueueTopQueriesStore.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.plugin.insights.core.service.store; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.PriorityBlockingQueue; + +/** + * Implementation of {@link TopQueriesStore} that uses a {@link PriorityBlockingQueue} + * to manage Top N search query records or groups. This implementation is thread-safe. + * + * @param The type of records that this store will manage. + */ +public class PriorityQueueTopQueriesStore implements TopQueriesStore { + + private final PriorityBlockingQueue topQueriesStore; + + /** + * Constructs a new {@code PriorityQueueTopQueriesStore} with the specified capacity. + * + * @param capacity the initial capacity of the store + * @param comparator comparator to use for the priority queue store + */ + public PriorityQueueTopQueriesStore(int capacity, Comparator comparator) { + this.topQueriesStore = new PriorityBlockingQueue<>(capacity, comparator); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean add(T queryRecord) { + return topQueriesStore.add(queryRecord); + } + + /** + * {@inheritDoc} + */ + @Override + public T poll() { + return topQueriesStore.poll(); + } + + /** + * {@inheritDoc} + */ + @Override + public T peek() { + return topQueriesStore.peek(); + } + + /** + * {@inheritDoc} + */ + @Override + public int size() { + return topQueriesStore.size(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isEmpty() { + return topQueriesStore.isEmpty(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean addAll(List records) { + return topQueriesStore.addAll(records); + } + + /** + * {@inheritDoc} + */ + @Override + public void clear() { + topQueriesStore.clear(); + } + + /** + * {@inheritDoc} + */ + @Override + public List getSnapshot() { + return new ArrayList<>(topQueriesStore); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean remove(T queryRecord) { + return topQueriesStore.remove(queryRecord); + } +} diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/store/TopQueriesStore.java b/src/main/java/org/opensearch/plugin/insights/core/service/store/TopQueriesStore.java new file mode 100644 index 0000000..75a2e1b --- /dev/null +++ b/src/main/java/org/opensearch/plugin/insights/core/service/store/TopQueriesStore.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.plugin.insights.core.service.store; + +import java.util.List; + +/** + * Interface for managing a store of top search query records. + * Implementations of this interface should provide thread-safe operations + * to manage and access the top search queries. + * + * @param The type of records that this store will manage. + */ +public interface TopQueriesStore { + + /** + * Adds a search query record to the store. + * + * @param queryRecord the search query record to be added + * @return {@code true} if the record was successfully added, otherwise {@code false} + */ + boolean add(T queryRecord); + + /** + * Retrieves and removes the top search query record from the store, + * or returns {@code null} if the store is empty. + * + * @return the top search query record, or {@code null} if the store is empty + */ + T poll(); + + /** + * Retrieves, but does not remove, the top search query record from the store, + * or returns {@code null} if the store is empty. + * + * @return the top search query record, or {@code null} if the store is empty + */ + T peek(); + + /** + * Returns the number of search query records currently in the store. + * + * @return the size of the store + */ + int size(); + + /** + * Returns {@code true} if the store is empty, otherwise {@code false}. + * + * @return {@code true} if the store is empty, otherwise {@code false} + */ + boolean isEmpty(); + + /** + * Adds a list of query records to the store. + * + * @param records the list of query records to add + * @return {@code true} if all records were added successfully, otherwise {@code false} + */ + boolean addAll(List records); + + /** + * Clears all the records from the store. + */ + void clear(); + + /** + * Returns a snapshot of the current elements in the store as a list. + * + * @return a list containing all the elements currently in the store + */ + List getSnapshot(); + + /** + * Removes the specified record from the store, if it exists. + * + * @param queryRecord the record to remove + * @return {@code true} if the record was present and removed, otherwise {@code false} + */ + boolean remove(T queryRecord); +} diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java similarity index 71% rename from src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java rename to src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java index e2270ad..828a005 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/QueryGrouperTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java @@ -6,14 +6,14 @@ * compatible open source license. */ -package org.opensearch.plugin.insights.core.service; +package org.opensearch.plugin.insights.core.service.grouper; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.PriorityBlockingQueue; import org.junit.Before; import org.opensearch.plugin.insights.QueryInsightsTestUtils; +import org.opensearch.plugin.insights.core.service.store.PriorityQueueTopQueriesStore; import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.Attribute; import org.opensearch.plugin.insights.rules.model.GroupingType; @@ -22,18 +22,18 @@ import org.opensearch.test.OpenSearchTestCase; /** - * Unit Tests for {@link QueryGrouper}. + * Unit Tests for {@link MinMaxHeapQueryGrouper}. */ -public class QueryGrouperTests extends OpenSearchTestCase { - private QueryGrouper queryGrouper; - private PriorityBlockingQueue topQueriesStore = new PriorityBlockingQueue<>( +public class MinMaxHeapQueryGrouperTests extends OpenSearchTestCase { + private MinMaxHeapQueryGrouper minMaxHeapQueryGrouper; + private PriorityQueueTopQueriesStore topQueriesStore = new PriorityQueueTopQueriesStore<>( 100, (a, b) -> SearchQueryRecord.compare(a, b, MetricType.LATENCY) ); @Before public void setup() { - queryGrouper = getQueryGroupingService(AggregationType.DEFAULT_AGGREGATION_TYPE, 10); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.DEFAULT_AGGREGATION_TYPE, 10); } public void testWithAllDifferentHashcodes() { @@ -42,7 +42,7 @@ public void testWithAllDifferentHashcodes() { SearchQueryRecord groupedRecord; Set hashcodeSet = new HashSet<>(); for (SearchQueryRecord record : records) { - groupedRecord = queryGrouper.addQueryToGroup(record); + groupedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); int hashcode = (int) groupedRecord.getAttributes().get(Attribute.QUERY_HASHCODE); hashcodeSet.add(hashcode); } @@ -56,7 +56,7 @@ public void testWithAllSameHashcodes() { SearchQueryRecord groupedRecord; Set hashcodeSet = new HashSet<>(); for (SearchQueryRecord record : records) { - groupedRecord = queryGrouper.addQueryToGroup(record); + groupedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); int hashcode = (int) groupedRecord.getAttributes().get(Attribute.QUERY_HASHCODE); hashcodeSet.add(hashcode); } @@ -67,11 +67,11 @@ public void testDrain() { int numOfRecords = 10; final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - int groupsBeforeDrain = queryGrouper.numberOfGroups(); - queryGrouper.drain(); - int groupsAfterDrain = queryGrouper.numberOfGroups(); + int groupsBeforeDrain = minMaxHeapQueryGrouper.numberOfGroups(); + minMaxHeapQueryGrouper.drain(); + int groupsAfterDrain = minMaxHeapQueryGrouper.numberOfGroups(); assertEquals(numOfRecords, groupsBeforeDrain); assertEquals(0, groupsAfterDrain); @@ -82,27 +82,27 @@ public void testChangeTopNSize() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - assertEquals(10, queryGrouper.numberOfTopGroups()); // Initially expects top 10 groups + assertEquals(10, minMaxHeapQueryGrouper.numberOfTopGroups()); // Initially expects top 10 groups - queryGrouper.updateTopNSize(5); - queryGrouper.drain(); // Clear previous state + minMaxHeapQueryGrouper.updateTopNSize(5); + minMaxHeapQueryGrouper.drain(); // Clear previous state for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - assertEquals(5, queryGrouper.numberOfTopGroups()); // After update, expects top 5 groups + assertEquals(5, minMaxHeapQueryGrouper.numberOfTopGroups()); // After update, expects top 5 groups } public void testEmptyPriorityQueues() { - int groupsBeforeDrain = queryGrouper.numberOfGroups(); + int groupsBeforeDrain = minMaxHeapQueryGrouper.numberOfGroups(); assertEquals(0, groupsBeforeDrain); - queryGrouper.drain(); - int groupsAfterDrain = queryGrouper.numberOfGroups(); + minMaxHeapQueryGrouper.drain(); + int groupsAfterDrain = minMaxHeapQueryGrouper.numberOfGroups(); assertEquals(0, groupsAfterDrain); // No groups should be present after draining } @@ -111,23 +111,23 @@ public void testAddRemoveFromMaxHeap() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - assertTrue(queryGrouper.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap + assertTrue(minMaxHeapQueryGrouper.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap - queryGrouper.updateTopNSize(5); // Change size to 5 - queryGrouper.drain(); // Clear previous state + minMaxHeapQueryGrouper.updateTopNSize(5); // Change size to 5 + minMaxHeapQueryGrouper.drain(); // Clear previous state for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - assertEquals(5, queryGrouper.numberOfTopGroups()); // Should be exactly 5 in the min heap + assertEquals(5, minMaxHeapQueryGrouper.numberOfTopGroups()); // Should be exactly 5 in the min heap } public void testInvalidGroupingType() { - QueryGrouper invalidGroupingService = new QueryGrouper( + MinMaxHeapQueryGrouper invalidGroupingService = new MinMaxHeapQueryGrouper( MetricType.LATENCY, GroupingType.NONE, AggregationType.DEFAULT_AGGREGATION_TYPE, @@ -143,10 +143,10 @@ public void testLargeNumberOfRecords() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - assertTrue(queryGrouper.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap + assertTrue(minMaxHeapQueryGrouper.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap } public void testChangeGroupingType() { @@ -154,15 +154,15 @@ public void testChangeGroupingType() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - int groupsBeforeChange = queryGrouper.numberOfGroups(); + int groupsBeforeChange = minMaxHeapQueryGrouper.numberOfGroups(); assertTrue(groupsBeforeChange > 0); - queryGrouper.setGroupingType(GroupingType.NONE); // Changing to NONE should clear groups + minMaxHeapQueryGrouper.setGroupingType(GroupingType.NONE); // Changing to NONE should clear groups - int groupsAfterChange = queryGrouper.numberOfGroups(); + int groupsAfterChange = minMaxHeapQueryGrouper.numberOfGroups(); assertEquals(0, groupsAfterChange); // Expect no groups after changing to NONE } @@ -171,16 +171,16 @@ public void testDrainWithMultipleGroupingTypes() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - int groupsBeforeDrain = queryGrouper.numberOfGroups(); + int groupsBeforeDrain = minMaxHeapQueryGrouper.numberOfGroups(); assertTrue(groupsBeforeDrain > 0); - queryGrouper.setGroupingType(GroupingType.SIMILARITY); - queryGrouper.drain(); + minMaxHeapQueryGrouper.setGroupingType(GroupingType.SIMILARITY); + minMaxHeapQueryGrouper.drain(); - int groupsAfterDrain = queryGrouper.numberOfGroups(); + int groupsAfterDrain = minMaxHeapQueryGrouper.numberOfGroups(); assertEquals(0, groupsAfterDrain); // After drain, groups should be cleared } @@ -189,21 +189,21 @@ public void testVaryingTopNSize() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - queryGrouper.updateTopNSize(15); - queryGrouper.drain(); // Clear previous state + minMaxHeapQueryGrouper.updateTopNSize(15); + minMaxHeapQueryGrouper.drain(); // Clear previous state for (SearchQueryRecord record : records) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } - assertEquals(15, queryGrouper.numberOfTopGroups()); // Should reflect the updated top N size + assertEquals(15, minMaxHeapQueryGrouper.numberOfTopGroups()); // Should reflect the updated top N size } public void testAddMeasurementSumAggregationLatency() { - queryGrouper = getQueryGroupingService(AggregationType.SUM, 10); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.SUM, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -213,7 +213,7 @@ public void testAddMeasurementSumAggregationLatency() { Number expectedSum = 0; for (SearchQueryRecord record : records) { - aggregatedRecord = queryGrouper.addQueryToGroup(record); + aggregatedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); } @@ -221,7 +221,7 @@ public void testAddMeasurementSumAggregationLatency() { } public void testAddMeasurementAverageAggregationLatency() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 10); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -233,7 +233,7 @@ public void testAddMeasurementAverageAggregationLatency() { int expectedCount = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); - aggregatedRecord = queryGrouper.addQueryToGroup(record); + aggregatedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); expectedCount += 1; } @@ -242,7 +242,7 @@ public void testAddMeasurementAverageAggregationLatency() { } public void testAddMeasurementNoneAggregationLatency() { - queryGrouper = getQueryGroupingService(AggregationType.NONE, 10); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.NONE, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -253,14 +253,20 @@ public void testAddMeasurementNoneAggregationLatency() { Number expectedValue = 0; for (SearchQueryRecord record : records) { expectedValue = record.getMeasurement(MetricType.LATENCY).longValue(); - lastRecord = queryGrouper.addQueryToGroup(record); + lastRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); } assertEquals(expectedValue, lastRecord.getMeasurement(MetricType.LATENCY)); } public void testAddMeasurementSumAggregationCpu() { - queryGrouper = new QueryGrouper(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.SUM, topQueriesStore, 10); + minMaxHeapQueryGrouper = new MinMaxHeapQueryGrouper( + MetricType.CPU, + GroupingType.SIMILARITY, + AggregationType.SUM, + topQueriesStore, + 10 + ); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -270,7 +276,7 @@ public void testAddMeasurementSumAggregationCpu() { Number expectedSum = 0; for (SearchQueryRecord record : records) { - aggregatedRecord = queryGrouper.addQueryToGroup(record); + aggregatedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); } @@ -278,7 +284,13 @@ public void testAddMeasurementSumAggregationCpu() { } public void testAddMeasurementAverageAggregationCpu() { - queryGrouper = new QueryGrouper(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.AVERAGE, topQueriesStore, 10); + minMaxHeapQueryGrouper = new MinMaxHeapQueryGrouper( + MetricType.CPU, + GroupingType.SIMILARITY, + AggregationType.AVERAGE, + topQueriesStore, + 10 + ); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -290,7 +302,7 @@ public void testAddMeasurementAverageAggregationCpu() { int expectedCount = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); - aggregatedRecord = queryGrouper.addQueryToGroup(record); + aggregatedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); expectedCount += 1; } @@ -299,7 +311,13 @@ public void testAddMeasurementAverageAggregationCpu() { } public void testAddMeasurementNoneAggregationCpu() { - queryGrouper = new QueryGrouper(MetricType.CPU, GroupingType.SIMILARITY, AggregationType.NONE, topQueriesStore, 10); + minMaxHeapQueryGrouper = new MinMaxHeapQueryGrouper( + MetricType.CPU, + GroupingType.SIMILARITY, + AggregationType.NONE, + topQueriesStore, + 10 + ); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -310,14 +328,14 @@ public void testAddMeasurementNoneAggregationCpu() { Number expectedValue = 0; for (SearchQueryRecord record : records) { expectedValue = record.getMeasurement(MetricType.CPU).longValue(); - lastRecord = queryGrouper.addQueryToGroup(record); + lastRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); } assertEquals(expectedValue, lastRecord.getMeasurement(MetricType.CPU)); } public void testNoneGroupingTypeIllegalArgumentException() { - queryGrouper = new QueryGrouper(MetricType.CPU, GroupingType.NONE, AggregationType.NONE, topQueriesStore, 10); + minMaxHeapQueryGrouper = new MinMaxHeapQueryGrouper(MetricType.CPU, GroupingType.NONE, AggregationType.NONE, topQueriesStore, 10); int numOfRecords = 10; List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords, AggregationType.NONE); @@ -328,13 +346,13 @@ public void testNoneGroupingTypeIllegalArgumentException() { Number expectedSum = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); - assertThrows(IllegalArgumentException.class, () -> { queryGrouper.addQueryToGroup(record); }); + assertThrows(IllegalArgumentException.class, () -> { minMaxHeapQueryGrouper.addQueryToGroup(record); }); } } // New query group not existing added to MIN public void testNewGroupAddedToMin() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 2, @@ -344,18 +362,18 @@ public void testNewGroupAddedToMin() { for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // New query group not existing added to MIN and overflows to MAX public void testNewGroupOverflowsMinToMax() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 2, @@ -365,18 +383,18 @@ public void testNewGroupOverflowsMinToMax() { for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // New query group not existing added to MIN and causes other group to overflow to MAX public void testNewGroupCausesOtherGroupOverflowMinToMax() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 2, @@ -386,18 +404,18 @@ public void testNewGroupCausesOtherGroupOverflowMinToMax() { for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // Existing query group update to MIN increases average public void testExistingGroupUpdateToMinIncreaseAverage() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -415,18 +433,18 @@ public void testExistingGroupUpdateToMinIncreaseAverage() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1200L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // Existing query group update to MIN decrease average - stay in MIN public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -444,18 +462,18 @@ public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(900L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // Existing query group update to MIN decrease average - overflows to MAX public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -473,18 +491,18 @@ public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // Existing query group update to MAX increases average - stay in MAX public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -502,18 +520,18 @@ public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // Existing query group update to MAX increases average - promote to MIN public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -531,18 +549,18 @@ public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1000L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } // Existing query group update to MAX decrease average public void testExistingGroupUpdateToMaxDecreaseAverage() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -560,17 +578,17 @@ public void testExistingGroupUpdateToMaxDecreaseAverage() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); } public void testSwitchGroupingTypeToNone() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -588,22 +606,22 @@ public void testSwitchGroupingTypeToNone() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(950L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(975L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); - queryGrouper.setGroupingType(GroupingType.NONE); - assertEquals(0, queryGrouper.numberOfTopGroups()); + minMaxHeapQueryGrouper.setGroupingType(GroupingType.NONE); + assertEquals(0, minMaxHeapQueryGrouper.numberOfTopGroups()); - assertThrows(IllegalArgumentException.class, () -> { queryGrouper.addQueryToGroup(allRecords1.get(0).get(0)); }); + assertThrows(IllegalArgumentException.class, () -> { minMaxHeapQueryGrouper.addQueryToGroup(allRecords1.get(0).get(0)); }); } public void testMultipleQueryGroupsUpdates() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 2); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -621,20 +639,20 @@ public void testMultipleQueryGroupsUpdates() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(2, queryGrouper.numberOfTopGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(850L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); assertEquals(1100L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); - assertEquals(3, queryGrouper.numberOfGroups()); + assertEquals(3, minMaxHeapQueryGrouper.numberOfGroups()); } public void testMaxGroupLimitReached() { - queryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 1); + minMaxHeapQueryGrouper = getQueryGroupingService(AggregationType.AVERAGE, 1); - queryGrouper.setMaxGroups(1); + minMaxHeapQueryGrouper.setMaxGroups(1); List> allRecords1 = QueryInsightsTestUtils.generateMultipleQueryInsightsRecordsWithMeasurement( 1, @@ -652,16 +670,16 @@ public void testMaxGroupLimitReached() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - queryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.addQueryToGroup(record); } } - assertEquals(1, queryGrouper.numberOfTopGroups()); + assertEquals(1, minMaxHeapQueryGrouper.numberOfTopGroups()); assertEquals(850L, topQueriesStore.poll().getMeasurement(MetricType.LATENCY)); - assertEquals(2, queryGrouper.numberOfGroups()); + assertEquals(2, minMaxHeapQueryGrouper.numberOfGroups()); } - private QueryGrouper getQueryGroupingService(AggregationType aggregationType, int topNSize) { - return new QueryGrouper(MetricType.LATENCY, GroupingType.SIMILARITY, aggregationType, topQueriesStore, topNSize); + private MinMaxHeapQueryGrouper getQueryGroupingService(AggregationType aggregationType, int topNSize) { + return new MinMaxHeapQueryGrouper(MetricType.LATENCY, GroupingType.SIMILARITY, aggregationType, topQueriesStore, topNSize); } } From 3147fa6ea7bdfd63f661beaa3ebf73e12f76bfdd Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 30 Aug 2024 16:35:53 -0700 Subject: [PATCH 20/29] Address review comments Signed-off-by: Siddhant Deshmukh --- .../core/service/TopQueriesService.java | 38 ++++++----- .../grouper/MinMaxHeapQueryGrouper.java | 10 +-- .../core/service/grouper/QueryGrouper.java | 8 +-- .../grouper/MinMaxHeapQueryGrouperTests.java | 66 +++++++++---------- 4 files changed, 62 insertions(+), 60 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index 0e748a9..1ce7b06 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -35,7 +35,9 @@ import org.opensearch.plugin.insights.core.exporter.QueryInsightsExporterFactory; import org.opensearch.plugin.insights.core.exporter.SinkType; import org.opensearch.plugin.insights.core.service.grouper.MinMaxHeapQueryGrouper; +import org.opensearch.plugin.insights.core.service.grouper.QueryGrouper; import org.opensearch.plugin.insights.core.service.store.PriorityQueueTopQueriesStore; +import org.opensearch.plugin.insights.core.service.store.TopQueriesStore; import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; @@ -69,7 +71,7 @@ public class TopQueriesService { /** * The internal thread-safe store that holds the top n queries insight data */ - private final PriorityQueueTopQueriesStore priorityQueueTopQueriesStore; + private final TopQueriesStore topQueriesStore; /** * The AtomicReference of a snapshot of the current window top queries for getters to consume @@ -96,7 +98,7 @@ public class TopQueriesService { */ private QueryInsightsExporter exporter; - private MinMaxHeapQueryGrouper minMaxHeapQueryGrouper; + private QueryGrouper queryGrouper; TopQueriesService( final MetricType metricType, @@ -111,14 +113,14 @@ public class TopQueriesService { this.windowSize = QueryInsightsSettings.DEFAULT_WINDOW_SIZE; this.windowStart = -1L; this.exporter = null; - priorityQueueTopQueriesStore = new PriorityQueueTopQueriesStore<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); + topQueriesStore = new PriorityQueueTopQueriesStore<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot = new AtomicReference<>(new ArrayList<>()); topQueriesHistorySnapshot = new AtomicReference<>(new ArrayList<>()); - minMaxHeapQueryGrouper = new MinMaxHeapQueryGrouper( + queryGrouper = new MinMaxHeapQueryGrouper( metricType, QueryInsightsSettings.DEFAULT_GROUPING_TYPE, AggregationType.AVERAGE, - priorityQueueTopQueriesStore, + topQueriesStore, topNSize ); } @@ -130,7 +132,7 @@ public class TopQueriesService { */ public void setTopNSize(final int topNSize) { this.topNSize = topNSize; - this.minMaxHeapQueryGrouper.updateTopNSize(topNSize); + this.queryGrouper.updateTopNSize(topNSize); } /** @@ -183,11 +185,11 @@ public void setWindowSize(final TimeValue windowSize) { } public void setGrouping(final GroupingType groupingType) { - minMaxHeapQueryGrouper.setGroupingType(groupingType); + queryGrouper.setGroupingType(groupingType); } public void setMaxGroups(final int maxGroups) { - minMaxHeapQueryGrouper.setMaxGroups(maxGroups); + queryGrouper.setMaxGroups(maxGroups); } /** @@ -321,21 +323,21 @@ void consumeRecords(final List records) { // add records in current window, if there are any, to the top n store addToTopNStore(recordsInThisWindow); // update the current window snapshot for getters to consume - final List newSnapShot = new ArrayList<>(priorityQueueTopQueriesStore.getSnapshot()); + final List newSnapShot = new ArrayList<>(topQueriesStore.getSnapshot()); newSnapShot.sort((a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot.set(newSnapShot); } private void addToTopNStore(final List records) { - if (minMaxHeapQueryGrouper.getGroupingType() != GroupingType.NONE) { + if (queryGrouper.getGroupingType() != GroupingType.NONE) { for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + queryGrouper.add(record); } } else { - priorityQueueTopQueriesStore.addAll(records); + topQueriesStore.addAll(records); // remove top elements for fix sizing priority queue - while (priorityQueueTopQueriesStore.size() > topNSize) { - priorityQueueTopQueriesStore.poll(); + while (topQueriesStore.size() > topNSize) { + topQueriesStore.poll(); } } } @@ -352,12 +354,12 @@ private void rotateWindowIfNecessary(final long newWindowStart) { final List history = new ArrayList<>(); // rotate the current window to history store only if the data belongs to the last window if (windowStart == newWindowStart - windowSize.getMillis()) { - history.addAll(priorityQueueTopQueriesStore.getSnapshot()); + history.addAll(topQueriesStore.getSnapshot()); } topQueriesHistorySnapshot.set(history); - priorityQueueTopQueriesStore.clear(); - if (minMaxHeapQueryGrouper.getGroupingType() != GroupingType.NONE) { - minMaxHeapQueryGrouper.drain(); + topQueriesStore.clear(); + if (queryGrouper.getGroupingType() != GroupingType.NONE) { + queryGrouper.drain(); } topQueriesCurrentSnapshot.set(new ArrayList<>()); windowStart = newWindowStart; diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java index 9f88bcc..a86cf03 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java @@ -15,7 +15,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.collect.Tuple; -import org.opensearch.plugin.insights.core.service.store.PriorityQueueTopQueriesStore; +import org.opensearch.plugin.insights.core.service.store.TopQueriesStore; import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.Attribute; import org.opensearch.plugin.insights.rules.model.GroupingType; @@ -58,7 +58,7 @@ public class MinMaxHeapQueryGrouper implements QueryGrouper { /** * Min heap to keep track of the Top N query groups and is passed from TopQueriesService as the topQueriesStore */ - private PriorityQueueTopQueriesStore minHeapTopQueriesStore; + private TopQueriesStore minHeapTopQueriesStore; /** * The Max heap is an overflow data structure used to manage records that exceed the capacity of the Min heap. * It stores all records not included in the Top N query results. When the aggregate measurement for one of these @@ -84,7 +84,7 @@ public MinMaxHeapQueryGrouper( MetricType metricType, GroupingType groupingType, AggregationType aggregationType, - PriorityQueueTopQueriesStore topQueriesStore, + TopQueriesStore topQueriesStore, int topNSize ) { this.groupingType = groupingType; @@ -104,7 +104,7 @@ public MinMaxHeapQueryGrouper( * @return return the search query record that represents the group */ @Override - public SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord) { + public SearchQueryRecord add(SearchQueryRecord searchQueryRecord) { if (groupingType == GroupingType.NONE) { throw new IllegalArgumentException("Do not use addQueryToGroup when GroupingType is None"); } @@ -157,7 +157,7 @@ public void drain() { } @Override - public PriorityQueueTopQueriesStore getMinHeapTopQueriesStore() { + public TopQueriesStore getTopQueriesStore() { return this.minHeapTopQueriesStore; } diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java index 6716643..4fb9557 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java @@ -8,7 +8,7 @@ package org.opensearch.plugin.insights.core.service.grouper; -import org.opensearch.plugin.insights.core.service.store.PriorityQueueTopQueriesStore; +import org.opensearch.plugin.insights.core.service.store.TopQueriesStore; import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; @@ -22,7 +22,7 @@ public interface QueryGrouper { * @param searchQueryRecord record to be added * @return the aggregate search query record representing the group */ - SearchQueryRecord addQueryToGroup(SearchQueryRecord searchQueryRecord); + SearchQueryRecord add(SearchQueryRecord searchQueryRecord); /** * Drain the internal grouping. Needs to be performed after every window or if a setting is changed. @@ -32,9 +32,9 @@ public interface QueryGrouper { /** * Get the min heap queue that holds the top N queries. * - * @return the PriorityBlockingQueue containing the top N queries + * @return the TopQueriesStore containing the top N queries */ - PriorityQueueTopQueriesStore getMinHeapTopQueriesStore(); + TopQueriesStore getTopQueriesStore(); /** * Set the grouping type for this grouper. diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java index 828a005..89ebc8d 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java @@ -42,7 +42,7 @@ public void testWithAllDifferentHashcodes() { SearchQueryRecord groupedRecord; Set hashcodeSet = new HashSet<>(); for (SearchQueryRecord record : records) { - groupedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); + groupedRecord = minMaxHeapQueryGrouper.add(record); int hashcode = (int) groupedRecord.getAttributes().get(Attribute.QUERY_HASHCODE); hashcodeSet.add(hashcode); } @@ -56,7 +56,7 @@ public void testWithAllSameHashcodes() { SearchQueryRecord groupedRecord; Set hashcodeSet = new HashSet<>(); for (SearchQueryRecord record : records) { - groupedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); + groupedRecord = minMaxHeapQueryGrouper.add(record); int hashcode = (int) groupedRecord.getAttributes().get(Attribute.QUERY_HASHCODE); hashcodeSet.add(hashcode); } @@ -67,7 +67,7 @@ public void testDrain() { int numOfRecords = 10; final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } int groupsBeforeDrain = minMaxHeapQueryGrouper.numberOfGroups(); minMaxHeapQueryGrouper.drain(); @@ -82,7 +82,7 @@ public void testChangeTopNSize() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } assertEquals(10, minMaxHeapQueryGrouper.numberOfTopGroups()); // Initially expects top 10 groups @@ -91,7 +91,7 @@ public void testChangeTopNSize() { minMaxHeapQueryGrouper.drain(); // Clear previous state for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } assertEquals(5, minMaxHeapQueryGrouper.numberOfTopGroups()); // After update, expects top 5 groups @@ -111,7 +111,7 @@ public void testAddRemoveFromMaxHeap() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } assertTrue(minMaxHeapQueryGrouper.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap @@ -120,7 +120,7 @@ public void testAddRemoveFromMaxHeap() { minMaxHeapQueryGrouper.drain(); // Clear previous state for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } assertEquals(5, minMaxHeapQueryGrouper.numberOfTopGroups()); // Should be exactly 5 in the min heap @@ -135,7 +135,7 @@ public void testInvalidGroupingType() { 10 ); SearchQueryRecord record = QueryInsightsTestUtils.generateQueryInsightRecords(1).get(0); - expectThrows(IllegalArgumentException.class, () -> invalidGroupingService.addQueryToGroup(record)); + expectThrows(IllegalArgumentException.class, () -> invalidGroupingService.add(record)); } public void testLargeNumberOfRecords() { @@ -143,7 +143,7 @@ public void testLargeNumberOfRecords() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } assertTrue(minMaxHeapQueryGrouper.numberOfTopGroups() <= 10); // Should be at most 10 in the min heap @@ -154,7 +154,7 @@ public void testChangeGroupingType() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } int groupsBeforeChange = minMaxHeapQueryGrouper.numberOfGroups(); @@ -171,7 +171,7 @@ public void testDrainWithMultipleGroupingTypes() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } int groupsBeforeDrain = minMaxHeapQueryGrouper.numberOfGroups(); @@ -189,14 +189,14 @@ public void testVaryingTopNSize() { final List records = QueryInsightsTestUtils.generateQueryInsightRecords(numOfRecords); for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } minMaxHeapQueryGrouper.updateTopNSize(15); minMaxHeapQueryGrouper.drain(); // Clear previous state for (SearchQueryRecord record : records) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } assertEquals(15, minMaxHeapQueryGrouper.numberOfTopGroups()); // Should reflect the updated top N size @@ -213,7 +213,7 @@ public void testAddMeasurementSumAggregationLatency() { Number expectedSum = 0; for (SearchQueryRecord record : records) { - aggregatedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); + aggregatedRecord = minMaxHeapQueryGrouper.add(record); expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); } @@ -233,7 +233,7 @@ public void testAddMeasurementAverageAggregationLatency() { int expectedCount = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.LATENCY).longValue(); - aggregatedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); + aggregatedRecord = minMaxHeapQueryGrouper.add(record); expectedCount += 1; } @@ -253,7 +253,7 @@ public void testAddMeasurementNoneAggregationLatency() { Number expectedValue = 0; for (SearchQueryRecord record : records) { expectedValue = record.getMeasurement(MetricType.LATENCY).longValue(); - lastRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); + lastRecord = minMaxHeapQueryGrouper.add(record); } assertEquals(expectedValue, lastRecord.getMeasurement(MetricType.LATENCY)); @@ -276,7 +276,7 @@ public void testAddMeasurementSumAggregationCpu() { Number expectedSum = 0; for (SearchQueryRecord record : records) { - aggregatedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); + aggregatedRecord = minMaxHeapQueryGrouper.add(record); expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); } @@ -302,7 +302,7 @@ public void testAddMeasurementAverageAggregationCpu() { int expectedCount = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); - aggregatedRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); + aggregatedRecord = minMaxHeapQueryGrouper.add(record); expectedCount += 1; } @@ -328,7 +328,7 @@ public void testAddMeasurementNoneAggregationCpu() { Number expectedValue = 0; for (SearchQueryRecord record : records) { expectedValue = record.getMeasurement(MetricType.CPU).longValue(); - lastRecord = minMaxHeapQueryGrouper.addQueryToGroup(record); + lastRecord = minMaxHeapQueryGrouper.add(record); } assertEquals(expectedValue, lastRecord.getMeasurement(MetricType.CPU)); @@ -346,7 +346,7 @@ public void testNoneGroupingTypeIllegalArgumentException() { Number expectedSum = 0; for (SearchQueryRecord record : records) { expectedSum = expectedSum.longValue() + record.getMeasurement(MetricType.CPU).longValue(); - assertThrows(IllegalArgumentException.class, () -> { minMaxHeapQueryGrouper.addQueryToGroup(record); }); + assertThrows(IllegalArgumentException.class, () -> { minMaxHeapQueryGrouper.add(record); }); } } @@ -362,7 +362,7 @@ public void testNewGroupAddedToMin() { for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -383,7 +383,7 @@ public void testNewGroupOverflowsMinToMax() { for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -404,7 +404,7 @@ public void testNewGroupCausesOtherGroupOverflowMinToMax() { for (List recordList : allRecords) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -433,7 +433,7 @@ public void testExistingGroupUpdateToMinIncreaseAverage() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -462,7 +462,7 @@ public void testExistingGroupUpdateToMinDecreaseAverageStayInMin() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -491,7 +491,7 @@ public void testExistingGroupUpdateToMinDecreaseAverageOverflowsToMax() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -520,7 +520,7 @@ public void testExistingGroupUpdateToMaxIncreaseAverageStayInMax() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -549,7 +549,7 @@ public void testExistingGroupUpdateToMaxIncreaseAveragePromoteToMin() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -578,7 +578,7 @@ public void testExistingGroupUpdateToMaxDecreaseAverage() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -606,7 +606,7 @@ public void testSwitchGroupingTypeToNone() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -617,7 +617,7 @@ public void testSwitchGroupingTypeToNone() { minMaxHeapQueryGrouper.setGroupingType(GroupingType.NONE); assertEquals(0, minMaxHeapQueryGrouper.numberOfTopGroups()); - assertThrows(IllegalArgumentException.class, () -> { minMaxHeapQueryGrouper.addQueryToGroup(allRecords1.get(0).get(0)); }); + assertThrows(IllegalArgumentException.class, () -> { minMaxHeapQueryGrouper.add(allRecords1.get(0).get(0)); }); } public void testMultipleQueryGroupsUpdates() { @@ -639,7 +639,7 @@ public void testMultipleQueryGroupsUpdates() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } @@ -670,7 +670,7 @@ public void testMaxGroupLimitReached() { for (List recordList : allRecords1) { for (SearchQueryRecord record : recordList) { - minMaxHeapQueryGrouper.addQueryToGroup(record); + minMaxHeapQueryGrouper.add(record); } } From 3dfe973b578cdb57b31b05b220b42423e415a346 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 30 Aug 2024 16:48:30 -0700 Subject: [PATCH 21/29] Removed unused interface Signed-off-by: Siddhant Deshmukh --- .../core/service/grouper/MinMaxHeapQueryGrouper.java | 5 ----- .../insights/core/service/grouper/QueryGrouper.java | 8 -------- 2 files changed, 13 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java index a86cf03..298e762 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java @@ -156,11 +156,6 @@ public void drain() { minHeapTopQueriesStore.clear(); } - @Override - public TopQueriesStore getTopQueriesStore() { - return this.minHeapTopQueriesStore; - } - /** * Set Grouping Type * @param newGroupingType grouping type diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java index 4fb9557..d863cd8 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java @@ -8,7 +8,6 @@ package org.opensearch.plugin.insights.core.service.grouper; -import org.opensearch.plugin.insights.core.service.store.TopQueriesStore; import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.SearchQueryRecord; @@ -29,13 +28,6 @@ public interface QueryGrouper { */ void drain(); - /** - * Get the min heap queue that holds the top N queries. - * - * @return the TopQueriesStore containing the top N queries - */ - TopQueriesStore getTopQueriesStore(); - /** * Set the grouping type for this grouper. * From 980eaa8c4e7afcac147c7ce9e7b118d976a127ec Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Fri, 30 Aug 2024 17:02:51 -0700 Subject: [PATCH 22/29] Rebase main and spotless Signed-off-by: Siddhant Deshmukh --- .../plugin/insights/core/listener/QueryInsightsListener.java | 5 +---- .../plugin/insights/core/service/QueryInsightsService.java | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index a5111ee..532217c 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -105,10 +105,7 @@ public QueryInsightsListener( this.queryInsightsService.setWindowSize(type, clusterService.getClusterSettings().get(getTopNWindowSizeSetting(type))); } - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(SEARCH_QUERY_METRICS_ENABLED_SETTING, v -> setSearchQueryMetricsEnabled(v)); - setSearchQueryMetricsEnabled(clusterService.getClusterSettings().get(SEARCH_QUERY_METRICS_ENABLED_SETTING)); - + // Settings endpoints set for grouping top n queries clusterService.getClusterSettings() .addSettingsUpdateConsumer( TOP_N_QUERIES_GROUP_BY, diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java index bda5809..f2ad46e 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java @@ -8,8 +8,8 @@ package org.opensearch.plugin.insights.core.service; -import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getExporterSettings; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_GROUPING_TYPE; +import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getExporterSettings; import java.io.IOException; import java.util.ArrayList; From 32cec07d6a73f3b4ab2206021d244aec13e57dc6 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Tue, 3 Sep 2024 12:41:44 -0700 Subject: [PATCH 23/29] Renaming variable Signed-off-by: Siddhant Deshmukh --- .../insights/core/service/grouper/MinMaxHeapQueryGrouper.java | 2 +- .../plugin/insights/settings/QueryInsightsSettings.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java index 298e762..dbf5de0 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java @@ -93,7 +93,7 @@ public MinMaxHeapQueryGrouper( this.groupIdToAggSearchQueryRecord = new ConcurrentHashMap<>(); this.minHeapTopQueriesStore = topQueriesStore; this.topNSize = topNSize; - this.maxGroups = QueryInsightsSettings.DEFAULT_MAX_GROUPS; + this.maxGroups = QueryInsightsSettings.DEFAULT_GROUPS_LIMIT; this.maxHeapQueryStore = new PriorityBlockingQueue<>(maxGroups, (a, b) -> SearchQueryRecord.compare(b, a, metricType)); } diff --git a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java index 12c4924..9254ebe 100644 --- a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java +++ b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java @@ -71,7 +71,7 @@ public class QueryInsightsSettings { public static final String PLUGINS_BASE_URI = "/_insights"; public static final GroupingType DEFAULT_GROUPING_TYPE = GroupingType.NONE; - public static final int DEFAULT_MAX_GROUPS = 100; + public static final int DEFAULT_GROUPS_LIMIT = 100; public static final int MAX_GROUPS_LIMIT = 10000; @@ -133,7 +133,7 @@ public class QueryInsightsSettings { */ public static final Setting TOP_N_QUERIES_MAX_GROUPS = Setting.intSetting( TOP_N_QUERIES_SETTING_PREFIX + ".max_groups", - DEFAULT_MAX_GROUPS, + DEFAULT_GROUPS_LIMIT, Setting.Property.NodeScope, Setting.Property.Dynamic ); From a04da0c3572af783a496524e84428d204d64b549 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Tue, 3 Sep 2024 14:39:58 -0700 Subject: [PATCH 24/29] Remove TopQueriesStore interface Signed-off-by: Siddhant Deshmukh --- .../core/service/TopQueriesService.java | 11 +- .../grouper/MinMaxHeapQueryGrouper.java | 5 +- .../store/PriorityQueueTopQueriesStore.java | 106 ------------------ .../core/service/store/TopQueriesStore.java | 86 -------------- .../grouper/MinMaxHeapQueryGrouperTests.java | 4 +- 5 files changed, 9 insertions(+), 203 deletions(-) delete mode 100644 src/main/java/org/opensearch/plugin/insights/core/service/store/PriorityQueueTopQueriesStore.java delete mode 100644 src/main/java/org/opensearch/plugin/insights/core/service/store/TopQueriesStore.java diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index 1ce7b06..cd9992d 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.List; import java.util.Locale; +import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -36,8 +37,6 @@ import org.opensearch.plugin.insights.core.exporter.SinkType; import org.opensearch.plugin.insights.core.service.grouper.MinMaxHeapQueryGrouper; import org.opensearch.plugin.insights.core.service.grouper.QueryGrouper; -import org.opensearch.plugin.insights.core.service.store.PriorityQueueTopQueriesStore; -import org.opensearch.plugin.insights.core.service.store.TopQueriesStore; import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.GroupingType; import org.opensearch.plugin.insights.rules.model.MetricType; @@ -71,7 +70,7 @@ public class TopQueriesService { /** * The internal thread-safe store that holds the top n queries insight data */ - private final TopQueriesStore topQueriesStore; + private final PriorityBlockingQueue topQueriesStore; /** * The AtomicReference of a snapshot of the current window top queries for getters to consume @@ -113,7 +112,7 @@ public class TopQueriesService { this.windowSize = QueryInsightsSettings.DEFAULT_WINDOW_SIZE; this.windowStart = -1L; this.exporter = null; - topQueriesStore = new PriorityQueueTopQueriesStore<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); + topQueriesStore = new PriorityBlockingQueue<>(topNSize, (a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot = new AtomicReference<>(new ArrayList<>()); topQueriesHistorySnapshot = new AtomicReference<>(new ArrayList<>()); queryGrouper = new MinMaxHeapQueryGrouper( @@ -323,7 +322,7 @@ void consumeRecords(final List records) { // add records in current window, if there are any, to the top n store addToTopNStore(recordsInThisWindow); // update the current window snapshot for getters to consume - final List newSnapShot = new ArrayList<>(topQueriesStore.getSnapshot()); + final List newSnapShot = new ArrayList<>(topQueriesStore); newSnapShot.sort((a, b) -> SearchQueryRecord.compare(a, b, metricType)); topQueriesCurrentSnapshot.set(newSnapShot); } @@ -354,7 +353,7 @@ private void rotateWindowIfNecessary(final long newWindowStart) { final List history = new ArrayList<>(); // rotate the current window to history store only if the data belongs to the last window if (windowStart == newWindowStart - windowSize.getMillis()) { - history.addAll(topQueriesStore.getSnapshot()); + history.addAll(topQueriesStore); } topQueriesHistorySnapshot.set(history); topQueriesStore.clear(); diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java index dbf5de0..a074280 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java @@ -15,7 +15,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.collect.Tuple; -import org.opensearch.plugin.insights.core.service.store.TopQueriesStore; import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.Attribute; import org.opensearch.plugin.insights.rules.model.GroupingType; @@ -58,7 +57,7 @@ public class MinMaxHeapQueryGrouper implements QueryGrouper { /** * Min heap to keep track of the Top N query groups and is passed from TopQueriesService as the topQueriesStore */ - private TopQueriesStore minHeapTopQueriesStore; + private PriorityBlockingQueue minHeapTopQueriesStore; /** * The Max heap is an overflow data structure used to manage records that exceed the capacity of the Min heap. * It stores all records not included in the Top N query results. When the aggregate measurement for one of these @@ -84,7 +83,7 @@ public MinMaxHeapQueryGrouper( MetricType metricType, GroupingType groupingType, AggregationType aggregationType, - TopQueriesStore topQueriesStore, + PriorityBlockingQueue topQueriesStore, int topNSize ) { this.groupingType = groupingType; diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/store/PriorityQueueTopQueriesStore.java b/src/main/java/org/opensearch/plugin/insights/core/service/store/PriorityQueueTopQueriesStore.java deleted file mode 100644 index 258ced1..0000000 --- a/src/main/java/org/opensearch/plugin/insights/core/service/store/PriorityQueueTopQueriesStore.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.plugin.insights.core.service.store; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.PriorityBlockingQueue; - -/** - * Implementation of {@link TopQueriesStore} that uses a {@link PriorityBlockingQueue} - * to manage Top N search query records or groups. This implementation is thread-safe. - * - * @param The type of records that this store will manage. - */ -public class PriorityQueueTopQueriesStore implements TopQueriesStore { - - private final PriorityBlockingQueue topQueriesStore; - - /** - * Constructs a new {@code PriorityQueueTopQueriesStore} with the specified capacity. - * - * @param capacity the initial capacity of the store - * @param comparator comparator to use for the priority queue store - */ - public PriorityQueueTopQueriesStore(int capacity, Comparator comparator) { - this.topQueriesStore = new PriorityBlockingQueue<>(capacity, comparator); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean add(T queryRecord) { - return topQueriesStore.add(queryRecord); - } - - /** - * {@inheritDoc} - */ - @Override - public T poll() { - return topQueriesStore.poll(); - } - - /** - * {@inheritDoc} - */ - @Override - public T peek() { - return topQueriesStore.peek(); - } - - /** - * {@inheritDoc} - */ - @Override - public int size() { - return topQueriesStore.size(); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean isEmpty() { - return topQueriesStore.isEmpty(); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean addAll(List records) { - return topQueriesStore.addAll(records); - } - - /** - * {@inheritDoc} - */ - @Override - public void clear() { - topQueriesStore.clear(); - } - - /** - * {@inheritDoc} - */ - @Override - public List getSnapshot() { - return new ArrayList<>(topQueriesStore); - } - - /** - * {@inheritDoc} - */ - @Override - public boolean remove(T queryRecord) { - return topQueriesStore.remove(queryRecord); - } -} diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/store/TopQueriesStore.java b/src/main/java/org/opensearch/plugin/insights/core/service/store/TopQueriesStore.java deleted file mode 100644 index 75a2e1b..0000000 --- a/src/main/java/org/opensearch/plugin/insights/core/service/store/TopQueriesStore.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.plugin.insights.core.service.store; - -import java.util.List; - -/** - * Interface for managing a store of top search query records. - * Implementations of this interface should provide thread-safe operations - * to manage and access the top search queries. - * - * @param The type of records that this store will manage. - */ -public interface TopQueriesStore { - - /** - * Adds a search query record to the store. - * - * @param queryRecord the search query record to be added - * @return {@code true} if the record was successfully added, otherwise {@code false} - */ - boolean add(T queryRecord); - - /** - * Retrieves and removes the top search query record from the store, - * or returns {@code null} if the store is empty. - * - * @return the top search query record, or {@code null} if the store is empty - */ - T poll(); - - /** - * Retrieves, but does not remove, the top search query record from the store, - * or returns {@code null} if the store is empty. - * - * @return the top search query record, or {@code null} if the store is empty - */ - T peek(); - - /** - * Returns the number of search query records currently in the store. - * - * @return the size of the store - */ - int size(); - - /** - * Returns {@code true} if the store is empty, otherwise {@code false}. - * - * @return {@code true} if the store is empty, otherwise {@code false} - */ - boolean isEmpty(); - - /** - * Adds a list of query records to the store. - * - * @param records the list of query records to add - * @return {@code true} if all records were added successfully, otherwise {@code false} - */ - boolean addAll(List records); - - /** - * Clears all the records from the store. - */ - void clear(); - - /** - * Returns a snapshot of the current elements in the store as a list. - * - * @return a list containing all the elements currently in the store - */ - List getSnapshot(); - - /** - * Removes the specified record from the store, if it exists. - * - * @param queryRecord the record to remove - * @return {@code true} if the record was present and removed, otherwise {@code false} - */ - boolean remove(T queryRecord); -} diff --git a/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java b/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java index 89ebc8d..f92f49a 100644 --- a/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java +++ b/src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouperTests.java @@ -11,9 +11,9 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.PriorityBlockingQueue; import org.junit.Before; import org.opensearch.plugin.insights.QueryInsightsTestUtils; -import org.opensearch.plugin.insights.core.service.store.PriorityQueueTopQueriesStore; import org.opensearch.plugin.insights.rules.model.AggregationType; import org.opensearch.plugin.insights.rules.model.Attribute; import org.opensearch.plugin.insights.rules.model.GroupingType; @@ -26,7 +26,7 @@ */ public class MinMaxHeapQueryGrouperTests extends OpenSearchTestCase { private MinMaxHeapQueryGrouper minMaxHeapQueryGrouper; - private PriorityQueueTopQueriesStore topQueriesStore = new PriorityQueueTopQueriesStore<>( + private PriorityBlockingQueue topQueriesStore = new PriorityBlockingQueue<>( 100, (a, b) -> SearchQueryRecord.compare(a, b, MetricType.LATENCY) ); From 2db647d7e25a942a37f992c12be3f0a54cae145d Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Tue, 3 Sep 2024 15:09:44 -0700 Subject: [PATCH 25/29] Drain top queries service on group change Signed-off-by: Siddhant Deshmukh --- .../core/service/TopQueriesService.java | 19 +++++++++++++++++-- .../grouper/MinMaxHeapQueryGrouper.java | 11 +++++++++-- .../core/service/grouper/QueryGrouper.java | 6 ++++-- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java index cd9992d..9bd2dd0 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/TopQueriesService.java @@ -184,11 +184,17 @@ public void setWindowSize(final TimeValue windowSize) { } public void setGrouping(final GroupingType groupingType) { - queryGrouper.setGroupingType(groupingType); + boolean changed = queryGrouper.setGroupingType(groupingType); + if (changed) { + drain(); + } } public void setMaxGroups(final int maxGroups) { - queryGrouper.setMaxGroups(maxGroups); + boolean changed = queryGrouper.setMaxGroups(maxGroups); + if (changed) { + drain(); + } } /** @@ -399,4 +405,13 @@ public List getTopQueriesCurrentSnapshot() { public void close() throws IOException { queryInsightsExporterFactory.closeExporter(this.exporter); } + + /** + * Drain internal stores. + */ + private void drain() { + topQueriesStore.clear(); + topQueriesHistorySnapshot.set(new ArrayList<>()); + topQueriesCurrentSnapshot.set(new ArrayList<>()); + } } diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java index a074280..cf274e3 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java @@ -158,13 +158,17 @@ public void drain() { /** * Set Grouping Type * @param newGroupingType grouping type + * @return grouping type changed */ @Override - public void setGroupingType(GroupingType newGroupingType) { + public boolean setGroupingType(GroupingType newGroupingType) { if (this.groupingType != newGroupingType) { + log.info("deshsid setting grouping type in grouper to : " + newGroupingType); this.groupingType = newGroupingType; drain(); + return true; } + return false; } /** @@ -180,13 +184,16 @@ public GroupingType getGroupingType() { * Set the maximum number of groups that should be tracked when calculating Top N groups. * If the value changes, reset the state of the query grouper service by draining all internal data. * @param maxGroups max number of groups + * @return max groups changed */ @Override - public void setMaxGroups(int maxGroups) { + public boolean setMaxGroups(int maxGroups) { if (this.maxGroups != maxGroups) { this.maxGroups = maxGroups; drain(); + return true; } + return false; } /** diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java index d863cd8..07ba769 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/QueryGrouper.java @@ -32,8 +32,9 @@ public interface QueryGrouper { * Set the grouping type for this grouper. * * @param groupingType the grouping type to set + * @return grouping type changed */ - void setGroupingType(GroupingType groupingType); + boolean setGroupingType(GroupingType groupingType); /** * Get the current grouping type for this grouper. @@ -46,8 +47,9 @@ public interface QueryGrouper { * Set the maximum number of groups allowed. * * @param maxGroups the maximum number of groups + * @return max groups changed */ - void setMaxGroups(int maxGroups); + boolean setMaxGroups(int maxGroups); /** * Update the top N size for the grouper. From 07ce827bf099c8de0949bb8e4268654aadf1340b Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Tue, 3 Sep 2024 17:16:53 -0700 Subject: [PATCH 26/29] Rename max groups setting and allow minimum 0 Signed-off-by: Siddhant Deshmukh --- .../plugin/insights/QueryInsightsPlugin.java | 2 +- .../core/listener/QueryInsightsListener.java | 8 ++++---- .../insights/core/service/QueryInsightsService.java | 4 ++-- .../core/service/grouper/MinMaxHeapQueryGrouper.java | 7 +++---- .../insights/settings/QueryInsightsSettings.java | 12 ++++++------ .../plugin/insights/QueryInsightsPluginTests.java | 2 +- .../plugin/insights/QueryInsightsTestUtils.java | 2 +- 7 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java b/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java index 4841108..58e835f 100644 --- a/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java +++ b/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java @@ -131,7 +131,7 @@ public List> getSettings() { QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE, QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS, QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY, - QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS, + QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N, QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING ); } diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index 532217c..ceb8d32 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -10,7 +10,7 @@ import static org.opensearch.plugin.insights.settings.QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY; -import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS; +import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNEnabledSetting; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNSizeSetting; import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.getTopNWindowSizeSetting; @@ -117,12 +117,12 @@ public QueryInsightsListener( clusterService.getClusterSettings() .addSettingsUpdateConsumer( - TOP_N_QUERIES_MAX_GROUPS, + TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N, v -> this.queryInsightsService.setMaximumGroups(v), v -> this.queryInsightsService.validateMaximumGroups(v) ); - this.queryInsightsService.validateMaximumGroups(clusterService.getClusterSettings().get(TOP_N_QUERIES_MAX_GROUPS)); - this.queryInsightsService.setMaximumGroups(clusterService.getClusterSettings().get(TOP_N_QUERIES_MAX_GROUPS)); + this.queryInsightsService.validateMaximumGroups(clusterService.getClusterSettings().get(TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N)); + this.queryInsightsService.setMaximumGroups(clusterService.getClusterSettings().get(TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N)); // Settings endpoints set for search query metrics clusterService.getClusterSettings() diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java index f2ad46e..35a83c0 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java @@ -233,9 +233,9 @@ public void setMaximumGroups(final int maxGroups) { * @param maxGroups maximum number of groups that should be tracked when calculating Top N groups */ public void validateMaximumGroups(final int maxGroups) { - if (maxGroups < 1 || maxGroups > QueryInsightsSettings.MAX_GROUPS_LIMIT) { + if (maxGroups < 0 || maxGroups > QueryInsightsSettings.MAX_GROUPS_EXCLUDING_TOPN_LIMIT) { throw new IllegalArgumentException( - "Max groups setting" + " should be between 1 and " + QueryInsightsSettings.MAX_GROUPS_LIMIT + ", was (" + maxGroups + ")" + "Max groups setting" + " should be between 0 and " + QueryInsightsSettings.MAX_GROUPS_EXCLUDING_TOPN_LIMIT + ", was (" + maxGroups + ")" ); } } diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java index cf274e3..3655944 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java @@ -8,7 +8,7 @@ package org.opensearch.plugin.insights.core.service.grouper; -import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS; +import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.PriorityBlockingQueue; @@ -92,7 +92,7 @@ public MinMaxHeapQueryGrouper( this.groupIdToAggSearchQueryRecord = new ConcurrentHashMap<>(); this.minHeapTopQueriesStore = topQueriesStore; this.topNSize = topNSize; - this.maxGroups = QueryInsightsSettings.DEFAULT_GROUPS_LIMIT; + this.maxGroups = QueryInsightsSettings.DEFAULT_GROUPS_EXCLUDING_TOPN_LIMIT; this.maxHeapQueryStore = new PriorityBlockingQueue<>(maxGroups, (a, b) -> SearchQueryRecord.compare(b, a, metricType)); } @@ -163,7 +163,6 @@ public void drain() { @Override public boolean setGroupingType(GroupingType newGroupingType) { if (this.groupingType != newGroupingType) { - log.info("deshsid setting grouping type in grouper to : " + newGroupingType); this.groupingType = newGroupingType; drain(); return true; @@ -255,7 +254,7 @@ private boolean checkMaxGroupsLimitReached(String groupId) { if (maxGroups <= maxHeapQueryStore.size()) { log.warn( "Exceeded [{}] setting threshold which is set at {}. Discarding new group with id {}.", - TOP_N_QUERIES_MAX_GROUPS.getKey(), + TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N.getKey(), maxGroups, groupId ); diff --git a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java index 9254ebe..89290bc 100644 --- a/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java +++ b/src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java @@ -71,9 +71,9 @@ public class QueryInsightsSettings { public static final String PLUGINS_BASE_URI = "/_insights"; public static final GroupingType DEFAULT_GROUPING_TYPE = GroupingType.NONE; - public static final int DEFAULT_GROUPS_LIMIT = 100; + public static final int DEFAULT_GROUPS_EXCLUDING_TOPN_LIMIT = 100; - public static final int MAX_GROUPS_LIMIT = 10000; + public static final int MAX_GROUPS_EXCLUDING_TOPN_LIMIT = 10000; /** * Settings for Top Queries @@ -129,11 +129,11 @@ public class QueryInsightsSettings { ); /** - * Define the group_by option for Top N queries to group queries. + * Define the max_groups_excluding_topn option for Top N queries to group queries. */ - public static final Setting TOP_N_QUERIES_MAX_GROUPS = Setting.intSetting( - TOP_N_QUERIES_SETTING_PREFIX + ".max_groups", - DEFAULT_GROUPS_LIMIT, + public static final Setting TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N = Setting.intSetting( + TOP_N_QUERIES_SETTING_PREFIX + ".max_groups_excluding_topn", + DEFAULT_GROUPS_EXCLUDING_TOPN_LIMIT, Setting.Property.NodeScope, Setting.Property.Dynamic ); diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java index 5799049..fca21e5 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java @@ -76,7 +76,7 @@ public void testGetSettings() { QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE, QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS, QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY, - QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS, + QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N, QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING ), queryInsightsPlugin.getSettings() diff --git a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java index 1a37940..6fb9dff 100644 --- a/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java +++ b/src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java @@ -323,7 +323,7 @@ public static void registerAllQueryInsightsSettings(ClusterSettings clusterSetti clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE); clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS); clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY); - clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS); + clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N); clusterSettings.registerSetting(QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING); } } From 357d2cd3b698dcfa0558f1bd95b6ea8ffcabc299 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Tue, 3 Sep 2024 17:39:46 -0700 Subject: [PATCH 27/29] Make write/read from io backword compatible Signed-off-by: Siddhant Deshmukh --- .../core/listener/QueryInsightsListener.java | 2 +- .../core/service/QueryInsightsService.java | 7 +++- .../insights/rules/model/MetricType.java | 17 +++++++++ .../rules/model/SearchQueryRecord.java | 35 ++++++++++++++----- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java index ceb8d32..0cd8412 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java +++ b/src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java @@ -117,7 +117,7 @@ public QueryInsightsListener( clusterService.getClusterSettings() .addSettingsUpdateConsumer( - TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N, + TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N, v -> this.queryInsightsService.setMaximumGroups(v), v -> this.queryInsightsService.validateMaximumGroups(v) ); diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java index 35a83c0..2b41856 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java @@ -235,7 +235,12 @@ public void setMaximumGroups(final int maxGroups) { public void validateMaximumGroups(final int maxGroups) { if (maxGroups < 0 || maxGroups > QueryInsightsSettings.MAX_GROUPS_EXCLUDING_TOPN_LIMIT) { throw new IllegalArgumentException( - "Max groups setting" + " should be between 0 and " + QueryInsightsSettings.MAX_GROUPS_EXCLUDING_TOPN_LIMIT + ", was (" + maxGroups + ")" + "Max groups setting" + + " should be between 0 and " + + QueryInsightsSettings.MAX_GROUPS_EXCLUDING_TOPN_LIMIT + + ", was (" + + maxGroups + + ")" ); } } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java b/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java index d6717e5..52b8331 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/MetricType.java @@ -96,4 +96,21 @@ public int compare(final Number a, final Number b) { } return -1; } + + /** + * Parse a value with the correct type based on MetricType + * + * @param o the generic object to parse + * @return {@link Number} + */ + Number parseValue(final Object o) { + switch (this) { + case LATENCY: + case CPU: + case MEMORY: + return (Long) o; + default: + return (Number) o; + } + } } diff --git a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java index ede43fc..42ee029 100644 --- a/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java +++ b/src/main/java/org/opensearch/plugin/insights/rules/model/SearchQueryRecord.java @@ -9,9 +9,11 @@ package org.opensearch.plugin.insights.rules.model; import java.io.IOException; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; +import org.opensearch.Version; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -41,11 +43,22 @@ public class SearchQueryRecord implements ToXContentObject, Writeable { */ public SearchQueryRecord(final StreamInput in) throws IOException, ClassCastException { this.timestamp = in.readLong(); - measurements = new LinkedHashMap<>(); - in.readOrderedMap(MetricType::readFromStream, Measurement::readFromStream) - .forEach(((metricType, measurement) -> measurements.put(metricType, measurement))); + if (in.getVersion().onOrAfter(Version.V_2_17_0)) { + measurements = new LinkedHashMap<>(); + in.readOrderedMap(MetricType::readFromStream, Measurement::readFromStream) + .forEach(((metricType, measurement) -> measurements.put(metricType, measurement))); + this.groupingId = null; + } else { + measurements = new HashMap<>(); + in.readMap(MetricType::readFromStream, StreamInput::readGenericValue).forEach((metricType, o) -> { + try { + measurements.put(metricType, new Measurement(metricType.parseValue(o))); + } catch (ClassCastException e) { + throw new ClassCastException("Error parsing value for metric type: " + metricType); + } + }); + } this.attributes = Attribute.readAttributeMap(in); - this.groupingId = null; } /** @@ -155,11 +168,15 @@ public XContentBuilder toXContent(final XContentBuilder builder, final ToXConten @Override public void writeTo(final StreamOutput out) throws IOException { out.writeLong(timestamp); - out.writeMap( - measurements, - (stream, metricType) -> MetricType.writeTo(out, metricType), - (stream, measurement) -> measurement.writeTo(out) - ); + if (out.getVersion().onOrAfter(Version.V_2_17_0)) { + out.writeMap( + measurements, + (stream, metricType) -> MetricType.writeTo(out, metricType), + (stream, measurement) -> measurement.writeTo(out) + ); + } else { + out.writeMap(measurements, (stream, metricType) -> MetricType.writeTo(out, metricType), StreamOutput::writeGenericValue); + } out.writeMap( attributes, (stream, attribute) -> Attribute.writeTo(out, attribute), From d89190999c9dbcded1f161356e2695b17f9f92ce Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Tue, 3 Sep 2024 18:03:19 -0700 Subject: [PATCH 28/29] Minor fix Signed-off-by: Siddhant Deshmukh --- .../insights/core/service/grouper/MinMaxHeapQueryGrouper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java index 3655944..be2f60f 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java @@ -251,7 +251,7 @@ private void addToMaxPQPromoteToMinPQ(SearchQueryRecord aggregateSearchQueryReco } private boolean checkMaxGroupsLimitReached(String groupId) { - if (maxGroups <= maxHeapQueryStore.size()) { + if (maxGroups <= maxHeapQueryStore.size() && minHeapTopQueriesStore.size() >= topNSize) { log.warn( "Exceeded [{}] setting threshold which is set at {}. Discarding new group with id {}.", TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N.getKey(), From 3493e654f48dc4e1751417fdde14f1a2c0a16e23 Mon Sep 17 00:00:00 2001 From: Siddhant Deshmukh Date: Wed, 4 Sep 2024 10:55:21 -0700 Subject: [PATCH 29/29] Refactor query grouper Signed-off-by: Siddhant Deshmukh --- .../grouper/MinMaxHeapQueryGrouper.java | 62 ++++++------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java index be2f60f..9b87f2d 100644 --- a/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java +++ b/src/main/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxHeapQueryGrouper.java @@ -115,14 +115,10 @@ public SearchQueryRecord add(SearchQueryRecord searchQueryRecord) { // 2) Existing group being updated to the grouping service // a. If present in min PQ // - remove the record from the min PQ - // - update the aggregate record (aggregate measurement could increase or decrease) - // - If max PQ contains elements, add to max PQ and promote any records to min PQ - // - If max PQ is empty, add to min PQ and overflow any records to max PQ // b. If present in max PQ // - remove the record from the max PQ - // - update the aggregate record (aggregate measurement could increase or decrease) - // - If min PQ is full, add to min PQ and overflow any records to max PQ - // - else, add to max PQ and promote any records to min PQ + // Add to min PQ and promote to max + // If max PQ is empty return else try to promote record from max to min if (!groupIdToAggSearchQueryRecord.containsKey(groupId)) { boolean maxGroupsLimitReached = checkMaxGroupsLimitReached(groupId); if (maxGroupsLimitReached) { @@ -131,15 +127,16 @@ public SearchQueryRecord add(SearchQueryRecord searchQueryRecord) { aggregateSearchQueryRecord = searchQueryRecord; aggregateSearchQueryRecord.setGroupingId(groupId); aggregateSearchQueryRecord.setMeasurementAggregation(metricType, aggregationType); - addToMinPQOverflowToMaxPQ(aggregateSearchQueryRecord, groupId); + addToMinPQ(aggregateSearchQueryRecord, groupId); } else { aggregateSearchQueryRecord = groupIdToAggSearchQueryRecord.get(groupId).v1(); boolean isPresentInMinPQ = groupIdToAggSearchQueryRecord.get(groupId).v2(); if (isPresentInMinPQ) { - updateToMinPQ(searchQueryRecord, aggregateSearchQueryRecord, groupId); + minHeapTopQueriesStore.remove(aggregateSearchQueryRecord); } else { - updateToMaxPQ(searchQueryRecord, aggregateSearchQueryRecord, groupId); + maxHeapQueryStore.remove(aggregateSearchQueryRecord); } + addAndPromote(searchQueryRecord, aggregateSearchQueryRecord, groupId); } return aggregateSearchQueryRecord; } @@ -204,49 +201,30 @@ public void updateTopNSize(int newSize) { this.topNSize = newSize; } - private void addToMinPQOverflowToMaxPQ(SearchQueryRecord searchQueryRecord, String groupId) { + private void addToMinPQ(SearchQueryRecord searchQueryRecord, String groupId) { minHeapTopQueriesStore.add(searchQueryRecord); groupIdToAggSearchQueryRecord.put(groupId, new Tuple<>(searchQueryRecord, true)); - - while (minHeapTopQueriesStore.size() > topNSize) { - SearchQueryRecord recordMovedFromMinToMax = minHeapTopQueriesStore.poll(); - maxHeapQueryStore.add(recordMovedFromMinToMax); - groupIdToAggSearchQueryRecord.put(recordMovedFromMinToMax.getGroupingId(), new Tuple<>(recordMovedFromMinToMax, false)); - } + overflow(); } - private void updateToMaxPQ(SearchQueryRecord searchQueryRecord, SearchQueryRecord aggregateSearchQueryRecord, String groupId) { - maxHeapQueryStore.remove(aggregateSearchQueryRecord); + private void addAndPromote(SearchQueryRecord searchQueryRecord, SearchQueryRecord aggregateSearchQueryRecord, String groupId) { Number measurementToAdd = searchQueryRecord.getMeasurement(metricType); aggregateSearchQueryRecord.addMeasurement(metricType, measurementToAdd); - - if (minHeapTopQueriesStore.size() >= topNSize) { - addToMinPQOverflowToMaxPQ(aggregateSearchQueryRecord, groupId); - } else { - addToMaxPQPromoteToMinPQ(aggregateSearchQueryRecord, groupId); + addToMinPQ(aggregateSearchQueryRecord, groupId); + if (maxHeapQueryStore.isEmpty()) { + return; } - } - - private void updateToMinPQ(SearchQueryRecord searchQueryRecord, SearchQueryRecord aggregateSearchQueryRecord, String groupId) { - minHeapTopQueriesStore.remove(aggregateSearchQueryRecord); - Number measurementToAdd = searchQueryRecord.getMeasurement(metricType); - aggregateSearchQueryRecord.addMeasurement(metricType, measurementToAdd); - - if (maxHeapQueryStore.size() > 0) { - addToMaxPQPromoteToMinPQ(aggregateSearchQueryRecord, groupId); - } else { - addToMinPQOverflowToMaxPQ(aggregateSearchQueryRecord, groupId); + if (SearchQueryRecord.compare(maxHeapQueryStore.peek(), minHeapTopQueriesStore.peek(), metricType) > 0) { + SearchQueryRecord recordMovedFromMaxToMin = maxHeapQueryStore.poll(); + addToMinPQ(recordMovedFromMaxToMin, recordMovedFromMaxToMin.getGroupingId()); } } - private void addToMaxPQPromoteToMinPQ(SearchQueryRecord aggregateSearchQueryRecord, String groupId) { - maxHeapQueryStore.add(aggregateSearchQueryRecord); - groupIdToAggSearchQueryRecord.put(groupId, new Tuple<>(aggregateSearchQueryRecord, false)); - - while (minHeapTopQueriesStore.size() < topNSize && !maxHeapQueryStore.isEmpty()) { - SearchQueryRecord recordMovedFromMaxToMin = maxHeapQueryStore.poll(); - minHeapTopQueriesStore.add(recordMovedFromMaxToMin); - groupIdToAggSearchQueryRecord.put(recordMovedFromMaxToMin.getGroupingId(), new Tuple<>(recordMovedFromMaxToMin, true)); + private void overflow() { + if (minHeapTopQueriesStore.size() > topNSize) { + SearchQueryRecord recordMovedFromMinToMax = minHeapTopQueriesStore.poll(); + maxHeapQueryStore.add(recordMovedFromMinToMax); + groupIdToAggSearchQueryRecord.put(recordMovedFromMinToMax.getGroupingId(), new Tuple<>(recordMovedFromMinToMax, false)); } }