Skip to content

Commit

Permalink
Enable inter-segment concurrency for low cardinality numeric terms aggs
Browse files Browse the repository at this point in the history
  • Loading branch information
@iverase
iverase committed May 6, 2024
1 parent a049496 commit 257e58d
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 1 deletion.
28 changes: 27 additions & 1 deletion server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,17 @@
package org.elasticsearch.search;

import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.OrdinalMap;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.action.search.SearchShardTask;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.cluster.routing.IndexRouting;
Expand All @@ -32,6 +35,7 @@
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.fielddata.FieldDataContext;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
import org.elasticsearch.index.mapper.IdLoader;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
Expand Down Expand Up @@ -245,12 +249,34 @@ static long getFieldCardinality(IndexFieldData<?> indexFieldData, DirectoryReade
if (ordinalMap != null) {
return ordinalMap.getValueCount();
}
if (directoryReader.leaves().size() == 0) {
if (directoryReader.leaves().isEmpty()) {
return 0;
}
return global.load(directoryReader.leaves().get(0)).getOrdinalsValues().getValueCount();
}
} else if (indexFieldData instanceof IndexNumericFieldData indexNumericFieldData) {
final IndexNumericFieldData.NumericType type = indexNumericFieldData.getNumericType();
try {
if (type == IndexNumericFieldData.NumericType.INT || type == IndexNumericFieldData.NumericType.SHORT) {
final IndexReader reader = directoryReader.getContext().reader();
final byte[] min = PointValues.getMinPackedValue(reader, indexFieldData.getFieldName());
final byte[] max = PointValues.getMaxPackedValue(reader, indexFieldData.getFieldName());
if (min != null && max != null) {
return NumericUtils.sortableBytesToInt(max, 0) - NumericUtils.sortableBytesToInt(min, 0) + 1;
}
} else if (type == IndexNumericFieldData.NumericType.LONG) {
final IndexReader reader = directoryReader.getContext().reader();
final byte[] min = PointValues.getMinPackedValue(reader, indexFieldData.getFieldName());
final byte[] max = PointValues.getMaxPackedValue(reader, indexFieldData.getFieldName());
if (min != null && max != null) {
return NumericUtils.sortableBytesToLong(max, 0) - NumericUtils.sortableBytesToLong(min, 0) + 1;
}
}
} catch (IOException ioe) {
return -1L;
}
}
//
return -1L;
}

Expand Down
58 changes: 58 additions & 0 deletions server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@
package org.elasticsearch.search;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
Expand Down Expand Up @@ -38,7 +42,9 @@
import org.elasticsearch.index.cache.query.QueryCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.plain.BinaryIndexFieldData;
import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
import org.elasticsearch.index.fielddata.plain.SortedOrdinalsIndexFieldData;
import org.elasticsearch.index.mapper.IdLoader;
import org.elasticsearch.index.mapper.MappedFieldType;
Expand Down Expand Up @@ -799,6 +805,58 @@ public void testGetFieldCardinality() throws IOException {
}
}

public void testGetFieldCardinalityNumeric() throws IOException {
try (BaseDirectoryWrapper dir = newDirectory()) {
final int numDocs = scaledRandomIntBetween(100, 200);
try (RandomIndexWriter w = new RandomIndexWriter(random(), dir, new IndexWriterConfig())) {
for (int i = 0; i < numDocs; ++i) {
Document doc = new Document();
doc.add(new LongField("long", i, Field.Store.NO));
doc.add(new IntField("int", i, Field.Store.NO));
doc.add(new SortedNumericDocValuesField("no_index", i));
w.addDocument(doc);
}
}
try (DirectoryReader reader = DirectoryReader.open(dir)) {
final SortedNumericIndexFieldData longFieldData = new SortedNumericIndexFieldData(
"long",
IndexNumericFieldData.NumericType.LONG,
IndexNumericFieldData.NumericType.LONG.getValuesSourceType(),
null,
true
);
assertEquals(numDocs, DefaultSearchContext.getFieldCardinality(longFieldData, reader));

final SortedNumericIndexFieldData integerFieldData = new SortedNumericIndexFieldData(
"int",
IndexNumericFieldData.NumericType.INT,
IndexNumericFieldData.NumericType.INT.getValuesSourceType(),
null,
true
);
assertEquals(numDocs, DefaultSearchContext.getFieldCardinality(integerFieldData, reader));

final SortedNumericIndexFieldData shortFieldData = new SortedNumericIndexFieldData(
"int",
IndexNumericFieldData.NumericType.SHORT,
IndexNumericFieldData.NumericType.SHORT.getValuesSourceType(),
null,
true
);
assertEquals(numDocs, DefaultSearchContext.getFieldCardinality(shortFieldData, reader));

final SortedNumericIndexFieldData noIndexFieldata = new SortedNumericIndexFieldData(
"no_index",
IndexNumericFieldData.NumericType.LONG,
IndexNumericFieldData.NumericType.LONG.getValuesSourceType(),
null,
false
);
assertEquals(-1, DefaultSearchContext.getFieldCardinality(noIndexFieldata, reader));
}
}
}

public void testGetFieldCardinalityUnmappedField() {
MapperService mapperService = mock(MapperService.class);
IndexService indexService = mock(IndexService.class);
Expand Down
13 changes: 13 additions & 0 deletions test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.OrdinalMap;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
Expand Down Expand Up @@ -1394,6 +1395,18 @@ long getCardinality(IndexReader reader, String field) {
subs[i] = sortedDocValues.termsEnum();
weights[i] = sortedDocValues.getValueCount();
}
case NUMERIC, SORTED_NUMERIC -> {
final byte[] min = PointValues.getMinPackedValue(reader, field);
final byte[] max = PointValues.getMaxPackedValue(reader, field);
if (min != null && max != null) {
if (min.length == 4) {
return NumericUtils.sortableBytesToInt(max, 0) - NumericUtils.sortableBytesToInt(min, 0);
} else if (min.length == 8) {
return NumericUtils.sortableBytesToLong(max, 0) - NumericUtils.sortableBytesToLong(min, 0);
}
}
return -1;
}
default -> {
return -1;
}
Expand Down

0 comments on commit 257e58d

Please sign in to comment.