diff --git a/CHANGELOG.md b/CHANGELOG.md index 67091eaa6b850..b2daaf93af8a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,7 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Optimize regexp-based include/exclude on aggregations when pattern matches prefixes ([#14371](https://github.com/opensearch-project/OpenSearch/pull/14371)) - Replace and block usages of org.apache.logging.log4j.util.Strings ([#15238](https://github.com/opensearch-project/OpenSearch/pull/15238)) - Remote publication using minimum node version for backward compatibility ([#15216](https://github.com/opensearch-project/OpenSearch/pull/15216)) - +- Updated `termsQuery` in `KeywordField` to set custom rewrite_override before executing query ### Deprecated diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml index a133060f07c6f..6d3fade1e05d8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml @@ -71,6 +71,56 @@ - '{ "index": { "_index": "test-iodvq", "_id": "3" } }' - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.987654321Z", "date": "2024-10-29T12:12:12.987Z" }' + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + terms: + some_keyword: ["400", "5"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + fuzzy: + some_keyword: { + value: "402" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + regexp: + some_keyword: { + value: "40*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + wildcard: + some_keyword: { + value: "ing*" + } + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -95,7 +145,6 @@ - match: { hits.total: 2 } - - do: search: rest_total_hits_as_int: true @@ -608,6 +657,56 @@ - '{ "index": { "_index": "test-index", "_id": "3" } }' - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }' + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + terms: + some_keyword: ["400", "5"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + fuzzy: + some_keyword: { + value: "402" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + regexp: + some_keyword: { + value: "40*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + wildcard: + some_keyword: { + value: "ing*" + } + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -1150,6 +1249,56 @@ - '{ "index": { "_index": "test-doc-values", "_id": "3" } }' - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }' + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + terms: + some_keyword: ["400", "5"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + fuzzy: + some_keyword: { + value: "402" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + regexp: + some_keyword: { + value: "40*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + wildcard: + some_keyword: { + value: "ing*" + } + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -1602,3 +1751,282 @@ } - match: { hits.total: 0 } + +--- +"search with rewrite_override": + - skip: + features: [ "headers" ] + version: " - 2.99.99" + reason: "rewrite_override parameter was added in 2.17.0" + - do: + indices.create: + index: test-rewrite + body: + mappings: + properties: + some_keyword: + type: keyword + index: true + doc_values: true + some_keyword_index: + type: keyword + index: true + doc_values: false + some_keyword_dv: + type: keyword + index: false + doc_values: true + + - do: + bulk: + index: test-rewrite + refresh: true + body: + - '{"index": {"_index": "test-rewrite", "_id": "1" }}' + - '{ "some_keyword": "ingesting some random keyword data", "some_keyword_index": "ingesting some random keyword data", "some_keyword_dv": "ingesting some random keyword data" }' + - '{ "index": { "_index": "test-rewrite", "_id": "2" }}' + - '{ "some_keyword": "400", "some_keyword_index": "400", "some_keyword_dv": "400" }' + - '{ "index": { "_index": "test-rewrite", "_id": "3" } }' + - '{ "some_keyword": "5", "some_keyword_index": "5", "some_keyword_dv": "5" }' + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + terms: + some_keyword: ["400", "5"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + terms: { + some_keyword_index: ["400", "5"], + rewrite_override: "index_only" + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + terms: { + some_keyword_dv: ["400", "5"], + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + fuzzy: + some_keyword: { + value: "402" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + fuzzy: + some_keyword_index: { + value: "402", + rewrite_override: "index_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + fuzzy: + some_keyword_dv: { + value: "402", + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + regexp: + some_keyword: { + value: "40*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + regexp: + some_keyword_index: { + value: "40*", + rewrite_override: "index_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + regexp: + some_keyword_dv: { + value: "40*", + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + wildcard: + some_keyword: { + value: "ing*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + wildcard: + some_keyword_index: { + value: "ing*", + rewrite_override: "index_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + wildcard: + some_keyword_dv: { + value: "ing*", + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + prefix: + some_keyword: "ing" + + - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + prefix: + some_keyword_index: { + value: "ing", + rewrite_override: "index_only" + } + + - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + prefix: + some_keyword_dv: { + value: "ing", + rewrite_override: "doc_values_only" + } + + - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + range: { + "some_keyword": { + "lt": 500 + } } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + range: { + "some_keyword_index": { + "lt": 500, + rewrite_override: "index_only" + + } } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-rewrite + body: + query: + range: { + "some_keyword_dv": { + "lt": 500, + rewrite_override: "doc_values_only" + + } } + + - match: { hits.total: 2 } diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java index 2116ac522b705..5a28706f46c68 100644 --- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java @@ -61,6 +61,7 @@ import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.similarity.SimilarityProvider; import org.opensearch.search.aggregations.support.CoreValuesSourceType; @@ -388,34 +389,57 @@ protected BytesRef indexedValueForSearch(Object value) { } @Override - public Query termsQuery(List values, QueryShardContext context) { + public Query termsQuery(List values, QueryShardContext.RewriteOverride rewriteOverride, QueryShardContext context) { failIfNotIndexedAndNoDocValues(); - // has index and doc_values enabled - if (isSearchable() && hasDocValues()) { - BytesRef[] bytesRefs = new BytesRef[values.size()]; - for (int i = 0; i < bytesRefs.length; i++) { - bytesRefs[i] = indexedValueForSearch(values.get(i)); - } - Query indexQuery = new TermInSetQuery(name(), bytesRefs); - Query dvQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); - return new IndexOrDocValuesQuery(indexQuery, dvQuery); + if (rewriteOverride == null) { + rewriteOverride = QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES; } - // if we only have doc_values enabled, we construct a new query with doc_values re-written - if (hasDocValues()) { - BytesRef[] bytesRefs = new BytesRef[values.size()]; - for (int i = 0; i < bytesRefs.length; i++) { - bytesRefs[i] = indexedValueForSearch(values.get(i)); - } - return new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); + Query query = null; + switch (rewriteOverride) { + case INDEX_OR_DOC_VALUES: + // has index and doc_values enabled + if (isSearchable() && hasDocValues()) { + BytesRef[] bytesRefs = new BytesRef[values.size()]; + for (int i = 0; i < bytesRefs.length; i++) { + bytesRefs[i] = indexedValueForSearch(values.get(i)); + } + Query indexQuery = new TermInSetQuery(name(), bytesRefs); + Query dvQuery = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); + query = new IndexOrDocValuesQuery(indexQuery, dvQuery); + } + // if we only have doc_values enabled, we construct a new query with doc_values re-written + else if (hasDocValues()) { + BytesRef[] bytesRefs = new BytesRef[values.size()]; + for (int i = 0; i < bytesRefs.length; i++) { + bytesRefs[i] = indexedValueForSearch(values.get(i)); + } + query = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); + } else { + query = super.termsQuery(values, context); + } + break; + case INDEX_ONLY: + failIfNotIndexed(); + query = super.termsQuery(values, context); + break; + case DOC_VALUES_ONLY: + failIfNoDocValues(); + BytesRef[] bytesRefs = new BytesRef[values.size()]; + for (int i = 0; i < bytesRefs.length; i++) { + bytesRefs[i] = indexedValueForSearch(values.get(i)); + } + query = new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, name(), bytesRefs); + break; } - // has index enabled, we're going to return the query as is - return super.termsQuery(values, context); + + return query; } @Override public Query prefixQuery( String value, @Nullable MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, boolean caseInsensitive, QueryShardContext context ) { @@ -428,21 +452,46 @@ public Query prefixQuery( ); } failIfNotIndexedAndNoDocValues(); - if (isSearchable() && hasDocValues()) { - Query indexQuery = super.prefixQuery(value, method, caseInsensitive, context); - Query dvQuery = super.prefixQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, context); - return new IndexOrDocValuesQuery(indexQuery, dvQuery); + if (rewriteOverride == null) { + rewriteOverride = QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES; } - if (hasDocValues()) { - if (caseInsensitive) { - return AutomatonQueries.caseInsensitivePrefixQuery( - (new Term(name(), indexedValueForSearch(value))), - MultiTermQuery.DOC_VALUES_REWRITE - ); - } - return new PrefixQuery(new Term(name(), indexedValueForSearch(value)), MultiTermQuery.DOC_VALUES_REWRITE); + Query query = null; + switch (rewriteOverride) { + case INDEX_OR_DOC_VALUES: + if (isSearchable() && hasDocValues()) { + Query indexQuery = super.prefixQuery(value, method, caseInsensitive, context); + Query dvQuery = super.prefixQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, context); + query = new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else if (hasDocValues()) { + if (caseInsensitive) { + return AutomatonQueries.caseInsensitivePrefixQuery( + (new Term(name(), indexedValueForSearch(value))), + MultiTermQuery.DOC_VALUES_REWRITE + ); + } + query = new PrefixQuery(new Term(name(), indexedValueForSearch(value)), MultiTermQuery.DOC_VALUES_REWRITE); + } else { + query = super.prefixQuery(value, method, caseInsensitive, context); + } + break; + case INDEX_ONLY: + failIfNotIndexed(); + query = super.prefixQuery(value, method, caseInsensitive, context); + break; + case DOC_VALUES_ONLY: + failIfNoDocValues(); + if (caseInsensitive) { + query = AutomatonQueries.caseInsensitivePrefixQuery( + (new Term(name(), indexedValueForSearch(value))), + MultiTermQuery.DOC_VALUES_REWRITE + ); + } else { + query = new PrefixQuery(new Term(name(), indexedValueForSearch(value)), MultiTermQuery.DOC_VALUES_REWRITE); + } + break; } - return super.prefixQuery(value, method, caseInsensitive, context); + + return query; } @Override @@ -452,6 +501,7 @@ public Query regexpQuery( int matchFlags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, QueryShardContext context ) { if (context.allowExpensiveQueries() == false) { @@ -460,33 +510,64 @@ public Query regexpQuery( ); } failIfNotIndexedAndNoDocValues(); - if (isSearchable() && hasDocValues()) { - Query indexQuery = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); - Query dvQuery = super.regexpQuery( - value, - syntaxFlags, - matchFlags, - maxDeterminizedStates, - MultiTermQuery.DOC_VALUES_REWRITE, - context - ); - return new IndexOrDocValuesQuery(indexQuery, dvQuery); + if (rewriteOverride == null) { + rewriteOverride = QueryRewriteContext.RewriteOverride.INDEX_OR_DOC_VALUES; } - if (hasDocValues()) { - return new RegexpQuery( - new Term(name(), indexedValueForSearch(value)), - syntaxFlags, - matchFlags, - RegexpQuery.DEFAULT_PROVIDER, - maxDeterminizedStates, - MultiTermQuery.DOC_VALUES_REWRITE - ); + Query query = null; + switch (rewriteOverride) { + case INDEX_OR_DOC_VALUES: + if (isSearchable() && hasDocValues()) { + Query indexQuery = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); + Query dvQuery = super.regexpQuery( + value, + syntaxFlags, + matchFlags, + maxDeterminizedStates, + MultiTermQuery.DOC_VALUES_REWRITE, + context + ); + query = new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else if (hasDocValues()) { + query = new RegexpQuery( + new Term(name(), indexedValueForSearch(value)), + syntaxFlags, + matchFlags, + RegexpQuery.DEFAULT_PROVIDER, + maxDeterminizedStates, + MultiTermQuery.DOC_VALUES_REWRITE + ); + } else { + query = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); + } + break; + case INDEX_ONLY: + failIfNotIndexed(); + query = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); + break; + case DOC_VALUES_ONLY: + failIfNoDocValues(); + query = new RegexpQuery( + new Term(name(), indexedValueForSearch(value)), + syntaxFlags, + matchFlags, + RegexpQuery.DEFAULT_PROVIDER, + maxDeterminizedStates, + MultiTermQuery.DOC_VALUES_REWRITE + ); + break; } - return super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); + return query; } @Override - public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + QueryShardContext.RewriteOverride rewriteOverride, + QueryShardContext context + ) { if (context.allowExpensiveQueries() == false) { throw new OpenSearchException( "[range] queries on [text] or [keyword] fields cannot be executed when '" @@ -495,41 +576,65 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower ); } failIfNotIndexedAndNoDocValues(); - if (isSearchable() && hasDocValues()) { - Query indexQuery = new TermRangeQuery( - name(), - lowerTerm == null ? null : indexedValueForSearch(lowerTerm), - upperTerm == null ? null : indexedValueForSearch(upperTerm), - includeLower, - includeUpper - ); - Query dvQuery = new TermRangeQuery( - name(), - lowerTerm == null ? null : indexedValueForSearch(lowerTerm), - upperTerm == null ? null : indexedValueForSearch(upperTerm), - includeLower, - includeUpper, - MultiTermQuery.DOC_VALUES_REWRITE - ); - return new IndexOrDocValuesQuery(indexQuery, dvQuery); + if (rewriteOverride == null) { + rewriteOverride = QueryRewriteContext.RewriteOverride.INDEX_OR_DOC_VALUES; } - if (hasDocValues()) { - return new TermRangeQuery( - name(), - lowerTerm == null ? null : indexedValueForSearch(lowerTerm), - upperTerm == null ? null : indexedValueForSearch(upperTerm), - includeLower, - includeUpper, - MultiTermQuery.DOC_VALUES_REWRITE - ); + Query query = null; + switch (rewriteOverride) { + case INDEX_OR_DOC_VALUES: + if (isSearchable() && hasDocValues()) { + Query indexQuery = new TermRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper + ); + Query dvQuery = new TermRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper, + MultiTermQuery.DOC_VALUES_REWRITE + ); + query = new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else if (hasDocValues()) { + query = new TermRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper, + MultiTermQuery.DOC_VALUES_REWRITE + ); + } else { + query = super.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); + } + break; + case INDEX_ONLY: + failIfNotIndexed(); + query = new TermRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper + ); + break; + case DOC_VALUES_ONLY: + failIfNoDocValues(); + query = new TermRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper, + MultiTermQuery.DOC_VALUES_REWRITE + ); + break; } - return new TermRangeQuery( - name(), - lowerTerm == null ? null : indexedValueForSearch(lowerTerm), - upperTerm == null ? null : indexedValueForSearch(upperTerm), - includeLower, - includeUpper - ); + return query; } @Override @@ -540,6 +645,7 @@ public Query fuzzyQuery( int maxExpansions, boolean transpositions, @Nullable MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, QueryShardContext context ) { failIfNotIndexedAndNoDocValues(); @@ -548,36 +654,61 @@ public Query fuzzyQuery( "[fuzzy] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to " + "false." ); } - if (isSearchable() && hasDocValues()) { - Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); - Query dvQuery = super.fuzzyQuery( - value, - fuzziness, - prefixLength, - maxExpansions, - transpositions, - MultiTermQuery.DOC_VALUES_REWRITE, - context - ); - return new IndexOrDocValuesQuery(indexQuery, dvQuery); + if (rewriteOverride == null) { + rewriteOverride = QueryRewriteContext.RewriteOverride.INDEX_OR_DOC_VALUES; } - if (hasDocValues()) { - return new FuzzyQuery( - new Term(name(), indexedValueForSearch(value)), - fuzziness.asDistance(BytesRefs.toString(value)), - prefixLength, - maxExpansions, - transpositions, - MultiTermQuery.DOC_VALUES_REWRITE - ); + Query query = null; + switch (rewriteOverride) { + case INDEX_OR_DOC_VALUES: + if (isSearchable() && hasDocValues()) { + Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); + Query dvQuery = super.fuzzyQuery( + value, + fuzziness, + prefixLength, + maxExpansions, + transpositions, + MultiTermQuery.DOC_VALUES_REWRITE, + context + ); + query = new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else if (hasDocValues()) { + query = new FuzzyQuery( + new Term(name(), indexedValueForSearch(value)), + fuzziness.asDistance(BytesRefs.toString(value)), + prefixLength, + maxExpansions, + transpositions, + MultiTermQuery.DOC_VALUES_REWRITE + ); + } else { + query = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); + } + break; + case INDEX_ONLY: + failIfNotIndexed(); + query = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); + break; + case DOC_VALUES_ONLY: + failIfNoDocValues(); + query = new FuzzyQuery( + new Term(name(), indexedValueForSearch(value)), + fuzziness.asDistance(BytesRefs.toString(value)), + prefixLength, + maxExpansions, + transpositions, + MultiTermQuery.DOC_VALUES_REWRITE + ); + break; } - return super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context); + return query; } @Override public Query wildcardQuery( String value, @Nullable MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, boolean caseInsensitive, QueryShardContext context ) { @@ -590,21 +721,44 @@ public Query wildcardQuery( // keyword field types are always normalized, so ignore case sensitivity and force normalize the // wildcard // query text - if (isSearchable() && hasDocValues()) { - Query indexQuery = super.wildcardQuery(value, method, caseInsensitive, true, context); - Query dvQuery = super.wildcardQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, true, context); - return new IndexOrDocValuesQuery(indexQuery, dvQuery); + if (rewriteOverride == null) { + rewriteOverride = QueryRewriteContext.RewriteOverride.INDEX_OR_DOC_VALUES; } - if (hasDocValues()) { - Term term; - value = normalizeWildcardPattern(name(), value, getTextSearchInfo().getSearchAnalyzer()); - term = new Term(name(), value); - if (caseInsensitive) { - return AutomatonQueries.caseInsensitiveWildcardQuery(term, method); - } - return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE); + Query query = null; + switch (rewriteOverride) { + case INDEX_OR_DOC_VALUES: + if (isSearchable() && hasDocValues()) { + Query indexQuery = super.wildcardQuery(value, method, caseInsensitive, true, context); + Query dvQuery = super.wildcardQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, true, context); + query = new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else if (hasDocValues()) { + Term term; + value = normalizeWildcardPattern(name(), value, getTextSearchInfo().getSearchAnalyzer()); + term = new Term(name(), value); + if (caseInsensitive) { + return AutomatonQueries.caseInsensitiveWildcardQuery(term, method); + } + query = new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE); + } else { + query = super.wildcardQuery(value, method, caseInsensitive, true, context); + } + break; + case INDEX_ONLY: + failIfNotIndexed(); + query = super.wildcardQuery(value, method, caseInsensitive, true, context); + break; + case DOC_VALUES_ONLY: + failIfNoDocValues(); + Term term; + value = normalizeWildcardPattern(name(), value, getTextSearchInfo().getSearchAnalyzer()); + term = new Term(name(), value); + if (caseInsensitive) { + return AutomatonQueries.caseInsensitiveWildcardQuery(term, method); + } + query = new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE); + break; } - return super.wildcardQuery(value, method, caseInsensitive, true, context); + return query; } } diff --git a/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java index 66d4654e543a2..ab3cafd2516c1 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/opensearch/index/mapper/MappedFieldType.java @@ -230,6 +230,14 @@ public Query termQueryCaseInsensitive(Object value, @Nullable QueryShardContext ); } + public Query termsQuery( + List values, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + @Nullable QueryShardContext context + ) { + return termsQuery(values, context); + } + /** Build a constant-scoring query that matches all values. The default implementation uses a * {@link ConstantScoreQuery} around a {@link BooleanQuery} whose {@link Occur#SHOULD} clauses * are generated with {@link #termQuery}. */ @@ -258,6 +266,20 @@ public Query rangeQuery( throw new IllegalArgumentException("Field [" + name + "] of type [" + typeName() + "] does not support range queries"); } + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ShapeRelation relation, + ZoneId timeZone, + DateMathParser parser, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + QueryShardContext context + ) { + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, relation, timeZone, parser, context); + } + public Query fuzzyQuery( Object value, Fuzziness fuzziness, @@ -286,6 +308,29 @@ public Query fuzzyQuery( ); } + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + @Nullable MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + QueryShardContext context + ) { + return fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); + } + + public Query prefixQuery( + String value, + @Nullable MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + boolean caseInsensitve, + QueryShardContext context + ) { + return prefixQuery(value, method, caseInsensitve, context); + } + // Case sensitive form of prefix query public final Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) { return prefixQuery(value, method, false, context); @@ -321,6 +366,16 @@ public Query wildcardQuery( ); } + public Query wildcardQuery( + String value, + @Nullable MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + boolean caseInsensitve, + QueryShardContext context + ) { + return wildcardQuery(value, method, caseInsensitve, context); + } + /** always normalizes the wildcard pattern to lowercase */ public Query normalizedWildcardQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) { throw new QueryShardException( @@ -343,6 +398,18 @@ public Query regexpQuery( ); } + public Query regexpQuery( + String value, + int syntaxFlags, + int matchFlags, + int maxDeterminizedStates, + @Nullable MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + QueryShardContext context + ) { + return regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context); + } + public Query existsQuery(QueryShardContext context) { if (hasDocValues()) { return new DocValuesFieldExistsQuery(name()); diff --git a/server/src/main/java/org/opensearch/index/mapper/SimpleMappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/SimpleMappedFieldType.java index 7ea5d49e234a5..815567e4cd263 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SimpleMappedFieldType.java +++ b/server/src/main/java/org/opensearch/index/mapper/SimpleMappedFieldType.java @@ -33,6 +33,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.search.Query; +import org.opensearch.common.Nullable; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.time.DateMathParser; import org.opensearch.index.query.QueryShardContext; @@ -78,6 +79,27 @@ public final Query rangeQuery( return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); } + @Override + public final Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ShapeRelation relation, + ZoneId timeZone, + DateMathParser parser, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + QueryShardContext context + ) { + if (relation == ShapeRelation.DISJOINT) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); + } + // We do not fail on non-null time zones and date parsers + // The reasoning is that on query parsers, you might want to set a time zone or format for date fields + // but then the API has no way to know which fields are dates and which fields are not dates + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, rewriteOverride, context); + } + /** * Same as {@link #rangeQuery(Object, Object, boolean, boolean, ShapeRelation, ZoneId, DateMathParser, QueryShardContext)} * but without the trouble of relations or date-specific options. @@ -86,4 +108,15 @@ protected Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLo throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support range queries"); } + protected Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + QueryShardContext context + ) { + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); + } + } diff --git a/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java b/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java index 682ccc13f769d..8cbb3872d98ea 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java +++ b/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java @@ -45,6 +45,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.automaton.Operations; import org.opensearch.OpenSearchException; +import org.opensearch.common.Nullable; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.unit.Fuzziness; @@ -110,7 +111,7 @@ public Query fuzzyQuery( int prefixLength, int maxExpansions, boolean transpositions, - MultiTermQuery.RewriteMethod method, + @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context ) { if (!context.allowExpensiveQueries()) { @@ -132,6 +133,20 @@ public Query fuzzyQuery( ); } + @Override + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + @Nullable MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + QueryShardContext context + ) { + return fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); + } + @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { if (context.allowExpensiveQueries() == false) { @@ -188,6 +203,17 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo return wildcardQuery(value, method, caseInsensitive, false, context); } + @Override + public Query wildcardQuery( + String value, + MultiTermQuery.RewriteMethod method, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + boolean caseInsensitive, + QueryShardContext context + ) { + return wildcardQuery(value, method, caseInsensitive, context); + } + /** always normalizes the wildcard pattern to lowercase */ @Override public Query normalizedWildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { @@ -279,4 +305,16 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower includeUpper ); } + + @Override + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + @Nullable QueryShardContext.RewriteOverride rewriteOverride, + QueryShardContext context + ) { + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); + } } diff --git a/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java index 93c32bbedcef4..e420784f558c0 100644 --- a/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java @@ -35,6 +35,7 @@ import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; +import org.opensearch.Version; import org.opensearch.common.unit.Fuzziness; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; @@ -78,6 +79,10 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder i private static final ParseField TRANSPOSITIONS_FIELD = new ParseField("transpositions"); private static final ParseField REWRITE_FIELD = new ParseField("rewrite"); + private static final ParseField REWRITE_OVERRIDE = new ParseField("rewrite_override"); + + private String rewriteOverride; + private final String fieldName; private final Object value; @@ -181,6 +186,9 @@ public FuzzyQueryBuilder(StreamInput in) throws IOException { maxExpansions = in.readVInt(); transpositions = in.readBoolean(); rewrite = in.readOptionalString(); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + rewriteOverride = in.readOptionalString(); + } } @Override @@ -192,6 +200,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeVInt(this.maxExpansions); out.writeBoolean(this.transpositions); out.writeOptionalString(this.rewrite); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalString(rewriteOverride); + } } @Override @@ -248,6 +259,11 @@ public String rewrite() { return this.rewrite; } + private FuzzyQueryBuilder rewriteOverride(String rewriteOverride) { + this.rewriteOverride = rewriteOverride; + return this; + } + @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); @@ -261,6 +277,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.field(REWRITE_FIELD.getPreferredName(), rewrite); } printBoostAndQueryName(builder); + if (rewriteOverride != null) { + builder.field(REWRITE_OVERRIDE.getPreferredName(), rewriteOverride); + } builder.endObject(); builder.endObject(); } @@ -273,6 +292,7 @@ public static FuzzyQueryBuilder fromXContent(XContentParser parser) throws IOExc int maxExpansions = FuzzyQueryBuilder.DEFAULT_MAX_EXPANSIONS; boolean transpositions = FuzzyQueryBuilder.DEFAULT_TRANSPOSITIONS; String rewrite = null; + String rewriteOverride = null; String queryName = null; float boost = AbstractQueryBuilder.DEFAULT_BOOST; String currentFieldName = null; @@ -303,6 +323,8 @@ public static FuzzyQueryBuilder fromXContent(XContentParser parser) throws IOExc transpositions = parser.booleanValue(); } else if (REWRITE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { rewrite = parser.textOrNull(); + } else if (REWRITE_OVERRIDE.match(currentFieldName, parser.getDeprecationHandler())) { + rewriteOverride = parser.textOrNull(); } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { queryName = parser.text(); } else { @@ -330,7 +352,8 @@ public static FuzzyQueryBuilder fromXContent(XContentParser parser) throws IOExc .transpositions(transpositions) .rewrite(rewrite) .boost(boost) - .queryName(queryName); + .queryName(queryName) + .rewriteOverride(rewriteOverride); } @Override @@ -357,17 +380,40 @@ protected Query doToQuery(QueryShardContext context) throws IOException { throw new IllegalStateException("Rewrite first"); } String rewrite = this.rewrite; - Query query = fieldType.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, null, context); + QueryShardContext.RewriteOverride rewriteOverrideMethod = QueryParsers.parseRewriteOverride( + rewriteOverride, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + LoggingDeprecationHandler.INSTANCE + ); + Query query = fieldType.fuzzyQuery( + value, + fuzziness, + prefixLength, + maxExpansions, + transpositions, + null, + rewriteOverrideMethod, + context + ); if (query instanceof MultiTermQuery) { MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(rewrite, null, LoggingDeprecationHandler.INSTANCE); - QueryParsers.setRewriteMethod((MultiTermQuery) query, rewriteMethod); + query = fieldType.fuzzyQuery( + value, + fuzziness, + prefixLength, + maxExpansions, + transpositions, + rewriteMethod, + rewriteOverrideMethod, + context + ); } return query; } @Override protected int doHashCode() { - return Objects.hash(fieldName, value, fuzziness, prefixLength, maxExpansions, transpositions, rewrite); + return Objects.hash(fieldName, value, fuzziness, prefixLength, maxExpansions, transpositions, rewrite, rewriteOverride); } @Override @@ -378,6 +424,7 @@ protected boolean doEquals(FuzzyQueryBuilder other) { && Objects.equals(prefixLength, other.prefixLength) && Objects.equals(maxExpansions, other.maxExpansions) && Objects.equals(transpositions, other.transpositions) - && Objects.equals(rewrite, other.rewrite); + && Objects.equals(rewrite, other.rewrite) + && Objects.equals(rewriteOverride, other.rewriteOverride); } } diff --git a/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java index ffc748bffb66e..668ed0283b21a 100644 --- a/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java @@ -36,6 +36,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; +import org.opensearch.Version; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -62,6 +63,10 @@ public class PrefixQueryBuilder extends AbstractQueryBuilder private static final ParseField PREFIX_FIELD = new ParseField("value"); private static final ParseField REWRITE_FIELD = new ParseField("rewrite"); + private static final ParseField REWRITE_OVERRIDE = new ParseField("rewrite_override"); + + private String rewriteOverride; + private final String fieldName; private final String value; @@ -98,6 +103,9 @@ public PrefixQueryBuilder(StreamInput in) throws IOException { value = in.readString(); rewrite = in.readOptionalString(); caseInsensitive = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + rewriteOverride = in.readOptionalString(); + } } @Override @@ -106,6 +114,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(value); out.writeOptionalString(rewrite); out.writeBoolean(caseInsensitive); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalString(rewriteOverride); + } } @Override @@ -131,6 +142,11 @@ public PrefixQueryBuilder rewrite(String rewrite) { return this; } + public PrefixQueryBuilder rewriteOverride(String rewriteOverride) { + this.rewriteOverride = rewriteOverride; + return this; + } + public String rewrite() { return this.rewrite; } @@ -147,6 +163,9 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio builder.field(CASE_INSENSITIVE_FIELD.getPreferredName(), caseInsensitive); } printBoostAndQueryName(builder); + if (rewriteOverride != null) { + builder.field(REWRITE_OVERRIDE.getPreferredName(), rewriteOverride); + } builder.endObject(); builder.endObject(); } @@ -155,6 +174,7 @@ public static PrefixQueryBuilder fromXContent(XContentParser parser) throws IOEx String fieldName = null; String value = null; String rewrite = null; + String rewriteOverride = null; String queryName = null; float boost = AbstractQueryBuilder.DEFAULT_BOOST; @@ -181,6 +201,8 @@ public static PrefixQueryBuilder fromXContent(XContentParser parser) throws IOEx rewrite = parser.textOrNull(); } else if (CASE_INSENSITIVE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { caseInsensitive = parser.booleanValue(); + } else if (REWRITE_OVERRIDE.match(currentFieldName, parser.getDeprecationHandler())) { + rewriteOverride = parser.textOrNull(); } else { throw new ParsingException( parser.getTokenLocation(), @@ -196,7 +218,11 @@ public static PrefixQueryBuilder fromXContent(XContentParser parser) throws IOEx } } - return new PrefixQueryBuilder(fieldName, value).rewrite(rewrite).boost(boost).queryName(queryName).caseInsensitive(caseInsensitive); + return new PrefixQueryBuilder(fieldName, value).rewrite(rewrite) + .boost(boost) + .queryName(queryName) + .caseInsensitive(caseInsensitive) + .rewriteOverride(rewriteOverride); } @Override @@ -215,7 +241,14 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws // This logic is correct for all field types, but by only applying it to constant // fields we also have the guarantee that it doesn't perform I/O, which is important // since rewrites might happen on a network thread. - Query query = fieldType.prefixQuery(value, null, caseInsensitive, context); // the rewrite method doesn't matter + QueryShardContext.RewriteOverride rewriteOverrideMethod = QueryParsers.parseRewriteOverride( + rewriteOverride, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + LoggingDeprecationHandler.INSTANCE + ); + Query query = fieldType.prefixQuery(value, null, rewriteOverrideMethod, caseInsensitive, context); // the rewrite method + // doesn't + // matter if (query instanceof MatchAllDocsQuery) { return new MatchAllQueryBuilder(); } else if (query instanceof MatchNoDocsQuery) { @@ -237,12 +270,17 @@ protected Query doToQuery(QueryShardContext context) throws IOException { if (fieldType == null) { throw new IllegalStateException("Rewrite first"); } - return fieldType.prefixQuery(value, method, caseInsensitive, context); + QueryShardContext.RewriteOverride rewriteOverrideMethod = QueryParsers.parseRewriteOverride( + rewriteOverride, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + LoggingDeprecationHandler.INSTANCE + ); + return fieldType.prefixQuery(value, method, rewriteOverrideMethod, caseInsensitive, context); } @Override protected final int doHashCode() { - return Objects.hash(fieldName, value, rewrite, caseInsensitive); + return Objects.hash(fieldName, value, rewrite, caseInsensitive, rewriteOverride); } @Override @@ -250,6 +288,7 @@ protected boolean doEquals(PrefixQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Objects.equals(value, other.value) && Objects.equals(rewrite, other.rewrite) + && Objects.equals(rewriteOverride, other.rewriteOverride) && Objects.equals(caseInsensitive, other.caseInsensitive); } } diff --git a/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java index 15a6d0b5a774e..8994f3d9d8770 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java @@ -159,4 +159,21 @@ public void onFailure(Exception e) { public boolean validate() { return validate; } + + /** A custom rewrite override for a query. Default executes the query as is and other values determine which structure to use at run-time. + * + * @opensearch.internal + */ + @PublicApi(since = "2.17.0") + public enum RewriteOverride { + + // don't override the rewrite, use default + INDEX_OR_DOC_VALUES, + + // use index structure + INDEX_ONLY, + + // use doc_values structure + DOC_VALUES_ONLY + } } diff --git a/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java index fdbef2c732361..a631ed351dde6 100644 --- a/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/RangeQueryBuilder.java @@ -35,9 +35,11 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.opensearch.Version; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateMathParser; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.Strings; @@ -47,6 +49,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.FieldNamesFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.query.support.QueryParsers; import java.io.IOException; import java.time.DateTimeException; @@ -75,6 +78,7 @@ public class RangeQueryBuilder extends AbstractQueryBuilder i private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); private static final ParseField FORMAT_FIELD = new ParseField("format"); private static final ParseField RELATION_FIELD = new ParseField("relation"); + private static final ParseField REWRITE_OVERRIDE = new ParseField("rewrite_override"); private final String fieldName; private Object from; @@ -85,6 +89,8 @@ public class RangeQueryBuilder extends AbstractQueryBuilder i private String format; private ShapeRelation relation; + private String rewriteOverride; + /** * A Query that matches documents within an range of terms. * @@ -116,6 +122,9 @@ public RangeQueryBuilder(StreamInput in) throws IOException { throw new IllegalArgumentException("[range] query does not support relation [" + relationString + "]"); } } + if (in.getVersion().after(Version.V_3_0_0)) { + rewriteOverride = in.readOptionalString(); + } } private boolean isRelationAllowed(ShapeRelation relation) { @@ -136,6 +145,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { relationString = this.relation.getRelationName(); } out.writeOptionalString(relationString); + if (out.getVersion().after(Version.V_3_0_0)) { + out.writeOptionalString(rewriteOverride); + } } /** @@ -271,6 +283,11 @@ public RangeQueryBuilder timeZone(String timeZone) { return this; } + public RangeQueryBuilder rewriteOverride(String rewriteOverride) { + this.rewriteOverride = rewriteOverride; + return this; + } + /** * In case of date field, gets the from/to fields timezone adjustment */ @@ -345,6 +362,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.field(RELATION_FIELD.getPreferredName(), relation.getRelationName()); } printBoostAndQueryName(builder); + if (rewriteOverride != null) { + builder.field(REWRITE_OVERRIDE.getPreferredName(), rewriteOverride); + } builder.endObject(); builder.endObject(); } @@ -360,6 +380,7 @@ public static RangeQueryBuilder fromXContent(XContentParser parser) throws IOExc String queryName = null; String format = null; String relation = null; + String rewriteOverride = null; String currentFieldName = null; XContentParser.Token token; @@ -403,6 +424,8 @@ public static RangeQueryBuilder fromXContent(XContentParser parser) throws IOExc relation = parser.text(); } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { queryName = parser.text(); + } else if (REWRITE_OVERRIDE.match(currentFieldName, parser.getDeprecationHandler())) { + rewriteOverride = parser.textOrNull(); } else { throw new ParsingException( parser.getTokenLocation(), @@ -432,6 +455,7 @@ public static RangeQueryBuilder fromXContent(XContentParser parser) throws IOExc if (relation != null) { rangeQuery.relation(relation); } + rangeQuery.rewriteOverride(rewriteOverride); return rangeQuery; } @@ -524,12 +548,27 @@ protected Query doToQuery(QueryShardContext context) throws IOException { throw new IllegalStateException("Rewrite first"); } DateMathParser forcedDateParser = getForceDateParser(); - return mapper.rangeQuery(from, to, includeLower, includeUpper, relation, timeZone, forcedDateParser, context); + QueryShardContext.RewriteOverride rewriteOverrideMethod = QueryParsers.parseRewriteOverride( + rewriteOverride, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + LoggingDeprecationHandler.INSTANCE + ); + return mapper.rangeQuery( + from, + to, + includeLower, + includeUpper, + relation, + timeZone, + forcedDateParser, + rewriteOverrideMethod, + context + ); } @Override protected int doHashCode() { - return Objects.hash(fieldName, from, to, timeZone, includeLower, includeUpper, format); + return Objects.hash(fieldName, from, to, timeZone, includeLower, includeUpper, format, rewriteOverride); } @Override @@ -540,6 +579,7 @@ protected boolean doEquals(RangeQueryBuilder other) { && Objects.equals(timeZone, other.timeZone) && Objects.equals(includeLower, other.includeLower) && Objects.equals(includeUpper, other.includeUpper) - && Objects.equals(format, other.format); + && Objects.equals(format, other.format) + && Objects.equals(rewriteOverride, other.rewriteOverride); } } diff --git a/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java index f0da4d5736c0f..726e9477a9347 100644 --- a/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java @@ -38,6 +38,7 @@ import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; +import org.opensearch.Version; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; @@ -73,6 +74,10 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder private static final ParseField REWRITE_FIELD = new ParseField("rewrite"); private static final ParseField VALUE_FIELD = new ParseField("value"); + private static final ParseField REWRITE_OVERRIDE = new ParseField("rewrite_override"); + + private String rewriteOverride; + private final String fieldName; private final String value; @@ -112,6 +117,9 @@ public RegexpQueryBuilder(StreamInput in) throws IOException { maxDeterminizedStates = in.readVInt(); rewrite = in.readOptionalString(); caseInsensitive = in.readBoolean(); + if (in.getVersion().after(Version.V_3_0_0)) { + rewriteOverride = in.readOptionalString(); + } } @Override @@ -122,6 +130,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeVInt(maxDeterminizedStates); out.writeOptionalString(rewrite); out.writeBoolean(caseInsensitive); + if (out.getVersion().after(Version.V_3_0_0)) { + out.writeOptionalString(rewriteOverride); + } } /** Returns the field name used in this query. */ @@ -172,6 +183,11 @@ public boolean caseInsensitive() { return this.caseInsensitive; } + public RegexpQueryBuilder rewriteOverride(String rewriteOverride) { + this.rewriteOverride = rewriteOverride; + return this; + } + /** * Sets the regexp maxDeterminizedStates. */ @@ -207,6 +223,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.field(REWRITE_FIELD.getPreferredName(), rewrite); } printBoostAndQueryName(builder); + if (rewriteOverride != null) { + builder.field(REWRITE_OVERRIDE.getPreferredName(), rewriteOverride); + } builder.endObject(); builder.endObject(); } @@ -215,6 +234,7 @@ public static RegexpQueryBuilder fromXContent(XContentParser parser) throws IOEx String fieldName = null; String rewrite = null; String value = null; + String rewriteOverride = null; float boost = AbstractQueryBuilder.DEFAULT_BOOST; int flagsValue = RegexpQueryBuilder.DEFAULT_FLAGS_VALUE; boolean caseInsensitive = DEFAULT_CASE_INSENSITIVITY; @@ -249,6 +269,8 @@ public static RegexpQueryBuilder fromXContent(XContentParser parser) throws IOEx caseInsensitive = parser.booleanValue(); } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { queryName = parser.text(); + } else if (REWRITE_OVERRIDE.match(currentFieldName, parser.getDeprecationHandler())) { + rewriteOverride = parser.textOrNull(); } else { throw new ParsingException( parser.getTokenLocation(), @@ -270,6 +292,7 @@ public static RegexpQueryBuilder fromXContent(XContentParser parser) throws IOEx .boost(boost) .queryName(queryName); result.caseInsensitive(caseInsensitive); + result.rewriteOverride(rewriteOverride); return result; } @@ -303,7 +326,20 @@ protected Query doToQuery(QueryShardContext context) throws QueryShardException, MappedFieldType fieldType = context.fieldMapper(fieldName); if (fieldType != null) { - query = fieldType.regexpQuery(value, sanitisedSyntaxFlag, matchFlagsValue, maxDeterminizedStates, method, context); + QueryShardContext.RewriteOverride rewriteOverrideMethod = QueryParsers.parseRewriteOverride( + rewriteOverride, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + LoggingDeprecationHandler.INSTANCE + ); + query = fieldType.regexpQuery( + value, + sanitisedSyntaxFlag, + matchFlagsValue, + maxDeterminizedStates, + method, + rewriteOverrideMethod, + context + ); } if (query == null) { if (method == null) { @@ -323,7 +359,7 @@ protected Query doToQuery(QueryShardContext context) throws QueryShardException, @Override protected int doHashCode() { - return Objects.hash(fieldName, value, syntaxFlagsValue, caseInsensitive, maxDeterminizedStates, rewrite); + return Objects.hash(fieldName, value, syntaxFlagsValue, caseInsensitive, maxDeterminizedStates, rewrite, rewriteOverride); } @Override @@ -333,6 +369,7 @@ protected boolean doEquals(RegexpQueryBuilder other) { && Objects.equals(syntaxFlagsValue, other.syntaxFlagsValue) && Objects.equals(caseInsensitive, other.caseInsensitive) && Objects.equals(maxDeterminizedStates, other.maxDeterminizedStates) - && Objects.equals(rewrite, other.rewrite); + && Objects.equals(rewrite, other.rewrite) + && Objects.equals(rewriteOverride, other.rewriteOverride); } } diff --git a/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java index 4b92d6a1f5460..65e60d76f253c 100644 --- a/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/TermsQueryBuilder.java @@ -42,6 +42,7 @@ import org.opensearch.client.Client; import org.opensearch.common.SetOnce; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; @@ -57,6 +58,7 @@ import org.opensearch.index.mapper.ConstantFieldType; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.query.support.QueryParsers; import org.opensearch.indices.TermsLookup; import java.io.IOException; @@ -87,6 +89,9 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { private final TermsLookup termsLookup; private final Supplier> supplier; + private static final ParseField REWRITE_OVERRIDE = new ParseField("rewrite_override"); + + private String rewriteOverride; private static final ParseField VALUE_TYPE_FIELD = new ParseField("value_type"); private ValueType valueType = ValueType.DEFAULT; @@ -225,9 +230,10 @@ public TermsQueryBuilder(String fieldName, Iterable values) { this.supplier = null; } - private TermsQueryBuilder(String fieldName, Iterable values, ValueType valueType) { + private TermsQueryBuilder(String fieldName, Iterable values, ValueType valueType, String rewriteOverride) { this(fieldName, values); this.valueType = valueType; + this.rewriteOverride = rewriteOverride; } private TermsQueryBuilder(String fieldName, Supplier> supplier) { @@ -237,9 +243,10 @@ private TermsQueryBuilder(String fieldName, Supplier> supplier) { this.supplier = supplier; } - private TermsQueryBuilder(String fieldName, Supplier> supplier, ValueType valueType) { + private TermsQueryBuilder(String fieldName, Supplier> supplier, ValueType valueType, String rewriteOverride) { this(fieldName, supplier); this.valueType = valueType; + this.rewriteOverride = rewriteOverride; } /** @@ -251,6 +258,9 @@ public TermsQueryBuilder(StreamInput in) throws IOException { termsLookup = in.readOptionalWriteable(TermsLookup::new); values = (List) in.readGenericValue(); this.supplier = null; + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + rewriteOverride = in.readOptionalString(); + } if (in.getVersion().onOrAfter(Version.V_2_17_0)) { valueType = in.readEnum(ValueType.class); } @@ -264,6 +274,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(fieldName); out.writeOptionalWriteable(termsLookup); out.writeGenericValue(values); + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeOptionalString(rewriteOverride); + } if (out.getVersion().onOrAfter(Version.V_2_17_0)) { out.writeEnum(valueType); } @@ -281,6 +294,11 @@ public TermsLookup termsLookup() { return this.termsLookup; } + public TermsQueryBuilder rewriteOverride(String rewriteOverride) { + this.rewriteOverride = rewriteOverride; + return this; + } + private static final Set> INTEGER_TYPES = new HashSet<>( Arrays.asList(Byte.class, Short.class, Integer.class, Long.class) ); @@ -414,6 +432,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.field(fieldName, convertBack(values)); } printBoostAndQueryName(builder); + if (rewriteOverride != null) { + builder.field(REWRITE_OVERRIDE.getPreferredName(), rewriteOverride); + } if (valueType != ValueType.DEFAULT) { builder.field(VALUE_TYPE_FIELD.getPreferredName(), valueType.type); } @@ -425,6 +446,8 @@ public static TermsQueryBuilder fromXContent(XContentParser parser) throws IOExc List values = null; TermsLookup termsLookup = null; + String rewriteOverride = null; + String queryName = null; float boost = AbstractQueryBuilder.DEFAULT_BOOST; @@ -465,6 +488,8 @@ public static TermsQueryBuilder fromXContent(XContentParser parser) throws IOExc boost = parser.floatValue(); } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { queryName = parser.text(); + } else if (REWRITE_OVERRIDE.match(currentFieldName, parser.getDeprecationHandler())) { + rewriteOverride = parser.textOrNull(); } else if (VALUE_TYPE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { valueTypeStr = parser.text(); } else { @@ -502,7 +527,10 @@ public static TermsQueryBuilder fromXContent(XContentParser parser) throws IOExc } } - return new TermsQueryBuilder(fieldName, values, termsLookup).boost(boost).queryName(queryName).valueType(valueType); + return new TermsQueryBuilder(fieldName, values, termsLookup).boost(boost) + .queryName(queryName) + .valueType(valueType) + .rewriteOverride(rewriteOverride); } static List parseValues(XContentParser parser) throws IOException { @@ -545,6 +573,11 @@ protected Query doToQuery(QueryShardContext context) throws IOException { if (fieldType == null) { throw new IllegalStateException("Rewrite first"); } + QueryShardContext.RewriteOverride rewriteOverrideMethod = QueryParsers.parseRewriteOverride( + rewriteOverride, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + LoggingDeprecationHandler.INSTANCE + ); if (valueType == ValueType.BITMAP) { if (values.size() == 1 && values.get(0) instanceof BytesArray) { if (fieldType instanceof NumberFieldMapper.NumberFieldType) { @@ -552,7 +585,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { } } } - return fieldType.termsQuery(values, context); + return fieldType.termsQuery(values, rewriteOverrideMethod, context); } private void fetch(TermsLookup termsLookup, Client client, ActionListener> actionListener) { @@ -583,7 +616,7 @@ private void fetch(TermsLookup termsLookup, Client client, ActionListener> supplier = new SetOnce<>(); queryRewriteContext.registerAsyncAction((client, listener) -> fetch(termsLookup, client, ActionListener.map(listener, list -> { supplier.set(list); return null; }))); - return new TermsQueryBuilder(this.fieldName, supplier::get, valueType); + return new TermsQueryBuilder(this.fieldName, supplier::get, valueType, rewriteOverride); } if (values == null || values.isEmpty()) { diff --git a/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java index d1fe4f0ba0264..8369b3d1f1b54 100644 --- a/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java @@ -36,6 +36,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; +import org.opensearch.Version; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -67,6 +68,9 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder unsearchable.termsQuery(Arrays.asList("foo", "bar"), null) + () -> unsearchable.termsQuery(Arrays.asList("foo", "bar"), QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, null) ); assertEquals( "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", @@ -175,7 +190,7 @@ public void testExistsQuery() { } public void testRangeQuery() { - MappedFieldType ft = new KeywordFieldType("field"); + KeywordFieldType ft = new KeywordFieldType("field"); Query indexExpected = new TermRangeQuery("field", BytesRefs.toBytesRef("foo"), BytesRefs.toBytesRef("bar"), true, false); Query dvExpected = new TermRangeQuery( @@ -188,19 +203,33 @@ public void testRangeQuery() { ); Query expected = new IndexOrDocValuesQuery(indexExpected, dvExpected); - Query actual = ft.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC); + Query actual = ft.rangeQuery("foo", "bar", true, false, QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, MOCK_QSC); assertEquals(expected, actual); - MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap()); - assertEquals(indexExpected, onlyIndexed.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC)); + KeywordFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap()); + assertEquals( + indexExpected, + onlyIndexed.rangeQuery("foo", "bar", true, false, QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, MOCK_QSC) + ); + assertEquals( + indexExpected, + onlyIndexed.rangeQuery("foo", "bar", true, false, QueryShardContext.RewriteOverride.INDEX_ONLY, MOCK_QSC) + ); - MappedFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap()); - assertEquals(dvExpected, onlyDocValues.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC)); + KeywordFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap()); + assertEquals( + dvExpected, + onlyDocValues.rangeQuery("foo", "bar", true, false, QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, MOCK_QSC) + ); + assertEquals( + dvExpected, + onlyDocValues.rangeQuery("foo", "bar", true, false, QueryShardContext.RewriteOverride.DOC_VALUES_ONLY, MOCK_QSC) + ); - MappedFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap()); + KeywordFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap()); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> unsearchable.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC) + () -> unsearchable.rangeQuery("foo", "bar", true, false, QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, MOCK_QSC) ); assertEquals( @@ -210,7 +239,14 @@ public void testRangeQuery() { OpenSearchException ee = expectThrows( OpenSearchException.class, - () -> ft.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_QSC_DISALLOW_EXPENSIVE) + () -> ft.rangeQuery( + "foo", + "bar", + true, + false, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + MOCK_QSC_DISALLOW_EXPENSIVE + ) ); assertEquals( "[range] queries on [text] or [keyword] fields cannot be executed when " + "'search.allow_expensive_queries' is set to false.", @@ -225,12 +261,43 @@ public void testRegexpQuery() { new RegexpQuery(new Term("field", "foo.*")), new RegexpQuery(new Term("field", "foo.*"), 0, 0, RegexpQuery.DEFAULT_PROVIDER, 10, MultiTermQuery.DOC_VALUES_REWRITE) ), - ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC) + ft.regexpQuery( + "foo.*", + 0, + 0, + 10, + MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + MOCK_QSC + ) ); Query indexExpected = new RegexpQuery(new Term("field", "foo.*")); MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap()); - assertEquals(indexExpected, onlyIndexed.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); + assertEquals( + indexExpected, + onlyIndexed.regexpQuery( + "foo.*", + 0, + 0, + 10, + MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + MOCK_QSC + ) + ); + assertEquals( + indexExpected, + onlyIndexed.regexpQuery( + "foo.*", + 0, + 0, + 10, + MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, + QueryShardContext.RewriteOverride.INDEX_ONLY, + MOCK_QSC + ) + ); Query dvExpected = new RegexpQuery( new Term("field", "foo.*"), @@ -241,12 +308,35 @@ public void testRegexpQuery() { MultiTermQuery.DOC_VALUES_REWRITE ); MappedFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap()); - assertEquals(dvExpected, onlyDocValues.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.DOC_VALUES_REWRITE, MOCK_QSC)); + assertEquals( + dvExpected, + onlyDocValues.regexpQuery( + "foo.*", + 0, + 0, + 10, + MultiTermQuery.DOC_VALUES_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + MOCK_QSC + ) + ); + assertEquals( + dvExpected, + onlyDocValues.regexpQuery( + "foo.*", + 0, + 0, + 10, + MultiTermQuery.DOC_VALUES_REWRITE, + QueryShardContext.RewriteOverride.DOC_VALUES_ONLY, + MOCK_QSC + ) + ); MappedFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap()); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> unsearchable.regexpQuery("foo.*", 0, 0, 10, null, MOCK_QSC) + () -> unsearchable.regexpQuery("foo.*", 0, 0, 10, null, QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, MOCK_QSC) ); assertEquals( "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", @@ -255,36 +345,100 @@ public void testRegexpQuery() { OpenSearchException ee = expectThrows( OpenSearchException.class, - () -> ft.regexpQuery("foo.*", randomInt(10), 0, randomInt(10) + 1, null, MOCK_QSC_DISALLOW_EXPENSIVE) + () -> ft.regexpQuery( + "foo.*", + randomInt(10), + 0, + randomInt(10) + 1, + null, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + MOCK_QSC_DISALLOW_EXPENSIVE + ) ); assertEquals("[regexp] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", ee.getMessage()); } public void testFuzzyQuery() { - MappedFieldType ft = new KeywordFieldType("field"); + KeywordFieldType ft = new KeywordFieldType("field"); assertEquals( new IndexOrDocValuesQuery( new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true), new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE) ), - ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC) + ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, MOCK_QSC) ); Query indexExpected = new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true); - MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap()); - assertEquals(indexExpected, onlyIndexed.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MOCK_QSC)); + KeywordFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap()); + assertEquals( + indexExpected, + onlyIndexed.fuzzyQuery( + "foo", + Fuzziness.fromEdits(2), + 1, + 50, + true, + FuzzyQuery.defaultRewriteMethod(50), + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + MOCK_QSC + ) + ); + assertEquals( + indexExpected, + onlyIndexed.fuzzyQuery( + "foo", + Fuzziness.fromEdits(2), + 1, + 50, + true, + FuzzyQuery.defaultRewriteMethod(50), + QueryShardContext.RewriteOverride.INDEX_ONLY, + MOCK_QSC + ) + ); Query dvExpected = new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE); - MappedFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap()); + KeywordFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap()); assertEquals( dvExpected, - onlyDocValues.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE, MOCK_QSC) + onlyDocValues.fuzzyQuery( + "foo", + Fuzziness.fromEdits(2), + 1, + 50, + true, + MultiTermQuery.DOC_VALUES_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + MOCK_QSC + ) + ); + assertEquals( + dvExpected, + onlyDocValues.fuzzyQuery( + "foo", + Fuzziness.fromEdits(2), + 1, + 50, + true, + MultiTermQuery.DOC_VALUES_REWRITE, + QueryShardContext.RewriteOverride.DOC_VALUES_ONLY, + MOCK_QSC + ) ); - MappedFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap()); + KeywordFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap()); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> unsearchable.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE, MOCK_QSC) + () -> unsearchable.fuzzyQuery( + "foo", + Fuzziness.fromEdits(2), + 1, + 50, + true, + MultiTermQuery.DOC_VALUES_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + MOCK_QSC + ) ); assertEquals( "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", @@ -293,13 +447,22 @@ public void testFuzzyQuery() { OpenSearchException ee = expectThrows( OpenSearchException.class, - () -> ft.fuzzyQuery("foo", Fuzziness.AUTO, randomInt(10) + 1, randomInt(10) + 1, randomBoolean(), MOCK_QSC_DISALLOW_EXPENSIVE) + () -> ft.fuzzyQuery( + "foo", + Fuzziness.AUTO, + randomInt(10) + 1, + randomInt(10) + 1, + randomBoolean(), + FuzzyQuery.defaultRewriteMethod(randomInt(10) + 1), + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + MOCK_QSC_DISALLOW_EXPENSIVE + ) ); assertEquals("[fuzzy] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", ee.getMessage()); } public void testWildCardQuery() { - MappedFieldType ft = new KeywordFieldType("field"); + KeywordFieldType ft = new KeywordFieldType("field"); Query expected = new IndexOrDocValuesQuery( new WildcardQuery(new Term("field", new BytesRef("foo*"))), new WildcardQuery( @@ -308,24 +471,77 @@ public void testWildCardQuery() { MultiTermQuery.DOC_VALUES_REWRITE ) ); - assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); + assertEquals( + expected, + ft.wildcardQuery( + "foo*", + MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + false, + MOCK_QSC + ) + ); Query indexExpected = new WildcardQuery(new Term("field", new BytesRef("foo*"))); - MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap()); - assertEquals(indexExpected, onlyIndexed.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); + KeywordFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap()); + assertEquals( + indexExpected, + onlyIndexed.wildcardQuery( + "foo*", + MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + false, + MOCK_QSC + ) + ); + assertEquals( + indexExpected, + onlyIndexed.wildcardQuery( + "foo*", + MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, + QueryShardContext.RewriteOverride.INDEX_ONLY, + false, + MOCK_QSC + ) + ); Query dvExpected = new WildcardQuery( new Term("field", new BytesRef("foo*")), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE ); - MappedFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap()); - assertEquals(dvExpected, onlyDocValues.wildcardQuery("foo*", MultiTermQuery.DOC_VALUES_REWRITE, MOCK_QSC)); + KeywordFieldType onlyDocValues = new KeywordFieldType("field", false, true, Collections.emptyMap()); + assertEquals( + dvExpected, + onlyDocValues.wildcardQuery( + "foo*", + MultiTermQuery.DOC_VALUES_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + false, + MOCK_QSC + ) + ); + assertEquals( + dvExpected, + onlyDocValues.wildcardQuery( + "foo*", + MultiTermQuery.DOC_VALUES_REWRITE, + QueryShardContext.RewriteOverride.DOC_VALUES_ONLY, + false, + MOCK_QSC + ) + ); - MappedFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap()); + KeywordFieldType unsearchable = new KeywordFieldType("field", false, false, Collections.emptyMap()); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> unsearchable.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC) + () -> unsearchable.wildcardQuery( + "foo*", + MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + false, + MOCK_QSC + ) ); assertEquals( "Cannot search on field [field] since it is both not indexed, and does not have doc_values " + "enabled.", @@ -334,7 +550,13 @@ public void testWildCardQuery() { OpenSearchException ee = expectThrows( OpenSearchException.class, - () -> ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC_DISALLOW_EXPENSIVE) + () -> ft.wildcardQuery( + "foo*", + MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES, + false, + MOCK_QSC_DISALLOW_EXPENSIVE + ) ); assertEquals("[wildcard] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", ee.getMessage()); } diff --git a/server/src/test/java/org/opensearch/index/query/FuzzyQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/FuzzyQueryBuilderTests.java index 42c905301c390..02f42aed8bb24 100644 --- a/server/src/test/java/org/opensearch/index/query/FuzzyQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/FuzzyQueryBuilderTests.java @@ -266,7 +266,8 @@ public void testFromJson() throws IOException { + " \"prefix_length\" : 0,\n" + " \"max_expansions\" : 100,\n" + " \"transpositions\" : false,\n" - + " \"boost\" : 42.0\n" + + " \"boost\" : 42.0,\n" + + " \"rewrite_override\" : \"index_only\"\n" + " }\n" + " }\n" + "}"; diff --git a/server/src/test/java/org/opensearch/index/query/PrefixQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/PrefixQueryBuilderTests.java index 52093e65d0122..56ab0c1b38c0b 100644 --- a/server/src/test/java/org/opensearch/index/query/PrefixQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/PrefixQueryBuilderTests.java @@ -115,7 +115,7 @@ public void testBlendedRewriteMethod() throws IOException { } public void testFromJson() throws IOException { - String json = "{ \"prefix\" : { \"user\" : { \"value\" : \"ki\", \"boost\" : 2.0" + "} }}"; + String json = "{ \"prefix\" : { \"user\" : { \"value\" : \"ki\", \"boost\" : 2.0 , \"rewrite_override\":\"index_only\"}}}"; PrefixQueryBuilder parsed = (PrefixQueryBuilder) parseQuery(json); checkGeneratedJson(json, parsed); diff --git a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java index e72be29b85b63..23b56a5fa4e87 100644 --- a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java @@ -420,7 +420,8 @@ public void testFromJson() throws IOException { + " \"include_lower\" : true,\n" + " \"include_upper\" : true,\n" + " \"time_zone\" : \"+01:00\",\n" - + " \"boost\" : 1.0\n" + + " \"boost\" : 1.0,\n" + + " \"rewrite_override\" : \"index_only\"\n" + " }\n" + " }\n" + "}"; diff --git a/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java index 30c8c07fa2c27..f8140715cf566 100644 --- a/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java @@ -122,7 +122,8 @@ public void testFromJson() throws IOException { + " \"flags_value\" : 7,\n" + " \"case_insensitive\" : true,\n" + " \"max_determinized_states\" : 20000,\n" - + " \"boost\" : 1.0\n" + + " \"boost\" : 1.0,\n" + + " \"rewrite_override\" : \"index_only\"\n" + " }\n" + " }\n" + "}"; diff --git a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java index cfe9fed3bc97c..179bfc2c3e013 100644 --- a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java @@ -307,7 +307,8 @@ public void testFromJson() throws IOException { String json = "{\n" + " \"terms\" : {\n" + " \"user\" : [ \"foobar\", \"opensearch\" ],\n" - + " \"boost\" : 1.0\n" + + " \"boost\" : 1.0,\n" + + " \"rewrite_override\": \"index_only\"\n" + " }\n" + "}"; @@ -378,7 +379,12 @@ public void testConversion() { } public void testRewriteIndexQueryToMatchNone() throws IOException { - TermsQueryBuilder query = new TermsQueryBuilder("_index", "does_not_exist", "also_does_not_exist"); + TermsQueryBuilder query = new TermsQueryBuilder( + "_index", + "does_not_exist", + "also_does_not_exist", + QueryShardContext.RewriteOverride.INDEX_OR_DOC_VALUES + ); QueryShardContext queryShardContext = createShardContext(); QueryBuilder rewritten = query.rewrite(queryShardContext); assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class)); diff --git a/server/src/test/java/org/opensearch/index/query/WildcardQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/WildcardQueryBuilderTests.java index f4ec241a64a26..0ff60dd3a3e6a 100644 --- a/server/src/test/java/org/opensearch/index/query/WildcardQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/WildcardQueryBuilderTests.java @@ -118,7 +118,8 @@ public void testEmptyValue() throws IOException { } public void testFromJson() throws IOException { - String json = "{ \"wildcard\" : { \"user\" : { \"wildcard\" : \"ki*y\", \"boost\" : 2.0" + " } }}"; + String json = + "{ \"wildcard\" : { \"user\" : { \"wildcard\" : \"ki*y\", \"boost\" : 2.0, \"rewrite_override\" : \"index_only\"} }}"; WildcardQueryBuilder parsed = (WildcardQueryBuilder) parseQuery(json); checkGeneratedJson(json, parsed); assertEquals(json, "ki*y", parsed.value());