From 14c9c7e063c16d97d070030f9e5270903a9fe499 Mon Sep 17 00:00:00 2001 From: mushao999 Date: Wed, 15 Dec 2021 11:36:38 +0800 Subject: [PATCH 01/11] optimize source filtering in SourceFieldMapper using xcontent filtering config --- .../index/mapper/SourceFieldMapper.java | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 75bcd1eab5432..dd35fd27baa85 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -14,34 +14,32 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.function.Function; +import java.util.Set; public class SourceFieldMapper extends MetadataFieldMapper { - public static final String NAME = "_source"; public static final String RECOVERY_SOURCE_NAME = "_recovery_source"; public static final String CONTENT_TYPE = "_source"; - private final Function, Map> filter; + private final CheckedBiFunction filter; + private final XContentParserConfiguration parserConfig; private static final SourceFieldMapper DEFAULT = new SourceFieldMapper(Defaults.ENABLED, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY); @@ -145,7 +143,19 @@ private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes) this.includes = includes; this.excludes = excludes; final boolean filtered = CollectionUtils.isEmpty(includes) == false || CollectionUtils.isEmpty(excludes) == false; - this.filter = enabled && filtered ? XContentMapValues.filter(includes, excludes) : null; + if (enabled && filtered) { + this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.of(includes), Set.of(excludes)); + this.filter = (sourceBytes, contentType) -> { + BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, sourceBytes.length())); + XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput); + XContentParser parser = XContentType.JSON.xContent().createParser(parserConfig, sourceBytes.streamInput()); + builder.copyCurrentStructure(parser); + return BytesReference.bytes(builder); + }; + } else { + this.parserConfig = null; + this.filter = null; + } this.complete = enabled && CollectionUtils.isEmpty(includes) && CollectionUtils.isEmpty(excludes); } @@ -181,14 +191,7 @@ public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nul if (enabled && originalSource != null) { // Percolate and tv APIs may not set the source and that is ok, because these APIs will not index any data if (filter != null) { - // we don't update the context source if we filter, we want to keep it as is... - Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); - Map filteredSource = filter.apply(mapTuple.v2()); - BytesStreamOutput bStream = new BytesStreamOutput(); - XContentType actualContentType = mapTuple.v1(); - XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource); - builder.close(); - return bStream.bytes(); + return filter.apply(originalSource,contentType); } else { return originalSource; } From 7de88d240fe4ab1789139fb86d8a13206c946ef1 Mon Sep 17 00:00:00 2001 From: mushao999 Date: Thu, 23 Dec 2021 14:14:24 +0800 Subject: [PATCH 02/11] update follow romseygeek's suggestions --- .../index/mapper/SourceFieldMapper.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index dd35fd27baa85..d7e7a6ac7e11b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -39,7 +39,6 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final String CONTENT_TYPE = "_source"; private final CheckedBiFunction filter; - private final XContentParserConfiguration parserConfig; private static final SourceFieldMapper DEFAULT = new SourceFieldMapper(Defaults.ENABLED, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY); @@ -144,7 +143,7 @@ private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes) this.excludes = excludes; final boolean filtered = CollectionUtils.isEmpty(includes) == false || CollectionUtils.isEmpty(excludes) == false; if (enabled && filtered) { - this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.of(includes), Set.of(excludes)); + final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.of(includes), Set.of(excludes)); this.filter = (sourceBytes, contentType) -> { BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, sourceBytes.length())); XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput); @@ -153,8 +152,7 @@ private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes) return BytesReference.bytes(builder); }; } else { - this.parserConfig = null; - this.filter = null; + this.filter = (sourceBytes, contentType) -> sourceBytes; } this.complete = enabled && CollectionUtils.isEmpty(includes) && CollectionUtils.isEmpty(excludes); } @@ -190,11 +188,7 @@ public void preParse(DocumentParserContext context) throws IOException { public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable XContentType contentType) throws IOException { if (enabled && originalSource != null) { // Percolate and tv APIs may not set the source and that is ok, because these APIs will not index any data - if (filter != null) { - return filter.apply(originalSource,contentType); - } else { - return originalSource; - } + return filter.apply(originalSource, contentType); } else { return null; } From 70a211b5d8c69661c609ba918b27b746c8918a04 Mon Sep 17 00:00:00 2001 From: mushao999 Date: Thu, 6 Jan 2022 11:26:36 +0800 Subject: [PATCH 03/11] format code --- .../org/elasticsearch/index/mapper/SourceFieldMapper.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index d7e7a6ac7e11b..b7b2db73593d9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -143,7 +143,10 @@ private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes) this.excludes = excludes; final boolean filtered = CollectionUtils.isEmpty(includes) == false || CollectionUtils.isEmpty(excludes) == false; if (enabled && filtered) { - final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.of(includes), Set.of(excludes)); + final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( + Set.of(includes), + Set.of(excludes) + ); this.filter = (sourceBytes, contentType) -> { BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, sourceBytes.length())); XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput); From 50b406f3a94b9d424808fa4bd82094bae268f01c Mon Sep 17 00:00:00 2001 From: mushao999 Date: Thu, 6 Jan 2022 19:29:04 +0800 Subject: [PATCH 04/11] use map filter if exclude has wildcard --- .../index/mapper/SourceFieldMapper.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index b7b2db73593d9..57e63dd11990b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -19,10 +19,14 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; @@ -31,7 +35,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Function; public class SourceFieldMapper extends MetadataFieldMapper { public static final String NAME = "_source"; @@ -143,17 +149,30 @@ private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes) this.excludes = excludes; final boolean filtered = CollectionUtils.isEmpty(includes) == false || CollectionUtils.isEmpty(excludes) == false; if (enabled && filtered) { - final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( - Set.of(includes), - Set.of(excludes) - ); - this.filter = (sourceBytes, contentType) -> { - BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, sourceBytes.length())); - XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput); - XContentParser parser = XContentType.JSON.xContent().createParser(parserConfig, sourceBytes.streamInput()); - builder.copyCurrentStructure(parser); - return BytesReference.bytes(builder); - }; + if ((CollectionUtils.isEmpty(excludes) == false) && Arrays.stream(excludes).filter(field -> field.contains("*")).count() > 0) { + this.filter = (originalSource, contentType) -> { + Function, Map> mapFilter = XContentMapValues.filter(includes, excludes); + Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); + Map filteredSource = mapFilter.apply(mapTuple.v2()); + BytesStreamOutput bStream = new BytesStreamOutput(); + XContentType actualContentType = mapTuple.v1(); + XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource); + builder.close(); + return bStream.bytes(); + }; + } else { + final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( + Set.of(includes), + Set.of(excludes) + ); + this.filter = (originalSource, contentType) -> { + BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, originalSource.length())); + XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput); + XContentParser parser = XContentType.JSON.xContent().createParser(parserConfig, originalSource.streamInput()); + builder.copyCurrentStructure(parser); + return BytesReference.bytes(builder); + }; + } } else { this.filter = (sourceBytes, contentType) -> sourceBytes; } From 004e69da5840186e790cc85246395ce414f761b1 Mon Sep 17 00:00:00 2001 From: mushao999 Date: Tue, 11 Jan 2022 12:01:27 +0800 Subject: [PATCH 05/11] introduce XContentFieldFilter --- .../common/xcontent/XContentFieldFilter.java | 24 +++++++++++ .../common/xcontent/XContentHelper.java | 38 ++++++++++++++++ .../index/mapper/SourceFieldMapper.java | 43 ++----------------- 3 files changed, 65 insertions(+), 40 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java new file mode 100644 index 0000000000000..c56029d889ed8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.xcontent; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; + +/** + * A filter that filter fields away from source + */ +public interface XContentFieldFilter { + /** + * filter source in {@link BytesReference} format and in {@link XContentType} content type + */ + BytesReference apply(BytesReference sourceBytes, XContentType xContentType) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 1eaac03641c62..f2e38abca944c 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -15,7 +15,10 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.xcontent.DeprecationHandler; @@ -35,11 +38,13 @@ import java.io.InputStream; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; @SuppressWarnings("unchecked") public class XContentHelper { @@ -575,4 +580,37 @@ public static void writeTo(StreamOutput out, XContentType xContentType) throws I out.writeVInt(xContentType.ordinal()); } } + + /** + * Construct {@link XContentFieldFilter} using given includes and excludes + * @param includes fields to keep, wildcard supported + * @param excludes fields to remove, wildcard supported + * @return filter using {@link XContentMapValues#filter(String[], String[])} if wildcard found in excludes, otherwise return filter using {@link XContentParser} + */ + public static XContentFieldFilter newFieldFilter(String[] includes, String[] excludes) { + if ((CollectionUtils.isEmpty(excludes) == false) && Arrays.stream(excludes).filter(field -> field.contains("*")).count() > 0) { + return (originalSource, contentType) -> { + Function, Map> mapFilter = XContentMapValues.filter(includes, excludes); + Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); + Map filteredSource = mapFilter.apply(mapTuple.v2()); + BytesStreamOutput bStream = new BytesStreamOutput(); + XContentType actualContentType = mapTuple.v1(); + XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource); + builder.close(); + return bStream.bytes(); + }; + } else { + final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( + Set.of(includes), + Set.of(excludes) + ); + return (originalSource, contentType) -> { + BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, originalSource.length())); + XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput); + XContentParser parser = XContentType.JSON.xContent().createParser(parserConfig, originalSource.streamInput()); + builder.copyCurrentStructure(parser); + return BytesReference.bytes(builder); + }; + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 57e63dd11990b..4b9ec9c6ca65a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -14,37 +14,27 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.XContentFieldFilter; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; public class SourceFieldMapper extends MetadataFieldMapper { public static final String NAME = "_source"; public static final String RECOVERY_SOURCE_NAME = "_recovery_source"; public static final String CONTENT_TYPE = "_source"; - private final CheckedBiFunction filter; + private final XContentFieldFilter filter; private static final SourceFieldMapper DEFAULT = new SourceFieldMapper(Defaults.ENABLED, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY); @@ -148,34 +138,7 @@ private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes) this.includes = includes; this.excludes = excludes; final boolean filtered = CollectionUtils.isEmpty(includes) == false || CollectionUtils.isEmpty(excludes) == false; - if (enabled && filtered) { - if ((CollectionUtils.isEmpty(excludes) == false) && Arrays.stream(excludes).filter(field -> field.contains("*")).count() > 0) { - this.filter = (originalSource, contentType) -> { - Function, Map> mapFilter = XContentMapValues.filter(includes, excludes); - Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); - Map filteredSource = mapFilter.apply(mapTuple.v2()); - BytesStreamOutput bStream = new BytesStreamOutput(); - XContentType actualContentType = mapTuple.v1(); - XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource); - builder.close(); - return bStream.bytes(); - }; - } else { - final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( - Set.of(includes), - Set.of(excludes) - ); - this.filter = (originalSource, contentType) -> { - BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, originalSource.length())); - XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput); - XContentParser parser = XContentType.JSON.xContent().createParser(parserConfig, originalSource.streamInput()); - builder.copyCurrentStructure(parser); - return BytesReference.bytes(builder); - }; - } - } else { - this.filter = (sourceBytes, contentType) -> sourceBytes; - } + this.filter = enabled && filtered ? XContentHelper.newFieldFilter(includes, excludes) : (sourceBytes, contentType) -> sourceBytes; this.complete = enabled && CollectionUtils.isEmpty(includes) && CollectionUtils.isEmpty(excludes); } From 05593c7ddb8aca134a240dae1f7d3bb7034010a8 Mon Sep 17 00:00:00 2001 From: mushao999 Date: Tue, 11 Jan 2022 14:33:49 +0800 Subject: [PATCH 06/11] use XContentFieldFilter in ShardGetService --- .../common/xcontent/XContentHelper.java | 13 ++++++++++--- .../elasticsearch/index/get/ShardGetService.java | 12 +++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index f2e38abca944c..efaa1403b1536 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -532,10 +532,17 @@ public static BytesReference toXContent(ToXContent toXContent, XContentType xCon */ @Deprecated public static XContentType xContentType(BytesReference bytes) { - if (bytes.hasArray()) { - return XContentFactory.xContentType(bytes.array(), bytes.arrayOffset(), bytes.length()); - } try { + Compressor compressor = CompressorFactory.compressor(bytes); + if (compressor != null) { + InputStream compressedStreamInput = compressor.threadLocalInputStream(bytes.streamInput()); + if (compressedStreamInput.markSupported() == false) { + compressedStreamInput = new BufferedInputStream(compressedStreamInput); + } + return XContentFactory.xContentType(compressedStreamInput); + } else if (bytes.hasArray()) { + return XContentFactory.xContentType(bytes.array(), bytes.arrayOffset(), bytes.length()); + } final InputStream inputStream = bytes.streamInput(); assert inputStream.markSupported(); return XContentFactory.xContentType(inputStream); diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 8153e3e406dfb..6eb605d89e8b1 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -17,9 +17,7 @@ import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; @@ -33,7 +31,6 @@ import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -253,15 +250,12 @@ private GetResult innerGetLoadFromStoredFields( if (fetchSourceContext.fetchSource() == false) { source = null; } else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) { - Map sourceAsMap; // TODO: The source might be parsed and available in the sourceLookup but that one uses unordered maps so different. // Do we care? - Tuple> typeMapTuple = XContentHelper.convertToMap(source, true); - XContentType sourceContentType = typeMapTuple.v1(); - sourceAsMap = typeMapTuple.v2(); - sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes()); + XContentType sourceContentType = XContentHelper.xContentType(source); try { - source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap)); + source = XContentHelper.newFieldFilter(fetchSourceContext.includes(), fetchSourceContext.excludes()) + .apply(source, sourceContentType); } catch (IOException e) { throw new ElasticsearchException("Failed to get id [" + id + "] with includes/excludes set", e); } From 96deb0007d0155d3e582b31eefd9a37b5cd278fb Mon Sep 17 00:00:00 2001 From: mushao999 Date: Tue, 11 Jan 2022 22:31:27 +0800 Subject: [PATCH 07/11] fix test problem --- .../common/xcontent/XContentHelper.java | 35 ++++++++++++------- .../index/get/ShardGetService.java | 2 +- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index efaa1403b1536..c2e72ec08189a 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -523,6 +523,24 @@ public static BytesReference toXContent(ToXContent toXContent, XContentType xCon } } + public static XContentType xContentTypeMayCompressed(BytesReference bytes) { + Compressor compressor = CompressorFactory.compressor(bytes); + if (compressor != null) { + try { + InputStream compressedStreamInput = compressor.threadLocalInputStream(bytes.streamInput()); + if (compressedStreamInput.markSupported() == false) { + compressedStreamInput = new BufferedInputStream(compressedStreamInput); + } + return XContentFactory.xContentType(compressedStreamInput); + } catch (IOException e) { + assert false : "Should not happen, we're just reading bytes from memory"; + throw new UncheckedIOException(e); + } + } else { + return XContentHelper.xContentType(bytes); + } + } + /** * Guesses the content type based on the provided bytes. * @@ -532,17 +550,10 @@ public static BytesReference toXContent(ToXContent toXContent, XContentType xCon */ @Deprecated public static XContentType xContentType(BytesReference bytes) { + if (bytes.hasArray()) { + return XContentFactory.xContentType(bytes.array(), bytes.arrayOffset(), bytes.length()); + } try { - Compressor compressor = CompressorFactory.compressor(bytes); - if (compressor != null) { - InputStream compressedStreamInput = compressor.threadLocalInputStream(bytes.streamInput()); - if (compressedStreamInput.markSupported() == false) { - compressedStreamInput = new BufferedInputStream(compressedStreamInput); - } - return XContentFactory.xContentType(compressedStreamInput); - } else if (bytes.hasArray()) { - return XContentFactory.xContentType(bytes.array(), bytes.arrayOffset(), bytes.length()); - } final InputStream inputStream = bytes.streamInput(); assert inputStream.markSupported(); return XContentFactory.xContentType(inputStream); @@ -613,8 +624,8 @@ public static XContentFieldFilter newFieldFilter(String[] includes, String[] exc ); return (originalSource, contentType) -> { BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, originalSource.length())); - XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput); - XContentParser parser = XContentType.JSON.xContent().createParser(parserConfig, originalSource.streamInput()); + XContentBuilder builder = new XContentBuilder(contentType.xContent(), streamOutput); + XContentParser parser = contentType.xContent().createParser(parserConfig, originalSource.streamInput()); builder.copyCurrentStructure(parser); return BytesReference.bytes(builder); }; diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 6eb605d89e8b1..89c5747c0c87b 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -252,7 +252,7 @@ private GetResult innerGetLoadFromStoredFields( } else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) { // TODO: The source might be parsed and available in the sourceLookup but that one uses unordered maps so different. // Do we care? - XContentType sourceContentType = XContentHelper.xContentType(source); + XContentType sourceContentType = XContentHelper.xContentTypeMayCompressed(source); try { source = XContentHelper.newFieldFilter(fetchSourceContext.includes(), fetchSourceContext.excludes()) .apply(source, sourceContentType); From cc48d67a4b8180cf9c7fa0742c9bce7d822780a1 Mon Sep 17 00:00:00 2001 From: mushao999 Date: Tue, 11 Jan 2022 22:37:59 +0800 Subject: [PATCH 08/11] move newFilter from XContentHelper to XContentFieldFilter --- .../common/xcontent/XContentFieldFilter.java | 46 +++++++++++++++++++ .../common/xcontent/XContentHelper.java | 38 --------------- .../index/get/ShardGetService.java | 3 +- .../index/mapper/SourceFieldMapper.java | 5 +- 4 files changed, 51 insertions(+), 41 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java index c56029d889ed8..9b444836e89ac 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java @@ -9,9 +9,21 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; /** * A filter that filter fields away from source @@ -21,4 +33,38 @@ public interface XContentFieldFilter { * filter source in {@link BytesReference} format and in {@link XContentType} content type */ BytesReference apply(BytesReference sourceBytes, XContentType xContentType) throws IOException; + + /** + * Construct {@link XContentFieldFilter} using given includes and excludes + * + * @param includes fields to keep, wildcard supported + * @param excludes fields to remove, wildcard supported + * @return filter using {@link XContentMapValues#filter(String[], String[])} if wildcard found in excludes, otherwise return filter using {@link XContentParser} + */ + static XContentFieldFilter newFieldFilter(String[] includes, String[] excludes) { + if ((CollectionUtils.isEmpty(excludes) == false) && Arrays.stream(excludes).filter(field -> field.contains("*")).count() > 0) { + return (originalSource, contentType) -> { + Function, Map> mapFilter = XContentMapValues.filter(includes, excludes); + Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); + Map filteredSource = mapFilter.apply(mapTuple.v2()); + BytesStreamOutput bStream = new BytesStreamOutput(); + XContentType actualContentType = mapTuple.v1(); + XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource); + builder.close(); + return bStream.bytes(); + }; + } else { + final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( + Set.of(includes), + Set.of(excludes) + ); + return (originalSource, contentType) -> { + BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, originalSource.length())); + XContentBuilder builder = new XContentBuilder(contentType.xContent(), streamOutput); + XContentParser parser = contentType.xContent().createParser(parserConfig, originalSource.streamInput()); + builder.copyCurrentStructure(parser); + return BytesReference.bytes(builder); + }; + } + } } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index c2e72ec08189a..ac8f24f60ddfe 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -15,10 +15,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.xcontent.DeprecationHandler; @@ -38,13 +35,11 @@ import java.io.InputStream; import java.io.UncheckedIOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; @SuppressWarnings("unchecked") public class XContentHelper { @@ -598,37 +593,4 @@ public static void writeTo(StreamOutput out, XContentType xContentType) throws I out.writeVInt(xContentType.ordinal()); } } - - /** - * Construct {@link XContentFieldFilter} using given includes and excludes - * @param includes fields to keep, wildcard supported - * @param excludes fields to remove, wildcard supported - * @return filter using {@link XContentMapValues#filter(String[], String[])} if wildcard found in excludes, otherwise return filter using {@link XContentParser} - */ - public static XContentFieldFilter newFieldFilter(String[] includes, String[] excludes) { - if ((CollectionUtils.isEmpty(excludes) == false) && Arrays.stream(excludes).filter(field -> field.contains("*")).count() > 0) { - return (originalSource, contentType) -> { - Function, Map> mapFilter = XContentMapValues.filter(includes, excludes); - Tuple> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType); - Map filteredSource = mapFilter.apply(mapTuple.v2()); - BytesStreamOutput bStream = new BytesStreamOutput(); - XContentType actualContentType = mapTuple.v1(); - XContentBuilder builder = XContentFactory.contentBuilder(actualContentType, bStream).map(filteredSource); - builder.close(); - return bStream.bytes(); - }; - } else { - final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( - Set.of(includes), - Set.of(excludes) - ); - return (originalSource, contentType) -> { - BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, originalSource.length())); - XContentBuilder builder = new XContentBuilder(contentType.xContent(), streamOutput); - XContentParser parser = contentType.xContent().createParser(parserConfig, originalSource.streamInput()); - builder.copyCurrentStructure(parser); - return BytesReference.bytes(builder); - }; - } - } } diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 89c5747c0c87b..8a687de947dfe 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.XContentFieldFilter; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexSettings; @@ -254,7 +255,7 @@ private GetResult innerGetLoadFromStoredFields( // Do we care? XContentType sourceContentType = XContentHelper.xContentTypeMayCompressed(source); try { - source = XContentHelper.newFieldFilter(fetchSourceContext.includes(), fetchSourceContext.excludes()) + source = XContentFieldFilter.newFieldFilter(fetchSourceContext.includes(), fetchSourceContext.excludes()) .apply(source, sourceContentType); } catch (IOException e) { throw new ElasticsearchException("Failed to get id [" + id + "] with includes/excludes set", e); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 4b9ec9c6ca65a..d30c24925a6c3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.XContentFieldFilter; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; @@ -138,7 +137,9 @@ private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes) this.includes = includes; this.excludes = excludes; final boolean filtered = CollectionUtils.isEmpty(includes) == false || CollectionUtils.isEmpty(excludes) == false; - this.filter = enabled && filtered ? XContentHelper.newFieldFilter(includes, excludes) : (sourceBytes, contentType) -> sourceBytes; + this.filter = enabled && filtered + ? XContentFieldFilter.newFieldFilter(includes, excludes) + : (sourceBytes, contentType) -> sourceBytes; this.complete = enabled && CollectionUtils.isEmpty(includes) && CollectionUtils.isEmpty(excludes); } From 5e0484525d9b98c6dcc96331bc5dae4b52aaeb06 Mon Sep 17 00:00:00 2001 From: mushao999 Date: Tue, 11 Jan 2022 22:55:51 +0800 Subject: [PATCH 09/11] fix checkstyle problem --- .../org/elasticsearch/common/xcontent/XContentFieldFilter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java index 9b444836e89ac..a2d1270b104c6 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java @@ -39,7 +39,8 @@ public interface XContentFieldFilter { * * @param includes fields to keep, wildcard supported * @param excludes fields to remove, wildcard supported - * @return filter using {@link XContentMapValues#filter(String[], String[])} if wildcard found in excludes, otherwise return filter using {@link XContentParser} + * @return filter using {@link XContentMapValues#filter(String[], String[])} if wildcard found in excludes + * , otherwise return filter using {@link XContentParser} */ static XContentFieldFilter newFieldFilter(String[] includes, String[] excludes) { if ((CollectionUtils.isEmpty(excludes) == false) && Arrays.stream(excludes).filter(field -> field.contains("*")).count() > 0) { From 1ed588f45cbb7bd1fee2bc29b5ca0c8302f99ad3 Mon Sep 17 00:00:00 2001 From: mushao999 Date: Tue, 11 Jan 2022 23:19:41 +0800 Subject: [PATCH 10/11] add javadoc for xContentTypeMayCompressed --- .../org/elasticsearch/common/xcontent/XContentHelper.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index ac8f24f60ddfe..b46464f034ba0 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -518,6 +518,14 @@ public static BytesReference toXContent(ToXContent toXContent, XContentType xCon } } + /** + * Guesses the content type based on the provided bytes which may be compressed. + * + * @deprecated the content type should not be guessed except for few cases where we effectively don't know the content type. + * The REST layer should move to reading the Content-Type header instead. There are other places where auto-detection may be needed. + * This method is deprecated to prevent usages of it from spreading further without specific reasons. + */ + @Deprecated public static XContentType xContentTypeMayCompressed(BytesReference bytes) { Compressor compressor = CompressorFactory.compressor(bytes); if (compressor != null) { From 872bf8a17e16fe52d1cd06e57039007099c0fd7c Mon Sep 17 00:00:00 2001 From: mushao999 Date: Wed, 12 Jan 2022 10:49:41 +0800 Subject: [PATCH 11/11] XContentType in XContentFieldFilter may be null --- .../elasticsearch/common/xcontent/XContentFieldFilter.java | 7 ++++++- .../java/org/elasticsearch/index/get/ShardGetService.java | 5 +---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java index a2d1270b104c6..af29edd0234b2 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentFieldFilter.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -31,8 +32,9 @@ public interface XContentFieldFilter { /** * filter source in {@link BytesReference} format and in {@link XContentType} content type + * note that xContentType may be null in some case, we should guess xContentType from sourceBytes in such cases */ - BytesReference apply(BytesReference sourceBytes, XContentType xContentType) throws IOException; + BytesReference apply(BytesReference sourceBytes, @Nullable XContentType xContentType) throws IOException; /** * Construct {@link XContentFieldFilter} using given includes and excludes @@ -60,6 +62,9 @@ static XContentFieldFilter newFieldFilter(String[] includes, String[] excludes) Set.of(excludes) ); return (originalSource, contentType) -> { + if (contentType == null) { + contentType = XContentHelper.xContentTypeMayCompressed(originalSource); + } BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, originalSource.length())); XContentBuilder builder = new XContentBuilder(contentType.xContent(), streamOutput); XContentParser parser = contentType.xContent().createParser(parserConfig, originalSource.streamInput()); diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 8a687de947dfe..992afdeb99881 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentFieldFilter; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; @@ -32,7 +31,6 @@ import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.HashMap; @@ -253,10 +251,9 @@ private GetResult innerGetLoadFromStoredFields( } else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) { // TODO: The source might be parsed and available in the sourceLookup but that one uses unordered maps so different. // Do we care? - XContentType sourceContentType = XContentHelper.xContentTypeMayCompressed(source); try { source = XContentFieldFilter.newFieldFilter(fetchSourceContext.includes(), fetchSourceContext.excludes()) - .apply(source, sourceContentType); + .apply(source, null); } catch (IOException e) { throw new ElasticsearchException("Failed to get id [" + id + "] with includes/excludes set", e); }