diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java new file mode 100644 index 0000000000000..553a3d321d7f7 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java @@ -0,0 +1,133 @@ +package org.elasticsearch.benchmark.search.fetch.subphase; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; +import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FetchSourcePhase; +import org.elasticsearch.search.lookup.SourceLookup; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +@Fork(1) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +public class FetchSourcePhaseBenchmark { + private BytesReference sourceBytes; + private FetchSourceContext fetchContext; + private Set includesSet; + private Set excludesSet; + private FilterPath[] includesFilters; + private FilterPath[] excludesFilters; + + @Param({ "tiny", "short", "one_4k_field", "one_4m_field" }) + private String source; + @Param({ "message" }) + private String includes; + @Param({ "" }) + private String excludes; + + @Setup + public void setup() throws IOException { + switch (source) { + case "tiny": + sourceBytes = new BytesArray("{\"message\": \"short\"}"); + break; + case "short": + sourceBytes = read300BytesExample(); + break; + case "one_4k_field": + sourceBytes = buildBigExample(String.join("", Collections.nCopies(1024, "huge"))); + break; + case "one_4m_field": + sourceBytes = buildBigExample(String.join("", Collections.nCopies(1024 * 1024, "huge"))); + break; + default: + throw new IllegalArgumentException("Unknown source [" + source + "]"); + } + fetchContext = new FetchSourceContext( + true, + Strings.splitStringByCommaToArray(includes), + Strings.splitStringByCommaToArray(excludes) + ); + includesSet = org.elasticsearch.core.Set.of(fetchContext.includes()); + excludesSet = org.elasticsearch.core.Set.of(fetchContext.excludes()); + includesFilters = FilterPath.compile(includesSet); + excludesFilters = FilterPath.compile(excludesSet); + } + + private BytesReference read300BytesExample() throws IOException { + return Streams.readFully(FetchSourcePhaseBenchmark.class.getResourceAsStream("300b_example.json")); + } + + private BytesReference buildBigExample(String extraText) throws IOException { + String bigger = read300BytesExample().utf8ToString(); + bigger = "{\"huge\": \"" + extraText + "\"," + bigger.substring(1); + return new BytesArray(bigger); + } + + @Benchmark + public BytesReference filterObjects() throws IOException { + SourceLookup lookup = new SourceLookup(); + lookup.setSource(sourceBytes); + Object value = lookup.filter(fetchContext); + return FetchSourcePhase.objectToBytes(value, XContentType.JSON, Math.min(1024, lookup.internalSourceRef().length())); + } + + @Benchmark + public BytesReference filterXContentOnParser() throws IOException { + BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, sourceBytes.length())); + XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput); + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + sourceBytes.streamInput(), + includesFilters, + excludesFilters + ) + ) { + builder.copyCurrentStructure(parser); + return BytesReference.bytes(builder); + } + } + + @Benchmark + public BytesReference filterXContentOnBuilder() throws IOException { + BytesStreamOutput streamOutput = new BytesStreamOutput(Math.min(1024, sourceBytes.length())); + XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), streamOutput, includesSet, excludesSet); + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, sourceBytes.streamInput()) + ) { + builder.copyCurrentStructure(parser); + return BytesReference.bytes(builder); + } + } +} diff --git a/benchmarks/src/main/resources/org/elasticsearch/benchmark/search/fetch/subphase/300b_example.json b/benchmarks/src/main/resources/org/elasticsearch/benchmark/search/fetch/subphase/300b_example.json new file mode 100644 index 0000000000000..8112244c213e8 --- /dev/null +++ b/benchmarks/src/main/resources/org/elasticsearch/benchmark/search/fetch/subphase/300b_example.json @@ -0,0 +1,20 @@ +{ + "@timestamp": "2099-11-15T14:12:12", + "http": { + "request": { + "method": "get" + }, + "response": { + "bytes": 1070000, + "status_code": 200 + }, + "version": "1.1" + }, + "message": "GET /search HTTP/1.1 200 1070000", + "source": { + "ip": "192.168.0.1" + }, + "user": { + "id": "user" + } +} \ No newline at end of file diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java index 79317f1a31f86..76b3456416714 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java @@ -8,6 +8,8 @@ package org.elasticsearch.common.xcontent; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -56,6 +58,17 @@ XContentParser createParser(NamedXContentRegistry xContentRegistry, XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, InputStream is) throws IOException; + /** + * Creates a parser over the provided input stream. + */ + XContentParser createParser( + NamedXContentRegistry xContentRegistry, + DeprecationHandler deprecationHandler, + InputStream is, + FilterPath[] includes, + FilterPath[] excludes + ) throws IOException; + /** * Creates a parser over the provided bytes. */ diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java index 631f16dcc818e..1ba38bd8d5de6 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.dataformat.cbor.CBORFactory; + import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -20,6 +21,7 @@ import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -79,6 +81,23 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, return new CborXContentParser(xContentRegistry, deprecationHandler, cborFactory.createParser(is)); } + @Override + public XContentParser createParser( + NamedXContentRegistry xContentRegistry, + DeprecationHandler deprecationHandler, + InputStream is, + FilterPath[] includes, + FilterPath[] excludes + ) throws IOException { + return new CborXContentParser( + xContentRegistry, + deprecationHandler, + cborFactory.createParser(is), + includes, + excludes + ); + } + @Override public XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, byte[] data) throws IOException { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java index 11c03f75d1f0e..6ae0f4b997eab 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java @@ -9,10 +9,12 @@ package org.elasticsearch.common.xcontent.cbor; import com.fasterxml.jackson.core.JsonParser; + import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContentParser; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; public class CborXContentParser extends JsonXContentParser { @@ -21,6 +23,16 @@ public CborXContentParser(NamedXContentRegistry xContentRegistry, super(xContentRegistry, deprecationHandler, parser); } + public CborXContentParser( + NamedXContentRegistry xContentRegistry, + DeprecationHandler deprecationHandler, + JsonParser parser, + FilterPath[] includes, + FilterPath[] excludes + ) { + super(xContentRegistry, deprecationHandler, parser, includes, excludes); + } + @Override public XContentType contentType() { return XContentType.CBOR; diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java index 8996c783f9fd9..f9d9de4ad0e8f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; + import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -19,6 +20,7 @@ import org.elasticsearch.common.xcontent.XContentGenerator; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -80,6 +82,23 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, return new JsonXContentParser(xContentRegistry, deprecationHandler, jsonFactory.createParser(is)); } + @Override + public XContentParser createParser( + NamedXContentRegistry xContentRegistry, + DeprecationHandler deprecationHandler, + InputStream is, + FilterPath[] include, + FilterPath[] exclude + ) throws IOException { + return new JsonXContentParser( + xContentRegistry, + deprecationHandler, + jsonFactory.createParser(is), + include, + exclude + ); + } + @Override public XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, byte[] data) throws IOException { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java index d07a0915bd3f9..a99f3cbbbcd8e 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java @@ -11,11 +11,15 @@ import com.fasterxml.jackson.core.JsonLocation; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.core.filter.FilteringParserDelegate; + import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.AbstractXContentParser; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; +import org.elasticsearch.common.xcontent.support.filtering.FilterPathBasedFilter; import org.elasticsearch.core.internal.io.IOUtils; import java.io.IOException; @@ -31,6 +35,30 @@ public JsonXContentParser(NamedXContentRegistry xContentRegistry, this.parser = parser; } + public JsonXContentParser( + NamedXContentRegistry xContentRegistry, + DeprecationHandler deprecationHandler, + JsonParser parser, + FilterPath[] include, + FilterPath[] exclude + ) { + super(xContentRegistry, deprecationHandler); + JsonParser filtered = parser; + if (exclude != null) { + for (FilterPath e : exclude) { + if (e.hasDoubleWildcard()) { + // Fixed in Jackson 2.13 - https://github.com/FasterXML/jackson-core/issues/700 + throw new UnsupportedOperationException("double wildcards are not supported in filtered excludes"); + } + } + filtered = new FilteringParserDelegate(filtered, new FilterPathBasedFilter(exclude, false), true, true); + } + if (include != null) { + filtered = new FilteringParserDelegate(filtered, new FilterPathBasedFilter(include, true), true, true); + } + this.parser = filtered; + } + @Override public XContentType contentType() { return XContentType.JSON; diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java index 2f8018decbce4..594de4af24dd3 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.fasterxml.jackson.dataformat.smile.SmileGenerator; + import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -20,6 +21,7 @@ import org.elasticsearch.common.xcontent.XContentGenerator; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -81,6 +83,23 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, return new SmileXContentParser(xContentRegistry, deprecationHandler, smileFactory.createParser(is)); } + @Override + public XContentParser createParser( + NamedXContentRegistry xContentRegistry, + DeprecationHandler deprecationHandler, + InputStream is, + FilterPath[] include, + FilterPath[] exclude + ) throws IOException { + return new SmileXContentParser( + xContentRegistry, + deprecationHandler, + smileFactory.createParser(is), + include, + exclude + ); + } + @Override public XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, byte[] data) throws IOException { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java index c8f988df1bdf6..5ddafd324906c 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java @@ -9,10 +9,12 @@ package org.elasticsearch.common.xcontent.smile; import com.fasterxml.jackson.core.JsonParser; + import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContentParser; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; public class SmileXContentParser extends JsonXContentParser { @@ -21,6 +23,16 @@ public SmileXContentParser(NamedXContentRegistry xContentRegistry, super(xContentRegistry, deprecationHandler, parser); } + public SmileXContentParser( + NamedXContentRegistry xContentRegistry, + DeprecationHandler deprecationHandler, + JsonParser parser, + FilterPath[] include, + FilterPath[] exclude + ) { + super(xContentRegistry, deprecationHandler, parser, include, exclude); + } + @Override public XContentType contentType() { return XContentType.SMILE; diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java index b500ee030d33e..d42b160c0ef63 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPath.java @@ -52,6 +52,13 @@ boolean isDoubleWildcard() { return doubleWildcard; } + public boolean hasDoubleWildcard() { + if (filter == null) { + return false; + } + return filter.indexOf("**") >= 0; + } + boolean isSimpleWildcard() { return simpleWildcard; } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java index 377a93e319faa..95a5ca520a380 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathBasedFilter.java @@ -21,6 +21,10 @@ public class FilterPathBasedFilter extends TokenFilter { * or value matches one of the filter paths. */ private static final TokenFilter MATCHING = new TokenFilter() { + @Override + public String toString() { + return "MATCHING"; + } }; /** @@ -28,6 +32,10 @@ public class FilterPathBasedFilter extends TokenFilter { * property names/values matches one of the filter paths. */ private static final TokenFilter NO_MATCHING = new TokenFilter() { + @Override + public String toString() { + return "NO_MATCHING"; + } }; private final FilterPath[] filters; diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java index 20bb858d2f0a9..22993880c6941 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -18,6 +19,7 @@ import org.elasticsearch.common.xcontent.XContentGenerator; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -74,6 +76,23 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, return new YamlXContentParser(xContentRegistry, deprecationHandler, yamlFactory.createParser(is)); } + @Override + public XContentParser createParser( + NamedXContentRegistry xContentRegistry, + DeprecationHandler deprecationHandler, + InputStream is, + FilterPath[] includes, + FilterPath[] excludes + ) throws IOException { + return new YamlXContentParser( + xContentRegistry, + deprecationHandler, + yamlFactory.createParser(is), + includes, + excludes + ); + } + @Override public XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, byte[] data) throws IOException { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java index 1b208e87d27c6..b55fd61ef415e 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java @@ -9,10 +9,12 @@ package org.elasticsearch.common.xcontent.yaml; import com.fasterxml.jackson.core.JsonParser; + import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContentParser; +import org.elasticsearch.common.xcontent.support.filtering.FilterPath; public class YamlXContentParser extends JsonXContentParser { @@ -21,6 +23,16 @@ public YamlXContentParser(NamedXContentRegistry xContentRegistry, super(xContentRegistry, deprecationHandler, parser); } + public YamlXContentParser( + NamedXContentRegistry xContentRegistry, + DeprecationHandler deprecationHandler, + JsonParser parser, + FilterPath[] includes, + FilterPath[] excludes + ) { + super(xContentRegistry, deprecationHandler, parser, includes, excludes); + } + @Override public XContentType contentType() { return XContentType.YAML; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java index ba9c5b0d9ab4a..bc7863d5bdf48 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; @@ -72,20 +73,7 @@ private void hitExecute(FetchSourceContext fetchSourceContext, HitContext hitCon try { final int initialCapacity = nestedHit ? 1024 : Math.min(1024, source.internalSourceRef().length()); - BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity); - XContentBuilder builder = new XContentBuilder(source.sourceContentType().xContent(), streamOutput); - if (value != null) { - builder.value(value); - } else { - // This happens if the source filtering could not find the specified in the _source. - // Just doing `builder.value(null)` is valid, but the xcontent validation can't detect what format - // it is. In certain cases, for example response serialization we fail if no xcontent type can't be - // detected. So instead we just return an empty top level object. Also this is in inline with what was - // being return in this situation in 5.x and earlier. - builder.startObject(); - builder.endObject(); - } - hitContext.hit().sourceRef(BytesReference.bytes(builder)); + hitContext.hit().sourceRef(objectToBytes(value, source.sourceContentType(), initialCapacity)); } catch (IOException e) { throw new ElasticsearchException("Error filtering source", e); } @@ -102,6 +90,23 @@ private static boolean containsFilters(FetchSourceContext context) { return context.includes().length != 0 || context.excludes().length != 0; } + public static BytesReference objectToBytes(Object value, XContentType xContentType, int initialCapacity) throws IOException { + BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity); + XContentBuilder builder = new XContentBuilder(xContentType.xContent(), streamOutput); + if (value != null) { + builder.value(value); + } else { + // This happens if the source filtering could not find the specified in the _source. + // Just doing `builder.value(null)` is valid, but the xcontent validation can't detect what format + // it is. In certain cases, for example response serialization we fail if no xcontent type can't be + // detected. So instead we just return an empty top level object. Also this is in inline with what was + // being return in this situation in 5.x and earlier. + builder.startObject(); + builder.endObject(); + } + return BytesReference.bytes(builder); + } + @SuppressWarnings("unchecked") private Map getNestedSource(Map sourceAsMap, HitContext hitContext) { for (SearchHit.NestedIdentity o = hitContext.hit().getNestedIdentity(); o != null; o = o.getChild()) { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index a97309f04ff22..828fe4f200b08 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.xcontent.support.AbstractFilteringTestCase; import java.io.IOException; +import java.util.Arrays; import java.util.Set; import static java.util.Collections.emptySet; @@ -30,8 +31,8 @@ public abstract class AbstractXContentFilteringTestCase extends AbstractFilteringTestCase { - protected final void testFilter(Builder expected, Builder actual, Set includes, Set excludes) throws IOException { - assertFilterResult(expected.apply(createBuilder()), actual.apply(createBuilder(includes, excludes))); + protected final void testFilter(Builder expected, Builder sample, Set includes, Set excludes) throws IOException { + assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes)); } protected abstract void assertFilterResult(XContentBuilder expected, XContentBuilder actual); @@ -47,8 +48,51 @@ private XContentBuilder createBuilder() throws IOException { return XContentBuilder.builder(getXContentType().xContent()); } - private XContentBuilder createBuilder(Set includes, Set excludes) throws IOException { - return XContentBuilder.builder(getXContentType().xContent(), includes, excludes); + private XContentBuilder filter(Builder sample, Set includes, Set excludes) throws IOException { + if (randomBoolean()) { + return filterOnBuilder(sample, includes, excludes); + } + FilterPath[] excludesFilter = FilterPath.compile(excludes); + if (excludesFilter != null && Arrays.stream(excludesFilter).anyMatch(FilterPath::hasDoubleWildcard)) { + /* + * If there are any double wildcard filters the parser based + * filtering produced weird invalid json. Just field names + * and no objects?! Weird. Anyway, we can't use it. + */ + return filterOnBuilder(sample, includes, excludes); + } + FilterPath[] includesFilter = FilterPath.compile(includes); + return filterOnParser(sample, includesFilter, excludesFilter); + } + + private XContentBuilder filterOnBuilder(Builder sample, Set includes, Set excludes) throws IOException { + return sample.apply(XContentBuilder.builder(getXContentType().xContent(), includes, excludes)); + } + + private XContentBuilder filterOnParser(Builder sample, FilterPath[] includes, FilterPath[] excludes) throws IOException { + try (XContentBuilder builtSample = sample.apply(createBuilder())) { + BytesReference sampleBytes = BytesReference.bytes(builtSample); + try ( + XContentParser parser = getXContentType().xContent() + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + sampleBytes.streamInput(), + includes, + excludes + ); + ) { + XContentBuilder result = createBuilder(); + if (sampleBytes.get(sampleBytes.length() - 1) == '\n') { + result.lfAtEnd(); + } + if (parser.nextToken() == null) { + // If the filter removed everything then emit an open/close + return result.startObject().endObject(); + } + return result.copyCurrentStructure(parser); + } + } } public void testSingleFieldObject() throws IOException {