diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index dfe6c9820a16a..eab51427876aa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -411,6 +411,55 @@ index param - nested array within array: - match: { hits.hits.0._source.path.to.some.3.id: [ 1000, 2000 ] } +--- +index param - nested array within array - disabled second pass: + - requires: + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + synthetic_source: + enable_second_doc_parsing_pass: false + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + path: + properties: + to: + properties: + some: + synthetic_source_keep: arrays + properties: + id: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "path": [ { "to": [ { "some" : [ { "id": 10 }, { "id": [1, 3, 2] } ] }, { "some": { "id": 100 } } ] }, { "to": { "some": { "id": [1000, 2000] } } } ] }' + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.hits.0._source.name: A } + - length: { hits.hits.0._source.path.to.some: 2} + - match: { hits.hits.0._source.path.to.some.0.id: 10 } + - match: { hits.hits.0._source.path.to.some.1.id: [ 1, 3, 2] } + + --- # 112156 stored field under object with store_array_source: diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 884ce38fba391..f5276bbe49b63 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -187,6 +187,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, + IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING, // validate that built-in similarities don't get redefined diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index f3f8ce4b8e7e4..347b44a22e7c0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -652,6 +652,13 @@ public Iterator> settings() { Property.Final ); + public static final Setting SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING = Setting.boolSetting( + "index.synthetic_source.enable_second_doc_parsing_pass", + true, + Property.IndexScope, + Property.Dynamic + ); + /** * Returns true if TSDB encoding is enabled. The default is true */ @@ -821,6 +828,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingDimensionFieldsLimit; private volatile boolean skipIgnoredSourceWrite; private volatile boolean skipIgnoredSourceRead; + private volatile boolean syntheticSourceSecondDocParsingPassEnabled; private final SourceFieldMapper.Mode indexMappingSourceMode; /** @@ -982,6 +990,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); + syntheticSourceSecondDocParsingPassEnabled = scopedSettings.get(SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING); indexMappingSourceMode = scopedSettings.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING); scopedSettings.addSettingsUpdateConsumer( @@ -1070,6 +1079,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this::setSkipIgnoredSourceWrite ); scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead); + scopedSettings.addSettingsUpdateConsumer( + SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING, + this::setSyntheticSourceSecondDocParsingPassEnabled + ); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { @@ -1662,6 +1675,14 @@ private void setSkipIgnoredSourceRead(boolean value) { this.skipIgnoredSourceRead = value; } + private void setSyntheticSourceSecondDocParsingPassEnabled(boolean syntheticSourceSecondDocParsingPassEnabled) { + this.syntheticSourceSecondDocParsingPassEnabled = syntheticSourceSecondDocParsingPassEnabled; + } + + public boolean isSyntheticSourceSecondDocParsingPassEnabled() { + return syntheticSourceSecondDocParsingPassEnabled; + } + public SourceFieldMapper.Mode getIndexMappingSourceMode() { return indexMappingSourceMode; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index ac236e5a7e5fd..2eec14bd1a8d6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -111,6 +111,7 @@ public int get() { private final Set ignoredFields; private final List ignoredFieldValues; private final List ignoredFieldsMissingValues; + private final boolean inArrayScopeEnabled; private boolean inArrayScope; private final Map> dynamicMappers; @@ -143,6 +144,7 @@ private DocumentParserContext( Set ignoreFields, List ignoredFieldValues, List ignoredFieldsWithNoSource, + boolean inArrayScopeEnabled, boolean inArrayScope, Map> dynamicMappers, Map dynamicObjectMappers, @@ -164,6 +166,7 @@ private DocumentParserContext( this.ignoredFields = ignoreFields; this.ignoredFieldValues = ignoredFieldValues; this.ignoredFieldsMissingValues = ignoredFieldsWithNoSource; + this.inArrayScopeEnabled = inArrayScopeEnabled; this.inArrayScope = inArrayScope; this.dynamicMappers = dynamicMappers; this.dynamicObjectMappers = dynamicObjectMappers; @@ -188,6 +191,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.ignoredFields, in.ignoredFieldValues, in.ignoredFieldsMissingValues, + in.inArrayScopeEnabled, in.inArrayScope, in.dynamicMappers, in.dynamicObjectMappers, @@ -219,6 +223,7 @@ protected DocumentParserContext( new HashSet<>(), new ArrayList<>(), new ArrayList<>(), + mappingParserContext.getIndexSettings().isSyntheticSourceSecondDocParsingPassEnabled(), false, new HashMap<>(), new HashMap<>(), @@ -371,7 +376,7 @@ public final Collection getIgnoredFieldsMiss * Applies to synthetic source only. */ public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException { - if (canAddIgnoredField() && mapper instanceof ObjectMapper) { + if (canAddIgnoredField() && mapper instanceof ObjectMapper && inArrayScopeEnabled) { boolean isNested = mapper instanceof NestedObjectMapper; if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) { DocumentParserContext subcontext = switchParser(parser()); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 357e1bca38e8f..ef03fd0ba6f0e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -333,6 +333,7 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME); replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); + replicatedSettings.add(IndexSettings.SYNTHETIC_SOURCE_SECOND_DOC_PARSING_PASS_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING);