From 14cd0e33ebfe6c84fcb09687c060e5a0f047c0a7 Mon Sep 17 00:00:00 2001 From: samuel-oci <97131656+samuel-oci@users.noreply.github.com> Date: Fri, 3 May 2024 09:31:01 -0700 Subject: [PATCH] FEATURE-13448: add use compound file setting (#13478) * add use compound file setting Signed-off-by: Samuel Herman * review feedback Signed-off-by: Samuel Herman --------- Signed-off-by: Samuel Herman --- CHANGELOG.md | 1 + .../common/settings/IndexScopedSettings.java | 1 + .../opensearch/index/engine/EngineConfig.java | 10 ++++ .../index/engine/InternalEngine.java | 2 +- .../index/engine/EngineConfigTests.java | 7 +++ .../index/engine/InternalEngineTests.java | 52 +++++++++++++++++++ 6 files changed, 72 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f013cedd82dc..a49ba7907c79d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added +- Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478)) - Constant Keyword Field ([#12285](https://github.com/opensearch-project/OpenSearch/pull/12285)) - Convert ingest processor supports ip type ([#12818](https://github.com/opensearch-project/OpenSearch/pull/12818)) - Add a counter to node stat api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768)) diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index c6c312d6b6eea..980c432774f6e 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -198,6 +198,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { EngineConfig.INDEX_CODEC_SETTING, EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING, EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS, + EngineConfig.INDEX_USE_COMPOUND_FILE, IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS, IndexSettings.DEFAULT_PIPELINE, IndexSettings.FINAL_PIPELINE, diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 8106b65bddeec..fe2ce470cda03 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -236,6 +236,12 @@ private static void doValidateCodecSettings(final String codec) { Property.Dynamic ); + public static final Setting INDEX_USE_COMPOUND_FILE = Setting.boolSetting( + "index.use_compound_file", + true, + Property.IndexScope + ); + private final TranslogConfig translogConfig; private final TranslogFactory translogFactory; @@ -494,6 +500,10 @@ public boolean isReadOnlyReplica() { return indexSettings.isSegRepEnabledOrRemoteNode() && isReadOnlyReplica; } + public boolean useCompoundFile() { + return indexSettings.getValue(INDEX_USE_COMPOUND_FILE); + } + /** * Returns the underlying startedPrimarySupplier. * @return the primary mode supplier. diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 7bacec22fc850..59165b936aec8 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -2341,7 +2341,7 @@ private IndexWriterConfig getIndexWriterConfig() { iwc.setSimilarity(engineConfig.getSimilarity()); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); - iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh + iwc.setUseCompoundFile(engineConfig.useCompoundFile()); if (config().getIndexSort() != null) { iwc.setIndexSort(config().getIndexSort()); } diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java index 9197b959a54f1..f42188d55bca3 100644 --- a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java +++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java @@ -32,6 +32,13 @@ public void setUp() throws Exception { defaultIndexSettings = IndexSettingsModule.newIndexSettings("test", defaultIndexMetadata.getSettings()); } + public void testEngineConfig_DefaultValueFoUseCompoundFile() { + EngineConfig config = new EngineConfig.Builder().indexSettings(defaultIndexSettings) + .retentionLeasesSupplier(() -> RetentionLeases.EMPTY) + .build(); + assertTrue(config.useCompoundFile()); + } + public void testEngineConfig_DefaultValueForReadOnlyEngine() { EngineConfig config = new EngineConfig.Builder().indexSettings(defaultIndexSettings) .retentionLeasesSupplier(() -> RetentionLeases.EMPTY) diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 74aef54987842..54a562642d4ab 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -342,6 +342,58 @@ public void testVerboseSegments() throws Exception { } } + public void testSegmentsWithUseCompoundFileFlag_true() throws IOException { + try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) { + ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); + engine.index(index); + engine.flush(); + final List segments = engine.segments(false); + assertThat(segments, hasSize(1)); + assertTrue(segments.get(0).compound); + boolean cfeCompoundFileFound = false; + boolean cfsCompoundFileFound = false; + for (final String fileName : store.readLastCommittedSegmentsInfo().files(true)) { + if (fileName.endsWith(".cfe")) { + cfeCompoundFileFound = true; + } + if (fileName.endsWith(".cfs")) { + cfsCompoundFileFound = true; + } + } + Assert.assertTrue(cfeCompoundFileFound); + Assert.assertTrue(cfsCompoundFileFound); + } + } + + public void testSegmentsWithUseCompoundFileFlag_false() throws IOException { + final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( + "test", + Settings.builder().put(defaultSettings.getSettings()).put(EngineConfig.INDEX_USE_COMPOUND_FILE.getKey(), false).build() + ); + try (Store store = createStore(); Engine engine = createEngine(indexSettings, store, createTempDir(), new TieredMergePolicy())) { + ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); + engine.index(index); + engine.flush(); + final List segments = engine.segments(false); + assertThat(segments, hasSize(1)); + assertFalse(segments.get(0).compound); + boolean cfeCompoundFileFound = false; + boolean cfsCompoundFileFound = false; + for (final String fileName : store.readLastCommittedSegmentsInfo().files(true)) { + if (fileName.endsWith(".cfe")) { + cfeCompoundFileFound = true; + } + if (fileName.endsWith(".cfs")) { + cfsCompoundFileFound = true; + } + } + Assert.assertFalse(cfeCompoundFileFound); + Assert.assertFalse(cfsCompoundFileFound); + } + } + public void testSegmentsWithMergeFlag() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) { ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);