From 3e9640eb4b2f6edfee6156431303ba2f8aebfaa6 Mon Sep 17 00:00:00 2001 From: rishavz_sagar Date: Wed, 22 Feb 2023 04:15:52 +0530 Subject: [PATCH] Fix last runtime millisec reset when applying index create block (#6419) Signed-off-by: Rishav Sagar Co-authored-by: Rishav Sagar --- .../decider/DiskThresholdDeciderIT.java | 35 +++++++++++++++++++ .../allocation/DiskThresholdMonitor.java | 3 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index d4537fa6a6bff..60d4a548805e2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -436,6 +436,41 @@ public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Excepti assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, hasSize(1)); } + public void testDiskMonitorResetLastRuntimeMilliSecOnlyInFirstCall() throws Exception { + final Settings settings = Settings.builder() + .put(DiskThresholdSettings.CLUSTER_CREATE_INDEX_BLOCK_AUTO_RELEASE.getKey(), false) + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), false) + .build(); + + internalCluster().startClusterManagerOnlyNode(settings); + internalCluster().startDataOnlyNodes(2, settings); + ensureStableCluster(3); + + final MockInternalClusterInfoService clusterInfoService = getMockInternalClusterInfoService(); + // Reduce disk space of all node. + clusterInfoService.setDiskUsageFunctionAndRefresh((discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, TOTAL_SPACE_BYTES, 0)); + + // Validate if cluster block is applied on the cluster + assertBusy(() -> { + ClusterState state = client().admin().cluster().prepareState().setLocal(true).get().getState(); + assertTrue(state.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); + }, 30L, TimeUnit.SECONDS); + + // User removes index create block. + Settings removeBlockSetting = Settings.builder().put(Metadata.SETTING_CREATE_INDEX_BLOCK_SETTING.getKey(), "false").build(); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(removeBlockSetting).get()); + // Free all the space + clusterInfoService.setDiskUsageFunctionAndRefresh( + (discoveryNode, fsInfoPath) -> setDiskUsage(fsInfoPath, TOTAL_SPACE_BYTES, TOTAL_SPACE_BYTES) + ); + + // Validate index create block is removed on the cluster + assertBusy(() -> { + ClusterState state = client().admin().cluster().prepareState().setLocal(true).get().getState(); + assertFalse(state.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id())); + }, 30L, TimeUnit.SECONDS); + } + private String populateNode(final String dataNodeName) throws Exception { final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); createAndPopulateIndex(indexName, dataNodeName); diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitor.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitor.java index 5dcacd51dbe2a..e531e875805a7 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitor.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitor.java @@ -375,7 +375,8 @@ public void onNewInfo(ClusterInfo info) { } // If all the nodes are breaching high disk watermark, we apply index create block to avoid red clusters. - if (nodesOverHighThreshold.size() == nodes.size()) { + if ((state.getBlocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id()) == false) + && nodesOverHighThreshold.size() == nodes.size()) { setIndexCreateBlock(listener, true); } else if (state.getBlocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id()) && diskThresholdSettings.isCreateIndexBlockAutoReleaseEnabled()) {