Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Remote Store] Removing feature flag to mark feature GA #9761

Merged
merged 21 commits into from
Sep 7, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
3c183a3
Initial commit, removing all uses of feature flag
shourya035 Sep 5, 2023
4ecfe6a
Removing segrep experimental flag marking completion of intergation w…
shourya035 Sep 5, 2023
c1ba952
Merge branch 'opensearch-project:main' into rs-remove-feature-flag
shourya035 Sep 5, 2023
0073da6
Fixing IndicesServiceTests
shourya035 Sep 5, 2023
58cd589
Changing version to retain BwC
shourya035 Sep 5, 2023
f095cb9
Merge branch 'main' into rs-remove-feature-flag
shourya035 Sep 6, 2023
923094d
Rebasing and fixing issues
shourya035 Sep 6, 2023
d636a3f
Fix compile issues
shourya035 Sep 6, 2023
8aafa1c
Addressing comments
shourya035 Sep 6, 2023
9b063c2
Merge branch 'opensearch-project:main' into rs-remove-feature-flag
shourya035 Sep 6, 2023
ff8d726
Merge branch 'opensearch-project:main' into rs-remove-feature-flag
shourya035 Sep 6, 2023
a25bc2a
Merge branch 'opensearch-project:main' into rs-remove-feature-flag
shourya035 Sep 6, 2023
81a0fb0
Addressing comments
shourya035 Sep 7, 2023
842899a
Merge branch 'opensearch-project:main' into rs-remove-feature-flag
shourya035 Sep 7, 2023
925cac9
Merge branch 'opensearch-project:main' into rs-remove-feature-flag
shourya035 Sep 7, 2023
3c83dcd
Merge branch 'main' into rs-remove-feature-flag
shourya035 Sep 7, 2023
d0628b3
Addressing comments
shourya035 Sep 7, 2023
cccf424
Merge branch 'opensearch-project:main' into rs-remove-feature-flag
shourya035 Sep 7, 2023
95e4f22
Merge branch 'main' into rs-remove-feature-flag
shourya035 Sep 7, 2023
dfd3635
Fixing conflicts
shourya035 Sep 7, 2023
224b56e
Retrigger integs
shourya035 Sep 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
import org.opensearch.common.lease.Releasable;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.index.shard.IndexShard;
import org.opensearch.index.shard.IndexShardState;
import org.opensearch.indices.IndicesService;
import org.opensearch.indices.replication.SegmentReplicationBaseIT;
import org.opensearch.indices.replication.common.ReplicationType;
import org.opensearch.plugins.Plugin;
Expand Down Expand Up @@ -264,9 +264,9 @@ public void testFailStaleReplica() throws Exception {
}

public void testWithDocumentReplicationEnabledIndex() throws Exception {
assumeTrue(
"Can't create DocRep index with remote store enabled. Skipping.",
Objects.equals(featureFlagSettings().get(FeatureFlags.REMOTE_STORE, "false"), "false")
assumeFalse(
"Skipping the test as its not compatible with segment replication with remote store. Cannot create DocRep indices with Remote store enabled",
segmentReplicationWithRemoteEnabled()
);
Settings settings = Settings.builder()
.put(MAX_REPLICATION_TIME_BACKPRESSURE_SETTING.getKey(), TimeValue.timeValueMillis(500))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,8 @@ public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends Abs
protected static final String TRANSLOG_REPOSITORY_NAME = "my-translog-repo-1";
protected static final String INDEX_NAME = "remote-store-test-idx-1";

@Override
protected Settings featureFlagSettings() {
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build();
}

@Before
public void setup() {
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
FeatureFlagSetter.set(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL);
internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REPOSITORY_NAME, TRANSLOG_REPOSITORY_NAME));
shourya035 marked this conversation as resolved.
Show resolved Hide resolved
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,8 @@
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.index.IndexSettings;
import org.opensearch.indices.replication.common.ReplicationType;
import org.opensearch.test.FeatureFlagSetter;
import org.opensearch.test.OpenSearchIntegTestCase;
import org.junit.After;
import org.junit.Before;
Expand Down Expand Up @@ -51,14 +49,8 @@ protected Settings nodeSettings(int nodeOriginal) {
return builder.build();
}

@Override
protected Settings featureFlagSettings() {
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build();
}

@Before
public void setup() {
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
internalCluster().startClusterManagerOnlyNode();
assertAcked(
clusterAdmin().preparePutRepository("my-segment-repo-1")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,7 @@ public Settings indexSettings() {

@Override
protected Settings featureFlagSettings() {
return Settings.builder()
.put(super.featureFlagSettings())
.put(FeatureFlags.REMOTE_STORE, "true")
.put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true")
.build();
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true").build();
}

@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9191")
shourya035 marked this conversation as resolved.
Show resolved Hide resolved
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,7 @@ protected Settings nodeSettings(int nodeOrdinal) {

@Override
protected Settings featureFlagSettings() {
return Settings.builder()
.put(super.featureFlagSettings())
.put(FeatureFlags.REMOTE_STORE, "true")
.put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true")
.build();
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true").build();
}

@Before
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.index.IndexSettings;
import org.opensearch.indices.replication.common.ReplicationType;
Expand Down Expand Up @@ -60,11 +59,7 @@ public void teardown() {

@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(FeatureFlags.REMOTE_STORE, "true")
.put(remoteStoreClusterSettings(BASE_REMOTE_REPO))
.build();
return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(remoteStoreClusterSettings(BASE_REMOTE_REPO)).build();
}

private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,7 @@ protected Settings nodeSettings(int nodeOrdinal) {

@Override
protected Settings featureFlagSettings() {
return Settings.builder()
.put(super.featureFlagSettings())
.put(FeatureFlags.REMOTE_STORE, "true")
.put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true")
.build();
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true").build();
}

shourya035 marked this conversation as resolved.
Show resolved Hide resolved
public Settings indexSettings() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,7 @@ protected boolean segmentReplicationWithRemoteEnabled() {

@Override
protected Settings featureFlagSettings() {
return Settings.builder()
.put(super.featureFlagSettings())
.put(FeatureFlags.REMOTE_STORE, "true")
.put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true")
.build();
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true").build();
}

@Before
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,7 @@ protected boolean segmentReplicationWithRemoteEnabled() {

@Override
protected Settings featureFlagSettings() {
return Settings.builder()
.put(super.featureFlagSettings())
.put(FeatureFlags.REMOTE_STORE, "true")
.put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true")
.build();
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true").build();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
import org.opensearch.common.action.ActionFuture;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
Expand All @@ -57,7 +56,6 @@
import org.opensearch.repositories.RepositoryShardId;
import org.opensearch.repositories.blobstore.BlobStoreRepository;
import org.opensearch.snapshots.mockstore.MockRepository;
import org.opensearch.test.FeatureFlagSetter;
import org.opensearch.test.OpenSearchIntegTestCase;

import java.nio.file.Path;
Expand Down Expand Up @@ -159,7 +157,6 @@ public void testCloneSnapshotIndex() throws Exception {

public void testCloneShallowSnapshotIndex() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
final String remoteStoreRepoName = "remote-store-repo-name";
internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName));
internalCluster().startDataOnlyNode();
Expand Down Expand Up @@ -206,7 +203,6 @@ public void testCloneShallowSnapshotIndex() throws Exception {

public void testShallowCloneNameAvailability() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
final String remoteStoreRepoName = "remote-store-repo-name";
internalCluster().startClusterManagerOnlyNode(
Settings.builder().put(LARGE_SNAPSHOT_POOL_SETTINGS).put(remoteStoreClusterSettings(remoteStoreRepoName)).build()
Expand Down Expand Up @@ -243,7 +239,6 @@ public void testShallowCloneNameAvailability() throws Exception {

public void testCloneAfterRepoShallowSettingEnabled() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
final String remoteStoreRepoName = "remote-store-repo-name";
internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName));
internalCluster().startDataOnlyNode();
Expand Down Expand Up @@ -280,7 +275,6 @@ public void testCloneAfterRepoShallowSettingEnabled() throws Exception {

public void testCloneAfterRepoShallowSettingDisabled() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
final String remoteStoreRepoName = "remote-store-repo-name";
internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName));
internalCluster().startDataOnlyNode();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,7 @@
import org.opensearch.common.action.ActionFuture;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
import org.opensearch.test.FeatureFlagSetter;
import org.opensearch.test.OpenSearchIntegTestCase;

import java.nio.file.Path;
Expand All @@ -41,7 +39,6 @@ public class DeleteSnapshotIT extends AbstractSnapshotIntegTestCase {

public void testDeleteSnapshot() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME));
internalCluster().startDataOnlyNode();

Expand Down Expand Up @@ -71,7 +68,6 @@ public void testDeleteSnapshot() throws Exception {

public void testDeleteShallowCopySnapshot() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME));
internalCluster().startDataOnlyNode();

Expand Down Expand Up @@ -102,8 +98,6 @@ public void testDeleteShallowCopySnapshot() throws Exception {
// Deleting multiple shallow copy snapshots as part of single delete call with repo having only shallow copy snapshots.
public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);

internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME));
internalCluster().startDataOnlyNode();
final Client clusterManagerClient = internalCluster().clusterManagerClient();
Expand Down Expand Up @@ -148,8 +142,6 @@ public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception {
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8610")
public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);

internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME));
final String dataNode = internalCluster().startDataOnlyNode();
ensureStableCluster(2);
Expand Down Expand Up @@ -236,8 +228,6 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception {
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8610")
public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);

internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME));
internalCluster().startDataOnlyNode();
final Client clusterManagerClient = internalCluster().clusterManagerClient();
Expand Down Expand Up @@ -298,8 +288,6 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception {

public void testRemoteStoreCleanupForDeletedIndex() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);

internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME));
internalCluster().startDataOnlyNode();
final Client clusterManagerClient = internalCluster().clusterManagerClient();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING.getKey(), 0) // We have tests that check by-timestamp order
.put(FeatureFlags.REMOTE_STORE, "true")
.put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true")
.put(remoteStoreClusterSettings("remote-store-repo-name"))
.build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import org.opensearch.common.action.ActionFuture;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.core.rest.RestStatus;
Expand Down Expand Up @@ -82,11 +81,6 @@
import static org.hamcrest.Matchers.nullValue;

public class RestoreSnapshotIT extends AbstractSnapshotIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(FeatureFlags.REMOTE_STORE, "true").build();
}

public void testParallelRestoreOperations() {
String indexName1 = "testindex1";
String indexName2 = "testindex2";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
import org.opensearch.common.action.ActionFuture;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.common.util.io.IOUtils;
import org.opensearch.core.common.Strings;
import org.opensearch.core.common.unit.ByteSizeUnit;
Expand Down Expand Up @@ -77,7 +76,6 @@ protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING.getKey(), 0) // We have tests that check by-timestamp order
.put(FeatureFlags.REMOTE_STORE, "true")
.build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -972,12 +972,8 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
registerHandler.accept(new RestCatAction(catActions));
registerHandler.accept(new RestDecommissionAction());
registerHandler.accept(new RestGetDecommissionStateAction());

// Remote Store APIs
if (FeatureFlags.isEnabled(FeatureFlags.REMOTE_STORE)) {
registerHandler.accept(new RestRemoteStoreStatsAction());
registerHandler.accept(new RestRestoreRemoteStoreAction());
}
registerHandler.accept(new RestRemoteStoreStatsAction());
registerHandler.accept(new RestRestoreRemoteStoreAction());
}

@Override
Expand Down
Loading
Loading