diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-security/roles.yml b/x-pack/plugin/ccr/qa/multi-cluster-with-security/roles.yml index ee9bfbf9cd9b2..700a2416c6609 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster-with-security/roles.yml +++ b/x-pack/plugin/ccr/qa/multi-cluster-with-security/roles.yml @@ -2,7 +2,7 @@ ccruser: cluster: - manage_ccr indices: - - names: [ 'index1', 'index3' ] + - names: [ 'allowed-index' ] privileges: - monitor - read diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java b/x-pack/plugin/ccr/qa/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java index 7b9e54ff04729..e8f4abd0bc849 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java @@ -60,37 +60,28 @@ protected boolean preserveClusterUponCompletion() { public void testFollowIndex() throws Exception { final int numDocs = 16; - final String indexName1 = "index1"; - final String indexName2 = "index2"; + final String allowedIndex = "allowed-index"; + final String unallowedIndex = "unallowed-index"; if (runningAgainstLeaderCluster) { logger.info("Running against leader cluster"); - Settings indexSettings = Settings.builder() - .put("index.soft_deletes.enabled", true) - .build(); - createIndex(indexName1, indexSettings); - createIndex(indexName2, indexSettings); + Settings indexSettings = Settings.builder().put("index.soft_deletes.enabled", true).build(); + createIndex(allowedIndex, indexSettings); + createIndex(unallowedIndex, indexSettings); for (int i = 0; i < numDocs; i++) { logger.info("Indexing doc [{}]", i); - index(indexName1, Integer.toString(i), "field", i); + index(allowedIndex, Integer.toString(i), "field", i); } for (int i = 0; i < numDocs; i++) { logger.info("Indexing doc [{}]", i); - index(indexName2, Integer.toString(i), "field", i); + index(unallowedIndex, Integer.toString(i), "field", i); } - refresh(indexName1); - verifyDocuments(adminClient(), indexName1, numDocs); + refresh(allowedIndex); + verifyDocuments(adminClient(), allowedIndex, numDocs); } else { - logger.info("Running against follow cluster"); - Settings indexSettings = Settings.builder() - .put("index.xpack.ccr.following_index", true) - .build(); - // TODO: remove mapping here when ccr syncs mappings too - createIndex(indexName1, indexSettings, "\"doc\": { \"properties\": { \"field\": { \"type\": \"long\" }}}"); - ensureYellow(indexName1); - followIndex("leader_cluster:" + indexName1, indexName1); - assertBusy(() -> verifyDocuments(client(), indexName1, numDocs)); + createAndFollowIndex("leader_cluster:" + allowedIndex, allowedIndex); + assertBusy(() -> verifyDocuments(client(), allowedIndex, numDocs)); assertThat(countCcrNodeTasks(), equalTo(1)); - assertOK(client().performRequest("POST", "/" + indexName1 + "/_xpack/ccr/_unfollow")); + assertOK(client().performRequest("POST", "/" + allowedIndex + "/_xpack/ccr/_unfollow")); // Make sure that there are no other ccr relates operations running: assertBusy(() -> { Map clusterState = toMap(adminClient().performRequest("GET", "/_cluster/state")); @@ -98,49 +89,10 @@ public void testFollowIndex() throws Exception { assertThat(tasks.size(), equalTo(0)); assertThat(countCcrNodeTasks(), equalTo(0)); }); - - // TODO: remove mapping here when ccr syncs mappings too - createIndex(indexName2, indexSettings, "\"doc\": { \"properties\": { \"field\": { \"type\": \"long\" }}}"); - ensureYellow(indexName2); - followIndex("leader_cluster:" + indexName2, indexName2); - // Verify that nothing has been replicated and no node tasks are running - // These node tasks should have been failed due to the fact that the user - // has no sufficient priviledges. - assertBusy(() -> assertThat(countCcrNodeTasks(), equalTo(0))); - verifyDocuments(adminClient(), indexName2, 0); - } - } - - public void testCreateAndFollowIndex() throws Exception { - final int numDocs = 16; - final String indexName1 = "index3"; - final String indexName2 = "index4"; - if (runningAgainstLeaderCluster) { - logger.info("Running against leader cluster"); - Settings indexSettings = Settings.builder() - .put("index.soft_deletes.enabled", true) - .build(); - createIndex(indexName1, indexSettings); - createIndex(indexName2, indexSettings); - for (int i = 0; i < numDocs; i++) { - logger.info("Indexing doc [{}]", i); - index(indexName1, Integer.toString(i), "field", i); - } - for (int i = 0; i < numDocs; i++) { - logger.info("Indexing doc [{}]", i); - index(indexName2, Integer.toString(i), "field", i); - } - refresh(indexName1); - verifyDocuments(adminClient(), indexName1, numDocs); - } else { - logger.info("Running against follow cluster"); - Settings indexSettings = Settings.builder() - .put("index.xpack.ccr.following_index", true) - .build(); - createAndFollowIndex("leader_cluster:" + indexName1, indexName1); - assertBusy(() -> verifyDocuments(client(), indexName1, numDocs)); + + followIndex("leader_cluster:" + allowedIndex, allowedIndex); assertThat(countCcrNodeTasks(), equalTo(1)); - assertOK(client().performRequest("POST", "/" + indexName1 + "/_xpack/ccr/_unfollow")); + assertOK(client().performRequest("POST", "/" + allowedIndex + "/_xpack/ccr/_unfollow")); // Make sure that there are no other ccr relates operations running: assertBusy(() -> { Map clusterState = toMap(adminClient().performRequest("GET", "/_cluster/state")); @@ -148,13 +100,17 @@ public void testCreateAndFollowIndex() throws Exception { assertThat(tasks.size(), equalTo(0)); assertThat(countCcrNodeTasks(), equalTo(0)); }); - - createAndFollowIndex("leader_cluster:" + indexName2, indexName2); + + createAndFollowIndex("leader_cluster:" + unallowedIndex, unallowedIndex); // Verify that nothing has been replicated and no node tasks are running // These node tasks should have been failed due to the fact that the user // has no sufficient priviledges. assertBusy(() -> assertThat(countCcrNodeTasks(), equalTo(0))); - verifyDocuments(adminClient(), indexName2, 0); + verifyDocuments(adminClient(), unallowedIndex, 0); + + followIndex("leader_cluster:" + unallowedIndex, unallowedIndex); + assertBusy(() -> assertThat(countCcrNodeTasks(), equalTo(0))); + verifyDocuments(adminClient(), unallowedIndex, 0); } } @@ -236,13 +192,4 @@ protected static void createIndex(String name, Settings settings, String mapping + ", \"mappings\" : {" + mapping + "} }", ContentType.APPLICATION_JSON))); } - private static void ensureYellow(String index) throws IOException { - Map params = new HashMap<>(); - params.put("wait_for_status", "yellow"); - params.put("wait_for_no_relocating_shards", "true"); - params.put("timeout", "30s"); - params.put("level", "shards"); - assertOK(adminClient().performRequest("GET", "_cluster/health/" + index, params)); - } - } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java index 8027e68bd99f7..f4736d48fefc9 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java @@ -234,6 +234,8 @@ private void createFollowIndex(IndexMetaData leaderIndexMetaData, Request reques } }, listener::onFailure); + // Can't use create index api here, because then index templates can alter the mappings / settings. + // And index templates could introduce settings / mappings that are incompatible with the leader index. clusterService.submitStateUpdateTask("follow_index_action", new AckedClusterStateUpdateTask(request, handler) { @Override @@ -249,26 +251,21 @@ public ClusterState execute(ClusterState currentState) throws Exception { } MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); - if (request.getFollowRequest().getFollowIndex() != null) { - IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndexMetaData); - - Settings.Builder settingsBuilder = Settings.builder(); - settingsBuilder.put(leaderIndexMetaData.getSettings()); - // TODO: do we want leader and follow index to have the same UUID? - // (Overwriting UUID here, because otherwise we can't follow indices in the same cluster) - settingsBuilder.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - settingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getFollowRequest().getFollowIndex()); - settingsBuilder.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); - - imdBuilder.settings(settingsBuilder); - imdBuilder.index(request.getFollowRequest().getFollowIndex()); - for (int shardId = 0 ; shardId < leaderIndexMetaData.getNumberOfShards(); shardId++) { - imdBuilder.putInSyncAllocationIds(shardId, new HashSet<>()); - } - mdBuilder.put(imdBuilder.build(), false); - } else { - mdBuilder.put(leaderIndexMetaData, false); + IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndexMetaData); + + Settings.Builder settingsBuilder = Settings.builder(); + settingsBuilder.put(leaderIndexMetaData.getSettings()); + // Overwriting UUID here, because otherwise we can't follow indices in the same cluster + settingsBuilder.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); + settingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getFollowRequest().getFollowIndex()); + settingsBuilder.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); + + imdBuilder.settings(settingsBuilder); + imdBuilder.index(request.getFollowRequest().getFollowIndex()); + for (int shardId = 0 ; shardId < leaderIndexMetaData.getNumberOfShards(); shardId++) { + imdBuilder.putInSyncAllocationIds(shardId, new HashSet<>()); } + mdBuilder.put(imdBuilder.build(), false); ClusterState.Builder builder = ClusterState.builder(currentState); builder.metaData(mdBuilder.build()); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/ShardChangesIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/ShardChangesIT.java index 28a17763c59e0..b8b3d4450c073 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/ShardChangesIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/ShardChangesIT.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; @@ -178,7 +177,7 @@ public void testFollowIndex() throws Exception { assertBusy(assertExpectedDocumentRunnable(i)); } - unFollowIndex("index2"); + unfollowIndex("index2"); client().execute(FollowIndexAction.INSTANCE, followRequest).get(); final int secondBatchNumDocs = randomIntBetween(2, 64); for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { @@ -200,7 +199,7 @@ public void testFollowIndex() throws Exception { for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { assertBusy(assertExpectedDocumentRunnable(i)); } - unFollowIndex("index2"); + unfollowIndex("index2"); } public void testFollowIndexWithNestedField() throws Exception { @@ -318,7 +317,7 @@ private CheckedRunnable assertTask(final int numberOfPrimaryShards, f }; } - private void unFollowIndex(String index) throws Exception { + private void unfollowIndex(String index) throws Exception { final UnfollowIndexAction.Request unfollowRequest = new UnfollowIndexAction.Request(); unfollowRequest.setFollowIndex(index); client().execute(UnfollowIndexAction.INSTANCE, unfollowRequest).get();