Skip to content

Commit

Permalink
iter
Browse files Browse the repository at this point in the history
  • Loading branch information
@martijnvg
martijnvg committed May 22, 2018
1 parent 139977e commit 6ef4681
Show file tree
Hide file tree
Showing 4 changed files with 42 additions and 99 deletions.
2 changes: 1 addition & 1 deletion x-pack/plugin/ccr/qa/multi-cluster-with-security/roles.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ ccruser:
cluster:
- manage_ccr
indices:
- names: [ 'index1', 'index3' ]
- names: [ 'allowed-index' ]
privileges:
- monitor
- read
Expand Down
97 changes: 22 additions & 75 deletions ...luster-with-security/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,101 +60,57 @@ protected boolean preserveClusterUponCompletion() {

public void testFollowIndex() throws Exception {
final int numDocs = 16;
final String indexName1 = "index1";
final String indexName2 = "index2";
final String allowedIndex = "allowed-index";
final String unallowedIndex = "unallowed-index";
if (runningAgainstLeaderCluster) {
logger.info("Running against leader cluster");
Settings indexSettings = Settings.builder()
.put("index.soft_deletes.enabled", true)
.build();
createIndex(indexName1, indexSettings);
createIndex(indexName2, indexSettings);
Settings indexSettings = Settings.builder().put("index.soft_deletes.enabled", true).build();
createIndex(allowedIndex, indexSettings);
createIndex(unallowedIndex, indexSettings);
for (int i = 0; i < numDocs; i++) {
logger.info("Indexing doc [{}]", i);
index(indexName1, Integer.toString(i), "field", i);
index(allowedIndex, Integer.toString(i), "field", i);
}
for (int i = 0; i < numDocs; i++) {
logger.info("Indexing doc [{}]", i);
index(indexName2, Integer.toString(i), "field", i);
index(unallowedIndex, Integer.toString(i), "field", i);
}
refresh(indexName1);
verifyDocuments(adminClient(), indexName1, numDocs);
refresh(allowedIndex);
verifyDocuments(adminClient(), allowedIndex, numDocs);
} else {
logger.info("Running against follow cluster");
Settings indexSettings = Settings.builder()
.put("index.xpack.ccr.following_index", true)
.build();
// TODO: remove mapping here when ccr syncs mappings too
createIndex(indexName1, indexSettings, "\"doc\": { \"properties\": { \"field\": { \"type\": \"long\" }}}");
ensureYellow(indexName1);
followIndex("leader_cluster:" + indexName1, indexName1);
assertBusy(() -> verifyDocuments(client(), indexName1, numDocs));
createAndFollowIndex("leader_cluster:" + allowedIndex, allowedIndex);
assertBusy(() -> verifyDocuments(client(), allowedIndex, numDocs));
assertThat(countCcrNodeTasks(), equalTo(1));
assertOK(client().performRequest("POST", "/" + indexName1 + "/_xpack/ccr/_unfollow"));
assertOK(client().performRequest("POST", "/" + allowedIndex + "/_xpack/ccr/_unfollow"));
// Make sure that there are no other ccr relates operations running:
assertBusy(() -> {
Map<String, Object> clusterState = toMap(adminClient().performRequest("GET", "/_cluster/state"));
List<?> tasks = (List<?>) XContentMapValues.extractValue("metadata.persistent_tasks.tasks", clusterState);
assertThat(tasks.size(), equalTo(0));
assertThat(countCcrNodeTasks(), equalTo(0));
});

// TODO: remove mapping here when ccr syncs mappings too
createIndex(indexName2, indexSettings, "\"doc\": { \"properties\": { \"field\": { \"type\": \"long\" }}}");
ensureYellow(indexName2);
followIndex("leader_cluster:" + indexName2, indexName2);
// Verify that nothing has been replicated and no node tasks are running
// These node tasks should have been failed due to the fact that the user
// has no sufficient priviledges.
assertBusy(() -> assertThat(countCcrNodeTasks(), equalTo(0)));
verifyDocuments(adminClient(), indexName2, 0);
}
}

public void testCreateAndFollowIndex() throws Exception {
final int numDocs = 16;
final String indexName1 = "index3";
final String indexName2 = "index4";
if (runningAgainstLeaderCluster) {
logger.info("Running against leader cluster");
Settings indexSettings = Settings.builder()
.put("index.soft_deletes.enabled", true)
.build();
createIndex(indexName1, indexSettings);
createIndex(indexName2, indexSettings);
for (int i = 0; i < numDocs; i++) {
logger.info("Indexing doc [{}]", i);
index(indexName1, Integer.toString(i), "field", i);
}
for (int i = 0; i < numDocs; i++) {
logger.info("Indexing doc [{}]", i);
index(indexName2, Integer.toString(i), "field", i);
}
refresh(indexName1);
verifyDocuments(adminClient(), indexName1, numDocs);
} else {
logger.info("Running against follow cluster");
Settings indexSettings = Settings.builder()
.put("index.xpack.ccr.following_index", true)
.build();
createAndFollowIndex("leader_cluster:" + indexName1, indexName1);
assertBusy(() -> verifyDocuments(client(), indexName1, numDocs));

followIndex("leader_cluster:" + allowedIndex, allowedIndex);
assertThat(countCcrNodeTasks(), equalTo(1));
assertOK(client().performRequest("POST", "/" + indexName1 + "/_xpack/ccr/_unfollow"));
assertOK(client().performRequest("POST", "/" + allowedIndex + "/_xpack/ccr/_unfollow"));
// Make sure that there are no other ccr relates operations running:
assertBusy(() -> {
Map<String, Object> clusterState = toMap(adminClient().performRequest("GET", "/_cluster/state"));
List<?> tasks = (List<?>) XContentMapValues.extractValue("metadata.persistent_tasks.tasks", clusterState);
assertThat(tasks.size(), equalTo(0));
assertThat(countCcrNodeTasks(), equalTo(0));
});

createAndFollowIndex("leader_cluster:" + indexName2, indexName2);
createAndFollowIndex("leader_cluster:" + unallowedIndex, unallowedIndex);
// Verify that nothing has been replicated and no node tasks are running
// These node tasks should have been failed due to the fact that the user
// has no sufficient priviledges.
assertBusy(() -> assertThat(countCcrNodeTasks(), equalTo(0)));
verifyDocuments(adminClient(), indexName2, 0);
verifyDocuments(adminClient(), unallowedIndex, 0);

followIndex("leader_cluster:" + unallowedIndex, unallowedIndex);
assertBusy(() -> assertThat(countCcrNodeTasks(), equalTo(0)));
verifyDocuments(adminClient(), unallowedIndex, 0);
}
}

Expand Down Expand Up @@ -236,13 +192,4 @@ protected static void createIndex(String name, Settings settings, String mapping
+ ", \"mappings\" : {" + mapping + "} }", ContentType.APPLICATION_JSON)));
}

private static void ensureYellow(String index) throws IOException {
Map<String, String> params = new HashMap<>();
params.put("wait_for_status", "yellow");
params.put("wait_for_no_relocating_shards", "true");
params.put("timeout", "30s");
params.put("level", "shards");
assertOK(adminClient().performRequest("GET", "_cluster/health/" + index, params));
}

}
35 changes: 16 additions & 19 deletions ...ugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,8 @@ private void createFollowIndex(IndexMetaData leaderIndexMetaData, Request reques
}
},
listener::onFailure);
// Can't use create index api here, because then index templates can alter the mappings / settings.
// And index templates could introduce settings / mappings that are incompatible with the leader index.
clusterService.submitStateUpdateTask("follow_index_action", new AckedClusterStateUpdateTask<Boolean>(request, handler) {

@Override
Expand All @@ -249,26 +251,21 @@ public ClusterState execute(ClusterState currentState) throws Exception {
}

MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
if (request.getFollowRequest().getFollowIndex() != null) {
IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndexMetaData);

Settings.Builder settingsBuilder = Settings.builder();
settingsBuilder.put(leaderIndexMetaData.getSettings());
// TODO: do we want leader and follow index to have the same UUID?
// (Overwriting UUID here, because otherwise we can't follow indices in the same cluster)
settingsBuilder.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
settingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getFollowRequest().getFollowIndex());
settingsBuilder.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true);

imdBuilder.settings(settingsBuilder);
imdBuilder.index(request.getFollowRequest().getFollowIndex());
for (int shardId = 0 ; shardId < leaderIndexMetaData.getNumberOfShards(); shardId++) {
imdBuilder.putInSyncAllocationIds(shardId, new HashSet<>());
}
mdBuilder.put(imdBuilder.build(), false);
} else {
mdBuilder.put(leaderIndexMetaData, false);
IndexMetaData.Builder imdBuilder = IndexMetaData.builder(leaderIndexMetaData);

Settings.Builder settingsBuilder = Settings.builder();
settingsBuilder.put(leaderIndexMetaData.getSettings());
// Overwriting UUID here, because otherwise we can't follow indices in the same cluster
settingsBuilder.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
settingsBuilder.put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getFollowRequest().getFollowIndex());
settingsBuilder.put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true);

imdBuilder.settings(settingsBuilder);
imdBuilder.index(request.getFollowRequest().getFollowIndex());
for (int shardId = 0 ; shardId < leaderIndexMetaData.getNumberOfShards(); shardId++) {
imdBuilder.putInSyncAllocationIds(shardId, new HashSet<>());
}
mdBuilder.put(imdBuilder.build(), false);

ClusterState.Builder builder = ClusterState.builder(currentState);
builder.metaData(mdBuilder.build());
Expand Down
7 changes: 3 additions & 4 deletions x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/ShardChangesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
Expand Down Expand Up @@ -178,7 +177,7 @@ public void testFollowIndex() throws Exception {
assertBusy(assertExpectedDocumentRunnable(i));
}

unFollowIndex("index2");
unfollowIndex("index2");
client().execute(FollowIndexAction.INSTANCE, followRequest).get();
final int secondBatchNumDocs = randomIntBetween(2, 64);
for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) {
Expand All @@ -200,7 +199,7 @@ public void testFollowIndex() throws Exception {
for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) {
assertBusy(assertExpectedDocumentRunnable(i));
}
unFollowIndex("index2");
unfollowIndex("index2");
}

public void testFollowIndexWithNestedField() throws Exception {
Expand Down Expand Up @@ -318,7 +317,7 @@ private CheckedRunnable<Exception> assertTask(final int numberOfPrimaryShards, f
};
}

private void unFollowIndex(String index) throws Exception {
private void unfollowIndex(String index) throws Exception {
final UnfollowIndexAction.Request unfollowRequest = new UnfollowIndexAction.Request();
unfollowRequest.setFollowIndex(index);
client().execute(UnfollowIndexAction.INSTANCE, unfollowRequest).get();
Expand Down

0 comments on commit 6ef4681

Please sign in to comment.