Skip to content

Commit

Permalink
fix stale remote cluster uuid state not purged from remote
Browse files Browse the repository at this point in the history 
Signed-off-by: bansvaru <bansvaru@amazon.com>
  • Loading branch information
@linuxpi
linuxpi committed Sep 13, 2023
1 parent b90a888 commit 378dcec
Showing 1 changed file with 72 additions and 32 deletions.
104 changes: 72 additions & 32 deletions server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,13 +392,20 @@ private void writeIndexMetadataAsync(
@Nullable
public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest)
throws IOException {
assert clusterState != null : "Last accepted cluster state is not set";
if (clusterState.nodes().isLocalNodeElectedClusterManager() == false) {
logger.error("Local node is not elected cluster manager. Exiting");
return null;
}
assert clusterState != null : "Last accepted cluster state is not set";
assert previousManifest != null : "Last cluster metadata manifest is not set";
return uploadManifest(clusterState, previousManifest.getIndices(), previousManifest.getPreviousClusterUUID(), true);
ClusterMetadataManifest committedManifest = uploadManifest(
clusterState,
previousManifest.getIndices(),
previousManifest.getPreviousClusterUUID(),
true
);
deleteStaleClusterUUID(clusterState, committedManifest);
return committedManifest;
}

@Override
Expand Down Expand Up @@ -661,30 +668,42 @@ private boolean isInvalidClusterUUID(ClusterMetadataManifest manifest) {
}

/**
* Fetch latest ClusterMetadataManifest file from remote state store
* Fetch ClusterMetadataManifest files from remote state store in order
*
* @param clusterUUID uuid of cluster state to refer to in remote
* @param clusterName name of the cluster
* @return latest ClusterMetadataManifest filename
* @param limit max no of files to fetch
* @return all manifest file names
*/
private Optional<String> getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException {
private List<BlobMetadata> getManifestFileNames(String clusterName, String clusterUUID, int limit) throws IllegalStateException {
try {
/**
* {@link BlobContainer#listBlobsByPrefixInSortedOrder} will get the latest manifest file
* {@link BlobContainer#listBlobsByPrefixInSortedOrder} will list the latest manifest file first
* as the manifest file name generated via {@link RemoteClusterStateService#getManifestFileName} ensures
* when sorted in LEXICOGRAPHIC order the latest uploaded manifest file comes on top.
*/
List<BlobMetadata> manifestFilesMetadata = manifestContainer(clusterName, clusterUUID).listBlobsByPrefixInSortedOrder(
return manifestContainer(clusterName, clusterUUID).listBlobsByPrefixInSortedOrder(
MANIFEST_FILE_PREFIX + DELIMITER,
1,
BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC
);
if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) {
return Optional.of(manifestFilesMetadata.get(0).name());
}
} catch (IOException e) {
throw new IllegalStateException("Error while fetching latest manifest file for remote cluster state", e);
}
}

/**
* Fetch latest ClusterMetadataManifest file from remote state store
*
* @param clusterUUID uuid of cluster state to refer to in remote
* @param clusterName name of the cluster
* @return latest ClusterMetadataManifest filename
*/
private Optional<String> getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException {
List<BlobMetadata> manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, 1);
if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) {
return Optional.of(manifestFilesMetadata.get(0).name());
}
logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID);
return Optional.empty();
}
Expand Down Expand Up @@ -729,32 +748,53 @@ public IndexMetadataTransferException(String errorDesc, Throwable cause) {

/**
* Purges all remote cluster state against provided cluster UUIDs
* @param clusterName name of the cluster
* @param clusterUUIDs clusteUUIDs for which the remote state needs to be purged
* @param clusterState current state of the cluster
* @param committedManifest last committed ClusterMetadataManifest
*/
public void deleteStaleClusterMetadata(String clusterName, List<String> clusterUUIDs) {
clusterUUIDs.forEach(clusterUUID -> {
getBlobStoreTransferService().deleteAsync(
ThreadPool.Names.REMOTE_PURGE,
getCusterMetadataBasePath(clusterName, clusterUUID),
new ActionListener<>() {
@Override
public void onResponse(Void unused) {
logger.info("Deleted all remote cluster metadata for cluster UUID - {}", clusterUUID);
public void deleteStaleClusterUUID(ClusterState clusterState, ClusterMetadataManifest committedManifest) throws IOException {
threadpool.executor(ThreadPool.Names.REMOTE_PURGE).execute(() -> {
String clusterName = clusterState.getClusterName().value();
Set<String> allClustersUUIDsInRemote = null;
try {
allClustersUUIDsInRemote = getAllClusterUUIDs(clusterState.getClusterName().value());
} catch (IOException e) {
throw new RuntimeException(e);
}
allClustersUUIDsInRemote.remove(committedManifest.getClusterUUID());
allClustersUUIDsInRemote.remove(committedManifest.getPreviousClusterUUID());
allClustersUUIDsInRemote.forEach(clusterUUID -> {
List<BlobMetadata> allManifestFiles = getManifestFileNames(clusterName, clusterUUID, Integer.MAX_VALUE);
Collections.reverse(allManifestFiles);
allManifestFiles.forEach(manifestFile -> {
try {
getBlobStoreTransferService().delete(getManifestFolderPath(clusterName, clusterUUID).add(manifestFile.name()));
} catch (IOException e) {
throw new RuntimeException(e);
}
});
// delete all index metdata
getBlobStoreTransferService().deleteAsync(
ThreadPool.Names.REMOTE_PURGE,
getCusterMetadataBasePath(clusterName, clusterUUID).add(INDEX_PATH_TOKEN),
new ActionListener<>() {
@Override
public void onResponse(Void unused) {
logger.info("Deleted all remote cluster metadata for cluster UUID - {}", clusterUUID);
}

@Override
public void onFailure(Exception e) {
logger.error(
new ParameterizedMessage(
"Exception occurred while deleting all remote cluster metadata for cluster UUID {}",
clusterUUID
),
e
);
@Override
public void onFailure(Exception e) {
logger.error(
new ParameterizedMessage(
"Exception occurred while deleting all remote cluster metadata for cluster UUID {}",
clusterUUID
),
e
);
}
}
}
);
);
});
});
}

Expand Down

0 comments on commit 378dcec

Please sign in to comment.