Skip to content

Commit

Permalink
Support use of IRSA for repository-s3 plugin credentials (opensearch-…
Browse files Browse the repository at this point in the history
…project#3475)

* Support use of IRSA for repository-s3 plugin credentials

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Address code review comments

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Address code review comments

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
  • Loading branch information
reta authored and Bukhtawar committed Jun 21, 2022
1 parent 2651637 commit dd6ace5
Show file tree
Hide file tree
Showing 11 changed files with 464 additions and 17 deletions.
1 change: 1 addition & 0 deletions plugins/repository-s3/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ versions << [
dependencies {
api "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
api "com.amazonaws:aws-java-sdk-core:${versions.aws}"
api "com.amazonaws:aws-java-sdk-sts:${versions.aws}"
api "com.amazonaws:jmespath-java:${versions.aws}"
api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
724bd22c0ff41c496469e18f9bea12bdfb2f7540
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,39 @@

package org.opensearch.repositories.s3;

import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;

import org.opensearch.common.Nullable;
import org.opensearch.common.concurrent.RefCountedReleasable;

import java.io.Closeable;
import java.io.IOException;

/**
* Handles the shutdown of the wrapped {@link AmazonS3Client} using reference
* counting.
*/
public class AmazonS3Reference extends RefCountedReleasable<AmazonS3> {

AmazonS3Reference(AmazonS3 client) {
super("AWS_S3_CLIENT", client, client::shutdown);
this(client, null);
}

AmazonS3Reference(AmazonS3WithCredentials client) {
this(client.client(), client.credentials());
}

AmazonS3Reference(AmazonS3 client, @Nullable AWSCredentialsProvider credentials) {
super("AWS_S3_CLIENT", client, () -> {
client.shutdown();
if (credentials instanceof Closeable) {
try {
((Closeable) credentials).close();
} catch (IOException e) {
/* Do nothing here */
}
}
});
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.repositories.s3;

import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;

import org.opensearch.common.Nullable;

/**
* The holder of the AmazonS3 and AWSCredentialsProvider
*/
final class AmazonS3WithCredentials {
private final AmazonS3 client;
private final AWSCredentialsProvider credentials;

private AmazonS3WithCredentials(final AmazonS3 client, @Nullable final AWSCredentialsProvider credentials) {
this.client = client;
this.credentials = credentials;
}

AmazonS3 client() {
return client;
}

AWSCredentialsProvider credentials() {
return credentials;
}

static AmazonS3WithCredentials create(final AmazonS3 client, @Nullable final AWSCredentialsProvider credentials) {
return new AmazonS3WithCredentials(client, credentials);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,29 @@ final class S3ClientSettings {
/** Placeholder client name for normalizing client settings in the repository settings. */
private static final String PLACEHOLDER_CLIENT = "placeholder";

// Properties to support using IAM Roles for Service Accounts (IRSA)

/** The identity token file for connecting to s3. */
static final Setting.AffixSetting<String> IDENTITY_TOKEN_FILE_SETTING = Setting.affixKeySetting(
PREFIX,
"identity_token_file",
key -> SecureSetting.simpleString(key, Property.NodeScope)
);

/** The role ARN (Amazon Resource Name) for connecting to s3. */
static final Setting.AffixSetting<SecureString> ROLE_ARN_SETTING = Setting.affixKeySetting(
PREFIX,
"role_arn",
key -> SecureSetting.secureString(key, null)
);

/** The role session name for connecting to s3. */
static final Setting.AffixSetting<SecureString> ROLE_SESSION_NAME_SETTING = Setting.affixKeySetting(
PREFIX,
"role_session_name",
key -> SecureSetting.secureString(key, null)
);

/** The access key (ie login id) for connecting to s3. */
static final Setting.AffixSetting<SecureString> ACCESS_KEY_SETTING = Setting.affixKeySetting(
PREFIX,
Expand Down Expand Up @@ -189,6 +212,9 @@ final class S3ClientSettings {
/** Credentials to authenticate with s3. */
final S3BasicCredentials credentials;

/** Credentials to authenticate with s3 using IAM Roles for Service Accounts (IRSA). */
final IrsaCredentials irsaCredentials;

/** The s3 endpoint the client should talk to, or empty string to use the default. */
final String endpoint;

Expand Down Expand Up @@ -221,6 +247,7 @@ final class S3ClientSettings {

private S3ClientSettings(
S3BasicCredentials credentials,
IrsaCredentials irsaCredentials,
String endpoint,
Protocol protocol,
int readTimeoutMillis,
Expand All @@ -233,6 +260,7 @@ private S3ClientSettings(
ProxySettings proxySettings
) {
this.credentials = credentials;
this.irsaCredentials = irsaCredentials;
this.endpoint = endpoint;
this.protocol = protocol;
this.readTimeoutMillis = readTimeoutMillis;
Expand Down Expand Up @@ -301,6 +329,7 @@ S3ClientSettings refine(Settings repositorySettings) {
validateInetAddressFor(newProxyHost);
return new S3ClientSettings(
newCredentials,
irsaCredentials,
newEndpoint,
newProtocol,
newReadTimeoutMillis,
Expand Down Expand Up @@ -396,12 +425,27 @@ private static S3BasicCredentials loadCredentials(Settings settings, String clie
}
}

private static IrsaCredentials loadIrsaCredentials(Settings settings, String clientName) {
String identityTokenFile = getConfigValue(settings, clientName, IDENTITY_TOKEN_FILE_SETTING);
try (
SecureString roleArn = getConfigValue(settings, clientName, ROLE_ARN_SETTING);
SecureString roleSessionName = getConfigValue(settings, clientName, ROLE_SESSION_NAME_SETTING)
) {
if (identityTokenFile.length() != 0 || roleArn.length() != 0 || roleSessionName.length() != 0) {
return new IrsaCredentials(identityTokenFile.toString(), roleArn.toString(), roleSessionName.toString());
}

return null;
}
}

// pkg private for tests
/** Parse settings for a single client. */
static S3ClientSettings getClientSettings(final Settings settings, final String clientName) {
final Protocol awsProtocol = getConfigValue(settings, clientName, PROTOCOL_SETTING);
return new S3ClientSettings(
S3ClientSettings.loadCredentials(settings, clientName),
S3ClientSettings.loadIrsaCredentials(settings, clientName),
getConfigValue(settings, clientName, ENDPOINT_SETTING),
awsProtocol,
Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()),
Expand Down Expand Up @@ -482,7 +526,8 @@ public boolean equals(final Object o) {
&& proxySettings.equals(that.proxySettings)
&& Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding)
&& Objects.equals(region, that.region)
&& Objects.equals(signerOverride, that.signerOverride);
&& Objects.equals(signerOverride, that.signerOverride)
&& Objects.equals(irsaCredentials, that.irsaCredentials);
}

@Override
Expand Down Expand Up @@ -512,4 +557,51 @@ private static <T> T getRepoSettingOrDefault(Setting.AffixSetting<T> setting, Se
}
return defaultValue;
}

/**
* Class to store IAM Roles for Service Accounts (IRSA) credentials
* See please: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
*/
static class IrsaCredentials {
private final String identityTokenFile;
private final String roleArn;
private final String roleSessionName;

IrsaCredentials(String identityTokenFile, String roleArn, String roleSessionName) {
this.identityTokenFile = Strings.isNullOrEmpty(identityTokenFile) ? null : identityTokenFile;
this.roleArn = Strings.isNullOrEmpty(roleArn) ? null : roleArn;
this.roleSessionName = Strings.isNullOrEmpty(roleSessionName) ? "s3-sdk-java-" + System.currentTimeMillis() : roleSessionName;
}

public String getIdentityTokenFile() {
return identityTokenFile;
}

public String getRoleArn() {
return roleArn;
}

public String getRoleSessionName() {
return roleSessionName;
}

@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
final IrsaCredentials that = (IrsaCredentials) o;
return Objects.equals(identityTokenFile, that.identityTokenFile)
&& Objects.equals(roleArn, that.roleArn)
&& Objects.equals(roleSessionName, that.roleSessionName);
}

@Override
public int hashCode() {
return Objects.hash(identityTokenFile, roleArn, roleSessionName);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,10 @@ public List<Setting<?>> getSettings() {
S3Repository.ACCESS_KEY_SETTING,
S3Repository.SECRET_KEY_SETTING,
S3ClientSettings.SIGNER_OVERRIDE,
S3ClientSettings.REGION
S3ClientSettings.REGION,
S3ClientSettings.ROLE_ARN_SETTING,
S3ClientSettings.IDENTITY_TOKEN_FILE_SETTING,
S3ClientSettings.ROLE_SESSION_NAME_SETTING
);
}

Expand Down
Loading

0 comments on commit dd6ace5

Please sign in to comment.