From 799181b33b13e563d40d51c7c7d9a900bf54cb45 Mon Sep 17 00:00:00 2001 From: Isuru Ranawaka Date: Sun, 21 Feb 2021 19:29:22 -0500 Subject: [PATCH 1/7] rebasing --- .../mft/admin/models/TransferCommand.java | 11 +- .../mft/admin/models/TransferRequest.java | 9 + .../apache/airavata/mft/agent/MFTAgent.java | 44 ++-- .../airavata/mft/agent/TransportMediator.java | 10 +- .../mft/agent/http/HttpServerHandler.java | 22 +- .../airavata/mft/agent/rpc/RPCParser.java | 25 +- .../src/main/resources/application.properties | 3 +- .../mft/api/handler/MFTApiHandler.java | 13 +- .../mft/controller/MFTController.java | 3 +- .../apache/airavata/mft/core/AuthZToken.java | 46 ++++ .../airavata/mft/core/api/Connector.java | 3 +- .../mft/core/api/MetadataCollector.java | 27 +- pom.xml | 1 + services/pom.xml | 5 + .../airavata/mft/secret/server/AppConfig.java | 34 ++- .../backend/custos/CustosException.java | 7 + .../backend/custos/CustosSecretBackend.java | 233 ++++++++++++++++++ .../auth/AgentAuthenticationHandler.java | 76 ++++++ .../backend/custos/auth/AuthConfig.java | 73 ++++++ .../custos/auth/AuthenticationHandler.java | 14 ++ .../src/main/resources/application.properties | 8 +- .../src/main/resources/applicationContext.xml | 4 +- .../distribution/conf/application.properties | 8 +- .../distribution/conf/applicationContext.xml | 4 +- .../src/main/proto/common/CredCommon.proto | 2 + .../azure/AzureMetadataCollector.java | 15 +- .../mft/transport/azure/AzureReceiver.java | 3 +- .../mft/transport/azure/AzureSender.java | 3 +- .../transport/box/BoxMetadataCollector.java | 11 +- .../mft/transport/box/BoxReceiver.java | 4 +- .../airavata/mft/transport/box/BoxSender.java | 4 +- .../dropbox/DropboxMetadataCollector.java | 11 +- .../transport/dropbox/DropboxReceiver.java | 5 +- .../mft/transport/dropbox/DropboxSender.java | 4 +- .../transport/ftp/FTPMetadataCollector.java | 16 +- .../mft/transport/ftp/FTPReceiver.java | 5 +- .../airavata/mft/transport/ftp/FTPSender.java | 4 +- .../transport/gcp/GCSMetadataCollector.java | 11 +- .../mft/transport/gcp/GCSReceiver.java | 5 +- .../airavata/mft/transport/gcp/GCSSender.java | 5 +- .../local/LocalMetadataCollector.java | 11 +- .../mft/transport/local/LocalReceiver.java | 4 +- .../mft/transport/local/LocalSender.java | 4 +- .../mft/transport/s3/S3MetadataCollector.java | 11 +- .../airavata/mft/transport/s3/S3Receiver.java | 4 +- .../airavata/mft/transport/s3/S3Sender.java | 3 +- .../transport/scp/SCPMetadataCollector.java | 25 +- .../mft/transport/scp/SCPReceiver.java | 20 +- .../airavata/mft/transport/scp/SCPSender.java | 5 +- 49 files changed, 734 insertions(+), 144 deletions(-) create mode 100644 core/src/main/java/org/apache/airavata/mft/core/AuthZToken.java create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosException.java create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AuthConfig.java create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AuthenticationHandler.java diff --git a/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferCommand.java b/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferCommand.java index b74b90a9..ada49edc 100644 --- a/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferCommand.java +++ b/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferCommand.java @@ -17,8 +17,6 @@ package org.apache.airavata.mft.admin.models; -import java.util.List; - public class TransferCommand { private String transferId; @@ -34,6 +32,7 @@ public class TransferCommand { private String destinationToken; private String destResourceBackend; private String destCredentialBackend; + private String mftAuthorizationToken; public String getTransferId() { return transferId; @@ -151,4 +150,12 @@ public TransferCommand setDestCredentialBackend(String destCredentialBackend) { this.destCredentialBackend = destCredentialBackend; return this; } + + public String getMftAuthorizationToken() { + return mftAuthorizationToken; + } + + public void setMftAuthorizationToken(String mftAuthorizationToken) { + this.mftAuthorizationToken = mftAuthorizationToken; + } } diff --git a/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferRequest.java b/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferRequest.java index 8512713f..80d39b98 100644 --- a/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferRequest.java +++ b/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferRequest.java @@ -33,6 +33,7 @@ public class TransferRequest { private String destinationToken; private String destResourceBackend; private String destCredentialBackend; + private String mftAuthorizationToken; private boolean affinityTransfer; private Map targetAgents; @@ -161,4 +162,12 @@ public TransferRequest setTargetAgents(Map targetAgents) { this.targetAgents = targetAgents; return this; } + + public String getMftAuthorizationToken() { + return mftAuthorizationToken; + } + + public void setMftAuthorizationToken(String mftAuthorizationToken) { + this.mftAuthorizationToken = mftAuthorizationToken; + } } diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java index 611f2f7d..d4280e05 100644 --- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java +++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java @@ -33,6 +33,7 @@ import org.apache.airavata.mft.agent.http.HttpServer; import org.apache.airavata.mft.agent.http.HttpTransferRequestsStore; import org.apache.airavata.mft.agent.rpc.RPCParser; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorResolver; import org.apache.airavata.mft.core.MetadataCollectorResolver; import org.apache.airavata.mft.core.api.Connector; @@ -64,6 +65,10 @@ public class MFTAgent implements CommandLineRunner { @org.springframework.beans.factory.annotation.Value("${agent.id}") private String agentId; + + @org.springframework.beans.factory.annotation.Value("${agent.secret}") + private String agentSecret; + @org.springframework.beans.factory.annotation.Value("${agent.host}") private String agentHost; @@ -116,8 +121,8 @@ public class MFTAgent implements CommandLineRunner { private HttpTransferRequestsStore transferRequestsStore; public void init() { - transferMessageCache = KVCache.newCache(mftConsulClient.getKvClient(), MFTConsulClient.AGENTS_TRANSFER_REQUEST_MESSAGE_PATH + agentId ); - rpcMessageCache = KVCache.newCache(mftConsulClient.getKvClient(), MFTConsulClient.AGENTS_RPC_REQUEST_MESSAGE_PATH + agentId ); + transferMessageCache = KVCache.newCache(mftConsulClient.getKvClient(), MFTConsulClient.AGENTS_TRANSFER_REQUEST_MESSAGE_PATH + agentId); + rpcMessageCache = KVCache.newCache(mftConsulClient.getKvClient(), MFTConsulClient.AGENTS_RPC_REQUEST_MESSAGE_PATH + agentId); } private void acceptRPCRequests() { @@ -159,13 +164,14 @@ private void acceptTransferRequests() { .setPublisher(agentId) .setDescription("Starting the transfer")); + AuthZToken authZToken = new AuthZToken(request.getMftAuthorizationToken(), agentId, agentSecret); Optional inConnectorOpt = ConnectorResolver.resolveConnector(request.getSourceType(), "IN"); Connector inConnector = inConnectorOpt.orElseThrow(() -> new Exception("Could not find an in connector for given input")); - inConnector.init(request.getSourceStorageId(), request.getSourceToken(), resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort); + inConnector.init(authZToken,request.getSourceStorageId(), request.getSourceToken(), resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort); Optional outConnectorOpt = ConnectorResolver.resolveConnector(request.getDestinationType(), "OUT"); Connector outConnector = outConnectorOpt.orElseThrow(() -> new Exception("Could not find an out connector for given input")); - outConnector.init(request.getDestinationStorageId(), request.getDestinationToken(), resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort); + outConnector.init(authZToken, request.getDestinationStorageId(), request.getDestinationToken(), resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort); Optional srcMetadataCollectorOp = MetadataCollectorResolver.resolveMetadataCollector(request.getSourceType()); MetadataCollector srcMetadataCollector = srcMetadataCollectorOp.orElseThrow(() -> new Exception("Could not find a metadata collector for source")); @@ -183,22 +189,22 @@ private void acceptTransferRequests() { .setDescription("Started the transfer")); - String transferId = mediator.transfer(request, inConnector, outConnector, srcMetadataCollector, dstMetadataCollector, - (id, st) -> { - try { - mftConsulClient.submitTransferStateToProcess(id, agentId, st.setPublisher(agentId)); - } catch (MFTConsulClientException e) { - logger.error("Failed while updating transfer state", e); + String transferId = mediator.transfer(authZToken,request, inConnector, outConnector, srcMetadataCollector, dstMetadataCollector, + (id, st) -> { + try { + mftConsulClient.submitTransferStateToProcess(id, agentId, st.setPublisher(agentId)); + } catch (MFTConsulClientException e) { + logger.error("Failed while updating transfer state", e); + } + }, + (id, transferSuccess) -> { + try { + // Delete scheduled key as the transfer completed / failed if it was placed in current session + mftConsulClient.getKvClient().deleteKey(MFTConsulClient.AGENTS_SCHEDULED_PATH + agentId + "/" + session + "/" + id); + } catch (Exception e) { + logger.error("Failed while deleting scheduled path for transfer {}", id); + } } - }, - (id, transferSuccess) -> { - try { - // Delete scheduled key as the transfer completed / failed if it was placed in current session - mftConsulClient.getKvClient().deleteKey(MFTConsulClient.AGENTS_SCHEDULED_PATH + agentId + "/" + session + "/" + id); - } catch (Exception e) { - logger.error("Failed while deleting scheduled path for transfer {}", id); - } - } ); logger.info("Started the transfer " + transferId); diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java b/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java index c1ac253e..a1c354f1 100644 --- a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java +++ b/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java @@ -48,11 +48,11 @@ public void destroy() { executor.shutdown(); } - public String transfer(TransferCommand command, Connector inConnector, Connector outConnector, MetadataCollector srcMetadataCollector, + public String transfer(AuthZToken authZToken, TransferCommand command, Connector inConnector, Connector outConnector, MetadataCollector srcMetadataCollector, MetadataCollector destMetadataCollector, BiConsumer onStatusCallback, BiConsumer exitingCallback) throws Exception { - FileResourceMetadata srcMetadata = srcMetadataCollector.getFileResourceMetadata( + FileResourceMetadata srcMetadata = srcMetadataCollector.getFileResourceMetadata(authZToken, command.getSourceStorageId(), command.getSourcePath(), command.getSourceToken()); @@ -129,15 +129,17 @@ public void run() { command.getDestinationPath(), command.getDestinationToken()); + if (!transferred) { logger.error("Transfer completed but resource is not available in destination"); throw new Exception("Transfer completed but resource is not available in destination"); } - FileResourceMetadata destMetadata = destMetadataCollector.getFileResourceMetadata( + FileResourceMetadata destMetadata = destMetadataCollector.getFileResourceMetadata(authZToken, command.getDestinationStorageId(), command.getDestinationPath(), - command.getDestinationToken()); + command.getDestinationToken()); + boolean doIntegrityVerify = true; diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java b/agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java index 57ee4f01..dde6f9c9 100644 --- a/agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java +++ b/agent/src/main/java/org/apache/airavata/mft/agent/http/HttpServerHandler.java @@ -18,19 +18,11 @@ package org.apache.airavata.mft.agent.http; import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelProgressiveFuture; -import io.netty.channel.ChannelProgressiveFutureListener; -import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.*; import io.netty.handler.codec.http.*; import io.netty.handler.stream.ChunkedStream; import io.netty.util.CharsetUtil; -import org.apache.airavata.mft.core.ConnectorContext; -import org.apache.airavata.mft.core.DoubleStreamingBuffer; -import org.apache.airavata.mft.core.FileResourceMetadata; -import org.apache.airavata.mft.core.TransferTask; +import org.apache.airavata.mft.core.*; import org.apache.airavata.mft.core.api.Connector; import org.apache.airavata.mft.core.api.MetadataCollector; import org.slf4j.Logger; @@ -41,9 +33,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; -import static io.netty.handler.codec.http.HttpMethod.*; +import static io.netty.handler.codec.http.HttpMethod.GET; import static io.netty.handler.codec.http.HttpResponseStatus.*; -import static io.netty.handler.codec.http.HttpVersion.*; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; public class HttpServerHandler extends SimpleChannelInboundHandler { @@ -84,7 +76,9 @@ public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) thr ConnectorParams params = httpTransferRequest.getConnectorParams(); - connector.init(params.getStorageId(), params.getCredentialToken(), params.getResourceServiceHost(), + AuthZToken authZToken = new AuthZToken(); + + connector.init(authZToken, params.getStorageId(), params.getCredentialToken(), params.getResourceServiceHost(), params.getResourceServicePort(), params.getSecretServiceHost(), params.getSecretServicePort()); metadataCollector.init(params.getResourceServiceHost(), params.getResourceServicePort(), @@ -101,7 +95,7 @@ public void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) thr return; } - FileResourceMetadata fileResourceMetadata = metadataCollector.getFileResourceMetadata(params.getStorageId(), + FileResourceMetadata fileResourceMetadata = metadataCollector.getFileResourceMetadata(authZToken, params.getStorageId(), httpTransferRequest.getTargetResourcePath(), params.getCredentialToken()); diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java b/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java index fc1bb2b5..33b9354c 100644 --- a/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java +++ b/agent/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java @@ -15,7 +15,7 @@ * limitations under the License. */ - package org.apache.airavata.mft.agent.rpc; +package org.apache.airavata.mft.agent.rpc; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.airavata.mft.admin.models.rpc.SyncRPCRequest; @@ -24,6 +24,7 @@ import org.apache.airavata.mft.agent.http.HttpTransferRequest; import org.apache.airavata.mft.agent.http.HttpTransferRequestsStore; import org.apache.airavata.mft.core.ConnectorResolver; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.MetadataCollectorResolver; @@ -73,12 +74,15 @@ public String resolveRPCRequest(SyncRPCRequest request) throws Exception { String resourceType = request.getParameters().get("resourceType"); String resourceToken = request.getParameters().get("resourceToken"); String mftAuthorizationToken = request.getParameters().get("mftAuthorizationToken"); + String agentId = request.getAgentId(); + String agentSecret = request.getParameters().get("agentSecret"); Optional metadataCollectorOp = MetadataCollectorResolver.resolveMetadataCollector(resourceType); if (metadataCollectorOp.isPresent()) { MetadataCollector metadataCollector = metadataCollectorOp.get(); metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort); - FileResourceMetadata fileResourceMetadata = metadataCollector.getFileResourceMetadata(resourceId, resourceToken); + FileResourceMetadata fileResourceMetadata = metadataCollector + .getFileResourceMetadata(new AuthZToken(mftAuthorizationToken, agentId, agentSecret), resourceId, resourceToken); return mapper.writeValueAsString(fileResourceMetadata); } break; @@ -89,12 +93,15 @@ public String resolveRPCRequest(SyncRPCRequest request) throws Exception { resourceToken = request.getParameters().get("resourceToken"); String childPath = request.getParameters().get("childPath"); mftAuthorizationToken = request.getParameters().get("mftAuthorizationToken"); + agentId = request.getAgentId(); + agentSecret = request.getParameters().get("agentSecret"); metadataCollectorOp = MetadataCollectorResolver.resolveMetadataCollector(resourceType); if (metadataCollectorOp.isPresent()) { MetadataCollector metadataCollector = metadataCollectorOp.get(); metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort); - FileResourceMetadata fileResourceMetadata = metadataCollector.getFileResourceMetadata(resourceId, childPath, resourceToken); + FileResourceMetadata fileResourceMetadata = metadataCollector + .getFileResourceMetadata(new AuthZToken(mftAuthorizationToken, agentId, agentSecret), resourceId, childPath, resourceToken); return mapper.writeValueAsString(fileResourceMetadata); } break; @@ -104,12 +111,15 @@ public String resolveRPCRequest(SyncRPCRequest request) throws Exception { resourceType = request.getParameters().get("resourceType"); resourceToken = request.getParameters().get("resourceToken"); mftAuthorizationToken = request.getParameters().get("mftAuthorizationToken"); + agentId = request.getAgentId(); + agentSecret = request.getParameters().get("agentSecret"); metadataCollectorOp = MetadataCollectorResolver.resolveMetadataCollector(resourceType); if (metadataCollectorOp.isPresent()) { MetadataCollector metadataCollector = metadataCollectorOp.get(); metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort); - DirectoryResourceMetadata dirResourceMetadata = metadataCollector.getDirectoryResourceMetadata(resourceId, resourceToken); + DirectoryResourceMetadata dirResourceMetadata = metadataCollector + .getDirectoryResourceMetadata(new AuthZToken(mftAuthorizationToken, agentId, agentSecret), resourceId, resourceToken); return mapper.writeValueAsString(dirResourceMetadata); } break; @@ -120,12 +130,15 @@ public String resolveRPCRequest(SyncRPCRequest request) throws Exception { resourceToken = request.getParameters().get("resourceToken"); childPath = request.getParameters().get("childPath"); mftAuthorizationToken = request.getParameters().get("mftAuthorizationToken"); + agentId = request.getAgentId(); + agentSecret = request.getParameters().get("agentSecret"); metadataCollectorOp = MetadataCollectorResolver.resolveMetadataCollector(resourceType); if (metadataCollectorOp.isPresent()) { MetadataCollector metadataCollector = metadataCollectorOp.get(); metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort); - DirectoryResourceMetadata dirResourceMetadata = metadataCollector.getDirectoryResourceMetadata(resourceId, childPath, resourceToken); + DirectoryResourceMetadata dirResourceMetadata = metadataCollector + .getDirectoryResourceMetadata(new AuthZToken(mftAuthorizationToken, agentId, agentSecret), resourceId, childPath, resourceToken); return mapper.writeValueAsString(dirResourceMetadata); } break; @@ -169,7 +182,7 @@ public SyncRPCResponse processRPCRequest(SyncRPCRequest request) { response.setResponseStatus(SyncRPCResponse.ResponseStatus.SUCCESS); } catch (Exception e) { logger.error("Errored while processing the rpc request for message {} and method {}", - request.getMessageId(), request.getMethod(), e); + request.getMessageId(), request.getMethod(), e); response.setErrorAsStr(e.getMessage()); response.setResponseStatus(SyncRPCResponse.ResponseStatus.FAIL); } diff --git a/agent/src/main/resources/application.properties b/agent/src/main/resources/application.properties index 038ded2e..5863e257 100644 --- a/agent/src/main/resources/application.properties +++ b/agent/src/main/resources/application.properties @@ -16,7 +16,8 @@ # spring.main.web-application-type=NONE -agent.id=agent0 +agent.id=mft-agent-a +agent.secret=jENXjJetKXvffqRjLK9PWXZWMegz7d7cJG8VXGHo agent.host=localhost agent.user=dimuthu agent.http.port=3333 diff --git a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java index 1a6a8b9a..0a6e5ef7 100644 --- a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java +++ b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java @@ -26,6 +26,7 @@ import org.apache.airavata.mft.admin.models.rpc.SyncRPCRequest; import org.apache.airavata.mft.admin.models.rpc.SyncRPCResponse; import org.apache.airavata.mft.api.service.*; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.MetadataCollectorResolver; @@ -76,6 +77,8 @@ public void submitTransfer(TransferApiRequest request, StreamObserver new Exception("Could not find a metadata collector for resource " + request.getResourceId())); metadataCollector.init(resourceServiceHost, resourceServicePort, secretServiceHost, secretServicePort); - - Boolean available = metadataCollector.isAvailable(request.getResourceId(), request.getResourceToken()); + AuthZToken authZToken = new AuthZToken(request.getMftAuthorizationToken()); + Boolean available = metadataCollector.isAvailable(authZToken, request.getResourceId(), request.getResourceToken()); responseObserver.onNext(ResourceAvailabilityResponse.newBuilder().setAvailable(available).build()); responseObserver.onCompleted(); @@ -227,9 +230,9 @@ public void getFileResourceMetadata(FetchResourceMetadataRequest request, Stream return; case FAIL: logger.error("Errored while processing the fetch file metadata response for resource id {}. Error msg : {}", - request.getResourceId(), rpcResponse.getErrorAsStr()); + request.getResourceId(), rpcResponse.getErrorAsStr()); responseObserver.onError(new Exception("Errored while processing the the fetch file metadata response. Error msg : " + - rpcResponse.getErrorAsStr())); + rpcResponse.getErrorAsStr())); } } catch (Exception e) { logger.error("Error while fetching resource metadata for file resource " + request.getResourceId(), e); diff --git a/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java b/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java index ec4240ca..3ab8edc4 100644 --- a/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java +++ b/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java @@ -233,7 +233,8 @@ private TransferCommand convertRequestToCommand(String transferId, TransferReque .setDestinationType(transferRequest.getDestinationType()) .setDestResourceBackend(transferRequest.getDestResourceBackend()) .setDestCredentialBackend(transferRequest.getDestCredentialBackend()) - .setTransferId(transferId); + .setTransferId(transferId) + .setMftAuthorizationToken(transferRequest.getMftAuthorizationToken()); return transferCommand; } diff --git a/core/src/main/java/org/apache/airavata/mft/core/AuthZToken.java b/core/src/main/java/org/apache/airavata/mft/core/AuthZToken.java new file mode 100644 index 00000000..5b38ffdf --- /dev/null +++ b/core/src/main/java/org/apache/airavata/mft/core/AuthZToken.java @@ -0,0 +1,46 @@ +package org.apache.airavata.mft.core; + +public class AuthZToken { + + private String mftAuthorizationToken; + private String agentId; + private String agentSecret; + + public AuthZToken(String mftAuthorizationToken, String agentId, String agentSecret) { + this.mftAuthorizationToken = mftAuthorizationToken; + this.agentId = agentId; + this.agentSecret = agentSecret; + } + + public AuthZToken(String mftAuthorizationToken) { + this.mftAuthorizationToken = mftAuthorizationToken; + } + + public AuthZToken(){ + + } + + public String getMftAuthorizationToken() { + return mftAuthorizationToken; + } + + public void setMftAuthorizationToken(String mftAuthorizationToken) { + this.mftAuthorizationToken = mftAuthorizationToken; + } + + public String getAgentId() { + return agentId; + } + + public void setAgentId(String agentId) { + this.agentId = agentId; + } + + public String getAgentSecret() { + return agentSecret; + } + + public void setAgentSecret(String agentSecret) { + this.agentSecret = agentSecret; + } +} diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/Connector.java b/core/src/main/java/org/apache/airavata/mft/core/api/Connector.java index 8264d584..738a886c 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/api/Connector.java +++ b/core/src/main/java/org/apache/airavata/mft/core/api/Connector.java @@ -17,10 +17,11 @@ package org.apache.airavata.mft.core.api; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; public interface Connector { - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception; public void destroy(); void startStream(String targetPath, ConnectorContext context) throws Exception; diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java b/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java index 761861fe..c7405bed 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java +++ b/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java @@ -17,6 +17,7 @@ package org.apache.airavata.mft.core.api; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; @@ -40,53 +41,53 @@ public interface MetadataCollector { * @return an object of {@link FileResourceMetadata} * @throws Exception if the resource id is not a File Resource type or the resource can't be fetched from the resource service */ - public FileResourceMetadata getFileResourceMetadata(String resourceId, String credentialToken) throws Exception; + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception; /* * Fetches a metadata of given File Resource inside a registered directory resource. Target file might be living in * multiple level below the parent directory * * @param parentResourceId parent directory resource id - * @param resourcePath path of the target resource. This should be a child path of the parent resource - * @param credentialToken credential token for the resource + * @param resourcePath path of the target resource. This should be a child path of the parent resource + * @param credentialToken credential token for the resource * @return an object of {@link FileResourceMetadata} * @throws Exception if the parent resource is not a Directory resource or the target resource is not a File Resource type - * or the resource can't be fetched from the resource service + * or the resource can't be fetched from the resource service */ - public FileResourceMetadata getFileResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception; + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception; /** * Fetches a metadata of given Directory Resource * - * @param resourceId id of the resource + * @param resourceId id of the resource * @param credentialToken credential token for the resource * @return an object of {@link DirectoryResourceMetadata} * @throws Exception if the resource id is not a Directory Resource type or the resource can't be fetched from the resource service */ - public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, String credentialToken) throws Exception; + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception; /** * Fetches a metadata of given Directory Resource inside a registered directory resource. Target directory might be living in * multiple level below the parent directory * * @param parentResourceId parent directory resource id - * @param resourcePath path of the target resource. This should be a child path of the parent resource - * @param credentialToken credential token for the resource + * @param resourcePath path of the target resource. This should be a child path of the parent resource + * @param credentialToken credential token for the resource * @return an object of {@link DirectoryResourceMetadata} * @throws Exception if the parent resource is not a Directory resource or the target resource is not a Directory Resource type - * or the resource can't be fetched from the resource service + * or the resource can't be fetched from the resource service */ - public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception; + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception; /** * Check whether the resource is available in the actual storage * - * @param resourceId id of the resource + * @param resourceId id of the resource * @param credentialToken credential token for the resource * @return true of the resource is available false otherwise * @throws Exception if the resource details can not be fetched from the resource service */ - public Boolean isAvailable(String resourceId, String credentialToken) throws Exception; + public Boolean isAvailable(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception; /** * Check whether the resource is available in the actual storage diff --git a/pom.xml b/pom.xml index 3a8a45bd..f729ead1 100755 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,7 @@ 0.1.55 0.27.0 2.5.1 + 1.0-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 813cd3be..b6b2fde6 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -43,6 +43,11 @@ protobuf-java ${protobuf.java} + + org.apache.custos + custos-java-sdk + ${custos.clients.version} + io.github.lognet grpc-spring-boot-starter diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java index 81f55521..aa0e98a8 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java @@ -17,14 +17,40 @@ package org.apache.airavata.mft.secret.server; -import org.apache.airavata.mft.secret.server.backend.SecretBackend; -import org.apache.airavata.mft.secret.server.backend.airavata.AiravataSecretBackend; -import org.apache.airavata.mft.secret.server.backend.sql.SQLSecretBackend; +import org.apache.airavata.mft.secret.server.backend.custos.auth.AgentAuthenticationHandler; +import org.apache.custos.clients.CustosClientProvider; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; @Configuration public class AppConfig { + + @Value("${custos.host}") + private String custosHost; + + @Value("${custos.port}") + private int custosPort; + + @Value("${custos.id}") + private String custosId; + + @Value("${custos.secret}") + private String custosSecret; + + @Bean + public CustosClientProvider custosClientProvider() { + return new CustosClientProvider.Builder().setServerHost(custosHost) + .setServerPort(custosPort) + .setClientId(custosId) + .setClientSec(custosSecret).build(); + } + + @Bean + public AgentAuthenticationHandler agentAuthenticationHandler() { + return new AgentAuthenticationHandler(this.custosId); + } + + } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosException.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosException.java new file mode 100644 index 00000000..f4c0e0f6 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosException.java @@ -0,0 +1,7 @@ +package org.apache.airavata.mft.secret.server.backend.custos; + +public class CustosException extends RuntimeException { + public CustosException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java new file mode 100644 index 00000000..8a666368 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java @@ -0,0 +1,233 @@ +package org.apache.airavata.mft.secret.server.backend.custos; + +import org.apache.airavata.mft.credential.stubs.azure.*; +import org.apache.airavata.mft.credential.stubs.box.*; +import org.apache.airavata.mft.credential.stubs.dropbox.*; +import org.apache.airavata.mft.credential.stubs.ftp.*; +import org.apache.airavata.mft.credential.stubs.gcs.*; +import org.apache.airavata.mft.credential.stubs.s3.*; +import org.apache.airavata.mft.credential.stubs.scp.*; +import org.apache.airavata.mft.secret.server.backend.SecretBackend; +import org.apache.airavata.mft.secret.server.backend.custos.auth.AgentAuthenticationHandler; +import org.apache.airavata.mft.secret.server.backend.custos.auth.AuthConfig; +import org.apache.custos.clients.CustosClientProvider; +import org.apache.custos.identity.management.client.IdentityManagementClient; +import org.apache.custos.resource.secret.management.client.ResourceSecretManagementAgentClient; +import org.apache.custos.resource.secret.management.client.ResourceSecretManagementClient; +import org.apache.custos.resource.secret.service.SSHCredential; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +import java.util.Optional; + +/** + * Handle Custos secret management operations + */ +public class CustosSecretBackend implements SecretBackend { + private static final Logger LOGGER = LoggerFactory.getLogger(CustosSecretBackend.class); + + + @Autowired + private AgentAuthenticationHandler handler; + + @Autowired + private CustosClientProvider clientProvider; + + @Value("${custos.id}") + private String custosId; + + private ResourceSecretManagementAgentClient csAgentClient; + + private ResourceSecretManagementClient csClient; + + private IdentityManagementClient identityClient; + + + @Override + public void init() { + try { + csAgentClient = (ResourceSecretManagementAgentClient) clientProvider + .getResourceSecretManagementClientForAgents(); + csClient = clientProvider.getResourceSecretManagementClient(); + identityClient = clientProvider.getIdentityManagementClient(); + } catch (Exception ex) { + LOGGER.error("Custos client initialization failed ", ex); + } + + } + + @Override + public void destroy() { + + } + + @Override + public Optional getSCPSecret(SCPSecretGetRequest request) throws Exception { + String agentId = request.getAuthzToken().getAgentId(); + String secret = request.getAuthzToken().getAgentSecret(); + Optional optionalAuthConfig = handler.authenticate(agentId, secret); + if (optionalAuthConfig.isPresent()) { + AuthConfig authConfig = optionalAuthConfig.get(); + SSHCredential sshCredential = csAgentClient. + getSSHCredential(request.getAuthzToken().getToken(), + authConfig.getAccessToken(), request.getSecretId(), false); + SCPSecret scpSecret = SCPSecret.newBuilder() + .setSecretId(sshCredential.getMetadata().getToken()) + .setPublicKey(sshCredential.getPublicKey()) + .setPassphrase(sshCredential.getPassphrase()) + .setPrivateKey(sshCredential.getPrivateKey()).build(); + LOGGER.info("Public key " + sshCredential.getPublicKey()); + return Optional.of(scpSecret); + + } else if (!request.getAuthzToken().getToken().isEmpty()) { + if (identityClient.isAuthenticated(request.getAuthzToken().getToken())) { + SSHCredential sshCredential = csClient.getSSHCredential(custosId, request.getSecretId(), false); + SCPSecret scpSecret = SCPSecret.newBuilder() + .setSecretId(sshCredential.getMetadata().getToken()) + .setPublicKey(sshCredential.getPublicKey()) + .setPassphrase(sshCredential.getPassphrase()) + .setPrivateKey(sshCredential.getPrivateKey()).build(); + LOGGER.info("Public key " + sshCredential.getPublicKey()); + return Optional.of(scpSecret); + } + } + return Optional.empty(); + } + + @Override + public SCPSecret createSCPSecret(SCPSecretCreateRequest request) { + return null; + } + + @Override + public boolean updateSCPSecret(SCPSecretUpdateRequest request) { + return false; + } + + @Override + public boolean deleteSCPSecret(SCPSecretDeleteRequest request) { + return false; + } + + @Override + public Optional getS3Secret(S3SecretGetRequest request) throws Exception { + return Optional.empty(); + } + + @Override + public S3Secret createS3Secret(S3SecretCreateRequest request) throws Exception { + return null; + } + + @Override + public boolean updateS3Secret(S3SecretUpdateRequest request) throws Exception { + return false; + } + + @Override + public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception { + return false; + } + + @Override + public Optional getBoxSecret(BoxSecretGetRequest request) throws Exception { + return Optional.empty(); + } + + @Override + public BoxSecret createBoxSecret(BoxSecretCreateRequest request) throws Exception { + return null; + } + + @Override + public boolean updateBoxSecret(BoxSecretUpdateRequest request) throws Exception { + return false; + } + + @Override + public boolean deleteBoxSecret(BoxSecretDeleteRequest request) throws Exception { + return false; + } + + @Override + public Optional getAzureSecret(AzureSecretGetRequest request) throws Exception { + return Optional.empty(); + } + + @Override + public AzureSecret createAzureSecret(AzureSecretCreateRequest request) throws Exception { + return null; + } + + @Override + public boolean updateAzureSecret(AzureSecretUpdateRequest request) throws Exception { + return false; + } + + @Override + public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws Exception { + return false; + } + + @Override + public Optional getGCSSecret(GCSSecretGetRequest request) throws Exception { + return Optional.empty(); + } + + @Override + public GCSSecret createGCSSecret(GCSSecretCreateRequest request) throws Exception { + return null; + } + + @Override + public boolean updateGCSSecret(GCSSecretUpdateRequest request) throws Exception { + return false; + } + + @Override + public boolean deleteGCSSecret(GCSSecretDeleteRequest request) throws Exception { + return false; + } + + @Override + public Optional getDropboxSecret(DropboxSecretGetRequest request) throws Exception { + return Optional.empty(); + } + + @Override + public DropboxSecret createDropboxSecret(DropboxSecretCreateRequest request) throws Exception { + return null; + } + + @Override + public boolean updateDropboxSecret(DropboxSecretUpdateRequest request) throws Exception { + return false; + } + + @Override + public boolean deleteDropboxSecret(DropboxSecretDeleteRequest request) throws Exception { + return false; + } + + @Override + public Optional getFTPSecret(FTPSecretGetRequest request) throws Exception { + return Optional.empty(); + } + + @Override + public FTPSecret createFTPSecret(FTPSecretCreateRequest request) throws Exception { + return null; + } + + @Override + public boolean updateFTPSecret(FTPSecretUpdateRequest request) throws Exception { + return false; + } + + @Override + public boolean deleteFTPSecret(FTPSecretDeleteRequest request) throws Exception { + return false; + } +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java new file mode 100644 index 00000000..9825f237 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java @@ -0,0 +1,76 @@ +package org.apache.airavata.mft.secret.server.backend.custos.auth; + +import com.google.protobuf.Struct; +import org.apache.airavata.mft.secret.server.backend.custos.CustosException; +import org.apache.custos.clients.CustosClientProvider; +import org.apache.custos.identity.management.client.IdentityManagementClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Handle agent authentication + */ +public class AgentAuthenticationHandler implements AuthenticationHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(AgentAuthenticationHandler.class); + + private static final String CLIENT_CREDENTIALS = "client_credentials"; + + private Map authCache = new ConcurrentHashMap(); + + private String custosId; + + @Autowired + private CustosClientProvider custosClientProvider; + + + public AgentAuthenticationHandler(String custosId) { + this.custosId = custosId; + } + + @Override + public Optional authenticate(String id, String secret) throws Exception { + try { + AuthConfig cachedAuthConfig = authCache.get(id); + AuthConfig authConfig = new AuthConfig(); + final boolean agentRequest = id != null & secret != null & !id.isEmpty() & !secret.isEmpty(); + if (cachedAuthConfig == null && agentRequest) { + IdentityManagementClient identityManagementClient = custosClientProvider.getIdentityManagementClient(); + Struct tokenResponse = identityManagementClient.getAgentToken(custosId, id, secret, + CLIENT_CREDENTIALS, null); + + if (tokenResponse.getFieldsMap() != null && !tokenResponse.getFieldsMap().isEmpty()) { + authConfig.setId(id); + authConfig.setSecret(secret); + tokenResponse.getFieldsMap().keySet().forEach(key -> { + String value = tokenResponse.getFieldsMap().get(key).getStringValue(); + if (key.trim().equals("access_token")) { + authConfig.setAccessToken(value); + } else if (key.trim().equals("refresh_token")) { + authConfig.setRefreshToken(value); + } else if (key.trim().equals("id_token")) { + authConfig.setIdToken(value); + } + }); + authCache.put(id, authConfig); + return Optional.of(authConfig); + } + } else if (id != null && agentRequest) { + return Optional.of(cachedAuthConfig); + + } + return Optional.empty(); + + } catch (Exception ex) { + String errorMsg = "Error occurred while authenticating agent " + id + " with Custos"; + LOGGER.error(errorMsg + ", reason :" + ex.getMessage()); + throw new CustosException(errorMsg, ex); + } + + } + +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AuthConfig.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AuthConfig.java new file mode 100644 index 00000000..39f2e983 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AuthConfig.java @@ -0,0 +1,73 @@ +package org.apache.airavata.mft.secret.server.backend.custos.auth; + +/** + * Represents the Auth object + */ +public class AuthConfig { + + private String id; + private String secret; + private String accessToken; + private String refreshToken; + private String idToken; + private String custosId; + + public AuthConfig(String id, String secret, String accessToken, String refreshToken, String idToken) { + this.id = id; + this.secret = secret; + this.accessToken = accessToken; + this.refreshToken = refreshToken; + this.idToken = idToken; + } + + public AuthConfig() { + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getSecret() { + return secret; + } + + public void setSecret(String secret) { + this.secret = secret; + } + + public String getAccessToken() { + return accessToken; + } + + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + } + + public String getRefreshToken() { + return refreshToken; + } + + public void setRefreshToken(String refreshToken) { + this.refreshToken = refreshToken; + } + + public String getIdToken() { + return idToken; + } + + public void setIdToken(String idToken) { + this.idToken = idToken; + } + + public String getCustosId() { + return custosId; + } + + public void setCustosId(String custosId) { + this.custosId = custosId; + } +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AuthenticationHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AuthenticationHandler.java new file mode 100644 index 00000000..c2b90f71 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AuthenticationHandler.java @@ -0,0 +1,14 @@ +package org.apache.airavata.mft.secret.server.backend.custos.auth; + +import java.util.Optional; + +/** + * Represents the authentication related functional interfaces + */ +public interface AuthenticationHandler { + + + Optional authenticate(String id, String secret) throws Exception; + + +} diff --git a/services/secret-service/server/src/main/resources/application.properties b/services/secret-service/server/src/main/resources/application.properties index 00cf2b5b..7a1a0f55 100644 --- a/services/secret-service/server/src/main/resources/application.properties +++ b/services/secret-service/server/src/main/resources/application.properties @@ -23,4 +23,10 @@ airavata.backend.credential.server.host=localhost airavata.backend.credential.server.port=8960 # Configurations for file Backend -file.backend.secret.file=secrets.json \ No newline at end of file +file.backend.secret.file=secrets.json + +#Configurations for Custos Backend +custos.host=custos.scigap.org +custos.port=31499 +custos.id=custos-2zuomcugra3ebgsqtzmf-10000514 +custos.secret=mupUhF4JL0S3IFHBjfhiTfLJS1NgSWfvkCj3l6c7 \ No newline at end of file diff --git a/services/secret-service/server/src/main/resources/applicationContext.xml b/services/secret-service/server/src/main/resources/applicationContext.xml index d84b110d..c815ce76 100644 --- a/services/secret-service/server/src/main/resources/applicationContext.xml +++ b/services/secret-service/server/src/main/resources/applicationContext.xml @@ -6,7 +6,9 @@ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd"> - + + \ No newline at end of file diff --git a/services/secret-service/server/src/main/resources/distribution/conf/application.properties b/services/secret-service/server/src/main/resources/distribution/conf/application.properties index 2b531f42..f904f17b 100644 --- a/services/secret-service/server/src/main/resources/distribution/conf/application.properties +++ b/services/secret-service/server/src/main/resources/distribution/conf/application.properties @@ -23,4 +23,10 @@ airavata.backend.credential.server.host=149.165.170.103 airavata.backend.credential.server.port=8960 # Configurations for file Backend -file.backend.secret.file=secrets.json \ No newline at end of file +file.backend.secret.file=secrets.json + +# Configurations for Custos Backend +custos.host=custos.scigap.org +custos.port=31499 +custos.id=custos-2zuomcugra3ebgsqtzmf-10000514 +custos.secret=mupUhF4JL0S3IFHBjfhiTfLJS1NgSWfvkCj3l6c7 \ No newline at end of file diff --git a/services/secret-service/server/src/main/resources/distribution/conf/applicationContext.xml b/services/secret-service/server/src/main/resources/distribution/conf/applicationContext.xml index d84b110d..c815ce76 100644 --- a/services/secret-service/server/src/main/resources/distribution/conf/applicationContext.xml +++ b/services/secret-service/server/src/main/resources/distribution/conf/applicationContext.xml @@ -6,7 +6,9 @@ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd"> - + + \ No newline at end of file diff --git a/services/secret-service/stub/src/main/proto/common/CredCommon.proto b/services/secret-service/stub/src/main/proto/common/CredCommon.proto index bb6ede92..e87977e3 100644 --- a/services/secret-service/stub/src/main/proto/common/CredCommon.proto +++ b/services/secret-service/stub/src/main/proto/common/CredCommon.proto @@ -22,4 +22,6 @@ package org.apache.airavata.mft.credential.stubs.common; message AuthToken { string token = 1; + string agentId = 2; + string agentSecret = 3; } \ No newline at end of file diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java index e043dc28..9ceeb4ff 100644 --- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java @@ -22,6 +22,7 @@ import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobProperties; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.ResourceTypes; @@ -47,7 +48,7 @@ public class AzureMetadataCollector implements MetadataCollector { boolean initialized = false; @Override - public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) { + public void init( String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) { this.resourceServiceHost = resourceServiceHost; this.resourceServicePort = resourceServicePort; this.secretServiceHost = secretServiceHost; @@ -62,10 +63,10 @@ private void checkInitialized() { } @Override - public FileResourceMetadata getFileResourceMetadata(String resourceId, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken,String resourceId, String credentialToken) throws Exception { checkInitialized(); - if (!isAvailable(resourceId, credentialToken)) { + if (!isAvailable(authZToken,resourceId, credentialToken)) { throw new Exception("Azure blob can not find for resource id " + resourceId); } @@ -98,21 +99,21 @@ public FileResourceMetadata getFileResourceMetadata(String resourceId, String cr } @Override - public FileResourceMetadata getFileResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public Boolean isAvailable(String resourceId, String credentialToken) throws Exception { + public Boolean isAvailable(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java index dab93371..a496fa5d 100644 --- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java @@ -22,6 +22,7 @@ import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.specialized.BlobInputStream; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -46,7 +47,7 @@ public class AzureReceiver implements Connector { BlobContainerClient containerClient; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { this.initialized = true; ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java index ef3249e7..e00c7996 100644 --- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java @@ -21,6 +21,7 @@ import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.specialized.BlockBlobClient; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -45,7 +46,7 @@ public class AzureSender implements Connector { BlobContainerClient containerClient; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { this.initialized = true; ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); diff --git a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java index 6d597a03..3c1da7b6 100644 --- a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java +++ b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java @@ -20,6 +20,7 @@ import com.box.sdk.BoxAPIConnection; import com.box.sdk.BoxFile; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.ResourceTypes; @@ -60,7 +61,7 @@ private void checkInitialized() { } @Override - public FileResourceMetadata getFileResourceMetadata(String resourceId, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); @@ -87,21 +88,21 @@ public FileResourceMetadata getFileResourceMetadata(String resourceId, String cr } @Override - public FileResourceMetadata getFileResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public Boolean isAvailable(String resourceId, String credentialToken) throws Exception { + public Boolean isAvailable(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); diff --git a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java index e041e358..be500552 100644 --- a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java +++ b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java @@ -20,6 +20,7 @@ import com.box.sdk.BoxAPIConnection; import com.box.sdk.BoxFile; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -45,7 +46,7 @@ public class BoxReceiver implements Connector { private BoxAPIConnection boxClient; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); @@ -54,6 +55,7 @@ public void init(String storageId, String credentialToken, String resourceServic boxClient = new BoxAPIConnection(boxSecret.getAccessToken()); } + @Override public void destroy() { diff --git a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java index 6cd50c51..9aa77c73 100644 --- a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java +++ b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java @@ -20,6 +20,7 @@ import com.box.sdk.BoxAPIConnection; import com.box.sdk.BoxFile; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -40,7 +41,7 @@ public class BoxSender implements Connector { private BoxAPIConnection boxClient; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); BoxSecret boxSecret = secretClient.box().getBoxSecret(BoxSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); @@ -48,6 +49,7 @@ public void init(String storageId, String credentialToken, String resourceServic boxClient = new BoxAPIConnection(boxSecret.getAccessToken()); } + @Override public void destroy() { diff --git a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java index 14f040d4..acb22a89 100644 --- a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java +++ b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java @@ -20,6 +20,7 @@ import com.dropbox.core.DbxRequestConfig; import com.dropbox.core.v2.DbxClientV2; import com.dropbox.core.v2.files.FileMetadata; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.ResourceTypes; @@ -60,7 +61,7 @@ private void checkInitialized() { } @Override - public FileResourceMetadata getFileResourceMetadata(String resourceId, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); DropboxResource dropboxResource = resourceClient.dropbox().getDropboxResource(DropboxResourceGetRequest.newBuilder().setResourceId(resourceId).build()); @@ -81,21 +82,21 @@ public FileResourceMetadata getFileResourceMetadata(String resourceId, String cr } @Override - public FileResourceMetadata getFileResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public Boolean isAvailable(String resourceId, String credentialToken) throws Exception { + public Boolean isAvailable(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); DropboxResource dropboxResource = resourceClient.dropbox().getDropboxResource(DropboxResourceGetRequest.newBuilder().setResourceId(resourceId).build()); diff --git a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java index d694f916..1f891c35 100644 --- a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java +++ b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java @@ -18,7 +18,9 @@ package org.apache.airavata.mft.transport.dropbox; import com.dropbox.core.DbxRequestConfig; +import com.dropbox.core.android.Auth; import com.dropbox.core.v2.DbxClientV2; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -43,7 +45,7 @@ public class DropboxReceiver implements Connector { private DbxClientV2 dbxClientV2; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); DropboxSecret dropboxSecret = secretClient.dropbox().getDropboxSecret(DropboxSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); @@ -51,6 +53,7 @@ public void init(String storageId, String credentialToken, String resourceServic dbxClientV2 = new DbxClientV2(config, dropboxSecret.getAccessToken()); } + @Override public void destroy() { diff --git a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java index da8f8b78..4a85ffff 100644 --- a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java +++ b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java @@ -21,6 +21,7 @@ import com.dropbox.core.v2.DbxClientV2; import com.dropbox.core.v2.files.FileMetadata; import com.dropbox.core.v2.files.WriteMode; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -43,7 +44,7 @@ public class DropboxSender implements Connector { private DbxClientV2 dbxClientV2; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); DropboxSecret dropboxSecret = secretClient.dropbox().getDropboxSecret(DropboxSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); @@ -52,6 +53,7 @@ public void init(String storageId, String credentialToken, String resourceServic dbxClientV2 = new DbxClientV2(config, dropboxSecret.getAccessToken()); } + @Override public void destroy() { diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java index cdd8ae03..5b047da4 100644 --- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java +++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java @@ -17,6 +17,7 @@ package org.apache.airavata.mft.transport.ftp; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.ResourceTypes; @@ -66,7 +67,7 @@ private void checkInitialized() { } @Override - public FileResourceMetadata getFileResourceMetadata(String resourceId, String credentialToken) { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); @@ -102,21 +103,22 @@ public FileResourceMetadata getFileResourceMetadata(String resourceId, String cr } @Override - public FileResourceMetadata getFileResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, String credentialToken) throws Exception { - throw new UnsupportedOperationException("Method not implemented"); } + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { + throw new UnsupportedOperationException("Method not implemented"); + } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public Boolean isAvailable(String resourceId, String credentialToken) { + public Boolean isAvailable(AuthZToken authZToken, String resourceId, String credentialToken) { checkInitialized(); @@ -150,7 +152,7 @@ public Boolean isAvailable(FTPResource ftpResource, String credentialToken) { ftpClient = FTPTransportUtil.getFTPClient(ftpResource.getFtpStorage(), ftpSecret); InputStream inputStream = null; - switch (ftpResource.getResourceCase().name()){ + switch (ftpResource.getResourceCase().name()) { case ResourceTypes.FILE: inputStream = ftpClient.retrieveFileStream(ftpResource.getFile().getResourcePath()); case ResourceTypes.DIRECTORY: diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java index f3b4ac89..1053fb38 100644 --- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java +++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java @@ -17,6 +17,8 @@ package org.apache.airavata.mft.transport.ftp; +import org.apache.airavata.mft.core.AuthZToken; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -45,7 +47,7 @@ public class FTPReceiver implements Connector { private FTPClient ftpClient; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { this.initialized = true; ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); @@ -57,6 +59,7 @@ public void init(String storageId, String credentialToken, String resourceServic this.ftpClient = FTPTransportUtil.getFTPClient(ftpStorage, ftpSecret); } + @Override public void destroy() { FTPTransportUtil.disconnectFTP(ftpClient); diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java index 5579cb86..32ae2380 100644 --- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java +++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java @@ -17,6 +17,7 @@ package org.apache.airavata.mft.transport.ftp; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.api.Connector; import org.apache.airavata.mft.credential.stubs.ftp.FTPSecret; @@ -42,7 +43,7 @@ public class FTPSender implements Connector { private FTPClient ftpClient; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { this.initialized = true; ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); @@ -54,6 +55,7 @@ public void init(String storageId, String credentialToken, String resourceServic this.ftpClient = FTPTransportUtil.getFTPClient(ftpStorage, ftpSecret); } + @Override public void destroy() { FTPTransportUtil.disconnectFTP(ftpClient); diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java index c753c0fa..64c80038 100644 --- a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java +++ b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java @@ -25,6 +25,7 @@ import com.google.api.services.storage.Storage; import com.google.api.services.storage.StorageScopes; import com.google.api.services.storage.model.StorageObject; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.ResourceTypes; @@ -71,7 +72,7 @@ private void checkInitialized() { } @Override - public FileResourceMetadata getFileResourceMetadata(String resourceId, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); GCSResource gcsResource = resourceClient.gcs().getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build()); @@ -102,21 +103,21 @@ public FileResourceMetadata getFileResourceMetadata(String resourceId, String cr } @Override - public FileResourceMetadata getFileResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public Boolean isAvailable(String resourceId, String credentialToken) throws Exception { + public Boolean isAvailable(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); GCSResource gcsResource = resourceClient.gcs().getGCSResource(GCSResourceGetRequest.newBuilder().setResourceId(resourceId).build()); diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSReceiver.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSReceiver.java index d734f92f..36de5e9a 100644 --- a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSReceiver.java +++ b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSReceiver.java @@ -24,6 +24,7 @@ import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.storage.Storage; import com.google.api.services.storage.StorageScopes; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -55,7 +56,7 @@ public class GCSReceiver implements Connector { private GCSStorage gcsStorage; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); this.gcsStorage = resourceClient.gcs().getGCSStorage(GCSStorageGetRequest.newBuilder().setStorageId(storageId).build()); @@ -74,6 +75,8 @@ public void init(String storageId, String credentialToken, String resourceServic storage = new Storage.Builder(transport, jsonFactory, credential).build(); } + + @Override public void destroy() { diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java index 4d0ef29d..6447ad94 100644 --- a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java +++ b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSSender.java @@ -30,7 +30,9 @@ import com.google.api.services.storage.model.StorageObject; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; +import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret; import org.apache.airavata.mft.credential.stubs.gcs.GCSSecretGetRequest; @@ -58,7 +60,7 @@ public class GCSSender implements Connector { private JsonObject jsonObject; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); this.gcsStorage = resourceClient.gcs().getGCSStorage(GCSStorageGetRequest.newBuilder().setStorageId(storageId).build()); @@ -79,6 +81,7 @@ public void init(String storageId, String credentialToken, String resourceServic storage = new Storage.Builder(transport, jsonFactory, credential).build(); } + @Override public void destroy() { diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java index 191a677d..31512595 100644 --- a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java +++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java @@ -17,6 +17,7 @@ package org.apache.airavata.mft.transport.local; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.ResourceTypes; @@ -60,7 +61,7 @@ private void checkInitialized() { } @Override - public FileResourceMetadata getFileResourceMetadata(String resourceId, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); LocalResource localResource = resourceClient.local().getLocalResource(LocalResourceGetRequest.newBuilder().setResourceId(resourceId).build()); @@ -95,21 +96,21 @@ public FileResourceMetadata getFileResourceMetadata(String resourceId, String cr } @Override - public FileResourceMetadata getFileResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public Boolean isAvailable(String resourceId, String credentialToken) throws Exception { + public Boolean isAvailable(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); LocalResource localResource = resourceClient.local().getLocalResource(LocalResourceGetRequest.newBuilder().setResourceId(resourceId).build()); return isAvailable(localResource, credentialToken); diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java index 53776ff5..c3acf100 100644 --- a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java +++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalReceiver.java @@ -17,6 +17,7 @@ package org.apache.airavata.mft.transport.local; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.api.Connector; import org.slf4j.Logger; @@ -31,11 +32,12 @@ public class LocalReceiver implements Connector { private boolean initialized; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { this.initialized = true; } + @Override public void destroy() { diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java index e4d5e480..dd537eea 100644 --- a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java +++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalSender.java @@ -17,6 +17,7 @@ package org.apache.airavata.mft.transport.local; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.api.Connector; import org.slf4j.Logger; @@ -30,12 +31,13 @@ public class LocalSender implements Connector { private boolean initialized; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { this.initialized = true; } + @Override public void destroy() { diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java index 741bd2ff..ff34e6af 100644 --- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java +++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java @@ -22,6 +22,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ObjectMetadata; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.ResourceTypes; @@ -62,7 +63,7 @@ private void checkInitialized() { } @Override - public FileResourceMetadata getFileResourceMetadata(String resourceId, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); @@ -88,21 +89,21 @@ public FileResourceMetadata getFileResourceMetadata(String resourceId, String cr } @Override - public FileResourceMetadata getFileResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { throw new UnsupportedOperationException("Method not implemented"); } @Override - public Boolean isAvailable(String resourceId, String credentialToken) throws Exception { + public Boolean isAvailable(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Receiver.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Receiver.java index 2e7e816b..de96c525 100644 --- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Receiver.java +++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Receiver.java @@ -23,6 +23,7 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -49,7 +50,7 @@ public class S3Receiver implements Connector { private S3Storage s3Storage; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, + public void init(AuthZToken authZToken,String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); @@ -65,6 +66,7 @@ public void init(String storageId, String credentialToken, String resourceServic .build(); } + @Override public void destroy() { diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Sender.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Sender.java index a7a640ca..b40f4aa7 100644 --- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Sender.java +++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Sender.java @@ -22,6 +22,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ObjectMetadata; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; @@ -46,7 +47,7 @@ public class S3Sender implements Connector { private S3Storage s3Storage; @Override - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { + public void init(AuthZToken authZToken,String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); this.s3Storage = resourceClient.s3().getS3Storage(S3StorageGetRequest.newBuilder().setStorageId(storageId).build()); diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java index 57597917..a5e1a2fb 100644 --- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java +++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java @@ -28,6 +28,7 @@ import net.schmizz.sshj.userauth.method.AuthPublickey; import net.schmizz.sshj.userauth.method.ChallengeResponseProvider; import net.schmizz.sshj.userauth.password.Resource; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; import org.apache.airavata.mft.core.ResourceTypes; @@ -80,7 +81,7 @@ private void checkInitialized() { } } - private FileResourceMetadata getFileResourceMetadata(SCPResource scpResource, SCPSecret scpSecret) throws Exception { + private FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, SCPResource scpResource, SCPSecret scpSecret) throws Exception { try (SSHClient sshClient = getSSHClient(scpResource, scpSecret)) { logger.info("Fetching metadata for resource {} in {}", scpResource.getFile().getResourcePath(), scpResource.getScpStorage().getHost()); @@ -120,7 +121,7 @@ private FileResourceMetadata getFileResourceMetadata(SCPResource scpResource, SC } } - public FileResourceMetadata getFileResourceMetadata(String resourceId, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); @@ -129,11 +130,11 @@ public FileResourceMetadata getFileResourceMetadata(String resourceId, String cr SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); SCPSecret scpSecret = secretClient.scp().getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); - return getFileResourceMetadata(scpResource, scpSecret); + return getFileResourceMetadata(authZToken,scpResource, scpSecret); } @Override - public FileResourceMetadata getFileResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public FileResourceMetadata getFileResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); SCPResource scpResource = resourceClient.scp().getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(parentResourceId).build()); @@ -145,10 +146,10 @@ public FileResourceMetadata getFileResourceMetadata(String parentResourceId, Str .setResourcePath(resourcePath).build()) .setScpStorage(scpResource.getScpStorage()).build(); - return getFileResourceMetadata(scpResource2, scpSecret); + return getFileResourceMetadata(authZToken, scpResource2, scpSecret); } - private DirectoryResourceMetadata getDirectoryResourceMetadata(SCPResource scpResource, SCPSecret scpSecret) throws Exception { + private DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, SCPResource scpResource, SCPSecret scpSecret) throws Exception { try (SSHClient sshClient = getSSHClient(scpResource, scpSecret)) { logger.info("Fetching metadata for resource {} in {}", scpResource.getFile().getResourcePath(), scpResource.getScpStorage().getHost()); @@ -159,7 +160,7 @@ private DirectoryResourceMetadata getDirectoryResourceMetadata(SCPResource scpRe sftpClient.close(); DirectoryResourceMetadata.Builder dirMetadataBuilder = DirectoryResourceMetadata.Builder.getBuilder() - .withLazyInitialized(false); + .withLazyInitialized(false); for (RemoteResourceInfo rri : lsOut) { if (rri.isDirectory()) { @@ -192,7 +193,7 @@ private DirectoryResourceMetadata getDirectoryResourceMetadata(SCPResource scpRe } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); SCPResource scpPResource = resourceClient.scp().getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build()); @@ -200,11 +201,11 @@ public DirectoryResourceMetadata getDirectoryResourceMetadata(String resourceId, SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); SCPSecret scpSecret = secretClient.scp().getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); - return getDirectoryResourceMetadata(scpPResource, scpSecret); + return getDirectoryResourceMetadata(authZToken,scpPResource, scpSecret); } @Override - public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResourceId, String resourcePath, String credentialToken) throws Exception { + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthZToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); SCPResource scpPResource = resourceClient.scp().getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(parentResourceId).build()); @@ -215,11 +216,11 @@ public DirectoryResourceMetadata getDirectoryResourceMetadata(String parentResou .setDirectory(DirectoryResource.newBuilder().setResourcePath(resourcePath).build()) .setScpStorage(scpPResource.getScpStorage()).build(); - return getDirectoryResourceMetadata(scpResource, scpSecret); + return getDirectoryResourceMetadata(authZToken,scpResource, scpSecret); } @Override - public Boolean isAvailable(String resourceId, String credentialToken) throws Exception { + public Boolean isAvailable(AuthZToken authZToken, String resourceId, String credentialToken) throws Exception { checkInitialized(); ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java index 516ff7dc..205402a7 100644 --- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java +++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java @@ -19,9 +19,11 @@ import com.jcraft.jsch.ChannelExec; import com.jcraft.jsch.Session; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.DoubleStreamingBuffer; import org.apache.airavata.mft.core.api.Connector; +import org.apache.airavata.mft.credential.stubs.common.AuthToken; import org.apache.airavata.mft.credential.stubs.scp.SCPSecret; import org.apache.airavata.mft.credential.stubs.scp.SCPSecretGetRequest; import org.apache.airavata.mft.resource.client.ResourceServiceClient; @@ -33,7 +35,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; public class SCPReceiver implements Connector { @@ -43,7 +47,7 @@ public class SCPReceiver implements Connector { private Session session; - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { if (initialized) { @@ -56,7 +60,13 @@ public void init(String storageId, String credentialToken, String resourceServic SCPStorage scpStorage = resourceClient.scp().getSCPStorage(SCPStorageGetRequest.newBuilder().setStorageId(storageId).build()); SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); - SCPSecret scpSecret = secretClient.scp().getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + AuthToken authToken = AuthToken.newBuilder() + .setToken(authZToken.getMftAuthorizationToken()).setAgentId(authZToken.getAgentId()) + .setAgentSecret(authZToken.getAgentSecret()) + .build(); + SCPSecret scpSecret = secretClient.scp().getSCPSecret(SCPSecretGetRequest + .newBuilder() + .setAuthzToken(authToken).setSecretId(credentialToken).build()); this.session = SCPTransportUtil.createSession( scpStorage.getUser(), @@ -64,9 +74,11 @@ public void init(String storageId, String credentialToken, String resourceServic scpStorage.getPort(), scpSecret.getPrivateKey().getBytes(), scpSecret.getPublicKey().getBytes(), - scpSecret.getPassphrase().equals("")? null : scpSecret.getPassphrase().getBytes()); + scpSecret.getPassphrase().equals("") ? null : scpSecret.getPassphrase().getBytes()); + } + public void destroy() { try { this.session.disconnect(); diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java index 01b8ed47..712816e3 100644 --- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java +++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java @@ -20,10 +20,12 @@ import com.jcraft.jsch.ChannelExec; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; +import org.apache.airavata.mft.core.AuthZToken; import org.apache.airavata.mft.core.ConnectorContext; import org.apache.airavata.mft.core.DoubleStreamingBuffer; import org.apache.airavata.mft.core.ResourceTypes; import org.apache.airavata.mft.core.api.Connector; +import org.apache.airavata.mft.credential.stubs.common.AuthToken; import org.apache.airavata.mft.credential.stubs.scp.SCPSecret; import org.apache.airavata.mft.credential.stubs.scp.SCPSecretGetRequest; import org.apache.airavata.mft.resource.client.ResourceServiceClient; @@ -47,7 +49,7 @@ public class SCPSender implements Connector { private Session session; - public void init(String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, + public void init(AuthZToken authZToken, String storageId, String credentialToken, String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception { if (initialized) { @@ -73,6 +75,7 @@ public void init(String storageId, String credentialToken, String resourceServic scpSecret.getPassphrase().equals("")? null : scpSecret.getPassphrase().getBytes()); } + public void destroy() { try { From bca18e3f140d76e590185a4a6ec5a45a486dafe2 Mon Sep 17 00:00:00 2001 From: Isuru Ranawaka Date: Wed, 3 Feb 2021 14:21:41 -0500 Subject: [PATCH 2/7] Custos Secret backend, passing AuthZToken --- agent/src/main/resources/application.properties | 2 +- .../server/src/main/resources/application.properties | 4 ++-- .../main/resources/distribution/conf/application.properties | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/agent/src/main/resources/application.properties b/agent/src/main/resources/application.properties index 5863e257..379794af 100644 --- a/agent/src/main/resources/application.properties +++ b/agent/src/main/resources/application.properties @@ -17,7 +17,7 @@ spring.main.web-application-type=NONE agent.id=mft-agent-a -agent.secret=jENXjJetKXvffqRjLK9PWXZWMegz7d7cJG8VXGHo +agent.secret=CHANGE_ME agent.host=localhost agent.user=dimuthu agent.http.port=3333 diff --git a/services/secret-service/server/src/main/resources/application.properties b/services/secret-service/server/src/main/resources/application.properties index 7a1a0f55..6c315cbb 100644 --- a/services/secret-service/server/src/main/resources/application.properties +++ b/services/secret-service/server/src/main/resources/application.properties @@ -28,5 +28,5 @@ file.backend.secret.file=secrets.json #Configurations for Custos Backend custos.host=custos.scigap.org custos.port=31499 -custos.id=custos-2zuomcugra3ebgsqtzmf-10000514 -custos.secret=mupUhF4JL0S3IFHBjfhiTfLJS1NgSWfvkCj3l6c7 \ No newline at end of file +custos.id=CHANGE_ME +custos.secret=CHANGE_ME \ No newline at end of file diff --git a/services/secret-service/server/src/main/resources/distribution/conf/application.properties b/services/secret-service/server/src/main/resources/distribution/conf/application.properties index f904f17b..e5b6af0f 100644 --- a/services/secret-service/server/src/main/resources/distribution/conf/application.properties +++ b/services/secret-service/server/src/main/resources/distribution/conf/application.properties @@ -28,5 +28,5 @@ file.backend.secret.file=secrets.json # Configurations for Custos Backend custos.host=custos.scigap.org custos.port=31499 -custos.id=custos-2zuomcugra3ebgsqtzmf-10000514 -custos.secret=mupUhF4JL0S3IFHBjfhiTfLJS1NgSWfvkCj3l6c7 \ No newline at end of file +custos.id=CHANGE_ME +custos.secret=CHANGE_ME \ No newline at end of file From 200815f26585be806e85b18d857e94c34195afd0 Mon Sep 17 00:00:00 2001 From: Isuru Ranawaka Date: Fri, 12 Feb 2021 08:50:20 -0500 Subject: [PATCH 3/7] implement secret backend get methods --- .../backend/custos/CustosSecretBackend.java | 154 +++++++++++++++++- 1 file changed, 152 insertions(+), 2 deletions(-) diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java index 8a666368..285422cf 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java @@ -14,12 +14,15 @@ import org.apache.custos.identity.management.client.IdentityManagementClient; import org.apache.custos.resource.secret.management.client.ResourceSecretManagementAgentClient; import org.apache.custos.resource.secret.management.client.ResourceSecretManagementClient; +import org.apache.custos.resource.secret.service.CredentialMap; +import org.apache.custos.resource.secret.service.PasswordCredential; import org.apache.custos.resource.secret.service.SSHCredential; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import java.util.Map; import java.util.Optional; /** @@ -78,7 +81,6 @@ public Optional getSCPSecret(SCPSecretGetRequest request) throws Exce .setPublicKey(sshCredential.getPublicKey()) .setPassphrase(sshCredential.getPassphrase()) .setPrivateKey(sshCredential.getPrivateKey()).build(); - LOGGER.info("Public key " + sshCredential.getPublicKey()); return Optional.of(scpSecret); } else if (!request.getAuthzToken().getToken().isEmpty()) { @@ -89,7 +91,6 @@ public Optional getSCPSecret(SCPSecretGetRequest request) throws Exce .setPublicKey(sshCredential.getPublicKey()) .setPassphrase(sshCredential.getPassphrase()) .setPrivateKey(sshCredential.getPrivateKey()).build(); - LOGGER.info("Public key " + sshCredential.getPublicKey()); return Optional.of(scpSecret); } } @@ -113,6 +114,31 @@ public boolean deleteSCPSecret(SCPSecretDeleteRequest request) { @Override public Optional getS3Secret(S3SecretGetRequest request) throws Exception { + String agentId = request.getAuthzToken().getAgentId(); + String secret = request.getAuthzToken().getAgentSecret(); + Optional optionalAuthConfig = handler.authenticate(agentId, secret); + if (optionalAuthConfig.isPresent()) { + AuthConfig authConfig = optionalAuthConfig.get(); + CredentialMap credentialMap = csAgentClient.getCredentialMap(request.getAuthzToken().getToken(), + authConfig.getAccessToken(), custosId, request.getSecretId()); + Map secretValues = credentialMap.getCredentialMapMap(); + S3Secret s3Secret = S3Secret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setAccessKey(secretValues.get("accessKey")) + .setSecretKey(secretValues.get("secretKey")).build(); + return Optional.of(s3Secret); + + } else if (!request.getAuthzToken().getToken().isEmpty()) { + if (identityClient.isAuthenticated(request.getAuthzToken().getToken())) { + CredentialMap credentialMap = csClient.getCredentialMap(custosId, request.getAuthzToken().getToken()); + Map secretValues = credentialMap.getCredentialMapMap(); + S3Secret s3Secret = S3Secret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setAccessKey(secretValues.get("accessKey")) + .setSecretKey(secretValues.get("secretKey")).build(); + return Optional.of(s3Secret); + } + } return Optional.empty(); } @@ -133,6 +159,29 @@ public boolean deleteS3Secret(S3SecretDeleteRequest request) throws Exception { @Override public Optional getBoxSecret(BoxSecretGetRequest request) throws Exception { + String agentId = request.getAuthzToken().getAgentId(); + String secret = request.getAuthzToken().getAgentSecret(); + Optional optionalAuthConfig = handler.authenticate(agentId, secret); + if (optionalAuthConfig.isPresent()) { + AuthConfig authConfig = optionalAuthConfig.get(); + CredentialMap credentialMap = csAgentClient.getCredentialMap(request.getAuthzToken().getToken(), + authConfig.getAccessToken(), custosId, request.getSecretId()); + Map secretValues = credentialMap.getCredentialMapMap(); + BoxSecret boxSecret = BoxSecret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setAccessToken(secretValues.get("accessToken")).build(); + return Optional.of(boxSecret); + + } else if (!request.getAuthzToken().getToken().isEmpty()) { + if (identityClient.isAuthenticated(request.getAuthzToken().getToken())) { + CredentialMap credentialMap = csClient.getCredentialMap(custosId, request.getAuthzToken().getToken()); + Map secretValues = credentialMap.getCredentialMapMap(); + BoxSecret boxSecret = BoxSecret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setAccessToken(secretValues.get("accessToken")).build(); + return Optional.of(boxSecret); + } + } return Optional.empty(); } @@ -153,6 +202,31 @@ public boolean deleteBoxSecret(BoxSecretDeleteRequest request) throws Exception @Override public Optional getAzureSecret(AzureSecretGetRequest request) throws Exception { + String agentId = request.getAuthzToken().getAgentId(); + String secret = request.getAuthzToken().getAgentSecret(); + Optional optionalAuthConfig = handler.authenticate(agentId, secret); + if (optionalAuthConfig.isPresent()) { + AuthConfig authConfig = optionalAuthConfig.get(); + CredentialMap credentialMap = csAgentClient.getCredentialMap(request.getAuthzToken().getToken(), + authConfig.getAccessToken(), custosId, request.getSecretId()); + Map secretValues = credentialMap.getCredentialMapMap(); + AzureSecret azureSecret = AzureSecret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setConnectionString(secretValues.get("connectionString")).build(); + + return Optional.of(azureSecret); + + } else if (!request.getAuthzToken().getToken().isEmpty()) { + if (identityClient.isAuthenticated(request.getAuthzToken().getToken())) { + CredentialMap credentialMap = csClient.getCredentialMap(custosId, request.getAuthzToken().getToken()); + Map secretValues = credentialMap.getCredentialMapMap(); + AzureSecret azureSecret = AzureSecret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setConnectionString(secretValues.get("connectionString")).build(); + + return Optional.of(azureSecret); + } + } return Optional.empty(); } @@ -173,6 +247,31 @@ public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws Except @Override public Optional getGCSSecret(GCSSecretGetRequest request) throws Exception { + String agentId = request.getAuthzToken().getAgentId(); + String secret = request.getAuthzToken().getAgentSecret(); + Optional optionalAuthConfig = handler.authenticate(agentId, secret); + if (optionalAuthConfig.isPresent()) { + AuthConfig authConfig = optionalAuthConfig.get(); + CredentialMap credentialMap = csAgentClient.getCredentialMap(request.getAuthzToken().getToken(), + authConfig.getAccessToken(), custosId, request.getSecretId()); + Map secretValues = credentialMap.getCredentialMapMap(); + GCSSecret gcsSecret = GCSSecret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setCredentialsJson(secretValues.get("credentialsJson")).build(); + + return Optional.of(gcsSecret); + + } else if (!request.getAuthzToken().getToken().isEmpty()) { + if (identityClient.isAuthenticated(request.getAuthzToken().getToken())) { + CredentialMap credentialMap = csClient.getCredentialMap(custosId, request.getAuthzToken().getToken()); + Map secretValues = credentialMap.getCredentialMapMap(); + GCSSecret gcsSecret = GCSSecret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setCredentialsJson(secretValues.get("credentialsJson")).build(); + + return Optional.of(gcsSecret); + } + } return Optional.empty(); } @@ -193,6 +292,31 @@ public boolean deleteGCSSecret(GCSSecretDeleteRequest request) throws Exception @Override public Optional getDropboxSecret(DropboxSecretGetRequest request) throws Exception { + String agentId = request.getAuthzToken().getAgentId(); + String secret = request.getAuthzToken().getAgentSecret(); + Optional optionalAuthConfig = handler.authenticate(agentId, secret); + if (optionalAuthConfig.isPresent()) { + AuthConfig authConfig = optionalAuthConfig.get(); + CredentialMap credentialMap = csAgentClient.getCredentialMap(request.getAuthzToken().getToken(), + authConfig.getAccessToken(), custosId, request.getSecretId()); + Map secretValues = credentialMap.getCredentialMapMap(); + DropboxSecret dropboxSecret = DropboxSecret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setAccessToken(secretValues.get("accessToken")).build(); + + return Optional.of(dropboxSecret); + + } else if (!request.getAuthzToken().getToken().isEmpty()) { + if (identityClient.isAuthenticated(request.getAuthzToken().getToken())) { + CredentialMap credentialMap = csClient.getCredentialMap(custosId, request.getAuthzToken().getToken()); + Map secretValues = credentialMap.getCredentialMapMap(); + DropboxSecret dropboxSecret = DropboxSecret.newBuilder() + .setSecretId(secretValues.get("secretId")) + .setAccessToken(secretValues.get("accessToken")).build(); + + return Optional.of(dropboxSecret); + } + } return Optional.empty(); } @@ -213,6 +337,32 @@ public boolean deleteDropboxSecret(DropboxSecretDeleteRequest request) throws Ex @Override public Optional getFTPSecret(FTPSecretGetRequest request) throws Exception { + String agentId = request.getAuthzToken().getAgentId(); + String secret = request.getAuthzToken().getAgentSecret(); + Optional optionalAuthConfig = handler.authenticate(agentId, secret); + if (optionalAuthConfig.isPresent()) { + AuthConfig authConfig = optionalAuthConfig.get(); + PasswordCredential passwordCredential = csAgentClient.getPasswordCredential(request.getAuthzToken().getToken(), + authConfig.getAccessToken(), custosId, request.getSecretId()); + FTPSecret ftpSecret = FTPSecret.newBuilder() + .setSecretId(request.getSecretId()) + .setPassword(passwordCredential.getPassword()) + .setUserId(passwordCredential.getUserId()) + .build(); + + return Optional.of(ftpSecret); + + } else if (!request.getAuthzToken().getToken().isEmpty()) { + if (identityClient.isAuthenticated(request.getAuthzToken().getToken())) { + PasswordCredential passwordCredential = csClient.getPasswordCredential(custosId, request.getAuthzToken().getToken()); + FTPSecret ftpSecret = FTPSecret.newBuilder() + .setSecretId(request.getSecretId()) + .setPassword(passwordCredential.getPassword()) + .setUserId(passwordCredential.getUserId()) + .build(); + return Optional.of(ftpSecret); + } + } return Optional.empty(); } From cbf5b9d44442cca92502cfb040a0e1da2b27b283 Mon Sep 17 00:00:00 2001 From: Isuru Ranawaka Date: Fri, 12 Feb 2021 10:38:19 -0500 Subject: [PATCH 4/7] Add cleanup methpds --- .../airavata/mft/secret/server/AppConfig.java | 6 ++++-- .../backend/custos/CustosSecretBackend.java | 13 ++++++++++++- .../auth/AgentAuthenticationHandler.java | 19 ++++++++++++++----- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java index aa0e98a8..d032bc12 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java @@ -23,6 +23,8 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.io.IOException; + @Configuration public class AppConfig { @@ -48,8 +50,8 @@ public CustosClientProvider custosClientProvider() { } @Bean - public AgentAuthenticationHandler agentAuthenticationHandler() { - return new AgentAuthenticationHandler(this.custosId); + public AgentAuthenticationHandler agentAuthenticationHandler(CustosClientProvider custosClientProvider) throws IOException { + return new AgentAuthenticationHandler(this.custosId, custosClientProvider); } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java index 285422cf..866d46a6 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import java.io.IOException; import java.util.Map; import java.util.Optional; @@ -63,7 +64,17 @@ public void init() { @Override public void destroy() { - + try { + this.csAgentClient.close(); + this.csClient.close(); + this.identityClient.close(); + } catch (IOException e) { + LOGGER.error("Error while closing agents"); + } finally { + this.csAgentClient = null; + this.csClient = null; + this.identityClient = null; + } } @Override diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java index 9825f237..51f6cc80 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java @@ -8,6 +8,8 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import java.io.Closeable; +import java.io.IOException; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; @@ -15,7 +17,7 @@ /** * Handle agent authentication */ -public class AgentAuthenticationHandler implements AuthenticationHandler { +public class AgentAuthenticationHandler implements AuthenticationHandler, Closeable { private static final Logger LOGGER = LoggerFactory.getLogger(AgentAuthenticationHandler.class); private static final String CLIENT_CREDENTIALS = "client_credentials"; @@ -24,22 +26,23 @@ public class AgentAuthenticationHandler implements AuthenticationHandler { private String custosId; - @Autowired - private CustosClientProvider custosClientProvider; + IdentityManagementClient identityManagementClient; - public AgentAuthenticationHandler(String custosId) { + + public AgentAuthenticationHandler(String custosId, CustosClientProvider custosClientProvider) throws IOException { this.custosId = custosId; + this.identityManagementClient = custosClientProvider.getIdentityManagementClient(); } @Override public Optional authenticate(String id, String secret) throws Exception { + IdentityManagementClient identityManagementClient = null; try { AuthConfig cachedAuthConfig = authCache.get(id); AuthConfig authConfig = new AuthConfig(); final boolean agentRequest = id != null & secret != null & !id.isEmpty() & !secret.isEmpty(); if (cachedAuthConfig == null && agentRequest) { - IdentityManagementClient identityManagementClient = custosClientProvider.getIdentityManagementClient(); Struct tokenResponse = identityManagementClient.getAgentToken(custosId, id, secret, CLIENT_CREDENTIALS, null); @@ -73,4 +76,10 @@ public Optional authenticate(String id, String secret) throws Except } + @Override + public void close() throws IOException { + if(this.identityManagementClient != null){ + this.identityManagementClient.close(); + } + } } From 07019597bbced7d57010dbbf595e6847665775a6 Mon Sep 17 00:00:00 2001 From: Isuru Ranawaka Date: Sun, 21 Feb 2021 22:20:59 -0500 Subject: [PATCH 5/7] Add agent authentication bean --- agent/src/main/resources/application.properties | 2 +- .../apache/airavata/mft/secret/server/AppConfig.java | 4 ++-- .../custos/auth/AgentAuthenticationHandler.java | 10 +++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/agent/src/main/resources/application.properties b/agent/src/main/resources/application.properties index 379794af..e7cbcef5 100644 --- a/agent/src/main/resources/application.properties +++ b/agent/src/main/resources/application.properties @@ -16,7 +16,7 @@ # spring.main.web-application-type=NONE -agent.id=mft-agent-a +agent.id=mft-agent-c agent.secret=CHANGE_ME agent.host=localhost agent.user=dimuthu diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java index d032bc12..89dd5eab 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java @@ -50,8 +50,8 @@ public CustosClientProvider custosClientProvider() { } @Bean - public AgentAuthenticationHandler agentAuthenticationHandler(CustosClientProvider custosClientProvider) throws IOException { - return new AgentAuthenticationHandler(this.custosId, custosClientProvider); + public AgentAuthenticationHandler agentAuthenticationHandler() throws IOException { + return new AgentAuthenticationHandler(custosId, custosClientProvider()); } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java index 51f6cc80..22f6e82c 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/auth/AgentAuthenticationHandler.java @@ -7,6 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import java.io.Closeable; import java.io.IOException; @@ -27,17 +28,20 @@ public class AgentAuthenticationHandler implements AuthenticationHandler, Closea private String custosId; - IdentityManagementClient identityManagementClient; + private IdentityManagementClient identityManagementClient; - public AgentAuthenticationHandler(String custosId, CustosClientProvider custosClientProvider) throws IOException { + @Autowired + private CustosClientProvider custosClientProvider; + + + public AgentAuthenticationHandler(@Value("${custos.id}") String custosId, @Autowired CustosClientProvider custosClientProvider) throws IOException { this.custosId = custosId; this.identityManagementClient = custosClientProvider.getIdentityManagementClient(); } @Override public Optional authenticate(String id, String secret) throws Exception { - IdentityManagementClient identityManagementClient = null; try { AuthConfig cachedAuthConfig = authCache.get(id); AuthConfig authConfig = new AuthConfig(); From 127ab06bf3559659f8a9b744bd7bd08228e72796 Mon Sep 17 00:00:00 2001 From: Isuru Ranawaka Date: Thu, 25 Feb 2021 22:21:26 -0500 Subject: [PATCH 6/7] Support dynamic client providers --- .../secret/server/backend/custos/CustosClientsFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosClientsFactory.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosClientsFactory.java index 6336ebd1..8ebac865 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosClientsFactory.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosClientsFactory.java @@ -12,6 +12,8 @@ public class CustosClientsFactory { private int custosPort; + private Map custosClientProviderMap = new ConcurrentHashMap<>(); + public CustosClientsFactory(String custosHost, int custosPort, String custosId, String custosSecret) { this.custosHost = custosHost; this.custosPort = custosPort; @@ -23,8 +25,6 @@ public CustosClientsFactory(String custosHost, int custosPort, String custosId, } - private Map custosClientProviderMap = new ConcurrentHashMap<>(); - public CustosClientProvider getCustosClientProvider(String custosId, String custosSecret) { From 82087eb2040864ed118d0323b7618263a1dc8e63 Mon Sep 17 00:00:00 2001 From: Isuru Ranawaka Date: Thu, 25 Feb 2021 22:43:30 -0500 Subject: [PATCH 7/7] Change property variable name --- .../backend/custos/CustosSecretBackend.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java index 0a2d879d..33bd15f5 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/custos/CustosSecretBackend.java @@ -113,7 +113,7 @@ public Optional getSCPSecret(SCPSecretGetRequest request) throws Exce case DELEGATEAUTH: DelegateAuth delegateAuth = request.getAuthzToken().getDelegateAuth(); ResourceSecretManagementClient csClient = getTenantResourceSecretManagementClient(delegateAuth); - SSHCredential sshCredential = csClient.getSSHCredential(delegateAuth.getClientId(), + SSHCredential sshCredential = csClient.getSSHCredential(delegateAuth.getPropertiesMap().get("PORTAL_CUSTOS_ID"), request.getSecretId(), false); SCPSecret scpSecret = SCPSecret.newBuilder() .setSecretId(sshCredential.getMetadata().getToken()) @@ -174,7 +174,7 @@ public Optional getS3Secret(S3SecretGetRequest request) throws Excepti case DELEGATEAUTH: DelegateAuth delegateAuth = request.getAuthzToken().getDelegateAuth(); ResourceSecretManagementClient csClient = getTenantResourceSecretManagementClient(delegateAuth); - CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getClientId(), + CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getPropertiesMap().get("PORTAL_CUSTOS_ID"), request.getSecretId()); Map secretValues = credentialMap.getCredentialMapMap(); S3Secret s3Secret = S3Secret.newBuilder() @@ -233,7 +233,8 @@ public Optional getBoxSecret(BoxSecretGetRequest request) throws Exce case DELEGATEAUTH: DelegateAuth delegateAuth = request.getAuthzToken().getDelegateAuth(); ResourceSecretManagementClient csClient = getTenantResourceSecretManagementClient(delegateAuth); - CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getClientId(), request.getSecretId()); + CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getPropertiesMap().get("PORTAL_CUSTOS_ID"), + request.getSecretId()); Map secretValues = credentialMap.getCredentialMapMap(); BoxSecret boxSecret = BoxSecret.newBuilder() .setSecretId(secretValues.get("secretId")) @@ -295,7 +296,8 @@ public Optional getAzureSecret(AzureSecretGetRequest request) throw case DELEGATEAUTH: DelegateAuth delegateAuth = request.getAuthzToken().getDelegateAuth(); ResourceSecretManagementClient csClient = getTenantResourceSecretManagementClient(delegateAuth); - CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getClientId(), request.getSecretId()); + CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getPropertiesMap().get("PORTAL_CUSTOS_ID"), + request.getSecretId()); Map secretValues = credentialMap.getCredentialMapMap(); AzureSecret azureSecret = AzureSecret.newBuilder() .setSecretId(secretValues.get("secretId")) @@ -356,7 +358,8 @@ public Optional getGCSSecret(GCSSecretGetRequest request) throws Exce case DELEGATEAUTH: DelegateAuth delegateAuth = request.getAuthzToken().getDelegateAuth(); ResourceSecretManagementClient csClient = getTenantResourceSecretManagementClient(delegateAuth); - CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getClientId(), request.getSecretId()); + CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getPropertiesMap().get("PORTAL_CUSTOS_ID"), + request.getSecretId()); Map secretValues = credentialMap.getCredentialMapMap(); GCSSecret gcsSecret = GCSSecret.newBuilder() .setSecretId(secretValues.get("secretId")) @@ -417,7 +420,8 @@ public Optional getDropboxSecret(DropboxSecretGetRequest request) case DELEGATEAUTH: DelegateAuth delegateAuth = request.getAuthzToken().getDelegateAuth(); ResourceSecretManagementClient csClient = getTenantResourceSecretManagementClient(delegateAuth); - CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getClientId(), request.getSecretId()); + CredentialMap credentialMap = csClient.getCredentialMap(delegateAuth.getPropertiesMap().get("PORTAL_CUSTOS_ID"), + request.getSecretId()); Map secretValues = credentialMap.getCredentialMapMap(); DropboxSecret dropboxSecret = DropboxSecret.newBuilder() .setSecretId(secretValues.get("secretId")) @@ -480,8 +484,9 @@ public Optional getFTPSecret(FTPSecretGetRequest request) throws Exce case DELEGATEAUTH: DelegateAuth delegateAuth = request.getAuthzToken().getDelegateAuth(); ResourceSecretManagementClient csClient = getTenantResourceSecretManagementClient(delegateAuth); - PasswordCredential passwordCredential = csClient.getPasswordCredential(delegateAuth.getClientId(), - request.getSecretId()); + PasswordCredential passwordCredential = csClient + .getPasswordCredential(delegateAuth.getPropertiesMap().get("PORTAL_CUSTOS_ID"), + request.getSecretId()); FTPSecret ftpSecret = FTPSecret.newBuilder() .setSecretId(request.getSecretId()) .setPassword(passwordCredential.getPassword()) @@ -509,8 +514,8 @@ public boolean deleteFTPSecret(FTPSecretDeleteRequest request) throws Exception private ResourceSecretManagementClient getTenantResourceSecretManagementClient(DelegateAuth delegateAuth) throws IOException { - String adminCustosId = delegateAuth.getPropertiesMap().get("ADMIN_TENANT_CUSTOS_ID"); - String adminCustosSecret = delegateAuth.getPropertiesMap().get("ADMIN_TENANT_CUSTOS_SECRET"); + String adminCustosId = delegateAuth.getClientId(); + String adminCustosSecret = delegateAuth.getClientSecret(); CustosClientProvider custosClientProvider = custosClientsFactory .getCustosClientProvider(adminCustosId, adminCustosSecret); return custosClientProvider