diff --git a/agent/pom.xml b/agent/pom.xml index c0ab77d7..310c41a7 100644 --- a/agent/pom.xml +++ b/agent/pom.xml @@ -84,6 +84,11 @@ mft-swift-transport ${project.version} + + org.apache.airavata + mft-odata-transport + ${project.version} + org.apache.airavata mft-common-clients diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java b/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java index 0323437c..0ff87d49 100644 --- a/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java +++ b/agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java @@ -195,7 +195,7 @@ public void transferSingleThread(String transferId, inConnector.complete(); outConnector.complete(); - logger.info("Completed streaming ransfer for transfer {}", transferId); + logger.info("Completed streaming transfer for transfer {}", transferId); } else { throw new Exception("No matching connector found to perform the transfer"); diff --git a/command-line/src/main/java/org/apache/airavata/mft/command/line/MainRunner.java b/command-line/src/main/java/org/apache/airavata/mft/command/line/MainRunner.java index 072c9889..b1002e9d 100644 --- a/command-line/src/main/java/org/apache/airavata/mft/command/line/MainRunner.java +++ b/command-line/src/main/java/org/apache/airavata/mft/command/line/MainRunner.java @@ -1,5 +1,6 @@ package org.apache.airavata.mft.command.line; +import org.apache.airavata.mft.command.line.sub.odata.ODataSubCommand; import org.apache.airavata.mft.command.line.sub.s3.S3SubCommand; import org.apache.airavata.mft.command.line.sub.swift.SwiftSubCommand; import org.apache.airavata.mft.command.line.sub.transfer.TransferSubCommand; @@ -8,7 +9,7 @@ @Command(name = "checksum", mixinStandardHelpOptions = true, version = "checksum 4.0", description = "Prints the checksum (SHA-256 by default) of a file to STDOUT.", - subcommands = {S3SubCommand.class, TransferSubCommand.class, SwiftSubCommand.class}) + subcommands = {S3SubCommand.class, TransferSubCommand.class, SwiftSubCommand.class, ODataSubCommand.class}) class MainRunner { public static void main(String... args) { diff --git a/command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataRemoteAddSubCommand.java b/command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataRemoteAddSubCommand.java new file mode 100644 index 00000000..226d1946 --- /dev/null +++ b/command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataRemoteAddSubCommand.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.command.line.sub.odata; + +import org.apache.airavata.mft.api.client.MFTApiClient; +import org.apache.airavata.mft.common.AuthToken; +import org.apache.airavata.mft.credential.stubs.odata.ODataSecret; +import org.apache.airavata.mft.credential.stubs.odata.ODataSecretCreateRequest; +import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage; +import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageCreateRequest; +import org.apache.airavata.mft.storage.stubs.storagesecret.StorageSecret; +import org.apache.airavata.mft.storage.stubs.storagesecret.StorageSecretCreateRequest; +import org.apache.airavata.mft.storage.stubs.storagesecret.StorageSecretServiceGrpc; +import picocli.CommandLine; + +import java.util.concurrent.Callable; + +@CommandLine.Command(name = "add") +public class ODataRemoteAddSubCommand implements Callable { + + @CommandLine.Option(names = {"-n", "--name"}, description = "Storage Name") + private String remoteName; + + @CommandLine.Option(names = {"-U", "--url"}, description = "Base URL for OData Endpoint") + private String baseURL; + + @CommandLine.Option(names = {"-u", "--user"}, description = "User Name") + private String userName; + + @CommandLine.Option(names = {"-p", "--password"}, description = "Password") + private String password; + + + @Override + public Integer call() throws Exception { + AuthToken authToken = AuthToken.newBuilder().build(); + + MFTApiClient mftApiClient = MFTApiClient.MFTApiClientBuilder.newBuilder().build(); + + ODataSecret oDataSecret = mftApiClient.getSecretServiceClient().odata().createODataSecret(ODataSecretCreateRequest.newBuilder() + .setAuthzToken(authToken).setPassword(password).setUserName(userName).build()); + + System.out.println("Created the OData secret " + oDataSecret.getSecretId()); + + ODataStorage oDataStorage = mftApiClient.getStorageServiceClient().odata().createODataStorage( + ODataStorageCreateRequest.newBuilder().setName(remoteName).setBaseUrl(baseURL).build()); + + System.out.println("Created OData storage " + oDataStorage.getStorageId()); + + StorageSecretServiceGrpc.StorageSecretServiceBlockingStub storageSecretClient = mftApiClient.getStorageServiceClient().storageSecret(); + + StorageSecret storageSecret = storageSecretClient.createStorageSecret(StorageSecretCreateRequest.newBuilder() + .setStorageId(oDataStorage.getStorageId()) + .setSecretId(oDataSecret.getSecretId()) + .setType(StorageSecret.StorageType.ODATA).build()); + + System.out.println("Successfully added OData remote endpoint"); + + return 0; + } +} diff --git a/command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataRemoteSubCommand.java b/command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataRemoteSubCommand.java new file mode 100644 index 00000000..3628a730 --- /dev/null +++ b/command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataRemoteSubCommand.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.command.line.sub.odata; + +import org.apache.airavata.mft.api.client.MFTApiClient; +import org.apache.airavata.mft.command.line.CommandLineUtil; +import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage; +import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageListRequest; +import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageListResponse; +import picocli.CommandLine; + +import java.util.List; + +@CommandLine.Command(name = "remote", subcommands = {ODataRemoteAddSubCommand.class}) +public class ODataRemoteSubCommand { + + @CommandLine.Command(name = "list") + void listS3Resource() { + System.out.println("Listing S3 Resource"); + MFTApiClient mftApiClient = MFTApiClient.MFTApiClientBuilder.newBuilder().build(); + + ODataStorageListResponse oDataStorageListResponse = mftApiClient.getStorageServiceClient().odata() + .listODataStorage(ODataStorageListRequest.newBuilder().setOffset(0).setLimit(10).build()); + + List storagesList = oDataStorageListResponse.getStoragesList(); + + int[] columnWidth = {40, 15, 55,}; + String[][] content = new String[storagesList.size() + 1][3]; + String[] headers = {"STORAGE ID", "NAME", "BASE URL"}; + content[0] = headers; + + + for (int i = 1; i <= storagesList.size(); i ++) { + ODataStorage storage = storagesList.get(i - 1); + content[i][0] = storage.getStorageId(); + content[i][1] = storage.getName(); + content[i][2] = storage.getBaseUrl(); + } + + CommandLineUtil.printTable(columnWidth, content); + } +} diff --git a/command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataSubCommand.java b/command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataSubCommand.java new file mode 100644 index 00000000..c2b4ae9b --- /dev/null +++ b/command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataSubCommand.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.command.line.sub.odata; + +import org.apache.airavata.mft.command.line.sub.swift.SwiftRemoteSubCommand; +import picocli.CommandLine; + +@CommandLine.Command(name = "odata", description = "Manage OData resources and credentials", + subcommands = {ODataRemoteSubCommand.class}) +public class ODataSubCommand { +} diff --git a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java index a3910a5c..ef5792ce 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java +++ b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java @@ -36,6 +36,9 @@ public static Optional resolveIncomingStreamingConne case "S3": className = "org.apache.airavata.mft.transport.s3.S3IncomingConnector"; break; + case "ODATA": + className = "org.apache.airavata.mft.transport.odata.ODataIncomingConnector"; + break; } if (className != null) { @@ -53,6 +56,10 @@ public static Optional resolveOutgoingStreamingConne case "SCP": className = "org.apache.airavata.mft.transport.scp.SCPOutgoingConnector"; break; + case "S3": + className = "org.apache.airavata.mft.transport.s3.S3OutgoingStreamingConnector"; + break; + } if (className != null) { diff --git a/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java index 9ab820a4..f6ac7c86 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java +++ b/core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java @@ -54,6 +54,9 @@ public static Optional resolveMetadataCollector(String type) case "SWIFT": className = "org.apache.airavata.mft.transport.swift.SwiftMetadataCollector"; break; + case "ODATA": + className = "org.apache.airavata.mft.transport.odata.ODataMetadataCollector"; + break; } if (className != null) { diff --git a/pom.xml b/pom.xml index dc00c6c4..b2254166 100755 --- a/pom.xml +++ b/pom.xml @@ -150,6 +150,7 @@ 2.5.1 2.5.0 2.6 + 4.5.13 diff --git a/services/resource-service/client/src/main/java/org/apache/airavata/mft/resource/client/StorageServiceClient.java b/services/resource-service/client/src/main/java/org/apache/airavata/mft/resource/client/StorageServiceClient.java index 73ea91a1..2e31c9d5 100644 --- a/services/resource-service/client/src/main/java/org/apache/airavata/mft/resource/client/StorageServiceClient.java +++ b/services/resource-service/client/src/main/java/org/apache/airavata/mft/resource/client/StorageServiceClient.java @@ -7,6 +7,7 @@ import org.apache.airavata.mft.resource.service.ftp.FTPStorageServiceGrpc; import org.apache.airavata.mft.resource.service.gcs.GCSStorageServiceGrpc; import org.apache.airavata.mft.resource.service.local.LocalStorageServiceGrpc; +import org.apache.airavata.mft.resource.service.odata.ODataStorageServiceGrpc; import org.apache.airavata.mft.resource.service.s3.S3StorageServiceGrpc; import org.apache.airavata.mft.resource.service.scp.SCPStorageServiceGrpc; import org.apache.airavata.mft.resource.service.swift.SwiftStorageServiceGrpc; @@ -63,6 +64,10 @@ public SwiftStorageServiceGrpc.SwiftStorageServiceBlockingStub swift() { return SwiftStorageServiceGrpc.newBlockingStub(channel); } + public ODataStorageServiceGrpc.ODataStorageServiceBlockingStub odata() { + return ODataStorageServiceGrpc.newBlockingStub(channel); + } + @Override public void close() throws IOException { diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java index 91d7d9f1..51a9408b 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java @@ -24,6 +24,7 @@ import org.apache.airavata.mft.resource.stubs.ftp.storage.*; import org.apache.airavata.mft.resource.stubs.gcs.storage.*; import org.apache.airavata.mft.resource.stubs.local.storage.*; +import org.apache.airavata.mft.resource.stubs.odata.storage.*; import org.apache.airavata.mft.resource.stubs.s3.storage.*; import org.apache.airavata.mft.resource.stubs.scp.storage.*; import org.apache.airavata.mft.resource.stubs.swift.storage.*; @@ -100,4 +101,10 @@ public interface ResourceBackend { SwiftStorage createSwiftStorage(SwiftStorageCreateRequest request) throws Exception; boolean updateSwiftStorage(SwiftStorageUpdateRequest request) throws Exception; boolean deleteSwiftStorage(SwiftStorageDeleteRequest request) throws Exception; + + public ODataStorageListResponse listODataStorage(ODataStorageListRequest request) throws Exception; + Optional getODataStorage(ODataStorageGetRequest request) throws Exception; + ODataStorage createODataStorage(ODataStorageCreateRequest request) throws Exception; + boolean updateODataStorage(ODataStorageUpdateRequest request) throws Exception; + boolean deleteODataStorage(ODataStorageDeleteRequest request) throws Exception; } diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java index 3201a204..3517d8a7 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java @@ -25,6 +25,7 @@ import org.apache.airavata.mft.resource.stubs.ftp.storage.*; import org.apache.airavata.mft.resource.stubs.gcs.storage.*; import org.apache.airavata.mft.resource.stubs.local.storage.*; +import org.apache.airavata.mft.resource.stubs.odata.storage.*; import org.apache.airavata.mft.resource.stubs.s3.storage.*; import org.apache.airavata.mft.resource.stubs.scp.storage.*; import org.apache.airavata.mft.resource.stubs.swift.storage.*; @@ -590,4 +591,29 @@ public boolean updateSwiftStorage(SwiftStorageUpdateRequest request) throws Exce public boolean deleteSwiftStorage(SwiftStorageDeleteRequest request) throws Exception { throw new UnsupportedOperationException("Operation is not supported in backend"); } + + @Override + public ODataStorageListResponse listODataStorage(ODataStorageListRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public Optional getODataStorage(ODataStorageGetRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public ODataStorage createODataStorage(ODataStorageCreateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean updateODataStorage(ODataStorageUpdateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean deleteODataStorage(ODataStorageDeleteRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } } diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java index d2b28c48..26d0e907 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java @@ -27,6 +27,7 @@ import org.apache.airavata.mft.resource.stubs.ftp.storage.*; import org.apache.airavata.mft.resource.stubs.gcs.storage.*; import org.apache.airavata.mft.resource.stubs.local.storage.*; +import org.apache.airavata.mft.resource.stubs.odata.storage.*; import org.apache.airavata.mft.resource.stubs.s3.storage.*; import org.apache.airavata.mft.resource.stubs.scp.storage.*; import org.apache.airavata.mft.resource.stubs.swift.storage.*; @@ -66,6 +67,9 @@ public class SQLResourceBackend implements ResourceBackend { @Autowired private StorageSecretRepository resourceSecretRepository; + @Autowired + private ODataStorageRepository odataStorageRepository; + private DozerBeanMapper mapper = new DozerBeanMapper(); @Override @@ -146,6 +150,12 @@ private GenericResource convertGenericResourceEntity(GenericResourceEntity resou builder.setSwiftStorage(swiftStorage.orElseThrow(() -> new Exception("Could not find a Swift storage with id " + resourceEty.getStorageId() + " for resource " + resourceEty.getResourceId()))); break; + case ODATA: + Optional odataStorage = getODataStorage(ODataStorageGetRequest.newBuilder() + .setStorageId(resourceEty.getStorageId()).build()); + builder.setOdataStorage(odataStorage.orElseThrow(() -> new Exception("Could not find a OData storage with id " + + resourceEty.getStorageId() + " for resource " + resourceEty.getResourceId()))); + break; } return builder.build(); @@ -493,4 +503,37 @@ public boolean deleteSwiftStorage(SwiftStorageDeleteRequest request) throws Exce resourceRepository.deleteByStorageIdAndStorageType(request.getStorageId(), GenericResourceEntity.StorageType.SWIFT); return true; } + + @Override + public ODataStorageListResponse listODataStorage(ODataStorageListRequest request) throws Exception { + ODataStorageListResponse.Builder respBuilder = ODataStorageListResponse.newBuilder(); + List all = odataStorageRepository.findAll(PageRequest.of(request.getOffset(), request.getLimit())); + all.forEach(ety -> respBuilder.addStorages(mapper.map(ety, ODataStorage.newBuilder().getClass()))); + return respBuilder.build(); + } + + @Override + public Optional getODataStorage(ODataStorageGetRequest request) throws Exception { + Optional entity = odataStorageRepository.findByStorageId(request.getStorageId()); + return entity.map(e -> mapper.map(e, ODataStorage.newBuilder().getClass()).build()); + } + + @Override + public ODataStorage createODataStorage(ODataStorageCreateRequest request) throws Exception { + ODataStorageEntity savedEntity = odataStorageRepository.save(mapper.map(request, ODataStorageEntity.class)); + return mapper.map(savedEntity, ODataStorage.newBuilder().getClass()).build(); + } + + @Override + public boolean updateODataStorage(ODataStorageUpdateRequest request) throws Exception { + odataStorageRepository.save(mapper.map(request, ODataStorageEntity.class)); + return true; + } + + @Override + public boolean deleteODataStorage(ODataStorageDeleteRequest request) throws Exception { + odataStorageRepository.deleteById(request.getStorageId()); + resourceRepository.deleteByStorageIdAndStorageType(request.getStorageId(), GenericResourceEntity.StorageType.SWIFT); + return true; + } } diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/GenericResourceEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/GenericResourceEntity.java index 4930cf3f..3e17ba25 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/GenericResourceEntity.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/GenericResourceEntity.java @@ -15,7 +15,7 @@ public enum ResourceType { } public enum StorageType { - S3, SCP, LOCAL, FTP, BOX, DROPBOX, GCS, AZURE, SWIFT; + S3, SCP, LOCAL, FTP, BOX, DROPBOX, GCS, AZURE, SWIFT, ODATA; } @Id diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/ODataStorageEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/ODataStorageEntity.java new file mode 100644 index 00000000..c48b41f7 --- /dev/null +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/ODataStorageEntity.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.resource.server.backend.sql.entity; + +import org.hibernate.annotations.GenericGenerator; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; + +@Entity +public class ODataStorageEntity { + + @Id + @Column(name = "S3_STORAGE_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String storageId; + + @Column(name = "BASE_URL") + private String baseUrl; + + @Column(name = "STORAGE_NAME") + private String name; + + public String getStorageId() { + return storageId; + } + + public void setStorageId(String storageId) { + this.storageId = storageId; + } + + public String getBaseUrl() { + return baseUrl; + } + + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/ODataStorageRepository.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/ODataStorageRepository.java new file mode 100644 index 00000000..2a08627e --- /dev/null +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/ODataStorageRepository.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.resource.server.backend.sql.repository; + +import org.apache.airavata.mft.resource.server.backend.sql.entity.ODataStorageEntity; +import org.springframework.data.domain.Pageable; +import org.springframework.data.repository.CrudRepository; + +import java.util.List; +import java.util.Optional; + +public interface ODataStorageRepository extends CrudRepository { + Optional findByStorageId(String storageId); + List findAll(Pageable pageable); +} diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ODataServiceHandler.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ODataServiceHandler.java new file mode 100644 index 00000000..391a9b28 --- /dev/null +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ODataServiceHandler.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.resource.server.handler; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.apache.airavata.mft.resource.server.backend.ResourceBackend; +import org.apache.airavata.mft.resource.service.odata.ODataStorageServiceGrpc; +import org.apache.airavata.mft.resource.stubs.odata.storage.*; +import org.lognet.springboot.grpc.GRpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +@GRpcService +public class ODataServiceHandler extends ODataStorageServiceGrpc.ODataStorageServiceImplBase { + + private static final Logger logger = LoggerFactory.getLogger(ODataServiceHandler.class); + + @Autowired + private ResourceBackend backend; + + @Override + public void listODataStorage(ODataStorageListRequest request, StreamObserver responseObserver) { + try { + ODataStorageListResponse response = this.backend.listODataStorage(request); + responseObserver.onNext(response); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error("Failed in retrieving OData storage list", e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed in retrieving OData storage list") + .asRuntimeException()); + } + } + + @Override + public void getODataStorage(ODataStorageGetRequest request, StreamObserver responseObserver) { + try { + this.backend.getODataStorage(request).ifPresentOrElse(resource -> { + responseObserver.onNext(resource); + responseObserver.onCompleted(); + }, () -> { + responseObserver.onError(Status.INTERNAL + .withDescription("No OData storage with id " + request.getStorageId()) + .asRuntimeException()); + }); + } catch (Exception e) { + logger.error("Failed in retrieving OData storage with id {}", request.getStorageId(), e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed in retrieving OData storage with id " + request.getStorageId()) + .asRuntimeException()); + } + } + + @Override + public void createODataStorage(ODataStorageCreateRequest request, StreamObserver responseObserver) { + try { + responseObserver.onNext(this.backend.createODataStorage(request)); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error("Failed in creating the OData storage", e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed in creating the OData storage") + .asRuntimeException()); + } + } + + @Override + public void updateODataStorage(ODataStorageUpdateRequest request, StreamObserver responseObserver) { + try { + this.backend.updateODataStorage(request); + responseObserver.onNext(ODataStorageUpdateResponse.newBuilder().setStorageId(request.getStorageId()).build()); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error("Failed in updating the OData storage {}", request.getStorageId(), e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed in updating the OData storage with id " + request.getStorageId()) + .asRuntimeException()); + } + } + + @Override + public void deleteODataStorage(ODataStorageDeleteRequest request, StreamObserver responseObserver) { + try { + boolean res = this.backend.deleteODataStorage(request); + if (res) { + responseObserver.onNext(ODataStorageDeleteResponse.newBuilder().setStatus(true).build()); + responseObserver.onCompleted(); + } else { + responseObserver.onError(new Exception("Failed to delete OData storage with id " + request.getStorageId())); + } + } catch (Exception e) { + logger.error("Failed in deleting the OData storage {}", request.getStorageId(), e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed in deleting the OData storage with id " + request.getStorageId()) + .asRuntimeException()); + } + } +} diff --git a/services/resource-service/server/src/main/resources/application.properties b/services/resource-service/server/src/main/resources/application.properties index 7cee619b..fb3d6e8b 100644 --- a/services/resource-service/server/src/main/resources/application.properties +++ b/services/resource-service/server/src/main/resources/application.properties @@ -15,6 +15,7 @@ # limitations under the License. # +spring.datasource.url=jdbc:h2:~/mft_resourcedb;DB_CLOSE_ON_EXIT=FALSE server.port=8089 grpc.port=7002 grpc.enableReflection=true diff --git a/services/resource-service/server/src/main/resources/distribution/conf/application.properties b/services/resource-service/server/src/main/resources/distribution/conf/application.properties index 4f024796..992025ad 100644 --- a/services/resource-service/server/src/main/resources/distribution/conf/application.properties +++ b/services/resource-service/server/src/main/resources/distribution/conf/application.properties @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +spring.datasource.url=jdbc:h2:~/mft_resourcedb;DB_CLOSE_ON_EXIT=FALSE server.port=8089 grpc.port=7002 diff --git a/services/resource-service/stub/src/main/proto/odata/ODataStorage.proto b/services/resource-service/stub/src/main/proto/odata/ODataStorage.proto new file mode 100644 index 00000000..a987319a --- /dev/null +++ b/services/resource-service/stub/src/main/proto/odata/ODataStorage.proto @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.mft.resource.stubs.odata.storage; + +message ODataStorage { + string storageId = 1; + string baseUrl = 2; + string name = 3; +} + +message ODataStorageListRequest { + int32 offset = 1; + int32 limit = 2; +} + +message ODataStorageListResponse { + repeated ODataStorage storages = 1; +} + +message ODataStorageGetRequest { + string storageId = 1; +} + +message ODataStorageCreateRequest { + string baseUrl = 1; + string storageId = 2; + string name = 3; +} + +message ODataStorageUpdateRequest { + string storageId = 1; + string baseUrl = 2; + string name = 3; +} + +message ODataStorageUpdateResponse { + string storageId = 1; +} + +message ODataStorageDeleteRequest { + string storageId = 1; +} + +message ODataStorageDeleteResponse { + bool status = 1; +} \ No newline at end of file diff --git a/services/resource-service/stub/src/main/proto/odata/ODataStorageService.proto b/services/resource-service/stub/src/main/proto/odata/ODataStorageService.proto new file mode 100644 index 00000000..ec0060c2 --- /dev/null +++ b/services/resource-service/stub/src/main/proto/odata/ODataStorageService.proto @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.mft.resource.service.odata; + +import "odata/ODataStorage.proto"; + +service ODataStorageService { + + // Storage + + rpc listODataStorage (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageListRequest) returns (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageListResponse); + + rpc getODataStorage (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageGetRequest) returns + (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage); + + rpc createODataStorage (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageCreateRequest) returns + (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage); + + rpc updateODataStorage (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageUpdateRequest) returns + (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageUpdateResponse); + + rpc deleteODataStorage (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageDeleteRequest) returns + (org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageDeleteResponse); +} \ No newline at end of file diff --git a/services/resource-service/stub/src/main/proto/resource/ResourceService.proto b/services/resource-service/stub/src/main/proto/resource/ResourceService.proto index c79f1e35..48def089 100644 --- a/services/resource-service/stub/src/main/proto/resource/ResourceService.proto +++ b/services/resource-service/stub/src/main/proto/resource/ResourceService.proto @@ -29,6 +29,7 @@ import "local/LocalStorage.proto"; import "s3/S3Storage.proto"; import "scp/SCPStorage.proto"; import "swift/SwiftStorage.proto"; +import "odata/ODataStorage.proto"; import "CredCommon.proto"; message FileResource { @@ -58,6 +59,7 @@ message GenericResource { org.apache.airavata.mft.resource.stubs.box.storage.BoxStorage boxStorage = 10; org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage azureStorage = 11; org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage swiftStorage = 12; + org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage odataStorage = 13; } } @@ -83,6 +85,7 @@ message GenericResourceCreateRequest { GCS = 6; AZURE = 7; SWIFT = 8; + ODATA = 9; } StorageType storageType = 5; diff --git a/services/resource-service/stub/src/main/proto/resourcesecretmap/StorageSecretMap.proto b/services/resource-service/stub/src/main/proto/resourcesecretmap/StorageSecretMap.proto index 7be8ae08..81599bd0 100644 --- a/services/resource-service/stub/src/main/proto/resourcesecretmap/StorageSecretMap.proto +++ b/services/resource-service/stub/src/main/proto/resourcesecretmap/StorageSecretMap.proto @@ -36,6 +36,7 @@ message StorageSecret { GCS = 6; AZURE = 7; SWIFT = 8; + ODATA = 9; } StorageType type = 4; } diff --git a/services/secret-service/client/src/main/java/org/apache/airavata/mft/secret/client/SecretServiceClient.java b/services/secret-service/client/src/main/java/org/apache/airavata/mft/secret/client/SecretServiceClient.java index a93a590a..e686d540 100644 --- a/services/secret-service/client/src/main/java/org/apache/airavata/mft/secret/client/SecretServiceClient.java +++ b/services/secret-service/client/src/main/java/org/apache/airavata/mft/secret/client/SecretServiceClient.java @@ -23,6 +23,7 @@ import org.apache.airavata.mft.credential.service.dropbox.DropboxSecretServiceGrpc; import org.apache.airavata.mft.credential.service.ftp.FTPSecretServiceGrpc; import org.apache.airavata.mft.credential.service.gcs.GCSSecretServiceGrpc; +import org.apache.airavata.mft.credential.service.odata.ODataSecretServiceGrpc; import org.apache.airavata.mft.credential.service.s3.S3SecretServiceGrpc; import org.apache.airavata.mft.credential.service.scp.SCPSecretServiceGrpc; import org.apache.airavata.mft.credential.service.swift.SwiftSecretServiceGrpc; @@ -70,6 +71,10 @@ public SwiftSecretServiceGrpc.SwiftSecretServiceBlockingStub swift() { return SwiftSecretServiceGrpc.newBlockingStub(channel); } + public ODataSecretServiceGrpc.ODataSecretServiceBlockingStub odata() { + return ODataSecretServiceGrpc.newBlockingStub(channel); + } + @Override public void close() throws IOException { this.channel.shutdown(); diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java index 7da5e14a..2fbdb317 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java @@ -22,6 +22,7 @@ import org.apache.airavata.mft.credential.stubs.dropbox.*; import org.apache.airavata.mft.credential.stubs.ftp.*; import org.apache.airavata.mft.credential.stubs.gcs.*; +import org.apache.airavata.mft.credential.stubs.odata.*; import org.apache.airavata.mft.credential.stubs.s3.*; import org.apache.airavata.mft.credential.stubs.scp.*; import org.apache.airavata.mft.credential.stubs.swift.*; @@ -72,4 +73,9 @@ public interface SecretBackend { SwiftSecret createSwiftSecret(SwiftSecretCreateRequest request) throws Exception; boolean updateSwiftSecret(SwiftSecretUpdateRequest request) throws Exception; boolean deleteSwiftSecret(SwiftSecretDeleteRequest request) throws Exception; + + Optional getODataSecret(ODataSecretGetRequest request) throws Exception; + ODataSecret createODataSecret(ODataSecretCreateRequest request) throws Exception; + boolean updateODataSecret(ODataSecretUpdateRequest request) throws Exception; + boolean deleteODataSecret(ODataSecretDeleteRequest request) throws Exception; } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java index dc3dcc02..501d95e1 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/file/FileBasedSecretBackend.java @@ -22,6 +22,7 @@ import org.apache.airavata.mft.credential.stubs.dropbox.*; import org.apache.airavata.mft.credential.stubs.ftp.*; import org.apache.airavata.mft.credential.stubs.gcs.*; +import org.apache.airavata.mft.credential.stubs.odata.*; import org.apache.airavata.mft.credential.stubs.s3.*; import org.apache.airavata.mft.credential.stubs.scp.*; import org.apache.airavata.mft.credential.stubs.swift.*; @@ -362,4 +363,24 @@ public boolean deleteSwiftSecret(SwiftSecretDeleteRequest request) throws Except throw new UnsupportedOperationException("Operation is not supported in backend"); } + @Override + public Optional getODataSecret(ODataSecretGetRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public ODataSecret createODataSecret(ODataSecretCreateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean updateODataSecret(ODataSecretUpdateRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + + @Override + public boolean deleteODataSecret(ODataSecretDeleteRequest request) throws Exception { + throw new UnsupportedOperationException("Operation is not supported in backend"); + } + } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java index 4cea50dd..fff66d8a 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java @@ -22,17 +22,20 @@ import org.apache.airavata.mft.credential.stubs.dropbox.*; import org.apache.airavata.mft.credential.stubs.ftp.*; import org.apache.airavata.mft.credential.stubs.gcs.*; +import org.apache.airavata.mft.credential.stubs.odata.*; import org.apache.airavata.mft.credential.stubs.s3.*; import org.apache.airavata.mft.credential.stubs.scp.*; import org.apache.airavata.mft.credential.stubs.swift.*; import org.apache.airavata.mft.secret.server.backend.SecretBackend; import org.apache.airavata.mft.secret.server.backend.sql.entity.FTPSecretEntity; +import org.apache.airavata.mft.secret.server.backend.sql.entity.ODataSecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.entity.S3SecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftAuthCredentialSecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftPasswordSecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftSecretEntity; import org.apache.airavata.mft.secret.server.backend.sql.repository.FTPSecretRepository; +import org.apache.airavata.mft.secret.server.backend.sql.repository.ODataSecretRepository; import org.apache.airavata.mft.secret.server.backend.sql.repository.S3SecretRepository; import org.apache.airavata.mft.secret.server.backend.sql.repository.SCPSecretRepository; import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftAuthCredentialSecretRepository; @@ -67,6 +70,9 @@ public class SQLSecretBackend implements SecretBackend { @Autowired private SwiftAuthCredentialSecretRepository swiftAuthCredentialSecretRepository; + @Autowired + private ODataSecretRepository odataSecretRepository; + private DozerBeanMapper mapper = new DozerBeanMapper(); @Override @@ -333,4 +339,28 @@ public boolean deleteFTPSecret(FTPSecretDeleteRequest request) { ftpSecretRepository.deleteById(request.getSecretId()); return true; } + + @Override + public Optional getODataSecret(ODataSecretGetRequest request) throws Exception { + Optional secretEty = odataSecretRepository.findBySecretId(request.getSecretId()); + return secretEty.map(odataSecretEntity -> mapper.map(odataSecretEntity, ODataSecret.newBuilder().getClass()).build()); + } + + @Override + public ODataSecret createODataSecret(ODataSecretCreateRequest request) throws Exception { + ODataSecretEntity savedEntity = odataSecretRepository.save(mapper.map(request, ODataSecretEntity.class)); + return mapper.map(savedEntity, ODataSecret.newBuilder().getClass()).build(); + } + + @Override + public boolean updateODataSecret(ODataSecretUpdateRequest request) throws Exception { + odataSecretRepository.save(mapper.map(request, ODataSecretEntity.class)); + return true; + } + + @Override + public boolean deleteODataSecret(ODataSecretDeleteRequest request) throws Exception { + odataSecretRepository.deleteById(request.getSecretId()); + return true; + } } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/ODataSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/ODataSecretEntity.java new file mode 100644 index 00000000..ed111550 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/ODataSecretEntity.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server.backend.sql.entity; + +import org.hibernate.annotations.GenericGenerator; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; + +@Entity +public class ODataSecretEntity { + + @Id + @Column(name = "SECRET_ID") + @GeneratedValue(generator = "uuid") + @GenericGenerator(name = "uuid", strategy = "uuid2") + private String secretId; + + @Column(name = "USER_NAME") + private String userName; + + @Column(name = "PASSWORD") + private String password; + + public String getSecretId() { + return secretId; + } + + public void setSecretId(String secretId) { + this.secretId = secretId; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/ODataSecretRepository.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/ODataSecretRepository.java new file mode 100644 index 00000000..566a24c8 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/ODataSecretRepository.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server.backend.sql.repository; + +import org.apache.airavata.mft.secret.server.backend.sql.entity.ODataSecretEntity; +import org.springframework.data.repository.CrudRepository; + +import java.util.Optional; + +public interface ODataSecretRepository extends CrudRepository { + Optional findBySecretId(String secretId); +} diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/S3SecretRepository.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/S3SecretRepository.java index 016b7900..56fbc2b6 100644 --- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/S3SecretRepository.java +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/S3SecretRepository.java @@ -7,5 +7,5 @@ import java.util.Optional; public interface S3SecretRepository extends CrudRepository { - Optional findBySecretId(String resourceId); + Optional findBySecretId(String secretId); } diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/ODataServiceHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/ODataServiceHandler.java new file mode 100644 index 00000000..14fcfbe7 --- /dev/null +++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/ODataServiceHandler.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.secret.server.handler; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.apache.airavata.mft.credential.service.odata.ODataSecretServiceGrpc; +import org.apache.airavata.mft.credential.stubs.odata.*; +import org.apache.airavata.mft.secret.server.backend.SecretBackend; +import org.lognet.springboot.grpc.GRpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +@GRpcService +public class ODataServiceHandler extends ODataSecretServiceGrpc.ODataSecretServiceImplBase { + + private static final Logger logger = LoggerFactory.getLogger(ODataServiceHandler.class); + + @Autowired + private SecretBackend backend; + @Override + public void getODataSecret(ODataSecretGetRequest request, StreamObserver responseObserver) { + try { + this.backend.getODataSecret(request).ifPresentOrElse(secret -> { + responseObserver.onNext(secret); + responseObserver.onCompleted(); + }, () -> { + responseObserver.onError(Status.INTERNAL + .withDescription("No OData Secret with id " + request.getSecretId()) + .asRuntimeException()); + }); + + } catch (Exception e) { + logger.error("Error in retrieving OData Secret with id " + request.getSecretId(), e); + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Error in retrieving OData Secret with id " + request.getSecretId()) + .asRuntimeException()); + } + super.getODataSecret(request, responseObserver); + } + + @Override + public void createODataSecret(ODataSecretCreateRequest request, StreamObserver responseObserver) { + try { + ODataSecret odataSecret = this.backend.createODataSecret(request); + responseObserver.onNext(odataSecret); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error("Error in creating OData Secret", e); + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Error in creating OData Secret") + .asRuntimeException()); + } + } + + @Override + public void updateODataSecret(ODataSecretUpdateRequest request, StreamObserver responseObserver) { + try { + this.backend.updateODataSecret(request); + responseObserver.onNext(ODataSecretUpdateResponse.newBuilder().setSecretId(request.getSecretId()).build()); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error("Error in updating OData Secret with id {}", request.getSecretId(), e); + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Error in updating OData Secret with id " + request.getSecretId()) + .asRuntimeException()); + } + } + + @Override + public void deleteODataSecret(ODataSecretDeleteRequest request, StreamObserver responseObserver) { + try { + this.backend.deleteODataSecret(request); + responseObserver.onNext(ODataSecretDeleteResponse.newBuilder().setStatus(true).build()); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error("Error in deleting OData Secret with id {}", request.getSecretId(), e); + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Error in deleting OData Secret with id " + request.getSecretId()) + .asRuntimeException()); + } + } +} diff --git a/services/secret-service/server/src/main/resources/application.properties b/services/secret-service/server/src/main/resources/application.properties index 2c8df2ef..4f483bf0 100644 --- a/services/secret-service/server/src/main/resources/application.properties +++ b/services/secret-service/server/src/main/resources/application.properties @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +spring.datasource.url=jdbc:h2:~/mft_secretdb;DB_CLOSE_ON_EXIT=FALSE server.port=8081 grpc.port=7003 diff --git a/services/secret-service/server/src/main/resources/distribution/conf/application.properties b/services/secret-service/server/src/main/resources/distribution/conf/application.properties index 10ebcc55..27370549 100644 --- a/services/secret-service/server/src/main/resources/distribution/conf/application.properties +++ b/services/secret-service/server/src/main/resources/distribution/conf/application.properties @@ -15,6 +15,8 @@ # limitations under the License. # +spring.datasource.url=jdbc:h2:~/mft_secretdb;DB_CLOSE_ON_EXIT=FALSE + server.port=8081 grpc.port=7003 diff --git a/services/secret-service/stub/src/main/proto/odata/ODataCredential.proto b/services/secret-service/stub/src/main/proto/odata/ODataCredential.proto new file mode 100644 index 00000000..5f5ca969 --- /dev/null +++ b/services/secret-service/stub/src/main/proto/odata/ODataCredential.proto @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.mft.credential.stubs.odata; + +import "CredCommon.proto"; + +message ODataSecret { + string secretId = 1; + string userName = 2; + string password = 3; +} + +message ODataSecretGetRequest { + string secretId = 1; + org.apache.airavata.mft.common.AuthToken authzToken = 2; +} + +message ODataSecretCreateRequest { + string userName = 1; + string password = 2; + org.apache.airavata.mft.common.AuthToken authzToken = 3; +} + +message ODataSecretUpdateRequest { + string secretId = 1; + string userName = 2; + string password = 3; + org.apache.airavata.mft.common.AuthToken authzToken = 4; +} + +message ODataSecretUpdateResponse { + string secretId = 1; +} + +message ODataSecretDeleteRequest { + string secretId = 1; + org.apache.airavata.mft.common.AuthToken authzToken = 2; +} + +message ODataSecretDeleteResponse { + bool status = 1; +} diff --git a/services/secret-service/stub/src/main/proto/odata/ODataSecretService.proto b/services/secret-service/stub/src/main/proto/odata/ODataSecretService.proto new file mode 100644 index 00000000..268f0f06 --- /dev/null +++ b/services/secret-service/stub/src/main/proto/odata/ODataSecretService.proto @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +option java_multiple_files = true; +package org.apache.airavata.mft.credential.service.odata; + +import "odata/ODataCredential.proto"; + +service ODataSecretService { + rpc getODataSecret (org.apache.airavata.mft.credential.stubs.odata.ODataSecretGetRequest) returns + (org.apache.airavata.mft.credential.stubs.odata.ODataSecret); + + rpc createODataSecret (org.apache.airavata.mft.credential.stubs.odata.ODataSecretCreateRequest) returns + (org.apache.airavata.mft.credential.stubs.odata.ODataSecret); + + rpc updateODataSecret (org.apache.airavata.mft.credential.stubs.odata.ODataSecretUpdateRequest) returns + (org.apache.airavata.mft.credential.stubs.odata.ODataSecretUpdateResponse); + + rpc deleteODataSecret (org.apache.airavata.mft.credential.stubs.odata.ODataSecretDeleteRequest) returns + (org.apache.airavata.mft.credential.stubs.odata.ODataSecretDeleteResponse); +} \ No newline at end of file diff --git a/transport/odata-transport/pom.xml b/transport/odata-transport/pom.xml new file mode 100644 index 00000000..706b2147 --- /dev/null +++ b/transport/odata-transport/pom.xml @@ -0,0 +1,53 @@ + + + + + mft-transport + org.apache.airavata + 0.01-SNAPSHOT + + 4.0.0 + + mft-odata-transport + + + 11 + 11 + + + + + org.apache.airavata + mft-core + ${project.version} + + + + org.apache.httpcomponents + httpclient + ${apache.http.client.version} + + + \ No newline at end of file diff --git a/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataIncomingConnector.java b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataIncomingConnector.java new file mode 100644 index 00000000..3626885a --- /dev/null +++ b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataIncomingConnector.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.odata; + +import org.apache.airavata.mft.core.api.ConnectorConfig; +import org.apache.airavata.mft.core.api.IncomingStreamingConnector; +import org.apache.airavata.mft.credential.stubs.odata.ODataSecret; +import org.apache.airavata.mft.credential.stubs.odata.ODataSecretGetRequest; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder; +import org.apache.airavata.mft.resource.stubs.common.GenericResource; +import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; +import org.apache.http.HttpEntity; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; + +public class ODataIncomingConnector implements IncomingStreamingConnector { + + private static final Logger logger = LoggerFactory.getLogger(ODataIncomingConnector.class); + + private CloseableHttpResponse response; + CloseableHttpClient client; + + private GenericResource resource; + private ODataStorage odataStorage; + + @Override + public void init(ConnectorConfig cc) throws Exception { + try (ResourceServiceClient resourceClient = ResourceServiceClientBuilder + .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) { + + resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder() + .setAuthzToken(cc.getAuthToken()) + .setResourceId(cc.getResourceId()).build()); + } + + if (resource.getStorageCase() != GenericResource.StorageCase.ODATASTORAGE) { + logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), cc.getResourceId()); + throw new Exception("Invalid storage type specified for resource " + cc.getResourceId()); + } + + odataStorage = resource.getOdataStorage(); + + try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient( + cc.getSecretServiceHost(), cc.getSecretServicePort())) { + + ODataSecret oDataSecret = secretClient.odata().getODataSecret(ODataSecretGetRequest.newBuilder() + .setAuthzToken(cc.getAuthToken()) + .setSecretId(cc.getCredentialToken()).build()); + + CredentialsProvider provider = new BasicCredentialsProvider(); + UsernamePasswordCredentials credentials + = new UsernamePasswordCredentials(oDataSecret.getUserName(), oDataSecret.getPassword()); + provider.setCredentials(AuthScope.ANY, credentials); + + client = HttpClientBuilder.create().setDefaultCredentialsProvider(provider).build(); + } + } + + @Override + public InputStream fetchInputStream() throws Exception { + + HttpGet httpGet = new HttpGet(odataStorage.getBaseUrl() + + "/Products('" + resource.getFile().getResourcePath() +"')/$value"); + response = client.execute(httpGet); + int statusCode = response.getStatusLine().getStatusCode(); + logger.info("Received status code {} for resource path {}", statusCode, resource.getFile().getResourcePath()); + + HttpEntity entity = response.getEntity(); + return entity.getContent(); + } + + @Override + public InputStream fetchInputStream(String childPath) throws Exception { + throw new UnsupportedOperationException("No child path structures available for OData"); + } + + @Override + public void complete() throws Exception { + if (response != null) { + response.close(); + } + + if (client != null) { + client.close(); + } + } +} diff --git a/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java new file mode 100644 index 00000000..63c8cca9 --- /dev/null +++ b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.odata; + +import org.apache.airavata.mft.common.AuthToken; +import org.apache.airavata.mft.core.DirectoryResourceMetadata; +import org.apache.airavata.mft.core.FileResourceMetadata; +import org.apache.airavata.mft.core.api.MetadataCollector; +import org.apache.airavata.mft.credential.stubs.odata.ODataSecret; +import org.apache.airavata.mft.credential.stubs.odata.ODataSecretGetRequest; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder; +import org.apache.airavata.mft.resource.stubs.common.GenericResource; +import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; +import org.apache.http.HttpEntity; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.StringReader; +import java.time.Instant; +import java.util.Optional; + +public class ODataMetadataCollector implements MetadataCollector { + + private static final Logger logger = LoggerFactory.getLogger(ODataMetadataCollector.class); + + private String resourceServiceHost; + private int resourceServicePort; + private String secretServiceHost; + private int secretServicePort; + + @Override + public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) { + this.resourceServiceHost = resourceServiceHost; + this.resourceServicePort = resourceServicePort; + this.secretServiceHost = secretServiceHost; + this.secretServicePort = secretServicePort; + } + + private CloseableHttpClient getHttpClient(ODataSecret oDataSecret) { + CredentialsProvider provider = new BasicCredentialsProvider(); + UsernamePasswordCredentials credentials + = new UsernamePasswordCredentials(oDataSecret.getUserName(), oDataSecret.getPassword()); + provider.setCredentials(AuthScope.ANY, credentials); + + return HttpClientBuilder.create().setDefaultCredentialsProvider(provider).build(); + } + + @Override + public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String resourceId, String credentialToken) throws Exception { + return findFileResourceMetadata(authZToken, resourceId, credentialToken) + .orElseThrow(() -> new Exception("Could not find a file resource entry for resource id " + resourceId)); + } + + @Override + public FileResourceMetadata getFileResourceMetadata(AuthToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { + throw new UnsupportedOperationException("OData does not have hierarchical structures"); + } + + @Override + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String resourceId, String credentialToken) throws Exception { + throw new UnsupportedOperationException("OData does not have directory structures"); + } + + @Override + public DirectoryResourceMetadata getDirectoryResourceMetadata(AuthToken authZToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { + throw new UnsupportedOperationException("OData does not have directory structures"); + } + + @Override + public Boolean isAvailable(AuthToken authZToken, String resourceId, String credentialToken) throws Exception { + return findFileResourceMetadata(authZToken, resourceId, credentialToken).isPresent(); + } + + @Override + public Boolean isAvailable(AuthToken authToken, String parentResourceId, String resourcePath, String credentialToken) throws Exception { + throw new UnsupportedOperationException("OData does not have directory structures"); + } + + private Optional findFileResourceMetadata(AuthToken authZToken, String resourceId, String credentialToken) throws Exception { + ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort); + GenericResource resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder().setResourceId(resourceId).build()); + + if (resource.getStorageCase() != GenericResource.StorageCase.ODATASTORAGE) { + logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), resourceId); + throw new Exception("Invalid storage type specified for resource " + resourceId); + } + + ODataStorage odataStorage = resource.getOdataStorage(); + + SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort); + ODataSecret oDataSecret = secretClient.odata().getODataSecret( + ODataSecretGetRequest.newBuilder().setSecretId(credentialToken).build()); + + try (CloseableHttpClient httpClient = getHttpClient(oDataSecret)) { + + HttpGet httpGet = new HttpGet(odataStorage.getBaseUrl() + + "/Products('" + resource.getFile().getResourcePath() +"')"); + + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != 200) { + logger.error("Failed while invoking get product information endpoint. Got code {}", statusCode); + throw new Exception("Failed while invoking get product information endpoint. Got code " + statusCode); + } + HttpEntity entity = response.getEntity(); + String responseString = EntityUtils.toString(entity, "UTF-8"); + return parseXML(responseString); + } + } + } + + private Optional parseXML(String xmlBody) { + + try { + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(new InputSource(new StringReader(xmlBody))); + doc.getDocumentElement().normalize(); + + System.out.print("Root element: "); + System.out.println(doc.getDocumentElement().getNodeName()); + NodeList properties = doc.getElementsByTagName("m:properties"); + + if (properties.getLength() == 1) { + + FileResourceMetadata.Builder builder = FileResourceMetadata.Builder.newBuilder(); + + Node propertyNode = properties.item(0); + NodeList childNodes = propertyNode.getChildNodes(); + for (int i = 0; i < childNodes.getLength(); i++) { + Node item = childNodes.item(i); + switch (item.getNodeName()) { + case "d:ContentLength": + builder.withResourceSize(Long.parseLong(item.getTextContent())); + break; + case "d:CreationDate": + builder.withCreatedTime(Instant.parse(item.getTextContent() + "Z").toEpochMilli()); + break; + case "d:ModificationDate": + builder.withUpdateTime(Instant.parse(item.getTextContent() + "Z").toEpochMilli()); + break; + case "d:Name": + builder.withFriendlyName(item.getTextContent()); + break; + case "d:Id": + builder.withResourcePath(item.getTextContent()); + break; + case "d:Checksum": + NodeList checksumNodes = item.getChildNodes(); + for (int j = 0; j < checksumNodes.getLength(); j++) { + Node item1 = checksumNodes.item(j); + if (item1.getNodeName().equals("d:Value")) { + builder.withMd5sum(item1.getTextContent()); + } + } + break; + } + } + + return Optional.of(builder.build()); + } + + } catch (Exception e) { + logger.warn("Failed while parsing provided XML body", e); + } + + return Optional.empty(); + } +} diff --git a/transport/pom.xml b/transport/pom.xml index c537b24a..2cc97299 100755 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -42,6 +42,7 @@ ftp-transport dropbox-transport swift-transport + odata-transport diff --git a/transport/s3-transport/pom.xml b/transport/s3-transport/pom.xml index 9142055f..4892b47d 100644 --- a/transport/s3-transport/pom.xml +++ b/transport/s3-transport/pom.xml @@ -37,7 +37,13 @@ org.apache.airavata mft-core - 0.01-SNAPSHOT + ${project.version} + + + + io.github.ci-cmg + aws-s3-outputstream + 1.1.0 diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingStreamingConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingStreamingConnector.java new file mode 100644 index 00000000..a0398554 --- /dev/null +++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingStreamingConnector.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.airavata.mft.transport.s3; + +import edu.colorado.cires.cmg.s3out.AwsS3ClientMultipartUpload; +import edu.colorado.cires.cmg.s3out.MultipartUploadRequest; +import edu.colorado.cires.cmg.s3out.S3ClientMultipartUpload; +import edu.colorado.cires.cmg.s3out.S3OutputStream; +import org.apache.airavata.mft.core.api.ConnectorConfig; +import org.apache.airavata.mft.core.api.OutgoingStreamingConnector; +import org.apache.airavata.mft.credential.stubs.s3.S3Secret; +import org.apache.airavata.mft.credential.stubs.s3.S3SecretGetRequest; +import org.apache.airavata.mft.resource.client.ResourceServiceClient; +import org.apache.airavata.mft.resource.client.ResourceServiceClientBuilder; +import org.apache.airavata.mft.resource.stubs.common.GenericResource; +import org.apache.airavata.mft.resource.stubs.common.GenericResourceGetRequest; +import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +import java.io.OutputStream; +import java.net.URI; + +/** NOTE: This implementation uses 3rd party buffering of output stream + * https://github.com/CI-CMG/aws-s3-outputstream until Amazon SDK supports + * https://github.com/aws/aws-sdk-java-v2/issues/3128 **/ + +public class S3OutgoingStreamingConnector implements OutgoingStreamingConnector { + + private static final Logger logger = LoggerFactory.getLogger(S3OutgoingStreamingConnector.class); + + private GenericResource resource; + private S3OutputStream s3OutputStream; + private S3ClientMultipartUpload s3; + + @Override + public void init(ConnectorConfig cc) throws Exception { + try (ResourceServiceClient resourceClient = ResourceServiceClientBuilder + .buildClient(cc.getResourceServiceHost(), cc.getResourceServicePort())) { + + resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder() + .setAuthzToken(cc.getAuthToken()) + .setResourceId(cc.getResourceId()).build()); + } + + if (resource.getStorageCase() != GenericResource.StorageCase.S3STORAGE) { + logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), cc.getResourceId()); + throw new Exception("Invalid storage type specified for resource " + cc.getResourceId()); + } + + S3Storage s3Storage = resource.getS3Storage(); + + S3Secret s3Secret; + + try (SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient( + cc.getSecretServiceHost(), cc.getSecretServicePort())) { + + s3Secret = secretClient.s3().getS3Secret(S3SecretGetRequest.newBuilder() + .setAuthzToken(cc.getAuthToken()) + .setSecretId(cc.getCredentialToken()).build()); + + AwsCredentials awsCreds; + if (s3Secret.getSessionToken() == null || s3Secret.getSessionToken().equals("")) { + awsCreds = AwsBasicCredentials.create(s3Secret.getAccessKey(), s3Secret.getSecretKey()); + } else { + awsCreds = AwsSessionCredentials.create(s3Secret.getAccessKey(), + s3Secret.getSecretKey(), + s3Secret.getSessionToken()); + } + + S3Client s3Client = S3Client.builder() + .region(Region.of(s3Storage.getRegion())).endpointOverride(new URI(s3Storage.getEndpoint())) + .credentialsProvider(() -> awsCreds) + .build(); + + this.s3 = AwsS3ClientMultipartUpload.builder().s3(s3Client).build(); + + } + } + + @Override + public void complete() throws Exception { + if (this.s3OutputStream != null) { + this.s3OutputStream.done(); + this.s3OutputStream.close(); + } + } + + @Override + public OutputStream fetchOutputStream() throws Exception { + this.s3OutputStream = S3OutputStream.builder() + .s3(s3) + .uploadRequest(MultipartUploadRequest.builder() + .bucket(resource.getS3Storage().getBucketName()) + .key(resource.getFile().getResourcePath()).build()) + .autoComplete(false) + .build(); + + logger.info("Initialized multipart upload for file {} in bucket {}", + resource.getFile().getResourcePath(), resource.getS3Storage().getBucketName()); + return this.s3OutputStream; + } + + @Override + public OutputStream fetchOutputStream(String childPath) throws Exception { + this.s3OutputStream = S3OutputStream.builder() + .s3(s3) + .uploadRequest(MultipartUploadRequest.builder() + .bucket(resource.getS3Storage().getBucketName()) + .key(resource.getFile().getResourcePath() + "/" + childPath).build()) + .autoComplete(false) + .build(); + + logger.info("Initialized multipart upload for file {} child path {} in bucket {}", + resource.getFile().getResourcePath(), childPath, resource.getS3Storage().getBucketName()); + return this.s3OutputStream; + } +}