OData Download API Support

agent/pom.xml

@@ -84,6 +84,11 @@
             <artifactId>mft-swift-transport</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>mft-odata-transport</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.airavata</groupId>
             <artifactId>mft-common-clients</artifactId>

agent/src/main/java/org/apache/airavata/mft/agent/TransportMediator.java

@@ -195,7 +195,7 @@ public void transferSingleThread(String transferId,
                 inConnector.complete();
                 outConnector.complete();
 
-                logger.info("Completed streaming ransfer for transfer {}", transferId);
+                logger.info("Completed streaming transfer for transfer {}", transferId);
 
             } else {
                 throw new Exception("No matching connector found to perform the transfer");

command-line/src/main/java/org/apache/airavata/mft/command/line/MainRunner.java

@@ -1,5 +1,6 @@
 package org.apache.airavata.mft.command.line;
 
+import org.apache.airavata.mft.command.line.sub.odata.ODataSubCommand;
 import org.apache.airavata.mft.command.line.sub.s3.S3SubCommand;
 import org.apache.airavata.mft.command.line.sub.swift.SwiftSubCommand;
 import org.apache.airavata.mft.command.line.sub.transfer.TransferSubCommand;
@@ -8,7 +9,7 @@
 
 @Command(name = "checksum", mixinStandardHelpOptions = true, version = "checksum 4.0",
         description = "Prints the checksum (SHA-256 by default) of a file to STDOUT.",
-        subcommands = {S3SubCommand.class, TransferSubCommand.class, SwiftSubCommand.class})
+        subcommands = {S3SubCommand.class, TransferSubCommand.class, SwiftSubCommand.class, ODataSubCommand.class})
 class MainRunner {
 
     public static void main(String... args) {

command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataRemoteAddSubCommand.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.command.line.sub.odata;
+
+import org.apache.airavata.mft.api.client.MFTApiClient;
+import org.apache.airavata.mft.common.AuthToken;
+import org.apache.airavata.mft.credential.stubs.odata.ODataSecret;
+import org.apache.airavata.mft.credential.stubs.odata.ODataSecretCreateRequest;
+import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage;
+import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageCreateRequest;
+import org.apache.airavata.mft.storage.stubs.storagesecret.StorageSecret;
+import org.apache.airavata.mft.storage.stubs.storagesecret.StorageSecretCreateRequest;
+import org.apache.airavata.mft.storage.stubs.storagesecret.StorageSecretServiceGrpc;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+@CommandLine.Command(name = "add")
+public class ODataRemoteAddSubCommand implements Callable<Integer> {
+
+    @CommandLine.Option(names = {"-n", "--name"}, description = "Storage Name")
+    private String remoteName;
+
+    @CommandLine.Option(names = {"-U", "--url"}, description = "Base URL for OData Endpoint")
+    private String baseURL;
+
+    @CommandLine.Option(names = {"-u", "--user"}, description = "User Name")
+    private String userName;
+
+    @CommandLine.Option(names = {"-p", "--password"}, description = "Password")
+    private String password;
+
+
+    @Override
+    public Integer call() throws Exception {
+        AuthToken authToken = AuthToken.newBuilder().build();
+
+        MFTApiClient mftApiClient = MFTApiClient.MFTApiClientBuilder.newBuilder().build();
+
+        ODataSecret oDataSecret = mftApiClient.getSecretServiceClient().odata().createODataSecret(ODataSecretCreateRequest.newBuilder()
+                .setAuthzToken(authToken).setPassword(password).setUserName(userName).build());
+
+        System.out.println("Created the OData secret " + oDataSecret.getSecretId());
+
+        ODataStorage oDataStorage = mftApiClient.getStorageServiceClient().odata().createODataStorage(
+                ODataStorageCreateRequest.newBuilder().setName(remoteName).setBaseUrl(baseURL).build());
+
+        System.out.println("Created OData storage " + oDataStorage.getStorageId());
+
+        StorageSecretServiceGrpc.StorageSecretServiceBlockingStub storageSecretClient = mftApiClient.getStorageServiceClient().storageSecret();
+
+        StorageSecret storageSecret = storageSecretClient.createStorageSecret(StorageSecretCreateRequest.newBuilder()
+                .setStorageId(oDataStorage.getStorageId())
+                .setSecretId(oDataSecret.getSecretId())
+                .setType(StorageSecret.StorageType.ODATA).build());
+
+        System.out.println("Successfully added OData remote endpoint");
+
+        return 0;
+    }
+}

command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataRemoteSubCommand.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.command.line.sub.odata;
+
+import org.apache.airavata.mft.api.client.MFTApiClient;
+import org.apache.airavata.mft.command.line.CommandLineUtil;
+import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage;
+import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageListRequest;
+import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageListResponse;
+import picocli.CommandLine;
+
+import java.util.List;
+
+@CommandLine.Command(name = "remote", subcommands = {ODataRemoteAddSubCommand.class})
+public class ODataRemoteSubCommand {
+
+    @CommandLine.Command(name = "list")
+    void listS3Resource() {
+        System.out.println("Listing S3 Resource");
+        MFTApiClient mftApiClient = MFTApiClient.MFTApiClientBuilder.newBuilder().build();
+
+        ODataStorageListResponse oDataStorageListResponse = mftApiClient.getStorageServiceClient().odata()
+                .listODataStorage(ODataStorageListRequest.newBuilder().setOffset(0).setLimit(10).build());
+
+        List<ODataStorage> storagesList = oDataStorageListResponse.getStoragesList();
+
+        int[] columnWidth = {40, 15, 55,};
+        String[][] content = new String[storagesList.size() + 1][3];
+        String[] headers = {"STORAGE ID", "NAME", "BASE URL"};
+        content[0] = headers;
+
+
+        for (int i = 1; i <= storagesList.size(); i ++) {
+            ODataStorage storage = storagesList.get(i - 1);
+            content[i][0] = storage.getStorageId();
+            content[i][1] = storage.getName();
+            content[i][2] = storage.getBaseUrl();
+        }
+
+        CommandLineUtil.printTable(columnWidth, content);
+    }
+}

command-line/src/main/java/org/apache/airavata/mft/command/line/sub/odata/ODataSubCommand.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.command.line.sub.odata;
+
+import org.apache.airavata.mft.command.line.sub.swift.SwiftRemoteSubCommand;
+import picocli.CommandLine;
+
+@CommandLine.Command(name = "odata", description = "Manage OData resources and credentials",
+        subcommands = {ODataRemoteSubCommand.class})
+public class ODataSubCommand {
+}

core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java

@@ -36,6 +36,9 @@ public static Optional<IncomingStreamingConnector> resolveIncomingStreamingConne
             case "S3":
                 className = "org.apache.airavata.mft.transport.s3.S3IncomingConnector";
                 break;
+            case "ODATA":
+                className = "org.apache.airavata.mft.transport.odata.ODataIncomingConnector";
+                break;
         }
 
         if (className != null) {
@@ -53,6 +56,10 @@ public static Optional<OutgoingStreamingConnector> resolveOutgoingStreamingConne
             case "SCP":
                 className = "org.apache.airavata.mft.transport.scp.SCPOutgoingConnector";
                 break;
+            case "S3":
+                className = "org.apache.airavata.mft.transport.s3.S3OutgoingStreamingConnector";
+                break;
+
         }
 
         if (className != null) {

core/src/main/java/org/apache/airavata/mft/core/MetadataCollectorResolver.java

@@ -54,6 +54,9 @@ public static Optional<MetadataCollector> resolveMetadataCollector(String type)
             case "SWIFT":
                 className = "org.apache.airavata.mft.transport.swift.SwiftMetadataCollector";
                 break;
+            case "ODATA":
+                className = "org.apache.airavata.mft.transport.odata.ODataMetadataCollector";
+                break;
         }
 
         if (className != null) {

pom.xml

@@ -150,6 +150,7 @@
         <mariadb.jdbc>2.5.1</mariadb.jdbc>
         <jclouds.version>2.5.0</jclouds.version>
         <commons.io.version>2.6</commons.io.version>
+        <apache.http.client.version>4.5.13</apache.http.client.version>
     </properties>
 
 </project>

services/resource-service/client/src/main/java/org/apache/airavata/mft/resource/client/StorageServiceClient.java

@@ -7,6 +7,7 @@
 import org.apache.airavata.mft.resource.service.ftp.FTPStorageServiceGrpc;
 import org.apache.airavata.mft.resource.service.gcs.GCSStorageServiceGrpc;
 import org.apache.airavata.mft.resource.service.local.LocalStorageServiceGrpc;
+import org.apache.airavata.mft.resource.service.odata.ODataStorageServiceGrpc;
 import org.apache.airavata.mft.resource.service.s3.S3StorageServiceGrpc;
 import org.apache.airavata.mft.resource.service.scp.SCPStorageServiceGrpc;
 import org.apache.airavata.mft.resource.service.swift.SwiftStorageServiceGrpc;
@@ -63,6 +64,10 @@ public SwiftStorageServiceGrpc.SwiftStorageServiceBlockingStub swift() {
         return SwiftStorageServiceGrpc.newBlockingStub(channel);
     }
 
+    public ODataStorageServiceGrpc.ODataStorageServiceBlockingStub odata() {
+        return ODataStorageServiceGrpc.newBlockingStub(channel);
+    }
+
     @Override
     public void close() throws IOException {
 

services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java

@@ -24,6 +24,7 @@
 import org.apache.airavata.mft.resource.stubs.ftp.storage.*;
 import org.apache.airavata.mft.resource.stubs.gcs.storage.*;
 import org.apache.airavata.mft.resource.stubs.local.storage.*;
+import org.apache.airavata.mft.resource.stubs.odata.storage.*;
 import org.apache.airavata.mft.resource.stubs.s3.storage.*;
 import org.apache.airavata.mft.resource.stubs.scp.storage.*;
 import org.apache.airavata.mft.resource.stubs.swift.storage.*;
@@ -100,4 +101,10 @@ public interface ResourceBackend {
     SwiftStorage createSwiftStorage(SwiftStorageCreateRequest request) throws Exception;
     boolean updateSwiftStorage(SwiftStorageUpdateRequest request) throws Exception;
     boolean deleteSwiftStorage(SwiftStorageDeleteRequest request) throws Exception;
+
+    public ODataStorageListResponse listODataStorage(ODataStorageListRequest request) throws Exception;
+    Optional<ODataStorage> getODataStorage(ODataStorageGetRequest request) throws Exception;
+    ODataStorage createODataStorage(ODataStorageCreateRequest request) throws Exception;
+    boolean updateODataStorage(ODataStorageUpdateRequest request) throws Exception;
+    boolean deleteODataStorage(ODataStorageDeleteRequest request) throws Exception;
 }

services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/file/FileBasedResourceBackend.java

@@ -25,6 +25,7 @@
 import org.apache.airavata.mft.resource.stubs.ftp.storage.*;
 import org.apache.airavata.mft.resource.stubs.gcs.storage.*;
 import org.apache.airavata.mft.resource.stubs.local.storage.*;
+import org.apache.airavata.mft.resource.stubs.odata.storage.*;
 import org.apache.airavata.mft.resource.stubs.s3.storage.*;
 import org.apache.airavata.mft.resource.stubs.scp.storage.*;
 import org.apache.airavata.mft.resource.stubs.swift.storage.*;
@@ -590,4 +591,29 @@ public boolean updateSwiftStorage(SwiftStorageUpdateRequest request) throws Exce
     public boolean deleteSwiftStorage(SwiftStorageDeleteRequest request) throws Exception {
         throw new UnsupportedOperationException("Operation is not supported in backend");
     }
+
+    @Override
+    public ODataStorageListResponse listODataStorage(ODataStorageListRequest request) throws Exception {
+        throw new UnsupportedOperationException("Operation is not supported in backend");
+    }
+
+    @Override
+    public Optional<ODataStorage> getODataStorage(ODataStorageGetRequest request) throws Exception {
+        throw new UnsupportedOperationException("Operation is not supported in backend");
+    }
+
+    @Override
+    public ODataStorage createODataStorage(ODataStorageCreateRequest request) throws Exception {
+        throw new UnsupportedOperationException("Operation is not supported in backend");
+    }
+
+    @Override
+    public boolean updateODataStorage(ODataStorageUpdateRequest request) throws Exception {
+        throw new UnsupportedOperationException("Operation is not supported in backend");
+    }
+
+    @Override
+    public boolean deleteODataStorage(ODataStorageDeleteRequest request) throws Exception {
+        throw new UnsupportedOperationException("Operation is not supported in backend");
+    }
 }

services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java

@@ -27,6 +27,7 @@
 import org.apache.airavata.mft.resource.stubs.ftp.storage.*;
 import org.apache.airavata.mft.resource.stubs.gcs.storage.*;
 import org.apache.airavata.mft.resource.stubs.local.storage.*;
+import org.apache.airavata.mft.resource.stubs.odata.storage.*;
 import org.apache.airavata.mft.resource.stubs.s3.storage.*;
 import org.apache.airavata.mft.resource.stubs.scp.storage.*;
 import org.apache.airavata.mft.resource.stubs.swift.storage.*;
@@ -66,6 +67,9 @@ public class SQLResourceBackend implements ResourceBackend {
     @Autowired
     private StorageSecretRepository resourceSecretRepository;
 
+    @Autowired
+    private ODataStorageRepository odataStorageRepository;
+
     private DozerBeanMapper mapper = new DozerBeanMapper();
 
     @Override
@@ -146,6 +150,12 @@ private GenericResource convertGenericResourceEntity(GenericResourceEntity resou
                 builder.setSwiftStorage(swiftStorage.orElseThrow(() -> new Exception("Could not find a Swift storage with id "
                         + resourceEty.getStorageId() + " for resource " + resourceEty.getResourceId())));
                 break;
+            case ODATA:
+                Optional<ODataStorage> odataStorage = getODataStorage(ODataStorageGetRequest.newBuilder()
+                        .setStorageId(resourceEty.getStorageId()).build());
+                builder.setOdataStorage(odataStorage.orElseThrow(() -> new Exception("Could not find a OData storage with id "
+                        + resourceEty.getStorageId() + " for resource " + resourceEty.getResourceId())));
+                break;
         }
 
         return builder.build();
@@ -493,4 +503,37 @@ public boolean deleteSwiftStorage(SwiftStorageDeleteRequest request) throws Exce
         resourceRepository.deleteByStorageIdAndStorageType(request.getStorageId(), GenericResourceEntity.StorageType.SWIFT);
         return true;
     }
+
+    @Override
+    public ODataStorageListResponse listODataStorage(ODataStorageListRequest request) throws Exception {
+        ODataStorageListResponse.Builder respBuilder = ODataStorageListResponse.newBuilder();
+        List<ODataStorageEntity> all = odataStorageRepository.findAll(PageRequest.of(request.getOffset(), request.getLimit()));
+        all.forEach(ety -> respBuilder.addStorages(mapper.map(ety, ODataStorage.newBuilder().getClass())));
+        return respBuilder.build();
+    }
+
+    @Override
+    public Optional<ODataStorage> getODataStorage(ODataStorageGetRequest request) throws Exception {
+        Optional<ODataStorageEntity> entity = odataStorageRepository.findByStorageId(request.getStorageId());
+        return entity.map(e -> mapper.map(e, ODataStorage.newBuilder().getClass()).build());
+    }
+
+    @Override
+    public ODataStorage createODataStorage(ODataStorageCreateRequest request) throws Exception {
+        ODataStorageEntity savedEntity = odataStorageRepository.save(mapper.map(request, ODataStorageEntity.class));
+        return mapper.map(savedEntity, ODataStorage.newBuilder().getClass()).build();
+    }
+
+    @Override
+    public boolean updateODataStorage(ODataStorageUpdateRequest request) throws Exception {
+        odataStorageRepository.save(mapper.map(request, ODataStorageEntity.class));
+        return true;
+    }
+
+    @Override
+    public boolean deleteODataStorage(ODataStorageDeleteRequest request) throws Exception {
+        odataStorageRepository.deleteById(request.getStorageId());
+        resourceRepository.deleteByStorageIdAndStorageType(request.getStorageId(), GenericResourceEntity.StorageType.SWIFT);
+        return true;
+    }
 }

services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/GenericResourceEntity.java

@@ -15,7 +15,7 @@ public enum ResourceType {
     }
 
     public enum StorageType {
-        S3, SCP, LOCAL, FTP, BOX, DROPBOX, GCS, AZURE, SWIFT;
+        S3, SCP, LOCAL, FTP, BOX, DROPBOX, GCS, AZURE, SWIFT, ODATA;
     }
 
     @Id