RestAssured test for the new harvesting client features #10909

Author: landreev

src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java

@@ -40,6 +40,8 @@ public class HarvestingClientsIT {
     private static final String ARCHIVE_DESCRIPTION = "RestAssured harvesting client test";
     private static final String CONTROL_OAI_SET = "controlTestSet2";
     private static final int DATASETS_IN_CONTROL_SET = 8;
+    private static final String DATACITE_ARCHIVE_URL = "https://oai.datacite.org";
+    private static final String DATACITE_OAI_URL = DATACITE_ARCHIVE_URL + "/oai";
     private static String normalUserAPIKey;
     private static String adminUserAPIKey;
     private static String harvestCollectionAlias;
@@ -303,4 +305,129 @@ private void harvestingClientRun(boolean allowHarvestingMissingCVV)  throws Inte
         // Fail if it hasn't completed in maxWait seconds
         assertTrue(i < maxWait);
     }
+    
+    /*
+     * Being able to harvest from DataCite (issue #10909, pr #11011) is an 
+     * important enough feature to warrant a dedicated test. 
+     * Just like the other tests above, this will rely on an external OAI 
+     * server, which is somewhat problematic inherently. However, of all the 
+     * external servers and services, DataCite can be safely considered to be 
+     * more reliable than most. 
+     * The test is super straightforward, with the goal of harvesting one 
+     * specific IQSS dataset (doi:10.7910/DVN/TJCLKP) from 
+     * https://oai.datacite.org/oai. As part of testing the overal functionality
+     * of being able to work with the quirks of the DataCite OAI service, it 
+     * tests 2 new features:
+     *    "useOaiIdentifiersAsPids": true,
+     *    "useListRecords": true,
+     * (both have useful applications in other scenarios, i.e. when harvesting
+     * from other sources, not just from DataCite!)
+     * 
+    */
+    @Test
+    public void testHarvestingFromDatacite()  throws InterruptedException {
+        String nickName = "philTJCLKP" + UtilIT.getRandomString(6);
+
+        // The magical string used as the name of our "pseudo set" is the
+        // native DataCite search API query that finds our dataset, base64-encoded. 
+        // i.e., 
+        //    native API: https://api.datacite.org/dois?query=doi:10.7910/DVN/TJCLKP
+        // encoded: 
+        //   echo "doi:10.7910/DVN/TJCLKP" | base64
+        //   ZG9pOjEwLjc5MTAvRFZOL1RKQ0xLUAo=
+        String pseudoSetName = "~ZG9pOjEwLjc5MTAvRFZOL1RKQ0xLUAo=";
+        
+        clientApiPath = String.format(HARVEST_CLIENTS_API+"%s", nickName);
+        String clientJson = String.format("{\"dataverseAlias\":\"root\","
+                + "\"type\":\"oai\","
+                + "\"harvestUrl\":\"%s\","
+                + "\"archiveUrl\":\"%s\","
+                + "\"set\":\"%s\","
+                + "\"useOaiIdentifiersAsPids\": true,"
+                + "\"useListRecords\": true,"
+                + "\"metadataFormat\":\"%s\"}", 
+                DATACITE_OAI_URL, DATACITE_ARCHIVE_URL, pseudoSetName, HARVEST_METADATA_FORMAT);
+        
+        Response createResponse = given()
+                .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
+                .body(clientJson)
+                .post(clientApiPath);
+        assertEquals(CREATED.getStatusCode(), createResponse.getStatusCode());
+        
+        // API TEST 1. Run the harvest using the configuration (client) we have 
+        // just created
+        
+        String runHarvestApiPath = String.format(HARVEST_CLIENTS_API+"%s/run", nickName);
+                
+        Response runResponse = given()
+                .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey)
+                .post(runHarvestApiPath);
+        assertEquals(ACCEPTED.getStatusCode(), runResponse.getStatusCode());
+        
+        // API TEST 2. As indicated by the ACCEPTED status code above, harvesting
+        // is an asynchronous operation that will be performed in the background.
+        // Verify that this "in progress" status is properly reported while it's 
+        // running, and that it completes in some reasonable amount of time. 
+        
+        int i = 0;
+        int maxWait=20; // a very conservative interval; this harvest has no business taking this long
+        
+        do {
+            // Give it an initial 2 sec. delay, to make sure the client state
+            // has been updated in the database, which can take some appreciable 
+            // amount of time on a heavily-loaded server running a full suite of
+            // tests:
+            Thread.sleep(2000L);
+            // keep checking the status of the client with the GET api:
+            Response getClientResponse = given()
+                .get(clientApiPath);
+        
+            assertEquals(OK.getStatusCode(), getClientResponse.getStatusCode());
+            JsonPath responseJsonPath = getClientResponse.body().jsonPath();
+            assertNotNull(responseJsonPath, "Invalid JSON in GET client response");
+            assertEquals(ApiConstants.STATUS_OK, responseJsonPath.getString("status"));
+            
+            String clientStatus = responseJsonPath.getString("data.status");
+            assertNotNull(clientStatus);
+            
+            if ("inProgress".equals(clientStatus) || "IN PROGRESS".equals(responseJsonPath.getString("data.lastResult"))) {
+                // we'll sleep for another second
+                i++;
+            } else {
+                logger.info("getClientResponse.prettyPrint: " 
+                        + getClientResponse.prettyPrint());
+                // Check the values in the response:
+                // a) Confirm that the harvest has completed: 
+                assertEquals("inActive", clientStatus, "Unexpected client status: "+clientStatus);
+                
+                // b) Confirm that it has actually succeeded:
+                assertEquals("SUCCESS", responseJsonPath.getString("data.lastResult"), "Last harvest not reported a success (took "+i+" seconds)");
+                String harvestTimeStamp = responseJsonPath.getString("data.lastHarvest");
+                assertNotNull(harvestTimeStamp); 
+                
+                // c) Confirm that the other timestamps match: 
+                assertEquals(harvestTimeStamp, responseJsonPath.getString("data.lastSuccessful"));
+                assertEquals(harvestTimeStamp, responseJsonPath.getString("data.lastNonEmpty"));
+                
+                // d) Confirm that the expected 1 dataset has been harvested, with no failures:
+                assertEquals(1, responseJsonPath.getInt("data.lastDatasetsHarvested"));
+                assertEquals(0, responseJsonPath.getInt("data.lastDatasetsFailed"));
+                assertEquals(0, responseJsonPath.getInt("data.lastDatasetsDeleted"));
+                
+                // ok, it looks like the harvest has completed successfully.
+                break;
+            }
+        } while (i<maxWait); 
+        
+        System.out.println("Waited " + i + " seconds for the harvest to complete.");
+        // Fail if it hasn't completed in maxWait seconds
+        assertTrue(i < maxWait);
+        
+        // @todo: maybe call native API and check specifically on 
+        // /api/datasets/:persistentId?persistentId=doi:10.7910/DVN/TJCLKP
+        // to verify that it has been properly imporated. 
+        
+        // No need to delete the client (and the harvested dataset with it) here, 
+        // it will be deleted by the @AfterEach cleanup() method
+    }
 }