diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 952f2c28238..517d8ebdbe9 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2784,6 +2784,115 @@ Each user can get a dump of their basic information in JSON format by passing in .. _pids-api: +Managing Harvesting Server and Sets +----------------------------------- + +This API can be used to manage the Harvesting sets that your installation makes available over OAI-PMH. For more information, see the :doc:`/admin/harvestserver` section of the Admin Guide. + +List All Harvesting Sets +~~~~~~~~~~~~~~~~~~~~~~~~ + +Shows all Harvesting Sets defined in the installation:: + + GET http://$SERVER/api/harvest/server/oaisets/ + +List A Specific Harvesting Set +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Shows a Harvesting Set with a defined specname:: + + GET http://$SERVER/api/harvest/server/oaisets/$specname + +Create a Harvesting Set +~~~~~~~~~~~~~~~~~~~~~~~ + +To create a harvesting set you must supply a JSON file that contains the following fields: + +- Name: Alpha-numeric may also contain -, _, or %, but no spaces. Must also be unique in the installation. +- Definition: A search query to select the datasets to be harvested. For example, a query containing authorName:YYY would include all datasets where ‘YYY’ is the authorName. +- Description: Text that describes the harvesting set. The description appears in the Manage Harvesting Sets dashboard and in API responses. This field is optional. + +An example JSON file would look like this: + + { + "name”:"ffAuthor", + "definition":"authorName:Finch, Fiona", + "description":"Fiona Finch’s Datasets" + } + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +.. code-block:: bashbloopbloopbloopbloopbloopbloopbloop + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + + curl -H X-Dataverse-key:$API_TOKEN -X POST "$SERVER_URL/api/harvest/server/oaisets/" --upload-file harvestset-finch.json + +The fully expanded example above (without the environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/harvest/server/oaisets/" --upload-file "harvestset-finch.json" + +Only users with superuser permissions may create harvesting sets. + +Modify an Existing Harvesting Set +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To modify a harvesting set, you must supply a JSON file that contains one or both of the following fields: + +- Definition: A search query to select the datasets to be harvested. For example, a query containing authorName:YYY would include all datasets where ‘YYY’ is the authorName. +- Description: Text that describes the harvesting set. The description appears in the Manage Harvesting Sets dashboard and in API responses. This field is optional. + +Note that you may not modify the name of an existing harvesting set. + +An example JSON file would look like this: + + { + "definition":"authorName:Finch, Fiona AND subject:trees", + "description":"Fiona Finch’s Datasets with subject of trees" + } + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export SPECNAME=ffAuthor + + curl -H X-Dataverse-key:$API_TOKEN -X PUT "$SERVER_URL/api/harvest/server/oaisets/$SPECNAME” --upload-file modify-harvestset-finch.json + +The fully expanded example above (without the environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/harvest/server/oaisets/ffAuthor” --upload-file "modify-harvestset-finch.json" + +Only users with superuser permissions may modify harvesting sets. + +Delete an Existing Harvesting Set +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To delete a harvesting set, use the set's database name. For example, to delete an existing harvesting set whose database name is “ffAuthor”: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export SPECNAME=ffAuthor + + curl -H X-Dataverse-key:$API_TOKEN -X DELETE "$SERVER_URL/api/harvest/server/oaisets/$SPECNAME” + +The fully expanded example above (without the environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/harvest/server/oaisets/ffAuthor” + +Only users with superuser permissions may delete harvesting sets. + PIDs ---- diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingServer.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingServer.java index b8950edc6a0..6df8cb9f1c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingServer.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingServer.java @@ -5,35 +5,20 @@ */ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; - import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.impl.CreateHarvestingClientCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetHarvestingClientCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateHarvestingClientCommand; -import edu.harvard.iq.dataverse.harvest.client.ClientHarvestRun; -import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean; -import edu.harvard.iq.dataverse.harvest.client.HarvestingClientServiceBean; import edu.harvard.iq.dataverse.harvest.server.OAISet; import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; -import edu.harvard.iq.dataverse.authorization.users.User; -import static edu.harvard.iq.dataverse.util.JsfHelper.JH; import javax.json.JsonObjectBuilder; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import java.io.IOException; import java.io.StringReader; import java.util.List; -import java.util.ResourceBundle; import java.util.logging.Logger; import java.util.regex.Pattern; import javax.ejb.EJB; import javax.ejb.Stateless; -import javax.faces.application.FacesMessage; import javax.json.Json; import javax.json.JsonReader; import javax.json.JsonArrayBuilder; @@ -119,8 +104,7 @@ public Response oaiSet(@PathParam("specname") String spec, @QueryParam("key") St * "description":$optional_set_description,"definition":$set_search_query_string}. */ @POST - @Path("{specname}") - public Response createOaiSet(String jsonBody, @PathParam("specname") String spec, @QueryParam("key") String apiKey) throws IOException, JsonParseException { + public Response createOaiSet(String jsonBody, @QueryParam("key") String apiKey) throws IOException, JsonParseException { /* * authorization modeled after the UI (aka HarvestingSetsPage) */ @@ -141,30 +125,33 @@ public Response createOaiSet(String jsonBody, @PathParam("specname") String spec JsonObject json = jrdr.readObject(); OAISet set = new OAISet(); - //Validating spec - if (!StringUtils.isEmpty(spec)) { - if (spec.length() > 30) { + + + String name, desc, defn; + + try { + name = json.getString("name"); + } catch (NullPointerException npe_name) { + return badRequest(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setspec.required")); + } + //Validating spec + if (!StringUtils.isEmpty(name)) { + if (name.length() > 30) { return badRequest(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setspec.sizelimit")); } - if (!Pattern.matches("^[a-zA-Z0-9\\_\\-]+$", spec)) { + if (!Pattern.matches("^[a-zA-Z0-9\\_\\-]+$", name)) { return badRequest(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setspec.invalid")); // If it passes the regex test, check } - if (oaiSetService.findBySpec(spec) != null) { + if (oaiSetService.findBySpec(name) != null) { return badRequest(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setspec.alreadyused")); } } else { return badRequest(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setspec.required")); } - set.setSpec(spec); - String name, desc, defn; - - try { - name = json.getString("name"); - } catch (NullPointerException npe_name) { - return badRequest(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setspec.required")); - } + + set.setSpec(name); try { defn = json.getString("definition"); } catch (NullPointerException npe_defn) { @@ -179,22 +166,77 @@ public Response createOaiSet(String jsonBody, @PathParam("specname") String spec set.setDescription(desc); set.setDefinition(defn); oaiSetService.save(set); - return created("/harvest/server/oaisets" + spec, oaiSetAsJson(set)); + return created("/harvest/server/oaisets" + name, oaiSetAsJson(set)); } } @PUT - @Path("{nickName}") + @Path("{specname}") public Response modifyOaiSet(String jsonBody, @PathParam("specname") String spec, @QueryParam("key") String apiKey) throws IOException, JsonParseException { - // TODO: - // ... - return created("/harvest/server/oaisets" + spec, null); + + AuthenticatedUser dvUser; + try { + dvUser = findAuthenticatedUserOrDie(); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + if (!dvUser.isSuperuser()) { + return badRequest(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setspec.superUser.required")); + } + + StringReader rdr = new StringReader(jsonBody); + + try (JsonReader jrdr = Json.createReader(rdr)) { + JsonObject json = jrdr.readObject(); + OAISet update; + //Validating spec + if (!StringUtils.isEmpty(spec)) { + update = oaiSetService.findBySpec(spec); + if (update == null) { + return badRequest(BundleUtil.getStringFromBundle("harvestserver.editSetDialog.setspec.notFound")); + } + + } else { + return badRequest(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setspec.required")); + } + + String desc, defn; + + try { + defn = json.getString("definition"); + } catch (NullPointerException npe_defn) { + defn = ""; // if they're updating description but not definition; + } + try { + desc = json.getString("description"); + } catch (NullPointerException npe_desc) { + desc = ""; //treating description as optional + } + if (defn.isEmpty() && desc.isEmpty()) { + return badRequest(BundleUtil.getStringFromBundle("harvestserver.newSetDialog.setspec.required")); + } + update.setDescription(desc); + update.setDefinition(defn); + oaiSetService.save(update); + return ok("/harvest/server/oaisets" + spec, oaiSetAsJson(update)); + } } @DELETE @Path("{specname}") public Response deleteOaiSet(@PathParam("specname") String spec, @QueryParam("key") String apiKey) { + + AuthenticatedUser dvUser; + try { + dvUser = findAuthenticatedUserOrDie(); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + if (!dvUser.isSuperuser()) { + return badRequest(BundleUtil.getStringFromBundle("harvestserver.deleteSetDialog.setspec.superUser.required")); + } + OAISet set = null; try { set = oaiSetService.findBySpec(spec); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index a4b53600052..97bbc22c2e1 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -575,6 +575,8 @@ harvestserver.newSetDialog.setspec.tip=A unique name (OAI setSpec) identifying t harvestserver.newSetDialog.setspec.helptext=Consists of letters, digits, underscores (_) and dashes (-). harvestserver.editSetDialog.setspec.helptext=The name can not be changed once the set has been created. harvestserver.editSetDialog.setspec.helptext.default=this is the default, unnamed set +harvestserver.editSetDialog.setspec.notFound=Set for editing not found! +harvestserver.editSetDialog.setspec.noChanges=Invalid update! You must provide a definition and/or description to be updated. harvestserver.newSetDialog.setspec.required=Name (OAI setSpec) cannot be empty! harvestserver.newSetDialog.setspec.invalid=Name (OAI setSpec) can contain only letters, digits, underscores (_) and dashes (-). harvestserver.newSetDialog.setspec.alreadyused=This set name (OAI setSpec) is already used. @@ -594,6 +596,7 @@ harvestserver.newSetDialog.btn.create=Create Set harvestserver.newSetDialog.success=Successfully created harvesting set "{0}". harvestserver.viewEditDialog.title=Edit Harvesting Set harvestserver.viewEditDialog.btn.save=Save Changes +harvestserver.deleteSetDialog.setspec.superUser.required=Only superusers may delete OAI sets. #dashboard-users.xhtml dashboard.card.users=Users diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index a1b70cc0f45..0e3c799b69d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -55,6 +55,11 @@ private String jsonForTestSpec(String name, String def) { String r = String.format("{\"name\":\"%s\",\"definition\":\"%s\"}", name, def);//description is optional return r; } + + private String jsonForEditSpec(String name, String def, String desc) { + String r = String.format("{\"name\":\"%s\",\"definition\":\"%s\",\"description\":\"%s\"}", name, def, desc); + return r; + } private String normalUserAPIKey; private String adminUserAPIKey; @@ -67,6 +72,7 @@ public void testSetCreation() { // make sure the set does not exist String u0 = String.format("/api/harvest/server/oaisets/%s", setName); + String createPath ="/api/harvest/server/oaisets/"; Response r0 = given() .get(u0); assertEquals(404, r0.getStatusCode()); @@ -75,28 +81,98 @@ public void testSetCreation() { Response r1 = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) .body(jsonForTestSpec(setName, def)) - .post(u0); + .post(createPath); assertEquals(400, r1.getStatusCode()); // try to create set as admin user, should succeed Response r2 = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(jsonForTestSpec(setName, def)) - .post(u0); + .post(createPath); assertEquals(201, r2.getStatusCode()); // try to create set with same name as admin user, should fail Response r3 = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(jsonForTestSpec(setName, def)) - .post(u0); + .post(createPath); assertEquals(400, r3.getStatusCode()); // try to export set as admin user, should succeed (under admin API, not checking that normal user will fail) Response r4 = UtilIT.exportOaiSet(setName); assertEquals(200, r4.getStatusCode()); + + // try to delete as normal user should fail + Response r5 = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) + .delete(u0); + logger.info("r5.getStatusCode(): " + r5.getStatusCode()); + assertEquals(400, r5.getStatusCode()); + + // try to delete as admin user should work + Response r6 = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .delete(u0); + logger.info("r6.getStatusCode(): " + r6.getStatusCode()); + assertEquals(200, r6.getStatusCode()); + + } + + @Test + public void testSetEdit() { + setupUsers(); + String setName = UtilIT.getRandomString(6); + String def = "*"; + + // make sure the set does not exist + String u0 = String.format("/api/harvest/server/oaisets/%s", setName); + String createPath ="/api/harvest/server/oaisets/"; + Response r0 = given() + .get(u0); + assertEquals(404, r0.getStatusCode()); + + + // try to create set as admin user, should succeed + Response r1 = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .body(jsonForTestSpec(setName, def)) + .post(createPath); + assertEquals(201, r1.getStatusCode()); + + + // try to edit as normal user should fail + Response r2 = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) + .body(jsonForEditSpec(setName, def,"")) + .put(u0); + logger.info("r2.getStatusCode(): " + r2.getStatusCode()); + assertEquals(400, r2.getStatusCode()); + + // try to edit as with blanks should fail + Response r3 = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .body(jsonForEditSpec(setName, "","")) + .put(u0); + logger.info("r3.getStatusCode(): " + r3.getStatusCode()); + assertEquals(400, r3.getStatusCode()); + + // try to edit as with something should pass + Response r4 = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .body(jsonForEditSpec(setName, "newDef","newDesc")) + .put(u0); + logger.info("r4 Status code: " + r4.getStatusCode()); + logger.info("r4.prettyPrint(): " + r4.prettyPrint()); + assertEquals(OK.getStatusCode(), r4.getStatusCode()); + + logger.info("u0: " + u0); + // now delete it... + Response r6 = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .delete(u0); + logger.info("r6.getStatusCode(): " + r6.getStatusCode()); + assertEquals(200, r6.getStatusCode()); - // TODO - get an answer to the question of if it's worth cleaning up (users, sets) or not } // A more elaborate test - we'll create and publish a dataset, then create an @@ -137,11 +213,11 @@ public void testOaiFunctionality() throws InterruptedException { String setName = identifier; String setQuery = "dsPersistentId:" + identifier; String apiPath = String.format("/api/harvest/server/oaisets/%s", setName); - + String createPath ="/api/harvest/server/oaisets/"; Response createSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(jsonForTestSpec(setName, setQuery)) - .post(apiPath); + .post(createPath); assertEquals(201, createSetResponse.getStatusCode()); // TODO: a) look up the set via native harvest/server api;