From 42b695871c4faab715d96de6413ef6dfb296b57a Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Fri, 20 Oct 2023 18:19:27 +0200 Subject: [PATCH 01/44] disable s3 tagging JVM option --- .../10022_upload_redirect_without_tagging.md | 1 + .../source/installation/config.rst | 8 +++ .../iq/dataverse/dataaccess/S3AccessIO.java | 11 ++- .../iq/dataverse/settings/JvmSettings.java | 2 + src/main/webapp/resources/js/fileupload.js | 70 ++++++++++--------- 5 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 doc/release-notes/10022_upload_redirect_without_tagging.md diff --git a/doc/release-notes/10022_upload_redirect_without_tagging.md b/doc/release-notes/10022_upload_redirect_without_tagging.md new file mode 100644 index 00000000000..379c7c8f1e0 --- /dev/null +++ b/doc/release-notes/10022_upload_redirect_without_tagging.md @@ -0,0 +1 @@ +If your S3 store does not support tagging and gives an error when redirecting uploads, from now on, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. Disabling the tagging can result in leftover files that are not used by your Dataverse instance and should be removed to preserve the storage space. To clean up the leftover files, you can use the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index ce8876b012c..2b5fd75977c 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -792,6 +792,13 @@ Larger installations may want to increase the number of open S3 connections allo ``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"`` +By default, when redirecting an upload to the S3 storage, the Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup when the file is not added to the dataset after upload (e.g., when the user cancels the operation). +If your S3 store does not support tagging and gives an error when redirecting uploads, you can disable that tag by using the ``dataverse.files..disable-tagging`` JVM option. For example: + +``./asadmin create-jvm-options "-Ddataverse.files..disable-tagging=true"`` + +Disabling the ``temp`` tag can result in leftover files that are not used by your Dataverse instance and should be removed to preserve the storage space. To clean up the leftover files, you can use the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. + In case you would like to configure Dataverse to use a custom S3 service instead of Amazon S3 services, please add the options for the custom URL and region as documented below. Please read above if your desired combination has been tested already and what other options have been set for a successful integration. @@ -825,6 +832,7 @@ List of S3 Storage Options dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` + dataverse.files..disable-tagging ``true``/``false`` Do not place the ``temp`` tag when redirecting the upload to the S3 server ``false`` =========================================== ================== =================================================================================== ============= .. table:: diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 822ada0b83e..c904651e81f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -39,6 +39,7 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.datavariable.DataVariable; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.FileUtil; import opennlp.tools.util.StringUtil; @@ -985,7 +986,10 @@ private String generateTemporaryS3UploadUrl(String key, Date expiration) throws GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key).withMethod(HttpMethod.PUT).withExpiration(expiration); //Require user to add this header to indicate a temporary file - generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId).orElse(false); + if (!taggingDisabled) { + generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + } URL presignedUrl; try { @@ -1034,7 +1038,10 @@ public JsonObjectBuilder generateTemporaryS3UploadUrls(String globalId, String s } else { JsonObjectBuilder urls = Json.createObjectBuilder(); InitiateMultipartUploadRequest initiationRequest = new InitiateMultipartUploadRequest(bucketName, key); - initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId).orElse(false); + if (!taggingDisabled) { + initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + } InitiateMultipartUploadResult initiationResponse = s3.initiateMultipartUpload(initiationRequest); String uploadId = initiationResponse.getUploadId(); for (int i = 1; i <= (fileSize / minPartSize) + (fileSize % minPartSize > 0 ? 1 : 0); i++) { diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index cc3272413c7..794ffd5e0af 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -50,6 +50,8 @@ public enum JvmSettings { UPLOADS_DIRECTORY(SCOPE_FILES, "uploads"), DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"), GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"), + SCOPE_DRIVER(SCOPE_FILES), + DISABLE_S3_TAGGING(SCOPE_DRIVER, "disable-tagging"), // SOLR INDEX SETTINGS SCOPE_SOLR(PREFIX, "solr"), diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index 08d6956b62c..03ec82f214b 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -192,41 +192,45 @@ var fileUpload = class fileUploadClass { progBar.html(''); progBar.append($('').attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all')); if(this.urls.hasOwnProperty("url")) { - $.ajax({ - url: this.urls.url, - headers: { "x-amz-tagging": "dv-state=temp" }, - type: 'PUT', - data: this.file, - context:this, - cache: false, - processData: false, - success: function() { - //ToDo - cancelling abandons the file. It is marked as temp so can be cleaned up later, but would be good to remove now (requires either sending a presigned delete URL or adding a callback to delete only a temp file - if(!cancelled) { - this.reportUpload(); - } - }, - error: function(jqXHR, textStatus, errorThrown) { - console.log('Failure: ' + jqXHR.status); - console.log('Failure: ' + errorThrown); - uploadFailure(jqXHR, thisFile); - }, - xhr: function() { - var myXhr = $.ajaxSettings.xhr(); - if (myXhr.upload) { - myXhr.upload.addEventListener('progress', function(e) { - if (e.lengthComputable) { - var doublelength = 2 * e.total; - progBar.children('progress').attr({ - value: e.loaded, - max: doublelength - }); - } - }); + const url = this.urls.url; + const request = { + url: url, + type: 'PUT', + data: this.file, + context:this, + cache: false, + processData: false, + success: function() { + //ToDo - cancelling abandons the file. It is marked as temp so can be cleaned up later, but would be good to remove now (requires either sending a presigned delete URL or adding a callback to delete only a temp file + if(!cancelled) { + this.reportUpload(); + } + }, + error: function(jqXHR, textStatus, errorThrown) { + console.log('Failure: ' + jqXHR.status); + console.log('Failure: ' + errorThrown); + uploadFailure(jqXHR, thisFile); + }, + xhr: function() { + var myXhr = $.ajaxSettings.xhr(); + if (myXhr.upload) { + myXhr.upload.addEventListener('progress', function(e) { + if (e.lengthComputable) { + var doublelength = 2 * e.total; + progBar.children('progress').attr({ + value: e.loaded, + max: doublelength + }); + } + }); + } + return myXhr; } - return myXhr; + }; + if (url.includes("x-amz-tagging")) { + request.headers = { "x-amz-tagging": "dv-state=temp" }; } - }); + $.ajax(request); } else { var loaded=[]; this.etags=[]; From 61ecb32e1e9836a613cea2f71e8376dee400ac4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Thu, 15 Feb 2024 15:03:33 +0100 Subject: [PATCH 02/44] CVOC : add HTTP request headers --- .../harvard/iq/dataverse/DatasetFieldServiceBean.java | 9 ++++++++- src/main/webapp/datasetFieldForEditFragment.xhtml | 1 + src/main/webapp/metadataFragment.xhtml | 4 ++++ src/main/webapp/search-include-fragment.xhtml | 4 ++++ src/main/webapp/search/advanced.xhtml | 1 + 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index ce2b00086ec..b05dd6b40c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -502,7 +502,14 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep HttpGet httpGet = new HttpGet(retrievalUri); //application/json+ld is for backward compatibility httpGet.addHeader("Accept", "application/ld+json, application/json+ld, application/json"); - + //Adding others custom HTTP request headers if exists + final JsonObject headers = cvocEntry.getJsonObject("headers"); + if (headers != null) { + final Set headerKeys = headers.keySet(); + for (final String hKey: headerKeys) { + httpGet.addHeader(hKey, headers.getString(hKey)); + } + } HttpResponse response = httpClient.execute(httpGet); String data = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); int statusCode = response.getStatusLine().getStatusCode(); diff --git a/src/main/webapp/datasetFieldForEditFragment.xhtml b/src/main/webapp/datasetFieldForEditFragment.xhtml index e72ee351ea0..d8c005366cb 100644 --- a/src/main/webapp/datasetFieldForEditFragment.xhtml +++ b/src/main/webapp/datasetFieldForEditFragment.xhtml @@ -42,6 +42,7 @@ + diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 200d2917b9a..1cf4ac0bc71 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -88,6 +88,7 @@ escape="#{dsf.datasetFieldType.isEscapeOutputText()}"> + @@ -100,6 +101,7 @@ escape="#{dsf.datasetFieldType.isEscapeOutputText()}"> + @@ -146,6 +148,7 @@ rendered="${cvocOnCvPart and cvPart.key.datasetFieldType.name.equals(cvocConf.get(cvPart.key.datasetFieldType.id).getString('term-uri-field'))}"> + @@ -156,6 +159,7 @@ rendered="${cvocOnDsf and cvPart.key.datasetFieldType.name.equals(cvocConf.get(dsf.datasetFieldType.id).getString('term-uri-field'))}"> + diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index af568170157..2eed454800d 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -213,6 +213,7 @@ + @@ -234,6 +235,7 @@ + @@ -384,6 +386,7 @@ + @@ -621,6 +624,7 @@ + diff --git a/src/main/webapp/search/advanced.xhtml b/src/main/webapp/search/advanced.xhtml index 06e662064e7..24b78ba0dc9 100644 --- a/src/main/webapp/search/advanced.xhtml +++ b/src/main/webapp/search/advanced.xhtml @@ -121,6 +121,7 @@ + From 25cf3ff5c54fe8d3ee767688e39ff1527d7ce0d0 Mon Sep 17 00:00:00 2001 From: zearaujo25 Date: Wed, 27 Mar 2024 18:29:53 -0300 Subject: [PATCH 03/44] 9887: Adding new endpoint to make superuser call idempotent --- .../edu/harvard/iq/dataverse/api/Admin.java | 32 +++++++++++++++---- .../edu/harvard/iq/dataverse/api/AdminIT.java | 14 +++++++- .../edu/harvard/iq/dataverse/api/RolesIT.java | 2 +- .../edu/harvard/iq/dataverse/api/UtilIT.java | 5 +++ 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index d098c2fe16a..86b4f5ee679 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1035,15 +1035,33 @@ public Response toggleSuperuser(@PathParam("identifier") String identifier) { ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "toggleSuperuser") .setInfo(identifier); try { - AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); - if (user.isDeactivated()) { - return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); - } + final AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); + return changeSuperUserStatus(user, !user.isSuperuser()); + } catch (Exception e) { + alr.setActionResult(ActionLogRecord.Result.InternalError); + alr.setInfo(alr.getInfo() + "// " + e.getMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + actionLogSvc.log(alr); + } + } - user.setSuperuser(!user.isSuperuser()); + private Response changeSuperUserStatus(AuthenticatedUser user, Boolean isSuperuser) { + if (user.isDeactivated()) { + return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); + } + user.setSuperuser(isSuperuser); + return ok("User " + user.getIdentifier() + " " + (user.isSuperuser() ? "set" : "removed") + + " as a superuser."); + } - return ok("User " + user.getIdentifier() + " " + (user.isSuperuser() ? "set" : "removed") - + " as a superuser."); + @Path("superuser/{identifier}") + @PUT + public Response changeSuperUserStatus(@PathParam("identifier") String identifier, Boolean isSuperUser) { + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "changeSuperUserStatus") + .setInfo(identifier + ":" + isSuperUser); + try { + return changeSuperUserStatus(authSvc.getAuthenticatedUser(identifier), isSuperUser); } catch (Exception e) { alr.setActionResult(ActionLogRecord.Result.InternalError); alr.setInfo(alr.getInfo() + "// " + e.getMessage()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index b9bf09cfe68..bb5c2ba34bd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -21,7 +21,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.BeforeAll; - +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Map; import java.util.UUID; import java.util.logging.Logger; @@ -895,4 +896,15 @@ private String createTestNonSuperuserApiToken() { createUserResponse.then().assertThat().statusCode(OK.getStatusCode()); return UtilIT.getApiTokenFromResponse(createUserResponse); } + + @ParameterizedTest + @ValueSource(booleans={true,false}) + public void testChangeSuperUserStatus(Boolean status) { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + Response toggleSuperuser = UtilIT.changeSuperUserStatus(username, status); + toggleSuperuser.then().assertThat() + .statusCode(OK.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java index 8b5ac917dea..d15fda3a1a1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java @@ -18,7 +18,7 @@ */ public class RolesIT { - private static final Logger logger = Logger.getLogger(AdminIT.class.getCanonicalName()); + private static final Logger logger = Logger.getLogger(RolesIT.class.getCanonicalName()); @BeforeAll public static void setUp() { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 080ca0c43e9..9a409ba97fb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1525,6 +1525,11 @@ static Response makeSuperUser(String username) { return response; } + static Response changeSuperUserStatus(String username, Boolean isSuperUser) { + Response response = given().body(isSuperUser).put("/api/admin/superuser/" + username); + return response; + } + static Response reindexDataset(String persistentId) { Response response = given() .get("/api/admin/index/dataset?persistentId=" + persistentId); From bd929670b70f8ee924ccef1cf91c89704622b766 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 28 Mar 2024 17:25:58 -0400 Subject: [PATCH 04/44] Always need file list for Globus --- .../edu/harvard/iq/dataverse/globus/GlobusServiceBean.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 9fda7a0f7f1..ebd1f943a50 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -1257,11 +1257,11 @@ public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse, Long fileId = Long.parseLong(idAsString); // If we need to create a GuestBookResponse record, we have to // look up the DataFile object for this file: + df = dataFileService.findCheapAndEasy(fileId); + selectedFiles.add(df); if (!doNotSaveGuestbookResponse) { - df = dataFileService.findCheapAndEasy(fileId); guestbookResponse.setDataFile(df); fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); - selectedFiles.add(df); } } catch (NumberFormatException nfe) { logger.warning( From 16ed8cc2255195c88375aa1e380c846f60d8706d Mon Sep 17 00:00:00 2001 From: zearaujo25 Date: Thu, 28 Mar 2024 19:11:27 -0300 Subject: [PATCH 05/44] 9887: Adding documentation and deprecation tag --- .../9887-new-superuser-status-endpoint.md | 1 + doc/sphinx-guides/source/api/native-api.rst | 23 ++++++++++++++++++- .../edu/harvard/iq/dataverse/api/Admin.java | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/9887-new-superuser-status-endpoint.md diff --git a/doc/release-notes/9887-new-superuser-status-endpoint.md b/doc/release-notes/9887-new-superuser-status-endpoint.md new file mode 100644 index 00000000000..4069d3cc04b --- /dev/null +++ b/doc/release-notes/9887-new-superuser-status-endpoint.md @@ -0,0 +1 @@ +Adding a new endpoint to make supersuser status change calls idempotent. Also Deprecating and renaming the old API. \ No newline at end of file diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 144e3ac8e5e..7fed31c5438 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5306,13 +5306,34 @@ Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "https://demo.datavers This action changes the identifier of user johnsmith to jsmith. -Make User a SuperUser +Toggle User superuser powers ~~~~~~~~~~~~~~~~~~~~~ Toggles superuser mode on the ``AuthenticatedUser`` whose ``identifier`` (without the ``@`` sign) is passed. :: POST http://$SERVER/api/admin/superuser/$identifier +.. note:: this endpoint is deprecated. Use the change superuser status endpoint + +.. _change-superuser-status: + +Change superuser status +~~~~~~~~~~~~~~~~~~~~~ +Changes a user superuser power status with a boolean value + +.. code-block:: bash + + export SERVER_URL=http://localhost:8080 + export USERNAME=jdoe + export IS_SUPERUSER=true + curl -X PUT "$SERVER/api/admin/superuser/$USERNAME" --data "$IS_SUPERUSER" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" --data "true" + .. _delete-a-user: Delete a User diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 86b4f5ee679..95dd8899978 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1030,6 +1030,7 @@ public Response deleteRole(@Context ContainerRequestContext crc, @PathParam("id" } @Path("superuser/{identifier}") + @Deprecated @POST public Response toggleSuperuser(@PathParam("identifier") String identifier) { ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "toggleSuperuser") From 5570bec5a2559e5607073cb4b01ac1c104016c58 Mon Sep 17 00:00:00 2001 From: zearaujo25 Date: Thu, 28 Mar 2024 19:20:16 -0300 Subject: [PATCH 06/44] 9887: Fixing documentation --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 7fed31c5438..eec5c17a52d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5307,7 +5307,7 @@ Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "https://demo.datavers This action changes the identifier of user johnsmith to jsmith. Toggle User superuser powers -~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Toggles superuser mode on the ``AuthenticatedUser`` whose ``identifier`` (without the ``@`` sign) is passed. :: @@ -5318,7 +5318,7 @@ Toggles superuser mode on the ``AuthenticatedUser`` whose ``identifier`` (withou .. _change-superuser-status: Change superuser status -~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~ Changes a user superuser power status with a boolean value .. code-block:: bash From e1bed704e0ff256f3d4ea8547f038ebe38e080cb Mon Sep 17 00:00:00 2001 From: Jose Lucas Cordeiro Date: Fri, 29 Mar 2024 12:51:30 -0300 Subject: [PATCH 07/44] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index eec5c17a52d..48d49ed40ba 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5306,8 +5306,8 @@ Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "https://demo.datavers This action changes the identifier of user johnsmith to jsmith. -Toggle User superuser powers -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Toggle Superuser Status +~~~~~~~~~~~~~~~~~~~~~~~ Toggles superuser mode on the ``AuthenticatedUser`` whose ``identifier`` (without the ``@`` sign) is passed. :: From e5aca34e452bd8ce1f8c8e5769d96e7b8883b214 Mon Sep 17 00:00:00 2001 From: Jose Lucas Cordeiro Date: Fri, 29 Mar 2024 12:51:44 -0300 Subject: [PATCH 08/44] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 48d49ed40ba..5205cccea5f 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5317,9 +5317,10 @@ Toggles superuser mode on the ``AuthenticatedUser`` whose ``identifier`` (withou .. _change-superuser-status: -Change superuser status -~~~~~~~~~~~~~~~~~~~~~~~ -Changes a user superuser power status with a boolean value +Set Superuser Status +~~~~~~~~~~~~~~~~~~~~ + +Specify the superuser status of a user with a boolean value. .. code-block:: bash From 4444c0a4d9b757f6f6a4a5ef5e85e455c5ba98fc Mon Sep 17 00:00:00 2001 From: Jose Lucas Cordeiro Date: Fri, 29 Mar 2024 12:52:25 -0300 Subject: [PATCH 09/44] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5205cccea5f..8808c0b5400 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5313,7 +5313,7 @@ Toggles superuser mode on the ``AuthenticatedUser`` whose ``identifier`` (withou POST http://$SERVER/api/admin/superuser/$identifier -.. note:: this endpoint is deprecated. Use the change superuser status endpoint +.. note:: This endpoint is deprecated. Use the set superuser status endpoint instead. .. _change-superuser-status: From 7008b350c28b67cafa8d2db8954532a1426e8971 Mon Sep 17 00:00:00 2001 From: Jose Lucas Cordeiro Date: Fri, 29 Mar 2024 12:52:57 -0300 Subject: [PATCH 10/44] Update doc/release-notes/9887-new-superuser-status-endpoint.md Co-authored-by: Philip Durbin --- doc/release-notes/9887-new-superuser-status-endpoint.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/9887-new-superuser-status-endpoint.md b/doc/release-notes/9887-new-superuser-status-endpoint.md index 4069d3cc04b..01b1f539f7a 100644 --- a/doc/release-notes/9887-new-superuser-status-endpoint.md +++ b/doc/release-notes/9887-new-superuser-status-endpoint.md @@ -1 +1 @@ -Adding a new endpoint to make supersuser status change calls idempotent. Also Deprecating and renaming the old API. \ No newline at end of file +The existing API endpoint for toggling the superuser status of a user has been deprecated in favor of a new API endpoint that allows you to explicitly and idempotently set the status as true or false. For details, see [the guides](https://dataverse-guide--10440.org.readthedocs.build/en/10440/api/native-api.html), #9887 and #10440. \ No newline at end of file From 532e7f0e738544413219d730f7763f5c59230623 Mon Sep 17 00:00:00 2001 From: zearaujo25 Date: Fri, 29 Mar 2024 14:12:22 -0300 Subject: [PATCH 11/44] 9887: code review --- doc/sphinx-guides/source/api/changelog.rst | 5 +++++ doc/sphinx-guides/source/api/native-api.rst | 2 +- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 025a2069d6e..5e55ff46235 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,6 +7,11 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.3 +---- + +- **/api/admin/superuser/{identifier}**: The POST endpoint is being deprecated in favor for the PUT,which we can specifically set the superuser status in the body. + v6.2 ---- diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8808c0b5400..f51eaf31da5 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5333,7 +5333,7 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash - curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" --data "true" + curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -d "true" .. _delete-a-user: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 95dd8899978..1e463a7e19d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1037,7 +1037,7 @@ public Response toggleSuperuser(@PathParam("identifier") String identifier) { .setInfo(identifier); try { final AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); - return changeSuperUserStatus(user, !user.isSuperuser()); + return setSuperuserStatus(user, !user.isSuperuser()); } catch (Exception e) { alr.setActionResult(ActionLogRecord.Result.InternalError); alr.setInfo(alr.getInfo() + "// " + e.getMessage()); @@ -1047,7 +1047,7 @@ public Response toggleSuperuser(@PathParam("identifier") String identifier) { } } - private Response changeSuperUserStatus(AuthenticatedUser user, Boolean isSuperuser) { + private Response setSuperuserStatus(AuthenticatedUser user, Boolean isSuperuser) { if (user.isDeactivated()) { return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); } @@ -1058,11 +1058,11 @@ private Response changeSuperUserStatus(AuthenticatedUser user, Boolean isSuperus @Path("superuser/{identifier}") @PUT - public Response changeSuperUserStatus(@PathParam("identifier") String identifier, Boolean isSuperUser) { + public Response setSuperuserStatus(@PathParam("identifier") String identifier, Boolean isSuperUser) { ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "changeSuperUserStatus") .setInfo(identifier + ":" + isSuperUser); try { - return changeSuperUserStatus(authSvc.getAuthenticatedUser(identifier), isSuperUser); + return setSuperuserStatus(authSvc.getAuthenticatedUser(identifier), isSuperUser); } catch (Exception e) { alr.setActionResult(ActionLogRecord.Result.InternalError); alr.setInfo(alr.getInfo() + "// " + e.getMessage()); From 74f6c4ac8802dadf9355ad9c4af5d3560255a5b2 Mon Sep 17 00:00:00 2001 From: Jose Lucas Cordeiro Date: Fri, 29 Mar 2024 15:19:13 -0300 Subject: [PATCH 12/44] Update doc/sphinx-guides/source/api/changelog.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 5e55ff46235..4c2d332857b 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -10,7 +10,7 @@ This API changelog is experimental and we would love feedback on its usefulness. v6.3 ---- -- **/api/admin/superuser/{identifier}**: The POST endpoint is being deprecated in favor for the PUT,which we can specifically set the superuser status in the body. +- **/api/admin/superuser/{identifier}**: The POST endpoint is being deprecated in favor for the PUT, which we can specifically set the superuser status in the body. v6.2 ---- From 9577295f50df9094291cedfeb66af37fe6c0db2d Mon Sep 17 00:00:00 2001 From: zearaujo25 Date: Fri, 29 Mar 2024 16:45:14 -0300 Subject: [PATCH 13/44] 9887: code review --- src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java | 2 +- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index bb5c2ba34bd..32c417a4fae 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -899,7 +899,7 @@ private String createTestNonSuperuserApiToken() { @ParameterizedTest @ValueSource(booleans={true,false}) - public void testChangeSuperUserStatus(Boolean status) { + public void testSetSuperUserStatus(Boolean status) { Response createUser = UtilIT.createRandomUser(); createUser.then().assertThat().statusCode(OK.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9a409ba97fb..ab0cb4c2426 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1520,6 +1520,7 @@ static Response getMetadataBlockFromDatasetVersion(String persistentId, String v .get("/api/datasets/:persistentId/versions/" + DS_VERSION_LATEST_PUBLISHED + "/metadata/citation?persistentId=" + persistentId); } + @Deprecated static Response makeSuperUser(String username) { Response response = given().post("/api/admin/superuser/" + username); return response; From 12fe4cd002b19309fd92f43a214ba4049c2688e4 Mon Sep 17 00:00:00 2001 From: zearaujo25 Date: Fri, 29 Mar 2024 17:10:39 -0300 Subject: [PATCH 14/44] 9887: renaming function --- src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java | 2 +- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index 32c417a4fae..44f062e8254 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -903,7 +903,7 @@ public void testSetSuperUserStatus(Boolean status) { Response createUser = UtilIT.createRandomUser(); createUser.then().assertThat().statusCode(OK.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); - Response toggleSuperuser = UtilIT.changeSuperUserStatus(username, status); + Response toggleSuperuser = UtilIT.setSuperuserStatus(username, status); toggleSuperuser.then().assertThat() .statusCode(OK.getStatusCode()); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index ab0cb4c2426..dc503a92c0a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1526,7 +1526,7 @@ static Response makeSuperUser(String username) { return response; } - static Response changeSuperUserStatus(String username, Boolean isSuperUser) { + static Response setSuperuserStatus(String username, Boolean isSuperUser) { Response response = given().body(isSuperUser).put("/api/admin/superuser/" + username); return response; } From 08fdc2cfefe00a914552827489b10b46e2eee149 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 29 Mar 2024 17:04:09 -0400 Subject: [PATCH 15/44] fix mixed logic --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 4c436715f0d..1d4a8895947 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3325,8 +3325,8 @@ public boolean validateFilesForDownload(boolean downloadOriginal){ } //Some are selected and there are non-downloadable ones or there are both downloadable and globus transferable files - if ((!(getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty()) - && (!getSelectedNonDownloadableFiles().isEmpty()) || (!getSelectedDownloadableFiles().isEmpty() && !getSelectedGlobusTransferableFiles().isEmpty()))) { + if (((!getSelectedDownloadableFiles().isEmpty() && !getSelectedGlobusTransferableFiles().isEmpty()) + || (!getSelectedNonDownloadableFiles().isEmpty()) && (!getSelectedDownloadableFiles().isEmpty() || !getSelectedGlobusTransferableFiles().isEmpty()))) { setValidateFilesOutcome("Mixed"); return true; } From a90254741dd67919f4c4013c821bae260bdab08f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 29 Mar 2024 17:06:02 -0400 Subject: [PATCH 16/44] further improve separation of download/transfer and messages --- .../edu/harvard/iq/dataverse/DatasetPage.java | 73 +++++++++++++------ src/main/java/propertyFiles/Bundle.properties | 12 +-- src/main/webapp/dataset.xhtml | 54 +++++++++----- src/main/webapp/filesFragment.xhtml | 4 +- 4 files changed, 93 insertions(+), 50 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 1d4a8895947..66db438a3dd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1234,8 +1234,17 @@ public boolean canDownloadFiles() { canDownloadFiles = false; for (FileMetadata fmd : workingVersion.getFileMetadatas()) { if (fileDownloadHelper.canDownloadFile(fmd)) { - canDownloadFiles = true; - break; + if (isVersionHasGlobus()) { + String driverId = DataAccess + .getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier()); + if (StorageIO.isDataverseAccessible(driverId)) { + canDownloadFiles = true; + break; + } + } else { + canDownloadFiles = true; + break; + } } } } @@ -3260,7 +3269,7 @@ public void startDownloadSelectedOriginal() { private void startDownload(boolean downloadOriginal){ boolean guestbookRequired = isDownloadPopupRequired(); - boolean validate = validateFilesForDownload(downloadOriginal); + boolean validate = validateFilesForDownload(downloadOriginal, false); if (validate) { updateGuestbookResponse(guestbookRequired, downloadOriginal, false); if(!guestbookRequired && !getValidateFilesOutcome().equals("Mixed")){ @@ -3283,7 +3292,7 @@ public void setValidateFilesOutcome(String validateFilesOutcome) { this.validateFilesOutcome = validateFilesOutcome; } - public boolean validateFilesForDownload(boolean downloadOriginal){ + public boolean validateFilesForDownload(boolean downloadOriginal, boolean isGlobusTransfer){ if (this.selectedFiles.isEmpty()) { PrimeFaces.current().executeScript("PF('selectFilesForDownload').show()"); return false; @@ -3300,33 +3309,39 @@ public boolean validateFilesForDownload(boolean downloadOriginal){ return false; } - for (FileMetadata fmd : getSelectedDownloadableFiles()) { - DataFile dataFile = fmd.getDataFile(); - if (downloadOriginal && dataFile.isTabularData()) { - bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); - } else { - bytes += dataFile.getFilesize(); + if (!isGlobusTransfer) { + for (FileMetadata fmd : getSelectedDownloadableFiles()) { + DataFile dataFile = fmd.getDataFile(); + if (downloadOriginal && dataFile.isTabularData()) { + bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); + } else { + bytes += dataFile.getFilesize(); + } } - } - //if there are two or more files, with a total size - //over the zip limit, post a "too large" popup - if (bytes > settingsWrapper.getZipDownloadLimit() && selectedDownloadableFiles.size() > 1) { - setValidateFilesOutcome("FailSize"); - return false; + // if there are two or more files, with a total size + // over the zip limit, post a "too large" popup + if (bytes > settingsWrapper.getZipDownloadLimit() && selectedDownloadableFiles.size() > 1) { + setValidateFilesOutcome("FailSize"); + return false; + } } - + // If some of the files were restricted and we had to drop them off the // list, and NONE of the files are left on the downloadable list - // - we show them a "you're out of luck" popup: - if (getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty() && !getSelectedNonDownloadableFiles().isEmpty()) { + // - we show them a "you're out of luck" popup + // Same for globus transfer + if ((!isGlobusTransfer + && (getSelectedDownloadableFiles().isEmpty() && !getSelectedNonDownloadableFiles().isEmpty())) + || (isGlobusTransfer && (getSelectedGlobusTransferableFiles().isEmpty() + && !getSelectedNonGlobusTransferableFiles().isEmpty()))) { setValidateFilesOutcome("FailRestricted"); return false; } - //Some are selected and there are non-downloadable ones or there are both downloadable and globus transferable files - if (((!getSelectedDownloadableFiles().isEmpty() && !getSelectedGlobusTransferableFiles().isEmpty()) - || (!getSelectedNonDownloadableFiles().isEmpty()) && (!getSelectedDownloadableFiles().isEmpty() || !getSelectedGlobusTransferableFiles().isEmpty()))) { + //For download or transfer, there are some that can be downloaded/transferred and some that can't + if ((!isGlobusTransfer && (!getSelectedNonDownloadableFiles().isEmpty() && !getSelectedDownloadableFiles().isEmpty())) || + (isGlobusTransfer && (!getSelectedNonGlobusTransferableFiles().isEmpty() && !getSelectedGlobusTransferableFiles().isEmpty()))) { setValidateFilesOutcome("Mixed"); return true; } @@ -6284,12 +6299,18 @@ public void clearSelectionEmbargo() { PrimeFaces.current().resetInputs("datasetForm:embargoInputs"); } - public boolean isCantDownloadDueToEmbargo() { + public boolean isCantDownloadDueToEmbargoOrDVAccess() { if (getSelectedNonDownloadableFiles() != null) { for (FileMetadata fmd : getSelectedNonDownloadableFiles()) { if (FileUtil.isActivelyEmbargoed(fmd)) { return true; } + if (isVersionHasGlobus()) { + if (StorageIO.isDataverseAccessible( + DataAccess.getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier()))) { + return true; + } + } } } return false; @@ -6381,7 +6402,11 @@ public void startGlobusTransfer(boolean transferAll, boolean popupShown) { } boolean guestbookRequired = isDownloadPopupRequired(); - boolean validated = validateFilesForDownload(true); + boolean validated = validateFilesForDownload(true, true); + + logger.info("SGT valid: " + validated); + logger.info("SGT outcome: " + getValidateFilesOutcome()); + logger.info("SGT popupshown: " + popupShown); if (validated) { globusTransferRequested = true; boolean mixed = "Mixed".equals(getValidateFilesOutcome()); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 692ab9e0686..33ba587ac9e 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1665,17 +1665,19 @@ dataset.noSelectedFiles=Please select one or more files. dataset.noSelectedFilesForDownload=Please select a file or files to be downloaded. dataset.noSelectedFilesForRequestAccess=Please select a file or files for access request. dataset.embargoedSelectedFilesForRequestAccess=Embargoed files cannot be accessed. Please select an unembargoed file or files for your access request. -dataset.inValidSelectedFilesForDownload=Restricted Files Selected -dataset.inValidSelectedFilesForDownloadWithEmbargo=Embargoed and/or Restricted Files Selected -dataset.noValidSelectedFilesForDownload=The selected file(s) may not be downloaded because you have not been granted access. +dataset.inValidSelectedFilesForDownload=Restricted Files, or Files only Acessible Via Globus Selected +dataset.inValidSelectedFilesForDownloadWithEmbargo=Embargoed and/or Restricted Files, or Files only acessible via Globus Selected +dataset.inValidSelectedFilesForTransferWithEmbargo=Embargoed and/or Restricted Files, or Files that are not Globus accessible Selected +dataset.noValidSelectedFilesForDownload=The selected file(s) may not be downloaded because you have not been granted access or the files can only be transferred via Globus. +dataset.noValidSelectedFilesForTransfer=The selected file(s) may not be transferred because you have not been granted access or the files are not Globus accessible. dataset.mixedSelectedFilesForDownload=The restricted file(s) selected may not be downloaded because you have not been granted access. -dataset.mixedSelectedFilesForDownloadWithEmbargo=The embargoed and/or restricted file(s) selected may not be downloaded because you have not been granted access. +dataset.mixedSelectedFilesForDownloadWithEmbargo=Any embargoed and/or restricted file(s) selected may not be downloaded because you have not been granted access. Some files may only be accessible via Globus. dataset.mixedSelectedFilesForTransfer=Some file(s) cannot be transferred. (They are restricted, embargoed, or not Globus accessible.) dataset.inValidSelectedFilesForTransfer=Ineligible Files Selected dataset.downloadUnrestricted=Click Continue to download the files you have access to download. dataset.transferUnrestricted=Click Continue to transfer the elligible files. -dataset.requestAccessToRestrictedFiles=You may request access to the restricted file(s) by clicking the Request Access button. +dataset.requestAccessToRestrictedFiles=You may request access any restricted file(s) by clicking the Request Access button. dataset.requestAccessToRestrictedFilesWithEmbargo=Embargoed files cannot be accessed during the embargo period. If your selection contains restricted files, you may request access to them by clicking the Request Access button. dataset.privateurl.infoMessageAuthor=Privately share this dataset before it is published: {0} dataset.privateurl.infoMessageReviewer=This unpublished dataset is being privately shared. diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 7b5db98b9dd..6424cde5ac2 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -32,7 +32,7 @@ and !permissionsWrapper.canIssuePublishDatasetCommand(DatasetPage.dataset)}"/> - @@ -1053,10 +1054,10 @@ - +

#{bundle['dataset.noValidSelectedFilesForDownload']}

-

#{DatasetPage.cantDownloadDueToEmbargo ? bundle['dataset.requestAccessToRestrictedFilesWithEmbargo'] : bundle['dataset.requestAccessToRestrictedFiles']}

+

#{DatasetPage.cantDownloadDueToEmbargoOrDVAccess ? bundle['dataset.requestAccessToRestrictedFilesWithEmbargo'] : bundle['dataset.requestAccessToRestrictedFiles']}
+ +

#{bundle['dataset.noValidSelectedFilesForTransfer']}

+ +

#{DatasetPage.cantDownloadDueToEmbargoOrDVAccess ? bundle['dataset.requestAccessToRestrictedFilesWithEmbargo'] : bundle['dataset.requestAccessToRestrictedFiles']}

+ +
+ +
+

#{bundle['file.zip.download.exceeds.limit.info']}

@@ -1085,8 +1097,8 @@ - -

#{DatasetPage.cantDownloadDueToEmbargo ? bundle['dataset.mixedSelectedFilesForDownloadWithEmbargo'] : bundle['dataset.mixedSelectedFilesForDownload']}

+ +

#{DatasetPage.cantDownloadDueToEmbargoOrDVAccess ? bundle['dataset.mixedSelectedFilesForDownloadWithEmbargo'] : bundle['dataset.mixedSelectedFilesForDownload']}

@@ -1970,7 +1982,11 @@ PF('downloadTooLarge').show(); } if (outcome ==='FailRestricted'){ - PF('downloadInvalid').show(); + if(isTransfer) { + PF('transferInvalid').show(); + } else { + PF('downloadInvalid').show(); + } } if (outcome ==='GuestbookRequired'){ PF('guestbookAndTermsPopup').show(); diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 58899ab7062..117710cfd53 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -456,7 +456,7 @@ #{bundle.download}
    -
  • +
    • -
  • +
  • Date: Thu, 4 Apr 2024 13:02:13 -0400 Subject: [PATCH 17/44] Wordsmithing tag guidance --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 12c7cc2f6f8..691b5ee12d7 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -792,12 +792,12 @@ Larger installations may want to increase the number of open S3 connections allo ``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"`` -By default, when redirecting an upload to the S3 storage, the Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup when the file is not added to the dataset after upload (e.g., when the user cancels the operation). +By default, when redirecting an upload to the S3 storage, the Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup if the file is not added to the dataset after upload (e.g., if the user cancels the operation). If your S3 store does not support tagging and gives an error when redirecting uploads, you can disable that tag by using the ``dataverse.files..disable-tagging`` JVM option. For example: ``./asadmin create-jvm-options "-Ddataverse.files..disable-tagging=true"`` -Disabling the ``temp`` tag can result in leftover files that are not used by your Dataverse instance and should be removed to preserve the storage space. To clean up the leftover files, you can use the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. +Disabling the ``temp`` tag makes it harder to identify abandoned files that are not used by your Dataverse instance (i.e. one cannot search for the temp tag in a delete script). These should still be removed to avoid wasting storage space. To clean up these files and any other leftover files, regardless of whether the temp tag is applied, you can use the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. In case you would like to configure Dataverse to use a custom S3 service instead of Amazon S3 services, please add the options for the custom URL and region as documented below. Please read above if your desired combination has From 9ef355b02ea1db7facf6b1f95e79d38f7ec972b3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 4 Apr 2024 13:03:20 -0400 Subject: [PATCH 18/44] Update config.rst --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 691b5ee12d7..7c83172f0b2 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -792,7 +792,7 @@ Larger installations may want to increase the number of open S3 connections allo ``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"`` -By default, when redirecting an upload to the S3 storage, the Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup if the file is not added to the dataset after upload (e.g., if the user cancels the operation). +By default, when redirecting an upload to the S3 storage, Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup if the file is not added to the dataset after upload (e.g., if the user cancels the operation). If your S3 store does not support tagging and gives an error when redirecting uploads, you can disable that tag by using the ``dataverse.files..disable-tagging`` JVM option. For example: ``./asadmin create-jvm-options "-Ddataverse.files..disable-tagging=true"`` From 596b5925c0675fe019f6fccf7e063759a5fcc232 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 4 Apr 2024 13:08:45 -0400 Subject: [PATCH 19/44] Update 10022_upload_redirect_without_tagging.md --- doc/release-notes/10022_upload_redirect_without_tagging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/10022_upload_redirect_without_tagging.md b/doc/release-notes/10022_upload_redirect_without_tagging.md index 379c7c8f1e0..107b05f2e2a 100644 --- a/doc/release-notes/10022_upload_redirect_without_tagging.md +++ b/doc/release-notes/10022_upload_redirect_without_tagging.md @@ -1 +1 @@ -If your S3 store does not support tagging and gives an error when redirecting uploads, from now on, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. Disabling the tagging can result in leftover files that are not used by your Dataverse instance and should be removed to preserve the storage space. To clean up the leftover files, you can use the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. +If your S3 store does not support tagging and gives an error when redirecting uploads, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. Disabling the tagging makes it harder to identify abandoned files (created in cases where the user does not complete the upload operation) with an external script but they can still be removed using the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. From be2b7838c21dccc2043125acb6568185968e7bc6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 4 Apr 2024 13:12:13 -0400 Subject: [PATCH 20/44] Update JvmSettings.java --- .../java/edu/harvard/iq/dataverse/settings/JvmSettings.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 794ffd5e0af..0a5662b0e98 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -50,6 +50,8 @@ public enum JvmSettings { UPLOADS_DIRECTORY(SCOPE_FILES, "uploads"), DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"), GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"), + + //STORAGE DRIVER SETTINGS SCOPE_DRIVER(SCOPE_FILES), DISABLE_S3_TAGGING(SCOPE_DRIVER, "disable-tagging"), From 1a81ae7ee5b0cde591b6d6409feff65222150fa3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 4 Apr 2024 15:18:01 -0400 Subject: [PATCH 21/44] fix issue rerunning downloads --- .../java/edu/harvard/iq/dataverse/util/URLTokenUtil.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java index a3293e0cd28..90557a530c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.util; import java.util.Arrays; +import java.util.Random; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -261,7 +262,9 @@ public JsonObject getParams(JsonObject toolParameters) { public static String getScriptForUrl(String url) { String msg = BundleUtil.getStringFromBundle("externaltools.enable.browser.popups"); - String script = "const newWin = window.open('" + url + "', target='_blank'); if (!newWin || newWin.closed || typeof newWin.closed == \"undefined\") {alert(\"" + msg + "\");}"; + String newWin = "newWin" + (new Random()).nextInt(1000000000); + //Always use a unique identifier so that more than one script can run (or one can be rerun) without conflicts + String script = String.format("const %1$s = window.open('" + url + "', target='_blank'); if (!%1$s || %1$s.closed || typeof %1$s.closed == \"undefined\") {alert(\"" + msg + "\");}", newWin); return script; } From 409ee0f6e33f3095cb2763242289da8ee4f54a94 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 11 Apr 2024 15:10:51 -0400 Subject: [PATCH 22/44] Update Bundle.properties --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 33ba587ac9e..d73a06823e7 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1677,7 +1677,7 @@ dataset.inValidSelectedFilesForTransfer=Ineligible Files Selected dataset.downloadUnrestricted=Click Continue to download the files you have access to download. dataset.transferUnrestricted=Click Continue to transfer the elligible files. -dataset.requestAccessToRestrictedFiles=You may request access any restricted file(s) by clicking the Request Access button. +dataset.requestAccessToRestrictedFiles=You may request access to any restricted file(s) by clicking the Request Access button. dataset.requestAccessToRestrictedFilesWithEmbargo=Embargoed files cannot be accessed during the embargo period. If your selection contains restricted files, you may request access to them by clicking the Request Access button. dataset.privateurl.infoMessageAuthor=Privately share this dataset before it is published: {0} dataset.privateurl.infoMessageReviewer=This unpublished dataset is being privately shared. From 4788c104b6a2c9f94d1b2e1cc016836dd53b2ded Mon Sep 17 00:00:00 2001 From: Jose Lucas Cordeiro Date: Thu, 11 Apr 2024 22:13:19 -0300 Subject: [PATCH 23/44] Adding header to documentation --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 52e3948532e..5b2c0586d1c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5433,13 +5433,13 @@ Specify the superuser status of a user with a boolean value. export SERVER_URL=http://localhost:8080 export USERNAME=jdoe export IS_SUPERUSER=true - curl -X PUT "$SERVER/api/admin/superuser/$USERNAME" --data "$IS_SUPERUSER" + curl -X PUT "$SERVER/api/admin/superuser/$USERNAME" -H "Content-Type: application/json" --data "$IS_SUPERUSER" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -d "true" + curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -H "Content-Type: application/json" -d "true" .. _delete-a-user: From 0d9e8df20bc7ee2019735c546aab77a2cdf0ebaa Mon Sep 17 00:00:00 2001 From: Jose Lucas Cordeiro Date: Fri, 12 Apr 2024 11:19:39 -0300 Subject: [PATCH 24/44] Changing header to text/plain --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5b2c0586d1c..958644f7515 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5439,7 +5439,7 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash - curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -H "Content-Type: application/json" -d "true" + curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -H "Content-Type: text/plain" -d "true" .. _delete-a-user: From 4857a888a784ffd0a568588d2ec2b71a8fe73702 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 12 Apr 2024 14:15:50 -0400 Subject: [PATCH 25/44] added isReleased to GET Dataverse reponse --- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 1 + src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index a0278d1a60f..38dab63384f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -292,6 +292,7 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re if (dv.getFilePIDsEnabled() != null) { bld.add("filePIDsEnabled", dv.getFilePIDsEnabled()); } + bld.add("isReleased", dv.isReleased()); return bld; } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index d7f7ff39cbd..0311bfde07e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -413,12 +413,13 @@ public void testMoveDataverse() { Response superuserResponse = UtilIT.makeSuperUser(username); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); - createDataverseResponse.prettyPrint(); + assertTrue(createDataverseResponse.prettyPrint().contains("isReleased\": false")); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); Integer dataverseId = UtilIT.getDataverseIdFromResponse(createDataverseResponse); Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);//.publishDataverseViaSword(dataverseAlias, apiToken); assertEquals(200, publishDataverse.getStatusCode()); + assertTrue(publishDataverse.prettyPrint().contains("isReleased\": true")); Response createDataverseResponse2 = UtilIT.createRandomDataverse(apiToken); createDataverseResponse2.prettyPrint(); From 19a8c3d35b4bc87e005b403858f232ebb7ccc22c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 12 Apr 2024 14:29:39 -0400 Subject: [PATCH 26/44] add release note --- ...dd-isreleased-to-get-dataverse-response.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md diff --git a/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md b/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md new file mode 100644 index 00000000000..5293c7267d0 --- /dev/null +++ b/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md @@ -0,0 +1,22 @@ +The Dataverse object returned by /api/dataverses has been extended to include "isReleased": {boolean}. +```javascript +{ + "status": "OK", + "data": { + "id": 32, + "alias": "dv6f645bb5", + "name": "dv6f645bb5", + "dataverseContacts": [ + { + "displayOrder": 0, + "contactEmail": "54180268@mailinator.com" + } + ], + "permissionRoot": true, + "dataverseType": "UNCATEGORIZED", + "ownerId": 1, + "creationDate": "2024-04-12T18:05:59Z", + "isReleased": true + } +} +``` \ No newline at end of file From 6684ae62ee7392386dcce682510caa7d1730e9b2 Mon Sep 17 00:00:00 2001 From: zearaujo25 Date: Fri, 12 Apr 2024 16:15:48 -0300 Subject: [PATCH 27/44] 9887: Changin body to string to allow empty Content-Type --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 1e463a7e19d..63249ddc151 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -17,6 +17,7 @@ import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.validation.EMailValidator; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.Template; @@ -1058,11 +1059,12 @@ private Response setSuperuserStatus(AuthenticatedUser user, Boolean isSuperuser) @Path("superuser/{identifier}") @PUT - public Response setSuperuserStatus(@PathParam("identifier") String identifier, Boolean isSuperUser) { + //using string instead of boolean so user doesnt need to add Content type header in their request + public Response setSuperuserStatus(@PathParam("identifier") String identifier, String isSuperUser) { ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "changeSuperUserStatus") .setInfo(identifier + ":" + isSuperUser); try { - return setSuperuserStatus(authSvc.getAuthenticatedUser(identifier), isSuperUser); + return setSuperuserStatus(authSvc.getAuthenticatedUser(identifier), StringUtil.isTrue(isSuperUser)); } catch (Exception e) { alr.setActionResult(ActionLogRecord.Result.InternalError); alr.setInfo(alr.getInfo() + "// " + e.getMessage()); From e3ecf8b846f09441961cad9c5b6cb3a60fba994d Mon Sep 17 00:00:00 2001 From: zearaujo25 Date: Fri, 12 Apr 2024 16:19:21 -0300 Subject: [PATCH 28/44] 9887: changing installation scrpt to sue new endpoint --- scripts/api/setup-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/api/setup-all.sh b/scripts/api/setup-all.sh index 5ddd9a35fdc..b7f962209e4 100755 --- a/scripts/api/setup-all.sh +++ b/scripts/api/setup-all.sh @@ -65,7 +65,7 @@ echo echo "Setting up the admin user (and as superuser)" adminResp=$(curl -s -H "Content-type:application/json" -X POST -d @"$SCRIPT_PATH"/data/user-admin.json "${DATAVERSE_URL}/api/builtin-users?password=$DV_SU_PASSWORD&key=burrito") echo "$adminResp" -curl -X POST "${DATAVERSE_URL}/api/admin/superuser/dataverseAdmin" +curl -X PUT "${DATAVERSE_URL}/api/admin/superuser/dataverseAdmin" -d "true" echo echo "Setting up the root dataverse" From a418b5814b74452bc43cfc9396b3b12c26bc7ccf Mon Sep 17 00:00:00 2001 From: Jose Lucas Cordeiro Date: Fri, 12 Apr 2024 16:31:28 -0300 Subject: [PATCH 29/44] 9887: Changin doc --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 958644f7515..ae1509c56f3 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5433,13 +5433,13 @@ Specify the superuser status of a user with a boolean value. export SERVER_URL=http://localhost:8080 export USERNAME=jdoe export IS_SUPERUSER=true - curl -X PUT "$SERVER/api/admin/superuser/$USERNAME" -H "Content-Type: application/json" --data "$IS_SUPERUSER" + curl -X PUT "$SERVER/api/admin/superuser/$USERNAME" -d "$IS_SUPERUSER" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -H "Content-Type: text/plain" -d "true" + curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -d "true" .. _delete-a-user: From 3f89c329bfe572b65901b248b7c639e5f206c975 Mon Sep 17 00:00:00 2001 From: zearaujo25 Date: Fri, 12 Apr 2024 16:34:25 -0300 Subject: [PATCH 30/44] 9887: renaming variable --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 63249ddc151..e1e86adacc6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1060,11 +1060,11 @@ private Response setSuperuserStatus(AuthenticatedUser user, Boolean isSuperuser) @Path("superuser/{identifier}") @PUT //using string instead of boolean so user doesnt need to add Content type header in their request - public Response setSuperuserStatus(@PathParam("identifier") String identifier, String isSuperUser) { + public Response setSuperuserStatus(@PathParam("identifier") String identifier, String isSuperuser) { ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "changeSuperUserStatus") - .setInfo(identifier + ":" + isSuperUser); + .setInfo(identifier + ":" + isSuperuser); try { - return setSuperuserStatus(authSvc.getAuthenticatedUser(identifier), StringUtil.isTrue(isSuperUser)); + return setSuperuserStatus(authSvc.getAuthenticatedUser(identifier), StringUtil.isTrue(isSuperuser)); } catch (Exception e) { alr.setActionResult(ActionLogRecord.Result.InternalError); alr.setInfo(alr.getInfo() + "// " + e.getMessage()); From 95ce8e0c39006451ab5b4fb736ddd84b4c7e9077 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 12 Apr 2024 16:41:11 -0400 Subject: [PATCH 31/44] tweak docs #9887 --- doc/sphinx-guides/source/api/changelog.rst | 2 +- doc/sphinx-guides/source/api/native-api.rst | 26 +++++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index abe42ba2e99..db994a629b3 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -10,7 +10,7 @@ This API changelog is experimental and we would love feedback on its usefulness. v6.3 ---- -- **/api/admin/superuser/{identifier}**: The POST endpoint is being deprecated in favor for the PUT, which we can specifically set the superuser status in the body. +- **/api/admin/superuser/{identifier}**: The POST endpoint that toggles superuser status has been deprecated in favor of a new PUT endpoint that allows you to specify true or false. See :ref:`set-superuser-status`. v6.2 ---- diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index ae1509c56f3..26cca81e0d1 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5415,31 +5415,43 @@ This action changes the identifier of user johnsmith to jsmith. Toggle Superuser Status ~~~~~~~~~~~~~~~~~~~~~~~ -Toggles superuser mode on the ``AuthenticatedUser`` whose ``identifier`` (without the ``@`` sign) is passed. :: +Toggle the superuser status of a user. - POST http://$SERVER/api/admin/superuser/$identifier +.. note:: This endpoint is deprecated as explained in :doc:`/api/changelog`. Please use the :ref:`set-superuser-status` endpoint instead. -.. note:: This endpoint is deprecated. Use the set superuser status endpoint instead. +.. code-block:: bash + + export SERVER_URL=http://localhost:8080 + export USERNAME=jdoe + curl -X POST "$SERVER_URL/api/admin/superuser/$USERNAME" -.. _change-superuser-status: +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X POST "http://localhost:8080/api/admin/superuser/jdoe" + +.. _set-superuser-status: Set Superuser Status ~~~~~~~~~~~~~~~~~~~~ -Specify the superuser status of a user with a boolean value. +Specify the superuser status of a user with a boolean value (``true`` or ``false``). + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. .. code-block:: bash export SERVER_URL=http://localhost:8080 export USERNAME=jdoe export IS_SUPERUSER=true - curl -X PUT "$SERVER/api/admin/superuser/$USERNAME" -d "$IS_SUPERUSER" + curl -X PUT "$SERVER_URL/api/admin/superuser/$USERNAME" -d "$IS_SUPERUSER" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -d "true" + curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -d true .. _delete-a-user: From 074498fd13c3112ea024d7ece0ea26def8fdde3e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 12 Apr 2024 16:45:46 -0400 Subject: [PATCH 32/44] switch BagIT to new "make superuser" API #9887 --- src/test/java/edu/harvard/iq/dataverse/api/BagIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java index 28f7fa28328..c80e321b228 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java @@ -53,8 +53,8 @@ public void testBagItExport() throws IOException { createUser.then().assertThat().statusCode(OK.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); - Response toggleSuperuser = UtilIT.makeSuperUser(username); - toggleSuperuser.then().assertThat() + Response makeSuperuser = UtilIT.setSuperuserStatus(username, true); + makeSuperuser.then().assertThat() .statusCode(OK.getStatusCode()); Response createDataverse = UtilIT.createRandomDataverse(apiToken); From a97b931214f9fce8368e1b7e6a5fcfc8dadcc0a9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 12 Apr 2024 16:49:54 -0400 Subject: [PATCH 33/44] reformat code (no-op) #9887 --- .../edu/harvard/iq/dataverse/api/Admin.java | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index e1e86adacc6..236cadcf128 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1030,49 +1030,49 @@ public Response deleteRole(@Context ContainerRequestContext crc, @PathParam("id" }, getRequestUser(crc)); } - @Path("superuser/{identifier}") - @Deprecated - @POST - public Response toggleSuperuser(@PathParam("identifier") String identifier) { - ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "toggleSuperuser") - .setInfo(identifier); - try { - final AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); - return setSuperuserStatus(user, !user.isSuperuser()); - } catch (Exception e) { - alr.setActionResult(ActionLogRecord.Result.InternalError); - alr.setInfo(alr.getInfo() + "// " + e.getMessage()); - return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } finally { - actionLogSvc.log(alr); - } - } + @Path("superuser/{identifier}") + @Deprecated + @POST + public Response toggleSuperuser(@PathParam("identifier") String identifier) { + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "toggleSuperuser") + .setInfo(identifier); + try { + final AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); + return setSuperuserStatus(user, !user.isSuperuser()); + } catch (Exception e) { + alr.setActionResult(ActionLogRecord.Result.InternalError); + alr.setInfo(alr.getInfo() + "// " + e.getMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + actionLogSvc.log(alr); + } + } - private Response setSuperuserStatus(AuthenticatedUser user, Boolean isSuperuser) { - if (user.isDeactivated()) { - return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); - } - user.setSuperuser(isSuperuser); - return ok("User " + user.getIdentifier() + " " + (user.isSuperuser() ? "set" : "removed") - + " as a superuser."); - } + private Response setSuperuserStatus(AuthenticatedUser user, Boolean isSuperuser) { + if (user.isDeactivated()) { + return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); + } + user.setSuperuser(isSuperuser); + return ok("User " + user.getIdentifier() + " " + (user.isSuperuser() ? "set" : "removed") + + " as a superuser."); + } - @Path("superuser/{identifier}") - @PUT - //using string instead of boolean so user doesnt need to add Content type header in their request - public Response setSuperuserStatus(@PathParam("identifier") String identifier, String isSuperuser) { - ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "changeSuperUserStatus") - .setInfo(identifier + ":" + isSuperuser); - try { + @Path("superuser/{identifier}") + @PUT + //using string instead of boolean so user doesnt need to add Content type header in their request + public Response setSuperuserStatus(@PathParam("identifier") String identifier, String isSuperuser) { + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "changeSuperUserStatus") + .setInfo(identifier + ":" + isSuperuser); + try { return setSuperuserStatus(authSvc.getAuthenticatedUser(identifier), StringUtil.isTrue(isSuperuser)); - } catch (Exception e) { - alr.setActionResult(ActionLogRecord.Result.InternalError); - alr.setInfo(alr.getInfo() + "// " + e.getMessage()); - return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } finally { - actionLogSvc.log(alr); - } - } + } catch (Exception e) { + alr.setActionResult(ActionLogRecord.Result.InternalError); + alr.setInfo(alr.getInfo() + "// " + e.getMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + actionLogSvc.log(alr); + } + } @GET @Path("validate/datasets") From 80ae69bc4f5d5ea649ad23a7b3aeef8aad3fb544 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 12 Apr 2024 16:52:55 -0400 Subject: [PATCH 34/44] make action log match method name, comment tweak #9887 --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 236cadcf128..da42a5c8994 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1059,9 +1059,9 @@ private Response setSuperuserStatus(AuthenticatedUser user, Boolean isSuperuser) @Path("superuser/{identifier}") @PUT - //using string instead of boolean so user doesnt need to add Content type header in their request + // Using string instead of boolean so user doesn't need to add a Content-type header in their request public Response setSuperuserStatus(@PathParam("identifier") String identifier, String isSuperuser) { - ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "changeSuperUserStatus") + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "setSuperuserStatus") .setInfo(identifier + ":" + isSuperuser); try { return setSuperuserStatus(authSvc.getAuthenticatedUser(identifier), StringUtil.isTrue(isSuperuser)); From 22e5b1aa2a224d2bd138f7c025118d37b0a645b1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 16 Apr 2024 16:54:47 -0400 Subject: [PATCH 35/44] tweak docs and release note #10022 --- .../10022_upload_redirect_without_tagging.md | 6 +++++- doc/sphinx-guides/source/api/native-api.rst | 2 +- .../source/developers/big-data-support.rst | 7 ++++++- doc/sphinx-guides/source/installation/config.rst | 10 ++++++---- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/release-notes/10022_upload_redirect_without_tagging.md b/doc/release-notes/10022_upload_redirect_without_tagging.md index 107b05f2e2a..7ff17f08f4c 100644 --- a/doc/release-notes/10022_upload_redirect_without_tagging.md +++ b/doc/release-notes/10022_upload_redirect_without_tagging.md @@ -1 +1,5 @@ -If your S3 store does not support tagging and gives an error when redirecting uploads, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. Disabling the tagging makes it harder to identify abandoned files (created in cases where the user does not complete the upload operation) with an external script but they can still be removed using the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. +If your S3 store does not support tagging and gives an error if you configure direct uploads, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. For more details see https://dataverse-guide--10029.org.readthedocs.build/en/10029/developers/big-data-support.html#s3-tags #10022 and #10029. + +## New config options + +- dataverse.files..disable-tagging diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8e2a9d7b886..848f031178a 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2013,7 +2013,7 @@ The fully expanded example above (without environment variables) looks like this .. _cleanup-storage-api: -Cleanup storage of a Dataset +Cleanup Storage of a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is an experimental feature and should be tested on your system before using it in production. diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 8d891e63317..4aba7881c1f 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -81,7 +81,12 @@ with the contents of the file cors.json as follows: Alternatively, you can enable CORS using the AWS S3 web interface, using json-encoded rules as in the example above. -Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 Tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and the new file(s) are added in the Dataverse installation. Note that not all S3 implementations support Tags: Minio does not. WIth such stores, direct upload works, but Tags are not used. +.. _s3-tags: + +S3 Tags and Direct Upload +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and new files are added in the Dataverse installation. Note that not all S3 implementations support tags. Minio, for example, does not. With such stores, direct upload may not work and you might need to disable tagging. For details, look for ``dataverse.files..disable-tagging`` under :ref:`list-of-s3-storage-options` in the Installation Guide. Trusted Remote Storage with the ``remote`` Store Type ----------------------------------------------------- diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index d32c65ef931..ae27a9727da 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1189,12 +1189,12 @@ Larger installations may want to increase the number of open S3 connections allo ``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"`` -By default, when redirecting an upload to the S3 storage, Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup if the file is not added to the dataset after upload (e.g., if the user cancels the operation). -If your S3 store does not support tagging and gives an error when redirecting uploads, you can disable that tag by using the ``dataverse.files..disable-tagging`` JVM option. For example: +By default, when direct upload to an S3 store is configured, Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup in case the file is not added to the dataset after upload (e.g., if the user cancels the operation). (See :ref:`s3-tags`.) +If your S3 store does not support tagging and gives an error when direct upload is configured, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. For example: ``./asadmin create-jvm-options "-Ddataverse.files..disable-tagging=true"`` -Disabling the ``temp`` tag makes it harder to identify abandoned files that are not used by your Dataverse instance (i.e. one cannot search for the temp tag in a delete script). These should still be removed to avoid wasting storage space. To clean up these files and any other leftover files, regardless of whether the temp tag is applied, you can use the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. +Disabling the ``temp`` tag makes it harder to identify abandoned files that are not used by your Dataverse instance (i.e. one cannot search for the ``temp`` tag in a delete script). These should still be removed to avoid wasting storage space. To clean up these files and any other leftover files, regardless of whether the ``temp`` tag is applied, you can use the :ref:`cleanup-storage-api` API endpoint. In case you would like to configure Dataverse to use a custom S3 service instead of Amazon S3 services, please add the options for the custom URL and region as documented below. Please read above if your desired combination has @@ -1202,6 +1202,8 @@ been tested already and what other options have been set for a successful integr Lastly, go ahead and restart your Payara server. With Dataverse deployed and the site online, you should be able to upload datasets and data files and see the corresponding files in your S3 bucket. Within a bucket, the folder structure emulates that found in local file storage. +.. _list-of-s3-storage-options: + List of S3 Storage Options ########################## @@ -1229,7 +1231,7 @@ List of S3 Storage Options dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` - dataverse.files..disable-tagging ``true``/``false`` Do not place the ``temp`` tag when redirecting the upload to the S3 server ``false`` + dataverse.files..disable-tagging ``true``/``false`` Do not place the ``temp`` tag when redirecting the upload to the S3 server. ``false`` =========================================== ================== =================================================================================== ============= .. table:: From 1c0915a1a98d0be30944424a15846d0bc90e04f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 17 Apr 2024 11:40:44 +0200 Subject: [PATCH 36/44] Add minimal release note snippet --- doc/release-notes/10316_cvoc_http_headers.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/10316_cvoc_http_headers.md diff --git a/doc/release-notes/10316_cvoc_http_headers.md b/doc/release-notes/10316_cvoc_http_headers.md new file mode 100644 index 00000000000..4b557383a2e --- /dev/null +++ b/doc/release-notes/10316_cvoc_http_headers.md @@ -0,0 +1,5 @@ +You are now able to add HTTP request headers required by the External Vocabulary Services you are implementing. + +A combined documentation can be found on pull request [#10404](https://github.com/IQSS/dataverse/pull/10404). + +For more information, see issue [#10316](https://github.com/IQSS/dataverse/issues/10316) and pull request [gddc/dataverse-external-vocab-support#19](https://github.com/gdcc/dataverse-external-vocab-support/pull/19). From f4da07d15df95d2a9e903dd63c30c6f35a830870 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 17 Apr 2024 09:46:15 -0400 Subject: [PATCH 37/44] address failing test - add sleep for reindexing --- src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index be8d5effbfe..b62b050b949 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -862,7 +862,7 @@ public void testNestedSubtree() { Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", apiToken); publishDataset.then().assertThat() .statusCode(OK.getStatusCode()); - + UtilIT.sleepForReindex(datasetPid, apiToken, 5); Response searchPublishedSubtreeWDS = UtilIT.search(searchPart, apiToken, "&subtree="+dataverseAlias); searchPublishedSubtreeWDS.prettyPrint(); searchPublishedSubtreeWDS.then().assertThat() From fa4a03cb023bbebe2464c02d15d5d312c4306bcd Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 17 Apr 2024 09:49:55 -0400 Subject: [PATCH 38/44] doc changes --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 66911aa0ad1..c1ea375d614 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -577,9 +577,9 @@ In general, the external vocabulary support mechanism may be a better choice for The specifics of the user interface for entering/selecting a vocabulary term and how that term is then displayed are managed by third-party Javascripts. The initial Javascripts that have been created provide auto-completion, displaying a list of choices that match what the user has typed so far, but other interfaces, such as displaying a tree of options for a hierarchical vocabulary, are possible. Similarly, existing scripts do relatively simple things for displaying a term - showing the term's name in the appropriate language and providing a link to an external URL with more information, but more sophisticated displays are possible. -Scripts supporting use of vocabularies from services supporting the SKOMOS protocol (see https://skosmos.org) and retrieving ORCIDs (from https://orcid.org) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.) +Scripts supporting use of vocabularies from services supporting the SKOMOS protocol (see https://skosmos.org), retrieving ORCIDs (from https://orcid.org), and using ROR (https://ror.org/) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.) -Configuration involves specifying which fields are to be mapped, whether free-text entries are allowed, which vocabulary(ies) should be used, what languages those vocabulary(ies) are available in, and several service protocol and service instance specific parameters. +Configuration involves specifying which fields are to be mapped, whether free-text entries are allowed, which vocabulary(ies) should be used, what languages those vocabulary(ies) are available in, and several service protocol and service instance specific parameters, including the ability to send HTTP headers on calls to the service. These are all defined in the :ref:`:CVocConf <:CVocConf>` setting as a JSON array. Details about the required elements as well as example JSON arrays are available at https://github.com/gdcc/dataverse-external-vocab-support, along with an example metadata block that can be used for testing. The scripts required can be hosted locally or retrieved dynamically from https://gdcc.github.io/ (similar to how dataverse-previewers work). From ccf34c9ed9e3e86a4a29bc8811ce04a6d8029bf8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 17 Apr 2024 10:24:15 -0400 Subject: [PATCH 39/44] rewrite :CVocConf docs, explain where to find readme.me #10316 --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 2 ++ doc/sphinx-guides/source/installation/config.rst | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 66911aa0ad1..7c31743b454 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -552,6 +552,8 @@ Great care must be taken when reloading a metadata block. Matching is done on fi The ability to reload metadata blocks means that SQL update scripts don't need to be written for these changes. See also the :doc:`/developers/sql-upgrade-scripts` section of the Developer Guide. +.. _using-external-vocabulary-services: + Using External Vocabulary Services ---------------------------------- diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index ff786e900cc..d79a1c4bfe6 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4458,9 +4458,13 @@ A boolean setting that, if true, will send an email and notification to users wh :CVocConf +++++++++ -A JSON-structured setting that configures Dataverse to associate specific metadatablock fields with external vocabulary services and specific vocabularies/sub-vocabularies managed by that service. More information about this capability is available at :doc:`/admin/metadatacustomization`. +The ``:CVocConf`` database setting is used to allow metadatablock fields to look up values in external vocabulary services. For example, you could configure the "Author Affiliation" field to look up organizations in the `Research Organization Registry (ROR) `_. For a high-level description of this feature, see :ref:`using-external-vocabulary-services` in the Admin Guide. -Scripts that implement this association for specific service protocols are maintained at https://github.com/gdcc/dataverse-external-vocab-support. That repository also includes a json-schema for validating the structure required by this setting along with an example metadatablock and sample :CVocConf setting values associating entries in the example block with ORCID and SKOSMOS based services. +The expected format for the ``:CVocConf`` database setting is JSON but the details are not documented here. Instead, please refer to `docs/readme.md `_ in the https://github.com/gdcc/dataverse-external-vocab-support repo. + +That repository also includes scripts that implement the lookup for specific service protocols, a JSON Schema for validating the structure required by this setting, and an example metadatablock with a sample ``:CVocConf`` config that associates fields in the example block with ORCID and SKOSMOS based services. + +The commands below should give you an idea of how to load the configuration, but you'll want to study the examples and make decisions about which configuration to use: ``wget https://gdcc.github.io/dataverse-external-vocab-support/examples/config/cvoc-conf.json`` From 6c30a9e72b59a715146c19d070634bc8ed06a7d7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 17 Apr 2024 13:19:31 -0400 Subject: [PATCH 40/44] remove logger.infos --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 66db438a3dd..3c0949dd9f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -6404,9 +6404,6 @@ public void startGlobusTransfer(boolean transferAll, boolean popupShown) { boolean validated = validateFilesForDownload(true, true); - logger.info("SGT valid: " + validated); - logger.info("SGT outcome: " + getValidateFilesOutcome()); - logger.info("SGT popupshown: " + popupShown); if (validated) { globusTransferRequested = true; boolean mixed = "Mixed".equals(getValidateFilesOutcome()); From e473d53bf73b2579f531c7d06d6c29b81bd8568c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 17 Apr 2024 14:53:33 -0400 Subject: [PATCH 41/44] explain how to avoid error when tags are disabled #10022 --- .../source/developers/big-data-support.rst | 4 ++-- .../source/developers/s3-direct-upload-api.rst | 6 ++++++ doc/sphinx-guides/source/installation/config.rst | 12 +++++++++++- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 15 +++++++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 4aba7881c1f..087cdb6303e 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -81,12 +81,12 @@ with the contents of the file cors.json as follows: Alternatively, you can enable CORS using the AWS S3 web interface, using json-encoded rules as in the example above. -.. _s3-tags: +.. _s3-tags-and-direct-upload: S3 Tags and Direct Upload ~~~~~~~~~~~~~~~~~~~~~~~~~ -Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and new files are added in the Dataverse installation. Note that not all S3 implementations support tags. Minio, for example, does not. With such stores, direct upload may not work and you might need to disable tagging. For details, look for ``dataverse.files..disable-tagging`` under :ref:`list-of-s3-storage-options` in the Installation Guide. +Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and new files are added in the Dataverse installation. Note that not all S3 implementations support tags. Minio, for example, does not. With such stores, direct upload may not work and you might need to disable tagging. For details, see :ref:`s3-tagging` in the Installation Guide. Trusted Remote Storage with the ``remote`` Store Type ----------------------------------------------------- diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index 0040c1fd3f0..33b8e434e6e 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -79,6 +79,12 @@ In the single part case, only one call to the supplied URL is required: curl -i -H 'x-amz-tagging:dv-state=temp' -X PUT -T "" +Or, if you have disabled S3 tagging (see :ref:`s3-tagging`), you should omit the header like this: + +.. code-block:: bash + + curl -i -X PUT -T "" + Note that without the ``-i`` flag, you should not expect any output from the command above. With the ``-i`` flag, you should expect to see a "200 OK" response. In the multipart case, the client must send each part and collect the 'eTag' responses from the server. The calls for this are the same as the one for the single part case except that each call should send a slice of the total file, with the last part containing the remaining bytes. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index ae27a9727da..75ae760aa4a 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1189,13 +1189,23 @@ Larger installations may want to increase the number of open S3 connections allo ``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"`` -By default, when direct upload to an S3 store is configured, Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup in case the file is not added to the dataset after upload (e.g., if the user cancels the operation). (See :ref:`s3-tags`.) +.. _s3-tagging: + +S3 Tagging +########## + +By default, when direct upload to an S3 store is configured, Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup in case the file is not added to the dataset after upload (e.g., if the user cancels the operation). (See :ref:`s3-tags-and-direct-upload`.) If your S3 store does not support tagging and gives an error when direct upload is configured, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. For example: ``./asadmin create-jvm-options "-Ddataverse.files..disable-tagging=true"`` Disabling the ``temp`` tag makes it harder to identify abandoned files that are not used by your Dataverse instance (i.e. one cannot search for the ``temp`` tag in a delete script). These should still be removed to avoid wasting storage space. To clean up these files and any other leftover files, regardless of whether the ``temp`` tag is applied, you can use the :ref:`cleanup-storage-api` API endpoint. +Note that if you disable tagging, you should should omit the ``x-amz-tagging:dv-state=temp`` header when using the :doc:`/developers/s3-direct-upload-api`, as noted in that section. + +Finalizing S3 Configuration +########################### + In case you would like to configure Dataverse to use a custom S3 service instead of Amazon S3 services, please add the options for the custom URL and region as documented below. Please read above if your desired combination has been tested already and what other options have been set for a successful integration. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 03f41fc409d..ccbbe8bd619 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2512,6 +2512,21 @@ static Response getUploadUrls(String idOrPersistentIdOfDataset, long sizeInBytes return requestSpecification.get("/api/datasets/" + idInPath + "/uploadurls?size=" + sizeInBytes + optionalQueryParam); } + /** + * If you set dataverse.files.localstack1.disable-tagging=true you will see + * an error like below. + * + * To avoid it, don't send the x-amz-tagging header. + */ + /* + + AccessDenied + There were headers present in the request which were not signed + 25ff2bb0-13c7-420e-8ae6-3d92677e4bd9 + 9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg= + x-amz-tagging + + */ static Response uploadFileDirect(String url, InputStream inputStream) { return given() .header("x-amz-tagging", "dv-state=temp") From 04f3dd5aec4b995c80773d65167819bb821d353f Mon Sep 17 00:00:00 2001 From: landreev Date: Thu, 18 Apr 2024 14:15:21 -0400 Subject: [PATCH 42/44] Allow guest user in requestGlobusDownload... a quick experiment to see if changes elsewhere may be needed --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 6d8fbe1808c..7b98c95e08c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3721,11 +3721,11 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P // ------------------------------------- // (1) Get the user from the ContainerRequestContext // ------------------------------------- - AuthenticatedUser authUser; + AuthenticatedUser authUser = null; try { authUser = getRequestAuthenticatedUserOrDie(crc); } catch (WrappedResponse e) { - return e.getResponse(); + logger.fine("guest user globus download"); } // ------------------------------------- // (2) Get the Dataset Id From e546989e0e5df2a4049a65db0b9880691d66b1e3 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 19 Apr 2024 10:04:07 -0400 Subject: [PATCH 43/44] Another minor guest user fix --- .../iq/dataverse/globus/GlobusServiceBean.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index ebd1f943a50..fb50214c259 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -985,10 +985,14 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro if (taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE")) { String comment = "Reason : " + taskStatus.split("#")[1] + "
    Short Description : " + taskStatus.split("#")[2]; - userNotificationService.sendNotification((AuthenticatedUser) authUser, new Timestamp(new Date().getTime()), - UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, dataset.getId(), comment, true); - globusLogger.info("Globus task failed during download process"); - } else { + if (authUser != null && authUser instanceof AuthenticatedUser) { + userNotificationService.sendNotification((AuthenticatedUser) authUser, new Timestamp(new Date().getTime()), + UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, dataset.getId(), comment, true); + } + + globusLogger.info("Globus task failed during download process: "+comment); + } else if (authUser != null && authUser instanceof AuthenticatedUser) { + boolean taskSkippedFiles = (task.getSkip_source_errors() == null) ? false : task.getSkip_source_errors(); if (!taskSkippedFiles) { userNotificationService.sendNotification((AuthenticatedUser) authUser, From 4a64ad0efa799a9cfccff5a4a1304e0fc3d8afed Mon Sep 17 00:00:00 2001 From: Juan Pablo Tosca Villanueva <142103991+jp-tosca@users.noreply.github.com> Date: Fri, 19 Apr 2024 12:16:22 -0400 Subject: [PATCH 44/44] Update native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 67f3a0e17b8..30ff90e0179 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -5879,7 +5879,7 @@ Superusers can add a new license by posting a JSON file adapted from this exampl .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - curl -X POST -H 'Content-Type: application/json' -H "X-Dataverse-key:$API_TOKEN" --data-binary @add-license.json "$SERVER_URL/api/licenses" + curl -X POST -H 'Content-Type: application/json' -H "X-Dataverse-key:$API_TOKEN" --upload-file add-license.json "$SERVER_URL/api/licenses" Superusers can change whether an existing license is active (usable for new dataset versions) or inactive (only allowed on already-published versions) specified by the license ``$ID``: