From 2903b0888a8d8591f68dec2afca089bee17bdcfe Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 7 Oct 2020 15:34:05 -0400 Subject: [PATCH 001/274] Consolidated file metadata on dataset pg by using file-info-fragment from file replace pg [ref #7081] --- src/main/webapp/editdatafiles.xhtml | 1 + src/main/webapp/file-info-fragment.xhtml | 40 +++++++++-- src/main/webapp/filesFragment.xhtml | 92 ++---------------------- 3 files changed, 41 insertions(+), 92 deletions(-) diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index d81e6da22a1..0965ec5520c 100644 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -43,6 +43,7 @@
+ diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index dff487454ea..a8ffd40cb7a 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -31,13 +31,43 @@
- #{fileMetadata.label} + + + #{fileMetadata.label} + + + + #{fileMetadata.label} +
+
- #{fileMetadata.dataFile.friendlyType} - - #{fileMetadata.dataFile.friendlySize} - - #{fileMetadata.fileDateToDisplay} -
#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue}
+ +
+ +
+ + + + + + + + +
+ +
+ +
+ + + +
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 62453b09fe5..bb7041bda66 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -365,93 +365,11 @@
-
-
-
- - - #{fileMetadata.label} - - - - - - - - #{fileMetadata.label} - - - - -
- -
-
- -
-
-
- -
- - - - #{fileMetadata.label} - - - - - #{fileMetadata.label} - - - - -
- -
- -
- - - - - - - - - - -
- -
-
- -
- - - -
- -
- -
- -
- - - - - - - - - - -
-
-
+ +             +             +             +            From f094b97dce164f28701492db089de3164c85ef14 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 8 Oct 2020 11:19:49 -0400 Subject: [PATCH 002/274] Fixed download count error on file-info-fragment, as well as cleaned up thumbnail code [ref #7081] --- .../edu/harvard/iq/dataverse/DatasetPage.java | 9 --------- .../harvard/iq/dataverse/EditDatafilesPage.java | 14 -------------- src/main/webapp/file-info-fragment.xhtml | 15 ++++++--------- 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 458fcf56ab0..43d945c657c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1311,15 +1311,6 @@ public boolean isSuperUser(){ } return false; } - /* - TODO/OPTIMIZATION: This is still costing us N SELECT FROM GuestbookResponse queries, - where N is the number of files. This could of course be replaced by a query that'll - look up all N at once... Not sure if it's worth it; especially now that N - will always be 10, for the initial page load. -- L.A. 4.2.1 - */ - public Long getGuestbookResponseCount(FileMetadata fileMetadata) { - return guestbookResponseService.getCountGuestbookResponsesByDataFileId(fileMetadata.getDataFile().getId()); - } /** * Check Dataset related permissions * diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 3a975d1018e..040988a6536 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2562,20 +2562,6 @@ public boolean isLocked() { return false; } - public boolean isThumbnailAvailable(FileMetadata fileMetadata) { - // new and optimized logic: - // - check download permission here (should be cached - so it's free!) - // - only then ask the file service if the thumbnail is available/exists. - // the service itself no longer checks download permissions. - if (!fileDownloadHelper.canDownloadFile(fileMetadata)) { - return false; - } - - return datafileService.isThumbnailAvailable(fileMetadata.getDataFile()); - } - - - private Boolean lockedFromEditsVar; public boolean isLockedFromEdits() { diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index a8ffd40cb7a..4fbfa4606f2 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -14,8 +14,8 @@
- - + @@ -24,7 +24,7 @@
-
+
@@ -48,16 +48,13 @@
- + - + +
From 56dc4c75ba9acf9c66906169bb9baa2dc74a7691 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 8 Oct 2020 15:52:24 -0400 Subject: [PATCH 003/274] Removed additional editMode render logic not needed in file table on dataset pg [ref #7081] --- src/main/webapp/file-info-fragment.xhtml | 1 + src/main/webapp/filesFragment.xhtml | 35 ++++++------------------ 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 4fbfa4606f2..ddc4b97a017 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -32,6 +32,7 @@
+ #{fileMetadata.label} diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index bb7041bda66..64f35a2bab3 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -43,7 +43,7 @@ value="#{DatasetPage.fileMetadatasSearch}" rowIndexVar="rowNum" rowKey="#{fileMetadata.dataFile.storageIdentifier}" selection="#{DatasetPage.selectedFiles}" var="fileMetadata" widgetVar="filesTable" - rendered="#{DatasetPage.fileDisplayTable and empty DatasetPage.editMode and (DatasetPage.workingVersion != null)}" + rendered="#{DatasetPage.fileDisplayTable and (DatasetPage.workingVersion != null)}" emptyMessage="#{DatasetPage.workingVersion.fileMetadatas.size() == 0 ? bundle['file.notFound.tip'] : bundle['file.notFound.search']}"> @@ -118,13 +118,9 @@
- -
1))}"> - - - +
- - + - - - - - - - - #{node.label} - - - - - #{node.label} - - - + + + + #{node.label} + (#{node.dataFile.friendlySize}) From 44722f51a12878d8e1bf01d452eed082f0b6a6e6 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 9 Oct 2020 12:27:56 -0400 Subject: [PATCH 004/274] Removed commented out isDuplicate render logic from checksum in dataset pg file table [ref #7081] --- src/main/webapp/file-info-fragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index ddc4b97a017..31068b7f1ae 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -58,7 +58,7 @@
- +
From 9eb7f9aef2dd38556d025431b840ae42783300e2 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 9 Oct 2020 12:50:37 -0400 Subject: [PATCH 005/274] Additional render logic clean up on file-info-fragment [ref #7081] --- src/main/webapp/file-info-fragment.xhtml | 36 +++++++++--------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 31068b7f1ae..64e5da35368 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -13,7 +13,7 @@
-
+
-
+
@@ -44,44 +44,34 @@
-
- -
- - + + - - - +
- - + +
-
- - - - - - - - - - + + + + + +
From 81384baec44f1a100ff84434a511bc7df93d975e Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 15 Oct 2020 12:56:42 -0400 Subject: [PATCH 006/274] Added checksum truncation function WIP [ref #7081 #6685 #5210] --- src/main/webapp/file-info-fragment.xhtml | 5 ++- .../resources/js/dv_rebind_bootstrap_ui.js | 39 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 64e5da35368..c14c8f98b60 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -53,7 +53,10 @@
- + #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} +
diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 15c14e7d8bd..31b49d5827a 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -29,6 +29,45 @@ function bind_bsui_components(){ // Disabled pagination links disabledLinks(); + // Truncate checksums + // TO-DO DETERMINE IF THIS IS PLACE IS OVERKILL, ON EVERY PG, EVERY REBIND ACTION ... + // PROBABLY NEEDED LIKE ALL THE OTHER FUNCTIONS FOR DYNAMIC AJAX UPDATES ON PG + checksumTruncate(); + + /* + * Truncate file checksums + */ + function checksumTruncate(){ + + // TO-DO MOVE THIS CHUNK OUT OF BIND_BSUI, TO BOTTOM OF THIS FILE + + $('span.checksum-truncate').each(function () { + + var checksumText = $(this).text(); + + console.log("Total Checksums..." + checksumText); + + $(this).css('color', 'red'); + + console.log("Characters..." + checksumText.length); + + if (checksumText.length > 35) { + $(this).text( checksumText.substr(0, 8) + '...' + checksumText.substr(checksumText.length-3, checksumText.length) ); + } + + // TO-DO WIRE UP CLICK TO COPY W/ CLIPBOARD.JS ... SEE JIM'S PR 5211 + // var clipboard = new Clipboard('button.checksum'); + // clipboard.on('success', function(e) { + // console.log(e); + // }); + // clipboard.on('error', function(e) { + // console.log(e); + // }); + + }); + + } + // Sharrre sharrre(); From 36ede87d12859a310d98e0afdfc57f61588997f7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 2 Nov 2020 14:56:54 -0500 Subject: [PATCH 007/274] Add ASCIIFoldingFilterFactory to handle non-ascii chars in search --- conf/solr/7.7.2/schema.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/solr/7.7.2/schema.xml b/conf/solr/7.7.2/schema.xml index da40a8e99fa..b0ecaba9c99 100644 --- a/conf/solr/7.7.2/schema.xml +++ b/conf/solr/7.7.2/schema.xml @@ -38,7 +38,7 @@ catchall "text" field, and use that for searching. --> - + +
From 35982b764575553c5a3fdc5f8f0f18f2274dbd8e Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 5 Jan 2021 15:23:27 -0500 Subject: [PATCH 009/274] Checksum truncate and click to copy added to file table on dataset pg [ref #5210] --- src/main/java/propertyFiles/Bundle.properties | 2 +- src/main/webapp/editFilesFragment.xhtml | 26 +++++----- src/main/webapp/file-info-fragment.xhtml | 12 ++--- src/main/webapp/filesFragment.xhtml | 2 +- .../resources/js/dv_rebind_bootstrap_ui.js | 50 ++++++++----------- 5 files changed, 41 insertions(+), 51 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 79af8899cc9..b27ab777ba6 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1557,7 +1557,7 @@ file.rsyncUpload.httpUploadDisabledDueToRsyncFileExisting=HTTP upload is disable file.rsyncUpload.httpUploadDisabledDueToRsyncFileExistingAndPublished=HTTP upload is disabled for this dataset because you have already uploaded files via rsync and published the dataset. file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttp=Upload with rsync + SSH is disabled for this dataset because you have already uploaded files via HTTP. If you would like to switch to rsync upload, then you must first remove all uploaded files from this dataset. Once this dataset is published, the chosen upload method is permanently locked in. file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttpAndPublished=Upload with rsync + SSH is disabled for this dataset because you have already uploaded files via HTTP and published the dataset. - +file.metaData.checksum.copy=Click to copy file.metaData.dataFile.dataTab.variables=Variables file.metaData.dataFile.dataTab.observations=Observations file.addDescription=Add file description... diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 833a6c2fd07..5ad2dc6dab0 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -326,8 +326,8 @@ -
-
+
+
-
+
@@ -369,20 +369,20 @@ style="width:60%; margin-bottom:.5em;" disabled="#{fileMetadata.dataFile.filePackage}" widgetVar="fileDirectoryName"/> -
+ -
- +
+ +
+ +
+ + +
-
- -
- - -
- +
-
+
@@ -52,15 +52,15 @@ -
+
#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} - - + data-clipboard-action="copy" + data-clipboard-text="#{fileMetadata.dataFile.checksumValue}" + title="#{bundle['file.metaData.checksum.copy']} #{fileMetadata.dataFile.checksumValue}">#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue}
-
+
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index b83d53f89a4..9f958cd451c 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -370,7 +370,7 @@             - +
'); + var clipboard = new Clipboard('span.checksum-truncate'); + clipboard.on('success', function(e) { + console.log(e); + }); + clipboard.on('error', function(e) { + console.log(e); + }); + }); +} /* * Select dataset/file citation onclick event From 8b9bb2c8c8c9dfd3ed036516566b54fee9045f5c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 17:10:31 -0500 Subject: [PATCH 010/274] updates for proper usage of "Dataverse" --- doc/sphinx-guides/source/admin/backups.rst | 4 +- doc/sphinx-guides/source/admin/dashboard.rst | 6 +-- .../source/admin/dataverses-datasets.rst | 48 +++++++++---------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/sphinx-guides/source/admin/backups.rst b/doc/sphinx-guides/source/admin/backups.rst index e11aa84ebfc..49cc9760622 100644 --- a/doc/sphinx-guides/source/admin/backups.rst +++ b/doc/sphinx-guides/source/admin/backups.rst @@ -4,8 +4,8 @@ Backups .. contents:: Contents: :local: -Running tape, or similar backups to ensure the long term preservation of all the data stored in the Dataverse is an implied responsibility that should be taken most seriously. +Running tape, or similar backups to ensure the long term preservation of all the data stored in the Dataverse Repository is an implied responsibility that should be taken most seriously. -*In addition* to running these disk-level backups, we have provided an experimental script that can be run on schedule (via a cron job or something similar) to create extra archival copies of all the Datafiles stored in the Dataverse on a remote storage server, accessible via an ssh connection. The script and some documentation can be found in ``scripts/backup/run_backup`` in the Dataverse source tree at https://github.com/IQSS/dataverse . Some degree of knowledge of system administration and Python is required. +*In addition* to running these disk-level backups, we have provided an experimental script that can be run on schedule (via a cron job or something similar) to create extra archival copies of all the Datafiles stored in the Dataverse Repository on a remote storage server, accessible via an ssh connection. The script and some documentation can be found in ``scripts/backup/run_backup`` in the Dataverse Project source tree at https://github.com/IQSS/dataverse . Some degree of knowledge of system administration and Python is required. Once again, the script is experimental and NOT a replacement of regular and reliable disk backups! diff --git a/doc/sphinx-guides/source/admin/dashboard.rst b/doc/sphinx-guides/source/admin/dashboard.rst index 5ee53790471..a3ab90110ea 100644 --- a/doc/sphinx-guides/source/admin/dashboard.rst +++ b/doc/sphinx-guides/source/admin/dashboard.rst @@ -1,7 +1,7 @@ Dashboard ========= -Dataverse offers a dashboard of administrative tools for superusers only. If you are a logged-in superuser, you can access it by clicking your username in the navbar, and then clicking "Dashboard" from the dropdown. You can verify that you are a superuser by checking the color of your username in the navbar. If it's red, you have the right permissions to use the Dashboard. Superusers can give other users the superuser status via :doc:`user-administration`. +The Dataverse Platform offers a dashboard of administrative tools for superusers only. If you are a logged-in superuser, you can access it by clicking your username in the navbar, and then clicking "Dashboard" from the dropdown. You can verify that you are a superuser by checking the color of your username in the navbar. If it's red, you have the right permissions to use the Dashboard. Superusers can give other users the superuser status via :doc:`user-administration`. .. contents:: Contents: :local: @@ -22,7 +22,7 @@ This dashboard tool allows you to define sets of local datasets to make availabl Metadata Export --------------- -This part of the Dashboard is simply a reminder message that metadata export happens through the Harvard Dataverse API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. +This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse Platform API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. Users ----- @@ -32,4 +32,4 @@ This dashboard tool allows you to search a list of all users of your Dataverse i Move Data --------- -This tool allows you to move datasets. To move dataverses, see the :doc:`dataverses-datasets` section. +This tool allows you to move datasets. To move Dataverse collections, see the :doc:`dataverses-datasets` section. diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index 9c122c25abc..1fe70e34997 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -4,46 +4,46 @@ Managing Datasets and Dataverses .. contents:: |toctitle| :local: -Dataverses ----------- +Dataverse Collections +--------------------- -Delete a Dataverse -^^^^^^^^^^^^^^^^^^ +Delete a Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Dataverses have to be empty to delete them. Navigate to the dataverse and click "Edit" and then "Delete Dataverse" to delete it. To delete a dataverse via API, see the :doc:`/api/native-api` section of the API Guide. +Dataverse collections have to be empty to delete them. Navigate to the Dataverse collection and click "Edit" and then "Delete Dataverse" to delete it. To delete a Dataverse collection via API, see the :doc:`/api/native-api` section of the API Guide. -Move a Dataverse -^^^^^^^^^^^^^^^^ +Move a Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Moves a dataverse whose id is passed to a new dataverse whose id is passed. The dataverse alias also may be used instead of the id. If the moved dataverse has a guestbook, template, metadata block, link, or featured dataverse that is not compatible with the destination dataverse, you will be informed and given the option to force the move and remove the association. Only accessible to superusers. :: +Moves a Dataverse collection whose id is passed to a new Dataverse collection whose id is passed. The Dataverse collection alias also may be used instead of the id. If the moved Dataverse collection has a guestbook, template, metadata block, link, or featured Dataverse collection that is not compatible with the destination Dataverse collection, you will be informed and given the option to force the move and remove the association. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/dataverses/$id/move/$destination-id -Link a Dataverse -^^^^^^^^^^^^^^^^ +Link a Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Creates a link between a dataverse and another dataverse (see the :ref:`dataverse-linking` section of the User Guide for more information). Only accessible to superusers. :: +Creates a link between a Dataverse collection and another Dataverse collection (see the :ref:`dataverse-linking` section of the User Guide for more information). Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X PUT http://$SERVER/api/dataverses/$linked-dataverse-alias/link/$linking-dataverse-alias -Unlink a Dataverse -^^^^^^^^^^^^^^^^^^ +Unlink a Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Removes a link between a dataverse and another dataverse. Only accessible to superusers. :: +Removes a link between a Dataverse collection and another Dataverse collection. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/dataverses/$linked-dataverse-alias/deleteLink/$linking-dataverse-alias -Add Dataverse RoleAssignments to Child Dataverses -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Add Dataverse Collection RoleAssignments to Dataverse Subcollections +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Recursively assigns the users and groups having a role(s),that are in the set configured to be inheritable via the :InheritParentRoleAssignments setting, on a specified dataverse to have the same role assignments on all of the dataverses that have been created within it. The response indicates success or failure and lists the individuals/groups and dataverses involved in the update. Only accessible to superusers. :: +Recursively assigns the users and groups having a role(s),that are in the set configured to be inheritable via the :InheritParentRoleAssignments setting, on a specified Dataverse collections to have the same role assignments on all of the Dataverse collections that have been created within it. The response indicates success or failure and lists the individuals/groups and Dataverse collections involved in the update. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/addRoleAssignmentsToChildren -Configure a Dataverse to store all new files in a specific file store -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Configure a Dataverse Collection to store all new files in a specific file store +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To direct new files (uploaded when datasets are created or edited) for all datasets in a given dataverse, the store can be specified via the API as shown below, or by editing the 'General Information' for a Dataverse on the Dataverse page. Only accessible to superusers. :: +To direct new files (uploaded when datasets are created or edited) for all datasets in a given Dataverse collection, the store can be specified via the API as shown below, or by editing the 'General Information' for a Dataverse collection on the Dataverse collection page. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X PUT -d $storageDriverLabel http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver @@ -70,21 +70,21 @@ Move a Dataset Superusers can move datasets using the dashboard. See also :doc:`dashboard`. -Moves a dataset whose id is passed to a dataverse whose alias is passed. If the moved dataset has a guestbook or a dataverse link that is not compatible with the destination dataverse, you will be informed and given the option to force the move (with ``forceMove=true`` as a query parameter) and remove the guestbook or link (or both). Only accessible to users with permission to publish the dataset in the original and destination dataverse. :: +Moves a dataset whose id is passed to a Dataverse collection whose alias is passed. If the moved dataset has a guestbook or a Dataverse collection link that is not compatible with the destination Dataverse collection, you will be informed and given the option to force the move (with ``forceMove=true`` as a query parameter) and remove the guestbook or link (or both). Only accessible to users with permission to publish the dataset in the original and destination Dataverse collection. :: curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://$SERVER/api/datasets/$id/move/$alias Link a Dataset ^^^^^^^^^^^^^^ -Creates a link between a dataset and a dataverse (see the :ref:`dataset-linking` section of the User Guide for more information). :: +Creates a link between a dataset and a Dataverse collection (see the :ref:`dataset-linking` section of the User Guide for more information). :: curl -H "X-Dataverse-key: $API_TOKEN" -X PUT http://$SERVER/api/datasets/$linked-dataset-id/link/$linking-dataverse-alias Unlink a Dataset ^^^^^^^^^^^^^^^^ -Removes a link between a dataset and a dataverse. Only accessible to superusers. :: +Removes a link between a dataset and a Dataverse collection. Only accessible to superusers. :: curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/datasets/$linked-dataset-id/deleteLink/$linking-dataverse-alias @@ -130,7 +130,7 @@ As a superuser, click "Update Current Version" when publishing. (This option is Diagnose Constraint Violations Issues in Datasets ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To identify invalid data values in specific datasets (if, for example, an attempt to edit a dataset results in a ConstraintViolationException in the server log), or to check all the datasets in the Dataverse for constraint violations, see :ref:`Dataset Validation ` in the :doc:`/api/native-api` section of the User Guide. +To identify invalid data values in specific datasets (if, for example, an attempt to edit a dataset results in a ConstraintViolationException in the server log), or to check all the datasets in the Dataverse installation for constraint violations, see :ref:`Dataset Validation ` in the :doc:`/api/native-api` section of the User Guide. Configure a Dataset to store all new files in a specific file store ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 694364f9bdefb3ca36513bb0c0c73a27a67bb7fd Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 17:17:55 -0500 Subject: [PATCH 011/274] updates for proper usage of Dataverse --- .../source/admin/external-tools.rst | 28 +++++++++---------- .../source/admin/geoconnect-worldmap.rst | 10 +++---- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst index 3e1965c48ea..4426ab2d37f 100644 --- a/doc/sphinx-guides/source/admin/external-tools.rst +++ b/doc/sphinx-guides/source/admin/external-tools.rst @@ -1,7 +1,7 @@ External Tools ============== -External tools can provide additional features that are not part of Dataverse itself, such as data file previews, visualization, and curation. +External tools can provide additional features that are not part of the Dataverse Platform itself, such as data file previews, visualization, and curation. .. contents:: |toctitle| :local: @@ -22,10 +22,10 @@ Inventory of External Tools Managing External Tools ----------------------- -Adding External Tools to Dataverse -+++++++++++++++++++++++++++++++++++ +Adding External Tools to a Dataverse Installation ++++++++++++++++++++++++++++++++++++++++++++++++++ -To add an external tool to your installation of Dataverse you must first download a JSON file for that tool, which we refer to as a "manifest". It should look something like this: +To add an external tool to your Dataverse installation you must first download a JSON file for that tool, which we refer to as a "manifest". It should look something like this: .. literalinclude:: ../_static/installation/files/root/external-tools/fabulousFileTool.json @@ -37,27 +37,27 @@ Configure the tool with the curl command below, making sure to replace the ``fab curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools --upload-file fabulousFileTool.json -Listing All External Tools in Dataverse -+++++++++++++++++++++++++++++++++++++++ +Listing All External Tools in a Dataverse Installation +++++++++++++++++++++++++++++++++++++++++++++++++++++++ -To list all the external tools that are available in Dataverse: +To list all the external tools that are available in a Dataverse installation: .. code-block:: bash curl http://localhost:8080/api/admin/externalTools -Showing an External Tool in Dataverse -+++++++++++++++++++++++++++++++++++++ +Showing an External Tool in a Dataverse Installation +++++++++++++++++++++++++++++++++++++++++++++++++++++ -To show one of the external tools that are available in Dataverse, pass its database id: +To show one of the external tools that are available in a Dataverse installation, pass its database id: .. code-block:: bash export TOOL_ID=1 curl http://localhost:8080/api/admin/externalTools/$TOOL_ID -Removing an External Tool From Dataverse -++++++++++++++++++++++++++++++++++++++++ +Removing an External Tool From a Dataverse Installation ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Assuming the external tool database id is "1", remove it with the following command: @@ -71,7 +71,7 @@ Assuming the external tool database id is "1", remove it with the following comm Testing External Tools ---------------------- -Once you have added an external tool to your installation of Dataverse, you will probably want to test it to make sure it is functioning properly. +Once you have added an external tool to your Dataverse installation, you will probably want to test it to make sure it is functioning properly. File Level vs. Dataset Level ++++++++++++++++++++++++++++ @@ -114,4 +114,4 @@ Writing Your Own External Tool If you plan to write a external tool, see the :doc:`/api/external-tools` section of the API Guide. -If you have an idea for an external tool, please let the Dataverse community know by posting about it on the dataverse-community mailing list: https://groups.google.com/forum/#!forum/dataverse-community +If you have an idea for an external tool, please let the Dataverse Project community know by posting about it on the dataverse-community mailing list: https://groups.google.com/forum/#!forum/dataverse-community diff --git a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst index fb8250c0650..87dad651c3e 100644 --- a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst +++ b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst @@ -1,7 +1,7 @@ Geoconnect and WorldMap ======================= -One of the optional components listed under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide is `Geoconnect `_, a piece of middleware that allows Dataverse users to create maps in `WorldMap `_ based on geospatial data stored in Dataverse. For more details on the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. +One of the optional components listed under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide is `Geoconnect `_, a piece of middleware that allows users to create maps in `WorldMap `_ based on geospatial data stored in a Dataverse installation. For more details on the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. .. contents:: |toctitle| :local: @@ -35,18 +35,18 @@ SQL commands to point a Dataverse installation at different Geoconnect servers: Removing Dead Explore Links --------------------------- -After a map has been created in WorldMap (assuming all the setup has been done), in Dataverse the file will display WorldMap as an explore option. In rare occasions, the map has been deleted on the WorldMap resulting in a dead link, a 404 page not found error. +After a map has been created in WorldMap (assuming all the setup has been done), in a Dataverse installation the file will display WorldMap as an explore option. In rare occasions, the map has been deleted on the WorldMap resulting in a dead link, a 404 page not found error. -Functionality has been added on the Dataverse side to iterate through all the maps Dataverse knows about (stored in the ``maplayermetadata`` database table) and to check for the existence of each map in WorldMap. The status code returned from WorldMap (200, 404, etc.) is recorded in Dataverse along with a timestamp of when the check was performed. To perform this check, you can execute the following ``curl`` command: +Functionality has been added on the Dataverse Platform side to iterate through all the maps the Dataverse installation knows about (stored in the ``maplayermetadata`` database table) and to check for the existence of each map in WorldMap. The status code returned from WorldMap (200, 404, etc.) is recorded in the Dataverse installation along with a timestamp of when the check was performed. To perform this check, you can execute the following ``curl`` command: ``curl -X POST http://localhost:8080/api/admin/geoconnect/mapLayerMetadatas/check`` The output above will contain the ``layerLink`` being checked as well as the HTTP response status code (200, 404, etc.) in the ``lastVerifiedStatus`` field. 200 means OK and 404 means not found. 500 might indicate that the map is only temporarily unavailable. The ``lastVerifiedStatus`` and ``lastVerifiedTime`` will be persisted to the ``maplayermetadata`` database table. -Armed with this information about WorldMap returning a 404 for a map, you may want to delete any record of the map on the Dataverse side removing WorldMap as an explore option (and so that thumbnail files are cleaned up). To accomplish this, use the following ``curl`` command, substituting the id of the file: +Armed with this information about WorldMap returning a 404 for a map, you may want to delete any record of the map on the Dataverse installation side removing WorldMap as an explore option (and so that thumbnail files are cleaned up). To accomplish this, use the following ``curl`` command, substituting the id of the file: ``curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://localhost:8080/api/files/{file_id}/map`` -End users can also run the ``DELETE`` command above (if they have permission to edit the dataset) but it's more likely that the sysadmin reading this guide will run it for them. It is recommended that you add the "check" command above to cron so that Dataverse periodically checks on the status of maps in WorldMap. In addition to the "check all maps" command above, you can also check an individual map with the following ``curl`` command, substituting the id of the row from the ``maplayermetadata`` table: +End users can also run the ``DELETE`` command above (if they have permission to edit the dataset) but it's more likely that the sysadmin reading this guide will run it for them. It is recommended that you add the "check" command above to cron so that the Dataverse installation periodically checks on the status of maps in WorldMap. In addition to the "check all maps" command above, you can also check an individual map with the following ``curl`` command, substituting the id of the row from the ``maplayermetadata`` table: ``curl -X POST http://localhost:8080/api/admin/geoconnect/mapLayerMetadatas/check/{id}`` From 49e9f8e24bb843506c356c7a7fefd3e296a9e44b Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 19:40:19 -0500 Subject: [PATCH 012/274] Update harvestserver.rst --- .../source/admin/harvestserver.rst | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index f1436926ea2..0936ba4bd36 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -4,12 +4,11 @@ Managing Harvesting Server and Sets .. contents:: |toctitle| :local: -Your Dataverse as an OAI server +Your Dataverse Installation as an OAI server ------------------------------- -As a harvesting *server*, your Dataverse can make some of the local -dataset metadata available to remote harvesting clients. These can be -other Dataverse instances, or any other clients that support OAI-PMH +As a harvesting *server*, your Dataverse installation can make some of the local dataset metadata available to remote harvesting clients. These can be +other Dataverse installations, or any other clients that support OAI-PMH harvesting protocol. Note that the terms "Harvesting Server" and "OAI Server" are being used interchangeably throughout this guide and in the inline help text. @@ -18,7 +17,7 @@ If you want to learn more about OAI-PMH, you could take a look at `DataCite OAI-PMH guide `_ or the `OAI-PMH protocol definition `_. -You might consider adding your OAI-enabled production instance of Dataverse to +You might consider adding your OAI-enabled Dataverse installation to `this shared list `_ of such instances. @@ -27,7 +26,7 @@ The email portion of :ref:`systemEmail` will be visible via OAI-PMH (from the "I How does it work? ----------------- -Only the published, unrestricted datasets in your Dataverse can +Only the published, unrestricted datasets in your Dataverse installation can be made harvestable. Remote clients normally keep their records in sync through scheduled incremental updates, daily or weekly, thus minimizing the load on your server. Note that it is only the metadata @@ -36,9 +35,9 @@ download the data files associated with the harvested datasets. Harvesting server can be enabled or disabled on the "Harvesting Server" page accessible via the :doc:`dashboard`. Harvesting server is by -default disabled on a brand new, "out of the box" Dataverse. +default disabled on a brand new, "out of the box" Dataverse installation. -The OAI-PMH endpoint can be accessed at ``http(s):///oai``. +The OAI-PMH endpoint can be accessed at ``http(s):///oai``. If you want other services to harvest your repository, point them to this URL. *Example URL for 'Identify' verb*: `demo.dataverse.org OAI `_ @@ -50,8 +49,7 @@ that will be available to remote harvesters as *OAI Sets*. Once again, the terms "OAI Set" and "Harvesting Set" are used interchangeably. Sets are defined by search queries. Any such query that finds any number of published, local (non-harvested) datasets can -be used to create an OAI set. Sets can overlap local dataverses, and -can include as few or as many of your local datasets as you wish. A +be used to create an OAI set. Sets can overlap local Dataverse collections, and can include as few or as many of your local datasets as you wish. A good way to master the Dataverse search query language is to experiment with the Advanced Search page. We also recommend that you consult the :doc:`/api/search` section of the API Guide. From 337998d0e5154dad662f64229acc6cde3669ee02 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 23:01:20 -0500 Subject: [PATCH 013/274] Update harvestserver.rst --- .../source/admin/harvestserver.rst | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index 0936ba4bd36..3550599a67c 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -50,7 +50,7 @@ the terms "OAI Set" and "Harvesting Set" are used interchangeably. Sets are defined by search queries. Any such query that finds any number of published, local (non-harvested) datasets can be used to create an OAI set. Sets can overlap local Dataverse collections, and can include as few or as many of your local datasets as you wish. A -good way to master the Dataverse search query language is to +good way to master the Dataverse Platform search query language is to experiment with the Advanced Search page. We also recommend that you consult the :doc:`/api/search` section of the API Guide. @@ -61,7 +61,7 @@ can go back and try to re-define the query. Some useful examples of search queries to define OAI sets: -- A good way to create a set that would include all your local, published datasets is to do so by the Unique Identifier authority registered to your Dataverse, for example: +- A good way to create a set that would include all your local, published datasets is to do so by the Unique Identifier authority registered to your Dataverse installation, for example: ``dsPersistentId:"doi:1234/"`` @@ -69,11 +69,11 @@ Some useful examples of search queries to define OAI sets: Note also that the search terms limiting the results to published and local datasets **are added to the query automatically**, so you don't need to worry about that. -- A query to create a set to include the datasets from a specific local dataverse: +- A query to create a set to include the datasets from a specific Dataverse collection: ``parentId:NNN`` - where NNN is the database id of the dataverse object (consult the Dataverse table of the SQL database used by the application to verify the database id). + where NNN is the database id of the Dataverse collection object (consult the Dataverse table of the SQL database used by the application to verify the database id). - A query to find all the dataset by a certain author: @@ -95,7 +95,7 @@ Some useful examples of search queries to define OAI sets: ``dsPersistentId:10.5000/XXYYZZ`` - For all dataverses with subjects of Astronomy and Astrophysics or Earth and Environmental Sciences: + For all Dataverse collections with subjects of Astronomy and Astrophysics or Earth and Environmental Sciences: ``(dvSubject:"Astronomy and Astrophysics" OR dvSubject:"Earth and Environmental Sciences")`` @@ -106,14 +106,14 @@ Some useful examples of search queries to define OAI sets: Important: New SOLR schema required! ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to be able to define OAI sets, your SOLR server must be upgraded with the search schema that came with the Dataverse release 4.5 (or later), and all your local datasets must be re-indexed, once the new schema is installed. +In order to be able to define OAI sets, your SOLR server must be upgraded with the search schema that came with release 4.5 (or later), and all your local datasets must be re-indexed, once the new schema is installed. OAI Set updates --------------- Every time a new harvesting set is created, or changes are made to an existing set, the contents of the set are automatically updated - the -Dataverse application will find the datasets defined by the query, and +Dataverse installation will find the datasets defined by the query, and attempt to run the metadata export on the ones that haven't been exported yet. Only the datasets for which the export has completed successfully, and the results cached on the filesystem are included in @@ -134,9 +134,7 @@ the sets every time any changes to the metadata are made. The OAI set will however be updated automatically by a scheduled metadata export job that runs every night (at 2AM, by default). This export timer is created and activated automatically every time the application is deployed -or restarted. Once again, this is new in Dataverse 4, and unlike DVN -v3, where export jobs had to be scheduled and activated by the admin -user. See the :doc:`/admin/metadataexport` section of the Admin Guide, for more information on the automated metadata exports. +or restarted. See the :doc:`/admin/metadataexport` section of the Admin Guide, for more information on the automated metadata exports. It is still possible however to make changes like this be immediately reflected in the OAI server, by going to the *Harvesting Server* page From d72a25c606049ad6c2baada8c194f9babc538fb8 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 5 Jan 2021 23:02:51 -0500 Subject: [PATCH 014/274] Update harvestclients.rst --- doc/sphinx-guides/source/admin/harvestclients.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 26af8eee32c..a3de6690b9f 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -4,10 +4,10 @@ Managing Harvesting Clients .. contents:: |toctitle| :local: -Your Dataverse as a Metadata Harvester --------------------------------------- +Your Dataverse Installation as a Metadata Harvester +--------------------------------------------------- -Harvesting is a process of exchanging metadata with other repositories. As a harvesting *client*, your Dataverse can gather metadata records from remote sources. These can be other Dataverse instances or other archives that support OAI-PMH, the standard harvesting protocol. Harvested metadata records will be indexed and made searchable by your users. Clicking on a harvested dataset in the search results takes the user to the original repository. Harvested datasets cannot be edited in your Dataverse installation. +Harvesting is a process of exchanging metadata with other repositories. As a harvesting *client*, your Dataverse installation can gather metadata records from remote sources. These can be other Dataverse installations or other archives that support OAI-PMH, the standard harvesting protocol. Harvested metadata records will be indexed and made searchable by your users. Clicking on a harvested dataset in the search results takes the user to the original repository. Harvested datasets cannot be edited in your Dataverse installation. Harvested records can be kept in sync with the original repository through scheduled incremental updates, daily or weekly. Alternatively, harvests can be run on demand, by the Admin. @@ -26,4 +26,4 @@ What if a Run Fails? Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara5/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. -Note that you'll want to run a minimum Dataverse version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. +Note that you'll want to run a minimum Dataverse Platform version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. From 5a1b4a5400c4df11c4b45cc187e9c19ebd540d00 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:04:05 -0500 Subject: [PATCH 015/274] Update integrations.rst --- .../source/admin/integrations.rst | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst index 9e43081bb45..65917d24891 100644 --- a/doc/sphinx-guides/source/admin/integrations.rst +++ b/doc/sphinx-guides/source/admin/integrations.rst @@ -1,7 +1,7 @@ Integrations ============ -Now that you've installed Dataverse, you might want to set up some integrations with other systems. Many of these integrations are open source and are cross listed in the :doc:`/api/apps` section of the API Guide. +Now that you've installed a Dataverse installation, you might want to set up some integrations with other systems. Many of these integrations are open source and are cross listed in the :doc:`/api/apps` section of the API Guide. .. contents:: Contents: :local: @@ -9,37 +9,37 @@ Now that you've installed Dataverse, you might want to set up some integrations Getting Data In --------------- -A variety of integrations are oriented toward making it easier for your researchers to deposit data into your installation of Dataverse. +A variety of integrations are oriented toward making it easier for your researchers to deposit data into your Dataverse installation. Dropbox +++++++ -If your researchers have data on Dropbox, you can make it easier for them to get it into Dataverse by setting the :ref:`dataverse.dropbox.key` JVM option described in the :doc:`/installation/config` section of the Installation Guide. +If your researchers have data on Dropbox, you can make it easier for them to get it into your Dataverse installation by setting the :ref:`dataverse.dropbox.key` JVM option described in the :doc:`/installation/config` section of the Installation Guide. Open Science Framework (OSF) ++++++++++++++++++++++++++++ The Center for Open Science's Open Science Framework (OSF) is an open source software project that facilitates open collaboration in science research across the lifespan of a scientific project. -For instructions on depositing data from OSF to your installation of Dataverse, your researchers can visit https://help.osf.io/hc/en-us/articles/360019737314-Connect-Dataverse-to-a-Project +For instructions on depositing data from OSF to your Dataverse installation, your researchers can visit https://help.osf.io/hc/en-us/articles/360019737314-Connect-Dataverse-to-a-Project RSpace ++++++ RSpace is an affordable and secure enterprise grade electronic lab notebook (ELN) for researchers to capture and organize data. -For instructions on depositing data from RSpace to your installation of Dataverse, your researchers can visit https://www.researchspace.com/help-and-support-resources/dataverse-integration/ +For instructions on depositing data from RSpace to your Dataverse installation, your researchers can visit https://www.researchspace.com/help-and-support-resources/dataverse-integration/ Open Journal Systems (OJS) ++++++++++++++++++++++++++ Open Journal Systems (OJS) is a journal management and publishing system that has been developed by the Public Knowledge Project to expand and improve access to research. -The OJS Dataverse Plugin adds data sharing and preservation to the OJS publication process. +The OJS Dataverse Project Plugin adds data sharing and preservation to the OJS publication process. As of this writing only OJS 2.x is supported and instructions for getting started can be found at https://github.com/pkp/ojs/tree/ojs-stable-2_4_8/plugins/generic/dataverse -If you are interested in OJS 3.x supporting deposit from Dataverse, please leave a comment on https://github.com/pkp/pkp-lib/issues/1822 +If you are interested in OJS 3.x supporting deposit to Dataverse installations, please leave a comment on https://github.com/pkp/pkp-lib/issues/1822 Renku +++++ @@ -47,7 +47,7 @@ Renku Renku is a platform that enables collaborative, reproducible and reusable (data)science. It allows researchers to automatically record the provenance of their research results and retain links to imported and exported data. Users -can organize their data in "Datasets", which can be exported to Dataverse via +can organize their data in "Datasets", which can be exported to a Dataverse installation via the command-line interface (CLI). Renku dataset documentation: https://renku-python.readthedocs.io/en/latest/commands.html#module-renku.cli.dataset @@ -63,9 +63,9 @@ Embedding Data on Websites OpenScholar +++++++++++ -`OpenScholar `_ is oriented toward hosting websites for academic institutions and offers `Dataverse Widgets `_ that can be added to web pages. See also: +`OpenScholar `_ is oriented toward hosting websites for academic institutions and offers `Dataverse Project Widgets `_ that can be added to web pages. See also: -- :ref:`openscholar-dataverse-level` (dataverse level) +- :ref:`openscholar-dataverse-level` (Dataverse collection level) - :ref:`openscholar-dataset-level` (dataset level) Analysis and Computation @@ -101,51 +101,51 @@ Whole Tale ++++++++++ `Whole Tale `_ enables researchers to analyze data using popular tools including Jupyter and RStudio with the ultimate goal of supporting publishing of reproducible research packages. Users can -`import data from Dataverse +`import data from a Dataverse installation `_ via identifier (e.g., DOI, URI, etc) or through the External Tools integration. For installation instructions, see the :doc:`external-tools` section or the `Integration `_ section of the Whole Tale User Guide. Binder ++++++ -Researchers can launch Jupyter Notebooks, RStudio, and other computational environments by entering the DOI of a Dataverse dataset on https://mybinder.org +Researchers can launch Jupyter Notebooks, RStudio, and other computational environments by entering the DOI of a dataset in a Dataverse installation on https://mybinder.org -Institutions can self host BinderHub. Dataverse is one of the supported `repository providers `_. +Institutions can self host BinderHub. The Dataverse Project is one of the supported `repository providers `_. Renku +++++ -Researchers can import Dataverse datasets into their Renku projects via the -command-line interface (CLI) by using the Dataverse DOI. See the `renku Dataset +Researchers can import datasets from a Dataverse installation into their Renku projects via the +command-line interface (CLI) by using the dataset's DOI. See the `renku Dataset documentation `_ -for details. Currently Dataverse ``>=4.8.x`` is required for the import to work. If you need -support for an earlier version of Dataverse, please get in touch with the Renku team at +for details. Currently Dataverse Platform ``>=4.8.x`` is required for the import to work. If you need +support for an earlier version of the Dataverse Platform, please get in touch with the Renku team at `Discourse `_ or `GitHub `_. Avgidea Data Search +++++++++++++++++++ -Researchers can use a Google Sheets add-on to search for Dataverse CSV data and then import that data into a sheet. See `Avgidea Data Search `_ for details. +Researchers can use a Google Sheets add-on to search for Dataverse installation's CSV data and then import that data into a sheet. See `Avgidea Data Search `_ for details. Discoverability --------------- -Integration with `DataCite `_ is built in to Dataverse. When datasets are published, metadata is sent to DataCite. You can futher increase the discoverability of your datasets by setting up additional integrations. +Integration with `DataCite `_ is built in to the Dataverse Platform. When datasets are published, metadata is sent to DataCite. You can further increase the discoverability of your datasets by setting up additional integrations. OAI-PMH (Harvesting) ++++++++++++++++++++ -Dataverse supports a protocol called OAI-PMH that facilitates harvesting datasets from one system into another. For details on harvesting, see the :doc:`harvestserver` section. +The Dataverse Platform supports a protocol called OAI-PMH that facilitates harvesting datasets from one system into another. For details on harvesting, see the :doc:`harvestserver` section. SHARE +++++ -`SHARE `_ is building a free, open, data set about research and scholarly activities across their life cycle. It's possible to add an installation of Dataverse as one of the `sources `_ they include if you contact the SHARE team. +`SHARE `_ is building a free, open, data set about research and scholarly activities across their life cycle. It's possible to add a Dataverse installation as one of the `sources `_ they include if you contact the SHARE team. Geodisy +++++++ -`Geodisy `_ will take your Dataverse Installation’s data, search for geospatial metadata and files, and copy them to a new system that allows for visual searching. Your original data and search methods are untouched; you have the benefit of both. For more information, please refer to `Geodisy's GitHub Repository. `_ +`Geodisy `_ will take your Dataverse installation’s data, search for geospatial metadata and files, and copy them to a new system that allows for visual searching. Your original data and search methods are untouched; you have the benefit of both. For more information, please refer to `Geodisy's GitHub Repository. `_ Research Data Preservation -------------------------- @@ -155,21 +155,21 @@ Archivematica `Archivematica `_ is an integrated suite of open-source tools for processing digital objects for long-term preservation, developed and maintained by Artefactual Systems Inc. Its configurable workflow is designed to produce system-independent, standards-based Archival Information Packages (AIPs) suitable for long-term storage and management. -Sponsored by the `Ontario Council of University Libraries (OCUL) `_, this technical integration enables users of Archivematica to select datasets from connected Dataverse instances and process them for long-term access and digital preservation. For more information and list of known issues, please refer to Artefactual's `release notes `_, `integration documentation `_, and the `project wiki `_. +Sponsored by the `Ontario Council of University Libraries (OCUL) `_, this technical integration enables users of Archivematica to select datasets from connected Dataverse installations and process them for long-term access and digital preservation. For more information and list of known issues, please refer to Artefactual's `release notes `_, `integration documentation `_, and the `project wiki `_. DuraCloud/Chronopolis +++++++++++++++++++++ -Dataverse can be configured to submit a copy of published Datasets, packaged as `Research Data Alliance conformant `_ zipped `BagIt `_ bags to the `Chronopolis `_ via `DuraCloud `_ +A Dataverse installation can be configured to submit a copy of published Datasets, packaged as `Research Data Alliance conformant `_ zipped `BagIt `_ bags to the `Chronopolis `_ via `DuraCloud `_ For details on how to configure this integration, look for "DuraCloud/Chronopolis" in the :doc:`/installation/config` section of the Installation Guide. Future Integrations ------------------- -The `Dataverse roadmap `_ is a good place to see integrations that the core Dataverse team is working on. +The `Dataverse Project Roadmap `_ is a good place to see integrations that the core Dataverse Project team is working on. -The `Community Dev `_ column of our project board is a good way to track integrations that are being worked on by the Dataverse community but many are not listed and if you have an idea for an integration, please ask on the `dataverse-community `_ mailing list if someone is already working on it. +The `Community Dev `_ column of our project board is a good way to track integrations that are being worked on by the Dataverse Community but many are not listed and if you have an idea for an integration, please ask on the `dataverse-community `_ mailing list if someone is already working on it. Many integrations take the form of "external tools". See the :doc:`external-tools` section for details. External tool makers should check out the :doc:`/api/external-tools` section of the API Guide. From 3eebc6c2a77223b99fd66143b0fb8296942addf0 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:05:32 -0500 Subject: [PATCH 016/274] proper use of Dataverse updates --- doc/sphinx-guides/source/admin/ip-groups.rst | 2 +- doc/sphinx-guides/source/admin/mail-groups.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/admin/ip-groups.rst b/doc/sphinx-guides/source/admin/ip-groups.rst index 642b19878f5..6d9f520cb4d 100644 --- a/doc/sphinx-guides/source/admin/ip-groups.rst +++ b/doc/sphinx-guides/source/admin/ip-groups.rst @@ -20,7 +20,7 @@ IP Groups must be expressed as ranges in IPv4 or IPv6 format. For illustrative p .. literalinclude:: ../_static/admin/ipGroupAll.json -Let's say you download the example above and edit it to give it a range used by your library, giving it a filename of ``ipGroup1.json`` and putting it in the ``/tmp`` directory. Next, load it into Dataverse using the following curl command: +Let's say you download the example above and edit it to give it a range used by your library, giving it a filename of ``ipGroup1.json`` and putting it in the ``/tmp`` directory. Next, load it into your Dataverse installation using the following curl command: ``curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/groups/ip --upload-file /tmp/ipGroup1.json`` diff --git a/doc/sphinx-guides/source/admin/mail-groups.rst b/doc/sphinx-guides/source/admin/mail-groups.rst index 3df529e4112..86fb4272dd3 100644 --- a/doc/sphinx-guides/source/admin/mail-groups.rst +++ b/doc/sphinx-guides/source/admin/mail-groups.rst @@ -66,7 +66,7 @@ Some hints: - Due to their excessive CPU usage, regular expressions should be used rarely. - Remember to properly escape "\" in your regular expression. Both Java and JSON are a bit picky about this. E.g. a character class "\d" would have to be escaped as "\\d". Plenty of tutorials on the web explain this in more detail. -- There is no way Dataverse can detect a wrong regular expression for you. Be sure to do extensive testing, as a misconfigured group could result in privilege escalation or an unexpected influx of support contacts. +- There is no way the Dataverse software can detect a wrong regular expression for you. Be sure to do extensive testing, as a misconfigured group could result in privilege escalation or an unexpected influx of support contacts. - Remember to enable the regular expression support for a group by adding ``"regex": true``! A short example for a group using regular expressions: From 3e2a620000634d7faa9cfa005c648156c7a68aed Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:13:18 -0500 Subject: [PATCH 017/274] Update make-data-count.rst --- .../source/admin/make-data-count.rst | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index a432a27fbe7..ac37d9eb11f 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -1,7 +1,7 @@ Make Data Count =============== -`Make Data Count`_ is a project to collect and standardize metrics on data use, especially views, downloads, and citations. Dataverse can integrate Make Data Count to collect and display usage metrics including counts of dataset views, file downloads, and dataset citations. +`Make Data Count`_ is a project to collect and standardize metrics on data use, especially views, downloads, and citations. The Dataverse Platform can integrate Make Data Count to collect and display usage metrics including counts of dataset views, file downloads, and dataset citations. .. contents:: Contents: :local: @@ -9,7 +9,7 @@ Make Data Count Introduction ------------ -`Make Data Count`_ is part of a broader Research Data Alliance (RDA) `Data Usage Metrics Working Group`_ which helped to produce a specification called the `COUNTER Code of Practice for Research Data`_ (`PDF`_, `HTML`_) that Dataverse makes every effort to comply with. The Code of Practice (CoP) is built on top of existing standards such as COUNTER and SUSHI that come out of the article publishing world. The Make Data Count project has emphasized that they would like feedback on the code of practice. You can keep up to date on the Make Data Count project by subscribing to their `newsletter`_. +`Make Data Count`_ is part of a broader Research Data Alliance (RDA) `Data Usage Metrics Working Group`_ which helped to produce a specification called the `COUNTER Code of Practice for Research Data`_ (`PDF`_, `HTML`_) that the Dataverse Platform makes every effort to comply with. The Code of Practice (CoP) is built on top of existing standards such as COUNTER and SUSHI that come out of the article publishing world. The Make Data Count project has emphasized that they would like feedback on the code of practice. You can keep up to date on the Make Data Count project by subscribing to their `newsletter`_. Architecture ------------ @@ -19,17 +19,17 @@ Dataverse installations who would like support for Make Data Count must install .. _Counter Processor: https://github.com/CDLUC3/counter-processor .. _DASH: https://cdluc3.github.io/dash/ -The diagram below shows how Counter Processor interacts with Dataverse and the DataCite hub, once configured. Installations of Dataverse using Handles rather than DOIs should note the limitations in the next section of this page. +The diagram below shows how Counter Processor interacts with your Dataverse installation and the DataCite hub, once configured. Dataverse installations using Handles rather than DOIs should note the limitations in the next section of this page. |makedatacount_components| The most important takeaways from the diagram are: -- Once enabled, Dataverse will log activity (views and downloads) to a specialized date-stamped file. -- You should run Counter Processor once a day to create reports in SUSHI (JSON) format that are saved to disk for Dataverse to process and that are sent to the DataCite hub. -- You should set up a cron job to have Dataverse process the daily SUSHI reports, updating the Dataverse database with the latest metrics. -- You should set up a cron job to have Dataverse pull the latest list of citations for each dataset on a periodic basis, perhaps weekly or daily. These citations come from Crossref via the DataCite hub. -- APIs are available in Dataverse to retrieve Make Data Count metrics: views, downloads, and citations. +- Once enabled, your Dataverse installation will log activity (views and downloads) to a specialized date-stamped file. +- You should run Counter Processor once a day to create reports in SUSHI (JSON) format that are saved to disk for your Dataverse installation to process and that are sent to the DataCite hub. +- You should set up a cron job to have your Dataverse installation process the daily SUSHI reports, updating the Dataverse installation database with the latest metrics. +- You should set up a cron job to have your Dataverse installation pull the latest list of citations for each dataset on a periodic basis, perhaps weekly or daily. These citations come from Crossref via the DataCite hub. +- APIs are available in the Dataverse Platform to retrieve Make Data Count metrics: views, downloads, and citations. Limitations for Dataverse Installations Using Handles Rather Than DOIs @@ -49,19 +49,19 @@ Data repositories using Handles and other identifiers are not supported by Make | | MDC citations | | +---------------------+---------------+------------------+ -This being said, the Dataverse usage logging can still generate logs and process those logs with Counter Processor to create json that details usage on a dataset level. Dataverse can ingest this locally generated json. +This being said, the Dataverse Platform usage logging can still generate logs and process those logs with Counter Processor to create json that details usage on a dataset level. Dataverse installations can ingest this locally generated json. When editing the ``counter-processor-config.yaml`` file mentioned below, make sure that the ``upload_to_hub`` boolean is set to ``False``. -Configuring Dataverse for Make Data Count Views and Downloads -------------------------------------------------------------- +Configuring Your Dataverse Installation for Make Data Count Views and Downloads +------------------------------------------------------------------------------- If you haven't already, follow the steps for installing Counter Processor in the :doc:`/installation/prerequisites` section of the Installation Guide. Enable Logging for Make Data Count ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To make Dataverse log dataset usage (views and downloads) for Make Data Count, you must set the ``:MDCLogPath`` database setting. See :ref:`:MDCLogPath` for details. +To make your Dataverse installation log dataset usage (views and downloads) for Make Data Count, you must set the ``:MDCLogPath`` database setting. See :ref:`:MDCLogPath` for details. If you wish to start logging in advance of setting up other components, or wish to log without display MDC metrics for any other reason, you can set the optional ``:DisplayMDCMetrics`` database setting to false. See :ref:`:DisplayMDCMetrics` for details. @@ -70,7 +70,7 @@ After you have your first day of logs, you can process them the next day. Enable or Disable Display of Make Data Count Metrics ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -By default, when MDC logging is enabled (when ``:MDCLogPath`` is set), Dataverse will display MDC metrics instead of it's internal (legacy) metrics. You can avoid this (e.g. to collect MDC metrics for some period of time before starting to display them) by setting ``:DisplayMDCMetrics`` to false. +By default, when MDC logging is enabled (when ``:MDCLogPath`` is set), your Dataverse installation will display MDC metrics instead of it's internal (legacy) metrics. You can avoid this (e.g. to collect MDC metrics for some period of time before starting to display them) by setting ``:DisplayMDCMetrics`` to false. The following discussion assumes ``:MDCLogPath`` has been set to ``/usr/local/payara5/glassfish/domains/domain1/logs/mdc`` @@ -94,7 +94,7 @@ Configure Counter Processor Populate Views and Downloads for the First Time ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Soon we will be setting up a cron job to run nightly but we start with a single successful configuration and manual run of Counter Processor and calls to Dataverse APIs. (The scripts discussed in the next section automate the steps described here, including creating empty log files if you're starting mid-month.) +Soon we will be setting up a cron job to run nightly but we start with a single successful configuration and manual run of Counter Processor and calls to your Dataverse installation's APIs. (The scripts discussed in the next section automate the steps described here, including creating empty log files if you're starting mid-month.) * Change to the directory where you installed Counter Processor. @@ -122,14 +122,14 @@ Soon we will be setting up a cron job to run nightly but we start with a single * Verify that views and downloads are available via API. - * Now that views and downloads have been recorded in the Dataverse database, you should make sure you can retrieve them from a dataset or two. Use the :ref:`Dataset Metrics ` endpoints in the :doc:`/api/native-api` section of the API Guide. + * Now that views and downloads have been recorded in the Dataverse installation's database, you should make sure you can retrieve them from a dataset or two. Use the :ref:`Dataset Metrics ` endpoints in the :doc:`/api/native-api` section of the API Guide. Populate Views and Downloads Nightly ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Running ``main.py`` to create the SUSHI JSON file and the subsequent calling of the Dataverse API to process it should be added as a cron job. +Running ``main.py`` to create the SUSHI JSON file and the subsequent calling of the Dataverse Platform API to process it should be added as a cron job. -Dataverse provides example scripts that run the steps to process new accesses and uploads and update Dataverse's database (`counter_daily.sh`) and to retrieve citations for all Datasets from DataCite (`counter_weekly.sh`). These scripts should be configured for your environment and can be run manually or as cron jobs. +The Dataverse Platform provides example scripts that run the steps to process new accesses and uploads and update your Dataverse installation's database (`counter_daily.sh`) and to retrieve citations for all Datasets from DataCite (`counter_weekly.sh`). These scripts should be configured for your environment and can be run manually or as cron jobs. Sending Usage Metrics to the DataCite Hub ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -141,12 +141,12 @@ Once you are satisfied with your testing, you should contact support@datacite.or .. _MDC-updateCitationsForDataset: -Configuring Dataverse for Make Data Count Citations ---------------------------------------------------- +Configuring Your Dataverse Installation for Make Data Count Citations +--------------------------------------------------------------------- -Please note: as explained in the note above about limitations, this feature is not available to installations of Dataverse that use Handles. +Please note: as explained in the note above about limitations, this feature is not available to Dataverse installations that use Handles. -To configure Dataverse to pull citations from the test vs. production DataCite server see :ref:`doi.dataciterestapiurlstring` in the Installation Guide. +To configure your Dataverse installation to pull citations from the test vs. production DataCite server see :ref:`doi.dataciterestapiurlstring` in the Installation Guide. Please note that in the curl example, Bash environment variables are used with the idea that you can set a few environment variables and copy and paste the examples as is. For example, "$DOI" could become "doi:10.5072/FK2/BL2IBM" by issuing the following export command from Bash: @@ -160,11 +160,11 @@ On some periodic basis (perhaps weekly) you should call the following curl comma ``curl -X POST "http://localhost:8080/api/admin/makeDataCount/:persistentId/updateCitationsForDataset?persistentId=$DOI"`` -Citations will be retrieved for each published dataset and recorded in the Dataverse database. +Citations will be retrieved for each published dataset and recorded in the your Dataverse installation's database. -For how to get the citations out of Dataverse, see "Retrieving Citations for a Dataset" under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. +For how to get the citations out of your Dataverse installation, see "Retrieving Citations for a Dataset" under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. -Please note that while Dataverse has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref. +Please note that while the Dataverse Platform has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref. Retrieving Make Data Count Metrics from the DataCite Hub -------------------------------------------------------- @@ -177,10 +177,10 @@ The following metrics can be downloaded directly from the DataCite hub (see http - Downloads for a Dataset - Citations for a Dataset (via Crossref) -Retrieving Make Data Count Metrics from Dataverse -------------------------------------------------- +Retrieving Make Data Count Metrics from a Dataverse Installation +---------------------------------------------------------------- -The Dataverse API endpoints for retrieving Make Data Count metrics are described below under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. +The Dataverse Platform API endpoints for retrieving Make Data Count metrics are described below under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. Please note that it is also possible to retrieve metrics from the DataCite hub itself via https://api.datacite.org From ea3fc02190ca88730fe7dd00b055a50e267c95da Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:32:23 -0500 Subject: [PATCH 018/274] updating tables is especially fun. --- .../source/admin/metadatacustomization.rst | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index a7c7e48cdd6..a2efa95deed 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -1,7 +1,7 @@ Metadata Customization ====================== -Dataverse has a flexible data-driven metadata system powered by "metadata blocks" that are listed in the :doc:`/user/appendix` section of the User Guide. In this section we explain the customization options. +The Dataverse software has a flexible data-driven metadata system powered by "metadata blocks" that are listed in the :doc:`/user/appendix` section of the User Guide. In this section we explain the customization options. .. contents:: |toctitle| :local: @@ -9,7 +9,7 @@ Dataverse has a flexible data-driven metadata system powered by "metadata blocks Introduction ------------ -Before you embark on customizing metadata in Dataverse you should make sure you are aware of the modest amount of customization that is available with the Dataverse web interface. It's possible to hide fields and make field required by clicking "Edit" at the dataverse level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Management page in the User Guide. +Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make field required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. Much more customization of metadata is possible, but this is an advanced topic so feedback on what is written below is very welcome. The possibilities for customization include: @@ -23,11 +23,11 @@ Much more customization of metadata is possible, but this is an advanced topic s - Changing how saved metadata values are displayed in the UI -Generally speaking it is safer to create your own custom metadata block rather than editing metadata blocks that ship with Dataverse, because changes to these blocks may be made in future releases of Dataverse. If you'd like to make improvements to any of the metadata blocks shipped with Dataverse, please open an issue at https://github.com/IQSS/dataverse/issues so it can be discussed before a pull request is made. Please note that the metadata blocks shipped with Dataverse are based on standards (e.g. DDI for social science) and you can learn more about these standards in the :doc:`/user/appendix` section of the User Guide. If you have developed your own custom metadata block that you think may be of interest to the Dataverse community, please create an issue and consider making a pull request as described in the :doc:`/developers/version-control` section of the Developer Guide. +Generally speaking it is safer to create your own custom metadata block rather than editing metadata blocks that ship with the Dataverse software, because changes to these blocks may be made in future releases. If you'd like to make improvements to any of the metadata blocks shipped with the Dataverse software, please open an issue at https://github.com/IQSS/dataverse/issues so it can be discussed before a pull request is made. Please note that the metadata blocks shipped with the Dataverse software are based on standards (e.g. DDI for social science) and you can learn more about these standards in the :doc:`/user/appendix` section of the User Guide. If you have developed your own custom metadata block that you think may be of interest to the Dataverse community, please create an issue and consider making a pull request as described in the :doc:`/developers/version-control` section of the Developer Guide. -In Dataverse 4, custom metadata are no longer defined as individual +In current versions of the Dataverse software, custom metadata are no longer defined as individual fields, as they were in Dataverse Network (DVN) 3.x, but in metadata blocks. -Dataverse 4 ships with a citation metadata block, which includes +The Dataverse software now ships with a citation metadata block, which includes mandatory fields needed for assigning persistent IDs to datasets, and domain specific metadata blocks. For a complete list, see the :doc:`/user/appendix` section of the User Guide. @@ -37,9 +37,9 @@ tab-separated value (TSV). [1]_\ :sup:`,`\ [2]_ While it is technically possible to define more than one metadata block in a TSV file, it is good organizational practice to define only one in each file. -The metadata block TSVs shipped with Dataverse are in `/tree/develop/scripts/api/data/metadatablocks -`__ and the corresponding ResourceBundle property files `/tree/develop/src/main/java `__ of the Dataverse GitHub repo. Human-readable copies are available in `this Google Sheets -document `__ but they tend to get out of sync with the TSV files, which should be considered authoritative. The Dataverse installation process operates on the TSVs, not the Google spreadsheet. +The metadata block TSVs shipped with the Dataverse software are in `/tree/develop/scripts/api/data/metadatablocks +`__ and the corresponding ResourceBundle property files `/tree/develop/src/main/java `__ of the Dataverse Software GitHub repo. Human-readable copies are available in `this Google Sheets +document `__ but they tend to get out of sync with the TSV files, which should be considered authoritative. The Dataverse Software installation process operates on the TSVs, not the Google spreadsheet. About the metadata block TSV ---------------------------- @@ -53,7 +53,7 @@ the metadata block TSV. - Cardinality: - - 0 or more per Dataverse + - 0 or more per Dataverse installation - 1 per Metadata Block definition @@ -104,11 +104,12 @@ Each of the three main sections own sets of properties: | dataverseAlias | If specified, this | Free text. For an | | | metadata block will | example, see | | | be available only to | custom_hbgdki.tsv. | -| | the dataverse | | +| | the Dataverse | +| | collection | | | designated here by | | | | its alias and to | | | | children of that | | -| | dataverse. | | +| | Dataverse collection. | | +-----------------------+-----------------------+-----------------------+ | displayName | Acts as a brief label | Should be relatively | | | for display related | brief. The limit is | @@ -254,7 +255,8 @@ Each of the three main sections own sets of properties: | | "Browse/Search | | | | Facets" when you edit | | | | "General Information" | | -| | for a dataverse. | | +| | for a Dataverse | | +| |collection. | | | | Setting this value to | | | | TRUE generally makes | | | | sense for enumerated | | @@ -407,7 +409,8 @@ FieldType definitions | | be entered into this field. | +-----------------------------------+-----------------------------------+ | textbox | Any text may be entered. For | -| | input, Dataverse presents a | +| | input, the Dataverse software | +| | presents a | | | multi-line area that accepts | | | newlines. While any HTML is | | | permitted, only a subset of HTML | @@ -503,12 +506,12 @@ These are common ways to use the displayFormat to control how values are display Metadata Block Setup -------------------- -Now that you understand the TSV format used for metadata blocks, the next step is to attempt to make improvements to existing metadata blocks or create entirely new metadata blocks. For either task, you should have a Dataverse environment set up for testing where you can drop the database frequently while you make edits to TSV files. Once you have tested your TSV files, you should consider making a pull request to contribute your improvement back to the community. +Now that you understand the TSV format used for metadata blocks, the next step is to attempt to make improvements to existing metadata blocks or create entirely new metadata blocks. For either task, you should have a Dataverse Software development environment set up for testing where you can drop the database frequently while you make edits to TSV files. Once you have tested your TSV files, you should consider making a pull request to contribute your improvement back to the community. Exploring Metadata Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~ -In addition to studying the TSV files themselves you might find the following highly experimental and subject-to-change API endpoints useful to understand the metadata blocks that have already been loaded into your installation of Dataverse: +In addition to studying the TSV files themselves you might find the following highly experimental and subject-to-change API endpoints useful to understand the metadata blocks that have already been loaded into your Dataverse installation: You can get a dump of metadata fields (yes, the output is odd, please open a issue) like this: @@ -533,7 +536,7 @@ To get a clean environment in Vagrant, you'll be running ``vagrant destroy``. In Editing TSV files ~~~~~~~~~~~~~~~~~ -Early in Dataverse 4 development metadata blocks were edited in the Google spreadsheet mentioned above and then exported in TSV format. This worked fine when there was only one person editing the Google spreadsheet but now that contributions are coming in from all over, the TSV files are edited directly. We are somewhat painfully aware that another format such as XML might make more sense these days. Please see https://github.com/IQSS/dataverse/issues/4451 for a discussion of non-TSV formats. +Early in Dataverse Software version 4 development, metadata blocks were edited in the Google spreadsheet mentioned above and then exported in TSV format. This worked fine when there was only one person editing the Google spreadsheet but now that contributions are coming in from all over, the TSV files are edited directly. We are somewhat painfully aware that another format such as XML might make more sense these days. Please see https://github.com/IQSS/dataverse/issues/4451 for a discussion of non-TSV formats. Please note that metadata fields share a common namespace so they must be unique. The following curl command will print list of metadata fields already available in the system: @@ -541,21 +544,21 @@ Please note that metadata fields share a common namespace so they must be unique We'll use this command again below to update the Solr schema to accomodate metadata fields we've added. -Loading TSV files into Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Loading TSV files into a Dataverse Installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A number of TSV files are loaded into Dataverse on every new installation, becoming the metadata blocks you see in the UI. For the list of metadata blocks that are included with Dataverse out of the box, see the :doc:`/user/appendix` section of the User Guide. +A number of TSV files are loaded into a newly-installed Dataverse installation, becoming the metadata blocks you see in the UI. For the list of metadata blocks that are included with the Dataverse Software out of the box, see the :doc:`/user/appendix` section of the User Guide. Along with TSV file, there are corresponding ResourceBundle property files with key=value pair `here `__. To add other language files, see the :doc:`/installation/config` for dataverse.lang.directory JVM Options section, and add a file, for example: "citation_lang.properties" to the path you specified for the ``dataverse.lang.directory`` JVM option, and then restart the app server. -If you are improving an existing metadata block, the Dataverse installation process will load the TSV for you, assuming you edited the TSV file in place. The TSV file for the Citation metadata block, for example, can be found at ``scripts/api/data/metadatablocks/citation.tsv``. +If you are improving an existing metadata block, the Dataverse Software installation process will load the TSV for you, assuming you edited the TSV file in place. The TSV file for the Citation metadata block, for example, can be found at ``scripts/api/data/metadatablocks/citation.tsv``. If any of the below mentioned property values are changed, corresponsing ResourceBundle property file has to be edited and stored under ``dataverse.lang.directory`` location - name, displayName property under #metadataBlock - name, title, description, watermark properties under #datasetfield - DatasetField, Value property under #controlledVocabulary -If you are creating a new custom metadata block (hopefully with the idea of contributing it back to the community if you feel like it would provide value to others), the Dataverse installation process won't know about your new TSV file so you must load it manually. The script that loads the TSV files into the system is ``scripts/api/setup-datasetfields.sh`` and contains a series of curl commands. Here's an example of the necessary curl command with the new custom metadata block in the "/tmp" directory. +If you are creating a new custom metadata block (hopefully with the idea of contributing it back to the community if you feel like it would provide value to others), the Dataverse Software installation process won't know about your new TSV file so you must load it manually. The script that loads the TSV files into the system is ``scripts/api/setup-datasetfields.sh`` and contains a series of curl commands. Here's an example of the necessary curl command with the new custom metadata block in the "/tmp" directory. ``curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file /tmp/new-metadata-block.tsv`` @@ -605,7 +608,7 @@ controlledvocabulary.language.marathi_(marathi)=Marathi (Mar\u0101\u1E6Dh\u012B) Enabling a Metadata Block ~~~~~~~~~~~~~~~~~~~~~~~~~ -Running a curl command like "load" example above should make the new custom metadata block available within the system but in order to start using the fields you must either enable it from the UI (see :ref:`general-information` section of Dataverse Management in the User Guide) or by running a curl command like the one below using a superuser API token. In the example below we are enabling the "journal" and "geospatial" metadata blocks for the root dataverse: +Running a curl command like "load" example above should make the new custom metadata block available within the system but in order to start using the fields you must either enable it from the UI (see :ref:`general-information` section of Dataverse Collection Management in the User Guide) or by running a curl command like the one below using a superuser API token. In the example below we are enabling the "journal" and "geospatial" metadata blocks for the root Dataverse collection: ``curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-type:application/json" -d "[\"journal\",\"geospatial\"]" http://localhost:8080/api/dataverses/:root/metadatablocks`` @@ -615,14 +618,14 @@ Updating the Solr Schema Once you have enabled a new metadata block you should be able to see the new fields in the GUI but before you can save the dataset, you must add additional fields to your Solr schema. -An API endpoint of Dataverse provides you with a generated set of all fields that need to be added to the Solr schema +An API endpoint of your Dataverse installation provides you with a generated set of all fields that need to be added to the Solr schema configuration, including any enabled metadata schemas: ``curl http://localhost:8080/api/admin/index/solr/schema`` For convenience and automation you can download and consider running :download:`updateSchemaMDB.sh <../../../../conf/solr/7.7.2/updateSchemaMDB.sh>`. It uses the API endpoint above and writes schema files to the filesystem (so be sure to run it on the Solr server itself as the Unix user who owns the Solr files) and then triggers a Solr reload. -By default, it will download from Dataverse at `http://localhost:8080` and reload Solr at `http://localhost:8983`. +By default, it will download from your Dataverse installation at `http://localhost:8080` and reload Solr at `http://localhost:8983`. You may use the following environment variables with this script or mix'n'match with options: ==================== ====== =============================================== ========================================================= @@ -644,7 +647,7 @@ Please note that if you are going to make a pull request updating ``conf/solr/7. Reloading a Metadata Block -------------------------- -As mentioned above, changes to metadata blocks that ship with Dataverse will be made over time to improve them and release notes will sometimes instruct you to reload an existing metadata block. The syntax for reloading is the same as reloading. Here's an example with the "citation" metadata block: +As mentioned above, changes to metadata blocks that ship with the Dataverse Software will be made over time to improve them and release notes will sometimes instruct you to reload an existing metadata block. The syntax for reloading is the same as reloading. Here's an example with the "citation" metadata block: ``curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv`` @@ -657,9 +660,9 @@ Tips from the Dataverse Community If there are tips that you feel are omitted from this document, please open an issue at https://github.com/IQSS/dataverse/issues and consider making a pull request to make improvements. You can find this document at https://github.com/IQSS/dataverse/blob/develop/doc/sphinx-guides/source/admin/metadatacustomization.rst -Alternatively, you are welcome to request "edit" access to this "Tips for Dataverse metadata blocks from the community" Google doc: https://docs.google.com/document/d/1XpblRw0v0SvV-Bq6njlN96WyHJ7tqG0WWejqBdl7hE0/edit?usp=sharing +Alternatively, you are welcome to request "edit" access to this "Tips for Dataverse Software metadata blocks from the community" Google doc: https://docs.google.com/document/d/1XpblRw0v0SvV-Bq6njlN96WyHJ7tqG0WWejqBdl7hE0/edit?usp=sharing -The thinking is that the tips can become issues and the issues can eventually be worked on as features to improve the Dataverse metadata system. +The thinking is that the tips can become issues and the issues can eventually be worked on as features to improve the Dataverse Software metadata system. Footnotes --------- @@ -679,7 +682,7 @@ Footnotes duplicated. See "Editing TSV files" for how to check for duplication. .. [5] - "displayoncreate" was "showabovefold" in Dataverse versions before 4.3.1 (see + "displayoncreate" was "showabovefold" in Dataverse Software versions before 4.3.1 (see `#3073 `__) but parsing is done based on column order rather than name so this only matters to the person reading the TSV file. From 4ec924e3af36c3e47890f35be18288a3152b793d Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:34:24 -0500 Subject: [PATCH 019/274] more updates --- doc/sphinx-guides/source/admin/metadataexport.rst | 6 +++--- doc/sphinx-guides/source/admin/monitoring.rst | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadataexport.rst b/doc/sphinx-guides/source/admin/metadataexport.rst index 60f5c96a4e2..55b1ee3617c 100644 --- a/doc/sphinx-guides/source/admin/metadataexport.rst +++ b/doc/sphinx-guides/source/admin/metadataexport.rst @@ -14,7 +14,7 @@ A scheduled timer job that runs nightly will attempt to export any published dat Batch exports through the API ----------------------------- -In addition to the automated exports, a Dataverse admin can start a batch job through the API. The following 2 API calls are provided: +In addition to the automated exports, a Dataverse installation admin can start a batch job through the API. The following 2 API calls are provided: ``curl http://localhost:8080/api/admin/metadata/exportAll`` @@ -28,12 +28,12 @@ Note, that creating, modifying, or re-exporting an OAI set will also attempt to Export Failures --------------- -An export batch job, whether started via the API, or by the application timer, will leave a detailed log in your configured logs directory. This is the same location where your main app server logs are found. The name of the log file is ``export_[timestamp].log`` - for example, *export_2016-08-23T03-35-23.log*. The log will contain the numbers of datasets processed successfully and those for which metadata export failed, with some information on the failures detected. Please attach this log file if you need to contact Dataverse support about metadata export problems. +An export batch job, whether started via the API, or by the application timer, will leave a detailed log in your configured logs directory. This is the same location where your main app server logs are found. The name of the log file is ``export_[timestamp].log`` - for example, *export_2016-08-23T03-35-23.log*. The log will contain the numbers of datasets processed successfully and those for which metadata export failed, with some information on the failures detected. Please attach this log file if you need to contact Dataverse Project support about metadata export problems. Downloading Metadata via GUI ---------------------------- -The :doc:`/user/dataset-management` section of the User Guide explains how end users can download the metadata formats above from the Dataverse GUI. +The :doc:`/user/dataset-management` section of the User Guide explains how end users can download the metadata formats above from your Dataverse installation's GUI. Downloading Metadata via API ---------------------------- diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst index 74b41c1578f..3d97c7ea8f7 100644 --- a/doc/sphinx-guides/source/admin/monitoring.rst +++ b/doc/sphinx-guides/source/admin/monitoring.rst @@ -90,7 +90,7 @@ Process the logs: Please note that load balancers (such as Amazon's ELB) might interfer with the ``LogFormat`` mentioned above. To start troubleshooting errors such as ``AWStats did not find any valid log lines that match your LogFormat parameter``, you might need to bump up the value of ``NbOfLinesForCorruptedLog`` in the config files above and re-try while you interate on your Apache and AWStats config. -Please note that the Dataverse team has attempted to parse Glassfish/Payara logs using AWStats but it didn't seem to just work and posts have been made at https://stackoverflow.com/questions/49134154/what-logformat-definition-does-awstats-require-to-parse-glassfish-http-access-logs and https://sourceforge.net/p/awstats/discussion/43428/thread/9b1befda/ that can be followed up on some day. +Please note that the Dataverse Project team has attempted to parse Glassfish/Payara logs using AWStats but it didn't seem to just work and posts have been made at https://stackoverflow.com/questions/49134154/what-logformat-definition-does-awstats-require-to-parse-glassfish-http-access-logs and https://sourceforge.net/p/awstats/discussion/43428/thread/9b1befda/ that can be followed up on some day. Database Connection Pool Used by App Server ------------------------------------------- @@ -113,7 +113,7 @@ Changes made to draft versions of datasets are logged in a folder called logs/ed Solr Indexing Failures Logging ------------------------------ -Failures occurring during the indexing of dataverses and datasets are logged in a folder called logs/process-failures. This logging will include instructions for manually re-running the failed processes. It may be advantageous to set up a automatic job to monitor new entries into this log folder so that indexes could be re-run. +Failures occurring during the indexing of Dataverse collections and datasets are logged in a folder called logs/process-failures. This logging will include instructions for manually re-running the failed processes. It may be advantageous to set up a automatic job to monitor new entries into this log folder so that indexes could be re-run. EJB Timers ---------- From 349afa758aad202263189c3617cc1b5ffbab03bd Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:39:19 -0500 Subject: [PATCH 020/274] more text updates --- .../admin/reporting-tools-and-queries.rst | 8 ++++---- .../source/admin/solr-search-index.rst | 20 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/sphinx-guides/source/admin/reporting-tools-and-queries.rst b/doc/sphinx-guides/source/admin/reporting-tools-and-queries.rst index 197339d767d..21539b9e41f 100644 --- a/doc/sphinx-guides/source/admin/reporting-tools-and-queries.rst +++ b/doc/sphinx-guides/source/admin/reporting-tools-and-queries.rst @@ -10,10 +10,10 @@ Reporting tools and queries created by members of the Dataverse community. * Matrix (): Collaboration Matrix is a visualization showing the connectedness and collaboration between authors and their affiliations. Visit https://rin.lipi.go.id/matrix/ to play with a production installation. -* Dataverse Web Report (): Creates interactive charts showing data extracted from the Dataverse Excel Report +* Dataverse Installation Web Report (): Creates interactive charts showing data extracted from the Dataverse installation Excel Report -* Dataverse Reports for Texas Digital Library (): A python3-based tool to generate and email statistical reports from Dataverse (https://dataverse.org/) using the native API and database queries. +* Dataverse Installation Reports for Texas Digital Library (): A python3-based tool to generate and email statistical reports from Dataverse (https://dataverse.org/) using the native API and database queries. -* dataverse-metrics (): Aggregates and visualizes metrics for installations of Dataverse around the world or a single Dataverse installation. +* dataverse-metrics (): Aggregates and visualizes metrics for Dataverse installations around the world or a single Dataverse installation. -* Useful queries from the Dataverse Community (): A community-generated and maintained document of postgresql queries for getting information about users and dataverses/datasets/files in your Dataverse installation. If you are trying to find out some information from Dataverse, chances are that someone else has had the same questions and it's now listed in this document. If it's not listed, please feel free to add it to the document. \ No newline at end of file +* Useful queries from the Dataverse Community (): A community-generated and maintained document of postgresql queries for getting information about users and dataverse collections,datasets, and files in your Dataverse installation. If you are trying to find out some information from your Dataverse installation, chances are that someone else has had the same questions and it's now listed in this document. If it's not listed, please feel free to add it to the document. \ No newline at end of file diff --git a/doc/sphinx-guides/source/admin/solr-search-index.rst b/doc/sphinx-guides/source/admin/solr-search-index.rst index d37b7eedb26..5dcc866b137 100644 --- a/doc/sphinx-guides/source/admin/solr-search-index.rst +++ b/doc/sphinx-guides/source/admin/solr-search-index.rst @@ -1,7 +1,7 @@ Solr Search Index ================= -Dataverse requires Solr to be operational at all times. If you stop Solr, you should see a error about this on the root dataverse page, which is powered by the search index Solr provides. You can set up Solr by following the steps in our Installation Guide's :doc:`/installation/prerequisites` and :doc:`/installation/config` sections explaining how to configure it. This section you're reading now is about the care and feeding of the search index. PostgreSQL is the "source of truth" and the Dataverse application will copy data from PostgreSQL into Solr. For this reason, the search index can be rebuilt at any time. Depending on the amount of data you have, this can be a slow process. You are encouraged to experiment with production data to get a sense of how long a full reindexing will take. +A Dataverse installation requires Solr to be operational at all times. If you stop Solr, you should see a error about this on the root Dataverse installation page, which is powered by the search index Solr provides. You can set up Solr by following the steps in our Installation Guide's :doc:`/installation/prerequisites` and :doc:`/installation/config` sections explaining how to configure it. This section you're reading now is about the care and feeding of the search index. PostgreSQL is the "source of truth" and the Dataverse installation will copy data from PostgreSQL into Solr. For this reason, the search index can be rebuilt at any time. Depending on the amount of data you have, this can be a slow process. You are encouraged to experiment with production data to get a sense of how long a full reindexing will take. .. contents:: Contents: :local: @@ -9,7 +9,7 @@ Dataverse requires Solr to be operational at all times. If you stop Solr, you sh Full Reindex ------------- -There are two ways to perform a full reindex of the Dataverse search index. Starting with a "clear" ensures a completely clean index but involves downtime. Reindexing in place doesn't involve downtime but does not ensure a completely clean index. +There are two ways to perform a full reindex of the Dataverse installation search index. Starting with a "clear" ensures a completely clean index but involves downtime. Reindexing in place doesn't involve downtime but does not ensure a completely clean index. Clear and Reindex +++++++++++++++++ @@ -29,7 +29,7 @@ Remove all Solr documents that are orphaned (ie not associated with objects in t Clearing Data from Solr ~~~~~~~~~~~~~~~~~~~~~~~ -Please note that the moment you issue this command, it will appear to end users looking at the root dataverse page that all data is gone! This is because the root dataverse page is powered by the search index. +Please note that the moment you issue this command, it will appear to end users looking at the root Dataverse installation page that all data is gone! This is because the root Dataverse installation page is powered by the search index. ``curl http://localhost:8080/api/admin/index/clear`` @@ -60,16 +60,16 @@ If indexing stops, this command should pick up where it left off based on which Manual Reindexing ----------------- -If you have made manual changes to a dataset in the database or wish to reindex a dataset that solr didn't want to index properly, it is possible to manually reindex dataverses and datasets. +If you have made manual changes to a dataset in the database or wish to reindex a dataset that solr didn't want to index properly, it is possible to manually reindex Dataverse collections and datasets. -Reindexing Dataverses -++++++++++++++++++++++ +Reindexing Dataverse Collections +++++++++++++++++++++++++++++++++ -Dataverses must be referenced by database object ID. If you have direct database access an SQL query such as +Dataverse collections must be referenced by database object ID. If you have direct database access an SQL query such as ``select id from dataverse where alias='datavarsealias';`` -should work, or you may click the Dataverse's "Edit" menu and look for dataverseId= in the URLs produced by the drop-down. Then, to re-index: +should work, or you may click the Dataverse Platform's "Edit" menu and look for dataverseId= in the URLs produced by the drop-down. Then, to re-index: ``curl http://localhost:8080/api/admin/index/dataverses/135`` @@ -89,8 +89,8 @@ To re-index a dataset by its database ID: Manually Querying Solr ---------------------- -If you suspect something isn't indexed properly in solr, you may bypass the Dataverse web interface and query the command line directly to verify what solr returns: +If you suspect something isn't indexed properly in solr, you may bypass the Dataverse installation's web interface and query the command line directly to verify what solr returns: ``curl "http://localhost:8983/solr/collection1/select?q=dsPersistentId:doi:10.15139/S3/HFV0AO"`` -to see the JSON you were hopefully expecting to see passed along to Dataverse. +to see the JSON you were hopefully expecting to see passed along to the Dataverse installation. From d2e1cdb120a5a82e18a8fae9d3804d0c458d7f87 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:47:19 -0500 Subject: [PATCH 021/274] more updates --- doc/sphinx-guides/source/admin/timers.rst | 16 ++++---- .../source/admin/troubleshooting.rst | 38 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/doc/sphinx-guides/source/admin/timers.rst b/doc/sphinx-guides/source/admin/timers.rst index d94f22da2a5..afb4f83515b 100644 --- a/doc/sphinx-guides/source/admin/timers.rst +++ b/doc/sphinx-guides/source/admin/timers.rst @@ -1,17 +1,17 @@ .. role:: fixedwidthplain -Dataverse Application Timers -============================ +Dataverse Installation Application Timers +========================================= -Dataverse uses timers to automatically run scheduled Harvest and Metadata export jobs. +Your Dataverse installation uses timers to automatically run scheduled Harvest and Metadata export jobs. .. contents:: |toctitle| :local: -Dedicated timer server in a Dataverse server cluster ----------------------------------------------------- +Dedicated timer server in a Dataverse Installation server cluster +----------------------------------------------------------------- -When running a Dataverse cluster - i.e. multiple Dataverse application +When running a Dataverse installation cluster - i.e. multiple Dataverse installation application servers talking to the same database - **only one** of them must act as the *dedicated timer server*. This is to avoid starting conflicting batch jobs on multiple nodes at the same time. @@ -22,7 +22,7 @@ The following JVM option instructs the application to act as the dedicated timer ``-Ddataverse.timerServer=true`` -**IMPORTANT:** Note that this option is automatically set by the Dataverse installer script. That means that when **configuring a multi-server cluster**, it will be the responsibility of the installer to remove the option from the :fixedwidthplain:`domain.xml` of every node except the one intended to be the timer server. +**IMPORTANT:** Note that this option is automatically set by the Dataverse Software installer script. That means that when **configuring a multi-server cluster**, it will be the responsibility of the installer to remove the option from the :fixedwidthplain:`domain.xml` of every node except the one intended to be the timer server. Harvesting Timers ----------------- @@ -65,7 +65,7 @@ This job is automatically scheduled to run once a week at 12:30AM local time on * Build and deploy the application -* Alternatively, you can insert an ejb-jar.xml file into a provided dataverse war file without building the application. +* Alternatively, you can insert an ejb-jar.xml file into a provided Dataverse Software war file without building the application. * Check if there is already an ejb-jar.xml file in the war file diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index d115554f7bc..99bf00c5647 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -3,15 +3,15 @@ Troubleshooting =============== -Sometimes Dataverse users get into trouble. Sometimes Dataverse itself gets into trouble. If something has gone wrong, this section is for you. +Sometimes a Dataverse installation's users get into trouble. Sometimes a Dataverse installation itself gets into trouble. If something has gone wrong, this section is for you. .. contents:: Contents: :local: -Using Dataverse APIs to Troubleshoot and Fix Problems ------------------------------------------------------ +Using Dataverse Installation APIs to Troubleshoot and Fix Problems +------------------------------------------------------------------ -See the :doc:`/api/intro` section of the API Guide for a high level overview of Dataverse APIs. Below are listed problems that support teams might encounter that can be handled via API (sometimes only via API). +See the :doc:`/api/intro` section of the API Guide for a high level overview of Dataverse Software APIs. Below are listed problems that support teams might encounter that can be handled via API (sometimes only via API). A Dataset Is Locked And Cannot Be Edited or Published ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -22,19 +22,19 @@ It's normal for the ingest process described in the :doc:`/user/tabulardatainges A dataset is locked with a lock of type ``finalizePublication`` while the persistent identifiers for the datafiles in the dataset are registered or updated, and/or while the physical files are being validated by recalculating the checksums and verifying them against the values stored in the database, before the publication process can be completed (Note that either of the two tasks can be disabled via database options - see :doc:`/installation/config`). If a dataset has been in this state for a long period of time, for hours or longer, it is somewhat safe to assume that it is stuck (for example, the process may have been interrupted by an application server restart, or a system crash), so you may want to remove the lock (to be safe, do restart the application server, to ensure that the job is no longer running in the background) and advise the user to try publishing again. See :doc:`dataverses-datasets` for more information on publishing. -If any files in the dataset fail the validation above the dataset will be left locked with a ``DatasetLock.Reason=FileValidationFailed``. The user will be notified that they need to contact their Dataverse support in order to address the issue before another attempt to publish can be made. The admin will have to address and fix the underlying problems (by either restoring the missing or corrupted files, or by purging the affected files from the dataset) before deleting the lock and advising the user to try to publish again. The goal of the validation framework is to catch these types of conditions while the dataset is still in DRAFT. +If any files in the dataset fail the validation above the dataset will be left locked with a ``DatasetLock.Reason=FileValidationFailed``. The user will be notified that they need to contact their Dataverse installation's support in order to address the issue before another attempt to publish can be made. The admin will have to address and fix the underlying problems (by either restoring the missing or corrupted files, or by purging the affected files from the dataset) before deleting the lock and advising the user to try to publish again. The goal of the validation framework is to catch these types of conditions while the dataset is still in DRAFT. During an attempt to publish a dataset, the validation will stop after encountering the first file that fails it. It is strongly recommended for the admin to review and verify *all* the files in the dataset, so that all the compromised files can be fixed before the lock is removed. We recommend using the ``/api/validate/dataset/files/{id}`` API. It will go through all the files for the dataset specified, and will report which ones have failed validation. see :ref:`Physical Files Validation in a Dataset ` in the :doc:`/api/native-api` section of the User Guide. -The following are two real life examples of problems that have resulted in corrupted datafiles during normal operation of Dataverse: +The following are two real life examples of problems that have resulted in corrupted datafiles during normal operation of a Dataverse installation: -1. Botched file deletes - while a datafile is in DRAFT, attempting to delete it from the dataset involves deleting both the ``DataFile`` database table entry, and the physical file. (Deleting a datafile from a *published* version merely removes it from the future versions - but keeps the file in the dataset). The problem we've observed in the early versions of Dataverse was a *partially successful* delete, where the database tansaction would fail (for whatever reason), but only after the physical file had already been deleted from the filesystem. Thus resulting in a datafile entry remaining in the dataset, but with the corresponding physical file missing. We believe we have addressed the issue that was making this condition possible, so it shouldn't happen again - but there may be a datafile in this state in your database. Assuming the user's intent was in fact to delete the file, the easiest solution is simply to confirm it and purge the datafile entity from the database. Otherwise the file needs to be restored from backups, or obtained from the user and copied back into storage. -2. Another issue we've observed: a failed tabular data ingest that leaves the datafile un-ingested, BUT with the physical file already replaced by the generated tab-delimited version of the data. This datafile will fail the validation because the checksum in the database matches the file in the original format (Stata, SPSS, etc.) as uploaded by the user. To fix: luckily, this is easily reversable, since the uploaded original should be saved in your storage, with the .orig extension. Simply swapping the .orig copy with the main file associated with the datafile will fix it. Similarly, we believe this condition should not happen again in Dataverse versions 4.20+, but you may have some legacy cases on your server. +1. Botched file deletes - while a datafile is in DRAFT, attempting to delete it from the dataset involves deleting both the ``DataFile`` database table entry, and the physical file. (Deleting a datafile from a *published* version merely removes it from the future versions - but keeps the file in the dataset). The problem we've observed in the early versions of the Dataverse Software was a *partially successful* delete, where the database transaction would fail (for whatever reason), but only after the physical file had already been deleted from the filesystem. Thus resulting in a datafile entry remaining in the dataset, but with the corresponding physical file missing. We believe we have addressed the issue that was making this condition possible, so it shouldn't happen again - but there may be a datafile in this state in your database. Assuming the user's intent was in fact to delete the file, the easiest solution is simply to confirm it and purge the datafile entity from the database. Otherwise the file needs to be restored from backups, or obtained from the user and copied back into storage. +2. Another issue we've observed: a failed tabular data ingest that leaves the datafile un-ingested, BUT with the physical file already replaced by the generated tab-delimited version of the data. This datafile will fail the validation because the checksum in the database matches the file in the original format (Stata, SPSS, etc.) as uploaded by the user. To fix: luckily, this is easily reversable, since the uploaded original should be saved in your storage, with the .orig extension. Simply swapping the .orig copy with the main file associated with the datafile will fix it. Similarly, we believe this condition should not happen again in Dataverse Software versions 4.20+, but you may have some legacy cases on your server. Someone Created Spam Datasets and I Need to Delete Them ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Depending on how open your installation of Dataverse is to the general public creating datasets, you may sometimes need to deal with spam datasets. +Depending on how open your Dataverse installation is to the general public creating datasets, you may sometimes need to deal with spam datasets. Look for "destroy" in the :doc:`/api/native-api` section of the API Guide. @@ -51,7 +51,7 @@ Ingest Long-Running Ingest Jobs Have Exhausted System Resources ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Ingest is both CPU- and memory-intensive, and depending on your system resources and the size and format of tabular data files uploaded, may render Dataverse unresponsive or nearly inoperable. It is possible to cancel these jobs by purging the ingest queue. +Ingest is both CPU- and memory-intensive, and depending on your system resources and the size and format of tabular data files uploaded, may render your Dataverse installation unresponsive or nearly inoperable. It is possible to cancel these jobs by purging the ingest queue. ``/usr/local/payara5/mq/bin/imqcmd -u admin query dst -t q -n DataverseIngest`` will query the DataverseIngest destination. The password, unless you have changed it, matches the username. @@ -61,7 +61,7 @@ Finally, list destinations to verify that the purge was successful:: ``/usr/local/payara5/mq/bin/imqcmd -u admin list dst`` -If you are still running Glassfish, substitute glassfish4 for payara5 above. If you have installed Dataverse in some other location, adjust the above paths accordingly. +If you are still running Glassfish, substitute glassfish4 for payara5 above. If you have installed your Dataverse installation in some other location, adjust the above paths accordingly. .. _troubleshooting-payara: @@ -73,7 +73,7 @@ Payara Finding the Payara Log File ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``/usr/local/payara5/glassfish/domains/domain1/logs/server.log`` is the main place to look when you encounter problems (assuming you installed Payara in the default directory). Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version or commit. Send more of the stack trace (the entire file if possible) to developers who can help (see "Getting Help", below) and be sure to say which version of Dataverse you are running. +``/usr/local/payara5/glassfish/domains/domain1/logs/server.log`` is the main place to look when you encounter problems (assuming you installed Payara in the default directory). Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version or commit. Send more of the stack trace (the entire file if possible) to developers who can help (see "Getting Help", below) and be sure to say which version of the Dataverse Software you have installed. .. _increase-payara-logging: @@ -92,7 +92,7 @@ Our guides focus on using the command line to manage Payara but you might be int Deployment fails, "EJB Timer Service not available" --------------------------------------------------- -Sometimes the Dataverse application fails to deploy, or Payara fails to restart once the application is deployed, with the following error message: :fixedwidthplain:`"remote failure: Error occurred during deployment: Exception while loading the app : EJB Timer Service is not available. Please see server.log for more details."` +Sometimes your Dataverse installation fails to deploy, or Payara fails to restart once the application is deployed, with the following error message: :fixedwidthplain:`"remote failure: Error occurred during deployment: Exception while loading the app : EJB Timer Service is not available. Please see server.log for more details."` We don't know what's causing this issue, but here's a known workaround: @@ -112,7 +112,7 @@ Note that it may or may not work on your system, so it is provided as an example Timer Not Working ----------------- -Dataverse relies on EJB timers to perform scheduled tasks: harvesting from remote servers, updating the local OAI sets and running metadata exports. (See :doc:`timers` for details.) If these scheduled jobs are not running on your server, you might experience the following symptoms: +Your Dataverse installation relies on EJB timers to perform scheduled tasks: harvesting from remote servers, updating the local OAI sets and running metadata exports. (See :doc:`timers` for details.) If these scheduled jobs are not running on your server, you might experience the following symptoms: If you are seeing the following in your server.log... @@ -125,15 +125,15 @@ followed by an Exception stack trace with these lines in it: :fixedwidthplain:`Exception Description: Could not deserialize object from byte array` ... -... you should reach out by opening an issue. In the good old days of running Dataverse 4 running on Glassfish 4, this -was a hint for an unsupported JDBC driver. In Dataverse 5 this would be a new regression and its cause would need to be +... you should reach out by opening an issue. In the good ol' days of running Dataverse Software version 4 running on Glassfish 4, this +was a hint for an unsupported JDBC driver. In Dataverse Software version 5 this would be a new regression and its cause would need to be investigated. Constraint Violations Issues ---------------------------- -In real life production use, it may be possible to end up in a situation where some values associated with the datasets in your database are no longer valid under the constraints enforced by the latest version of Dataverse. This is not very likely to happen, but if it does, the symptoms will be as follows: Some datasets can no longer be edited, long exception stack traces logged in the app server log, caused by:: +In real life production use, it may be possible to end up in a situation where some values associated with the datasets in your database are no longer valid under the constraints enforced by the later versions of the Dataverse Software. This is not very likely to happen, but if it does, the symptoms will be as follows: Some datasets can no longer be edited, long exception stack traces logged in the app server log, caused by:: javax.validation.ConstraintViolationException: Bean Validation constraint(s) violated while executing Automatic Bean Validation on callback event:'preUpdate'. @@ -141,12 +141,12 @@ In real life production use, it may be possible to end up in a situation where s (contrary to what the message suggests, there are no specific "details" anywhere in the stack trace that would explain what values violate which constraints) -To identifiy the specific invalid values in the affected datasets, or to check all the datasets in the Dataverse for constraint violations, see :ref:`Dataset Validation ` in the :doc:`/api/native-api` section of the User Guide. +To identify the specific invalid values in the affected datasets, or to check all the datasets in the Dataverse installation for constraint violations, see :ref:`Dataset Validation ` in the :doc:`/api/native-api` section of the User Guide. Many Files with a File Type of "Unknown", "Application", or "Binary" -------------------------------------------------------------------- -From the home page of a Dataverse installation you can get a count of files by file type by clicking "Files" and then scrolling down to "File Type". If you see a lot of files that are "Unknown", "Application", or "Binary" you can have Dataverse attempt to redetect the file type by using the :ref:`Redetect File Type ` API endpoint. +From the home page of a Dataverse installation you can get a count of files by file type by clicking "Files" and then scrolling down to "File Type". If you see a lot of files that are "Unknown", "Application", or "Binary" you can have the Dataverse instalation attempt to redetect the file type by using the :ref:`Redetect File Type ` API endpoint. Getting Help ------------ From 5652040259ae62e1a4afc94a6acc5fa8a54f0baa Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 6 Jan 2021 12:48:34 -0500 Subject: [PATCH 022/274] last update of admin guide --- doc/sphinx-guides/source/admin/user-administration.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/admin/user-administration.rst b/doc/sphinx-guides/source/admin/user-administration.rst index d9907a94f43..bc9be64775f 100644 --- a/doc/sphinx-guides/source/admin/user-administration.rst +++ b/doc/sphinx-guides/source/admin/user-administration.rst @@ -9,7 +9,7 @@ This section focuses on user administration tools and tasks. Manage Users ------------ -The Manage Users table gives the network administrator a list of all user accounts in table form. You can access it by clicking the "Manage Users" button on the :doc:`dashboard`, which is linked from the header of all Dataverse pages (if you're logged in as an administrator). It lists username, full name, email address, affiliation, the authentication method they use, the roles their account has been granted, and whether or not they have Superuser status. +The Manage Users table gives the network administrator a list of all user accounts in table form. You can access it by clicking the "Manage Users" button on the :doc:`dashboard`, which is linked from the header of all Dataverse installation pages (if you're logged in as an administrator). It lists username, full name, email address, affiliation, the authentication method they use, the roles their account has been granted, and whether or not they have Superuser status. Users are listed alphabetically by username. The search bar above the table allows you to search for a specific user. It performs a right-truncated wildcard search of the Username, Name, and Email columns. This means, if you search "baseba" then it will search those three columns for any string of text that begins with "baseba", e.g. "baseball" or "baseballfan". @@ -47,9 +47,9 @@ See :ref:`change-identifier-label` Confirm Email ------------- -Dataverse encourages builtin/local users to verify their email address upon signup or email change so that sysadmins can be assured that users can be contacted. +A Dataverse installation encourages builtin/local users to verify their email address upon signup or email change so that sysadmins can be assured that users can be contacted. -The app will send a standard welcome email with a URL the user can click, which, when activated, will store a ``lastconfirmed`` timestamp in the ``authenticateduser`` table of the database. Any time this is "null" for a user (immediately after signup and/or changing of their Dataverse email address), their current email on file is considered to not be verified. The link that is sent expires after a time (the default is 24 hours), but this is configurable by a superuser via the ``:MinutesUntilConfirmEmailTokenExpires`` config option. +The app will send a standard welcome email with a URL the user can click, which, when activated, will store a ``lastconfirmed`` timestamp in the ``authenticateduser`` table of the database. Any time this is "null" for a user (immediately after signup and/or changing of their Dataverse installation email address), their current email on file is considered to not be verified. The link that is sent expires after a time (the default is 24 hours), but this is configurable by a superuser via the ``:MinutesUntilConfirmEmailTokenExpires`` config option. Should users' URL token expire, they will see a "Verify Email" button on the account information page to send another URL. From 870fcfac1764d9082605758970653ececd3dfa1b Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 10:46:38 -0500 Subject: [PATCH 023/274] more updates --- doc/sphinx-guides/source/api/apps.rst | 26 +++++++++++++------------- doc/sphinx-guides/source/api/auth.rst | 6 +++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index dca641b1feb..0428dee9278 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -1,9 +1,9 @@ Apps ==== -The introduction of Dataverse APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools-and-queries` sections of the Admin Guide. +The introduction of Dataverse Platform APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools-and-queries` sections of the Admin Guide. -The apps below are open source and demonstrate how to use Dataverse APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse APIs in Python, Javascript, R, and Java. +The apps below are open source and demonstrate how to use Dataverse Platform APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse Platform APIs in Python, Javascript, R, and Java. .. contents:: |toctitle| :local: @@ -42,7 +42,7 @@ https://github.com/IQSS/TwoRavens Python ------ -Please note that there are multiple Python modules for Dataverse APIs listed in the :doc:`client-libraries` section. +Please note that there are multiple Python modules for Dataverse Platform APIs listed in the :doc:`client-libraries` section. dataverse-sample-data ~~~~~~~~~~~~~~~~~~~~~ @@ -54,28 +54,28 @@ https://github.com/IQSS/dataverse-sample-data Texas Digital Library dataverse-reports ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Dataverse Reports for Texas Digital Library generates and emails statistical reports for an installation of Dataverse using the native API and database queries. +Dataverse Installation Reports from Texas Digital Library generates and emails statistical reports for a Dataverse installation using the native API and database queries. https://github.com/TexasDigitalLibrary/dataverse-reports OSF ~~~ -OSF allows you to view, download, and upload files to and from a Dataverse dataset from an Open Science Framework (OSF) project. +OSF allows you to view, download, and upload files to and from a dataset in a Dataverse installation from an Open Science Framework (OSF) project. https://github.com/CenterForOpenScience/osf.io/tree/develop/addons/dataverse Geoconnect ~~~~~~~~~~ -Geoconnect allows Dataverse files to be visualized and explored on `WorldMap `_. Read more about it in the :doc:`/user/data-exploration/worldmap` section of the User Guide. +Geoconnect allows files from a Dataverse installation to be visualized and explored on `WorldMap `_. Read more about it in the :doc:`/user/data-exploration/worldmap` section of the User Guide. https://github.com/IQSS/geoconnect dataverse-metrics ~~~~~~~~~~~~~~~~~ -dataverse-metrics aggregates and visualizes metrics across multiple Dataverse installations but can also be used with a single installation +dataverse-metrics aggregates and visualizes metrics across multiple Dataverse installations but can also be used with a single installation. https://github.com/IQSS/dataverse-metrics @@ -97,26 +97,26 @@ repo2docker ~~~~~~~~~~~ repo2docker is a command line tool that allows you to create and start a -Docker image from a code repository that follows the [reproducible executable environment specification](https://repo2docker.readthedocs.io/en/latest/specification.html). repo2docker supports Dataverse DOIs to find and retrieve datasets. +Docker image from a code repository that follows the [reproducible executable environment specification](https://repo2docker.readthedocs.io/en/latest/specification.html). repo2docker supports Dataverse installation DOIs to find and retrieve datasets. https://github.com/jupyter/repo2docker/blob/master/repo2docker/contentproviders/dataverse.py Java ---- -Please note that there is a Java library for Dataverse APIs listed in the :doc:`client-libraries` section. +Please note that there is a Java library for Dataverse Platform APIs listed in the :doc:`client-libraries` section. DVUploader ~~~~~~~~~~ -The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse API to upload files to a specified Dataset. Files can be specified by name, or the DVUploader can upload all files in a directory or recursively from a directory tree. The DVUploader can also verify that uploaded files match their local sources by comparing the local and remote fixity checksums. Source code, release 1.0.0- jar file, and documentation are available on GitHub. DVUploader's creation was supported by the Texas Digital Library. +The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse Platform API to upload files to a specified Dataset. Files can be specified by name, or the DVUploader can upload all files in a directory or recursively from a directory tree. The DVUploader can also verify that uploaded files match their local sources by comparing the local and remote fixity checksums. Source code, release 1.0.0- jar file, and documentation are available on GitHub. DVUploader's creation was supported by the Texas Digital Library. https://github.com/IQSS/dataverse-uploader Dataverse for Android ~~~~~~~~~~~~~~~~~~~~~ -Dataverse for Android makes use of Dataverse's Search API. +Dataverse Software on Android makes use of a Dataverse installation's Search API. https://github.com/IQSS/dataverse-android @@ -126,13 +126,13 @@ PHP OJS ~~~ -The Open Journal Systems (OJS) Dataverse Plugin adds data sharing and preservation to the OJS publication process. +The Open Journal Systems (OJS) Dataverse Software Plugin adds data sharing and preservation to the OJS publication process. https://github.com/pkp/ojs/tree/ojs-stable-2_4_8/plugins/generic/dataverse OpenScholar ~~~~~~~~~~~ -The Dataverse module from OpenScholar allows Dataverse widgets to be easily embedded in its web pages: +The Dataverse Software module from OpenScholar allows a Dataverse installation's widgets to be easily embedded in its web pages: https://github.com/openscholar/openscholar/tree/SCHOLAR-3.x/openscholar/modules/os_features/os_dataverse diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index 6d53e52d3b4..99a8a6a843b 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -1,7 +1,7 @@ API Tokens and Authentication ============================= -An API token is similar to a password and allows you to authenticate to Dataverse APIs to perform actions as you. Many Dataverse APIs require the use of an API token. +An API token is similar to a password and allows you to authenticate to Dataverse Platform APIs to perform actions as you. Many Dataverse Platform APIs require the use of an API token. .. contents:: |toctitle| :local: @@ -23,7 +23,7 @@ Passing Your API Token as an HTTP Header (Preferred) or a Query Parameter See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. -There are two ways to pass your API token to Dataverse APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example. +There are two ways to pass your API token to Dataverse Platform APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example. .. code-block:: bash @@ -60,4 +60,4 @@ Use of the ``X-Dataverse-key`` HTTP header form is preferred to passing ``key`` Resetting Your API Token ------------------------ -You can reset your API Token from your account page in Dataverse as described in the :doc:`/user/account` section of the User Guide. +You can reset your API Token from your account page in your Dataverse installation as described in the :doc:`/user/account` section of the User Guide. From 8e7ea59ab185b57f2cf72b6e820788bcd23ed3b6 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 10:51:18 -0500 Subject: [PATCH 024/274] more updates --- .../source/api/client-libraries.rst | 16 ++++++++-------- doc/sphinx-guides/source/api/dataaccess.rst | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index 64726c77f58..406064424cb 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -1,9 +1,9 @@ Client Libraries ================ -Currently there are client libraries for Python, Javascript, R, and Java that can be used to develop against Dataverse APIs. We use the term "client library" on this page but "Dataverse SDK" (software development kit) is another way of describing these resources. They are designed to help developers express Dataverse concepts more easily in the languages listed below. For support on any of these client libraries, please consult each project's README. +Currently there are client libraries for Python, Javascript, R, and Java that can be used to develop against Dataverse Software APIs. We use the term "client library" on this page but "Dataverse Software SDK" (software development kit) is another way of describing these resources. They are designed to help developers express Dataverse Software concepts more easily in the languages listed below. For support on any of these client libraries, please consult each project's README. -Because Dataverse is a SWORD server, additional client libraries exist for Java, Ruby, and PHP per the :doc:`/api/sword` page. +Because a Dataverse installation is a SWORD server, additional client libraries exist for Java, Ruby, and PHP per the :doc:`/api/sword` page. .. contents:: |toctitle| :local: @@ -11,29 +11,29 @@ Because Dataverse is a SWORD server, additional client libraries exist for Java, Python ------ -There are two Python modules for interacting with Dataverse APIs. +There are two Python modules for interacting with Dataverse Software APIs. `pyDataverse `_ had its initial release in 2019 and can be installed with ``pip install pyDataverse``. The module is developed by `Stefan Kasberger `_ from `AUSSDA - The Austrian Social Science Data Archive `_. -`dataverse-client-python `_ had its initial release in 2015. `Robert Liebowitz `_ created this library while at the `Center for Open Science (COS) `_ and the COS uses it to integrate the `Open Science Framework (OSF) `_ with Dataverse via an add-on which itself is open source and listed on the :doc:`/api/apps` page. +`dataverse-client-python `_ had its initial release in 2015. `Robert Liebowitz `_ created this library while at the `Center for Open Science (COS) `_ and the COS uses it to integrate the `Open Science Framework (OSF) `_ with a Dataverse installation via an add-on which itself is open source and listed on the :doc:`/api/apps` page. Javascript ---------- -https://github.com/IQSS/dataverse-client-javascript is the official Javascript package for Dataverse APIs. It can be found on npm at https://www.npmjs.com/package/js-dataverse +https://github.com/IQSS/dataverse-client-javascript is the official Javascript package for Dataverse Software APIs. It can be found on npm at https://www.npmjs.com/package/js-dataverse It was created and is maintained by `The Agile Monkeys `_. R - -https://github.com/IQSS/dataverse-client-r is the official R package for Dataverse APIs. The latest release can be installed from `CRAN `_. +https://github.com/IQSS/dataverse-client-r is the official R package for Dataverse Software APIs. The latest release can be installed from `CRAN `_. -The package is currently maintained by `Will Beasley `_. It was created by `Thomas Leeper `_ whose dataverse can be found at https://dataverse.harvard.edu/dataverse/leeper +The package is currently maintained by `Will Beasley `_. It was created by `Thomas Leeper `_ whose Dataverse collection can be found at https://dataverse.harvard.edu/dataverse/leeper Java ---- -https://github.com/IQSS/dataverse-client-java is the official Java library for Dataverse APIs. +https://github.com/IQSS/dataverse-client-java is the official Java library for Dataverse Platform APIs. `Richard Adams `_ from `ResearchSpace `_ created and maintains this library. diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 3c20233f6e6..9bafde1d819 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -1,7 +1,7 @@ Data Access API =============== -The Data Access API provides programmatic download access to the files stored under Dataverse. +The Data Access API provides programmatic download access to the files stored in a Dataverse installation. More advanced features of the Access API include format-specific transformations (thumbnail generation/resizing for images; converting tabular data into alternative file formats) and access to the data-level metadata that describes the contents of the tabular files. .. contents:: |toctitle| @@ -286,7 +286,7 @@ Preprocessed Data ``/api/access/datafile/$id/metadata/preprocessed`` -This method provides the "preprocessed data" - a summary record that describes the values of the data vectors in the tabular file, in JSON. These metadata values are used by TwoRavens, the companion data exploration utility of the Dataverse application. Please note that this format might change in the future. +This method provides the "preprocessed data" - a summary record that describes the values of the data vectors in the tabular file, in JSON. These metadata values are used by TwoRavens, an external tool that integrates with a Dataverse installation. Please note that this format might change in the future. Authentication and Authorization -------------------------------- From b2f3bfaf6c382cc822edcdf04828234f7d71d881 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 11:30:00 -0500 Subject: [PATCH 025/274] yet more updates --- .../source/api/external-tools.rst | 40 +++++++++---------- doc/sphinx-guides/source/api/faq.rst | 18 ++++----- .../source/api/getting-started.rst | 26 ++++++------ 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index 9c261154150..5ea4a1aa83b 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -1,7 +1,7 @@ Building External Tools ======================= -External tools can provide additional features that are not part of Dataverse itself, such as data exploration. Thank you for your interest in building an external tool for Dataverse! +External tools can provide additional features that are not part of the Dataverse Platform itself, such as data exploration. Thank you for your interest in building an external tool for the Dataverse Platform! .. contents:: |toctitle| :local: @@ -9,11 +9,11 @@ External tools can provide additional features that are not part of Dataverse it Introduction ------------ -External tools are additional applications the user can access or open from Dataverse to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse application. +External tools are additional applications the user can access or open from your Dataverse installation to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse Platform. -Once you have created the external tool itself (which is most of the work!), you need to teach Dataverse how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org +Once you have created the external tool itself (which is most of the work!), you need to teach a Dataverse installation how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org -In short, you will be creating a manifest in JSON format that describes not only how to construct URLs for your tool, but also what types of files your tool operates on, where it should appear in the Dataverse web interfaces, etc. +In short, you will be creating a manifest in JSON format that describes not only how to construct URLs for your tool, but also what types of files your tool operates on, where it should appear in the Dataverse installation web interfaces, etc. The possibilities for external tools are endless. Let's look at some examples to get your creative juices flowing. Then we'll look at a complete list of parameters you can use when creating the manifest file for your tool. @@ -37,18 +37,18 @@ Note: This is the same list that appears in the :doc:`/admin/external-tools` sec How External Tools Are Presented to Users ----------------------------------------- -An external tool can appear in Dataverse in a variety of ways: +An external tool can appear in your Dataverse installation in a variety of ways: - as an explore, preview, or configure option for a file - as an explore option for a dataset - as an embedded preview on the file landing page -See also the :ref:`testing-external-tools` section of the Admin Guide for some perspective on how installations of Dataverse will expect to test your tool before announcing it to their users. +See also the :ref:`testing-external-tools` section of the Admin Guide for some perspective on how Dataverse installations will expect to test your tool before announcing it to their users. Creating an External Tool Manifest ---------------------------------- -External tools must be expressed in an external tool manifest file, a specific JSON format Dataverse requires. As the author of an external tool, you are expected to provide this JSON file and installation instructions on a web page for your tool. +External tools must be expressed in an external tool manifest file, a specific JSON format a Dataverse installation requires. As the author of an external tool, you are expected to provide this JSON file and installation instructions on a web page for your tool. Examples of Manifests +++++++++++++++++++++ @@ -78,9 +78,9 @@ Terminology =========================== ========== Term Definition =========================== ========== - external tool manifest A **JSON file** the defines the URL constructed by Dataverse when users click explore or configure tool options. External tool makers are asked to host this JSON file on a website (no app store yet, sorry) and explain how to use install and use the tool. Examples include :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` and :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` as well as the real world examples above such as Data Explorer. + external tool manifest A **JSON file** the defines the URL constructed by a Dataverse installation when users click explore or configure tool options. External tool makers are asked to host this JSON file on a website (no app store yet, sorry) and explain how to use install and use the tool. Examples include :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` and :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` as well as the real world examples above such as Data Explorer. - displayName The **name** of the tool in the Dataverse web interface. For example, "Data Explorer". + displayName The **name** of the tool in the Dataverse installation web interface. For example, "Data Explorer". description The **description** of the tool, which appears in a popup (for configure tools only) so the user who clicked the tool can learn about the tool before being redirected the tool in a new tab in their browser. HTML is supported. @@ -102,7 +102,7 @@ Terminology reserved words A **set of strings surrounded by curly braces** such as ``{fileId}`` or ``{datasetId}`` that will be inserted into query parameters. See the table below for a complete list. - toolName A **name** of an external tool that is used to differentiate between external tools and also used in bundle.properties for localization in the Dataverse web interface. For example, the toolName for Data Explorer is ``explorer``. For the Data Curation Tool the toolName is ``dct``. This is an optional parameter in the manifest JSON file. + toolName A **name** of an external tool that is used to differentiate between external tools and also used in bundle.properties for localization in the Dataverse installation web interface. For example, the toolName for Data Explorer is ``explorer``. For the Data Curation Tool the toolName is ``dct``. This is an optional parameter in the manifest JSON file. =========================== ========== Reserved Words @@ -118,9 +118,9 @@ Reserved Words ``{fileId}`` depends The database ID of a file the user clicks "Explore" or "Configure" on. For example, ``42``. This reserved word is **required for file level tools** unless you use ``{filePid}`` instead. - ``{filePid}`` depends The Persistent ID (DOI or Handle) of a file the user clicks "Explore" or "Configure" on. For example, ``doi:10.7910/DVN/TJCLKP/3VSTKY``. Note that not all installations of Dataverse have Persistent IDs (PIDs) enabled at the file level. This reserved word is **required for file level tools** unless you use ``{fileId}`` instead. + ``{filePid}`` depends The Persistent ID (DOI or Handle) of a file the user clicks "Explore" or "Configure" on. For example, ``doi:10.7910/DVN/TJCLKP/3VSTKY``. Note that not all Dataverse installations have Persistent IDs (PIDs) enabled at the file level. This reserved word is **required for file level tools** unless you use ``{fileId}`` instead. - ``{apiToken}`` optional The Dataverse API token of the user launching the external tool, if available. Please note that API tokens should be treated with the same care as a password. For example, ``f3465b0c-f830-4bc7-879f-06c0745a5a5c``. + ``{apiToken}`` optional The Dataverse installation's API token of the user launching the external tool, if available. Please note that API tokens should be treated with the same care as a password. For example, ``f3465b0c-f830-4bc7-879f-06c0745a5a5c``. ``{datasetId}`` depends The database ID of the dataset. For example, ``42``. This reseved word is **required for dataset level tools** unless you use ``{datasetPid}`` instead. @@ -128,13 +128,13 @@ Reserved Words ``{datasetVersion}`` optional The friendly version number ( or \:draft ) of the dataset version the file level tool is being launched from. For example, ``1.0`` or ``:draft``. - ``{localeCode}`` optional The code for the language ("en" for English, "fr" for French, etc.) that user has selected from the language toggle in Dataverse. See also :ref:`i18n`. + ``{localeCode}`` optional The code for the language ("en" for English, "fr" for French, etc.) that user has selected from the language toggle in a Dataverse installation. See also :ref:`i18n`. =========================== ========== =========== Internationalization of Your External Tool ++++++++++++++++++++++++++++++++++++++++++ -The name and description of your tool can be localized and made available in different languages in the Dataverse web interface. Use the ``toolName`` parameter in the manifest JSON file and then add that toolName to bundle.properties. +The name and description of your tool can be localized and made available in different languages in your Dataverse installation's web interface. Use the ``toolName`` parameter in the manifest JSON file and then add that toolName to bundle.properties. For example, if the ``toolName`` of your external tool is ``fabulous`` then the lines in Bundle.properties should be: @@ -150,9 +150,9 @@ Again, you can use :download:`fabulousFileTool.json <../_static/installation/fil Testing Your External Tool -------------------------- -As the author of an external tool, you are not expected to learn how to install and operate Dataverse. There's a very good chance your tool can be added to a server Dataverse developers use for testing if you reach out on any of the channels listed under :ref:`getting-help-developers` in the Developer Guide. +As the author of an external tool, you are not expected to learn how to install and operate a Dataverse installation. There's a very good chance your tool can be added to a server Dataverse Community developers use for testing if you reach out on any of the channels listed under :ref:`getting-help-developers` in the Developer Guide. -By all means, if you'd like to install Dataverse yourself, a number of developer-centric options are available. For example, there's a script to spin up Dataverse on EC2 at https://github.com/IQSS/dataverse-sample-data . The process for using curl to add your external tool to a Dataverse installation is documented under :ref:`managing-external-tools` in the Admin Guide. +By all means, if you'd like to install a Dataverse installation yourself, a number of developer-centric options are available. For example, there's a script to spin up a Dataverse installation on EC2 at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible . The process for using curl to add your external tool to your Dataverse installation is documented under :ref:`managing-external-tools` in the Admin Guide. Spreading the Word About Your External Tool ------------------------------------------- @@ -162,19 +162,19 @@ Adding Your Tool to the Inventory of External Tools Once you've gotten your tool working, please make a pull request to update the list of tools above! You are also welcome to download :download:`dataverse-external-tools.tsv <../_static/admin/dataverse-external-tools.tsv>`, add your tool to the TSV file, create and issue at https://github.com/IQSS/dataverse/issues , and then upload your TSV file there. -Unless your tool runs entirely in a browser, you may have integrated server-side software with Dataverse. If so, please double check that your software is listed in the :doc:`/admin/integrations` section of the Admin Guide and if not, please open an issue or pull request to add it. Thanks! +Unless your tool runs entirely in a browser, you may have integrated server-side software with your Dataverse installation. If so, please double check that your software is listed in the :doc:`/admin/integrations` section of the Admin Guide and if not, please open an issue or pull request to add it. Thanks! -If you've thought to yourself that there ought to be an app store for Dataverse external tools, you're not alone. Please see https://github.com/IQSS/dataverse/issues/5688 :) +If you've thought to yourself that there ought to be an app store for Dataverse Platform external tools, you're not alone. Please see https://github.com/IQSS/dataverse/issues/5688 :) Demoing Your External Tool ++++++++++++++++++++++++++ -https://demo.dataverse.org is the place to play around with Dataverse and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible which already includes a number of the tools listed above. +https://demo.dataverse.org is the place to play around with the Dataverse Platform and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible which already includes a number of the tools listed above. Announcing Your External Tool +++++++++++++++++++++++++++++ You are welcome to announce your external tool at https://groups.google.com/forum/#!forum/dataverse-community -If you're too shy, we'll do it for you. We'll probably tweet about it too. Thank you for your contribution to Dataverse! +If you're too shy, we'll do it for you. We'll probably tweet about it too. Thank you for your contribution to the Dataverse Project! diff --git a/doc/sphinx-guides/source/api/faq.rst b/doc/sphinx-guides/source/api/faq.rst index a3994501f6c..d580692a9f5 100644 --- a/doc/sphinx-guides/source/api/faq.rst +++ b/doc/sphinx-guides/source/api/faq.rst @@ -11,13 +11,13 @@ What is an API? See "What is an API?" in the :doc:`intro` section. -What Are Common Use Cases for Dataverse APIs? ---------------------------------------------- +What Are Common Use Cases for Dataverse Platform APIs? +---------------------------------------------------------- See the :doc:`getting-started` section for common use cases for researchers and curators. Other types of API users should find starting points at :ref:`types-of-api-users`. -Where Can I Find Examples of Using Dataverse APIs? --------------------------------------------------- +Where Can I Find Examples of Using Dataverse Platform APIs? +----------------------------------------------------------- See the :doc:`getting-started` section links to examples using curl. @@ -26,7 +26,7 @@ For examples in Javascript, Python, R, and Java, and PHP, see the :doc:`apps` an When Should I Use the Native API vs. the SWORD API? --------------------------------------------------- -The :doc:`sword` is based on a standard, works fine, and is fully supported, but much more development effort has been going into the :doc:`native-api`, which is not based on a standard. It is specific to Dataverse. +The :doc:`sword` is based on a standard, works fine, and is fully supported, but much more development effort has been going into the :doc:`native-api`, which is not based on a standard. It is specific to the Dataverse Platform. SWORD uses XML. The Native API uses JSON. @@ -54,9 +54,9 @@ Note that when multiple query parameters are used (such as ``persistentId`` and Where is the Comprehensive List of All API Functionality? --------------------------------------------------------- -There are so many Dataverse APIs that a single page in this guide would probably be overwhelming. See :ref:`list-of-dataverse-apis` for links to various pages. +There are so many Dataverse Platform APIs that a single page in this guide would probably be overwhelming. See :ref:`list-of-dataverse-apis` for links to various pages. -It is possible to get a complete list of API functionality in Swagger/OpenAPI format if you deploy Dataverse to Payara 5+. For details, see https://github.com/IQSS/dataverse/issues/5794 +It is possible to get a complete list of API functionality in Swagger/OpenAPI format if you deploy Dataverse Software version 5+. For details, see https://github.com/IQSS/dataverse/issues/5794 Is There a Changelog of API Functionality That Has Been Added Over Time? ------------------------------------------------------------------------ @@ -70,8 +70,8 @@ What Functionality is GUI Only and Not Available Via API The following tasks cannot currently be automated via API because no API exists for them. The web interface should be used instead for these GUI-only features: -- Setting a logo image, URL, and tagline when creating a dataverse. -- Editing properties of an existing dataverse. +- Setting a logo image, URL, and tagline when creating a Dataverse collection. +- Editing properties of an existing Dataverse collection. - Set "Enable Access Request" for Terms of Use: https://groups.google.com/d/msg/dataverse-community/oKdesT9rFGc/qM6wrsnnBAAJ - Downloading a guestbook. - Set guestbook_id for a dataset: https://groups.google.com/d/msg/dataverse-community/oKdesT9rFGc/qM6wrsnnBAAJ diff --git a/doc/sphinx-guides/source/api/getting-started.rst b/doc/sphinx-guides/source/api/getting-started.rst index 1c96a0af542..0b4fd42ad4a 100644 --- a/doc/sphinx-guides/source/api/getting-started.rst +++ b/doc/sphinx-guides/source/api/getting-started.rst @@ -1,7 +1,7 @@ Getting Started with APIs ========================= -If you are a researcher or curator who wants to automate parts of your workflow, this section should help you get started. The :doc:`intro` section lists resources for other groups who may be interested in Dataverse APIs such as developers of integrations and support teams. +If you are a researcher or curator who wants to automate parts of your workflow, this section should help you get started. The :doc:`intro` section lists resources for other groups who may be interested in Dataverse Platform APIs such as developers of integrations and support teams. .. contents:: |toctitle| :local: @@ -9,14 +9,14 @@ If you are a researcher or curator who wants to automate parts of your workflow, Servers You Can Test With ------------------------- -Rather than using a production installation of Dataverse, API users are welcome to use http://demo.dataverse.org for testing. You can email support@dataverse.org if you have any trouble with this server. +Rather than using a production Dataverse installation, API users are welcome to use http://demo.dataverse.org for testing. You can email support@dataverse.org if you have any trouble with this server. If you would rather have full control over your own test server, deployments to AWS, Docker, Vagrant, and more are covered in the :doc:`/developers/index` and the :doc:`/installation/index`. Getting an API Token -------------------- -Many Dataverse APIs require an API token. +Many Dataverse Platform APIs require an API token. Once you have identified a server to test with, create an account, click on your name, and get your API token. For more details, see the :doc:`auth` section. @@ -59,8 +59,8 @@ If you don't like curl, don't have curl, or want to use a different programming Depositing Data --------------- -Creating a Dataverse -~~~~~~~~~~~~~~~~~~~~ +Creating a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See :ref:`create-dataverse-api`. @@ -74,8 +74,8 @@ Uploading Files See :ref:`add-file-api`. -Publishing a Dataverse -~~~~~~~~~~~~~~~~~~~~~~ +Publishing a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See :ref:`publish-dataverse-api`. @@ -94,8 +94,8 @@ A quick example search for the word "data" is https://demo.dataverse.org/api/sea See the :doc:`search` section for details. -Finding Recently Published Dataverses, Datasets, and Files -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Finding Recently Published Dataverse Collections, Datasets, and Files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See :ref:`search-date-range`. @@ -117,8 +117,8 @@ Dataset metadata is available in a variety of formats listed at :ref:`metadata-e See :ref:`export-dataset-metadata-api`. -Listing the Contents of a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Listing the Contents of a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See :ref:`show-contents-of-a-dataverse-api`. @@ -143,11 +143,11 @@ See :ref:`list-role-assignments-on-a-dataverse-api`. Beyond "Getting Started" Tasks ------------------------------ -In addition to the tasks listed above, Dataverse supports many other operations via API. +In addition to the tasks listed above, your Dataverse installation supports many other operations via API. See :ref:`list-of-dataverse-apis` and :ref:`types-of-api-users` to get oriented. -If you're looking for some inspiration for how you can use Dataverse APIs, there are open source projects that integrate with Dataverse listed in the :doc:`apps` section. +If you're looking for some inspiration for how you can use the Dataverse Platform APIs, there are open source projects that integrate with the Dataverse Platform listed in the :doc:`apps` section. Getting Help ------------- From b6e1edb5a32c8bce86d3b0bc30b1199eacee7095 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 12:53:08 -0500 Subject: [PATCH 026/274] annnnd more updates --- doc/sphinx-guides/source/api/intro.rst | 34 ++++++++++++------------ doc/sphinx-guides/source/api/metrics.rst | 16 +++++------ doc/sphinx-guides/source/api/search.rst | 22 +++++++-------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index 6ce77f5fe7d..122f99854b9 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -1,16 +1,16 @@ Introduction ============ -Dataverse APIs allow users to accomplish many tasks such as... +Dataverse Platform APIs allow users to accomplish many tasks such as... - creating datasets - uploading files - publishing datasets - and much, much more -... all without using the Dataverse web interface. +... all without using the Dataverse installation's web interface. -APIs open the door for integrations between Dataverse and other software. For a list, see the :doc:`/admin/integrations` section of the Admin Guide. +APIs open the door for integrations between the Dataverse Platform and other software. For a list, see the :doc:`/admin/integrations` section of the Admin Guide. .. contents:: |toctitle| :local: @@ -18,7 +18,7 @@ APIs open the door for integrations between Dataverse and other software. For a What is an API? --------------- -API stands for "Application Programming Interface" and an example is Dataverse's "file upload" API. In the diagram below, we can see that while users can click a button within Dataverse's web interface to upload a file, there are many other ways to get files into Dataverse, all using an API that allows for uploading of files. +API stands for "Application Programming Interface" and an example is the Dataverse Platform's "file upload" API. In the diagram below, we can see that while users can click a button within Dataverse installation's web interface to upload a file, there are many other ways to get files into a Dataverse installation, all using an API that allows for uploading of files. .. graphviz:: @@ -65,34 +65,34 @@ API stands for "Application Programming Interface" and an example is Dataverse's The components above that use the "file" upload API are: - DvUploader is terminal-based application for uploading files that is described in the :doc:`/user/dataset-management` section of the User Guide. -- OJS, OSF, and RSpace are all web applications that can integrate with Dataverse and are described in "Getting Data In" in the :doc:`/admin/integrations` section of the Admin Guide. +- OJS, OSF, and RSpace are all web applications that can integrate with a Dataverse installation and are described in "Getting Data In" in the :doc:`/admin/integrations` section of the Admin Guide. - The script in the diagram can be as simple as a single line of code that is run in a terminal. You can copy and paste "one-liners" like this from the guide. See the :doc:`getting-started` section for examples using a tool called "curl". The diagram above shows only a few examples of software using a specific API but many more APIs are available. .. _types-of-api-users: -Types of Dataverse API Users ----------------------------- +Types of Dataverse Platform API Users +------------------------------------- This guide is intended to serve multiple audiences but pointers various sections of the guide are provided below based on the type of API user you are. -API Users Within a Single Installation of Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +API Users Within a Single Dataverse Installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Each installation of Dataverse will have its own groups of people interested in APIs. +Each Dataverse installation will have its own groups of people interested in APIs. Users of Integrations and Apps ++++++++++++++++++++++++++++++ Integrations and apps can take many forms but two examples are: -- Using Open Science Framework (OSF), a web application, to deposit and publish data into Dataverse. -- Using DVUploader, a terminal-based desktop application, to upload files into Dataverse. +- Using Open Science Framework (OSF), a web application, to deposit and publish data into a Dataverse installation. +- Using DVUploader, a terminal-based desktop application, to upload files into a Dataverse installation. -In both examples, users need to obtain an API Token to authenticate with Dataverse. +In both examples, users need to obtain an API Token to authenticate with a Dataverse installation. -|Start| A good starting point is "API Tokens" in the :doc:`/user/account` section of the User Guide. DvUploader is documented in the :doc:`/user/dataset-management` section of the User Guide. The integrations that are enabled depend on your installation of Dataverse. You can find a list in the :doc:`/admin/integrations` section of the Admin Guide. +|Start| A good starting point is "API Tokens" in the :doc:`/user/account` section of the User Guide. DvUploader is documented in the :doc:`/user/dataset-management` section of the User Guide. The integrations that are enabled depend on your Dataverse installation. You can find a list in the :doc:`/admin/integrations` section of the Admin Guide. Power Users +++++++++++ @@ -104,16 +104,16 @@ Power users may be researchers or curators who are comfortable with automating p Support Teams and Superusers ++++++++++++++++++++++++++++ -Support teams that answer questions about their installation of Dataverse should familiarize themselves with the :doc:`getting-started` section to get a sense of common tasks that researchers and curators might be trying to accomplish by using Dataverse APIs. +Support teams that answer questions about their Dataverse installation should familiarize themselves with the :doc:`getting-started` section to get a sense of common tasks that researchers and curators might be trying to accomplish by using Dataverse Platform APIs. -Superusers of an installation of Dataverse have access a superuser dashboard described in the :doc:`/admin/dashboard` section of the Admin Guide but some operations can only be done via API. +Superusers of a Dataverse installation have access a superuser dashboard described in the :doc:`/admin/dashboard` section of the Admin Guide but some operations can only be done via API. |Start| A good starting point for both groups is the :doc:`getting-started` section of this guide followed by the :doc:`/admin/troubleshooting` section of the Admin Guide. Sysadmins +++++++++ -Sysadmins often write scripts to automate tasks and Dataverse APIs make this possible. Sysadmins have control over the server that Dataverse is running on and may be called upon to execute API commands that are limited to "localhost" (the server itself) for security reasons. +Sysadmins often write scripts to automate tasks and Dataverse Platform APIs make this possible. Sysadmins have control over the server that the Dataverse installation is running on and may be called upon to execute API commands that are limited to "localhost" (the server itself) for security reasons. |Start| A good starting point for sysadmins is "Blocking API Endpoints" in the :doc:`/installation/config` section of the Installation Guide, followed by the :doc:`getting-started` section of this guide, followed by the :doc:`/admin/troubleshooting` section of the Admin Guide. diff --git a/doc/sphinx-guides/source/api/metrics.rst b/doc/sphinx-guides/source/api/metrics.rst index 70a55a27cc8..e29cb3a2c9e 100755 --- a/doc/sphinx-guides/source/api/metrics.rst +++ b/doc/sphinx-guides/source/api/metrics.rst @@ -1,7 +1,7 @@ Metrics API =========== -The Metrics API provides counts of downloads, datasets created, files uploaded, and more, as described below. Dataverse also supports Make Data Count, which is described in the :doc:`/admin/make-data-count` section of the Admin Guide. +The Metrics API provides counts of downloads, datasets created, files uploaded, and more, as described below. The Dataverse Platform also supports Make Data Count, which is described in the :doc:`/admin/make-data-count` section of the Admin Guide. .. contents:: |toctitle| :local: @@ -15,7 +15,7 @@ The Metrics API provides counts of downloads, datasets created, files uploaded, Total ----- -Returns a count of various objects in dataverse over all-time:: +Returns a count of various objects in a Dataverse installation over all-time:: GET https://$SERVER/api/info/metrics/$type @@ -26,7 +26,7 @@ Example: ``curl https://demo.dataverse.org/api/info/metrics/downloads`` To-Month -------- -Returns a count of various objects in dataverse up to a specified month ``$YYYY-DD`` in YYYY-MM format (e.g. ``2018-01``):: +Returns a count of various objects in a Dataverse installation up to a specified month ``$YYYY-DD`` in YYYY-MM format (e.g. ``2018-01``):: GET https://$SERVER/api/info/metrics/$type/toMonth/$YYYY-DD @@ -38,7 +38,7 @@ Example: ``curl https://demo.dataverse.org/api/info/metrics/dataverses/toMonth/2 Past Days --------- -Returns a count of various objects in dataverse for the past ``$days`` (e.g. ``30``):: +Returns a count of various objects in a Dataverse installation for the past ``$days`` (e.g. ``30``):: GET https://$SERVER/api/info/metrics/$type/pastDays/$days @@ -47,13 +47,13 @@ Returns a count of various objects in dataverse for the past ``$days`` (e.g. ``3 Example: ``curl https://demo.dataverse.org/api/info/metrics/datasets/pastDays/30`` -Dataverse Specific Metrics --------------------------- +Dataverse Collection Specific Metrics +------------------------------------- By Subject ~~~~~~~~~~~~~~~ -Returns the number of dataverses by each subject:: +Returns the number of Dataverse collections in a Dataverse installation by each subject:: GET https://$SERVER/api/info/metrics/dataverses/bySubject @@ -61,7 +61,7 @@ Returns the number of dataverses by each subject:: By Category ~~~~~~~~~~~~~~~~~~~~~~ -Returns the number of dataverses by each category:: +Returns the number of Dataverse collections by each category:: GET https://$SERVER/api/info/metrics/dataverses/byCategory diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 25bad2b8091..60e799f00fc 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -4,7 +4,7 @@ Search API .. contents:: |toctitle| :local: -The Search API supports the same searching, sorting, and faceting operations as the Dataverse web interface. +The Search API supports the same searching, sorting, and faceting operations as the Dataverse Platform's web interface. To search unpublished content, you must pass in an API token as described in the :doc:`auth` section. @@ -15,7 +15,7 @@ The parameters and JSON response are partly inspired by the `GitHub Search API < .. _CORS: https://www.w3.org/TR/cors/ -Please note that in Dataverse 4.3 and older the "citation" field wrapped the persistent ID URL in an ```` tag but this has been changed to plaintext. If you want the old value with HTML in it, a new field called "citationHtml" can be used. +Please note that in Dataverse software version 4.3 and older the "citation" field wrapped the persistent ID URL in an ```` tag but this has been changed to plaintext. If you want the old value with HTML in it, a new field called "citationHtml" can be used. Parameters @@ -25,8 +25,8 @@ Parameters Name Type Description =============== ======= =========== q string The search term or terms. Using "title:data" will search only the "title" field. "*" can be used as a wildcard either alone or adjacent to a term (i.e. "bird*"). For example, https://demo.dataverse.org/api/search?q=title:data . For a list of fields to search, please see https://github.com/IQSS/dataverse/issues/2558 (for now). -type string Can be either "dataverse", "dataset", or "file". Multiple "type" parameters can be used to include multiple types (i.e. ``type=dataset&type=file``). If omitted, all types will be returned. For example, https://demo.dataverse.org/api/search?q=*&type=dataset -subtree string The identifier of the dataverse to which the search should be narrowed. The subtree of this dataverse and all its children will be searched. Multiple "subtree" parameters can be used to include multiple Dataverses. For example, https://demo.dataverse.org/api/search?q=data&subtree=birds&subtree=cats . +type string Can be either "Dataverse", "dataset", or "file". Multiple "type" parameters can be used to include multiple types (i.e. ``type=dataset&type=file``). If omitted, all types will be returned. For example, https://demo.dataverse.org/api/search?q=*&type=dataset +subtree string The identifier of the Dataverse collection to which the search should be narrowed. The subtree of this Dataverse collection and all its children will be searched. Multiple "subtree" parameters can be used to include multiple Dataverse collections. For example, https://demo.dataverse.org/api/search?q=data&subtree=birds&subtree=cats . sort string The sort field. Supported values include "name" and "date". See example under "order". order string The order in which to sort. Can either be "asc" or "desc". For example, https://demo.dataverse.org/api/search?q=data&sort=name&order=asc per_page int The number of results to return per request. The default is 10. The max is 1000. See :ref:`iteration example `. @@ -96,7 +96,7 @@ https://demo.dataverse.org/api/search?q=trees "url":"https://demo.dataverse.org/dataverse/birds", "image_url":"https://demo.dataverse.org/api/access/dvCardImage/2", "identifier":"birds", - "description":"A bird dataverse with some trees", + "description":"A bird Dataverse collection with some trees", "published_at":"2016-05-10T12:57:27Z" }, { @@ -152,7 +152,7 @@ Advanced Search Examples https://demo.dataverse.org/api/search?q=finch&show_relevance=true&show_facets=true&fq=publicationDate:2016&subtree=birds -In this example, ``show_relevance=true`` matches per field are shown. Available facets are shown with ``show_facets=true`` and of the facets is being used with ``fq=publicationDate:2016``. The search is being narrowed to the dataverse with the identifier "birds" with the parameter ``subtree=birds``. +In this example, ``show_relevance=true`` matches per field are shown. Available facets are shown with ``show_facets=true`` and of the facets is being used with ``fq=publicationDate:2016``. The search is being narrowed to the Dataverse collection with the identifier "birds" with the parameter ``subtree=birds``. .. code-block:: json @@ -171,13 +171,13 @@ In this example, ``show_relevance=true`` matches per field are shown. Available "url":"https://demo.dataverse.org/dataverse/finches", "image_url":"https://demo.dataverse.org/api/access/dvCardImage/3", "identifier":"finches", - "description":"A dataverse with finches", + "description":"A Dataverse collection with finches", "published_at":"2016-05-10T12:57:38Z", "matches":[ { "description":{ "snippets":[ - "A dataverse with finches" + "A Dataverse collection with finches" ] } }, @@ -283,10 +283,10 @@ The above example ``fq=publicationStatus:Published`` retrieves only "RELEASED" v "global_id": "doi:10.70122/FK2/GUAS41", "description": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.", "published_at": "2019-12-24T08:05:02Z", - "publisher": "mdmizanur rahman Dataverse", + "publisher": "mdmizanur rahman Dataverse collection", "citationHtml": "Finch, Fiona, 2019, \"Darwin's Finches\", https://doi.org/10.70122/FK2/GUAS41, Demo Dataverse, V1", "identifier_of_dataverse": "rahman", - "name_of_dataverse": "mdmizanur rahman Dataverse", + "name_of_dataverse": "mdmizanur rahman Dataverse collection", "citation": "Finch, Fiona, 2019, \"Darwin's Finches\", https://doi.org/10.70122/FK2/GUAS41, Demo Dataverse, V1", "storageIdentifier": "file://10.70122/FK2/GUAS41", "subjects": [ @@ -352,7 +352,7 @@ The above example ``fq=publicationStatus:Published`` retrieves only "RELEASED" v Date Range Search Example ------------------------- -Below is an example of searching across a date range of dataverses, datasets, and files that were published in 2018. +Below is an example of searching across a date range of Dataverse collections, datasets, and files that were published in 2018. `https://demo.dataverse.org/api/search?q=*&per_page=1000&sort=date&order=asc&q=*&fq=dateSort:[2018-01-01T00\:00\:00Z+TO+2019-01-01T00\:00\:00Z] `_ From 2c20ff09b45a85231de7405dca2cde9e13b46b3d Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 7 Jan 2021 18:14:03 -0500 Subject: [PATCH 027/274] more edits. --- doc/sphinx-guides/source/api/intro.rst | 34 ++-- doc/sphinx-guides/source/api/native-api.rst | 176 ++++++++++---------- 2 files changed, 105 insertions(+), 105 deletions(-) diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index 122f99854b9..988d401eddf 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -1,7 +1,7 @@ Introduction ============ -Dataverse Platform APIs allow users to accomplish many tasks such as... +The Dataverse Platform APIs allow users to accomplish many tasks such as... - creating datasets - uploading files @@ -120,37 +120,37 @@ Sysadmins often write scripts to automate tasks and Dataverse Platform APIs make In House Developers +++++++++++++++++++ -Some organizations that run Dataverse employ developers who are tasked with using Dataverse APIs to accomplish specific tasks such as building custom integrations with in house systems or creating reports specific to the organization's needs. +Some organizations that run a Dataverse installation employ developers who are tasked with using the Dataverse installation's APIs to accomplish specific tasks such as building custom integrations with in house systems or creating reports specific to the organization's needs. |Start| A good starting point for in house developers is the :doc:`getting-started` section. API Users Across the Dataverse Project ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The Dataverse project loves contributors! Depending on your interests and skills, you might fall into one or more of the groups below. +The Dataverse Project loves contributors! Depending on your interests and skills, you might fall into one or more of the groups below. Developers of Integrations, External Tools, and Apps ++++++++++++++++++++++++++++++++++++++++++++++++++++ -One of the primary purposes for Dataverse APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: +One of the primary purposes for Dataverse Platform APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: - The :doc:`/admin/integrations` section of the Admin Guide. - The :doc:`/api/external-tools` section this guide. - The :doc:`apps` section of this guide. -|Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with Dataverse, followed by the :doc:`getting-started` section. +|Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with the Dataverse Platform, followed by the :doc:`getting-started` section. -Developers of Dataverse API Client Libraries -++++++++++++++++++++++++++++++++++++++++++++ +Developers of Dataverse Platform API Client Libraries ++++++++++++++++++++++++++++++++++++++++++++++++++++++ -A client library helps developers using a specific programming language such as Python, Javascript, R, or Java interact with Dataverse APIs in a manner that is idiomatic for their language. For example, a Python programmer may want to +A client library helps developers using a specific programming language such as Python, Javascript, R, or Java interact with Dataverse Platform APIs in a manner that is idiomatic for their language. For example, a Python programmer may want to |Start| A good starting point is the :doc:`client-libraries` section, followed by the :doc:`getting-started` section. -Developers of Dataverse Itself -++++++++++++++++++++++++++++++ +Developers of The Dataverse Software Itself ++++++++++++++++++++++++++++++++++++++++++++ -Developers working on Dataverse itself use Dataverse APIs when adding features, fixing bugs, and testing those features and bug fixes. +Developers working on the Dataverse Software itself use the APIs when adding features, fixing bugs, and testing those features and bug fixes. |Start| A good starting point is the :doc:`/developers/testing` section of the Developer Guide. @@ -178,7 +178,7 @@ See :doc:`auth`. Lists of Dataverse APIs ~~~~~~~~~~~~~~~~~~~~~~~ -- :doc:`search`: For searching dataverses, datasets, and files. +- :doc:`search`: For searching dataverse collections, datasets, and files. - :doc:`dataaccess`: For downloading and subsetting data. - :doc:`native-api`: For performing most tasks that are possible in the GUI. See :doc:`getting-started` for the most common commands which operate on endpoints with names like: @@ -208,7 +208,7 @@ Please note that some APIs are only documented in other guides that are more sui Client Libraries ~~~~~~~~~~~~~~~~ -See :doc:`client-libraries` for how to use Dataverse APIs from Python, R, and Java. +See :doc:`client-libraries` for how to use Dataverse Platform APIs from Python, R, Java, and Javascript. Examples ~~~~~~~~ @@ -225,11 +225,11 @@ See :doc:`faq`. Getting Help ------------ -Dataverse API questions are on topic in all the usual places: +Dataverse Platform API questions are on topic in all the usual places: - The dataverse-community Google Group: https://groups.google.com/forum/#!forum/dataverse-community -- Dataverse community calls: https://dataverse.org/community-calls -- The Dataverse chat room: http://chat.dataverse.org -- The Dataverse ticketing system: support@dataverse.org +- Dataverse Project community calls: https://dataverse.org/community-calls +- The Dataverse Project chat room: http://chat.dataverse.org +- The Dataverse Project ticketing system: support@dataverse.org After your question has been answered, you are welcome to help improve the :doc:`faq` section of this guide. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c9301ae5999..d420a41568f 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1,7 +1,7 @@ Native API ========== -Dataverse 4 exposes most of its GUI functionality via a REST-based API. This section describes that functionality. Most API endpoints require an API token that can be passed as the ``X-Dataverse-key`` HTTP header or in the URL as the ``key`` query parameter. +The Dataverse Platform exposes most of its GUI functionality via a REST-based API. This section describes that functionality. Most API endpoints require an API token that can be passed as the ``X-Dataverse-key`` HTTP header or in the URL as the ``key`` query parameter. .. note:: |CORS| Some API endpoint allow CORS_ (cross-origin resource sharing), which makes them usable from scripts running in web browsers. These endpoints are marked with a *CORS* badge. @@ -9,25 +9,25 @@ Dataverse 4 exposes most of its GUI functionality via a REST-based API. This sec .. _CORS: https://www.w3.org/TR/cors/ -.. warning:: Dataverse 4's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. +.. warning:: The Dataverse Platform's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. .. contents:: |toctitle| :local: -Dataverses ----------- +Dataverse Collections +--------------------- .. _create-dataverse-api: -Create a Dataverse -~~~~~~~~~~~~~~~~~~ +Create a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A dataverse is a container for datasets and other dataverses as explained in the :doc:`/user/dataverse-management` section of the User Guide. +A Dataverse collection is a container for datasets and other Dataverse collections as explained in the :doc:`/user/dataverse-management` section of the User Guide. -The steps for creating a dataverse are: +The steps for creating a Dataverse collection are: -- Prepare a JSON file containing the name, description, etc, of the dataverse you'd like to create. -- Figure out the alias or database id of the "parent" dataverse into which you will be creating your new dataverse. +- Prepare a JSON file containing the name, description, etc, of the Dataverse collection you'd like to create. +- Figure out the alias or database id of the "parent" Dataverse collection into which you will be creating your new Dataverse collection. - Execute a curl command or equivalent. Download :download:`dataverse-complete.json <../_static/api/dataverse-complete.json>` file and modify it to suit your needs. The fields ``name``, ``alias``, and ``dataverseContacts`` are required. The controlled vocabulary for ``dataverseType`` is the following: @@ -46,7 +46,7 @@ Download :download:`dataverse-complete.json <../_static/api/dataverse-complete.j The curl command below assumes you have kept the name "dataverse-complete.json" and that this file is in your current working directory. -Next you need to figure out the alias or database id of the "parent" dataverse into which you will be creating your new dataverse. Out of the box the top level dataverse has an alias of "root" and a database id of "1" but your installation may vary. The easiest way to determine the alias of your root dataverse is to click "Advanced Search" and look at the URL. You may also choose a parent under the root. +Next you need to figure out the alias or database id of the "parent" Dataverse collection into which you will be creating your new Dataverse collection. Out of the box the top level Dataverse collection has an alias of "root" and a database id of "1" but your installation may vary. The easiest way to determine the alias of your root Dataverse collection is to click "Advanced Search" and look at the URL. You may also choose a parent under the root. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -64,16 +64,16 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root --upload-file dataverse-complete.json -You should expect an HTTP 200 response and JSON beginning with "status":"OK" followed by a representation of the newly-created dataverse. +You should expect an HTTP 200 response and JSON beginning with "status":"OK" followed by a representation of the newly-created Dataverse collection. .. _view-dataverse: -View a Dataverse -~~~~~~~~~~~~~~~~ +View a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| View a JSON representation of the dataverse identified by ``$id``. ``$id`` can be the database ID of the dataverse, its alias, or the special value ``:root`` for the root dataverse. +|CORS| View a JSON representation of the Dataverse collection identified by ``$id``. ``$id`` can be the database ID of the Dataverse collection, its alias, or the special value ``:root`` for the root Dataverse collection. -To view a published dataverse: +To view a published Dataverse collection: .. code-block:: bash @@ -88,7 +88,7 @@ The fully expanded example above (without environment variables) looks like this curl https://demo.dataverse.org/api/dataverses/root -To view an unpublished dataverse: +To view an unpublished Dataverse collection: .. code-block:: bash @@ -104,12 +104,12 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root -Delete a Dataverse -~~~~~~~~~~~~~~~~~~ +Delete a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Before you may delete a dataverse you must first delete or move all of its contents elsewhere. +Before you may delete a Dataverse collection you must first delete or move all of its contents elsewhere. -Deletes the dataverse whose database ID or alias is given: +Deletes the Dataverse collection whose database ID or alias is given: .. code-block:: bash @@ -127,10 +127,10 @@ The fully expanded example above (without environment variables) looks like this .. _show-contents-of-a-dataverse-api: -Show Contents of a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Show Contents of a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Lists all the dataverses and datasets directly under a dataverse (direct children only, not recursive) specified by database id or alias. If you pass your API token and have access, unpublished dataverses and datasets will be included in the response. +|CORS| Lists all the Dataverse collections and datasets directly under a Dataverse collection (direct children only, not recursive) specified by database id or alias. If you pass your API token and have access, unpublished Dataverse collections and datasets will be included in the response. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -148,10 +148,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/contents -Report the data (file) size of a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Report the data (file) size of a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Shows the combined size in bytes of all the files uploaded into the dataverse ``id``: +Shows the combined size in bytes of all the files uploaded into the Dataverse collection ``id``: .. code-block:: bash @@ -167,13 +167,13 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/storagesize -The size of published and unpublished files will be summed both in the dataverse specified and beneath all its sub-dataverses, recursively. -By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by Dataverse - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. +The size of published and unpublished files will be summed both in the Dataverse collection specified and beneath all its sub-collections, recursively. +By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by the Dataverse installation - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. -List Roles Defined in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Roles Defined in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -All the roles defined directly in the dataverse identified by ``id``: +All the roles defined directly in the Dataverse collection identified by ``id``: .. code-block:: bash @@ -189,10 +189,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/roles -List Facets Configured for a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Facets Configured for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| List all the facets for a given dataverse ``id``: +|CORS| List all the facets for a given Dataverse collection ``id``: .. code-block:: bash @@ -208,10 +208,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/facets -Set Facets for a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Set Facets for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Assign search facets for a given dataverse identified by ``id``: +Assign search facets for a given Dataverse collection identified by ``id``: .. code-block:: bash @@ -229,10 +229,10 @@ The fully expanded example above (without environment variables) looks like this Where ``facets.json`` contains a JSON encoded list of metadata keys (e.g. ``["authorName","authorAffiliation"]``). -Create a New Role in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create a New Role in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Creates a new role under dataverse ``id``. Needs a json file with the role description: +Creates a new role under Dataverse collection ``id``. Needs a json file with the role description: .. code-block:: bash @@ -261,10 +261,10 @@ Where ``roles.json`` looks like this:: .. _list-role-assignments-on-a-dataverse-api: -List Role Assignments in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Role Assignments in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -List all the role assignments at the given dataverse: +List all the role assignments at the given Dataverse collection: .. code-block:: bash @@ -280,10 +280,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/assignments -Assign Default Role to User Creating a Dataset in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Assign Default Role to User Creating a Dataset in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Assign a default role to a user creating a dataset in a dataverse ``id`` where ``roleAlias`` is the database alias of the role to be assigned: +Assign a default role to a user creating a dataset in a Dataverse collection ``id`` where ``roleAlias`` is the database alias of the role to be assigned: .. code-block:: bash @@ -300,12 +300,12 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X PUT https://demo.dataverse.org/api/dataverses/root/defaultContributorRole/curator -Note: You may use "none" as the ``ROLE_ALIAS``. This will prevent a user who creates a dataset from having any role on that dataset. It is not recommended for dataverses with human contributors. +Note: You may use "none" as the ``ROLE_ALIAS``. This will prevent a user who creates a dataset from having any role on that dataset. It is not recommended for Dataverse collections with human contributors. .. _assign-role-on-a-dataverse-api: -Assign a New Role on a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Assign a New Role on a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Assigns a new role, based on the POSTed JSON: @@ -332,8 +332,8 @@ POSTed JSON example (the content of ``role.json`` file):: .. _revoke-role-on-a-dataverse-api: -Delete Role Assignment from a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Delete Role Assignment from a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Delete the assignment whose id is ``$id``: @@ -352,12 +352,12 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X DELETE https://demo.dataverse.org/api/dataverses/root/assignments/6 -List Metadata Blocks Defined on a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Metadata Blocks Defined on a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Get the metadata blocks defined on a dataverse which determine which field are available to authors when they create and edit datasets within that dataverse. This feature is described in :ref:`general-information` section of Dataverse Management of the User Guide. +|CORS| Get the metadata blocks defined on a Dataverse collection which determine which field are available to authors when they create and edit datasets within that Dataverse collection. This feature is described in :ref:`general-information` section of Dataverse Collection Management of the User Guide. -Please note that an API token is only required if the dataverse has not been published. +Please note that an API token is only required if the Dataverse collection has not been published. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -375,12 +375,12 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/metadatablocks -Define Metadata Blocks for a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Define Metadata Blocks for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can define the metadata blocks available to authors within a dataverse. +You can define the metadata blocks available to authors within a Dataverse collection. -The metadata blocks that are available with a default installation of Dataverse are in :download:`define-metadatablocks.json <../_static/api/define-metadatablocks.json>` (also shown below) and you should download this file and edit it to meet your needs. Please note that the "citation" metadata block is required. You must have "EditDataverse" permission on the dataverse. +The metadata blocks that are available with a default Dataverse installation are in :download:`define-metadatablocks.json <../_static/api/define-metadatablocks.json>` (also shown below) and you should download this file and edit it to meet your needs. Please note that the "citation" metadata block is required. You must have "EditDataverse" permission on the Dataverse collection. .. literalinclude:: ../_static/api/define-metadatablocks.json @@ -400,10 +400,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-type:application/json" --upload-file define-metadatablocks.json https://demo.dataverse.org/api/dataverses/root/metadatablocks -Determine if a Dataverse Inherits Its Metadata Blocks from Its Parent -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Determine if a Dataverse Collection Inherits Its Metadata Blocks from Its Parent +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get whether the dataverse is a metadata block root, or does it uses its parent blocks: +Get whether the Dataverse collection is a metadata block root, or does it uses its parent blocks: .. code-block:: bash @@ -419,10 +419,10 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/metadatablocks/isRoot -Configure a Dataverse to Inherit Its Metadata Blocks from Its Parent -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configure a Dataverse Collection to Inherit Its Metadata Blocks from Its Parent +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Set whether the dataverse is a metadata block root, or does it uses its parent blocks. Possible +Set whether the Dataverse collection is a metadata block root, or does it uses its parent blocks. Possible values are ``true`` and ``false`` (both are valid JSON expressions): .. code-block:: bash @@ -444,8 +444,8 @@ The fully expanded example above (without environment variables) looks like this .. _create-dataset-command: -Create a Dataset in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create a Dataset in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A dataset is a container for files as explained in the :doc:`/user/dataset-management` section of the User Guide. @@ -457,11 +457,11 @@ To create a dataset, you must supply a JSON file that contains at least the foll - Description - Subject -As a starting point, you can download :download:`dataset-finch1.json <../../../../scripts/search/tests/data/dataset-finch1.json>` and modify it to meet your needs. (In addition to this minimal example, you can download :download:`dataset-create-new-all-default-fields.json <../../../../scripts/api/data/dataset-create-new-all-default-fields.json>` which populates all of the metadata fields that ship with Dataverse.) +As a starting point, you can download :download:`dataset-finch1.json <../../../../scripts/search/tests/data/dataset-finch1.json>` and modify it to meet your needs. (In addition to this minimal example, you can download :download:`dataset-create-new-all-default-fields.json <../../../../scripts/api/data/dataset-create-new-all-default-fields.json>` which populates all of the metadata fields that ship with a Dataverse installation.) The curl command below assumes you have kept the name "dataset-finch1.json" and that this file is in your current working directory. -Next you need to figure out the alias or database id of the "parent" dataverse into which you will be creating your new dataset. Out of the box the top level dataverse has an alias of "root" and a database id of "1" but your installation may vary. The easiest way to determine the alias of your root dataverse is to click "Advanced Search" and look at the URL. You may also choose a parent dataverse under the root dataverse. +Next you need to figure out the alias or database id of the "parent" Dataverse collection into which you will be creating your new dataset. Out of the box the top level Dataverse collection has an alias of "root" and a database id of "1" but your installation may vary. The easiest way to determine the alias of your root Dataverse collection is to click "Advanced Search" and look at the URL. You may also choose a parent Dataverse collection under the root Dataverse collection. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -481,14 +481,14 @@ The fully expanded example above (without the environment variables) looks like You should expect an HTTP 200 ("OK") response and JSON indicating the database ID and Persistent ID (PID such as DOI or Handle) that has been assigned to your newly created dataset. -.. note:: Only a Dataverse account with superuser permissions is allowed to include files when creating a dataset via this API. Adding files this way only adds their file metadata to the database, you will need to manually add the physical files to the file system. +.. note:: Only a Dataverse installation account with superuser permissions is allowed to include files when creating a dataset via this API. Adding files this way only adds their file metadata to the database, you will need to manually add the physical files to the file system. -Import a Dataset into a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Import a Dataset into a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: This action requires a Dataverse account with super-user permissions. +.. note:: This action requires a Dataverse installation account with super-user permissions. -To import a dataset with an existing persistent identifier (PID), the dataset's metadata should be prepared in Dataverse's native JSON format. The PID is provided as a parameter at the URL. The following line imports a dataset with the PID ``PERSISTENT_IDENTIFIER`` to Dataverse, and then releases it: +To import a dataset with an existing persistent identifier (PID), the dataset's metadata should be prepared in Dataverse installation's native JSON format. The PID is provided as a parameter at the URL. The following line imports a dataset with the PID ``PERSISTENT_IDENTIFIER`` to the Dataverse installation, and then releases it: .. code-block:: bash @@ -507,7 +507,7 @@ The fully expanded example above (without environment variables) looks like this The ``pid`` parameter holds a persistent identifier (such as a DOI or Handle). The import will fail if no PID is provided, or if the provided PID fails validation. -The optional ``release`` parameter tells Dataverse to immediately publish the dataset. If the parameter is changed to ``no``, the imported dataset will remain in ``DRAFT`` status. +The optional ``release`` parameter tells the Dataverse installation to immediately publish the dataset. If the parameter is changed to ``no``, the imported dataset will remain in ``DRAFT`` status. The JSON format is the same as that supported by the native API's :ref:`create dataset command`, although it also allows packages. For example: @@ -519,17 +519,17 @@ Before calling the API, make sure the data files referenced by the ``POST``\ ed .. warning:: - * This API does not cover staging files (with correct contents, checksums, sizes, etc.) in the corresponding places in the Dataverse filestore. + * This API does not cover staging files (with correct contents, checksums, sizes, etc.) in the corresponding places in the Dataverse installation's filestore. * This API endpoint does not support importing *files'* persistent identifiers. - * A Dataverse server can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. + * A Dataverse installation can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. -Import a Dataset into a Dataverse with a DDI file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Import a Dataset into a Dataverse Installation with a DDI file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: This action requires a Dataverse account with super-user permissions. +.. note:: This action requires a Dataverse installation account with super-user permissions. -To import a dataset with an existing persistent identifier (PID), you have to provide the PID as a parameter at the URL. The following line imports a dataset with the PID ``PERSISTENT_IDENTIFIER`` to Dataverse, and then releases it: +To import a dataset with an existing persistent identifier (PID), you have to provide the PID as a parameter at the URL. The following line imports a dataset with the PID ``PERSISTENT_IDENTIFIER`` to the Dataverse installation, and then releases it: .. code-block:: bash @@ -548,21 +548,21 @@ The fully expanded example above (without environment variables) looks like this The optional ``pid`` parameter holds a persistent identifier (such as a DOI or Handle). The import will fail if the provided PID fails validation. -The optional ``release`` parameter tells Dataverse to immediately publish the dataset. If the parameter is changed to ``no``, the imported dataset will remain in ``DRAFT`` status. +The optional ``release`` parameter tells the Dataverse installation to immediately publish the dataset. If the parameter is changed to ``no``, the imported dataset will remain in ``DRAFT`` status. The file is a DDI xml file. .. warning:: * This API does not handle files related to the DDI file. - * A Dataverse server can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. + * A Dataverse installation can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. .. _publish-dataverse-api: -Publish a Dataverse -~~~~~~~~~~~~~~~~~~~ +Publish a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to publish a dataverse, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. +In order to publish a Dataverse collection, you must know either its "alias" (which the GUI calls an "identifier") or its database ID. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -585,7 +585,7 @@ You should expect a 200 ("OK") response and JSON output. Datasets -------- -**Note** Creation of new datasets is done with a ``POST`` onto dataverses. See Dataverses_ section. +**Note** Creation of new datasets is done with a ``POST`` onto a Dataverse collection. See Dataverse_ section. **Note** In all commands below, dataset versions can be referred to as: From 3ea71dd0c3fbfd008831d82e84142ad498ab5ca9 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 8 Jan 2021 13:25:08 -0500 Subject: [PATCH 028/274] Checksum/unf truncation, click to copy, for dataset pg and dataverse pg, plus file metadata code clean up [ref #5210] --- .../search/SearchIncludeFragment.java | 14 ---------- src/main/webapp/editFilesFragment.xhtml | 4 +-- src/main/webapp/file-info-fragment.xhtml | 15 ++++++----- src/main/webapp/filesFragment.xhtml | 19 +++++++------ .../resources/js/dv_rebind_bootstrap_ui.js | 27 ++++++++++++------- src/main/webapp/search-include-fragment.xhtml | 25 +++++++++-------- 6 files changed, 51 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index ac70072b7bb..9219e8b3664 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1236,20 +1236,6 @@ public String dataFileSizeDisplay(DataFile datafile) { } - public String dataFileChecksumDisplay(DataFile datafile) { - if (datafile == null) { - return ""; - } - - if (datafile.getChecksumValue() != null && !StringUtils.isEmpty(datafile.getChecksumValue())) { - if (datafile.getChecksumType() != null) { - return " " + datafile.getChecksumType() + ": " + datafile.getChecksumValue() + " "; - } - } - - return ""; - } - public void setDisplayCardValues() { Set harvestedDatasetIds = null; diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 5ad2dc6dab0..e4e6950c87b 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -374,8 +374,8 @@ -
- +
+
diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index b10383b55ed..4a96327c736 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -52,18 +52,21 @@ -
- + #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} + title="#{bundle['file.metaData.checksum.copy']} #{fileMetadata.dataFile.checksumValue}">#{fileMetadata.dataFile.checksumValue}
- +
- + #{fileMetadata.dataFile.unf}
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 9f958cd451c..0a77a40d881 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -372,6 +372,15 @@ +
@@ -471,16 +480,6 @@ disabled="#{DatasetPage.locked}"> #{bundle['file.requestAccess']} - - diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 76d6d02a906..131f3634b60 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -156,22 +156,29 @@ function sharrre(){ } /* - * Truncate file checksums + * Truncate file checksums + click to copy */ function checksumTruncate(){ $('span.checksum-truncate').each(function () { var checksumText = $(this).text(); - if (checksumText.length > 35) { - $(this).text(checksumText.substr(0, 8) + '...' + checksumText.substr(checksumText.length-3, checksumText.length)); + if (checksumText.length > 25) { + // COUNT ":" IN UNF VERSION LABEL + var labelCount = (checksumText.match(/:/g) || []).length; + // COUNT "=" IN UNF SUFFIX + var suffixCount = (checksumText.match(/=/g) || []).length; + // TRUNCATE MIDDLE W/ "..." + FIRST/LAST 3 CHARACTERS + $(this).text(checksumText.substr(0, labelCount + 3) + '...' + checksumText.substr((checksumText.length - suffixCount - 3), checksumText.length)); } + // ADD CLIPBOARD ICON $(this).append(' '); - var clipboard = new Clipboard('span.checksum-truncate'); - clipboard.on('success', function(e) { - console.log(e); - }); - clipboard.on('error', function(e) { - console.log(e); - }); + }); + // USE CLIPBOARD.JS TO COPY + var clipboard = new Clipboard('span.checksum-truncate'); + clipboard.on('success', function(e) { + console.log(e); + }); + clipboard.on('error', function(e) { + console.log(e); }); } diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index 3260495c4e7..956e57242af 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -601,23 +601,26 @@
-
- +
+
-
- - - - - - - +
+ + + + + + #{result.fileChecksumValue} -
From 5efe3f0b14d61453ce649d5a1958a000aee162b7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 8 Jan 2021 15:49:51 -0500 Subject: [PATCH 029/274] placeholder release notes --- doc/release-notes/820-non-ascii-chars-in-search.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/release-notes/820-non-ascii-chars-in-search.md diff --git a/doc/release-notes/820-non-ascii-chars-in-search.md b/doc/release-notes/820-non-ascii-chars-in-search.md new file mode 100644 index 00000000000..c449a68cb11 --- /dev/null +++ b/doc/release-notes/820-non-ascii-chars-in-search.md @@ -0,0 +1,8 @@ +### Search with non-ascii characters + +Many languages include characters that have close analogs in ascii, e.g. (á, à, â, ç, é, è, ê, ë, í, ó, ö, ú, ù, û, ü…). This release changes the default Solr configuration to allow search to match words based on these associations, e.g. a search for Mercè would match the word Merce in a Dataset, and vice versa. This should generally be helpful, but can result in false positives.,e.g. "canon" will be found searching for "cañon". + +## Upgrade Instructions + +1. You will need to replace or modify your `schema.xml` and restart solr. Re-indexing is required to get full-functionality from this change - the standard instructions for an incremental reindex could be added here. + \ No newline at end of file From 7ad07b6871647c06dc2c4effa6f16a213ece44c3 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 8 Jan 2021 16:47:18 -0500 Subject: [PATCH 030/274] lots of updates in a very long file --- doc/sphinx-guides/source/api/native-api.rst | 106 ++++++++++---------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d420a41568f..9abe5dea3b4 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -585,7 +585,7 @@ You should expect a 200 ("OK") response and JSON output. Datasets -------- -**Note** Creation of new datasets is done with a ``POST`` onto a Dataverse collection. See Dataverse_ section. +**Note** Creation of new datasets is done with a ``POST`` onto a Dataverse collection. See the Dataverse Collections section above. **Note** In all commands below, dataset versions can be referred to as: @@ -760,12 +760,12 @@ The fully expanded example above (without environment variables) looks like this Schema.org JSON-LD ^^^^^^^^^^^^^^^^^^ -Please note that the ``schema.org`` format has changed in backwards-incompatible ways after Dataverse 4.9.4: +Please note that the ``schema.org`` format has changed in backwards-incompatible ways after Dataverse software version 4.9.4: - "description" was a single string and now it is an array of strings. - "citation" was an array of strings and now it is an array of objects. -Both forms are valid according to Google's Structured Data Testing Tool at https://search.google.com/structured-data/testing-tool . (This tool will report "The property affiliation is not recognized by Google for an object of type Thing" and this known issue is being tracked at https://github.com/IQSS/dataverse/issues/5029 .) Schema.org JSON-LD is an evolving standard that permits a great deal of flexibility. For example, https://schema.org/docs/gs.html#schemaorg_expected indicates that even when objects are expected, it's ok to just use text. As with all metadata export formats, we will try to keep the Schema.org JSON-LD format Dataverse emits backward-compatible to made integrations more stable, despite the flexibility that's afforded by the standard. +Both forms are valid according to Google's Structured Data Testing Tool at https://search.google.com/structured-data/testing-tool . (This tool will report "The property affiliation is not recognized by Google for an object of type Thing" and this known issue is being tracked at https://github.com/IQSS/dataverse/issues/5029 .) Schema.org JSON-LD is an evolving standard that permits a great deal of flexibility. For example, https://schema.org/docs/gs.html#schemaorg_expected indicates that even when objects are expected, it's ok to just use text. As with all metadata export formats, we will try to keep the Schema.org JSON-LD format your Dataverse installation emits backward-compatible to made integrations more stable, despite the flexibility that's afforded by the standard. List Files in a Dataset ~~~~~~~~~~~~~~~~~~~~~~~ @@ -830,9 +830,9 @@ Update Metadata For a Dataset Updates the metadata for a dataset. If a draft of the dataset already exists, the metadata of that draft is overwritten; otherwise, a new draft is created with this metadata. -You must download a JSON representation of the dataset, edit the JSON you download, and then send the updated JSON to the Dataverse server. +You must download a JSON representation of the dataset, edit the JSON you download, and then send the updated JSON to the Dataverse installation. -For example, after making your edits, your JSON file might look like :download:`dataset-update-metadata.json <../_static/api/dataset-update-metadata.json>` which you would send to Dataverse like this: +For example, after making your edits, your JSON file might look like :download:`dataset-update-metadata.json <../_static/api/dataset-update-metadata.json>` which you would send to the Dataverse installation like this: .. code-block:: bash @@ -868,7 +868,7 @@ Now that the resulting JSON file only contains the ``metadataBlocks`` key, you c vi dataset-update-metadata.json -Now that you've made edits to the metadata in your JSON file, you can send it to Dataverse as described above. +Now that you've made edits to the metadata in your JSON file, you can send it to a Dataverse installation as described above. Edit Dataset Metadata ~~~~~~~~~~~~~~~~~~~~~ @@ -933,7 +933,7 @@ For these deletes your JSON file must include an exact match of those dataset fi Publish a Dataset ~~~~~~~~~~~~~~~~~ -When publishing a dataset it's good to be aware of Dataverse's versioning system, which is described in the :doc:`/user/dataset-management` section of the User Guide. +When publishing a dataset it's good to be aware of the Dataverse Platform's versioning system, which is described in the :doc:`/user/dataset-management` section of the User Guide. If this is the first version of the dataset, its version number will be set to ``1.0``. Otherwise, the new dataset version number is determined by the most recent version number and the ``type`` parameter. Passing ``type=minor`` increases the minor version number (2.3 is updated to 2.4). Passing ``type=major`` increases the major version number (2.3 is updated to 3.0). (Superusers can pass ``type=updatecurrent`` to update metadata without changing the version number.) @@ -1180,7 +1180,7 @@ The fully expanded example above (without environment variables) looks like this You should expect a 201 ("CREATED") response and JSON indicating the database id that has been assigned to your newly uploaded file. -Please note that it's possible to "trick" Dataverse into giving a file a content type (MIME type) of your choosing. For example, you can make a text file be treated like a video file with ``-F 'file=@README.txt;type=video/mpeg4'``, for example. If Dataverse does not properly detect a file type, specifying the content type via API like this a potential workaround. +Please note that it's possible to "trick" a Dataverse installation into giving a file a content type (MIME type) of your choosing. For example, you can make a text file be treated like a video file with ``-F 'file=@README.txt;type=video/mpeg4'``, for example. If the Dataverse installation does not properly detect a file type, specifying the content type via API like this a potential workaround. The curl syntax above to upload a file is tricky and a Python version is provided below. (Please note that it depends on libraries such as "requests" that you may need to install but this task is out of scope for this guide.) Here are some parameters you can set in the script: @@ -1200,7 +1200,7 @@ In practice, you only need one the ``dataset_id`` or the ``persistentId``. The e # -------------------------------------------------- # Update the 4 params below to run this code # -------------------------------------------------- - dataverse_server = 'https://your dataverse server' # no trailing slash + dataverse_server = 'https://your dataverse installation' # no trailing slash api_key = 'api key' dataset_id = 1 # database id of the dataset persistentId = 'doi:10.5072/FK2/6XACVA' # doi or hdl of the dataset @@ -1286,7 +1286,7 @@ The fully expanded example above (without environment variables) looks like this curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/datasets/24/storagesize The size of published and unpublished files will be summed in the dataset specified. -By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by Dataverse - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. Because this deals with unpublished files the token supplied must have permission to view unpublished drafts. +By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by the Dataverse installation - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. Because this deals with unpublished files the token supplied must have permission to view unpublished drafts. Get the size of Downloading all the files of a Dataset Version @@ -1363,7 +1363,7 @@ The review process can sometimes resemble a tennis match, with the authors submi Link a Dataset ~~~~~~~~~~~~~~ -Creates a link between a dataset and a dataverse (see :ref:`dataset-linking` section of Dataverse Management in the User Guide for more information): +Creates a link between a dataset and a Dataverse collection (see :ref:`dataset-linking` section of Dataverse Collection Management in the User Guide for more information): .. code-block:: bash @@ -1494,7 +1494,7 @@ If the dataset is not locked (or if there is no lock of the specified type), the Dataset Metrics ~~~~~~~~~~~~~~~ -Please note that these dataset level metrics are only available if support for Make Data Count has been enabled in your installation of Dataverse. See the :ref:`Dataset Metrics ` in the :doc:`/user/dataset-management` section of the User Guide and the :doc:`/admin/make-data-count` section of the Admin Guide for details. +Please note that these dataset level metrics are only available if support for Make Data Count has been enabled in your Dataverse installation. See the :ref:`Dataset Metrics ` in the :doc:`/user/dataset-management` section of the User Guide and the :doc:`/admin/make-data-count` section of the Admin Guide for details. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -1652,7 +1652,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/24/destroy -Calling the destroy endpoint is permanent and irreversible. It will remove the dataset and its datafiles, then re-index the parent dataverse in Solr. This endpoint requires the API token of a superuser. +Calling the destroy endpoint is permanent and irreversible. It will remove the dataset and its datafiles, then re-index the parent Dataverse collection in Solr. This endpoint requires the API token of a superuser. Configure a Dataset to Use a Specific File Store ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1773,7 +1773,7 @@ The fully expanded example above (without environment variables) looks like this Reingest a File ~~~~~~~~~~~~~~~ -Attempt to ingest an existing datafile as tabular data. This API can be used on a file that was not ingested as tabular back when it was uploaded. For example, a Stata v.14 file that was uploaded before ingest support for Stata 14 was added (in Dataverse v.4.9). It can also be used on a file that failed to ingest due to a bug in the ingest plugin that has since been fixed (hence the name "reingest"). +Attempt to ingest an existing datafile as tabular data. This API can be used on a file that was not ingested as tabular back when it was uploaded. For example, a Stata v.14 file that was uploaded before ingest support for Stata 14 was added (in Dataverse software v.4.9). It can also be used on a file that failed to ingest due to a bug in the ingest plugin that has since been fixed (hence the name "reingest"). Note that this requires "superuser" credentials. @@ -1816,7 +1816,7 @@ Note: at present, the API cannot be used on a file that's already successfully i Redetect File Type ~~~~~~~~~~~~~~~~~~ -Dataverse uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have Dataverse attempt to redetect the file type. +The Dataverse Platform uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have the Dataverse Platform attempt to redetect the file type. When using the curl command below, you can pass ``dryRun=true`` if you don't want any changes to be saved to the database. Change this to ``dryRun=false`` (or omit it) to save the change. @@ -2301,7 +2301,7 @@ In order to delete a token use:: Builtin Users ------------- -Builtin users are known as "Username/Email and Password" users in the :doc:`/user/account` of the User Guide. Dataverse stores a password (encrypted, of course) for these users, which differs from "remote" users such as Shibboleth or OAuth users where the password is stored elsewhere. See also :ref:`auth-modes` section of Configuration in the Installation Guide. It's a valid configuration of Dataverse to not use builtin users at all. +Builtin users are known as "Username/Email and Password" users in the :doc:`/user/account` of the User Guide. The Dataverse installation stores a password (encrypted, of course) for these users, which differs from "remote" users such as Shibboleth or OAuth users where the password is stored elsewhere. See also :ref:`auth-modes` section of Configuration in the Installation Guide. It's a valid configuration of a Dataverse installation to not use builtin users at all. Create a Builtin User ~~~~~~~~~~~~~~~~~~~~~ @@ -2321,10 +2321,10 @@ Optionally, you may use a third query parameter "sendEmailNotification=false" to Roles ----- -Create a New Role in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Create a New Role in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Creates a new role in dataverse object whose Id is ``dataverseIdtf`` (that's an id/alias):: +Creates a new role in Dataverse collection object whose Id is ``dataverseIdtf`` (that's an id/alias):: POST http://$SERVER/api/roles?dvo=$dataverseIdtf&key=$apiKey @@ -2348,9 +2348,9 @@ Explicit Groups Create New Explicit Group ~~~~~~~~~~~~~~~~~~~~~~~~~ -Explicit groups list their members explicitly. These groups are defined in dataverses, which is why their API endpoint is under ``api/dataverses/$id/``, where ``$id`` is the id of the dataverse. +Explicit groups list their members explicitly. These groups are defined in Dataverse collections, which is why their API endpoint is under ``api/dataverses/$id/``, where ``$id`` is the id of the Dataverse collection. -Create a new explicit group under dataverse ``$id``:: +Create a new explicit group under Dataverse collection ``$id``:: POST http://$server/api/dataverses/$id/groups @@ -2362,31 +2362,31 @@ Data being POSTed is json-formatted description of the group:: "aliasInOwner":"ccs" } -List Explicit Groups in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Explicit Groups in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -List explicit groups under dataverse ``$id``:: +List explicit groups under Dataverse collection ``$id``:: GET http://$server/api/dataverses/$id/groups -Show Single Group in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Show Single Group in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Show group ``$groupAlias`` under dataverse ``$dv``:: GET http://$server/api/dataverses/$dv/groups/$groupAlias -Update Group in a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Update Group in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Update group ``$groupAlias`` under dataverse ``$dv``. The request body is the same as the create group one, except that the group alias cannot be changed. Thus, the field ``aliasInOwner`` is ignored. :: +Update group ``$groupAlias`` under Dataverse collection ``$dv``. The request body is the same as the create group one, except that the group alias cannot be changed. Thus, the field ``aliasInOwner`` is ignored. :: PUT http://$server/api/dataverses/$dv/groups/$groupAlias -Delete Group from a Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Delete Group from a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Delete group ``$groupAlias`` under dataverse ``$dv``:: +Delete group ``$groupAlias`` under Dataverse collection ``$dv``:: DELETE http://$server/api/dataverses/$dv/groups/$groupAlias @@ -2421,10 +2421,10 @@ Management of Shibboleth groups via API is documented in the :doc:`/installation Info ---- -Show Dataverse Version and Build Number -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Show Dataverse Software Version and Build Number +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Get the Dataverse version. The response contains the version and build numbers: +|CORS| Get the Dataverse installation version. The response contains the version and build numbers: .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. @@ -2440,10 +2440,10 @@ The fully expanded example above (without environment variables) looks like this curl https://demo.dataverse.org/api/info/version -Show Dataverse Server Name -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Show Dataverse Installation Server Name +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the server name. This is useful when a Dataverse system is composed of multiple app servers behind a load balancer: +Get the server name. This is useful when a Dataverse installation is composed of multiple app servers behind a load balancer: .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. @@ -2611,7 +2611,7 @@ The fully expanded example above (without environment variables) looks like this Delete a PID ~~~~~~~~~~~~ -Delete PID (this is only possible for PIDs that are in the "draft" state) and within Dataverse, set ``globalidcreatetime`` to null and ``identifierregistered`` to false. A superuser API token is required. +Delete PID (this is only possible for PIDs that are in the "draft" state) and within a Dataverse installation, set ``globalidcreatetime`` to null and ``identifierregistered`` to false. A superuser API token is required. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. @@ -3005,14 +3005,14 @@ List all role assignments of a role assignee (i.e. a user or a group):: Note that ``identifier`` can contain slashes (e.g. ``&ip/localhost-users``). -List Permissions a User Has on a Dataverse or Dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List Permissions a User Has on a Dataverse Collection or Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -List permissions a user (based on API Token used) has on a dataverse or dataset:: +List permissions a user (based on API Token used) has on a Dataverse collection or dataset:: GET http://$SERVER/api/admin/permissions/$identifier -The ``$identifier`` can be a dataverse alias or database id or a dataset persistent ID or database id. +The ``$identifier`` can be a Dataverse collection alias or database id or a dataset persistent ID or database id. Show Role Assignee ~~~~~~~~~~~~~~~~~~ @@ -3028,7 +3028,7 @@ The ``$identifier`` should start with an ``@`` if it's a user. Groups start with Saved Search ~~~~~~~~~~~~ -The Saved Search, Linked Dataverses, and Linked Datasets features shipped with Dataverse 4.0, but as a "`superuser-only `_" because they are **experimental** (see `#1364 `_, `#1813 `_, `#1840 `_, `#1890 `_, `#1939 `_, `#2167 `_, `#2186 `_, `#2053 `_, and `#2543 `_). The following API endpoints were added to help people with access to the "admin" API make use of these features in their current form. There is a known issue (`#1364 `_) that once a link to a dataverse or dataset is created, it cannot be removed (apart from database manipulation and reindexing) which is why a ``DELETE`` endpoint for saved searches is neither documented nor functional. The Linked Dataverses feature is `powered by Saved Search `_ and therefore requires that the "makelinks" endpoint be executed on a periodic basis as well. +The Saved Search, Linked Dataverses, and Linked Datasets features shipped with Dataverse 4.0, but as a "`superuser-only `_" because they are **experimental** (see `#1364 `_, `#1813 `_, `#1840 `_, `#1890 `_, `#1939 `_, `#2167 `_, `#2186 `_, `#2053 `_, and `#2543 `_). The following API endpoints were added to help people with access to the "admin" API make use of these features in their current form. There is a known issue (`#1364 `_) that once a link to a Dataverse collection or dataset is created, it cannot be removed (apart from database manipulation and reindexing) which is why a ``DELETE`` endpoint for saved searches is neither documented nor functional. The Linked Dataverse collections feature is `powered by Saved Search `_ and therefore requires that the "makelinks" endpoint be executed on a periodic basis as well. List all saved searches. :: @@ -3038,11 +3038,11 @@ List a saved search by database id. :: GET http://$SERVER/api/admin/savedsearches/$id -Execute a saved search by database id and make links to dataverses and datasets that are found. The JSON response indicates which dataverses and datasets were newly linked versus already linked. The ``debug=true`` query parameter adds to the JSON response extra information about the saved search being executed (which you could also get by listing the saved search). :: +Execute a saved search by database id and make links to Dataverse collections and datasets that are found. The JSON response indicates which Dataverse collections and datasets were newly linked versus already linked. The ``debug=true`` query parameter adds to the JSON response extra information about the saved search being executed (which you could also get by listing the saved search). :: PUT http://$SERVER/api/admin/savedsearches/makelinks/$id?debug=true -Execute all saved searches and make links to dataverses and datasets that are found. ``debug`` works as described above. This happens automatically with a timer. For details, see :ref:`saved-search-timer` in the Admin Guide. :: +Execute all saved searches and make links to Dataverse collections and datasets that are found. ``debug`` works as described above. This happens automatically with a timer. For details, see :ref:`saved-search-timer` in the Admin Guide. :: PUT http://$SERVER/api/admin/savedsearches/makelinks/all?debug=true @@ -3100,7 +3100,7 @@ if validation fails, will report the specific database entity and the offending If the optional argument ``variables=true`` is specified, the API will also validate the metadata associated with any tabular data files found in the dataset specified. (For example: an invalid or empty variable name). -Validate all the datasets in the Dataverse, report any constraint violations found:: +Validate all the datasets in the Dataverse installation, report any constraint violations found:: curl $SERVER_URL/api/admin/validate/datasets @@ -3119,7 +3119,7 @@ This API streams its output in real time, i.e. it will start producing the outpu ] } -Note that if you are attempting to validate a very large number of datasets in your Dataverse, this API may time out - subject to the timeout limit set in your app server configuration. If this is a production Dataverse instance serving large amounts of data, you most likely have that timeout set to some high value already. But if you need to increase it, it can be done with the asadmin command. For example:: +Note that if you are attempting to validate a very large number of datasets in your Dataverse installation, this API may time out - subject to the timeout limit set in your app server configuration. If this is a production Dataverse installation serving large amounts of data, you most likely have that timeout set to some high value already. But if you need to increase it, it can be done with the asadmin command. For example:: asadmin set server-config.network-config.protocols.protocol.http-listener-1.http.request-timeout-seconds=3600 @@ -3161,7 +3161,7 @@ Unset the default workflow for ``triggerType``. After this call, dataset release DELETE http://$SERVER/api/admin/workflows/default/$triggerType -Set the whitelist of IP addresses separated by a semicolon (``;``) allowed to resume workflows. Request body is a list of IP addresses allowed to send "resume workflow" messages to this Dataverse instance:: +Set the whitelist of IP addresses separated by a semicolon (``;``) allowed to resume workflows. Request body is a list of IP addresses allowed to send "resume workflow" messages to this Dataverse installation:: PUT http://$SERVER/api/admin/workflows/ip-whitelist @@ -3190,13 +3190,13 @@ Clear a specific metric cache. Currently this must match the name of the row in CORS -Inherit Dataverse Role Assignments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Inherit Dataverse Collection Role Assignments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Recursively applies the role assignments of the specified dataverse, for the roles specified by the ``:InheritParentRoleAssignments`` setting, to all dataverses contained within it:: +Recursively applies the role assignments of the specified Dataverse collection, for the roles specified by the ``:InheritParentRoleAssignments`` setting, to all Dataverse collections contained within it:: GET http://$SERVER/api/admin/dataverse/{dataverse alias}/addRoleAssignmentsToChildren -Note: setting ``:InheritParentRoleAssignments`` will automatically trigger inheritance of the parent dataverse's role assignments for a newly created dataverse. Hence this API call is intended as a way to update existing child dataverses or to update children after a change in role assignments has been made on a parent dataverse. +Note: setting ``:InheritParentRoleAssignments`` will automatically trigger inheritance of the parent Dataverse collection's role assignments for a newly created Dataverse collection. Hence this API call is intended as a way to update existing child Dataverse collections or to update children after a change in role assignments has been made on a parent Dataverse collection. From 283769218eb7ddebc54c6478047caf798cd5afeb Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 8 Jan 2021 16:54:35 -0500 Subject: [PATCH 031/274] updates to SWORD doc --- doc/sphinx-guides/source/api/sword.rst | 48 +++++++++++++------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index f031f78f822..0d09f9ac047 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -9,11 +9,11 @@ SWORD_ stands for "Simple Web-service Offering Repository Deposit" and is a "pro About ----- -Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`. +Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse Software 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`. -Dataverse implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. +The Dataverse Platform implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. -As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse 4.0 datasets can also be created via JSON using the "native" API. SWORD is limited to the dozen or so fields listed below in the crosswalk, but the native API allows you to populate all metadata fields available in Dataverse. +As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse Software 4.0 datasets can also be created via JSON using the "native" API. SWORD is limited to the dozen or so fields listed below in the crosswalk, but the native API allows you to populate all metadata fields available in a Dataverse installation. .. _SWORD: http://en.wikipedia.org/wiki/SWORD_%28protocol%29 @@ -30,7 +30,7 @@ Backward incompatible changes For better security than in DVN 3.x, usernames and passwords are no longer accepted. The use of an API token is required. -Differences in Dataverse 4 from DVN 3.x lead to a few minor backward incompatible changes in the Dataverse implementation of SWORD, which are listed below. Old ``v1`` URLs should continue to work but the ``Service Document`` will contain a deprecation warning and responses will contain ``v1.1`` URLs. See also :ref:`known-issues`. +Differences in Dataverse Platform 4 from DVN 3.x lead to a few minor backward incompatible changes in the Dataverse Platform implementation of SWORD, which are listed below. Old ``v1`` URLs should continue to work but the ``Service Document`` will contain a deprecation warning and responses will contain ``v1.1`` URLs. See also :ref:`known-issues`. - Newly required fields when creating/editing datasets for compliance with the `Joint Declaration for Data Citation principles `_. @@ -40,16 +40,16 @@ Differences in Dataverse 4 from DVN 3.x lead to a few minor backward incompatibl - Deaccessioning is no longer supported. An alternative will be developed at https://github.com/IQSS/dataverse/issues/778 -- The Service Document will show a single API Terms of Use rather than root level and dataverse level Deposit Terms of Use. +- The Service Document will show a single API Terms of Use rather than root level and Dataverse collection level Deposit Terms of Use. New features as of v1.1 ----------------------- -- Dataverse 4 supports API tokens and requires them to be used for APIs instead of a username and password. In the ``curl`` examples below, you will see ``curl -u $API_TOKEN:`` showing that you should send your API token as the username and nothing as the password. For example, ``curl -u 54b143b5-d001-4254-afc0-a1c0f6a5b5a7:``. +- The Dataverse Software supports API tokens and requires them to be used for APIs instead of a username and password. In the ``curl`` examples below, you will see ``curl -u $API_TOKEN:`` showing that you should send your API token as the username and nothing as the password. For example, ``curl -u 54b143b5-d001-4254-afc0-a1c0f6a5b5a7:``. -- SWORD operations no longer require "admin" permission. In order to use any SWORD operation in DVN 3.x, you had to be an "admin" on a dataverse (the container for your dataset) and similar rules were applied in Dataverse 4.4 and earlier (the ``EditDataverse`` permission was required). The SWORD API has now been fully integrated with the Dataverse 4 permission model such that any action you have permission to perform in the GUI or "native" API you are able to perform via SWORD. This means that even a user with a "Contributor" role can operate on datasets via SWORD. Note that users with the "Contributor" role do not have the ``PublishDataset`` permission and will not be able publish their datasets via any mechanism, GUI or API. +- SWORD operations no longer require "admin" permission. In order to use any SWORD operation in DVN 3.x, you had to be an "admin" on a Dataverse collection (the container for your dataset) and similar rules were applied in Dataverse Software 4.4 and earlier (the ``EditDataverse`` permission was required). The SWORD API has now been fully integrated with the Dataverse Software permission model such that any action you have permission to perform in the GUI or "native" API you are able to perform via SWORD. This means that even a user with a "Contributor" role can operate on datasets via SWORD. Note that users with the "Contributor" role do not have the ``PublishDataset`` permission and will not be able publish their datasets via any mechanism, GUI or API. -- Dataverses can be published via SWORD. +- Dataverse collections can be published via SWORD. - Datasets versions will only be increased to the next minor version (i.e. 1.1) rather than a major version (2.0) if possible. This depends on the nature of the change. Adding or removing a file, for example, requires a major version bump. @@ -73,14 +73,14 @@ curl examples Retrieve SWORD service document ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The service document enumerates the dataverses ("collections" from a SWORD perspective) the user can deposit data into. The "collectionPolicy" element for each dataverse contains the Terms of Use. Any user with an API token can use this API endpoint. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. +The service document enumerates the Dataverse collections (also "collections" from a SWORD perspective) the user can deposit data into. The "collectionPolicy" element for each Dataverse collections contains the Terms of Use. Any user with an API token can use this API endpoint. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. ``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/service-document`` Create a dataset with an Atom entry ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To create a dataset, you must have the "Dataset Creator" role (the ``AddDataset`` permission) on a dataverse. Practically speaking, you should first retrieve the service document to list the dataverses into which you are authorized to deposit data. +To create a dataset, you must have the "Dataset Creator" role (the ``AddDataset`` permission) on a Dataverse collection. Practically speaking, you should first retrieve the service document to list the Dataverse collections into which you are authorized to deposit data. ``curl -u $API_TOKEN: --data-binary "@path/to/atom-entry-study.xml" -H "Content-Type: application/atom+xml" https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS`` @@ -88,10 +88,10 @@ Example Atom entry (XML) .. literalinclude:: sword-atom-entry.xml -Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse DB Element Crosswalk -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse Project DB Element Crosswalk +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ -|DC (terms: namespace) | Dataverse DB Element | Required | Note | +|DC (terms: namespace) | Dataverse Software DB Element | Required | Note | +=============================+==============================================+==============+=============================================================================================================================================================+ |dcterms:title | title | Y | Title of the Dataset. | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -115,17 +115,17 @@ Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse DB Element Crosswalk +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ |dcterms:coverage | otherGeographicCoverage | | General information on the geographic coverage of the Dataset. | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ -|dcterms:license | license | | Set the license to CC0 (default in Dataverse for new Datasets), otherwise enter "NONE" and fill in the dcterms:rights field. | +|dcterms:license | license | | Set the license to CC0 (default in a Dataverse installation for new Datasets), otherwise enter "NONE" and fill in the dcterms:rights field. | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ |dcterms:rights | termsofuse | | If not using CC0, enter any terms of use or restrictions for the Dataset. | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ |dcterms:isReferencedBy | publicationCitation | | The publication (journal article, book, other work) that uses this dataset (include citation, permanent identifier (DOI), and permanent URL). | +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+ -List datasets in a dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List datasets in a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You must have permission to add datasets in a dataverse (the dataverse should appear in the service document) to list the datasets inside. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. +You must have permission to add datasets in a Dataverse collection (the Dataverse collection should appear in the service document) to list the datasets inside. Institution-wide Shibboleth groups are not respected because membership in such a group can only be set via a browser. ``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS`` @@ -173,17 +173,17 @@ You must have the ``DeleteDatasetDraft`` permission (Contributor role or above s ``curl -u $API_TOKEN: -i -X DELETE https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/study/doi:TEST/12345`` -Determine if a dataverse has been published -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Determine if a Dataverse Collection has been published +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This API endpoint is the same as the "list datasets in a dataverse" endpoint documented above and the same permissions apply but it is documented here separately to point out that you can look for a boolean called ``dataverseHasBeenReleased`` to know if a dataverse has been released, which is required for publishing a dataset. +This API endpoint is the same as the "list datasets in a Dataverse collection" endpoint documented above and the same permissions apply but it is documented here separately to point out that you can look for a boolean called ``dataverseHasBeenReleased`` to know if a Dataverse collection has been released, which is required for publishing a dataset. ``curl -u $API_TOKEN: https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/collection/dataverse/$DATAVERSE_ALIAS`` -Publish a dataverse -~~~~~~~~~~~~~~~~~~~ +Publish a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``cat /dev/null`` and ``--data-binary @-`` arguments are used to send zero-length content to the API, which is required by the upstream library to process the ``In-Progress: false`` header. You must have the ``PublishDataverse`` permission (Admin role) on the dataverse to publish it. +The ``cat /dev/null`` and ``--data-binary @-`` arguments are used to send zero-length content to the API, which is required by the upstream library to process the ``In-Progress: false`` header. You must have the ``PublishDataverse`` permission (Admin role) on the Dataverse collection to publish it. ``cat /dev/null | curl -u $API_TOKEN: -X POST -H "In-Progress: false" --data-binary @- https://$HOSTNAME/dvn/api/data-deposit/v1.1/swordv2/edit/dataverse/$DATAVERSE_ALIAS`` @@ -203,7 +203,7 @@ Known issues - The Service Document does not honor groups within groups: https://github.com/IQSS/dataverse/issues/3056 -- Should see all the fields filled in for a dataset regardless of what the parent dataverse specifies: https://github.com/IQSS/dataverse/issues/756 +- Should see all the fields filled in for a dataset regardless of what the parent Dataverse collection specifies: https://github.com/IQSS/dataverse/issues/756 - SWORD 2.0 Profile 6.4 "Retrieving the content" has not been implemented: https://github.com/IQSS/dataverse/issues/183 From d0397848abc540bc92345a38449c8cf74710d0de Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 11 Jan 2021 10:12:15 -0500 Subject: [PATCH 032/274] Applied truncation to UNF, moved tabular metadata labels to bundle [ref #5210 #6685] --- .../iq/dataverse/search/SearchIncludeFragment.java | 6 +++--- src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/filesFragment.xhtml | 3 ++- .../webapp/resources/js/dv_rebind_bootstrap_ui.js | 11 +++++++++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 9219e8b3664..61859700077 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1213,14 +1213,14 @@ public String tabularDataDisplayInfo(DataFile datafile) { Long varNumber = datatable.getVarQuantity(); Long obsNumber = datatable.getCaseQuantity(); if (varNumber != null && varNumber.intValue() != 0) { - ret = ret.concat(varNumber + " Variables"); + ret = ret.concat(varNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.variables")); if (obsNumber != null && obsNumber.intValue() != 0) { - ret = ret.concat(", " + obsNumber + " Observations"); + ret = ret.concat(", " + obsNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.observations")); } ret = ret.concat(" - "); } if (unf != null && !unf.equals("")) { - ret = ret.concat("UNF: " + unf); + ret = ret.concat(BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.unf") + ": " + unf); } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b27ab777ba6..8c8a9e5310a 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1558,6 +1558,7 @@ file.rsyncUpload.httpUploadDisabledDueToRsyncFileExistingAndPublished=HTTP uploa file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttp=Upload with rsync + SSH is disabled for this dataset because you have already uploaded files via HTTP. If you would like to switch to rsync upload, then you must first remove all uploaded files from this dataset. Once this dataset is published, the chosen upload method is permanently locked in. file.rsyncUpload.rsyncUploadDisabledDueFileUploadedViaHttpAndPublished=Upload with rsync + SSH is disabled for this dataset because you have already uploaded files via HTTP and published the dataset. file.metaData.checksum.copy=Click to copy +file.metaData.dataFile.dataTab.unf=UNF file.metaData.dataFile.dataTab.variables=Variables file.metaData.dataFile.dataTab.observations=Observations file.addDescription=Add file description... diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 0a77a40d881..5c3cdcdc24f 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -35,11 +35,12 @@ + 25) { // COUNT ":" IN UNF VERSION LABEL - var labelCount = (checksumText.match(/:/g) || []).length; + var labelIndex = checksumText.indexOf(':'); // COUNT "=" IN UNF SUFFIX var suffixCount = (checksumText.match(/=/g) || []).length; + // TRUNCATE MIDDLE W/ "..." + FIRST/LAST 3 CHARACTERS - $(this).text(checksumText.substr(0, labelCount + 3) + '...' + checksumText.substr((checksumText.length - suffixCount - 3), checksumText.length)); + // CHECK IF UNF LABEL + if (labelIndex > 0) { + $(this).text(checksumText.substr(0, (labelIndex + 3) + 3) + '...' + checksumText.substr((checksumText.length - suffixCount - 3), checksumText.length)); + } + else { + $(this).text(checksumText.substr(0, 3) + '...' + checksumText.substr((checksumText.length - 3), checksumText.length)); + } } // ADD CLIPBOARD ICON $(this).append(' '); From d16b6ee4bdb8b5fb282200391bcf78963d8f7247 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 11 Jan 2021 12:15:36 -0500 Subject: [PATCH 033/274] This is the reason search cards were not fully initialized for tabular files. (#7312) --- src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index a9a16e60ae2..094e20edf28 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -497,7 +497,7 @@ public DataFile findCheapAndEasy(Long id) { // If content type indicates it's tabular data, spend 2 extra queries // looking up the data table and tabular tags objects: - if (MIME_TYPE_TSV.equalsIgnoreCase(contentType)) { + if (MIME_TYPE_TSV.equalsIgnoreCase(contentType) || MIME_TYPE_TSV_ALT.equalsIgnoreCase(contentType)) { Object[] dtResult; try { dtResult = (Object[]) em.createNativeQuery("SELECT ID, UNF, CASEQUANTITY, VARQUANTITY, ORIGINALFILEFORMAT, ORIGINALFILESIZE FROM dataTable WHERE DATAFILE_ID = " + id).getSingleResult(); From 064edbae1c88109a1d54dcad5385983e9cfc8c1c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 13:29:42 -0500 Subject: [PATCH 034/274] unf updates --- doc/sphinx-guides/source/developers/unf/index.rst | 6 +++--- doc/sphinx-guides/source/developers/unf/unf-v5.rst | 4 ++-- doc/sphinx-guides/source/developers/unf/unf-v6.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/developers/unf/index.rst b/doc/sphinx-guides/source/developers/unf/index.rst index af9265a7fc0..237d1e9f138 100644 --- a/doc/sphinx-guides/source/developers/unf/index.rst +++ b/doc/sphinx-guides/source/developers/unf/index.rst @@ -20,10 +20,10 @@ computed. The signature is thus independent of the storage format. E.g., the same data object stored in, say, SPSS and Stata, will have the same UNF. -Early versions of Dataverse were using the first released +Early versions of the Dataverse Software were using the first released implementation of the UNF algorithm (v.3, implemented in R). Starting -with Dataverse 2.0 and throughout the 3.* lifecycle, UNF v.5 -(implemented in Java) was used. Dataverse 4.0 uses the latest release, +with Dataverse Software version 2.0 and throughout the 3.* lifecycle, UNF v.5 +(implemented in Java) was used. The Dataverse Software version 4.0 uses the latest release, UNF v.6. Two parallel implementation, in R and Java, will be available, for cross-validation. diff --git a/doc/sphinx-guides/source/developers/unf/unf-v5.rst b/doc/sphinx-guides/source/developers/unf/unf-v5.rst index 29b4556b1f9..cbbde2e18f8 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v5.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v5.rst @@ -8,8 +8,8 @@ UNF Version 5 **Important Update:** -UNF Version 5 has been in use by the Dataverse project since 2009. It was built into every version of the DVN, starting with 2.0 and up to 3.6.2. However, some problems were recently found in that implementation. Namely, in certain cases data normalization is not implemented fully to the spec. UNF signatures it generates are still reasonably strong statistically; however, this means that at least some of our signatures are not independently verifiable. I.e., if somebody fully implements their own version of UNF calculator, for certain datasets it would calculate signatures different from those generated by the DVN. Unless of course they implement it with the exact same bugs as ours. +UNF Version 5 has been in use by the Dataverse Project since 2009. It was built into every version of the DVN, starting with 2.0 and up to 3.6.2. However, some problems were recently found in that implementation. Namely, in certain cases data normalization is not implemented fully to the spec. UNF signatures it generates are still reasonably strong statistically; however, this means that at least some of our signatures are not independently verifiable. I.e., if somebody fully implements their own version of UNF calculator, for certain datasets it would calculate signatures different from those generated by the DVN. Unless of course they implement it with the exact same bugs as ours. -To address this, the Project is about to release UNF Version 6. The release date is still being discussed. It may coincide with the release of Dataverse 4.0. Alternatively, the production version of DVN 3.6.3 may get upgraded to use UNF v6 prior to that. This will be announced shortly. In the process, we are solving another problem with UNF v5 - this time we've made an effort to offer very implementer-friendly documentation that describes the algorithm fully and unambiguously. So if you are interested in implementing your own version of a UNF calculator, (something we would like to encourage!) please proceed directly to the Version 6 documentation. +To address this, the Project is about to release UNF Version 6. The release date is still being discussed. It may coincide with the release of Dataverse Software version 4.0. Alternatively, the production version of DVN 3.6.3 may get upgraded to use UNF v6 prior to that. This will be announced shortly. In the process, we are solving another problem with UNF v5 - this time we've made an effort to offer very implementer-friendly documentation that describes the algorithm fully and unambiguously. So if you are interested in implementing your own version of a UNF calculator, (something we would like to encourage!) please proceed directly to the Version 6 documentation. Going forward, we are going to offer a preserved version of the Version 5 library and, possibly, an online UNF v5 calculator, for the purposes of validating vectors and data sets for which published Version 5 UNFs exist. diff --git a/doc/sphinx-guides/source/developers/unf/unf-v6.rst b/doc/sphinx-guides/source/developers/unf/unf-v6.rst index b5c898a9a08..9648bae47c8 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v6.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v6.rst @@ -160,7 +160,7 @@ For example, to specify a non-default precision the parameter it is specified us Note: we do not recommend truncating character strings at fewer bytes than the default ``128`` (the ``X`` parameter). At the very least this number **must** be high enough so that the printable UNFs of individual variables or files are not truncated, when calculating combined UNFs of files or datasets, respectively. -It should also be noted that the Dataverse application never calculates UNFs with any non-default parameters. And we are not aware of anyone else actually doing so. If you are considering creating your own implementation of the UNF, it may be worth trying to create a simplified, defaults-only version first. Such an implementation would be sufficient to independently verify Dataverse-produced UNFs, among other things. +It should also be noted that the Dataverse Software never calculates UNFs with any non-default parameters. And we are not aware of anyone else actually doing so. If you are considering creating your own implementation of the UNF, it may be worth trying to create a simplified, defaults-only version first. Such an implementation would be sufficient to independently verify Dataverse Software-produced UNFs, among other things. .. _note2: From 1a87191b978d3f06b0f7ff0d9b23bebfc8b8511e Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 13:39:23 -0500 Subject: [PATCH 035/274] more updates --- .../source/developers/aux-file-support.rst | 2 +- .../source/developers/big-data-support.rst | 58 +++++++++---------- .../source/developers/coding-style.rst | 6 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/doc/sphinx-guides/source/developers/aux-file-support.rst b/doc/sphinx-guides/source/developers/aux-file-support.rst index be21b56c245..260c4493590 100644 --- a/doc/sphinx-guides/source/developers/aux-file-support.rst +++ b/doc/sphinx-guides/source/developers/aux-file-support.rst @@ -1,7 +1,7 @@ Auxiliary File Support ====================== -Auxiliary file support is experimental. Auxiliary files in Dataverse are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (OpenDP.io). In future versions, this approach may become more broadly used and supported. +Auxiliary file support is experimental. Auxiliary files in the Dataverse Software are being added to support depositing and downloading differentially private metadata, as part of the OpenDP project (OpenDP.io). In future versions, this approach may become more broadly used and supported. Adding an Auxiliary File to a Datafile -------------------------------------- diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 11cfbfcde08..37a14b63f34 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -11,18 +11,18 @@ Various components need to be installed and/or configured for big data support. S3 Direct Upload and Download ----------------------------- -A lightweight option for supporting file sizes beyond a few gigabytes - a size that can cause performance issues when uploaded through the Dataverse server itself - is to configure an S3 store to provide direct upload and download via 'pre-signed URLs'. When these options are configured, file uploads and downloads are made directly to and from a configured S3 store using secure (https) connections that enforce Dataverse's access controls. (The upload and download URLs are signed with a unique key that only allows access for a short time period and Dataverse will only generate such a URL if the user has permission to upload/download the specific file in question.) +A lightweight option for supporting file sizes beyond a few gigabytes - a size that can cause performance issues when uploaded through a Dataverse installation itself - is to configure an S3 store to provide direct upload and download via 'pre-signed URLs'. When these options are configured, file uploads and downloads are made directly to and from a configured S3 store using secure (https) connections that enforce a Dataverse installation's access controls. (The upload and download URLs are signed with a unique key that only allows access for a short time period and a Dataverse installation will only generate such a URL if the user has permission to upload/download the specific file in question.) -This option can handle files >40GB and could be appropriate for files up to a TB. Other options can scale farther, but this option has the advantages that it is simple to configure and does not require any user training - uploads and downloads are done via the same interface as normal uploads to Dataverse. +This option can handle files >40GB and could be appropriate for files up to a TB. Other options can scale farther, but this option has the advantages that it is simple to configure and does not require any user training - uploads and downloads are done via the same interface as normal uploads to a Dataverse installation. -To configure these options, an administrator must set two JVM options for the Dataverse server using the same process as for other configuration options: +To configure these options, an administrator must set two JVM options for the Dataverse installation using the same process as for other configuration options: ``./asadmin create-jvm-options "-Ddataverse.files..download-redirect=true"`` ``./asadmin create-jvm-options "-Ddataverse.files..upload-redirect=true"`` -With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific dataverses) while configuring only direct download, or no direct access for another store. +With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific Dataverse collections) while configuring only direct download, or no direct access for another store. The direct upload option now switches between uploading the file in one piece (up to 1 GB by default) and sending it as multiple parts. The default can be changed by setting: @@ -32,12 +32,12 @@ For AWS, the minimum allowed part size is 5*1024*1024 bytes and the maximum is 5 It is also possible to set file upload size limits per store. See the :MaxFileUploadSizeInBytes setting described in the :doc:`/installation/config` guide. -At present, one potential drawback for direct-upload is that files are only partially 'ingested', tabular and FITS files are processed, but zip files are not unzipped, and the file contents are not inspected to evaluate their mimetype. This could be appropriate for large files, or it may be useful to completely turn off ingest processing for performance reasons (ingest processing requires a copy of the file to be retrieved by Dataverse from the S3 store). A store using direct upload can be configured to disable all ingest processing for files above a given size limit: +At present, one potential drawback for direct-upload is that files are only partially 'ingested', tabular and FITS files are processed, but zip files are not unzipped, and the file contents are not inspected to evaluate their mimetype. This could be appropriate for large files, or it may be useful to completely turn off ingest processing for performance reasons (ingest processing requires a copy of the file to be retrieved by the Dataverse installation from the S3 store). A store using direct upload can be configured to disable all ingest processing for files above a given size limit: ``./asadmin create-jvm-options "-Ddataverse.files..ingestsizelimit="`` -**IMPORTANT:** One additional step that is required to enable direct uploads via Dataverse and for direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. +**IMPORTANT:** One additional step that is required to enable direct uploads via a Dataverse installation and for direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. The example below shows how to enable CORS rules (to support upload and download) on a bucket using the AWS CLI command line tool. Note that you may want to limit the AllowedOrigins and/or AllowedHeaders further. https://github.com/GlobalDataverseCommunityConsortium/dataverse-previewers/wiki/Using-Previewers-with-download-redirects-from-S3 has some additional information about doing this. ``aws s3api put-bucket-cors --bucket --cors-configuration file://cors.json`` @@ -59,7 +59,7 @@ with the contents of the file cors.json as follows: Alternatively, you can enable CORS using the AWS S3 web interface, using json-encoded rules as in the example above. -Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 Tags to aid in identifying/removing such files. Upon upload, files are given a "dv-status":"temp" tag which is removed when the dataset changes are saved and the new file(s) are added in Dataverse. Note that not all S3 implementations support Tags: Minio does not. WIth such stores, direct upload works, but Tags are not used. +Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 Tags to aid in identifying/removing such files. Upon upload, files are given a "dv-status":"temp" tag which is removed when the dataset changes are saved and the new file(s) are added in the Dataverse installation. Note that not all S3 implementations support Tags: Minio does not. WIth such stores, direct upload works, but Tags are not used. Data Capture Module (DCM) ------------------------- @@ -71,37 +71,37 @@ DCM was developed and tested using Glassfish but these docs have been updated wi Install a DCM ~~~~~~~~~~~~~ -Installation instructions can be found at https://github.com/sbgrid/data-capture-module/blob/master/doc/installation.md. Note that shared storage (posix or AWS S3) between Dataverse and your DCM is required. You cannot use a DCM with Swift at this point in time. +Installation instructions can be found at https://github.com/sbgrid/data-capture-module/blob/master/doc/installation.md. Note that shared storage (posix or AWS S3) between your Dataverse installation and your DCM is required. You cannot use a DCM with Swift at this point in time. .. FIXME: Explain what ``dataverse.files.dcm-s3-bucket-name`` is for and what it has to do with ``dataverse.files.s3.bucket-name``. -Once you have installed a DCM, you will need to configure two database settings on the Dataverse side. These settings are documented in the :doc:`/installation/config` section of the Installation Guide: +Once you have installed a DCM, you will need to configure two database settings on the Dataverse installation side. These settings are documented in the :doc:`/installation/config` section of the Installation Guide: - ``:DataCaptureModuleUrl`` should be set to the URL of a DCM you installed. - ``:UploadMethods`` should include ``dcm/rsync+ssh``. -This will allow your Dataverse installation to communicate with your DCM, so that Dataverse can download rsync scripts for your users. +This will allow your Dataverse installation to communicate with your DCM, so that your Dataverse installation can download rsync scripts for your users. -Downloading rsync scripts via Dataverse API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Downloading rsync scripts via Your Dataverse Installation's API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The rsync script can be downloaded from Dataverse via API using an authorized API token. In the curl example below, substitute ``$PERSISTENT_ID`` with a DOI or Handle: +The rsync script can be downloaded from your Dataverse installation via API using an authorized API token. In the curl example below, substitute ``$PERSISTENT_ID`` with a DOI or Handle: ``curl -H "X-Dataverse-key: $API_TOKEN" $DV_BASE_URL/api/datasets/:persistentId/dataCaptureModule/rsync?persistentId=$PERSISTENT_ID`` -How a DCM reports checksum success or failure to Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +How a DCM reports checksum success or failure to your Dataverse Installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Once the user uploads files to a DCM, that DCM will perform checksum validation and report to Dataverse the results of that validation. The DCM must be configured to pass the API token of a superuser. The implementation details, which are subject to change, are below. +Once the user uploads files to a DCM, that DCM will perform checksum validation and report to your Dataverse installation the results of that validation. The DCM must be configured to pass the API token of a superuser. The implementation details, which are subject to change, are below. -The JSON that a DCM sends to Dataverse on successful checksum validation looks something like the contents of :download:`checksumValidationSuccess.json <../_static/installation/files/root/big-data-support/checksumValidationSuccess.json>` below: +The JSON that a DCM sends to your Dataverse installation on successful checksum validation looks something like the contents of :download:`checksumValidationSuccess.json <../_static/installation/files/root/big-data-support/checksumValidationSuccess.json>` below: .. literalinclude:: ../_static/installation/files/root/big-data-support/checksumValidationSuccess.json :language: json - ``status`` - The valid strings to send are ``validation passed`` and ``validation failed``. -- ``uploadFolder`` - This is the directory on disk where Dataverse should attempt to find the files that a DCM has moved into place. There should always be a ``files.sha`` file and a least one data file. ``files.sha`` is a manifest of all the data files and their checksums. The ``uploadFolder`` directory is inside the directory where data is stored for the dataset and may have the same name as the "identifier" of the persistent id (DOI or Handle). For example, you would send ``"uploadFolder": "DNXV2H"`` in the JSON file when the absolute path to this directory is ``/usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/DNXV2H/DNXV2H``. -- ``totalSize`` - Dataverse will use this value to represent the total size in bytes of all the files in the "package" that's created. If 360 data files and one ``files.sha`` manifest file are in the ``uploadFolder``, this value is the sum of the 360 data files. +- ``uploadFolder`` - This is the directory on disk where your Dataverse installation should attempt to find the files that a DCM has moved into place. There should always be a ``files.sha`` file and a least one data file. ``files.sha`` is a manifest of all the data files and their checksums. The ``uploadFolder`` directory is inside the directory where data is stored for the dataset and may have the same name as the "identifier" of the persistent id (DOI or Handle). For example, you would send ``"uploadFolder": "DNXV2H"`` in the JSON file when the absolute path to this directory is ``/usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/DNXV2H/DNXV2H``. +- ``totalSize`` - Your Dataverse installation will use this value to represent the total size in bytes of all the files in the "package" that's created. If 360 data files and one ``files.sha`` manifest file are in the ``uploadFolder``, this value is the sum of the 360 data files. Here's the syntax for sending the JSON. @@ -115,12 +115,12 @@ Steps to set up a DCM mock for Development See instructions at https://github.com/sbgrid/data-capture-module/blob/master/doc/mock.md -Add Dataverse settings to use mock (same as using DCM, noted above): +Add Dataverse Installation settings to use mock (same as using DCM, noted above): - ``curl http://localhost:8080/api/admin/settings/:DataCaptureModuleUrl -X PUT -d "http://localhost:5000"`` - ``curl http://localhost:8080/api/admin/settings/:UploadMethods -X PUT -d "dcm/rsync+ssh"`` -At this point you should be able to download a placeholder rsync script. Dataverse is then waiting for news from the DCM about if checksum validation has succeeded or not. First, you have to put files in place, which is usually the job of the DCM. You should substitute "X1METO" for the "identifier" of the dataset you create. You must also use the proper path for where you store files in your dev environment. +At this point you should be able to download a placeholder rsync script. Your Dataverse installation is then waiting for news from the DCM about if checksum validation has succeeded or not. First, you have to put files in place, which is usually the job of the DCM. You should substitute "X1METO" for the "identifier" of the dataset you create. You must also use the proper path for where you store files in your dev environment. - ``mkdir /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO`` - ``mkdir /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO`` @@ -130,7 +130,7 @@ At this point you should be able to download a placeholder rsync script. Dataver -Now the files are in place and you need to send JSON to Dataverse with a success or failure message as described above. Make a copy of ``doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json`` and put the identifier in place such as "X1METO" under "uploadFolder"). Then use curl as described above to send the JSON. +Now the files are in place and you need to send JSON to your Dataverse installation with a success or failure message as described above. Make a copy of ``doc/sphinx-guides/source/_static/installation/files/root/big-data-support/checksumValidationSuccess.json`` and put the identifier in place such as "X1METO" under "uploadFolder"). Then use curl as described above to send the JSON. Troubleshooting ~~~~~~~~~~~~~~~ @@ -155,7 +155,7 @@ Optional steps for setting up the S3 Docker DCM Variant ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Before: the default bucket for DCM to hold files in S3 is named test-dcm. It is coded into `post_upload_s3.bash` (line 30). Change to a different bucket if needed. -- Also Note: With the new support for multiple file store in Dataverse, DCM requires a store with id="s3" and DCM will only work with this store. +- Also Note: With the new support for multiple file store in the Dataverse Software, DCM requires a store with id="s3" and DCM will only work with this store. - Add AWS bucket info to dcmsrv - Add AWS credentials to ``~/.aws/credentials`` @@ -164,7 +164,7 @@ Optional steps for setting up the S3 Docker DCM Variant - ``aws_access_key_id =`` - ``aws_secret_access_key =`` -- Dataverse configuration (on dvsrv): +- Dataverse installation configuration (on dvsrv): - Set S3 as the storage driver @@ -175,7 +175,7 @@ Optional steps for setting up the S3 Docker DCM Variant - ``./asadmin create-jvm-options "\-Ddataverse.files.s3.label=s3"`` - - Add AWS bucket info to Dataverse + - Add AWS bucket info to your Dataverse installation - Add AWS credentials to ``~/.aws/credentials`` - ``[default]`` @@ -187,13 +187,13 @@ Optional steps for setting up the S3 Docker DCM Variant - ``[default]`` - ``region = us-east-1`` - - Add the S3 bucket names to Dataverse + - Add the S3 bucket names to your Dataverse installation - - S3 bucket for Dataverse + - S3 bucket for your Dataverse installation - ``/usr/local/payara5/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.s3.bucket-name=iqsstestdcmbucket"`` - - S3 bucket for DCM (as Dataverse needs to do the copy over) + - S3 bucket for DCM (as your Dataverse installation needs to do the copy over) - ``/usr/local/payara5/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.dcm-s3-bucket-name=test-dcm"`` @@ -255,7 +255,7 @@ Configuring the RSAL Mock Info for configuring the RSAL Mock: https://github.com/sbgrid/rsal/tree/master/mocks -Also, to configure Dataverse to use the new workflow you must do the following (see also the :doc:`workflows` section): +Also, to configure your Dataverse installation to use the new workflow you must do the following (see also the :doc:`workflows` section): 1. Configure the RSAL URL: diff --git a/doc/sphinx-guides/source/developers/coding-style.rst b/doc/sphinx-guides/source/developers/coding-style.rst index f0a05991fc0..0c00f611a7f 100755 --- a/doc/sphinx-guides/source/developers/coding-style.rst +++ b/doc/sphinx-guides/source/developers/coding-style.rst @@ -2,7 +2,7 @@ Coding Style ============ -Like all development teams, the `Dataverse developers at IQSS `_ have their habits and styles when it comes to writing code. Let's attempt to get on the same page. :) +Like all development teams, the `Dataverse Project developers at IQSS `_ have their habits and styles when it comes to writing code. Let's attempt to get on the same page. :) .. contents:: |toctitle| :local: @@ -62,7 +62,7 @@ As you probably gathered from the :doc:`dev-environment` section, IQSS has stand Checking Your Formatting With Checkstyle ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The easiest way to adopt Dataverse coding style is to use Netbeans as your IDE, avoid change the default Netbeans formatting settings, and only reformat code you've changed, as described above. +The easiest way to adopt the Dataverse Project coding style is to use Netbeans as your IDE, avoid change the default Netbeans formatting settings, and only reformat code you've changed, as described above. If you do not use Netbeans, you are encouraged to check the formatting of your code using Checkstyle. @@ -97,7 +97,7 @@ Special strings should be defined as public constants. For example, ``DatasetFie Avoid Hard-Coding User-Facing Messaging in English ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -There is an ongoing effort to translate Dataverse into various languages. Look for "lang" or "languages" in the :doc:`/installation/config` section of the Installation Guide for details if you'd like to help or play around with this feature. +There is an ongoing effort to translate the Dataverse Software into various languages. Look for "lang" or "languages" in the :doc:`/installation/config` section of the Installation Guide for details if you'd like to help or play around with this feature. The translation effort is hampered if you hard code user-facing messages in English in the Java code. Put English strings in ``Bundle.properties`` and use ``BundleUtil`` to pull them out. This is especially important for messages that appear in the UI. We are aware that the API has many, many hard coded English strings in it. If you touch a method in the API and notice English strings, you are strongly encouraged to use that opportunity to move the English to ``Bundle.properties``. From 09b9f608ed51b16c075958ec7e807f7eb6c62b18 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 13:44:25 -0500 Subject: [PATCH 036/274] more updates --- .../source/developers/configuration.rst | 8 ++++---- doc/sphinx-guides/source/developers/containers.rst | 2 +- .../source/developers/dependencies.rst | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/sphinx-guides/source/developers/configuration.rst b/doc/sphinx-guides/source/developers/configuration.rst index d058a1e1705..9c971668201 100644 --- a/doc/sphinx-guides/source/developers/configuration.rst +++ b/doc/sphinx-guides/source/developers/configuration.rst @@ -4,7 +4,7 @@ Consuming Configuration .. contents:: |toctitle| :local: -Dataverse uses different types of configuration: +The Dataverse Software uses different types of configuration: 1. JVM system properties 2. Simple database value settings @@ -25,11 +25,11 @@ Developers have accessed the simple properties via 3. ``SystemConfig.xxx()`` for specially treated settings, maybe mixed from 1 and 2 and other sources. 4. ``SettingsWrapper``, reading from 2 and 3 for use in frontend pages based on JSF -As of Dataverse 5.3, we start to streamline our efforts into using a more consistent approach, also bringing joy and +As of Dataverse Software version 5.3, we start to streamline our efforts into using a more consistent approach, also bringing joy and happiness to all the system administrators out there. This will be done by adopting the use of `MicroProfile Config `_ over time. -So far we streamlined configuration of these Dataverse parts: +So far we streamlined configuration of these Dataverse Software parts: - ✅ Database Connection @@ -54,7 +54,7 @@ System administrators benefit from: - Lots of database settings have been introduced in the past, but should be more easily configurable and not rely on a database connection. -- Running Dataverse in containers gets much easier when configuration can be provisioned in a +- Running a Dataverse installation in containers gets much easier when configuration can be provisioned in a streamlined fashion, mitigating the need for scripting glue and distinguishing between setting types. - Classic installations have a profit, too: we can enable using a single config file, e.g. living in ``/etc/dataverse/config.properties``. diff --git a/doc/sphinx-guides/source/developers/containers.rst b/doc/sphinx-guides/source/developers/containers.rst index ecf82533160..64c7710f0f5 100755 --- a/doc/sphinx-guides/source/developers/containers.rst +++ b/doc/sphinx-guides/source/developers/containers.rst @@ -2,7 +2,7 @@ Docker, Kubernetes, and Containers ================================== -The Dataverse community is exploring the use of Docker, Kubernetes, and other container-related technologies. The primary community-lead projects to watch are: +The Dataverse Community is exploring the use of Docker, Kubernetes, and other container-related technologies. The primary community-lead projects to watch are: - https://github.com/IQSS/dataverse-kubernetes - https://github.com/IQSS/dataverse-docker diff --git a/doc/sphinx-guides/source/developers/dependencies.rst b/doc/sphinx-guides/source/developers/dependencies.rst index 599606e7714..09d29324d2c 100644 --- a/doc/sphinx-guides/source/developers/dependencies.rst +++ b/doc/sphinx-guides/source/developers/dependencies.rst @@ -5,13 +5,13 @@ Dependency Management .. contents:: |toctitle| :local: -Dataverse is (currently) a Jakarta EE 8 based application, that uses a lot of additional libraries for special purposes. +The Dataverse Software is (currently) a Jakarta EE 8 based application, that uses a lot of additional libraries for special purposes. This includes features like support for SWORD-API, S3 storage and many others. Besides the code that glues together the single pieces, any developer needs to describe used dependencies for the Maven-based build system. As is familiar to any Maven user, this happens inside the "Project Object Model" (POM) living in ``pom.xml`` at the root of the project repository. Recursive and convergent dependency resolution makes dependency -management with Maven very easy. But sometimes, in projects with many complex dependencies like Dataverse, you have +management with Maven very easy. But sometimes, in projects with many complex dependencies like the Dataverse software, you have to help Maven make the right choices. Terms @@ -19,7 +19,7 @@ Terms As a developer, you should familiarize yourself with the following terms: -- **Direct dependencies**: things *you use* yourself in your own code for Dataverse. +- **Direct dependencies**: things *you use* yourself in your own code for the Dataverse Software. - **Transitive dependencies**: things *others use* for things you use, pulled in recursively. See also: `Maven docs `_. @@ -167,7 +167,7 @@ A reduced example, only showing bits relevant to the above cases and usage of an import + Jackson is used by AWS SDK and others, but we also use it in the Dataverse Software. --> com.fasterxml.jackson jackson-bom @@ -175,7 +175,7 @@ A reduced example, only showing bits relevant to the above cases and usage of an import pom - + joda-time joda-time @@ -230,9 +230,9 @@ Repositories Maven receives all dependencies from *repositories*. Those can be public like `Maven Central `_ and others, but you can also use a private repository on premises or in the cloud. Last but not least, you can use -local repositories, which can live next to your application code (see ``local_lib`` dir within Dataverse codebase). +local repositories, which can live next to your application code (see ``local_lib`` dir within the Dataverse Software codebase). -Repositories are defined within the Dataverse POM like this: +Repositories are defined within the Dataverse Software POM like this: .. code:: xml From 1810940deefc7d6eb7bb8c299304c0e20684613f Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 14:18:44 -0500 Subject: [PATCH 037/274] Update deployment.rst --- .../source/developers/deployment.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/developers/deployment.rst b/doc/sphinx-guides/source/developers/deployment.rst index 3bb4b99913c..c65ad265add 100755 --- a/doc/sphinx-guides/source/developers/deployment.rst +++ b/doc/sphinx-guides/source/developers/deployment.rst @@ -2,15 +2,15 @@ Deployment ========== -Developers often only deploy Dataverse to their :doc:`dev-environment` but it can be useful to deploy Dataverse to cloud services such as Amazon Web Services (AWS). +Developers often only deploy the Dataverse software to their :doc:`dev-environment` but it can be useful to deploy the Dataverse software to cloud services such as Amazon Web Services (AWS). .. contents:: |toctitle| :local: -Deploying Dataverse to Amazon Web Services (AWS) ------------------------------------------------- +Deploying the Dataverse Software to Amazon Web Services (AWS) +------------------------------------------------------------- -We have written scripts to deploy Dataverse to Amazon Web Services (AWS) but they require some setup. +We have written scripts to deploy the Dataverse Software to Amazon Web Services (AWS) but they require some setup. Install AWS CLI ~~~~~~~~~~~~~~~ @@ -73,14 +73,14 @@ If you are having trouble configuring the files manually as described above, see Configure Ansible File (Optional) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to configure Dataverse settings such as the password of the dataverseAdmin user, download https://raw.githubusercontent.com/GlobalDataverseCommunityConsortium/dataverse-ansible/master/defaults/main.yml and edit the file to your liking. +In order to configure Dataverse installation settings such as the password of the dataverseAdmin user, download https://raw.githubusercontent.com/GlobalDataverseCommunityConsortium/dataverse-ansible/master/defaults/main.yml and edit the file to your liking. You can skip this step if you're fine with the values in the "main.yml" file in the link above. Download and Run the "Create Instance" Script ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Once you have done the configuration above, you are ready to try running the "ec2-create-instance.sh" script to spin up Dataverse in AWS. +Once you have done the configuration above, you are ready to try running the "ec2-create-instance.sh" script to spin up a Dataverse installation in AWS. Download `ec2-create-instance.sh`_ and put it somewhere reasonable. For the purpose of these instructions we'll assume it's in the "Downloads" directory in your home directory. @@ -98,7 +98,7 @@ ec2-create-instance accepts a number of command-line switches, including: ``bash ~/Downloads/ec2-create-instance.sh -b develop -r https://github.com/scholarsportal/dataverse.git -g main.yml`` -You will need to wait for 15 minutes or so until the deployment is finished, longer if you've enabled sample data and/or the API test suite. Eventually, the output should tell you how to access the installation of Dataverse in a web browser or via SSH. It will also provide instructions on how to delete the instance when you are finished with it. Please be aware that AWS charges per minute for a running instance. You may also delete your instance from https://console.aws.amazon.com/console/home?region=us-east-1 . +You will need to wait for 15 minutes or so until the deployment is finished, longer if you've enabled sample data and/or the API test suite. Eventually, the output should tell you how to access the Dataverse installation in a web browser or via SSH. It will also provide instructions on how to delete the instance when you are finished with it. Please be aware that AWS charges per minute for a running instance. You may also delete your instance from https://console.aws.amazon.com/console/home?region=us-east-1 . Caveat Recipiens ~~~~~~~~~~~~~~~~ @@ -111,7 +111,7 @@ Migrating Datafiles from Local Storage to S3 A number of pilot Dataverse installations start on local storage, then administrators are tasked with migrating datafiles into S3 or similar object stores. The files may be copied with a command-line utility such as `s3cmd`. You will want to retain the local file hierarchy, keeping the authority (for example: 10.5072) at the bucket "root." -The below example queries may assist with updating dataset and datafile locations in the Dataverse PostgresQL database. Depending on the initial version of Dataverse and subsequent upgrade path, Datafile storage identifiers may or may not include a ``file://`` prefix, so you'll want to catch both cases. +The below example queries may assist with updating dataset and datafile locations in the Dataverse installation's PostgresQL database. Depending on the initial version of the Dataverse software and subsequent upgrade path, Datafile storage identifiers may or may not include a ``file://`` prefix, so you'll want to catch both cases. To Update Dataset Location to S3, Assuming a ``file://`` Prefix ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 1b27ec324fa45968e329b6ae39411b7d9f58b064 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 11 Jan 2021 15:08:28 -0500 Subject: [PATCH 038/274] Separate unf display from variables + observations on dataverse pg result cards [ref #5210 #6685] --- .../search/SearchIncludeFragment.java | 27 ++++++++++++++----- src/main/webapp/search-include-fragment.xhtml | 12 ++++++--- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 61859700077..73aacbdce2d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1201,7 +1201,7 @@ public SearchException getSearchException() { } public String tabularDataDisplayInfo(DataFile datafile) { - String ret = ""; + String tabInfo = ""; if (datafile == null) { return null; @@ -1209,22 +1209,35 @@ public String tabularDataDisplayInfo(DataFile datafile) { if (datafile.isTabularData() && datafile.getDataTable() != null) { DataTable datatable = datafile.getDataTable(); - String unf = datatable.getUnf(); Long varNumber = datatable.getVarQuantity(); Long obsNumber = datatable.getCaseQuantity(); if (varNumber != null && varNumber.intValue() != 0) { - ret = ret.concat(varNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.variables")); + tabInfo = tabInfo.concat(varNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.variables")); if (obsNumber != null && obsNumber.intValue() != 0) { - ret = ret.concat(", " + obsNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.observations")); + tabInfo = tabInfo.concat(", " + obsNumber + " " + BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.observations")); } - ret = ret.concat(" - "); } + } + + return tabInfo; + } + + public String tabularDataUnfDisplay(DataFile datafile) { + String tabUnf = ""; + + if (datafile == null) { + return null; + } + + if (datafile.isTabularData() && datafile.getDataTable() != null) { + DataTable datatable = datafile.getDataTable(); + String unf = datatable.getUnf(); if (unf != null && !unf.equals("")) { - ret = ret.concat(BundleUtil.getStringFromBundle("file.metaData.dataFile.dataTab.unf") + ": " + unf); + tabUnf = tabUnf.concat(unf); } } - return ret; + return tabUnf; } public String dataFileSizeDisplay(DataFile datafile) { diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index 956e57242af..d233ee59078 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -619,9 +619,15 @@ title="#{bundle['file.metaData.checksum.copy']} #{result.fileChecksumValue}" jsf:rendered="#{!result.harvested and !SearchIncludeFragment.isTabular(result.entity) and !(empty result.fileChecksumValue)}">#{result.fileChecksumValue} - - + + + + + #{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)}
From 16a3efad80f995f95a199bf9eb59740a69146f85 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 11 Jan 2021 15:20:24 -0500 Subject: [PATCH 039/274] Overwrite css from primefaces component.css for overflow property to be visible for datatables [ref #5210 #6685] --- src/main/webapp/resources/css/structure.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index dff45133c9f..1b94bcb29bb 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -862,6 +862,9 @@ div.manageTable.ui-datatable div.ui-datatable-tablewrapper, div.manageTable.ui-d div[id$="filesTable"].ui-datatable thead th.col-file-action, div[id$="filesTable"].ui-datatable tbody td.col-file-action {overflow:visible !important;} thead.ui-datatable-scrollable-theadclone {display:none} +/* DATATABE + OVERFLOW VISIBLE */ +.ui-datatable thead th, .ui-datatable tbody td, .ui-datatable tfoot td, .ui-datatable tfoot th {overflow:visible;} + /* FILE */ #file-preview-icon-block > #border-block {/*border:1px solid #DDD;border-radius:4px; padding: 10px;*/} #file-preview-icon-block > #border-block > #preview-square {/*position: relative; overflow:hidden; padding-bottom:100%;*/} From aae992507223dcec08dbe58f99090195e9bc6a18 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 11 Jan 2021 16:33:48 -0500 Subject: [PATCH 040/274] Fixed float UI issue in file table on dataset pg [ref #5210, #6685] --- src/main/webapp/file-info-fragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 4a96327c736..045128fc166 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -29,7 +29,7 @@
-
+
From 3a76134dc0de2bc855014b699d9e14d3ee165eb0 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 11 Jan 2021 17:57:48 -0500 Subject: [PATCH 041/274] more updates --- .../source/developers/dev-environment.rst | 30 +++++++++---------- .../source/developers/documentation.rst | 16 +++++----- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 9abfd385c28..192aac2576c 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -10,7 +10,7 @@ These instructions are purposefully opinionated and terse to help you get your d Quick Start ----------- -The quickest way to get Dataverse running is to use Vagrant as described in the :doc:`tools` section, but for day to day development work, we recommended the following setup. +The quickest way to get the Dataverse software running is to use Vagrant as described in the :doc:`tools` section, but for day to day development work, we recommended the following setup. Set Up Dependencies ------------------- @@ -25,7 +25,7 @@ Windows is not well supported, unfortunately, but Vagrant and Minishift environm Install Java ~~~~~~~~~~~~ -Dataverse requires Java 8. +The Dataverse software requires Java 8. We suggest downloading OpenJDK from https://adoptopenjdk.net @@ -36,29 +36,29 @@ Install Netbeans or Maven NetBeans IDE is recommended, and can be downloaded from http://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Jakarta EE projects, and includes a required build tool, Maven. -Below we describe how to build the Dataverse war file with Netbeans but if you prefer to use only Maven, you can find installation instructions in the :doc:`tools` section. +Below we describe how to build the Dataverse software war file with Netbeans but if you prefer to use only Maven, you can find installation instructions in the :doc:`tools` section. Install Homebrew (Mac Only) ~~~~~~~~~~~~~~~~~~~~~~~~~~~ On Mac, install Homebrew to simplify the steps below: https://brew.sh -Clone the Dataverse Git Repo -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Clone the Dataverse Software Git Repo +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Fork https://github.com/IQSS/dataverse and then clone your fork like this: ``git clone git@github.com:[YOUR GITHUB USERNAME]/dataverse.git`` -Build the Dataverse War File +Build the Dataverse Software War File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you installed Netbeans, follow these steps: -- Launch Netbeans and click "File" and then "Open Project". Navigate to where you put the Dataverse code and double-click "dataverse" to open the project. +- Launch Netbeans and click "File" and then "Open Project". Navigate to where you put the Dataverse software code and double-click "Dataverse" to open the project. - If you see "resolve project problems," go ahead and let Netbeans try to resolve them. This will probably including downloading dependencies, which can take a while. - Allow Netbeans to install nb-javac (required for Java 8 and below). -- Select "dataverse" under Projects and click "Run" in the menu and then "Build Project (dataverse)". Check back for "BUILD SUCCESS" at the end. +- Select "Dataverse" under Projects and click "Run" in the menu and then "Build Project (Dataverse)". Check back for "BUILD SUCCESS" at the end. If you installed Maven instead of Netbeans, run ``mvn package``. Check for "BUILD SUCCESS" at the end. @@ -155,10 +155,10 @@ To install Solr, execute the following commands: ``bin/solr create_core -c collection1 -d server/solr/collection1/conf`` -Run the Dataverse Installer Script ----------------------------------- +Run the Dataverse Software Installer Script +------------------------------------------- -Navigate to the directory where you cloned the Dataverse git repo change directories to the ``scripts/installer`` directory like this: +Navigate to the directory where you cloned the Dataverse software git repo change directories to the ``scripts/installer`` directory like this: ``cd scripts/installer`` @@ -176,10 +176,10 @@ Finally, run the installer (see also :download:`README_python.txt <../../../../s ``python3 install.py`` -Verify Dataverse is Running -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Verify the Dataverse Software is Running +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -After the script has finished, you should be able to log into Dataverse with the following credentials: +After the script has finished, you should be able to log into your Dataverse installation with the following credentials: - http://localhost:8080 - username: dataverseAdmin @@ -197,7 +197,7 @@ This will disable DOI registration by using a fake (in-code) DOI provider. Pleas Next Steps ---------- -If you can log in to Dataverse, great! If not, please see the :doc:`troubleshooting` section. For further assitance, please see "Getting Help" in the :doc:`intro` section. +If you can log in to the Dataverse installation, great! If not, please see the :doc:`troubleshooting` section. For further assitance, please see "Getting Help" in the :doc:`intro` section. You're almost ready to start hacking on code. Now that the installer script has you up and running, you need to continue on to the :doc:`tips` section to get set up to deploy code from your IDE or the command line. diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index 01f7cad1ecc..d2101f37587 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -19,17 +19,17 @@ If you find a typo or a small error in the documentation you can fix it using Gi - `user`_ - Find the file you want to edit under one of the folders above. -- Click the pencil icon in the upper-right corner. If this is your first contribution to Dataverse, the hover text over the pencil icon will say "Fork this project and edit this file". +- Click the pencil icon in the upper-right corner. If this is your first contribution to the Dataverse Project, the hover text over the pencil icon will say "Fork this project and edit this file". - Make changes to the file and preview them. - In the **Commit changes** box, enter a description of the changes you have made and click **Propose file change**. - Under the **Write** tab, delete the long welcome message and write a few words about what you fixed. - Click **Create Pull Request**. -That's it! Thank you for your contribution! Your pull request will be added manually to the main Dataverse project board at https://github.com/orgs/IQSS/projects/2 and will go through code review and QA before it is merged into the "develop" branch. Along the way, developers might suggest changes or make them on your behalf. Once your pull request has been merged you will be listed as a contributor at https://github.com/IQSS/dataverse/graphs/contributors +That's it! Thank you for your contribution! Your pull request will be added manually to the main Dataverse Project board at https://github.com/orgs/IQSS/projects/2 and will go through code review and QA before it is merged into the "develop" branch. Along the way, developers might suggest changes or make them on your behalf. Once your pull request has been merged you will be listed as a contributor at https://github.com/IQSS/dataverse/graphs/contributors Please see https://github.com/IQSS/dataverse/pull/5857 for an example of a quick fix that was merged (the "Files changed" tab shows how a typo was fixed). -If you would like to read more about the Dataverse project's use of GitHub, please see the :doc:`version-control` section. For bug fixes and features we request that you create an issue before making a pull request but this is not at all necessary for quick fixes to the documentation. +If you would like to read more about the Dataverse Project's use of GitHub, please see the :doc:`version-control` section. For bug fixes and features we request that you create an issue before making a pull request but this is not at all necessary for quick fixes to the documentation. .. _admin: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/admin .. _api: https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source/api @@ -40,7 +40,7 @@ If you would like to read more about the Dataverse project's use of GitHub, plea Other Changes (Sphinx) ---------------------- -The documentation for Dataverse was written using Sphinx (http://sphinx-doc.org/). +The documentation for the Dataverse Project was written using Sphinx (http://sphinx-doc.org/). If you are interested in suggesting changes or updates we recommend that you create the html files using Sphinx locally and then submit a pull request through GitHub. Here are the instructions on how to proceed: @@ -71,12 +71,12 @@ This is all you need. You should now be able to build HTML/pdf documentation fro Using Sphinx ~~~~~~~~~~~~ -First, you will need to make a fork of the dataverse repository in GitHub. Then, you will need to make a clone of your fork so you can manipulate the files outside GitHub. +First, you will need to make a fork of the Dataverse software repository in GitHub. Then, you will need to make a clone of your fork so you can manipulate the files outside GitHub. To edit the existing documentation: - Create a branch (refer to http://guides.dataverse.org/en/latest/developers/version-control.html > *Create a New Branch off the develop Branch*) to record the changes you are about to perform. -- Go to ~/dataverse/doc/sphinx-guides/source directory inside your clone. There, you will find the .rst files that correspond to the guides in the dataverse page (http://guides.dataverse.org/en/latest/). +- Go to ~/dataverse/doc/sphinx-guides/source directory inside your clone. There, you will find the .rst files that correspond to the guides in the Dataverse Software Guides page (http://guides.dataverse.org/en/latest/). - Using your preferred text editor, open and edit the necessary files, or create new ones. **NOTE:** When adding ReStructured Text (RST) `cross references `_, use the hyphen character (``-``) as the word separator for the cross reference label. For example, ``my-reference-label`` would be the preferred label for a cross reference as opposed to, for example, ``my_reference_label``. @@ -90,7 +90,7 @@ Once you are done, open a terminal and change directories to ~/dataverse/doc/sph After sphinx is done processing the files you should notice that the html folder in ~/dataverse/doc/sphinx-guides/build directory has been updated. You can click on the files in the html folder to preview the changes. -Now you can make a commit with the changes to your own fork in GitHub and submit a pull request to the original (upstream) dataverse repository. +Now you can make a commit with the changes to your own fork in GitHub and submit a pull request to the original (upstream) Dataverse Software repository. Table of Contents ----------------- @@ -126,7 +126,7 @@ This has been tested and works on Mac, Linux, and Windows. Versions -------- -For installations hosting their own copies of the guides, note that as each version of Dataverse is released, there is an updated version of the guides released with it. Google and other search engines index all versions, which may confuse users who discover your guides in the search results as to which version they should be looking at. When learning about your installation from the search results, it is best to be viewing the *latest* version. +For installations hosting their own copies of the guides, note that as each version of the Dataverse Software is released, there is an updated version of the guides released with it. Google and other search engines index all versions, which may confuse users who discover your guides in the search results as to which version they should be looking at. When learning about your installation from the search results, it is best to be viewing the *latest* version. In order to make it clear to the crawlers that we only want the latest version discoverable in their search results, we suggest adding this to your ``robots.txt`` file:: From 6b4fb249a3a4a5bf84ab9fb5c67eec3b7e8f4415 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 12 Jan 2021 16:33:32 -0500 Subject: [PATCH 042/274] Undo on bug testing revision mistakenly left in java function [ref #5210, #6685] --- .../harvard/iq/dataverse/search/SearchIncludeFragment.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 73aacbdce2d..f7a3b874c24 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -1204,7 +1204,7 @@ public String tabularDataDisplayInfo(DataFile datafile) { String tabInfo = ""; if (datafile == null) { - return null; + return ""; } if (datafile.isTabularData() && datafile.getDataTable() != null) { @@ -1226,7 +1226,7 @@ public String tabularDataUnfDisplay(DataFile datafile) { String tabUnf = ""; if (datafile == null) { - return null; + return ""; } if (datafile.isTabularData() && datafile.getDataTable() != null) { From f45d0353627590723052b7c6dae293741b048679 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:25:12 -0500 Subject: [PATCH 043/274] more updates --- .../source/developers/geospatial.rst | 22 ++++++++-------- doc/sphinx-guides/source/developers/intro.rst | 26 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/sphinx-guides/source/developers/geospatial.rst b/doc/sphinx-guides/source/developers/geospatial.rst index 13680a52358..1cbe85cd12e 100644 --- a/doc/sphinx-guides/source/developers/geospatial.rst +++ b/doc/sphinx-guides/source/developers/geospatial.rst @@ -8,12 +8,12 @@ Geospatial Data Geoconnect ---------- -Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. To set up a Geoconnect development environment, you can follow the steps outlined in the `local_setup.md `_ guide. You will need Python and a few other prerequisites. +Geoconnect works as a middle layer, allowing geospatial data files in your Dataverse installation to be visualized with Harvard WorldMap. To set up a Geoconnect development environment, you can follow the steps outlined in the `local_setup.md `_ guide. You will need Python and a few other prerequisites. -As mentioned under the :ref:`architecture` section of Preparation in the Installation Guide, Geoconnect is an optional component of Dataverse, so this section is only necessary to follow it you are working on an issue related to this feature. +As mentioned under the :ref:`architecture` section of Preparation in the Installation Guide, Geoconnect is an optional component of the Dataverse Platform, so this section is only necessary to follow it you are working on an issue related to this feature. -How Dataverse Ingests Shapefiles --------------------------------- +How Your Dataverse Installation Ingests Shapefiles +-------------------------------------------------- A shapefile is a set of files, often uploaded/transferred in ``.zip`` format. This set may contain up to fifteen files. A minimum of three specific files (``.shp``, ``.shx``, ``.dbf``) are needed to be a valid shapefile and a fourth file (``.prj``) is required for WorldMap -- or any type of meaningful visualization. @@ -27,7 +27,7 @@ For ingest and connecting to WorldMap, four files are the minimum required: Ingest ~~~~~~ -When uploaded to Dataverse, the ``.zip`` is unpacked (same as all ``.zip`` files). Shapefile sets are recognized by the same base name and specific extensions. These individual files constitute a shapefile set. The first four are the minimum required (``.shp``, ``.shx``, ``.dbf``, ``.prj``) +When uploaded to a Dataverse installation, the ``.zip`` is unpacked (same as all ``.zip`` files). Shapefile sets are recognized by the same base name and specific extensions. These individual files constitute a shapefile set. The first four are the minimum required (``.shp``, ``.shx``, ``.dbf``, ``.prj``) For example: @@ -38,19 +38,19 @@ For example: - bicycles.sbx (NOT required extension) - bicycles.sbn (NOT required extension) -Upon recognition of the four required files, Dataverse will group them as well as any other relevant files into a shapefile set. Files with these extensions will be included in the shapefile set: +Upon recognition of the four required files, the Dataverse installation will group them as well as any other relevant files into a shapefile set. Files with these extensions will be included in the shapefile set: - Required: ``.shp``, ``.shx``, ``.dbf``, ``.prj`` - Optional: ``.sbn``, ``.sbx``, ``.fbn``, ``.fbx``, ``.ain``, ``.aih``, ``.ixs``, ``.mxs``, ``.atx``, ``.cpg``, ``shp.xml`` -Then Dataverse creates a new ``.zip`` with mimetype as a shapefile. The shapefile set will persist as this new ``.zip``. +Then the Dataverse installation creates a new ``.zip`` with mimetype as a shapefile. The shapefile set will persist as this new ``.zip``. Example ~~~~~~~ **1a.** Original ``.zip`` contents: -A file named ``bikes_and_subways.zip`` is uploaded to the Dataverse. This ``.zip`` contains the following files. +A file named ``bikes_and_subways.zip`` is uploaded to the Dataverse installation. This ``.zip`` contains the following files. - ``bicycles.shp`` (shapefile set #1) - ``bicycles.shx`` (shapefile set #1) @@ -66,9 +66,9 @@ A file named ``bikes_and_subways.zip`` is uploaded to the Dataverse. This ``.zip - ``subway_line.prj`` (shapefile set #2) - ``subway_line.dbf`` (shapefile set #2) -**1b.** Dataverse unzips and re-zips files: +**1b.** The Dataverse installation unzips and re-zips files: -Upon ingest, Dataverse unpacks the file ``bikes_and_subways.zip``. Upon recognizing the shapefile sets, it groups those files together into new ``.zip`` files: +Upon ingest, the Dataverse installation unpacks the file ``bikes_and_subways.zip``. Upon recognizing the shapefile sets, it groups those files together into new ``.zip`` files: - files making up the "bicycles" shapefile become a new ``.zip`` - files making up the "subway_line" shapefile become a new ``.zip`` @@ -76,7 +76,7 @@ Upon ingest, Dataverse unpacks the file ``bikes_and_subways.zip``. Upon recogniz To ensure that a shapefile set remains intact, individual files such as ``bicycles.sbn`` are kept in the set -- even though they are not used for mapping. -**1c.** Dataverse final file listing: +**1c.** The Dataverse installation final file listing: - ``bicycles.zip`` (contains shapefile set #1: ``bicycles.shp``, ``bicycles.shx``, ``bicycles.prj``, ``bicycles.dbf``, ``bicycles.sbx``, ``bicycles.sbn``) - ``bicycles.txt`` (separate, not part of a shapefile set) diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index e7ed6a12a37..a4e7b312bc9 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -2,7 +2,7 @@ Introduction ============ -Welcome! `Dataverse `_ is an `open source `_ project that loves `contributors `_! +Welcome! `The Dataverse Project `_ is an `open source `_ project that loves `contributors `_! .. contents:: |toctitle| :local: @@ -10,7 +10,7 @@ Welcome! `Dataverse `_ is an `open source `_ application that is compiled into a WAR file and deployed to an application server (app server) which is configured to work with a relational database (PostgreSQL) and a search engine (Solr). +The Dataverse software is a `Jakarta EE `_ application that is compiled into a WAR file and deployed to an application server (app server) which is configured to work with a relational database (PostgreSQL) and a search engine (Solr). We make use of a variety of Jakarta EE technologies such as JPA, JAX-RS, JMS, and JSF. The front end is built using PrimeFaces and Bootstrap. @@ -33,7 +33,7 @@ In addition, we start to adopt parts of Eclipse MicroProfile, namely `MicroProfi Roadmap ------- -For the Dataverse development roadmap, please see https://www.iq.harvard.edu/roadmap-dataverse-project +For the Dataverse software development roadmap, please see https://www.iq.harvard.edu/roadmap-dataverse-project Kanban Board ------------ @@ -48,7 +48,7 @@ We use GitHub Issues as our issue tracker: https://github.com/IQSS/dataverse/iss Related Guides -------------- -If you are a developer who wants to make use of Dataverse APIs, please see the :doc:`/api/index`. If you have front-end UI questions, please see the :doc:`/style/index`. +If you are a developer who wants to make use of the Dataverse Software APIs, please see the :doc:`/api/index`. If you have front-end UI questions, please see the :doc:`/style/index`. If you are a sysadmin who likes to code, you may be interested in hacking on installation scripts mentioned in the :doc:`/installation/index`. We validate the installation scripts with :doc:`/developers/tools` such as `Vagrant `_ and Docker (see the :doc:`containers` section). @@ -57,20 +57,20 @@ Related Projects As a developer, you also may be interested in these projects related to Dataverse: -- External Tools - add additional features to Dataverse without modifying the core: :doc:`/api/external-tools` -- Dataverse API client libraries - use Dataverse APIs from various languages: :doc:`/api/client-libraries` -- DVUploader - a stand-alone command-line Java application that uses the Dataverse API to support upload of files from local disk to a Dataset: https://github.com/IQSS/dataverse-uploader +- External Tools - add additional features to the Dataverse Software without modifying the core: :doc:`/api/external-tools` +- Dataverse Software API client libraries - use Dataverse Software APIs from various languages: :doc:`/api/client-libraries` +- DVUploader - a stand-alone command-line Java application that uses the Dataverse Software API to support upload of files from local disk to a Dataset: https://github.com/IQSS/dataverse-uploader - dataverse-sample-data - populate your Dataverse installation with sample data: https://github.com/IQSS/dataverse-sample-data -- dataverse-metrics - aggregate and visualize metrics for installations of Dataverse around the world: https://github.com/IQSS/dataverse-metrics +- dataverse-metrics - aggregate and visualize metrics for Dataverse installations around the world: https://github.com/IQSS/dataverse-metrics - Configuration management scripts - Ansible, Puppet, etc.: See :ref:`advanced` section in the Installation Guide. - :doc:`/developers/unf/index` (Java) - a Universal Numerical Fingerprint: https://github.com/IQSS/UNF -- GeoConnect (Python) - create a map by uploading files to Dataverse: https://github.com/IQSS/geoconnect +- GeoConnect (Python) - create a map by uploading files to a Dataverse installation: https://github.com/IQSS/geoconnect - `DataTags `_ (Java and Scala) - tag datasets with privacy levels: https://github.com/IQSS/DataTags - `TwoRavens `_ (Javascript) - a `d3.js `_ interface for exploring data and running Zelig models: https://github.com/IQSS/TwoRavens -- `Zelig `_ (R) - run statistical models on files uploaded to Dataverse: https://github.com/IQSS/Zelig +- `Zelig `_ (R) - run statistical models on files uploaded to a Dataverse installation: https://github.com/IQSS/Zelig - `Matrix `_ - a visualization showing the connectedness and collaboration between authors and their affiliations. -- Third party apps - make use of Dataverse APIs: :doc:`/api/apps` -- chat.dataverse.org - chat interface for Dataverse users and developers: https://github.com/IQSS/chat.dataverse.org +- Third party apps - make use of Dataverse installation APIs: :doc:`/api/apps` +- chat.dataverse.org - chat interface for Dataverse Project users and developers: https://github.com/IQSS/chat.dataverse.org - [Your project here] :) ---- From c69abed0dfb282b4d04cbeb3bf9fd84f1628b10e Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:27:45 -0500 Subject: [PATCH 044/274] Update make-data-count.rst --- .../source/developers/make-data-count.rst | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/developers/make-data-count.rst b/doc/sphinx-guides/source/developers/make-data-count.rst index dac1b074d68..f604e2c6c62 100644 --- a/doc/sphinx-guides/source/developers/make-data-count.rst +++ b/doc/sphinx-guides/source/developers/make-data-count.rst @@ -1,7 +1,7 @@ Make Data Count =============== -Support for Make Data Count is a feature of Dataverse that is described in the :doc:`/admin/make-data-count` section of the Admin Guide. In order for developers to work on the feature, they must install Counter Processor, a Python 3 application, as described below. Counter Processor can be found at https://github.com/CDLUC3/counter-processor +Support for Make Data Count is a feature of the Dataverse Software that is described in the :doc:`/admin/make-data-count` section of the Admin Guide. In order for developers to work on the feature, they must install Counter Processor, a Python 3 application, as described below. Counter Processor can be found at https://github.com/CDLUC3/counter-processor .. contents:: |toctitle| :local: @@ -9,7 +9,7 @@ Support for Make Data Count is a feature of Dataverse that is described in the : Architecture ------------ -There are many components involved in Dataverse's architecture for Make Data Count as shown in the diagram below. +There are many components involved in the Dataverse software's architecture for Make Data Count as shown in the diagram below. |makedatacount_components| @@ -28,20 +28,20 @@ To insert a citation you could insert a row like below, changing "72" in the exa Full Setup ~~~~~~~~~~ -The recommended way to work on the Make Data Count feature is to spin up an EC2 instance that has both Dataverse and Counter Processor installed. Go to the :doc:`deployment` page for details on how to spin up an EC2 instance and make sure that your Ansible file is configured to install Counter Processor before running the "create" script. +The recommended way to work on the Make Data Count feature is to spin up an EC2 instance that has both the Dataverse software and Counter Processor installed. Go to the :doc:`deployment` page for details on how to spin up an EC2 instance and make sure that your Ansible file is configured to install Counter Processor before running the "create" script. (Alternatively, you can try installing Counter Processor in Vagrant. :download:`setup-counter-processor.sh <../../../../scripts/vagrant/setup-counter-processor.sh>` might help you get it installed.) -After you have spun to your EC2 instance, set ``:MDCLogPath`` so that Dataverse creates a log for Counter Processor to operate on. For more on this database setting, see the :doc:`/installation/config` section of the Installation Guide. +After you have spun to your EC2 instance, set ``:MDCLogPath`` so that the Dataverse installation creates a log for Counter Processor to operate on. For more on this database setting, see the :doc:`/installation/config` section of the Installation Guide. -Next you need to have Dataverse add some entries to the log that Counter Processor will operate on. To do this, click on some published datasets and download some files. +Next you need to have the Dataverse installation add some entries to the log that Counter Processor will operate on. To do this, click on some published datasets and download some files. Next you should run Counter Processor to convert the log into a SUSHI report, which is in JSON format. Before running Counter Processor, you need to put a configuration file into place. As a starting point use :download:`counter-processor-config.yaml <../../../../scripts/vagrant/counter-processor-config.yaml>` and edit the file, paying particular attention to the following settings: - ``log_name_pattern`` You might want something like ``/usr/local/payara5/glassfish/domains/domain1/logs/counter_(yyyy-mm-dd).log`` - ``year_month`` You should probably set this to the current month. - ``output_file`` This needs to be a directory that the "dataverse" Unix user can read but that the "counter" user can write to. In dev, you can probably get away with "/tmp" as the directory. -- ``platform`` Out of the box from Counter Processor this is set to ``Dash`` but this should be changed to match the name of your Dataverse. Examples are "Harvard Dataverse" for Harvard University or "LibraData" for the University of Virginia. +- ``platform`` Out of the box from Counter Processor this is set to ``Dash`` but this should be changed to match the name of your Dataverse installation. Examples are "Harvard Dataverse Repository" for Harvard University or "LibraData" for the University of Virginia. - ``upload_to_hub`` This should be "False" unless you are testing sending SUSHI reports to the DataCite hub. - ``simulate_date`` You should probably set this to tomorrow. @@ -73,10 +73,10 @@ If all this is working and you want to send data to the test instance of the Dat For how to put citations into your dev database and how to get them out again, see :ref:`MDC-updateCitationsForDataset` section in Make Data Count of the Admin Guide. -Testing Make Data Count and Dataverse -------------------------------------- +Testing Make Data Count and Your Dataverse Installation +------------------------------------------------------- -A developer running Counter Processor alongside Dataverse for development or testing purposes will notice that once the raw Dataverse logs have been processed, there is no straightforward way to re-test those same logs. +A developer running Counter Processor alongside the Dataverse installation for development or testing purposes will notice that once the raw Dataverse installation logs have been processed, there is no straightforward way to re-test those same logs. The first thing to fix is to clear two files from Counter Processor ``state`` folder, ``statefile.json`` and ``counter_db_[yyyy-mm].sqlite3`` @@ -88,7 +88,7 @@ To get the ``REPORT_ID``, look at the logs generated in ``/usr/local/counter-pro To read more about the Make Data Count api, see https://github.com/datacite/sashimi -You can compare the MDC metrics display with Dataverse's original by toggling the ``:DisplayMDCMetrics`` setting (true by default to display MDC metrics). +You can compare the MDC metrics display with the Dataverse installation's original by toggling the ``:DisplayMDCMetrics`` setting (true by default to display MDC metrics). Resources --------- From 0ad21ce37510f70e94b344c9b1e546b29d115545 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:29:52 -0500 Subject: [PATCH 045/274] YET MORE UPDATES --- doc/sphinx-guides/source/developers/remote-users.rst | 2 +- doc/sphinx-guides/source/developers/selinux.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/remote-users.rst b/doc/sphinx-guides/source/developers/remote-users.rst index 66af0c71eda..6e658c15543 100755 --- a/doc/sphinx-guides/source/developers/remote-users.rst +++ b/doc/sphinx-guides/source/developers/remote-users.rst @@ -8,7 +8,7 @@ Shibboleth and OAuth Shibboleth and OAuth -------------------- -If you are working on anything related to users, please keep in mind that your changes will likely affect Shibboleth and OAuth users. For some background on user accounts in Dataverse, see :ref:`auth-modes` section of Configuration in the Installation Guide. +If you are working on anything related to users, please keep in mind that your changes will likely affect Shibboleth and OAuth users. For some background on user accounts in the Dataverse software, see :ref:`auth-modes` section of Configuration in the Installation Guide. Rather than setting up Shibboleth on your laptop, developers are advised to simply add a value to their database to enable Shibboleth "dev mode" like this: diff --git a/doc/sphinx-guides/source/developers/selinux.rst b/doc/sphinx-guides/source/developers/selinux.rst index d7f5b0d7519..52fd62cc308 100644 --- a/doc/sphinx-guides/source/developers/selinux.rst +++ b/doc/sphinx-guides/source/developers/selinux.rst @@ -44,7 +44,7 @@ Use ``semodule -l | grep shibboleth`` to see if the ``shibboleth.te`` rules are Exercising SELinux denials ~~~~~~~~~~~~~~~~~~~~~~~~~~ -As of this writing, there are two optional components of Dataverse that are known not to work with SELinux out of the box with SELinux: Shibboleth and rApache. +As of this writing, there are two optional components of the Dataverse software that are known not to work with SELinux out of the box with SELinux: Shibboleth and rApache. We will be exercising SELinux denials with Shibboleth, and the SELinux-related issues are expected out the box: From 92bc34bf33e91bba85312d306b4299182a93df66 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:43:56 -0500 Subject: [PATCH 046/274] updates, updates everywhere --- .../source/developers/sql-upgrade-scripts.rst | 6 +-- .../source/developers/testing.rst | 38 +++++++++---------- doc/sphinx-guides/source/developers/tips.rst | 30 +++++++-------- doc/sphinx-guides/source/developers/tools.rst | 16 ++++---- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index 6d49f927a9e..bf27242f79e 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -2,7 +2,7 @@ SQL Upgrade Scripts =================== -The database schema for Dataverse is constantly evolving and we have adopted a tool called Flyway to help keep your development environment up to date and in working order. As you make changes to the database schema (changes to ``@Entity`` classes), you must write SQL upgrade scripts when needed and follow Flyway file naming conventions. +The database schema for the Dataverse software is constantly evolving and we have adopted a tool called Flyway to help keep your development environment up to date and in working order. As you make changes to the database schema (changes to ``@Entity`` classes), you must write SQL upgrade scripts when needed and follow Flyway file naming conventions. .. contents:: |toctitle| :local: @@ -30,7 +30,7 @@ Create a new file called something like ``V4.11.0.1__5565-sanitize-directory-lab The SQL migration script you wrote will be part of the war file and executed when the war file is deployed. To see a history of Flyway database migrations that have been applied, look at the ``flyway_schema_history`` table. -As with any task related to Dataverse development, if you need any help writing SQL upgrade scripts, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. +As with any task related to the development of the Dataverse software, if you need any help writing SQL upgrade scripts, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. Troubleshooting --------------- @@ -38,7 +38,7 @@ Troubleshooting Renaming SQL Upgrade Scripts ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Please note that if you need to rename your script (because a new version of Dataverse was released, for example), you will see the error "FlywayException: Validate failed: Detected applied migration not resolved locally" when you attempt to deploy and deployment will fail. +Please note that if you need to rename your script (because a new version of the Dataverse software was released, for example), you will see the error "FlywayException: Validate failed: Detected applied migration not resolved locally" when you attempt to deploy and deployment will fail. To resolve this problem, delete the old migration from the ``flyway_schema_history`` table and attempt to redeploy. diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 4ec1ff5cec0..c2e14749dd8 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -2,7 +2,7 @@ Testing ======= -In order to keep our codebase healthy, the Dataverse project encourages developers to write automated tests in the form of unit tests and integration tests. We also welcome ideas for how to improve our automated testing. +In order to keep our codebase healthy, the Dataverse Project encourages developers to write automated tests in the form of unit tests and integration tests. We also welcome ideas for how to improve our automated testing. .. contents:: |toctitle| :local: @@ -113,16 +113,16 @@ Generally speaking, unit tests have been flagged as non-essential because they a Integration Tests ----------------- -Unit tests are fantastic for low level testing of logic but aren't especially real-world-applicable because they do not exercise Dataverse as it runs in production with a database and other runtime dependencies. We test in-depth by also writing integration tests to exercise a running system. +Unit tests are fantastic for low level testing of logic but aren't especially real-world-applicable because they do not exercise the Dataverse software as it runs in production with a database and other runtime dependencies. We test in-depth by also writing integration tests to exercise a running system. Unfortunately, the term "integration tests" can mean different things to -different people. For our purposes, an integration test can have to flavors: +different people. For our purposes, an integration test can have two flavors: 1. Be an API Test: - - Exercise Dataverse APIs. + - Exercise the Dataverse Software APIs. - Running not automatically on developers' laptops. - - Operate on an installation of Dataverse that is running and able to talk to both PostgreSQL and Solr. + - Operate on a Dataverse installation that is running and able to talk to both PostgreSQL and Solr. - Written using REST Assured. 2. Be a `Testcontainers `__ Test: @@ -173,32 +173,32 @@ Alternatively, you can run tests against the app server running on your laptop b Getting Set Up to Run REST Assured Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Unit tests are run automatically on every build, but dev environments and servers require special setup to run REST Assured tests. In short, Dataverse needs to be placed into an insecure mode that allows arbitrary users and datasets to be created and destroyed. This differs greatly from the out-of-the-box behavior of Dataverse, which we strive to keep secure for sysadmins installing the software for their institutions in a production environment. +Unit tests are run automatically on every build, but dev environments and servers require special setup to run REST Assured tests. In short, the Dataverse software needs to be placed into an insecure mode that allows arbitrary users and datasets to be created and destroyed. This differs greatly from the out-of-the-box behavior of the Dataverse software, which we strive to keep secure for sysadmins installing the software for their institutions in a production environment. -The :doc:`dev-environment` section currently refers developers here for advice on getting set up to run REST Assured tests, but we'd like to add some sort of "dev" flag to the installer to put Dataverse in "insecure" mode, with lots of scary warnings that this dev mode should not be used in production. +The :doc:`dev-environment` section currently refers developers here for advice on getting set up to run REST Assured tests, but we'd like to add some sort of "dev" flag to the installer to put the Dataverse software in "insecure" mode, with lots of scary warnings that this dev mode should not be used in production. The instructions below assume a relatively static dev environment on a Mac. There is a newer "all in one" Docker-based approach documented in the :doc:`/developers/containers` section under "Docker" that you may like to play with as well. The Burrito Key ^^^^^^^^^^^^^^^ -For reasons that have been lost to the mists of time, Dataverse really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database): +For reasons that have been lost to the mists of time, the Dataverse software really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database): ``curl -X PUT -d 'burrito' http://localhost:8080/api/admin/settings/BuiltinUsers.KEY`` -Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as dataverses, datasets, and files. +Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as Dataverse collections, datasets, and files. -Root Dataverse Permissions -^^^^^^^^^^^^^^^^^^^^^^^^^^ +Root Dataverse Collection Permissions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In your browser, log in as dataverseAdmin (password: admin) and click the "Edit" button for your root dataverse. Navigate to Permissions, then the Edit Access button. Under "Who can add to this dataverse?" choose "Anyone with a Dataverse account can add sub dataverses and datasets" if it isn't set to this already. +In your browser, log in as dataverseAdmin (password: admin) and click the "Edit" button for your root Dataverse collection. Navigate to Permissions, then the Edit Access button. Under "Who can add to this Dataverse collection?" choose "Anyone with a Dataverse installation account can add sub Dataverse collections and datasets" if it isn't set to this already. Alternatively, this same step can be done with this script: ``scripts/search/tests/grant-authusers-add-on-root`` -Publish Root Dataverse -^^^^^^^^^^^^^^^^^^^^^^ +Publish Root Dataverse Collection +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The root dataverse must be published for some of the REST Assured tests to run. +The root Dataverse collection must be published for some of the REST Assured tests to run. dataverse.siteUrl ^^^^^^^^^^^^^^^^^ @@ -235,7 +235,7 @@ Remember, it’s only a test (and it's not graded)! Some guidelines to bear in m - Let the code do the labor; automate everything that happens when you run your test file. - Just as with any development, if you’re stuck: ask for help! -To execute existing integration tests on your local Dataverse, a helpful command line tool to use is `Maven `_. You should have Maven installed as per the `Development Environment `_ guide, but if not it’s easily done via Homebrew: ``brew install maven``. +To execute existing integration tests on your local Dataverse installation, a helpful command line tool to use is `Maven `_. You should have Maven installed as per the `Development Environment `_ guide, but if not it’s easily done via Homebrew: ``brew install maven``. Once installed, you may run commands with ``mvn [options] [] []``. @@ -410,7 +410,7 @@ Enhance build time by caching dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In the future, CI builds in ephemeral build environments and Docker builds can benefit from caching all dependencies and plugins. -As Dataverse is a huge project, build times can be enhanced by avoiding re-downloading everything when the Maven POM is unchanged. +As the Dataverse Project is a huge project, build times can be enhanced by avoiding re-downloading everything when the Maven POM is unchanged. To seed the cache, use the following Maven goal before using Maven in (optional) offline mode in your scripts: .. code:: shell @@ -456,9 +456,9 @@ Accessibility Testing Accessibility Policy ~~~~~~~~~~~~~~~~~~~~ -Dataverse aims to improve the user experience for those with disabilities, and are in the process of following the recommendations of the `Harvard University Digital Accessibility Policy `__, which use the Worldwide Web Consortium’s Web Content Accessibility Guidelines version 2.1, Level AA Conformance (WCAG 2.1 Level AA) as the standard. +The Dataverse Project aims to improve the user experience for those with disabilities, and are in the process of following the recommendations of the `Harvard University Digital Accessibility Policy `__, which use the Worldwide Web Consortium’s Web Content Accessibility Guidelines version 2.1, Level AA Conformance (WCAG 2.1 Level AA) as the standard. -To report an accessibility issue with Dataverse, you can create a new issue in our GitHub repo at: https://github.com/IQSS/dataverse/issues/ +To report an accessibility issue with the Dataverse software, you can create a new issue in our GitHub repo at: https://github.com/IQSS/dataverse/issues/ Accessibility Tools ~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index 47265c97eff..5d38c2f8aba 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -10,16 +10,16 @@ If you just followed the steps in :doc:`dev-environment` for the first time, you Iterating on Code and Redeploying --------------------------------- -When you followed the steps in the :doc:`dev-environment` section, the war file was deployed to Payara by the Dataverse installation script. That's fine but once you're ready to make a change to the code you will need to get comfortable with undeploying and redeploying code (a war file) to Payara. +When you followed the steps in the :doc:`dev-environment` section, the war file was deployed to Payara by the Dataverse software installation script. That's fine but once you're ready to make a change to the code you will need to get comfortable with undeploying and redeploying code (a war file) to Payara. -It's certainly possible to manage deployment and undeployment of the war file via the command line using the ``asadmin`` command that ships with Payara (that's what the Dataverse installation script uses and the steps are documented below), but we recommend getting set up with an IDE such as Netbeans to manage deployment for you. +It's certainly possible to manage deployment and undeployment of the war file via the command line using the ``asadmin`` command that ships with Payara (that's what the Dataverse software installation script uses and the steps are documented below), but we recommend getting set up with an IDE such as Netbeans to manage deployment for you. -Undeploy the war File from the Dataverse Installation Script -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Undeploy the war File from the Dataverse Software Installation Script +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Because the initial deployment of the war file was done outside of Netbeans by the Dataverse installation script, it's a good idea to undeploy that war file to give Netbeans a clean slate to work with. +Because the initial deployment of the war file was done outside of Netbeans by the Dataverse Software installation script, it's a good idea to undeploy that war file to give Netbeans a clean slate to work with. -Assuming you installed Payara in ``/usr/local/payara5``, run the following ``asadmin`` command to see the version of Dataverse that the Dataverse installation script deployed: +Assuming you installed Payara in ``/usr/local/payara5``, run the following ``asadmin`` command to see the version of the Dataverse Software that the Dataverse Software installation script deployed: ``/usr/local/payara5/bin/asadmin list-applications`` @@ -38,7 +38,7 @@ Please note that if you are on a Mac, Netbeans may be unable to start Payara due At this point you can manage Payara using Netbeans. Click "Window" and then "Services". Expand "Servers" and right-click Payara to stop and then start it so that it appears in the Output window. Note that you can expand "Payara" and "Applications" to see if any applications are deployed. -Ensure that Dataverse Will Be Deployed to Payara +Ensure that the Dataverse Software Will Be Deployed to Payara ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Click "Window" and then "Projects". Click "File" and then "Project Properties (dataverse)". Click "Run" and change "Server" from "No Server Selected" to your installation of Payara. Click OK. @@ -81,7 +81,7 @@ For faster iteration while working on JSF pages, it is highly recommended that y Database Schema Exploration --------------------------- -With over 100 tables, the Dataverse PostgreSQL database ("dvndb") can be somewhat daunting for newcomers. Here are some tips for coming up to speed. (See also the :doc:`sql-upgrade-scripts` section.) +With over 100 tables, the Dataverse Software PostgreSQL database ("dvndb") can be somewhat daunting for newcomers. Here are some tips for coming up to speed. (See also the :doc:`sql-upgrade-scripts` section.) pgAdmin ~~~~~~~~ @@ -109,13 +109,13 @@ For the ``asadmin`` commands below, we assume you have already changed directori There are four steps to this process: 1. Build the war file: ``mvn package`` -2. Check which version of Dataverse is deployed: ``./asadmin list-applications`` -3. Undeploy the Dataverse application (if necessary): ``./asadmin undeploy dataverse-VERSION`` +2. Check which version of the Dataverse software is deployed: ``./asadmin list-applications`` +3. Undeploy the Dataverse software application (if necessary): ``./asadmin undeploy dataverse-VERSION`` 4. Copy the war file to the server (if necessary) 5. Deploy the new code: ``./asadmin deploy /path/to/dataverse-VERSION.war`` -Running the Dataverse Installation Script in Non-Interactive Mode ------------------------------------------------------------------ +Running the Dataverse Software Installation Script in Non-Interactive Mode +-------------------------------------------------------------------------- Rather than running the installer in "interactive" mode, it's possible to put the values in a file. See "non-interactive mode" in the :doc:`/installation/installation-main` section of the Installation Guide. @@ -128,9 +128,9 @@ Solr ---- .. TODO: This section should be moved into a dedicated guide about Solr for developers. It should be extended with - information about the way Solr is used within Dataverse, ideally explaining concepts and links to upstream docs. + information about the way Solr is used within the Dataverse software, ideally explaining concepts and links to upstream docs. -Once some dataverses, datasets, and files have been created and indexed, you can experiment with searches directly from Solr at http://localhost:8983/solr/#/collection1/query and look at the JSON output of searches, such as this wildcard search: http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&indent=true . You can also get JSON output of static fields Solr knows about: http://localhost:8983/solr/collection1/schema/fields +Once some Dataverse collections, datasets, and files have been created and indexed, you can experiment with searches directly from Solr at http://localhost:8983/solr/#/collection1/query and look at the JSON output of searches, such as this wildcard search: http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&indent=true . You can also get JSON output of static fields Solr knows about: http://localhost:8983/solr/collection1/schema/fields You can simply double-click "start.jar" rather that running ``java -jar start.jar`` from the command line. Figuring out how to stop Solr after double-clicking it is an exercise for the reader. @@ -165,7 +165,7 @@ When you prefer manual updates, there is another script, see above: :ref:`custom Sample Data ----------- -You may want to populate your **non-production** installation(s) of Dataverse with sample data. You have a couple options: +You may want to populate your **non-production** Dataverse installations with sample data. You have a couple options: - Code in https://github.com/IQSS/dataverse-sample-data (recommended). This set of sample data includes several common data types, data subsetted from production datasets in dataverse.harvard.edu, datasets with file hierarchy, and more. - Scripts called from ``scripts/deploy/phoenix.dataverse.org/post``. diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst index 8fc86796742..49e92c1b5cf 100755 --- a/doc/sphinx-guides/source/developers/tools.rst +++ b/doc/sphinx-guides/source/developers/tools.rst @@ -30,11 +30,11 @@ With Maven installed you can run ``mvn package`` and ``mvn test`` from the comma Vagrant +++++++ -Vagrant allows you to spin up a virtual machine running Dataverse on your development workstation. You'll need to install Vagrant from https://www.vagrantup.com and VirtualBox from https://www.virtualbox.org. +Vagrant allows you to spin up a virtual machine running the Dataverse software on your development workstation. You'll need to install Vagrant from https://www.vagrantup.com and VirtualBox from https://www.virtualbox.org. We assume you have already cloned the repo from https://github.com/IQSS/dataverse as explained in the :doc:`/developers/dev-environment` section. -From the root of the git repo (where the ``Vagrantfile`` is), run ``vagrant up`` and eventually you should be able to reach an installation of Dataverse at http://localhost:8888 (the ``forwarded_port`` indicated in the ``Vagrantfile``). +From the root of the git repo (where the ``Vagrantfile`` is), run ``vagrant up`` and eventually you should be able to reach a Dataverse installation at http://localhost:8888 (the ``forwarded_port`` indicated in the ``Vagrantfile``). Please note that running ``vagrant up`` for the first time should run the ``downloads/download.sh`` script for you to download required software such as an app server, Solr, etc. However, these dependencies change over time so it's a place to look if ``vagrant up`` was working but later fails. @@ -73,7 +73,7 @@ The first time you run ``./pagekite.py`` a file at ``~/.pagekite.rc`` will be created. You can edit this file to configure PageKite to serve up port 8080 (the default app server HTTP port) or the port of your choosing. -According to https://pagekite.net/support/free-for-foss/ PageKite (very generously!) offers free accounts to developers writing software the meets http://opensource.org/docs/definition.php such as Dataverse. +According to https://pagekite.net/support/free-for-foss/ PageKite (very generously!) offers free accounts to developers writing software the meets http://opensource.org/docs/definition.php such as the Dataverse Project. MSV +++ @@ -160,7 +160,7 @@ jmap allows you to look at the contents of the java heap. It can be used to crea will output a list of all classes, sorted by the number of instances of each individual class, with the size in bytes. This can be very useful when looking for memory leaks in the application. Another useful tool is ``jstat``, that can be used in combination with ``jmap`` to monitor the effectiveness of garbage collection in reclaiming allocated memory. -In the example script below we stress running Dataverse applicatione with GET requests to a specific dataverse page, use ``jmap`` to see how many Dataverse, Dataset and DataFile class object get allocated, then run ``jstat`` to see how the numbers are affected by both "Young Generation" and "Full" garbage collection runs (``YGC`` and ``FGC`` respectively): +In the example script below we stress running Dataverse software applicatione with GET requests to a specific page in a Dataverse installation, use ``jmap`` to see how many Dataverse collection, Dataset and DataFile class object get allocated, then run ``jstat`` to see how the numbers are affected by both "Young Generation" and "Full" garbage collection runs (``YGC`` and ``FGC`` respectively): (This is script is provided **as an example only**! You will have to experiment and expand it to suit any specific needs and any specific problem you may be trying to diagnose, and this is just to give an idea of how to go about it) @@ -173,7 +173,7 @@ In the example script below we stress running Dataverse applicatione with GET re while : do - # Access the dataverse xxx 10 times in a row: + # Access the Dataverse collection xxx 10 times in a row: for ((i = 0; i < 10; i++)) do # hide the output, standard and stderr: @@ -186,7 +186,7 @@ In the example script below we stress running Dataverse applicatione with GET re jmap -histo $id > /tmp/jmap.histo.out - # grep the output for Dataverse, Dataset and DataFile classes: + # grep the output for Dataverse Collection, Dataset and DataFile classes: grep '\.Dataverse$' /tmp/jmap.histo.out grep '\.Dataset$' /tmp/jmap.histo.out grep '\.DataFile$' /tmp/jmap.histo.out @@ -237,7 +237,7 @@ The script above will run until you stop it, and will output something like: How to analyze the output, what to look for: -First, look at the numbers in the jmap output. In the example above, you can immediately see, after the first three iterations, that every 10 dataverse page loads results in the increase of the number of Dataset classes by 160. I.e., each page load leaves 16 of these on the heap. We can also see that each of the 10 page load cycles increased the heap by roughly 3GB; that each cycle resulted in a couple of YG (young generation) garbage collections, and in the old generation allocation being almost 70% full. These numbers in the example are clearly quite high and are an indication of some problematic memory allocation by the dataverse page - if this is the result of something you have added to the page, you probably would want to investigate and fix it. However, overly generous memory use **is not the same as a leak** necessarily. What you want to see now is how much of this allocation can be reclaimed by "Full GC". If all of it gets freed by ``FGC``, it is not the end of the world (even though you do not want your system to spend too much time running ``FGC``; it costs CPU cycles, and actually freezes the application while it's in progress!). It is however a **really** serious problem, if you determine that a growing portion of the old. gen. memory (``"O"`` in the ``jmap`` output) is not getting freed, even by ``FGC``. This *is* a real leak now, i.e. something is leaving behind some objects that are still referenced and thus off limits to garbage collector. So look for the lines where the ``FGC`` counter is incremented. For example, the first ``FGC`` in the example output above: +First, look at the numbers in the jmap output. In the example above, you can immediately see, after the first three iterations, that every 10 Dataverse installation page loads results in the increase of the number of Dataset classes by 160. I.e., each page load leaves 16 of these on the heap. We can also see that each of the 10 page load cycles increased the heap by roughly 3GB; that each cycle resulted in a couple of YG (young generation) garbage collections, and in the old generation allocation being almost 70% full. These numbers in the example are clearly quite high and are an indication of some problematic memory allocation by the Dataverse installation page - if this is the result of something you have added to the page, you probably would want to investigate and fix it. However, overly generous memory use **is not the same as a leak** necessarily. What you want to see now is how much of this allocation can be reclaimed by "Full GC". If all of it gets freed by ``FGC``, it is not the end of the world (even though you do not want your system to spend too much time running ``FGC``; it costs CPU cycles, and actually freezes the application while it's in progress!). It is however a **really** serious problem, if you determine that a growing portion of the old. gen. memory (``"O"`` in the ``jmap`` output) is not getting freed, even by ``FGC``. This *is* a real leak now, i.e. something is leaving behind some objects that are still referenced and thus off limits to garbage collector. So look for the lines where the ``FGC`` counter is incremented. For example, the first ``FGC`` in the example output above: .. code-block:: none @@ -257,7 +257,7 @@ First, look at the numbers in the jmap output. In the example above, you can imm 0.00 100.00 71.95 20.12 ? 22 25.034 1 4.455 29.489 Wed Aug 14 23:21:40 EDT 2019 -We can see that the first ``FGC`` resulted in reducing the ``"O"`` by almost 7GB, from 15GB down to 8GB (from 88% to 20% full). The number of Dataset classes has not budged at all - it has grown by the same 160 objects as before (very suspicious!). To complicate matters, ``FGC`` does not **guarantee** to free everything that can be freed - it will balance how much the system needs memory vs. how much it is willing to spend in terms of CPU cycles performing GC (remember, the application freezes while ``FGC`` is running!). So you should not assume that the "20% full" number above means that you have 20% of your stack already wasted and unrecoverable. Instead, look for the next **minium** value of ``"O"``; then for the next, etc. Now compare these consecutive miniums. With the above test (this is an output of a real experiment, a particularly memory-hungry feature added to the dataverse page), the minimums sequence (of old. gen. usage, in %) was looking as follows: +We can see that the first ``FGC`` resulted in reducing the ``"O"`` by almost 7GB, from 15GB down to 8GB (from 88% to 20% full). The number of Dataset classes has not budged at all - it has grown by the same 160 objects as before (very suspicious!). To complicate matters, ``FGC`` does not **guarantee** to free everything that can be freed - it will balance how much the system needs memory vs. how much it is willing to spend in terms of CPU cycles performing GC (remember, the application freezes while ``FGC`` is running!). So you should not assume that the "20% full" number above means that you have 20% of your stack already wasted and unrecoverable. Instead, look for the next **minium** value of ``"O"``; then for the next, etc. Now compare these consecutive miniums. With the above test (this is an output of a real experiment, a particularly memory-hungry feature added to the Dataverse installation page), the minimums sequence (of old. gen. usage, in %) was looking as follows: .. code-block:: none From c8e4404af36c45f787c41d721d0fd4bce0516462 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:48:43 -0500 Subject: [PATCH 047/274] more updates --- .../source/developers/troubleshooting.rst | 16 +++++------ .../source/developers/version-control.rst | 28 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/doc/sphinx-guides/source/developers/troubleshooting.rst b/doc/sphinx-guides/source/developers/troubleshooting.rst index 2964684f7a9..a53a08ce77f 100755 --- a/doc/sphinx-guides/source/developers/troubleshooting.rst +++ b/doc/sphinx-guides/source/developers/troubleshooting.rst @@ -2,7 +2,7 @@ Troubleshooting =============== -Over in the :doc:`dev-environment` section we described the "happy path" of when everything goes right as you set up your Dataverse development environment. Here are some common problems and solutions for when things go wrong. +Over in the :doc:`dev-environment` section we described the "happy path" of when everything goes right as you set up your Dataverse software development environment. Here are some common problems and solutions for when things go wrong. .. contents:: |toctitle| :local: @@ -14,7 +14,7 @@ For unknown reasons, Netbeans will sometimes change the following line under ``s ``/`` -Sometimes Netbeans will change ``/`` to ``/dataverse``. Sometimes it will delete the line entirely. Either way, you will see very strange behavior when attempting to click around Dataverse in a browser. The homepage will load but icons will be missing. Any other page will fail to load entirely and you'll see an app server error. +Sometimes Netbeans will change ``/`` to ``/dataverse``. Sometimes it will delete the line entirely. Either way, you will see very strange behavior when attempting to click around the Dataverse installation in a browser. The homepage will load but icons will be missing. Any other page will fail to load entirely and you'll see an app server error. The solution is to put the file back to how it was before Netbeans touched it. If anyone knows of an open Netbeans bug about this, please let us know. @@ -41,7 +41,7 @@ This command helps verify what host your domain is using to send mail. Even if i 2. From the left-side panel, select **JavaMail Sessions** 3. You should see one session named **mail/notifyMailSession** -- click on that. -From this window you can modify certain fields of your Dataverse's notifyMailSession, which is the JavaMail session for outgoing system email (such as on user signup or data publication). Two of the most important fields we need are: +From this window you can modify certain fields of your Dataverse installation's notifyMailSession, which is the JavaMail session for outgoing system email (such as on user signup or data publication). Two of the most important fields we need are: - **Mail Host:** The DNS name of the default mail server (e.g. smtp.gmail.com) - **Default User:** The username provided to your Mail Host when you connect to it (e.g. johndoe@gmail.com) @@ -50,7 +50,7 @@ Most of the other defaults can safely be left as is. **Default Sender Address** If your user credentials for the SMTP server require a password, you'll need to configure some **Additional Properties** at the bottom. -**IMPORTANT:** Before continuing, it's highly recommended that your Default User account does NOT use a password you share with other accounts, as one of the additional properties includes entering the Default User's password (without concealing it on screen). For smtp.gmail.com you can safely use an `app password `_ or create an extra Gmail account for use with your Dataverse dev environment. +**IMPORTANT:** Before continuing, it's highly recommended that your Default User account does NOT use a password you share with other accounts, as one of the additional properties includes entering the Default User's password (without concealing it on screen). For smtp.gmail.com you can safely use an `app password `_ or create an extra Gmail account for use with your Dataverse software development environment. Authenticating yourself to a Mail Host can be tricky. As an example, we'll walk through setting up our JavaMail Session to use smtp.gmail.com as a host by way of SSL on port 465. Use the Add Property button to generate a blank property for each name/value pair. @@ -74,9 +74,9 @@ The mail session can also be set from command line. To use this method, you will - Delete: ``./asadmin delete-javamail-resource mail/MyMailSession`` - Create (remove brackets and replace the variables inside): ``./asadmin create-javamail-resource --mailhost [smtp.gmail.com] --mailuser [test\@test\.com] --fromaddress [test\@test\.com] --property mail.smtp.auth=[true]:mail.smtp.password=[password]:mail.smtp.port=[465]:mail.smtp.socketFactory.port=[465]:mail.smtp.socketFactory.fallback=[false]:mail.smtp.socketFactory.class=[javax.net.ssl.SSLSocketFactory] mail/notifyMailSession`` -These properties can be tailored to your own preferred mail service, but if all else fails these settings work fine with Dataverse development environments for your localhost. +These properties can be tailored to your own preferred mail service, but if all else fails these settings work fine with Dataverse software development environments for your localhost. -+ If you're seeing a "Relay access denied" error in your app server logs when Dataverse attempts to send an email, double check your user/password credentials for the Mail Host you're using. ++ If you're seeing a "Relay access denied" error in your app server logs when the Dataverse installation attempts to send an email, double check your user/password credentials for the Mail Host you're using. + If you're seeing a "Connection refused" / similar error upon email sending, try another port. As another example, here is how to create a Mail Host via command line for Amazon SES: @@ -97,8 +97,8 @@ A script called :download:`dev-rebuild.sh <../../../../scripts/dev/dev-rebuild.s - Deploys the war file located in the ``target`` directory. - Runs ``setup-all.sh`` in insecure mode so tests will pass. - Runs post-install SQL statements. -- Publishes the root dataverse. -- Adjusts permissions on on the root dataverse so tests will pass. +- Publishes the root Dataverse collection. +- Adjusts permissions on on the root Dataverse collection so tests will pass. To execute the script, make sure you have built a war file already and then ``cd`` to the root of the source tree and run ``scripts/dev/dev-rebuild.sh``. Feedback on this script is welcome! diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst index eaaf0fa1911..e6161966563 100644 --- a/doc/sphinx-guides/source/developers/version-control.rst +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -2,16 +2,16 @@ Version Control ================== -The Dataverse Project uses git for version control and GitHub for hosting. On this page we'll explain where to find the code, our branching strategey, advice on how to make a pull request, and other git tips. +The Dataverse Project uses git for version control and GitHub for hosting. On this page we'll explain where to find the code, our branching strategy, advice on how to make a pull request, and other git tips. .. contents:: |toctitle| :local: -Where to Find the Dataverse Code --------------------------------- +Where to Find the Dataverse Software Code +----------------------------------------- -The main Dataverse code at https://github.com/IQSS/dataverse but as explained in the :doc:`intro` section under "Related Projects", there are many other code bases you can hack on if you wish! +The main Dataverse software code at https://github.com/IQSS/dataverse but as explained in the :doc:`intro` section under "Related Projects", there are many other code bases you can hack on if you wish! Branching Strategy ------------------ @@ -19,7 +19,7 @@ Branching Strategy Goals ~~~~~ -The goals of the Dataverse branching strategy are: +The goals of the Dataverse software branching strategy are: - allow for concurrent development - only ship stable code @@ -32,7 +32,7 @@ Branches The "master" Branch ******************* -The "`master `_" branch represents released versions of Dataverse. As mentioned in the :doc:`making-releases` section, at release time we update the master branch to include all the code for that release. Commits are never made directly to master. Rather, master is updated only when we merge code into it from the "develop" branch. +The "`master `_" branch represents released versions of the Dataverse software. As mentioned in the :doc:`making-releases` section, at release time we update the master branch to include all the code for that release. Commits are never made directly to master. Rather, master is updated only when we merge code into it from the "develop" branch. The "develop" Branch ******************** @@ -60,7 +60,7 @@ Find or Create a GitHub Issue For guidance on which issue to work on, please ask! Also, see https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md -Let's say you want to tackle https://github.com/IQSS/dataverse/issues/3728 which points out a typo in a page of Dataverse's documentation. +Let's say you want to tackle https://github.com/IQSS/dataverse/issues/3728 which points out a typo in a page of the Dataverse software's documentation. If you tell us your GitHub username we are happy to add you to the "read only" team at https://github.com/orgs/IQSS/teams/dataverse-readonly/members so that we can assign the issue to you while you're working on it. You can also tell us if you'd like to be added to the `Dataverse Community Contributors spreadsheet `_. @@ -77,7 +77,7 @@ Making a commit (or several commits) to that branch. Ideally the first line of y Push Your Branch to GitHub ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Push your feature branch to your fork of Dataverse. Your git command may look something like ``git push origin 3728-doc-apipolicy-fix``. +Push your feature branch to your fork of the Dataverse software. Your git command may look something like ``git push origin 3728-doc-apipolicy-fix``. Make a Pull Request ~~~~~~~~~~~~~~~~~~~ @@ -98,8 +98,8 @@ Summary of Git commands This section provides sequences of Git commands for two scenarios: -* preparing the first request, when the IQSS Dataverse repository and the forked repository are identical -* creating an additional request after some time, when the IQSS Dataverse repository is ahead of the forked repository +* preparing the first request, when the IQSS Dataverse software repository and the forked repository are identical +* creating an additional request after some time, when the IQSS Dataverse software repository is ahead of the forked repository In the examples we use 123-COOL-FEATURE as the name of the feature branch, and https://github.com/YOUR_NAME/dataverse.git as your forked repository's URL. In practice modify both accordingly. @@ -157,7 +157,7 @@ How to Resolve Conflicts in Your Pull Request Unfortunately, pull requests can quickly become "stale" and unmergable as other pull requests are merged into the develop branch ahead of you. This is completely normal, and often occurs because other developers made their pull requests before you did. -The Dataverse team may ping you to ask you to merge the latest from the develop branch into your branch and resolve merge conflicts. If this sounds daunting, please just say so and we will assist you. +The Dataverse Project team may ping you to ask you to merge the latest from the develop branch into your branch and resolve merge conflicts. If this sounds daunting, please just say so and we will assist you. If you'd like to resolve the merge conflicts yourself, here are some steps to do so that make use of GitHub Desktop and Netbeans. @@ -169,7 +169,7 @@ If you'd like to resolve the merge conflicts yourself, here are some steps to do **In Netbeans:** -4. Click Window -> Favorites and open your local Dataverse project folder in the Favorites panel. +4. Click Window -> Favorites and open your local Dataverse software project folder in the Favorites panel. 5. In this file browser, you can follow the red cylinder icon to find files with merge conflicts. 6. Double click the red merge conflicted file. 7. Right click on the red tab for that file and select Git -> Resolve Conflicts. @@ -182,14 +182,14 @@ If you'd like to resolve the merge conflicts yourself, here are some steps to do **In GitHub Issues:** -11. Leave a comment for the Dataverse team that you have resolved the merge conflicts. +11. Leave a comment for the Dataverse Project team that you have resolved the merge conflicts. Adding Commits to a Pull Request from a Fork -------------------------------------------- By default, when a pull request is made from a fork, "Allow edits from maintainers" is checked as explained at https://help.github.com/articles/allowing-changes-to-a-pull-request-branch-created-from-a-fork/ -This is a nice feature of GitHub because it means that the core dev team for Dataverse can make small (or even large) changes to a pull request from a contributor to help the pull request along on its way to QA and being merged. +This is a nice feature of GitHub because it means that the core dev team for the Dataverse Project can make small (or even large) changes to a pull request from a contributor to help the pull request along on its way to QA and being merged. GitHub documents how to make changes to a fork at https://help.github.com/articles/committing-changes-to-a-pull-request-branch-created-from-a-fork/ but as of this writing the steps involve making a new clone of the repo. This works but you might find it more convenient to add a "remote" to your existing clone. The example below uses the fork at https://github.com/OdumInstitute/dataverse and the branch ``4709-postgresql_96`` but the technique can be applied to any fork and branch: From 0aa8c5e7e934e514805ddb434c229f2cf9b16aa9 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 22:53:35 -0500 Subject: [PATCH 048/274] more changes --- .../source/developers/windows.rst | 10 +++++----- .../source/developers/workflows.rst | 20 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index ec5d4cb971f..0cfc329377e 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -4,13 +4,13 @@ Windows Development Development on Windows is not well supported, unfortunately. You will have a much easier time if you develop on Mac or Linux as described under :doc:`dev-environment` section. -Vagrant commands appear below and were tested on Windows 10 but the Vagrant environement is currently broken. Please see https://github.com/IQSS/dataverse/issues/6849 +Vagrant commands appear below and were tested on Windows 10 but the Vagrant environment is currently broken. Please see https://github.com/IQSS/dataverse/issues/6849 .. contents:: |toctitle| :local: -Running Dataverse in Vagrant ----------------------------- +Running the Dataverse Software in Vagrant +----------------------------------------- Install Vagrant ~~~~~~~~~~~~~~~ @@ -69,7 +69,7 @@ The ``dataverse`` directory you changed is the one you just cloned. Vagrant will ``vagrant up`` -After a long while you hopefully will have Dataverse installed at http://localhost:8888 +After a long while you hopefully will have a Dataverse installation available at http://localhost:8888 Improving Windows Support ------------------------- @@ -82,4 +82,4 @@ We have been unable to get Windows Subsystem for Linux (WSL) to work. We tried f Discussion and Feedback ~~~~~~~~~~~~~~~~~~~~~~~ -For more discussion of Windows support for Dataverse development see our community list thread `"Do you want to develop on Windows?" `_ We would be happy to incorporate feedback from Windows developers into this page. The :doc:`documentation` section describes how. +For more discussion of Windows support for Dataverse software development see our community list thread `"Do you want to develop on Windows?" `_ We would be happy to incorporate feedback from Windows developers into this page. The :doc:`documentation` section describes how. diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index b9090b86be3..f823f8459b1 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -1,7 +1,7 @@ Workflows ================ -Dataverse has a flexible workflow mechanism that can be used to trigger actions before and after Dataset publication. +The Dataverse software has a flexible workflow mechanism that can be used to trigger actions before and after Dataset publication. .. contents:: |toctitle| :local: @@ -10,35 +10,35 @@ Dataverse has a flexible workflow mechanism that can be used to trigger actions Introduction ------------ -Dataverse can perform two sequences of actions when datasets are published: one prior to publishing (marked by a ``PrePublishDataset`` trigger), and one after the publication has succeeded (``PostPublishDataset``). The pre-publish workflow is useful for having an external system prepare a dataset for being publicly accessed (a possibly lengthy activity that requires moving files around, uploading videos to a streaming server, etc.), or to start an approval process. A post-publish workflow might be used for sending notifications about the newly published dataset. +The Dataverse software can perform two sequences of actions when datasets are published: one prior to publishing (marked by a ``PrePublishDataset`` trigger), and one after the publication has succeeded (``PostPublishDataset``). The pre-publish workflow is useful for having an external system prepare a dataset for being publicly accessed (a possibly lengthy activity that requires moving files around, uploading videos to a streaming server, etc.), or to start an approval process. A post-publish workflow might be used for sending notifications about the newly published dataset. -Workflow steps are created using *step providers*. Dataverse ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers. This allows installations to implement functionality they need without changing the Dataverse source code. +Workflow steps are created using *step providers*. The Dataverse software ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers. This allows installations to implement functionality they need without changing the Dataverse software source code. -Steps can be internal (say, writing some data to the log) or external. External steps involve Dataverse sending a request to an external system, and waiting for the system to reply. The wait period is arbitrary, and so allows the external system unbounded operation time. This is useful, e.g., for steps that require human intervension, such as manual approval of a dataset publication. +Steps can be internal (say, writing some data to the log) or external. External steps involve the Dataverse software sending a request to an external system, and waiting for the system to reply. The wait period is arbitrary, and so allows the external system unbounded operation time. This is useful, e.g., for steps that require human intervention, such as manual approval of a dataset publication. -The external system reports the step result back to dataverse, by sending a HTTP ``POST`` command to ``api/workflows/{invocation-id}``. The body of the request is passed to the paused step for further processing. +The external system reports the step result back to the Dataverse installation, by sending a HTTP ``POST`` command to ``api/workflows/{invocation-id}``. The body of the request is passed to the paused step for further processing. -If a step in a workflow fails, Dataverse make an effort to roll back all the steps that preceded it. Some actions, such as writing to the log, cannot be rolled back. If such an action has a public external effect (e.g. send an EMail to a mailing list) it is advisable to put it in the post-release workflow. +If a step in a workflow fails, the Dataverse installation makes an effort to roll back all the steps that preceded it. Some actions, such as writing to the log, cannot be rolled back. If such an action has a public external effect (e.g. send an EMail to a mailing list) it is advisable to put it in the post-release workflow. .. tip:: - For invoking external systems using a REST api, Dataverse's internal step + For invoking external systems using a REST api, the Dataverse software's internal step provider offers a step for sending and receiving customizable HTTP requests. It's called *http/sr*, and is detailed below. Administration ~~~~~~~~~~~~~~ -A Dataverse instance stores a set of workflows in its database. Workflows can be managed using the ``api/admin/workflows/`` endpoints of the :doc:`/api/native-api`. Sample workflow files are available in ``scripts/api/data/workflows``. +A Dataverse installation stores a set of workflows in its database. Workflows can be managed using the ``api/admin/workflows/`` endpoints of the :doc:`/api/native-api`. Sample workflow files are available in ``scripts/api/data/workflows``. At the moment, defining a workflow for each trigger is done for the entire instance, using the endpoint ``api/admin/workflows/default/«trigger type»``. -In order to prevent unauthorized resuming of workflows, Dataverse maintains a "white list" of IP addresses from which resume requests are honored. This list is maintained using the ``/api/admin/workflows/ip-whitelist`` endpoint of the :doc:`/api/native-api`. By default, Dataverse honors resume requests from localhost only (``127.0.0.1;::1``), so set-ups that use a single server work with no additional configuration. +In order to prevent unauthorized resuming of workflows, the Dataverse installation maintains a "white list" of IP addresses from which resume requests are honored. This list is maintained using the ``/api/admin/workflows/ip-whitelist`` endpoint of the :doc:`/api/native-api`. By default, the Dataverse installation honors resume requests from localhost only (``127.0.0.1;::1``), so set-ups that use a single server work with no additional configuration. Available Steps ~~~~~~~~~~~~~~~ -Dataverse has an internal step provider, whose id is ``:internal``. It offers the following steps: +The Dataverse software has an internal step provider, whose id is ``:internal``. It offers the following steps: log +++ From 2f4fadd5ac09ca5afaf9bd65bf316a30fcb6186b Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 23:00:23 -0500 Subject: [PATCH 049/274] onto the installation guide! --- doc/sphinx-guides/source/installation/config.rst | 8 ++++---- doc/sphinx-guides/source/installation/external-tools.rst | 2 +- doc/sphinx-guides/source/installation/geoconnect.rst | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 4a877eabff7..09bc1647d19 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2,9 +2,9 @@ Configuration ============= -Now that you've successfully logged into Dataverse with a superuser account after going through a basic :doc:`installation-main`, you'll need to secure and configure your installation. +Now that you've successfully logged into your Dataverse installation with a superuser account after going through a basic :doc:`installation-main`, you'll need to secure and configure your installation. -Settings within Dataverse itself are managed via JVM options or by manipulating values in the ``setting`` table directly or through API calls. +Settings within your Dataverse installation itself are managed via JVM options or by manipulating values in the ``setting`` table directly or through API calls. Once you have finished securing and configuring your Dataverse installation, you may proceed to the :doc:`/admin/index` for more information on the ongoing administration of a Dataverse installation. Advanced configuration topics are covered in the :doc:`r-rapache-tworavens`, :doc:`shibboleth` and :doc:`oauth2` sections. @@ -45,9 +45,9 @@ To avoid having your users send credentials in the clear, it's strongly recommen Recording User IP Addresses +++++++++++++++++++++++++++ -By default, Dataverse captures the IP address from which requests originate. This is used for multiple purposes including controlling access to the admin API, IP-based user groups and Make Data Count reporting. When Dataverse is configured behind a proxy such as a load balancer, this default setup may not capture the correct IP address. In this case all the incoming requests will be logged in the access logs, MDC logs etc., as if they are all coming from the IP address(es) of the load balancer itself. Proxies usually save the original address in an added HTTP header, from which it can be extracted. For example, AWS LB records the "true" original address in the standard ``X-Forwarded-For`` header. If your Dataverse is running behind an IP-masking proxy, but you would like to use IP groups, or record the true geographical location of the incoming requests with Make Data Count, you may enable the IP address lookup from the proxy header using the JVM option ``dataverse.useripaddresssourceheader``, described further below. +By default, the Dataverse installation captures the IP address from which requests originate. This is used for multiple purposes including controlling access to the admin API, IP-based user groups and Make Data Count reporting. When the Dataverse installation is configured behind a proxy such as a load balancer, this default setup may not capture the correct IP address. In this case all the incoming requests will be logged in the access logs, MDC logs etc., as if they are all coming from the IP address(es) of the load balancer itself. Proxies usually save the original address in an added HTTP header, from which it can be extracted. For example, AWS LB records the "true" original address in the standard ``X-Forwarded-For`` header. If your Dataverse installation is running behind an IP-masking proxy, but you would like to use IP groups, or record the true geographical location of the incoming requests with Make Data Count, you may enable the IP address lookup from the proxy header using the JVM option ``dataverse.useripaddresssourceheader``, described further below. -Before doing so however, you must absolutely **consider the security risks involved**! This option must be enabled **only** on a Dataverse that is in fact fully behind a proxy that properly, and consistently, adds the ``X-Forwarded-For`` (or a similar) header to every request it forwards. Consider the implications of activating this option on a Dataverse that is not running behind a proxy, *or running behind one, but still accessible from the insecure locations bypassing the proxy*: Anyone can now add the header above to an incoming reqest, supplying an arbitrary IP address that Dataverse will trust as the true origin of the call. Thus giving an attacker an easy way to, for example, get in a privileged IP group. The implications could be even more severe if an attacker were able to pretend to be coming from ``localhost``, if Dataverse is configured to trust localhost connections for unrestricted access to the admin API! We have addressed this by making it so that Dataverse should never accept ``localhost``, ``127.0.0.1``, ``0:0:0:0:0:0:0:1`` etc. when supplied in such a header. But if you have reasons to still find this risk unacceptable, you may want to consider turning open localhost access to the API off (See :ref:`Securing Your Installation ` for more information.) +Before doing so however, you must absolutely **consider the security risks involved**! This option must be enabled **only** on a Dataverse installation that is in fact fully behind a proxy that properly, and consistently, adds the ``X-Forwarded-For`` (or a similar) header to every request it forwards. Consider the implications of activating this option on a Dataverse installation that is not running behind a proxy, *or running behind one, but still accessible from the insecure locations bypassing the proxy*: Anyone can now add the header above to an incoming request, supplying an arbitrary IP address that the Dataverse installation will trust as the true origin of the call. Thus giving an attacker an easy way to, for example, get in a privileged IP group. The implications could be even more severe if an attacker were able to pretend to be coming from ``localhost``, if a Dataverse installation is configured to trust localhost connections for unrestricted access to the admin API! We have addressed this by making it so that Dataverse installation should never accept ``localhost``, ``127.0.0.1``, ``0:0:0:0:0:0:0:1`` etc. when supplied in such a header. But if you have reasons to still find this risk unacceptable, you may want to consider turning open localhost access to the API off (See :ref:`Securing Your Installation ` for more information.) This is how to verify that your proxy or load balancer, etc. is handling the originating address headers properly and securely: Make sure access logging is enabled in your application server (Payara) configuration. (```` in the ``domain.xml``). Add the address header to the access log format. For example, on a system behind AWS ELB, you may want to use something like ``%client.name% %datetime% %request% %status% %response.length% %header.referer% %header.x-forwarded-for%``. Once enabled, access the Dataverse from outside the LB. You should now see the real IP address of your remote client in the access log. For example, something like: ``"1.2.3.4" "01/Jun/2020:12:00:00 -0500" "GET /dataverse.xhtml HTTP/1.1" 200 81082 "NULL-REFERER" "128.64.32.16"`` diff --git a/doc/sphinx-guides/source/installation/external-tools.rst b/doc/sphinx-guides/source/installation/external-tools.rst index 028802fd736..2517a2c14c8 100644 --- a/doc/sphinx-guides/source/installation/external-tools.rst +++ b/doc/sphinx-guides/source/installation/external-tools.rst @@ -1,7 +1,7 @@ External Tools ============== -External tools can provide additional features that are not part of Dataverse itself, such as data exploration. +External tools can provide additional features that are not part of the Dataverse software itself, such as data exploration. .. contents:: |toctitle| :local: diff --git a/doc/sphinx-guides/source/installation/geoconnect.rst b/doc/sphinx-guides/source/installation/geoconnect.rst index 14a12b50a3f..430c0eb807c 100644 --- a/doc/sphinx-guides/source/installation/geoconnect.rst +++ b/doc/sphinx-guides/source/installation/geoconnect.rst @@ -4,7 +4,7 @@ Geoconnect .. contents:: |toctitle| :local: -Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. +Geoconnect works as a middle layer, allowing geospatial data files in a Dataverse installation to be visualized with Harvard WorldMap. To understand the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. @@ -12,6 +12,6 @@ As of this writing, the README at https://github.com/IQSS/geoconnect recommends To set up a Geoconnect development environment, you can follow the steps outlined in the `local_setup.md `_ guide. Although those instructions are for a local development setup, they may assist in installing Geoconnect in your production environment. -Harvard Dataverse runs Geoconnect on Heroku. To make use of Heroku, you will need a Heroku account, as well as a few other prerequisites. Follow the instructions outlined in the `heroku_setup.md `_ guide. The `heroku.py `_ settings file may also be adapted for other environments. Please note, for the production environment, remember to set ``DEBUG=False``. +The Harvard Dataverse Repository runs Geoconnect on Heroku. To make use of Heroku, you will need a Heroku account, as well as a few other prerequisites. Follow the instructions outlined in the `heroku_setup.md `_ guide. The `heroku.py `_ settings file may also be adapted for other environments. Please note, for the production environment, remember to set ``DEBUG=False``. See also the :doc:`/admin/geoconnect-worldmap` section of the Admin Guide. From b81bef41da0e4164df4d508fae2b556c55dabbf6 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 12 Jan 2021 23:35:47 -0500 Subject: [PATCH 050/274] more updates --- .../source/installation/config.rst | 257 +++++++++--------- 1 file changed, 128 insertions(+), 129 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 09bc1647d19..37f419f2cbd 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -49,16 +49,16 @@ By default, the Dataverse installation captures the IP address from which reques Before doing so however, you must absolutely **consider the security risks involved**! This option must be enabled **only** on a Dataverse installation that is in fact fully behind a proxy that properly, and consistently, adds the ``X-Forwarded-For`` (or a similar) header to every request it forwards. Consider the implications of activating this option on a Dataverse installation that is not running behind a proxy, *or running behind one, but still accessible from the insecure locations bypassing the proxy*: Anyone can now add the header above to an incoming request, supplying an arbitrary IP address that the Dataverse installation will trust as the true origin of the call. Thus giving an attacker an easy way to, for example, get in a privileged IP group. The implications could be even more severe if an attacker were able to pretend to be coming from ``localhost``, if a Dataverse installation is configured to trust localhost connections for unrestricted access to the admin API! We have addressed this by making it so that Dataverse installation should never accept ``localhost``, ``127.0.0.1``, ``0:0:0:0:0:0:0:1`` etc. when supplied in such a header. But if you have reasons to still find this risk unacceptable, you may want to consider turning open localhost access to the API off (See :ref:`Securing Your Installation ` for more information.) -This is how to verify that your proxy or load balancer, etc. is handling the originating address headers properly and securely: Make sure access logging is enabled in your application server (Payara) configuration. (```` in the ``domain.xml``). Add the address header to the access log format. For example, on a system behind AWS ELB, you may want to use something like ``%client.name% %datetime% %request% %status% %response.length% %header.referer% %header.x-forwarded-for%``. Once enabled, access the Dataverse from outside the LB. You should now see the real IP address of your remote client in the access log. For example, something like: +This is how to verify that your proxy or load balancer, etc. is handling the originating address headers properly and securely: Make sure access logging is enabled in your application server (Payara) configuration. (```` in the ``domain.xml``). Add the address header to the access log format. For example, on a system behind AWS ELB, you may want to use something like ``%client.name% %datetime% %request% %status% %response.length% %header.referer% %header.x-forwarded-for%``. Once enabled, access the Dataverse installation from outside the LB. You should now see the real IP address of your remote client in the access log. For example, something like: ``"1.2.3.4" "01/Jun/2020:12:00:00 -0500" "GET /dataverse.xhtml HTTP/1.1" 200 81082 "NULL-REFERER" "128.64.32.16"`` In this example, ``128.64.32.16`` is your remote address (that you should verify), and ``1.2.3.4`` is the address of your LB. If you're not seeing your remote address in the log, do not activate the JVM option! Also, verify that all the entries in the log have this header populated. The only entries in the access log that you should be seeing without this header (logged as ``"NULL-HEADER-X-FORWARDED-FOR"``) are local requests, made from localhost, etc. In this case, since the request is not coming through the proxy, the local IP address should be logged as the primary one (as the first value in the log entry, ``%client.name%``). If you see any requests coming in from remote, insecure subnets without this header - do not use the JVM option! -Once you are ready, enable the :ref:`JVM option `. Verify that the remote locations are properly tracked in your MDC metrics, and/or your IP groups are working. As a final test, if your Dataverse is allowing unrestricted localhost access to the admin API, imitate an attack in which a malicious request is pretending to be coming from ``127.0.0.1``. Try the following from a remote, insecure location: +Once you are ready, enable the :ref:`JVM option `. Verify that the remote locations are properly tracked in your MDC metrics, and/or your IP groups are working. As a final test, if your Dataverse installation is allowing unrestricted localhost access to the admin API, imitate an attack in which a malicious request is pretending to be coming from ``127.0.0.1``. Try the following from a remote, insecure location: ``curl https://your.dataverse.edu/api/admin/settings --header "X-FORWARDED-FOR: 127.0.0.1"`` -First of all, confirm that access is denied! If you are in fact able to access the settings api from a location outside the proxy, **something is seriously wrong**, so please let us know, and stop using the JVM option. Otherwise check the access log entry for the header value. What you should see is something like ``"127.0.0.1, 128.64.32.16"``. Where the second address should be the real IP of your remote client. The fact that the "fake" ``127.0.0.1`` you sent over is present in the header is perfectly ok. This is the proper proxy behavior - it preserves any incoming values in the ``X-Forwarded-Header``, if supplied, and adds the detected incoming address to it, *on the right*. It is only this rightmost comma-separated value that Dataverse should ever be using. +First of all, confirm that access is denied! If you are in fact able to access the settings api from a location outside the proxy, **something is seriously wrong**, so please let us know, and stop using the JVM option. Otherwise check the access log entry for the header value. What you should see is something like ``"127.0.0.1, 128.64.32.16"``. Where the second address should be the real IP of your remote client. The fact that the "fake" ``127.0.0.1`` you sent over is present in the header is perfectly ok. This is the proper proxy behavior - it preserves any incoming values in the ``X-Forwarded-Header``, if supplied, and adds the detected incoming address to it, *on the right*. It is only this rightmost comma-separated value that Dataverse installation should ever be using. Still feel like activating this option in your configuration? - Have fun and be safe! @@ -71,7 +71,7 @@ Privacy Considerations Email Privacy ^^^^^^^^^^^^^ -Out of the box, Dataverse will list email addresses of the contacts for datasets when users visit a dataset page and click the "Export Metadata" button. Additionally, out of the box, Dataverse will list email addresses of dataverse contacts via API (see :ref:`View a Dataverse ` in the :doc:`/api/native-api` section of the API Guide). If you would like to exclude these email addresses from export, set :ref:`:ExcludeEmailFromExport <:ExcludeEmailFromExport>` to true. +Out of the box, your Dataverse installation will list email addresses of the contacts for datasets when users visit a dataset page and click the "Export Metadata" button. Additionally, out of the box, the Dataverse installation will list email addresses of Dataverse collection contacts via API (see :ref:`View a Dataverse Collection ` in the :doc:`/api/native-api` section of the API Guide). If you would like to exclude these email addresses from export, set :ref:`:ExcludeEmailFromExport <:ExcludeEmailFromExport>` to true. Additional Recommendations ++++++++++++++++++++++++++ @@ -86,9 +86,9 @@ Related to this is that you should remove ``/root/.payara/pass`` to ensure that Enforce Strong Passwords for User Accounts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Dataverse only stores passwords (as salted hash, and using a strong hashing algorithm) for "builtin" users. You can increase the password complexity rules to meet your security needs. If you have configured your Dataverse installation to allow login from remote authentication providers such as Shibboleth, ORCID, GitHub or Google, you do not have any control over those remote providers' password complexity rules. See the :ref:`auth-modes` section below for more on login options. +Your Dataverse installation only stores passwords (as salted hash, and using a strong hashing algorithm) for "builtin" users. You can increase the password complexity rules to meet your security needs. If you have configured your Dataverse installation to allow login from remote authentication providers such as Shibboleth, ORCID, GitHub or Google, you do not have any control over those remote providers' password complexity rules. See the :ref:`auth-modes` section below for more on login options. -Even if you are satisfied with the out-of-the-box password complexity rules Dataverse ships with, for the "dataverseAdmin" account you should use a strong password so the hash cannot easily be cracked through dictionary attacks. +Even if you are satisfied with the out-of-the-box password complexity rules the Dataverse software ships with, for the "dataverseAdmin" account you should use a strong password so the hash cannot easily be cracked through dictionary attacks. Password complexity rules for "builtin" accounts can be adjusted with a variety of settings documented below. Here's a list: @@ -106,15 +106,15 @@ Password complexity rules for "builtin" accounts can be adjusted with a variety Network Ports ------------- -Remember how under "Decisions to Make" in the :doc:`prep` section we mentioned you'll need to make a decision about whether or not to introduce a proxy in front of Dataverse such as Apache or nginx? The time has come to make that decision. +Remember how under "Decisions to Make" in the :doc:`prep` section we mentioned you'll need to make a decision about whether or not to introduce a proxy in front of the Dataverse software such as Apache or nginx? The time has come to make that decision. -The need to redirect port HTTP (port 80) to HTTPS (port 443) for security has already been mentioned above and the fact that Payara puts these services on 8080 and 8181, respectively, was touched on in the :doc:`installation-main` section. In production, you don't want to tell your users to use Dataverse on ports 8080 and 8181. You should have them use the standard HTTPS port, which is 443. +The need to redirect port HTTP (port 80) to HTTPS (port 443) for security has already been mentioned above and the fact that Payara puts these services on 8080 and 8181, respectively, was touched on in the :doc:`installation-main` section. In production, you don't want to tell your users to use your Dataverse installation on ports 8080 and 8181. You should have them use the standard HTTPS port, which is 443. -Your decision to proxy or not should primarily be driven by which features of Dataverse you'd like to use. If you'd like to use Shibboleth, the decision is easy because proxying or "fronting" Payara with Apache is required. The details are covered in the :doc:`shibboleth` section. +Your decision to proxy or not should primarily be driven by which features of the Dataverse software you'd like to use. If you'd like to use Shibboleth, the decision is easy because proxying or "fronting" Payara with Apache is required. The details are covered in the :doc:`shibboleth` section. If you'd like to use TwoRavens, you should also consider fronting with Apache because you will be required to install an Apache anyway to make use of the rApache module. For details, see the :doc:`r-rapache-tworavens` section. -Even if you have no interest in Shibboleth nor TwoRavens, you may want to front Dataverse with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse team `_, but the process of adding a certificate to Payara is arduous and not for the faint of heart. The Dataverse team cannot provide much help with adding certificates to Payara beyond linking to `tips `_ on the web. +Even if you have no interest in Shibboleth nor TwoRavens, you may want to front your Dataverse installation with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse Project team `_, but the process of adding a certificate to Payara is arduous and not for the faint of heart. The Dataverse Project team cannot provide much help with adding certificates to Payara beyond linking to `tips `_ on the web. Still not convinced you should put Payara behind another web server? Even if you manage to get your SSL certificate into Payara, how are you going to run Payara on low ports such as 80 and 443? Are you going to run Payara as root? Bad idea. This is a security risk. Under "Additional Recommendations" under "Securing Your Installation" above you are advised to configure Payara to run as a user other than root. @@ -126,7 +126,7 @@ If you really don't want to front Payara with any proxy (not recommended), you c ``./asadmin set server-config.network-config.network-listeners.network-listener.http-listener-2.port=443`` -What about port 80? Even if you don't front Dataverse with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Payara uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Payara can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: http://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to +What about port 80? Even if you don't front your Dataverse installation with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Payara uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Payara can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: http://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to If you are running an installation with Apache and Payara on the same server, and would like to restrict Payara from responding to any requests to port 8080 from external hosts (in other words, not through Apache), you can restrict the AJP listener to localhost only with: @@ -134,19 +134,19 @@ If you are running an installation with Apache and Payara on the same server, an You should **NOT** use the configuration option above if you are running in a load-balanced environment, or otherwise have the web server on a different host than the application server. -Root Dataverse Permissions --------------------------- +Root Dataverse Collection Permissions +------------------------------------- -The user who creates a dataverse is given the "Admin" role on that dataverse. The root dataverse is created automatically for you by the installer and the "Admin" is the superuser account ("dataverseAdmin") we used in the :doc:`installation-main` section to confirm that we can log in. These next steps of configuring the root dataverse require the "Admin" role on the root dataverse, but not the much more powerful superuser attribute. In short, users with the "Admin" role are subject to the permission system. A superuser, on the other hand, completely bypasses the permission system. You can give non-superusers the "Admin" role on the root dataverse if you'd like them to configure the root dataverse. +The user who creates a Dataverse collection is given the "Admin" role on that Dataverse collection. The root Dataverse collection is created automatically for you by the installer and the "Admin" is the superuser account ("dataverseAdmin") we used in the :doc:`installation-main` section to confirm that we can log in. These next steps of configuring the root Dataverse collection require the "Admin" role on the root Dataverse collection, but not the much more powerful superuser attribute. In short, users with the "Admin" role are subject to the permission system. A superuser, on the other hand, completely bypasses the permission system. You can give non-superusers the "Admin" role on the root Dataverse collection if you'd like them to configure the root Dataverse collection. -In order for non-superusers to start creating dataverses or datasets, you need click "Edit" then "Permissions" and make choices about which users can add dataverses or datasets within the root dataverse. (There is an API endpoint for this operation as well.) Again, the user who creates a dataverse will be granted the "Admin" role on that dataverse. Non-superusers who are not "Admin" on the root dataverse will not be able to do anything useful until the root dataverse has been published. +In order for non-superusers to start creating Dataverse collections or datasets, you need click "Edit" then "Permissions" and make choices about which users can add Dataverse collections or datasets within the root Dataverse collection. (There is an API endpoint for this operation as well.) Again, the user who creates a Dataverse collection will be granted the "Admin" role on that Dataverse collection. Non-superusers who are not "Admin" on the root Dataverse collection will not be able to do anything useful until the root Dataverse collection has been published. -As the person installing Dataverse you may or may not be a local metadata expert. You may want to have others sign up for accounts and grant them the "Admin" role at the root dataverse to configure metadata fields, templates, browse/search facets, guestbooks, etc. For more on these topics, consult the :doc:`/user/dataverse-management` section of the User Guide. +As the person installing the Dataverse software, you may or may not be a local metadata expert. You may want to have others sign up for accounts and grant them the "Admin" role at the root Dataverse collection to configure metadata fields, templates, browse/search facets, guestbooks, etc. For more on these topics, consult the :doc:`/user/dataverse-management` section of the User Guide. Persistent Identifiers and Publishing Datasets ---------------------------------------------- -Persistent identifiers are a required and integral part of the Dataverse platform. They provide a URL that is guaranteed to resolve to the datasets or files they represent. Dataverse currently supports creating identifiers using DOI and Handle. +Persistent identifiers are a required and integral part of the Dataverse Platform. They provide a URL that is guaranteed to resolve to the datasets or files they represent. The Dataverse software currently supports creating identifiers using DOI and Handle. By default, the installer configures a default DOI namespace (10.5072) with DataCite as the registration provider. Please note that as of the release 4.9.3, we can no longer use EZID as the provider. Unlike EZID, DataCite requires that you register for a test account, configured with your own prefix (please contact support@datacite.org). Once you receive the login name, password, and prefix for the account, configure the credentials in your domain.xml, as the following two JVM options:: @@ -157,16 +157,16 @@ and restart Payara. The prefix can be configured via the API (where it is referr ``curl -X PUT -d 10.xxxx http://localhost:8080/api/admin/settings/:Authority`` -Once this is done, you will be able to publish datasets and files, but the persistent identifiers will not be citable, and they will only resolve from the DataCite test environment (and then only if the Dataverse from which you published them is accessible - DOIs minted from your laptop will not resolve). Note that any datasets or files created using the test configuration cannot be directly migrated and would need to be created again once a valid DOI namespace is configured. +Once this is done, you will be able to publish datasets and files, but the persistent identifiers will not be citable, and they will only resolve from the DataCite test environment (and then only if the Dataverse installation from which you published them is accessible - DOIs minted from your laptop will not resolve). Note that any datasets or files created using the test configuration cannot be directly migrated and would need to be created again once a valid DOI namespace is configured. To properly configure persistent identifiers for a production installation, an account and associated namespace must be acquired for a fee from a DOI or HDL provider. **DataCite** (https://www.datacite.org) is the recommended DOI provider (see https://dataverse.org/global-dataverse-community-consortium for more on joining DataCite) but **EZID** (http://ezid.cdlib.org) is an option for the University of California according to https://www.cdlib.org/cdlinfo/2017/08/04/ezid-doi-service-is-evolving/ . **Handle.Net** (https://www.handle.net) is the HDL provider. -Once you have your DOI or Handle account credentials and a namespace, configure Dataverse to use them using the JVM options and database settings below. +Once you have your DOI or Handle account credentials and a namespace, configure your Dataverse installation to use them using the JVM options and database settings below. -Configuring Dataverse for DOIs -++++++++++++++++++++++++++++++ +Configuring Your Dataverse Installation for DOIs +++++++++++++++++++++++++++++++++++++++++++++++++ -By default Dataverse attempts to register DOIs for each dataset and file under a test authority, though you must apply for your own credentials as explained above. +By default, your Dataverse installation attempts to register DOIs for each dataset and file under a test authority, though you must apply for your own credentials as explained above. Here are the configuration options for DOIs: @@ -187,8 +187,8 @@ Here are the configuration options for DOIs: - :ref:`:DataFilePIDFormat <:DataFilePIDFormat>` (optional) - :ref:`:FilePIDsEnabled <:FilePIDsEnabled>` (optional, defaults to true) -Configuring Dataverse for Handles -+++++++++++++++++++++++++++++++++ +Configuring Your Dataverse Installation for Handles ++++++++++++++++++++++++++++++++++++++++++++++++++++ Here are the configuration options for handles: @@ -213,17 +213,17 @@ Note: If you are **minting your own handles** and plan to set up your own handle Auth Modes: Local vs. Remote vs. Both ------------------------------------- -There are three valid configurations or modes for authenticating users to Dataverse: +There are three valid configurations or modes for authenticating users to your Dataverse installation: Local Only Auth +++++++++++++++ -Out of the box, Dataverse is configured in "local only" mode. The "dataverseAdmin" superuser account mentioned in the :doc:`/installation/installation-main` section is an example of a local account. Internally, these accounts are called "builtin" because they are built in to the Dataverse application itself. +Out of the box, your Dataverse installation is configured in "local only" mode. The "dataverseAdmin" superuser account mentioned in the :doc:`/installation/installation-main` section is an example of a local account. Internally, these accounts are called "builtin" because they are built in to the Dataverse software application itself. Both Local and Remote Auth ++++++++++++++++++++++++++ -The ``authenticationproviderrow`` database table controls which "authentication providers" are available within Dataverse. Out of the box, a single row with an id of "builtin" will be present. For each user in Dataverse, the ``authenticateduserlookup`` table will have a value under ``authenticationproviderid`` that matches this id. For example, the default "dataverseAdmin" user will have the value "builtin" under ``authenticationproviderid``. Why is this important? Users are tied to a specific authentication provider but conversion mechanisms are available to switch a user from one authentication provider to the other. As explained in the :doc:`/user/account` section of the User Guide, a graphical workflow is provided for end users to convert from the "builtin" authentication provider to a remote provider. Conversion from a remote authentication provider to the builtin provider can be performed by a sysadmin with access to the "admin" API. See the :doc:`/api/native-api` section of the API Guide for how to list users and authentication providers as JSON. +The ``authenticationproviderrow`` database table controls which "authentication providers" are available within a Dataverse installation. Out of the box, a single row with an id of "builtin" will be present. For each user in a Dataverse installation, the ``authenticateduserlookup`` table will have a value under ``authenticationproviderid`` that matches this id. For example, the default "dataverseAdmin" user will have the value "builtin" under ``authenticationproviderid``. Why is this important? Users are tied to a specific authentication provider but conversion mechanisms are available to switch a user from one authentication provider to the other. As explained in the :doc:`/user/account` section of the User Guide, a graphical workflow is provided for end users to convert from the "builtin" authentication provider to a remote provider. Conversion from a remote authentication provider to the builtin provider can be performed by a sysadmin with access to the "admin" API. See the :doc:`/api/native-api` section of the API Guide for how to list users and authentication providers as JSON. Adding and enabling a second authentication provider (:ref:`native-api-add-auth-provider` and :ref:`api-toggle-auth-provider`) will result in the Log In page showing additional providers for your users to choose from. By default, the Log In page will show the "builtin" provider, but you can adjust this via the :ref:`conf-default-auth-provider` configuration option. Further customization can be achieved by setting :ref:`conf-allow-signup` to "false", thus preventing users from creating local accounts via the web interface. Please note that local accounts can also be created through the API by enabling the ``builtin-users`` endpoint (:ref:`:BlockedApiEndpoints`) and setting the ``BuiltinUsers.KEY`` database setting (:ref:`BuiltinUsers.KEY`). @@ -237,28 +237,28 @@ As for the "Remote only" authentication mode, it means that: - Shibboleth or OAuth has been enabled. - ``:AllowSignUp`` is set to "false" to prevent users from creating local accounts via the web interface. - ``:DefaultAuthProvider`` has been set to use the desired authentication provider -- The "builtin" authentication provider has been disabled (:ref:`api-toggle-auth-provider`). Note that disabling the "builtin" authentication provider means that the API endpoint for converting an account from a remote auth provider will not work. Converting directly from one remote authentication provider to another (i.e. from GitHub to Google) is not supported. Conversion from remote is always to "builtin". Then the user initiates a conversion from "builtin" to remote. Note that longer term, the plan is to permit multiple login options to the same Dataverse account per https://github.com/IQSS/dataverse/issues/3487 (so all this talk of conversion will be moot) but for now users can only use a single login option, as explained in the :doc:`/user/account` section of the User Guide. In short, "remote only" might work for you if you only plan to use a single remote authentication provider such that no conversion between remote authentication providers will be necessary. +- The "builtin" authentication provider has been disabled (:ref:`api-toggle-auth-provider`). Note that disabling the "builtin" authentication provider means that the API endpoint for converting an account from a remote auth provider will not work. Converting directly from one remote authentication provider to another (i.e. from GitHub to Google) is not supported. Conversion from remote is always to "builtin". Then the user initiates a conversion from "builtin" to remote. Note that longer term, the plan is to permit multiple login options to the same Dataverse installation account per https://github.com/IQSS/dataverse/issues/3487 (so all this talk of conversion will be moot) but for now users can only use a single login option, as explained in the :doc:`/user/account` section of the User Guide. In short, "remote only" might work for you if you only plan to use a single remote authentication provider such that no conversion between remote authentication providers will be necessary. File Storage: Using a Local Filesystem and/or Swift and/or S3 object stores --------------------------------------------------------------------------- By default, a Dataverse installation stores all data files (files uploaded by end users) on the filesystem at ``/usr/local/payara5/glassfish/domains/domain1/files``. This path can vary based on answers you gave to the installer (see the :ref:`dataverse-installer` section of the Installation Guide) or afterward by reconfiguring the ``dataverse.files.directory`` JVM option described below. -Dataverse can alternately store files in a Swift or S3-compatible object store, and can now be configured to support multiple stores at once. With a multi-store configuration, the location for new files can be controlled on a per-dataverse basis. +A Dataverse installation can alternately store files in a Swift or S3-compatible object store, and can now be configured to support multiple stores at once. With a multi-store configuration, the location for new files can be controlled on a per-Dataverse collection basis. The following sections describe how to set up various types of stores and how to configure for multiple stores. Multi-store Basics ++++++++++++++++++ -To support multiple stores, Dataverse now requires an id, type, and label for each store (even for a single store configuration). These are configured by defining two required jvm options: +To support multiple stores, a Dataverse installation now requires an id, type, and label for each store (even for a single store configuration). These are configured by defining two required jvm options: .. code-block:: none ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files..type=" ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files..label=
- + From 56122d5646c819793553ecf5fca455cb6c3c1faa Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 29 Jan 2021 12:33:20 -0500 Subject: [PATCH 103/274] updates platform to software, release notes --- doc/release-notes/7501-guides-update | 6 +++++ doc/sphinx-guides/source/admin/dashboard.rst | 4 ++-- .../source/admin/external-tools.rst | 2 +- .../source/admin/harvestclients.rst | 2 +- .../source/admin/harvestserver.rst | 2 +- .../source/admin/integrations.rst | 8 +++---- .../source/admin/make-data-count.rst | 16 ++++++------- .../source/admin/solr-search-index.rst | 2 +- doc/sphinx-guides/source/api/apps.rst | 10 ++++---- doc/sphinx-guides/source/api/auth.rst | 4 ++-- .../source/api/client-libraries.rst | 2 +- .../source/api/external-tools.rst | 8 +++---- doc/sphinx-guides/source/api/faq.rst | 8 +++---- .../source/api/getting-started.rst | 6 ++--- doc/sphinx-guides/source/api/intro.rst | 24 +++++++++---------- doc/sphinx-guides/source/api/metrics.rst | 2 +- doc/sphinx-guides/source/api/native-api.rst | 8 +++---- doc/sphinx-guides/source/api/search.rst | 2 +- doc/sphinx-guides/source/api/sword.rst | 4 ++-- .../source/installation/config.rst | 2 +- 20 files changed, 64 insertions(+), 58 deletions(-) create mode 100644 doc/release-notes/7501-guides-update diff --git a/doc/release-notes/7501-guides-update b/doc/release-notes/7501-guides-update new file mode 100644 index 00000000000..6ad08a34446 --- /dev/null +++ b/doc/release-notes/7501-guides-update @@ -0,0 +1,6 @@ +## Release Highlights + +The Dataverse Guides have been updated following Merce's recent note to the Dataverse Community: + +https://groups.google.com/g/dataverse-community/c/pD-aFrpXMPo + diff --git a/doc/sphinx-guides/source/admin/dashboard.rst b/doc/sphinx-guides/source/admin/dashboard.rst index a3ab90110ea..520126c5299 100644 --- a/doc/sphinx-guides/source/admin/dashboard.rst +++ b/doc/sphinx-guides/source/admin/dashboard.rst @@ -1,7 +1,7 @@ Dashboard ========= -The Dataverse Platform offers a dashboard of administrative tools for superusers only. If you are a logged-in superuser, you can access it by clicking your username in the navbar, and then clicking "Dashboard" from the dropdown. You can verify that you are a superuser by checking the color of your username in the navbar. If it's red, you have the right permissions to use the Dashboard. Superusers can give other users the superuser status via :doc:`user-administration`. +The Dataverse software offers a dashboard of administrative tools for superusers only. If you are a logged-in superuser, you can access it by clicking your username in the navbar, and then clicking "Dashboard" from the dropdown. You can verify that you are a superuser by checking the color of your username in the navbar. If it's red, you have the right permissions to use the Dashboard. Superusers can give other users the superuser status via :doc:`user-administration`. .. contents:: Contents: :local: @@ -22,7 +22,7 @@ This dashboard tool allows you to define sets of local datasets to make availabl Metadata Export --------------- -This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse Platform API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. +This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse software API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. Users ----- diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst index 4426ab2d37f..6e35d5f1d92 100644 --- a/doc/sphinx-guides/source/admin/external-tools.rst +++ b/doc/sphinx-guides/source/admin/external-tools.rst @@ -1,7 +1,7 @@ External Tools ============== -External tools can provide additional features that are not part of the Dataverse Platform itself, such as data file previews, visualization, and curation. +External tools can provide additional features that are not part of the Dataverse software itself, such as data file previews, visualization, and curation. .. contents:: |toctitle| :local: diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index a3de6690b9f..c62c64b80fe 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -26,4 +26,4 @@ What if a Run Fails? Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara5/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. -Note that you'll want to run a minimum Dataverse Platform version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. +Note that you'll want to run a minimum Dataverse software version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index 3550599a67c..71c272d3056 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -50,7 +50,7 @@ the terms "OAI Set" and "Harvesting Set" are used interchangeably. Sets are defined by search queries. Any such query that finds any number of published, local (non-harvested) datasets can be used to create an OAI set. Sets can overlap local Dataverse collections, and can include as few or as many of your local datasets as you wish. A -good way to master the Dataverse Platform search query language is to +good way to master the Dataverse software search query language is to experiment with the Advanced Search page. We also recommend that you consult the :doc:`/api/search` section of the API Guide. diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst index 72e86ba5807..b1683639c9a 100644 --- a/doc/sphinx-guides/source/admin/integrations.rst +++ b/doc/sphinx-guides/source/admin/integrations.rst @@ -111,8 +111,8 @@ Researchers can import datasets from a Dataverse installation into their Renku p command-line interface (CLI) by using the dataset's DOI. See the `renku Dataset documentation `_ -for details. Currently Dataverse Platform ``>=4.8.x`` is required for the import to work. If you need -support for an earlier version of the Dataverse Platform, please get in touch with the Renku team at +for details. Currently Dataverse software ``>=4.8.x`` is required for the import to work. If you need +support for an earlier version of the Dataverse software, please get in touch with the Renku team at `Discourse `_ or `GitHub `_. Avgidea Data Search @@ -123,12 +123,12 @@ Researchers can use a Google Sheets add-on to search for Dataverse installation' Discoverability --------------- -Integration with `DataCite `_ is built in to the Dataverse Platform. When datasets are published, metadata is sent to DataCite. You can further increase the discoverability of your datasets by setting up additional integrations. +Integration with `DataCite `_ is built in to the Dataverse software. When datasets are published, metadata is sent to DataCite. You can further increase the discoverability of your datasets by setting up additional integrations. OAI-PMH (Harvesting) ++++++++++++++++++++ -The Dataverse Platform supports a protocol called OAI-PMH that facilitates harvesting datasets from one system into another. For details on harvesting, see the :doc:`harvestserver` section. +The Dataverse software supports a protocol called OAI-PMH that facilitates harvesting datasets from one system into another. For details on harvesting, see the :doc:`harvestserver` section. SHARE +++++ diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index ac37d9eb11f..e2942d8f24a 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -1,7 +1,7 @@ Make Data Count =============== -`Make Data Count`_ is a project to collect and standardize metrics on data use, especially views, downloads, and citations. The Dataverse Platform can integrate Make Data Count to collect and display usage metrics including counts of dataset views, file downloads, and dataset citations. +`Make Data Count`_ is a project to collect and standardize metrics on data use, especially views, downloads, and citations. The Dataverse software can integrate Make Data Count to collect and display usage metrics including counts of dataset views, file downloads, and dataset citations. .. contents:: Contents: :local: @@ -9,7 +9,7 @@ Make Data Count Introduction ------------ -`Make Data Count`_ is part of a broader Research Data Alliance (RDA) `Data Usage Metrics Working Group`_ which helped to produce a specification called the `COUNTER Code of Practice for Research Data`_ (`PDF`_, `HTML`_) that the Dataverse Platform makes every effort to comply with. The Code of Practice (CoP) is built on top of existing standards such as COUNTER and SUSHI that come out of the article publishing world. The Make Data Count project has emphasized that they would like feedback on the code of practice. You can keep up to date on the Make Data Count project by subscribing to their `newsletter`_. +`Make Data Count`_ is part of a broader Research Data Alliance (RDA) `Data Usage Metrics Working Group`_ which helped to produce a specification called the `COUNTER Code of Practice for Research Data`_ (`PDF`_, `HTML`_) that the Dataverse software makes every effort to comply with. The Code of Practice (CoP) is built on top of existing standards such as COUNTER and SUSHI that come out of the article publishing world. The Make Data Count project has emphasized that they would like feedback on the code of practice. You can keep up to date on the Make Data Count project by subscribing to their `newsletter`_. Architecture ------------ @@ -29,7 +29,7 @@ The most important takeaways from the diagram are: - You should run Counter Processor once a day to create reports in SUSHI (JSON) format that are saved to disk for your Dataverse installation to process and that are sent to the DataCite hub. - You should set up a cron job to have your Dataverse installation process the daily SUSHI reports, updating the Dataverse installation database with the latest metrics. - You should set up a cron job to have your Dataverse installation pull the latest list of citations for each dataset on a periodic basis, perhaps weekly or daily. These citations come from Crossref via the DataCite hub. -- APIs are available in the Dataverse Platform to retrieve Make Data Count metrics: views, downloads, and citations. +- APIs are available in the Dataverse software to retrieve Make Data Count metrics: views, downloads, and citations. Limitations for Dataverse Installations Using Handles Rather Than DOIs @@ -49,7 +49,7 @@ Data repositories using Handles and other identifiers are not supported by Make | | MDC citations | | +---------------------+---------------+------------------+ -This being said, the Dataverse Platform usage logging can still generate logs and process those logs with Counter Processor to create json that details usage on a dataset level. Dataverse installations can ingest this locally generated json. +This being said, the Dataverse software usage logging can still generate logs and process those logs with Counter Processor to create json that details usage on a dataset level. Dataverse installations can ingest this locally generated json. When editing the ``counter-processor-config.yaml`` file mentioned below, make sure that the ``upload_to_hub`` boolean is set to ``False``. @@ -127,9 +127,9 @@ Soon we will be setting up a cron job to run nightly but we start with a single Populate Views and Downloads Nightly ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Running ``main.py`` to create the SUSHI JSON file and the subsequent calling of the Dataverse Platform API to process it should be added as a cron job. +Running ``main.py`` to create the SUSHI JSON file and the subsequent calling of the Dataverse software API to process it should be added as a cron job. -The Dataverse Platform provides example scripts that run the steps to process new accesses and uploads and update your Dataverse installation's database (`counter_daily.sh`) and to retrieve citations for all Datasets from DataCite (`counter_weekly.sh`). These scripts should be configured for your environment and can be run manually or as cron jobs. +The Dataverse software provides example scripts that run the steps to process new accesses and uploads and update your Dataverse installation's database (`counter_daily.sh`) and to retrieve citations for all Datasets from DataCite (`counter_weekly.sh`). These scripts should be configured for your environment and can be run manually or as cron jobs. Sending Usage Metrics to the DataCite Hub ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -164,7 +164,7 @@ Citations will be retrieved for each published dataset and recorded in the your For how to get the citations out of your Dataverse installation, see "Retrieving Citations for a Dataset" under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. -Please note that while the Dataverse Platform has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref. +Please note that while the Dataverse software has a metadata field for "Related Dataset" this information is not currently sent as a citation to Crossref. Retrieving Make Data Count Metrics from the DataCite Hub -------------------------------------------------------- @@ -180,7 +180,7 @@ The following metrics can be downloaded directly from the DataCite hub (see http Retrieving Make Data Count Metrics from a Dataverse Installation ---------------------------------------------------------------- -The Dataverse Platform API endpoints for retrieving Make Data Count metrics are described below under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. +The Dataverse software API endpoints for retrieving Make Data Count metrics are described below under :ref:`Dataset Metrics ` in the :doc:`/api/native-api` section of the API Guide. Please note that it is also possible to retrieve metrics from the DataCite hub itself via https://api.datacite.org diff --git a/doc/sphinx-guides/source/admin/solr-search-index.rst b/doc/sphinx-guides/source/admin/solr-search-index.rst index 5dcc866b137..5ce36179e6b 100644 --- a/doc/sphinx-guides/source/admin/solr-search-index.rst +++ b/doc/sphinx-guides/source/admin/solr-search-index.rst @@ -69,7 +69,7 @@ Dataverse collections must be referenced by database object ID. If you have dire ``select id from dataverse where alias='datavarsealias';`` -should work, or you may click the Dataverse Platform's "Edit" menu and look for dataverseId= in the URLs produced by the drop-down. Then, to re-index: +should work, or you may click the Dataverse software's "Edit" menu and look for dataverseId= in the URLs produced by the drop-down. Then, to re-index: ``curl http://localhost:8080/api/admin/index/dataverses/135`` diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index a0eb8645a41..3ac158f15cb 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -1,9 +1,9 @@ Apps ==== -The introduction of Dataverse Platform APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools-and-queries` sections of the Admin Guide. +The introduction of Dataverse software APIs has fostered the development of a variety of software applications that are listed in the :doc:`/admin/integrations`, :doc:`/admin/external-tools`, and :doc:`/admin/reporting-tools-and-queries` sections of the Admin Guide. -The apps below are open source and demonstrate how to use Dataverse Platform APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse Platform APIs in Python, Javascript, R, and Java. +The apps below are open source and demonstrate how to use Dataverse software APIs. Some of these apps are built on :doc:`/api/client-libraries` that are available for Dataverse software APIs in Python, Javascript, R, and Java. .. contents:: |toctitle| :local: @@ -42,7 +42,7 @@ https://github.com/IQSS/TwoRavens Python ------ -Please note that there are multiple Python modules for Dataverse Platform APIs listed in the :doc:`client-libraries` section. +Please note that there are multiple Python modules for Dataverse software APIs listed in the :doc:`client-libraries` section. dataverse-sample-data ~~~~~~~~~~~~~~~~~~~~~ @@ -97,12 +97,12 @@ https://github.com/jupyter/repo2docker/blob/master/repo2docker/contentproviders/ Java ---- -Please note that there is a Java library for Dataverse Platform APIs listed in the :doc:`client-libraries` section. +Please note that there is a Java library for Dataverse software APIs listed in the :doc:`client-libraries` section. DVUploader ~~~~~~~~~~ -The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse Platform API to upload files to a specified Dataset. Files can be specified by name, or the DVUploader can upload all files in a directory or recursively from a directory tree. The DVUploader can also verify that uploaded files match their local sources by comparing the local and remote fixity checksums. Source code, release 1.0.0- jar file, and documentation are available on GitHub. DVUploader's creation was supported by the Texas Digital Library. +The open-source DVUploader tool is a stand-alone command-line Java application that uses the Dataverse software API to upload files to a specified Dataset. Files can be specified by name, or the DVUploader can upload all files in a directory or recursively from a directory tree. The DVUploader can also verify that uploaded files match their local sources by comparing the local and remote fixity checksums. Source code, release 1.0.0- jar file, and documentation are available on GitHub. DVUploader's creation was supported by the Texas Digital Library. https://github.com/IQSS/dataverse-uploader diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index e21555a07be..9bcbd125719 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -1,7 +1,7 @@ API Tokens and Authentication ============================= -An API token is similar to a password and allows you to authenticate to Dataverse Platform APIs to perform actions as you. Many Dataverse Platform APIs require the use of an API token. +An API token is similar to a password and allows you to authenticate to Dataverse software APIs to perform actions as you. Many Dataverse software APIs require the use of an API token. .. contents:: |toctitle| :local: @@ -25,7 +25,7 @@ Note: The SWORD API uses a different way of passing the API token. Please see :r See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. -There are two ways to pass your API token to Dataverse Platform APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example. +There are two ways to pass your API token to Dataverse software APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example. .. code-block:: bash diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index 406064424cb..d6a803a4f0b 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -34,6 +34,6 @@ The package is currently maintained by `Will Beasley `_ from `ResearchSpace `_ created and maintains this library. diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index 5ea4a1aa83b..f7a0e051803 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -1,7 +1,7 @@ Building External Tools ======================= -External tools can provide additional features that are not part of the Dataverse Platform itself, such as data exploration. Thank you for your interest in building an external tool for the Dataverse Platform! +External tools can provide additional features that are not part of the Dataverse software itself, such as data exploration. Thank you for your interest in building an external tool for the Dataverse software! .. contents:: |toctitle| :local: @@ -9,7 +9,7 @@ External tools can provide additional features that are not part of the Datavers Introduction ------------ -External tools are additional applications the user can access or open from your Dataverse installation to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse Platform. +External tools are additional applications the user can access or open from your Dataverse installation to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse software. Once you have created the external tool itself (which is most of the work!), you need to teach a Dataverse installation how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org @@ -164,12 +164,12 @@ Once you've gotten your tool working, please make a pull request to update the l Unless your tool runs entirely in a browser, you may have integrated server-side software with your Dataverse installation. If so, please double check that your software is listed in the :doc:`/admin/integrations` section of the Admin Guide and if not, please open an issue or pull request to add it. Thanks! -If you've thought to yourself that there ought to be an app store for Dataverse Platform external tools, you're not alone. Please see https://github.com/IQSS/dataverse/issues/5688 :) +If you've thought to yourself that there ought to be an app store for Dataverse software external tools, you're not alone. Please see https://github.com/IQSS/dataverse/issues/5688 :) Demoing Your External Tool ++++++++++++++++++++++++++ -https://demo.dataverse.org is the place to play around with the Dataverse Platform and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible which already includes a number of the tools listed above. +https://demo.dataverse.org is the place to play around with the Dataverse software and your tool can be included. Please email support@dataverse.org to start the conversation about adding your tool. Additionally, you are welcome to open an issue at https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible which already includes a number of the tools listed above. Announcing Your External Tool +++++++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/api/faq.rst b/doc/sphinx-guides/source/api/faq.rst index e49a6648dd8..b73fbe694a1 100644 --- a/doc/sphinx-guides/source/api/faq.rst +++ b/doc/sphinx-guides/source/api/faq.rst @@ -11,12 +11,12 @@ What is an API? See "What is an API?" in the :doc:`intro` section. -What Are Common Use Cases for Dataverse Platform APIs? +What Are Common Use Cases for Dataverse software APIs? ---------------------------------------------------------- See the :doc:`getting-started` section for common use cases for researchers and curators. Other types of API users should find starting points at :ref:`types-of-api-users`. -Where Can I Find Examples of Using Dataverse Platform APIs? +Where Can I Find Examples of Using Dataverse software APIs? ----------------------------------------------------------- See the :doc:`getting-started` section links to examples using curl. @@ -26,7 +26,7 @@ For examples in Javascript, Python, R, and Java, and PHP, see the :doc:`apps` an When Should I Use the Native API vs. the SWORD API? --------------------------------------------------- -The :doc:`sword` is based on a standard, works fine, and is fully supported, but much more development effort has been going into the :doc:`native-api`, which is not based on a standard. It is specific to the Dataverse Platform. +The :doc:`sword` is based on a standard, works fine, and is fully supported, but much more development effort has been going into the :doc:`native-api`, which is not based on a standard. It is specific to the Dataverse software. SWORD uses XML. The Native API uses JSON. @@ -54,7 +54,7 @@ Note that when multiple query parameters are used (such as ``persistentId`` and Where is the Comprehensive List of All API Functionality? --------------------------------------------------------- -There are so many Dataverse Platform APIs that a single page in this guide would probably be overwhelming. See :ref:`list-of-dataverse-apis` for links to various pages. +There are so many Dataverse software APIs that a single page in this guide would probably be overwhelming. See :ref:`list-of-dataverse-apis` for links to various pages. It is possible to get a complete list of API functionality in Swagger/OpenAPI format if you deploy Dataverse Software version 5+. For details, see https://github.com/IQSS/dataverse/issues/5794 diff --git a/doc/sphinx-guides/source/api/getting-started.rst b/doc/sphinx-guides/source/api/getting-started.rst index 0b4fd42ad4a..27fd2a18c5d 100644 --- a/doc/sphinx-guides/source/api/getting-started.rst +++ b/doc/sphinx-guides/source/api/getting-started.rst @@ -1,7 +1,7 @@ Getting Started with APIs ========================= -If you are a researcher or curator who wants to automate parts of your workflow, this section should help you get started. The :doc:`intro` section lists resources for other groups who may be interested in Dataverse Platform APIs such as developers of integrations and support teams. +If you are a researcher or curator who wants to automate parts of your workflow, this section should help you get started. The :doc:`intro` section lists resources for other groups who may be interested in Dataverse software APIs such as developers of integrations and support teams. .. contents:: |toctitle| :local: @@ -16,7 +16,7 @@ If you would rather have full control over your own test server, deployments to Getting an API Token -------------------- -Many Dataverse Platform APIs require an API token. +Many Dataverse software APIs require an API token. Once you have identified a server to test with, create an account, click on your name, and get your API token. For more details, see the :doc:`auth` section. @@ -147,7 +147,7 @@ In addition to the tasks listed above, your Dataverse installation supports many See :ref:`list-of-dataverse-apis` and :ref:`types-of-api-users` to get oriented. -If you're looking for some inspiration for how you can use the Dataverse Platform APIs, there are open source projects that integrate with the Dataverse Platform listed in the :doc:`apps` section. +If you're looking for some inspiration for how you can use the Dataverse software APIs, there are open source projects that integrate with the Dataverse software listed in the :doc:`apps` section. Getting Help ------------- diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst index e096d658368..03f950f0ec3 100755 --- a/doc/sphinx-guides/source/api/intro.rst +++ b/doc/sphinx-guides/source/api/intro.rst @@ -1,7 +1,7 @@ Introduction ============ -The Dataverse Platform APIs allow users to accomplish many tasks such as... +The Dataverse software APIs allow users to accomplish many tasks such as... - creating datasets - uploading files @@ -10,7 +10,7 @@ The Dataverse Platform APIs allow users to accomplish many tasks such as... ... all without using the Dataverse installation's web interface. -APIs open the door for integrations between the Dataverse Platform and other software. For a list, see the :doc:`/admin/integrations` section of the Admin Guide. +APIs open the door for integrations between the Dataverse software and other software. For a list, see the :doc:`/admin/integrations` section of the Admin Guide. .. contents:: |toctitle| :local: @@ -18,7 +18,7 @@ APIs open the door for integrations between the Dataverse Platform and other sof What is an API? --------------- -API stands for "Application Programming Interface" and an example is the Dataverse Platform's "file upload" API. In the diagram below, we can see that while users can click a button within Dataverse installation's web interface to upload a file, there are many other ways to get files into a Dataverse installation, all using an API that allows for uploading of files. +API stands for "Application Programming Interface" and an example is the Dataverse software's "file upload" API. In the diagram below, we can see that while users can click a button within Dataverse installation's web interface to upload a file, there are many other ways to get files into a Dataverse installation, all using an API that allows for uploading of files. .. graphviz:: @@ -72,7 +72,7 @@ The diagram above shows only a few examples of software using a specific API but .. _types-of-api-users: -Types of Dataverse Platform API Users +Types of Dataverse software API Users ------------------------------------- This guide is intended to serve multiple audiences but pointers various sections of the guide are provided below based on the type of API user you are. @@ -104,7 +104,7 @@ Power users may be researchers or curators who are comfortable with automating p Support Teams and Superusers ++++++++++++++++++++++++++++ -Support teams that answer questions about their Dataverse installation should familiarize themselves with the :doc:`getting-started` section to get a sense of common tasks that researchers and curators might be trying to accomplish by using Dataverse Platform APIs. +Support teams that answer questions about their Dataverse installation should familiarize themselves with the :doc:`getting-started` section to get a sense of common tasks that researchers and curators might be trying to accomplish by using Dataverse software APIs. Superusers of a Dataverse installation have access a superuser dashboard described in the :doc:`/admin/dashboard` section of the Admin Guide but some operations can only be done via API. @@ -113,7 +113,7 @@ Superusers of a Dataverse installation have access a superuser dashboard describ Sysadmins +++++++++ -Sysadmins often write scripts to automate tasks and Dataverse Platform APIs make this possible. Sysadmins have control over the server that the Dataverse installation is running on and may be called upon to execute API commands that are limited to "localhost" (the server itself) for security reasons. +Sysadmins often write scripts to automate tasks and Dataverse software APIs make this possible. Sysadmins have control over the server that the Dataverse installation is running on and may be called upon to execute API commands that are limited to "localhost" (the server itself) for security reasons. |Start| A good starting point for sysadmins is "Blocking API Endpoints" in the :doc:`/installation/config` section of the Installation Guide, followed by the :doc:`getting-started` section of this guide, followed by the :doc:`/admin/troubleshooting` section of the Admin Guide. @@ -132,18 +132,18 @@ The Dataverse Project loves contributors! Depending on your interests and skills Developers of Integrations, External Tools, and Apps ++++++++++++++++++++++++++++++++++++++++++++++++++++ -One of the primary purposes for Dataverse Platform APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: +One of the primary purposes for Dataverse software APIs in the first place is to enable integrations with third party software. Integrations are listed in the following places: - The :doc:`/admin/integrations` section of the Admin Guide. - The :doc:`/api/external-tools` section this guide. - The :doc:`apps` section of this guide. -|Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with the Dataverse Platform, followed by the :doc:`getting-started` section. +|Start| Good starting points are the three sections above to get a sense of third-party software that already integrates with the Dataverse software, followed by the :doc:`getting-started` section. -Developers of Dataverse Platform API Client Libraries +Developers of Dataverse software API Client Libraries +++++++++++++++++++++++++++++++++++++++++++++++++++++ -A client library helps developers using a specific programming language such as Python, Javascript, R, or Java interact with Dataverse Platform APIs in a manner that is idiomatic for their language. For example, a Python programmer may want to +A client library helps developers using a specific programming language such as Python, Javascript, R, or Java interact with Dataverse software APIs in a manner that is idiomatic for their language. For example, a Python programmer may want to |Start| A good starting point is the :doc:`client-libraries` section, followed by the :doc:`getting-started` section. @@ -207,7 +207,7 @@ Please note that some APIs are only documented in other guides that are more sui Client Libraries ~~~~~~~~~~~~~~~~ -See :doc:`client-libraries` for how to use Dataverse Platform APIs from Python, R, Java, and Javascript. +See :doc:`client-libraries` for how to use Dataverse software APIs from Python, R, Java, and Javascript. Examples ~~~~~~~~ @@ -224,7 +224,7 @@ See :doc:`faq`. Getting Help ------------ -Dataverse Platform API questions are on topic in all the usual places: +Dataverse software API questions are on topic in all the usual places: - The dataverse-community Google Group: https://groups.google.com/forum/#!forum/dataverse-community - Dataverse Project community calls: https://dataverse.org/community-calls diff --git a/doc/sphinx-guides/source/api/metrics.rst b/doc/sphinx-guides/source/api/metrics.rst index e29cb3a2c9e..b273ef9eaff 100755 --- a/doc/sphinx-guides/source/api/metrics.rst +++ b/doc/sphinx-guides/source/api/metrics.rst @@ -1,7 +1,7 @@ Metrics API =========== -The Metrics API provides counts of downloads, datasets created, files uploaded, and more, as described below. The Dataverse Platform also supports Make Data Count, which is described in the :doc:`/admin/make-data-count` section of the Admin Guide. +The Metrics API provides counts of downloads, datasets created, files uploaded, and more, as described below. The Dataverse software also supports Make Data Count, which is described in the :doc:`/admin/make-data-count` section of the Admin Guide. .. contents:: |toctitle| :local: diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 9abe5dea3b4..d4712dd7a86 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1,7 +1,7 @@ Native API ========== -The Dataverse Platform exposes most of its GUI functionality via a REST-based API. This section describes that functionality. Most API endpoints require an API token that can be passed as the ``X-Dataverse-key`` HTTP header or in the URL as the ``key`` query parameter. +The Dataverse software exposes most of its GUI functionality via a REST-based API. This section describes that functionality. Most API endpoints require an API token that can be passed as the ``X-Dataverse-key`` HTTP header or in the URL as the ``key`` query parameter. .. note:: |CORS| Some API endpoint allow CORS_ (cross-origin resource sharing), which makes them usable from scripts running in web browsers. These endpoints are marked with a *CORS* badge. @@ -9,7 +9,7 @@ The Dataverse Platform exposes most of its GUI functionality via a REST-based AP .. _CORS: https://www.w3.org/TR/cors/ -.. warning:: The Dataverse Platform's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. +.. warning:: The Dataverse software's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. .. contents:: |toctitle| :local: @@ -933,7 +933,7 @@ For these deletes your JSON file must include an exact match of those dataset fi Publish a Dataset ~~~~~~~~~~~~~~~~~ -When publishing a dataset it's good to be aware of the Dataverse Platform's versioning system, which is described in the :doc:`/user/dataset-management` section of the User Guide. +When publishing a dataset it's good to be aware of the Dataverse software's versioning system, which is described in the :doc:`/user/dataset-management` section of the User Guide. If this is the first version of the dataset, its version number will be set to ``1.0``. Otherwise, the new dataset version number is determined by the most recent version number and the ``type`` parameter. Passing ``type=minor`` increases the minor version number (2.3 is updated to 2.4). Passing ``type=major`` increases the major version number (2.3 is updated to 3.0). (Superusers can pass ``type=updatecurrent`` to update metadata without changing the version number.) @@ -1816,7 +1816,7 @@ Note: at present, the API cannot be used on a file that's already successfully i Redetect File Type ~~~~~~~~~~~~~~~~~~ -The Dataverse Platform uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have the Dataverse Platform attempt to redetect the file type. +The Dataverse software uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have the Dataverse software attempt to redetect the file type. When using the curl command below, you can pass ``dryRun=true`` if you don't want any changes to be saved to the database. Change this to ``dryRun=false`` (or omit it) to save the change. diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 60e799f00fc..15b964f867a 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -4,7 +4,7 @@ Search API .. contents:: |toctitle| :local: -The Search API supports the same searching, sorting, and faceting operations as the Dataverse Platform's web interface. +The Search API supports the same searching, sorting, and faceting operations as the Dataverse software's web interface. To search unpublished content, you must pass in an API token as described in the :doc:`auth` section. diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index cd49238a432..d58e003e636 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -11,7 +11,7 @@ About Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse Software 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`. -The Dataverse Platform implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. +The Dataverse software implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse Software 4.0 datasets can also be created via JSON using the "native" API. SWORD is limited to the dozen or so fields listed below in the crosswalk, but the native API allows you to populate all metadata fields available in a Dataverse installation. @@ -49,7 +49,7 @@ Backward incompatible changes For better security than in DVN 3.x, usernames and passwords are no longer accepted. The use of an API token is required. -Differences in Dataverse Platform 4 from DVN 3.x lead to a few minor backward incompatible changes in the Dataverse Platform implementation of SWORD, which are listed below. Old ``v1`` URLs should continue to work but the ``Service Document`` will contain a deprecation warning and responses will contain ``v1.1`` URLs. See also :ref:`known-issues`. +Differences in Dataverse software 4 from DVN 3.x lead to a few minor backward incompatible changes in the Dataverse software implementation of SWORD, which are listed below. Old ``v1`` URLs should continue to work but the ``Service Document`` will contain a deprecation warning and responses will contain ``v1.1`` URLs. See also :ref:`known-issues`. - Newly required fields when creating/editing datasets for compliance with the `Joint Declaration for Data Citation principles `_. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9393233295a..4c5130f524b 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -146,7 +146,7 @@ As the person installing the Dataverse software, you may or may not be a local m Persistent Identifiers and Publishing Datasets ---------------------------------------------- -Persistent identifiers are a required and integral part of the Dataverse Platform. They provide a URL that is guaranteed to resolve to the datasets or files they represent. The Dataverse software currently supports creating identifiers using DOI and Handle. +Persistent identifiers are a required and integral part of the Dataverse software. They provide a URL that is guaranteed to resolve to the datasets or files they represent. The Dataverse software currently supports creating identifiers using DOI and Handle. By default, the installer configures a default DOI namespace (10.5072) with DataCite as the registration provider. Please note that as of the release 4.9.3, we can no longer use EZID as the provider. Unlike EZID, DataCite requires that you register for a test account, configured with your own prefix (please contact support@datacite.org). Once you receive the login name, password, and prefix for the account, configure the credentials in your domain.xml, as the following two JVM options:: From 495ba605ca16a34e8aa98a9a41155d1ac54b8d1f Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 29 Jan 2021 15:12:27 -0500 Subject: [PATCH 104/274] Create 7501-guides-update.md --- doc/release-notes/7501-guides-update.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/release-notes/7501-guides-update.md diff --git a/doc/release-notes/7501-guides-update.md b/doc/release-notes/7501-guides-update.md new file mode 100644 index 00000000000..6ad08a34446 --- /dev/null +++ b/doc/release-notes/7501-guides-update.md @@ -0,0 +1,6 @@ +## Release Highlights + +The Dataverse Guides have been updated following Merce's recent note to the Dataverse Community: + +https://groups.google.com/g/dataverse-community/c/pD-aFrpXMPo + From 43b5c35f78d8ac554c7ee8d936386914bf53e692 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 29 Jan 2021 15:47:47 -0500 Subject: [PATCH 105/274] reverting tracking changes --- conf/docker-aio/run-test-suite.sh | 4 +- .../7398-saved-search-performance.md | 30 +++++++ doc/release-notes/7518-guides-updates.md | 5 ++ .../admin/dataverse-external-tools.tsv | 2 +- doc/sphinx-guides/source/admin/dashboard.rst | 4 + .../source/admin/harvestclients.rst | 4 + doc/sphinx-guides/source/api/apps.rst | 2 +- .../source/installation/prerequisites.rst | 5 +- .../installation/r-rapache-tworavens.rst | 2 +- scripts/issues/7398/delete_dld.txt | 21 +++++ scripts/issues/7398/delete_ss.txt | 18 ++++ scripts/issues/7398/dld_for_deletion.txt | 19 +++++ scripts/issues/7398/ss_for_deletion.txt | 10 +++ .../harvard/iq/dataverse/DataversePage.java | 38 +-------- .../savedsearch/SavedSearchServiceBean.java | 84 +++++++++---------- src/main/java/propertyFiles/Bundle.properties | 1 + ....3.0.2__7512-update-notification-types.sql | 6 ++ src/main/webapp/dataset.xhtml | 3 + src/main/webapp/dataverseuser.xhtml | 9 +- src/main/webapp/filesFragment.xhtml | 15 ---- src/main/webapp/resources/js/clipboard.min.js | 6 +- .../resources/js/dv_rebind_bootstrap_ui.js | 10 +++ .../iq/dataverse/api/ExternalToolsIT.java | 2 +- 23 files changed, 188 insertions(+), 112 deletions(-) create mode 100644 doc/release-notes/7398-saved-search-performance.md create mode 100644 doc/release-notes/7518-guides-updates.md create mode 100644 scripts/issues/7398/delete_dld.txt create mode 100644 scripts/issues/7398/delete_ss.txt create mode 100644 scripts/issues/7398/dld_for_deletion.txt create mode 100644 scripts/issues/7398/ss_for_deletion.txt create mode 100644 src/main/resources/db/migration/V5.3.0.2__7512-update-notification-types.sql mode change 100755 => 100644 src/main/webapp/resources/js/clipboard.min.js diff --git a/conf/docker-aio/run-test-suite.sh b/conf/docker-aio/run-test-suite.sh index d0dd5d5318c..2b24f6c90b2 100755 --- a/conf/docker-aio/run-test-suite.sh +++ b/conf/docker-aio/run-test-suite.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # This is the canonical list of which "IT" tests are expected to pass. dvurl=$1 @@ -8,4 +8,4 @@ fi # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment. # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml. -source maven/maven.sh mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT -Ddataverse.test.baseurl=$dvurl +source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT -Ddataverse.test.baseurl=$dvurl diff --git a/doc/release-notes/7398-saved-search-performance.md b/doc/release-notes/7398-saved-search-performance.md new file mode 100644 index 00000000000..4986524ed4f --- /dev/null +++ b/doc/release-notes/7398-saved-search-performance.md @@ -0,0 +1,30 @@ +## Release Highlights + +### Saved Search Performance Improvements + +A refactoring has greatly improved Saved Search performance in the application. If your installation has multiple, potentially long-running Saved Searches in place, this greatly improves the probability that those search jobs will complete without timing out. + +## Notes for Dataverse Installation Administrators + +### DB Cleanup for Saved Searches + +A previous version of dataverse changed the indexing logic so that when a user links a dataverse, its children are also indexed as linked. This means that the children do not need to be separately linked, and in this version we removed the logic that creates a saved search to create those links when a dataverse is linked. + +We recommend cleaning up the db to a) remove these saved searches and b) remove the links for the objects. We can do this via a few queries, which are available in the folder here: + +https://github.com/IQSS/dataverse/raw/develop/scripts/issues/7398/ + +There are four sets of queries available, and they should be run in this order: + +- ss_for_deletion.txt to identify the Saved Searches to be deleted +- delete_ss.txt to delete the Saved Searches identified in the previous query +- dld_for_deletion.txt to identify the linked datasets and dataverses to be deleted +- delete_dld.txt to delete the linked datasets and dataverses identified in the previous queries + +Note: removing these saved searches and links should not affect what users will see as linked due to the aforementioned indexing change. Similarly, not removing these saved searches and links should not affect anything, but is a cleanup of unnecessary rows in the database. + +## Additional Upgrade Instructions + +X\. (Optional, but recommended) DB Cleanup + +Perform the DB Cleanup for Saved Searches and Linked Objects, summarized in the "Notes for Dataverse Installation Administrators" section above. diff --git a/doc/release-notes/7518-guides-updates.md b/doc/release-notes/7518-guides-updates.md new file mode 100644 index 00000000000..b64e0c3a0ee --- /dev/null +++ b/doc/release-notes/7518-guides-updates.md @@ -0,0 +1,5 @@ +## Guides Updates + +The Dataverse Software Guides have been updated to follow recent changes to how different terms are used across the Dataverse Project. For more information, see Mercè's note the community: + + diff --git a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv index 881cf448221..521d61735dc 100644 --- a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv +++ b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv @@ -1,6 +1,6 @@ Tool Type Scope Description TwoRavens explore file A system of interlocking statistical tools for data exploration, analysis, and meta-analysis: http://2ra.vn. See the :doc:`/user/data-exploration/tworavens` section of the User Guide for more information on TwoRavens from the user perspective and the :doc:`/installation/r-rapache-tworavens` section of the Installation Guide. -Data Explorer explore file A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Explorer for the instructions on adding Data Explorer to your Dataverse; and the :doc:`/installation/prerequisites` section of the Installation Guide for the instructions on how to set up **basic R configuration required** (specifically, Dataverse uses R to generate .prep metadata files that are needed to run Data Explorer). +Data Explorer explore file A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse. Whole Tale explore dataset A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide `_. File Previewers explore file A set of tools that display the content of files - including audio, html, `Hypothes.is `_ annotations, images, PDF, text, video, tabular data, and spreadsheets - allowing them to be viewed without downloading. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/GlobalDataverseCommunityConsortium/dataverse-previewers Data Curation Tool configure file A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions. diff --git a/doc/sphinx-guides/source/admin/dashboard.rst b/doc/sphinx-guides/source/admin/dashboard.rst index 520126c5299..6472c9e0e0f 100644 --- a/doc/sphinx-guides/source/admin/dashboard.rst +++ b/doc/sphinx-guides/source/admin/dashboard.rst @@ -22,7 +22,11 @@ This dashboard tool allows you to define sets of local datasets to make availabl Metadata Export --------------- +<<<<<<< HEAD This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse software API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. +======= +This part of the Dashboard is simply a reminder message that metadata export happens through the Dataverse Software API. See the :doc:`metadataexport` section and the :doc:`/api/native-api` section of the API Guide for more details. +>>>>>>> parent of f9f1faff9... updating branch from tracking Users ----- diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index c62c64b80fe..78a79178175 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -26,4 +26,8 @@ What if a Run Fails? Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara5/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. +<<<<<<< HEAD Note that you'll want to run a minimum Dataverse software version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. +======= +Note that you'll want to run a minimum Dataverse software version of 4.6, optimally 4.18 or beyond, for the best OAI-PMH interoperability. +>>>>>>> parent of f9f1faff9... updating branch from tracking diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index 3ac158f15cb..0000527a86e 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -16,7 +16,7 @@ Data Explorer Data Explorer is a GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. -https://github.com/scholarsportal/Dataverse-Data-Explorer +https://github.com/scholarsportal/Dataverse-Data-Explorer-v2 Data Curation Tool ~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index fe439018e40..84fe012efd1 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -303,10 +303,7 @@ R The Dataverse software uses `R `_ to handle tabular data files. The instructions below describe a **minimal** R installation. It will allow you to ingest R (.RData) files as tabular -data; to export tabular data as .RData files; and to run `Data -Explorer `_ -(specifically, R is used to generate .prep metadata files that Data -Explorer uses). R can be considered an optional component, meaning +data and to export tabular data as .RData files. R can be considered an optional component, meaning that if you don't have R installed, you will still be able to run and use the Dataverse software - but the functionality specific to tabular data mentioned above will not be available to your users. **Note** that if diff --git a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst index 72e6034d5ad..48dd3a6da95 100644 --- a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst +++ b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst @@ -25,7 +25,7 @@ correct versions of the required third party R packages. setup described in the "Prerequisites" portion of the Installation Guide. Meaning that once completed, it installs everything needed to run TwoRavens, PLUS all the libraries and components required to -ingest RData files, export as RData, and use Data Explorer.** +ingest RData files and export as RData.** diff --git a/scripts/issues/7398/delete_dld.txt b/scripts/issues/7398/delete_dld.txt new file mode 100644 index 00000000000..30edf42774b --- /dev/null +++ b/scripts/issues/7398/delete_dld.txt @@ -0,0 +1,21 @@ +-- these queries will delete linked objects identified using the query in dld_for_deletion + +begin; + +delete from datasetlinkingdataverse where id in ( +select dld.id +from datasetlinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld +where dld.dataset_id = dvo.id +and dld.linkingdataverse_id = dvld.linkingdataverse_id +and dvo.owner_id = dvld.dataverse_id +); + +delete from dataverselinkingdataverse where id in ( +select dld.id +from dataverselinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld +where dld.dataverse_id = dvo.id +and dld.linkingdataverse_id = dvld.linkingdataverse_id +and dvo.owner_id = dvld.dataverse_id +); + +commit; \ No newline at end of file diff --git a/scripts/issues/7398/delete_ss.txt b/scripts/issues/7398/delete_ss.txt new file mode 100644 index 00000000000..3cf053ce5af --- /dev/null +++ b/scripts/issues/7398/delete_ss.txt @@ -0,0 +1,18 @@ +-- these queries will delete the saved searches identified using the ss_for_deletion query + +begin; + +create temporary table delete_ss on commit drop as ( +Select ss.id +from savedsearch ss, savedsearchfilterquery ssfq, dataverselinkingdataverse dld +where ss.id = ssfq.savedsearch_id +and ss.definitionpoint_id = dld.linkingdataverse_id +and dld.dataverse_id = rtrim(reverse(split_part(reverse(ssfq.filterquery),'/',1)),'"')::integer +and ss.query='*' +and ssfq.filterquery like 'subtreePaths%' +); + +delete from savedsearchfilterquery where savedsearch_id in (select id from delete_ss); +delete from savedsearch where id in (select id from delete_ss); + +commit; diff --git a/scripts/issues/7398/dld_for_deletion.txt b/scripts/issues/7398/dld_for_deletion.txt new file mode 100644 index 00000000000..51aa9ce1458 --- /dev/null +++ b/scripts/issues/7398/dld_for_deletion.txt @@ -0,0 +1,19 @@ +-- these queries will show you the linked objects that will get deleted + +-- datasets +select dld.dataset_id, dvo.owner_id, dld.linkingdataverse_id, +dvld.dataverse_id, dvld.linkingdataverse_id +from datasetlinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld +where dld.dataset_id = dvo.id +and dld.linkingdataverse_id = dvld.linkingdataverse_id +and dvo.owner_id = dvld.dataverse_id +order by dld.linkingdataverse_id; + +-- dataverses +select dld.dataverse_id, dvo.owner_id, dld.linkingdataverse_id, +dvld.dataverse_id, dvld.linkingdataverse_id +from dataverselinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld +where dld.dataverse_id = dvo.id +and dld.linkingdataverse_id = dvld.linkingdataverse_id +and dvo.owner_id = dvld.dataverse_id +order by dld.linkingdataverse_id; diff --git a/scripts/issues/7398/ss_for_deletion.txt b/scripts/issues/7398/ss_for_deletion.txt new file mode 100644 index 00000000000..11ad8bbc49a --- /dev/null +++ b/scripts/issues/7398/ss_for_deletion.txt @@ -0,0 +1,10 @@ +-- this query will show you the saved searches that will get deleted + +select ss.id, ss.definitionpoint_id, dld.dataverse_id, ssfq.filterquery +from savedsearch ss, savedsearchfilterquery ssfq, dataverselinkingdataverse dld +where ss.id = ssfq.savedsearch_id +and ss.definitionpoint_id = dld.linkingdataverse_id +and dld.dataverse_id = rtrim(reverse(split_part(reverse(ssfq.filterquery),'/',1)),'"')::integer +and ss.query='*' +and ssfq.filterquery like 'subtreePaths%' +order by ss.definitionpoint_id; \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 165c1759b5e..34efc928ab9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -787,18 +787,9 @@ public String saveLinkedDataverse() { return ""; } - AuthenticatedUser savedSearchCreator = getAuthenticatedUser(); - if (savedSearchCreator == null) { - String msg = BundleUtil.getStringFromBundle("dataverse.link.user"); - logger.severe(msg); - JsfHelper.addErrorMessage(msg); - return returnRedirect(); - } - linkingDataverse = dataverseService.find(linkingDataverseId); LinkDataverseCommand cmd = new LinkDataverseCommand(dvRequestService.getDataverseRequest(), linkingDataverse, dataverse); - //LinkDvObjectCommand cmd = new LinkDvObjectCommand (session.getUser(), linkingDataverse, dataverse); try { commandEngine.submit(cmd); } catch (CommandException ex) { @@ -808,32 +799,9 @@ public String saveLinkedDataverse() { JsfHelper.addErrorMessage(msg); return returnRedirect(); } - - SavedSearch savedSearchOfChildren = createSavedSearchForChildren(savedSearchCreator); - - boolean createLinksAndIndexRightNow = false; - if (createLinksAndIndexRightNow) { - try { - // create links (does indexing) right now (might be expensive) - boolean debug = false; - DataverseRequest dataverseRequest = new DataverseRequest(savedSearchCreator, SavedSearchServiceBean.getHttpServletRequest()); - savedSearchService.makeLinksForSingleSavedSearch(dataverseRequest, savedSearchOfChildren, debug); - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataverse.linked.success", getSuccessMessageArguments())); - return returnRedirect(); - } catch (SearchException | CommandException ex) { - // error: solr is down, etc. can't link children right now - JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("dataverse.linked.internalerror", getSuccessMessageArguments())); - String msg = dataverse.getDisplayName() + " has been successfully linked to " + linkingDataverse.getDisplayName() + " but contents will not appear until an internal error has been fixed."; - logger.log(Level.SEVERE, "{0} {1}", new Object[]{msg, ex}); - //JsfHelper.addErrorMessage(msg); - return returnRedirect(); - } - } else { - // defer: please wait for the next timer/cron job - //JsfHelper.addSuccessMessage(dataverse.getDisplayName() + " has been successfully linked to " + linkingDataverse.getDisplayName() + ". Please wait for its contents to appear."); - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataverse.linked.success.wait", getSuccessMessageArguments())); - return returnRedirect(); - } + + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataverse.linked.success.wait", getSuccessMessageArguments())); + return returnRedirect(); } private List getSuccessMessageArguments() { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java index 20cb1a8629a..a495842e40d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/savedsearch/SavedSearchServiceBean.java @@ -8,6 +8,7 @@ import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.search.SearchServiceBean; import edu.harvard.iq.dataverse.search.SolrQueryResponse; @@ -20,6 +21,7 @@ import edu.harvard.iq.dataverse.search.SortBy; import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.logging.Level; @@ -171,35 +173,40 @@ public JsonObjectBuilder makeLinksForSingleSavedSearch(DataverseRequest dvReq, S JsonArrayBuilder savedSearchArrayBuilder = Json.createArrayBuilder(); JsonArrayBuilder infoPerHit = Json.createArrayBuilder(); SolrQueryResponse queryResponse = findHits(savedSearch); + + List skipList = new ArrayList(); // a list for the definition point itself and already linked objects + skipList.add(savedSearch.getDefinitionPoint().getId()); - // get linked objects and add to a list TypedQuery typedQuery = em.createNamedQuery("DataverseLinkingDataverse.findIdsByLinkingDataverseId", Long.class) .setParameter("linkingDataverseId", savedSearch.getDefinitionPoint().getId()); - List alreadyLinkedObjectIds = typedQuery.getResultList(); + skipList.addAll(typedQuery.getResultList()); typedQuery = em.createNamedQuery("DatasetLinkingDataverse.findIdsByLinkingDataverseId", Long.class) .setParameter("linkingDataverseId", savedSearch.getDefinitionPoint().getId()); - alreadyLinkedObjectIds.addAll(typedQuery.getResultList()); - + skipList.addAll(typedQuery.getResultList()); + for (SolrSearchResult solrSearchResult : queryResponse.getSolrSearchResults()) { JsonObjectBuilder hitInfo = Json.createObjectBuilder(); hitInfo.add("name", solrSearchResult.getNameSort()); hitInfo.add("dvObjectId", solrSearchResult.getEntityId()); + + if (skipList.contains(solrSearchResult.getEntityId())) { + hitInfo.add(resultString, "Skipping because would link to itself or an already linked entity."); + infoPerHit.add(hitInfo); + continue; + } DvObject dvObjectThatDefinitionPointWillLinkTo = dvObjectService.findDvObject(solrSearchResult.getEntityId()); if (dvObjectThatDefinitionPointWillLinkTo == null) { hitInfo.add(resultString, "Could not find DvObject with id " + solrSearchResult.getEntityId()); infoPerHit.add(hitInfo); - break; - } + continue; + } + if (dvObjectThatDefinitionPointWillLinkTo.isInstanceofDataverse()) { Dataverse dataverseToLinkTo = (Dataverse) dvObjectThatDefinitionPointWillLinkTo; - if (wouldResultInLinkingToItself(savedSearch.getDefinitionPoint(), dataverseToLinkTo)) { - hitInfo.add(resultString, "Skipping because dataverse id " + dataverseToLinkTo.getId() + " would link to itself."); - } else if (alreadyLinkedToTheDataverse(alreadyLinkedObjectIds, dataverseToLinkTo)) { - hitInfo.add(resultString, "Skipping because dataverse " + savedSearch.getDefinitionPoint().getId() + " already links to dataverse " + dataverseToLinkTo.getId() + "."); - } else if (dataverseToLinkToIsAlreadyPartOfTheSubtree(savedSearch.getDefinitionPoint(), dataverseToLinkTo)) { + if (dataverseToLinkToIsAlreadyPartOfTheSubtree(savedSearch.getDefinitionPoint(), dataverseToLinkTo)) { hitInfo.add(resultString, "Skipping because " + dataverseToLinkTo + " is already part of the subtree for " + savedSearch.getDefinitionPoint()); } else { DataverseLinkingDataverse link = commandEngine.submitInNewTransaction(new LinkDataverseCommand(dvReq, savedSearch.getDefinitionPoint(), dataverseToLinkTo)); @@ -207,29 +214,22 @@ public JsonObjectBuilder makeLinksForSingleSavedSearch(DataverseRequest dvReq, S } } else if (dvObjectThatDefinitionPointWillLinkTo.isInstanceofDataset()) { Dataset datasetToLinkTo = (Dataset) dvObjectThatDefinitionPointWillLinkTo; - if (alreadyLinkedToTheDataset(alreadyLinkedObjectIds, datasetToLinkTo)) { - hitInfo.add(resultString, "Skipping because dataverse " + savedSearch.getDefinitionPoint() + " already links to dataset " + datasetToLinkTo + "."); - } else if (datasetToLinkToIsAlreadyPartOfTheSubtree(savedSearch.getDefinitionPoint(), datasetToLinkTo)) { + if (datasetToLinkToIsAlreadyPartOfTheSubtree(savedSearch.getDefinitionPoint(), datasetToLinkTo)) { // already there from normal search/browse hitInfo.add(resultString, "Skipping because dataset " + datasetToLinkTo.getId() + " is already part of the subtree for " + savedSearch.getDefinitionPoint().getAlias()); } else if (datasetAncestorAlreadyLinked(savedSearch.getDefinitionPoint(), datasetToLinkTo)) { hitInfo.add(resultString, "FIXME: implement this?"); - } else if (!datasetToLinkTo.isReleased()) { - hitInfo.add(resultString, "Skipping because dataset " + datasetToLinkTo.getId() + " is not released " ); } else { DatasetLinkingDataverse link = commandEngine.submitInNewTransaction(new LinkDatasetCommand(dvReq, savedSearch.getDefinitionPoint(), datasetToLinkTo)); - alreadyLinkedObjectIds.add(datasetToLinkTo.getId()); // because search results could produce two hits (published and draft) hitInfo.add(resultString, "Persisted DatasetLinkingDataverse id " + link.getId() + " link of " + link.getDataset() + " to " + link.getLinkingDataverse()); } - } else if (dvObjectThatDefinitionPointWillLinkTo.isInstanceofDataFile()) { - hitInfo.add(resultString, "Skipping because the search matched a file. The matched file id was " + dvObjectThatDefinitionPointWillLinkTo.getId() + "."); } else { hitInfo.add(resultString, "Unexpected DvObject type."); } infoPerHit.add(hitInfo); } - + JsonObjectBuilder info = getInfo(savedSearch, infoPerHit); if (debugFlag) { info.add("debug", getDebugInfo(savedSearch)); @@ -237,13 +237,12 @@ public JsonObjectBuilder makeLinksForSingleSavedSearch(DataverseRequest dvReq, S savedSearchArrayBuilder.add(info); response.add("hits for saved search id " + savedSearch.getId(), savedSearchArrayBuilder); - Date end = new Date(); - logger.info("SAVED SEARCH (" + savedSearch.getId() + ") total time in ms: " + (end.getTime() - start.getTime())); + logger.info("SAVED SEARCH (" + savedSearch.getId() + ") total time in ms: " + (new Date().getTime() - start.getTime())); return response; } private SolrQueryResponse findHits(SavedSearch savedSearch) throws SearchException { - String sortField = SearchFields.RELEVANCE; + String sortField = SearchFields.TYPE; // first return dataverses, then datasets String sortOrder = SortBy.DESCENDING; SortBy sortBy = new SortBy(sortField, sortOrder); int paginationStart = 0; @@ -251,16 +250,23 @@ private SolrQueryResponse findHits(SavedSearch savedSearch) throws SearchExcepti int numResultsPerPage = Integer.MAX_VALUE; List dataverses = new ArrayList<>(); dataverses.add(savedSearch.getDefinitionPoint()); + + // since saved search can only link Dataverses and Datasets, we can limit our search + List searchFilterQueries = savedSearch.getFilterQueriesAsStrings(); + searchFilterQueries.add("dvObjectType:(dataverses OR datasets)"); + + // run the search as GuestUser to only link published objects SolrQueryResponse solrQueryResponse = searchService.search( - new DataverseRequest(savedSearch.getCreator(), getHttpServletRequest()), + new DataverseRequest(GuestUser.get(), getHttpServletRequest()), dataverses, savedSearch.getQuery(), - savedSearch.getFilterQueriesAsStrings(), + searchFilterQueries, sortBy.getField(), sortBy.getOrder(), paginationStart, dataRelatedToMe, - numResultsPerPage + numResultsPerPage, + false // do not retrieve entities ); return solrQueryResponse; } @@ -289,18 +295,6 @@ private JsonArrayBuilder getFilterQueries(SavedSearch savedSearch) { return filterQueriesArrayBuilder; } - private boolean alreadyLinkedToTheDataverse(List alreadyLinkedObjectIds, Dataverse dataverseToLinkTo) { - return alreadyLinkedObjectIds.contains(dataverseToLinkTo.getId()); - } - - private boolean alreadyLinkedToTheDataset(List alreadyLinkedObjectIds, Dataset linkToThisDataset) { - return alreadyLinkedObjectIds.contains(linkToThisDataset.getId()); - } - - private static boolean wouldResultInLinkingToItself(Dataverse savedSearchDefinitionPoint, Dataverse dataverseToLinkTo) { - return savedSearchDefinitionPoint.equals(dataverseToLinkTo); - } - private boolean datasetToLinkToIsAlreadyPartOfTheSubtree(Dataverse definitionPoint, Dataset datasetWeMayLinkTo) { Dataverse ancestor = datasetWeMayLinkTo.getOwner(); while (ancestor != null) { @@ -351,14 +345,12 @@ public static HttpServletRequest getHttpServletRequest() { * Saved Search was created. The default IP address in the * DataverseRequest constructor is used instead, which as of this * writing is 0.0.0.0 to mean "undefined". Is this a feature or a bug? - * What is the expected interplay between Saved Search and IP Groups? - * Users might be surprised to see certain DvObjects in the results of - * their query when creating the Saved Search and later find that those - * DvObjects, which are only visible due to an IP Groups membership, are - * not found by Saved Search when executed by cron, for example. As of - * this writing Saved Search is a superuser-only feature so perhaps IP - * Groups are irrelevant because all DvObjects are discoverable to - * superusers. + * This is not an issue for the search itself, since it is now run as the + * Guest User, but would present a problem if the user does not have + * permission to create links and could only create the saved search due to + * a granted permission from the IP Group. + * As of this writing Saved Search is a superuser-only feature; so IP + * Groups are irrelevant because all superusers can create links. */ return null; } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 3d3349a0b52..2a73ea97307 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -54,6 +54,7 @@ harvested=Harvested apply=Apply add=Add delete=Delete +copyClipboard=Copy to Clipboard yes=Yes no=No previous=Previous diff --git a/src/main/resources/db/migration/V5.3.0.2__7512-update-notification-types.sql b/src/main/resources/db/migration/V5.3.0.2__7512-update-notification-types.sql new file mode 100644 index 00000000000..c063aba38f8 --- /dev/null +++ b/src/main/resources/db/migration/V5.3.0.2__7512-update-notification-types.sql @@ -0,0 +1,6 @@ +--In support of removing world map we are deleting notifications related to world map +--and updating notifications to reflect the current enumeration of types on UserNotification + +delete from usernotification where type = 5 or type = 6; + +update usernotification set type = type - 2 where type > 6; \ No newline at end of file diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 65a30280ceb..53c383381d4 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -974,6 +974,9 @@
+ - -
-
-
- -

- - - - - - - - - -

- -

- - - - - - - - - - -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- #{bundle['user.username']} - - #{DataverseUserPage.currentUser.identifier.replaceFirst("@", "")} -
- #{bundle['user.firstName']} - - #{DataverseUserPage.currentUser.firstName} -
- #{bundle['user.lastName']} - - #{DataverseUserPage.currentUser.lastName} -
- #{bundle.email} - -

- #{DataverseUserPage.currentUser.email} -   - - #{bundle['confirmEmail.notVerified']} - - - #{bundle['confirmEmail.verified']} - -

-   - - #{bundle['confirmEmail.submitRequest']} - -
- #{DataverseUserPage.userAuthProvider.persistentIdName} - -   - - - -
- #{bundle.affiliation} - - #{DataverseUserPage.currentUser.affiliation} -
- #{bundle['user.position']} - - #{DataverseUserPage.currentUser.position} -
-
- @@ -473,6 +327,152 @@
+ +
+
+ + +
+
+
+ +

+ + + + + + + + + +

+ +

+ + + + + + + + + + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ #{bundle['user.username']} + + #{DataverseUserPage.currentUser.identifier.replaceFirst("@", "")} +
+ #{bundle['user.firstName']} + + #{DataverseUserPage.currentUser.firstName} +
+ #{bundle['user.lastName']} + + #{DataverseUserPage.currentUser.lastName} +
+ #{bundle.email} + +

+ #{DataverseUserPage.currentUser.email} +   + + #{bundle['confirmEmail.notVerified']} + + + #{bundle['confirmEmail.verified']} + +

+   + + #{bundle['confirmEmail.submitRequest']} + +
+ #{DataverseUserPage.userAuthProvider.persistentIdName} + +   + + + +
+ #{bundle.affiliation} + + #{DataverseUserPage.currentUser.affiliation} +
+ #{bundle['user.position']} + + #{DataverseUserPage.currentUser.position} +
+
+

From 4d430c266f4b652cfec681deb3e0b9b56108bbe1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 3 Feb 2021 13:47:45 -0500 Subject: [PATCH 138/274] #7534 fixing default values --- .../authorization/providers/builtin/DataverseUserPage.java | 2 +- src/main/webapp/dataverseuser.xhtml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 9d34a6c4e63..d3c2cbf711b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -132,7 +132,7 @@ public enum EditMode { private Long dataverseId; private List notificationsList; private int activeIndex; - private String selectTab = "somedata"; + private String selectTab = "dataRelatedToMe"; UIInput usernameField; diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index a339ba66a69..ba25798e081 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -39,7 +39,7 @@ - + From 6a2e472bdbd19255aa2587f5f4d1c8cdf95326b6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 3 Feb 2021 15:13:47 -0500 Subject: [PATCH 139/274] #7534 clean up comments --- .../authorization/providers/builtin/DataverseUserPage.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index d3c2cbf711b..204d93b5b8f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -176,17 +176,14 @@ public String init() { mydatapage.init(); activeIndex = 0; break; - // case "groupsRoles": - // activeIndex = 2; - // break; case "accountInfo": activeIndex = 2; - // activeIndex = 3; break; case "apiTokenTab": activeIndex = 3; break; default: + //TODO: Do we need to call mydatapage.init(); here too? activeIndex = 0; break; } From 481c3b4bb03209bca88de6a6b70673e86145f791 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 3 Feb 2021 19:23:03 -0500 Subject: [PATCH 140/274] implements changes to the API path as discussed (#7084) --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 8 ++++---- .../java/edu/harvard/iq/dataverse/util/FileUtil.java | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index e0ed28a9f3b..52790256bea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -468,12 +468,12 @@ public Response getVersionFiles( @PathParam("id") String datasetId, @PathParam(" } @GET - @Path("{id}/versions/{versionId}/fileaccess") + @Path("{id}/dirindex") @Produces("text/html") - //public Response getFileAccessFolderView(@PathParam("id") String datasetId, @PathParam("versionId") String versionId, @PathParam("name") String name, @QueryParam("folder") String folderName, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { - public Response getFileAccessFolderView(@PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("folder") String folderName, @QueryParam("original") Boolean originals, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { + public Response getFileAccessFolderView(@PathParam("id") String datasetId, @QueryParam("version") String versionId, @QueryParam("folder") String folderName, @QueryParam("original") Boolean originals, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { - folderName = folderName == null ? "" : folderName; + folderName = folderName == null ? "" : folderName; + versionId = versionId == null ? ":latest-published" : versionId; DatasetVersion version; try { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 58a1e7e4c14..04d539ffb98 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1935,12 +1935,16 @@ public static String formatFolderListingHtml(String folderName, DatasetVersion v StringBuilder sb = new StringBuilder(); - sb.append(HtmlFormatUtil.formatTag("Index of folder /" + folderName + " in dataset " + persistentId, HTML_H1)); + String versionTag = version.getFriendlyVersionNumber(); + versionTag = "DRAFT".equals(versionTag) ? "Draft Version" : "v. " + versionTag; + sb.append(HtmlFormatUtil.formatTag("Index of folder /" + folderName + + " in dataset " + persistentId + + " (" + versionTag + ")", HTML_H1)); sb.append("\n"); sb.append(formatFolderListingTableHtml(folderName, fileMetadatas, apiLocation, originals)); String body = sb.toString(); - + return formatDoc(title, body); } @@ -2034,8 +2038,8 @@ private static String getFolderAccessUrl(DatasetVersion version, String currentF } return apiLocation + "/api/datasets/" + datasetId + - "/versions/" + versionTag + - "/fileaccess?folder=" + subFolder + + "/dirindex/?version=" + versionTag + "&" + + "folder=" + subFolder + (originals ? "&original=true" : ""); } From cdaf121610990e60aa2a0c2eab5c0da0d97a4f73 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 3 Feb 2021 19:26:31 -0500 Subject: [PATCH 141/274] Changed the release note to reflect the modifications to the API path (#7084) --- doc/release-notes/7084-crawlable-file-access.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7084-crawlable-file-access.md b/doc/release-notes/7084-crawlable-file-access.md index 111dbeb0547..a8d5660566f 100644 --- a/doc/release-notes/7084-crawlable-file-access.md +++ b/doc/release-notes/7084-crawlable-file-access.md @@ -5,7 +5,7 @@ A new api offers *crawlable* access view of the folders and files within a datset: ``` - /api/datasets//versions//fileaccess[?folder=] + /api/datasets//dirindex ``` will output a simple html listing, based on the standard Apache From b774f07e697ac3fb063d158a567f9161bf8e8218 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 3 Feb 2021 20:04:05 -0500 Subject: [PATCH 142/274] changed the api guide to reflect the latest mods. #7084 --- doc/sphinx-guides/source/api/native-api.rst | 43 ++++++++++----------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 926470a93e7..5773e1fdbab 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -786,34 +786,26 @@ The fully expanded example above (without environment variables) looks like this curl https://demo.dataverse.org/api/datasets/24/versions/1.0/files -File Access Tree View in a Dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +View Dataset Files and Folders as a Directory Index +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Lists all the file metadata, for the given dataset and version: +|CORS| Provides a *crawlable* view of files and folders within the given dataset and version: .. code-block:: bash - export SERVER_URL=https://demo.dataverse.org - export ID=24 - export VERSION=1.0 - - curl $SERVER_URL/api/datasets/$ID/versions/$VERSION/fileaccess - -The fully expanded example above (without environment variables) looks like this: - -.. code-block:: bash - - curl https://demo.dataverse.org/api/datasets/24/versions/1.0/fileaccess + curl $SERVER_URL/api/datasets/$ID/dirindex Optional parameters: -* ``folder`` Specifies a subfolder within the dataset. +* ``folder`` - A subfolder within the dataset (default: top-level view of the dataset) +* ``version`` - Specifies the version (default: latest published version) +* ``original=true`` - Download original versions of ingested tabular files. This API outputs a simple html listing, based on the standard Apache directory index, with Access API download links for individual files, and recursive calls to the API above for sub-folders. -Using this API, ``wget -recursive`` (or similar crawling client) can +Using this API, ``wget --recursive`` (or similar crawling client) can be used to download all the files in a dataset, preserving the file names and folder structure; without having to use the download-as-zip API. In addition to being faster (zipping is a relatively @@ -834,7 +826,7 @@ or, as viewed as a tree on the dataset page: .. |image2| image:: ./img/dataset_page_tree_view.png -The fileaccess API for the top-level folder view output will be as follows: +The output of the API for the top-level folder (``/api/datasets/{dataset}/dirindex/``) will be as follows: .. |image3| image:: ./img/index_view_top.png @@ -844,15 +836,15 @@ with the underlying html source: Index of folder / -

Index of folder / in dataset doi:XXX/YY/ZZZZ

+

Index of folder / in dataset doi:XXX/YY/ZZZZ (v. MM)

- +
NameLast ModifiedSizeDescription
subfolder/ - -  
subfolder/ - -  
testfile.txt13-January-2021 22:3519 B 
-The ``/fileaccess?folder=subfolder`` link above will produce the following view: +The ``/dirindex?folder=subfolder`` link above will produce the following view: .. |image4| image:: ./img/index_view_subfolder.png @@ -862,16 +854,23 @@ with the html source as follows: Index of folder /subfolder -

Index of folder /subfolder in dataset doi:XXX/YY/ZZZZ

+

Index of folder /subfolder in dataset doi:XXX/YY/ZZZZ (v. MM)

- +
NameLast ModifiedSizeDescription
50by1000.tab11-January-2021 09:31102.5 KB 
50by1000.tab11-January-2021 09:31102.5 KB 
+An example of a ``wget`` command line for crawling ("recursive downloading") of the files and folders in a dataset: +.. code-block:: bash + + wget -r -nH --cut-dirs=3 --content-disposition https://demo.dataverse.org/api/datasets/24/dirindex +.. note:: In addition to the files and folders in the dataset, the command line above will also save the directory index of each folder, in a separate folder "dirindex". +.. note:: When running in recursive mode ``wget`` respects the "Robot Exclusion Standard" rules, as specified in the ``robots.txt``. If a Dataverse installation is running with a restrictive robot access rules, they may need to be adjusted to allow recursive downloading via using this API. + List All Metadata Blocks for a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From a853e314312010f95991d7e0012b9715aeb3d5e2 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 3 Feb 2021 20:08:13 -0500 Subject: [PATCH 143/274] a (#7084) --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5773e1fdbab..d57b4ea5f4d 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -805,7 +805,7 @@ This API outputs a simple html listing, based on the standard Apache directory index, with Access API download links for individual files, and recursive calls to the API above for sub-folders. -Using this API, ``wget --recursive`` (or similar crawling client) can +Using this API, ``wget --recursive`` (or a similar crawling client) can be used to download all the files in a dataset, preserving the file names and folder structure; without having to use the download-as-zip API. In addition to being faster (zipping is a relatively From c0d5c86c1a7f65708af4db04860c0fcc7487d477 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Wed, 3 Feb 2021 21:26:18 -0500 Subject: [PATCH 144/274] escape from HTML: the sequel --- doc/release-notes/4259-java-11.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/release-notes/4259-java-11.md b/doc/release-notes/4259-java-11.md index 89af24305cd..eada46f3751 100644 --- a/doc/release-notes/4259-java-11.md +++ b/doc/release-notes/4259-java-11.md @@ -27,20 +27,20 @@ There are some things to note and keep in mind regarding the move to Java 11: For installations that were previously running on Dataverse 4.x, the following JVM Options have been added in support of Java 11: -- [9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED -- [9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED -- [9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED -- [9|]--add-opens=java.base/java.lang=ALL-UNNAMED -- [9|]--add-opens=java.base/java.net=ALL-UNNAMED -- [9|]--add-opens=java.base/java.nio=ALL-UNNAMED -- [9|]--add-opens=java.base/java.util=ALL-UNNAMED -- [9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED -- [9|]--add-opens=java.management/sun.management=ALL-UNNAMED -- [9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED -- [9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED -- [9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED -- [9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED -- [9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED +- `[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED` +- `[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED` +- `[9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED` +- `[9|]--add-opens=java.base/java.lang=ALL-UNNAMED` +- `[9|]--add-opens=java.base/java.net=ALL-UNNAMED` +- `[9|]--add-opens=java.base/java.nio=ALL-UNNAMED` +- `[9|]--add-opens=java.base/java.util=ALL-UNNAMED` +- `[9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED` +- `[9|]--add-opens=java.management/sun.management=ALL-UNNAMED` +- `[9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED` +- `[9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED` +- `[9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED` +- `[9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED` +- `[9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED` ## Upgrade Instructions From 61c5cba10846f3a67b13c6c12b9dce26b68605f5 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Feb 2021 10:37:23 -0500 Subject: [PATCH 145/274] updated the images for the guide to reflect the minor cosmetic change (version number added to the dir. index) #7084 --- .../source/api/img/index_view_subfolder.png | Bin 23816 -> 25559 bytes .../source/api/img/index_view_top.png | Bin 24369 -> 25521 bytes doc/sphinx-guides/source/api/native-api.rst | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/img/index_view_subfolder.png b/doc/sphinx-guides/source/api/img/index_view_subfolder.png index c793fe6135c1824506a62c37d24319a4e93013a9..6c940ad23541bddd19672927fd0c6be5b9eafbb1 100644 GIT binary patch delta 22429 zcmc$`byQVt_ckhmB2pp>n-)-zZs`_~mPR@Rq`PC$xdD}Kq(eGHx;NcO=cc=J184C( z@B90n_dVk~t&FaL{Jr9Y&8YQZFiN16YVP?0An0*E z=iCou_%8(#G0HHvpvb}<-h8#A5zNJrAZCH`?$!K0f8~1D&$-}Fo$5ltcQ$FLpAka! z6rHKw24eoRjP}=R)r1Z@wjjdh;sPGohBzK1GfpsyWghz|sv@_Kv*5npaBy?JaBW%d z%c9PiAzTCuaIYsH>{JW}X2cj+L(@t1beOM~w-tYYDvt3K!w5 zJ#Y1Q%3u-a^|~&DD<}e#;p~bR;P)S=*IpO69+LKl*hLClZA{|3@BKEpIO=6;!`Jor zCSmTHHna z&xh?xaw=XqEKa0v*sy8c@lwqqx?b}ZoB!lv|6_xoaiLQ`yK%d!qUx{DiMrlV+(2fMMY*IYfBnFi$$rB*qhbMn|~F-Rn`Un)3zfm|$q8n3nrW5;W@i|Yv2gmqKFMS$5Br)Ae2dx@*x zbaQ4N56eli(twkQc{(q6o+mO~(5*_oR(&hmt(umN{5>SNG&Kht+U(*bKjcg+#pc>~ z2@W~cqjJ12y-$eGcKhgO}9vvt)JcY*4w#Ign_`3 zwQM&W>!l#*FA7%{cDW{W0Z;9z6eE{fZ6DIE#k4zwiFVRkgW}(7w21ghA{^`zY~o@G zQrFMB;7TYsHq6+d&&KAtR5cny4Y)l-voEUpvqev@9Rajqa*mx!nsgRj)e!D{*cmY) z2LM-<-fYjZ5tDegsxXBh97@6Xa$z60j7EMR1{ozDFBeJYK$q^=hWKw*Yn7mLf|bW# z_dXFPLUY$3oBiNpZ@d9|$IwIdx6x>%(+O%C6XmWiHf?lF{9RmA_{jj>=9Sg4fqFh` ze?1*RB~R!)^ZG>@-Piq$W{lSNu7piMD{^+b`e7;C02d`@=$?2TBmEOaJ7Iiy9n=(= znh927v|r{_NmHao$tgqC#~|XT>vfr}P)T|$@l^IQB0$$w&SO|jwI<)+@q?(wMQSps z=gII$jiz;jBWcxMqlJ$I0ez>P!4EkI@AJm40eIGV48#$rA%0P8r0B;Eclr$+2mNBh zkgwYs+VeQVN4eB>SqiXyz`A%8#LaM5J!4f!^xk5dT-byt)N9IhJ!NzwX6}RQ$Zb(i z-udaRr?tso{6XokrojrT|UN4YaVdAu>_yrBL0yo)lj{ddgnaWV@%MFMh8ITTIOSzR`c zw_v|xnTbyHum!2u4Tq=m;sAkW3AL>$Wos}i*uDa^qEWZkws`O=2%H;vBK>jI z)XT6p?jbnre>c-tQB*fF;VNLXzveih&DmK|@xgKPw?4+ItBvlT{axUTQqa@Sr1SI#-M=AL*Yho*LFI z=UH0`ALp0+)v8=7nEO~jX0bqVsNF*7gq8908)Lz`){6NFg45KW)^%l;>s>T6v_Uv4 z$!O25XPN)}-g^IuWB?3M28~DO$Z89z&M7ybFR7q+P0{5&HhULIkuSF$nsl}{;+2Nm z`+7*{&}mI9bs&DXVo;#$$?Kc3{aO3LQ;IOUk>z@l*@~f_?!&i$VJS}S>ZP|tLILxs zXn7j$cSCr6BRqo>YqaKL!Q8zVqjvrx-AiMrMA zRbPG!u~hXk7E@Q-eJo8h@pUl!A(h9k1H7`stBbVOEdjb~Avd`Y~J#grMVVJ19I5l(Ppjq7(f;pG+{%?9g*`iR6J9QMGoM&r9r1CQ|}= z?gR(m4F`dEAY^K%kuYk8we)H<{5 zT26S{M&$)biu}EkBzu=@jOTAg({5K347rY34UZE{QbPtqBr)Jx%c_<JOBp7S;kg#Y1eTS(#h?T1EY$-{&3*$0sTf--)L#iF$3u~_}=@IZYjI5m#+%Y(qJef`Kf)l`IP z#!>mNDdVJBI)i~aKI-{LK_pJ?5 zAF-3H8(mY#ICssIf}ptA^DR_J5dwYuG%TI2*0IB2#3fiS^Jom@WDTsNmwY2gPc`+s z@!}W-s<1XL*?*-V#V5~q?Bv3y@kF{U@i#$OU&}#>oCLA$`yIbzCZio(V4?C2pII;6 zv4Ej(?YGo(CDC#@D@mgn_xvsNLg}o>svizTucEfUy17ghWLdnF@=%pejW931YBi(b z4>nzkGwVdOvs-A}yhSdTqn}ef@#8m1aa5gqS}ikJit)DkheuMF3enZ-<{4@&qL^1~ zZlrp}%7K>%XC#5o%PlRn5fFdb=|~aMR)BLu8#(w?AkL}sd)~RjUFpk2GOVmdD8J2V zt-Voi-qW(S#lpQY!iWZsFob%5pGs|ifIw1%`ZnS%emL5RnjJf`#C=Z63(}SUDU&vdo9+Qp;#J ze&yC^q=vQV_(=0;tBWuO$1A62bf@yzNYy=l z2M7d#Yxe#ct!vFzz(iX~|Ak+<$ILj@k({%MN9rl-W=Z*a$mPI79UrnlPWTT7r(nLw z0j(vpC5Gb)twsMVe(L#|gU??FjUr6hLU?HNETeR^F*&j- z4T<4Z)%lHC)WZ$F4j9c-Pb>Dwn+4xQef$$Jx(OZ|0d~pNlx^2FC=XQ>U*BoRNJv1? z7}6{G$nA+}a+{%GwE(@~(ExuYp^y4LNXM)QW+crtBd)SJ7?aV-(a~_zn+URv!+PgA z$B7N=GvP4cbtAu2@{1CO=iS%}?1p2gs&_-)tbwqZEj1lgmA4$Qm6?dmAKzaf>%6@+ zpu$2C-7~Ut6i+E-&EeT?Vimt>M#lchf_5$4dmQ-5fF)PpHbtRi1je12Bp1xQ9Z{m| zstO)Z53r&*(60-k*{-A8JX(;Lz_f-YGnMp@9Q75z*K~Vxjjz&8fVcLdGlU5dG8yBc zo!2_M1$}2Y_y}Vl4<6U(JtxQ!6KkNB-6#f^BId?ZDcjxQlNruB^g1%{$(xDz6l61V zNSA-Qud|8Ra%lHUm$56#$m+g`e1}vx8d76 zPoHca+hUWL98u2QF(vtMSC)L0Hx8*I9*QLGvwUPsEWu*$6CXR9yk#22M@^L|rVQwQ zo9khMqPZv64>|Rj5-e8QfV3p*bv$M>+C!~S2|#otY!zoySB#1gU)oMavnDI7H$&rq zZVscdkTk*OO_XnJTQuT}0AYa$E3c1ZJ?z;h2;kmB0uZ%Z(h8t-Akjj@L>H zdD0TP%H(SmmGus-;<;n^Zlu&kV{?(_iCAobrs{5s1g1`es;YzR^-_)8Y$;M)Q&Z%E zCS5IT{R`{NIy(7HY!oV6Gf&p{^pXStCY3e)t#9XQJ$b;*=>*CAry{JOL&P=3+m7{g zKPU#=ZR1iOa0({Mf|b{)tw-OjEVS>y^{)h2a(CKqEZ;qsmQB-s$MIxHEG%;3H4C zVS$k2uKYS3eyU>0o?=g49Gbvzgw3)h+J}P%Cf?q>DEWjT*!sh(6jv1e%Bb@O-O=;! zu){(@&GrBji}xgg{KhoXVjc|`A`9+Y`MJftf2z@!8zjGOoy=GYjcq=Z<2!73+#De? zv>KWD+Ih;9frFHyOdZwiJQU6<4zO<(Ow$TL{Pq=+bTEhXX1E<~^XhPU(rBho=|*Nq z?PqAK8TY%Xl780wiuu&CHG#@6JwW_X&4GXT7pF=LpGJ_oAyq4wq2u2u z-RL7yx~ywAbHdiqcZ4I{!aBQxSYX7Xnd}YkMkNMubKu|B$tRq}oEvEKfyhFNQ+K|D zGbX~B_A7Vq-$OUWBGro}VaCrlmIdw<^>7Z7C{xrf9;V`&NGq~#8oJGl%uRU^tke9c z8?5Lcz!`X!&|WW64)gEyQY4-1Kb59m9wLTxY};JT>)L*c4zqjqe@9{I=Ub|#!9x<$Em%Xg^B zNaly1w2R7?m5y;0J)YPCLEB`h`hZr=o+DxR&yM$vZ%)nE-kE+0vX|zg+)z+^lPWy0 zw({9Gf2c&aX|`_3vu=9jkxgtM8>(FRK+q55)aUEw2zq=Izb?NDyj|e+)gL@x$4|ca zp9@%gTm#sU>+S2LfSYYy-q))h*eKSf1rFIaf5;U=BzW#PuY6g$-J%ubw^xHAW+Tz? zd~sw#kabkh#OdB(71{(xlVQZ+2GOR(%W!{<`Yx&HD=F>g$v}H*wUuM9D3#!V81)OA zwf|zS_qB+5J5-VyRxbxIHxzx*JD1l<`@No8cVJKk^Z7;F?jxV3HPY5wolz_;M?8}q zlGe7iShP#uE`Re!Y|AGJk)1|HY^IuZRYSm2)W%cMEM*U0w3TA6R_Q)5^7~{F93;%` z@r9Q9@c5M3k7gC>K&ZO5XVJ)veaVKxhJ)zD%Og(P`}dfFB{@W$h`{=h1;QK zl^TU%Co{6CcvNXomCydjCLf9UTAFu^&RPlkxTv!;#{>ab)Pekwh-nPvgNEK1nesTVL$IN@hvmPqIL1D1S{nV*q!C(<*Ra!L zZE+KsKL7kQvv1%j_G{`o_e3ttwC8wF3%*{}JeA_dMWuU1}^!#At!&eun-K*VRM#QkSFkj5CIe0M`_7#lrhmWdQT#xWU zS0I6y;@Hx-`LEE>g?9>COV!BNjMfVhh))#W0WWy3xLcmZHXu*=RN3Pxlt$;6swIu^DUm+BGe(zty|yFHrO$yTp1AI# zD?jvGvZIhqXK8_|XvErwOKL(1Uac}y$^n-fSaIm-gI67Ei-2J%|1dee( z-LoNGfQsqGQcy`)XhC?AUjh2t-%Cq92^>7gN_;BSlFZoAij9yDyxuM<<1{0DyxWZ1 z8zAw$#8?t1PAuz!OhJ3B^D(0hyjyOXu}XqB-X`x=!IT z`0rwq8fV`kcNz`ypVO;^QEDgT*NQuCLjkq2{&*^uP zWlvz0x@-n410qjl^g2x`;Z#PC6{K4$%?a4z*XB~{HTpA6x%2BRBBa`Q;d@d##p1!! zfV5TlWN$T_ovr@XxmQ59Om^~K9#HH1XLFwAyqYE0PbuTT27;jfueAaryPwnge zx;m*aAIZ-w&1{?CA>Gx>iv&O%&JmPsa{WXvB|0#+l6IbQVN`!mMOf&nqq6^LsO3C+ z-=5QYI^{P90~tB(mGn|^jp8j^@C!oSCP^!>t~|+mzndLH*m+jo|o=t zMrKSa0kqgNJN4|}3@Ekx(5P~9yRoFlUw(_st!-Wr_e}O?`I?e!@_}Z*jh)`{7{Dm5 z-70t#B?qyWwc6*DRqB|F@#yCfMc^TC>3#6fC$!5KVB&^kc?|TZ=ogHYnI#t+Bnfiv zrUqRy27cmSY6seKhOj5w0-rr=<|>^WRKathQM-JHib0O?s*Z8;&w3Ji-ka6;w?&+R37`rI{?OY%cI@Bi{>^LKyDaeLB6-Nw`oto0rcdP6 z++0p$Bl4mvPOeO=T#ctUjh8Mt!1*gvR>#Y2HO0@YRH={!HlQWd#|CvDI9%~h_o)cn z@pTA#=DX6K@w<~kYM!8md_MVmqyRGos?(Lx|_XPvP>?G1ZugF@pOfX}}*b*RpiDGl_{XakKaaOgX zdL_QKtn2lKIpv9_QNDmxz(-)v|5!LI+Riat9PSrqb48JJ_pxP*KKmG*)>OMYjwiFa zH4MiXN!+h~1UtVbdT#tlL#l`Wnosp?$85$xV{b4>WR=lcM0e1-VfTzw+w zvwGSEJpuRnfgJ(Y#Nw3>-Uu<~_T?JrP@P2f>s*hZxwa*8EfUrk#yHRDCrkRpY`d4~ zoc-hz4(HJU_MbdnUmMw@AijDUaTq5s5>Ag(m6&@OA$V8OiOKbvr*C_dSc8 zGxvYQ_~_gtQM$2+UEMV%f*QX<^?hRUvp}od?$t*L@nPsLI+C0$|6RQ^R|u$n$2-0b(X+*uRINRI6jQN=eQ9OizFRWjuNv%OmZMf3*w4;Brr(?! zEP?Xo^6_!*#sky2IsKcEk7T@|{Ux{hY%oENw!}!swFlHPZEl3cXXOX!IcfqgqH`)F z>ozzIv9?-+*U18ou}iW{Ey;D;lF8u|r{`r7VlJ_x%(I;bf1usLhI}_RaWL~3rX1^# z-g9b2)|`^JVchu3qnoKEx-8Uzm!dmVGx|l4y@iY~D!_sRG}j2eM_Zk>+r!sa7vgp9 zWXsV_HYL`oY)BYbXTi7}wdc=*7w>LIyG_~Ss1#?4IN>FuvXS!3m>J-7$ip>Sw!k=RUih;!O{?Y17;ovHWAh z)B`7lbyg|MU|f8G{FmH!?YwN%yze!l(R80+$14r!HqWfAc&eNozSb0aX>_m)O|JKv zcPc&@4Da&K7W3*~gDpttNXjwyunVLOy-jZE1y(|L)gOnl37J0fe8@s0r$B%m)N&HK z>J&~B$%K7W>I!0@@oX}`8&=p!t+q@Hchn^;Lfu_>6xN(vLJZUD8HBSk@pY?C_++_Y<>RbDRqXEskydV(hwi!S)Ng2{fk^oLqZu`jo|-JTGA;-!2iaqmit{#~#5l?e?~?mn zee_(4P6CQY-*sDRV4b$*)y&Q4p|JKi%r!D*A$Ks|up8AqK2l~`$)g_D);xuuv#EB* zmmFEEOdf^t(o;eM1^H6s&+*EXA!)ZDki5F2m9OSDh#qJ%yWXkwY{|B)ZXw1)DKP%g z((#}m+4|$V2$aif$m2+l;f>Ynafi>_kTK&^%K&S&(o;TOHteVz3w@*SI~fc)6lxPp zm7GRn`ddm==t$4K=B=Y!)IT3?sAsFKJ%w05p^EKLBlH5}`It;LRp2q~+UJrI2!~sB zK65mTuvXKdx?$*{O_B%;eS-z*_84AqL@9L8k^LUSQv zA?EWa7uch>x+8sQ=4+nR@w?#a=@eK%u9<5z!z^iem|~m27F3OlTftIj4p#ffTu_ue zLp{w3locQ_-#2M=9*3h~?4AL7ilGd^{-K_pY=->WC9(FDVpNF0^AgFJ;?RTPCmv3?2w0`J4wl*OPb8^33?5w~F zq-oK0Cmvn-;Kx&w#JaV))z8*f;++IkQiewr+bIIGPKz0N&oGxdVmkr-fEeGflMvz7 z2`(jZqD>#{40e9o%V)UColv$z7VcbAMdt+N%siWAW{$*Vaw#$@)y@IpXLHp*31h{6 z>rA}fZtc!W`NQCRESw&#PMj6*(%^BBZF^jg5Fp{>pA$3+<1Rr}Yc2dL_%_ zaIna->ij8RnJ;LS>VxppGbMiARTSL)2d19rbf2GPX+tTmVrxs^8a`iNP43KPWQNsy zco5?ZTzts3GxB|+)Av#B$T7ri>LM56fV6K^I(cwOX(>u;5dyp{Z!$hTZh6;V zxWzZSf&RUcVNP=8W@+~klSyk*wn-V%YAfa-byI8onwQt0x(#jH0{oKW<@?`hx}c8{ z6b)ZfD2dbETPb#6;C{xX0s5e6k9kt#Z zuOP>S79;4BqXY66xiRmCGE7ArW``-0z#95CmPMAWdn;h+$MVs7D>MNAWMVXO_T+tn z%F+9UvG|=l>PJ+>cHNF8g)5@T!asl=pAZua2$NB6o~$o~3n2zsf3lNJWYIy)9M_c) zRL&w^SH?X^PUvvP68HFrAK$b`L&*rm4ZRdLjRlY{p ztZ@l^jE?*@uo8yFblOQ;EBS5tg438~OZE6$wYcHP>sp`RF)V%dtxQ#KV;;^HG@8z` zI)iS)W11h^-WxAv7UZ3j^d9F-SbnUnPL#Md(3Anp2pyXV;yO|8eq5RBouKXuXoQ|c}71wkk%eaXYe%I8g!s6TPhzFiT}YeLbTur?e-5q z1H5jo0%nyD$cPhT`+u<|;jlHr4c7HrKTw8`v0~z6T`8hbi9LV#KlCQ=XErEbv5vKY z#|j)MT<|u69k_ zB?bZatgHWdqYxWPdmg8#g#Q1u++PMn(&O&Ck!~(j|L04Q$cHdMQdR$uLi|5F@qV`P z2JGhi`2nr!f6}tpMM1h2o5KsL|7AE5*r6L5*vtQe?ZE&2TmRo1|9`-C;I9$-|7iUG z0o#H9zXtwawgdl~-TOuMzZv*IE4)b}{RNL>*~N8su)Z1u^5cfB+d$gNa{-X3wd_^1 z6o{S~{!YV|Nn()v>v?z_dmnS#J^QT98ujWFQJBG_c-M-b^#bcV{0dIOfI z{2-p;bLk1@o|Y0$J)-vup4o-s4iG8E*- zRT)O|S3il$G{LDkn>I(^X^6Zh?@bF_?~G$b>WK`^ae+ zEDhf;ZNqXrgdXPcgAB54+a%XU@bRh`HJ#sG<{*oWKEoQMTMei2+?84;|t1|=MNue`iQ6cNGww`ESe1RC9 z*l)bHvQVa+y#@@zJn1j&+s9|x5!E2fJvI;0hy;`z)9fd-o#z`6bo8|d+YS*XGBuLt9EZ&LE(*py zdDh^f3TWCDX(z7tik9R-zw{c)fqRYKN) zJSTtTgDpZDS_l38DNBBA}`ph{^!Ql8X zw?7koah~Fseq;!BmWeH(BX$86eU?Bp7VW;W_@lZ55=fQiaU{eu_S*`qALv2V?@1H} z>T0+39wSb<4B9mAdfD7}h+{*AJO*wba!9UnpgZCG#4Pj?^*#*T8Ui~YbH>(-(xRgO zT^KDjX$faBv{2LP_Z#mLGzlJ+N6Hm{G|Y?gj$){^48u8EX>x(&=Pp#>KD%ZAVD-2E z|L|-h(az$#usjbZxE5@{Jp!Z9n=}S_f2AW*bRE|}0tpL>zc@_D29U=)P;E(r`QL0y z)*cI1Z6QrXkQbfXF1T{E7dvX&pG&v1Q(dBQ&f17-J!l@vzG~d)SO7VtOPP@ZPw)AP z@s`!V)ML%gJgiAPPL+7ksMa8mb4Y#9h1Cb&hX#_e3`wdtG{8kZ-lCk`iow?RY#z9~ zDV;Vee57QGM#U1rAyjg`mSl~E&RC<@f>=|@l;6<2dZ}$QlYmzEvs?;lygh2O@=f9~ z#AU{5FbdChhyldvV3f#M$SmMCn&<$st;YOVjGoWcGF$0*9Cql@qALi_4QhZ?7`ov! z8;4DT+>M$@_K^=QuVY$XFO_Ngi{8_6Pm=U}(ONUY7^j8r$QpL;3Bs|$0)D(KR<9tF zwlES-3vl`rs6BHMjGuyVs1`A{QO)|GXXt1!;NT@DCqrM_;`V@og|Z1YbmFH zG?^lhd*(r?9iD;xYnQp>@O(s?se?P1C2FbN89Ui)vyr? z%wS@IOigEKPiu?`kYW;gxE*#oC~*A~0zWyuyFG8x+hVpQ)%pBQ?4DwaKe_sbTK1H=)R#+73aX>u_SZq(D15GtBvKvEc_m=h|%^qfB^3EC4 z+!VeGY-h*W8eeP5htA3(%@ePCIHIt`cWNYHM68bmu0UpsIEP_|q1K6h_k>QPSQ-Rb zH*;`Ujft{)P*zHf&G^R$Q9)h;d!p+xNHRsT?XtfHbUT^MzK?~&@UdaNazJ2_yPNg9S@Fv=M+3*Y0*0hnpT<~2%$>M3*^Co_vT}q-;=p>kgW$;vf}Nr6O=hSg~K~x z)K{a(e)sI}3~hnO^|Le9O$|>ncl!dWKLAj-o!J>ILt-K*j#|m`C3&9{;nP!sfzLvG z)}3Eq>}p!JVk{BiArbaDaCGtZ<7<#hR_6ppq?hNneOX)FP=!=Bkj_sB^c9^iJOroqLa{37zb!z?F`h=-CXqS`P-(1;3hL0DA@cQ_9_V!Sf zp5R{G6LAkHu`n{5p6Ow|(tXnW;ufcy1wIW`_@xbf*b4@@5;-P!&_O_lTC}1f&);C+ z_dP`M+cK&eQE(O~F^-F?0doC5d2_7`0UQQn&hxJ8@sB%E4d`7+EB!oDeDo2`>}?2_ zWMzrDnfDMVPy6IbYUQ&_R}GU9i0(zph7dFJY}=Sn_4dcCB|WB4) z?nI0@Bf;m=aFzkrT8IKT5BJl6;zMrfyFJ>j%4vop*GVjv0#nu5JAj$llz-RSQ z)LT)7pjuLaE*|9a8bW}=;##v=uN07Lhc4*u8Wcq4#r2WuGgEzD>!N1==9%@k4-pTM zl(nF)PtNQxGb4D+l+{Y(tT3Eq!hIso^S({$x8z*`%RJp;;wd)ibXFr96pwfoJF^}c zPG*KG;e6$qTz2*}*}~Ab8re4cqvE@}g2Ky1)TXMgACPzEA5jLAbQ~*;cawp`WOfH3 zzcxN=lBr-B1}^g#Fz4w^A@M~L#)2&IjWd=(*_Rwy6n?^8Es%$P;6&6G|B>fERUWez zC&2`IY+>vAFkWnz6#NPUZa_prwdR@ky!i1qm z`eS4kK~}Piy+;Ia;sG%Dl!B+qAcpEmih}-0k#~WY7w>%DE04dk**UX2YT0t$(#E_n;g6C%}Lg4LAGUMXu183olxhUa7V zjcpYxo^4Lq@lY-B`6x5MJ2_n374Y-j+E$B}&kDfkuw{G^ zlM|shr0?O^NeT`VfEt}n^qwuM4rZ&!yhI;sgL{%9UQb(;;Ll5tc{+T4ufH|oWisD| zjfEfL*3tq?N#^>R;B|sYl_aJm2eqGGW1^Utp-;`P=Sr@Jy^4bqhV5f}i zYE`F{iM%S1XtvORbv(!8)(h3rMn+@7_#(%w`2j^=X0putW0=0V;2Q{?&a9Q>8%+CO zg%52aKk-uIsT=9iI*LCSjnMgZQkcE>?q;56(|~r@f_ikv3~eRk#S~hIKrD-vMo5!1 zT>~bmvkg%1D^Z&%PV_0Bk|YR0Xk9MYnKpG0~ky(-^ol#@G_Jo?G}A{-F0!I9;A~i&3`c7jMGFAjl=CNlAH_ zEG0cP;27oA?z1UM7}sfQj2&etx3{i_n0uW-!VDeO8v}{lu($oKe|V`nZ@;uVxl0XH zwtn!%UPEymZgr@Q;5hu|n^S$TiG4-TpbWurV*Hxb<}kH1SMw%K3P&MLSY)3T!<57Z zXjA{OyeRVkogTv0m4G*!Pv}izh5q5ex7&(5?;+X1OEHp8g2!V{dasAV0*2g;h3z(e z1a-RKc1^&dwq@YfL6(MI#e&-=bO+8ycFb+-?Q+P51T*7K%uH^%W+Id6IsJKlTrES` z#r};P#%w9gWPwS?*u@el@PP}%J}z4<0BH?d+j*?Kc@qE2naZfmJ(Ip8;}bgV)yU{s z`?yL<*UWeW1RoT2nX@}~**C}Mg!g&UJCQqSe_?Z*S+oO_SyV8IfqN5Z;AQVT6Qcj1 z{$wU7beTU4&j~VwX40pGw*rqkCeBw*X72nMsnE2%@30!9)@eeZ?P@aSY7C?+dv^SU zy{{{*n?%U29|)XdbzLFb{s$%?*=Mvxok9_eE8GF*K+tP+dppeoL+a{Bw(8&%rUh~=AR zeD3Z@GCVTA9Z*Wfl+NC5ibIBur+(0dMh8iyUi~fh;%bs1=WFL!&Q>R$>4u@<69Z4# zP=e!@v5cg0Hu3zoSiP@IH5(eELsT$j)>yn67`5^>%@lD2>2(f(kPCLwNs)XSzSfoi zyOi0Hpj<}}NN+*rGxnM%jD8fcyy%hb+y>fq}-*1bgd@AjdVXg(_}%Y@B(+mhCU;h=GlAl3*J10?iyKqL{)7wveCf$yBk8k>H zW@>{sr6@@1w#h$$>&Sfj69BqAFd_|T10?QP52UNFO~YL5Pph)NNq$V zu4GNaNf?wM)!()>s?CRAHt!*G^;Z-^%TzE~7W*xVbePp6&eduoI zB5UyJAOT?FSX$q@hXqpNX!#A1RX@NbdyhAQvg>@wJ(n)*KN4;79q)d2dcYwVBr?|= z_xpFK1_dv9AA08`%;YECsS@E|N0CL}=2yLW*o-9UPJii5A9J9nzq;}zG-o@L4HYgz zD=4{#x%9pYrOR*SVL$K6|NfUN7JT~LXZz*@|-wprzEQxynnX;5t>7Q=}0c3s@dCpN^8Z(^Fsg1x$Y+|osxNg3+@k(;I&GRD}4CZH}D4t8HuN$mFhNm@vC63 z(xLoq$-jp}$hY|!N9Kx-+IIUEjayiD zU}S~ihXZHQG35YwT#7vfdMWR*;2&lw$CX8UnpI^|Nwu24K+zGKGEJ_E<_pGujijC) z`H-pj%_?jjJ*m%4y^bHj@h_<#xsUUjfY3)JnLle&MAYrsjaV*`#xk z^utr>{7;_z^YCA%*mpWh^`*Naxv=_To+WZ@v)-z@n)<^|B7G3U#RfIvhNhnP`dAxK zpBWT@r;!J$FOEUH`6&nt%f#Du62PtR)kpW?`_ouU=ROGLqJgJ>&-i=z=pGLTRJ9$) z@8#KJRr{cH`zdgHoYvNg&O{7?T8%a!o%v5d+gA{ixx>82iy&4tORXS;M7+OEapN>v zuyDi<3a8!F9frjWr+OKFX{9sLD4c=#JWfusT~F!Bh;$H0Yvt7fMTMr(is#!UO{CSJ zs{5Xq%M9w-CC+mJPCnceirM4_jd4naHUup zOk3|tn|1n6IO%NtyqGacs4%x~JW%L^&y;xOS@O)FReAoY$vf*EZ2Hr9eFd9dFyU@A zhPq?i-OiAi=iXiancl)B)^J~mRpU{y`m}P4U0+z;@0;6!h?qtsVkuxvZG=v{DUksx z*H>Zt=1sRp=DUlSOp&K>QZp6ZEFxICzVYIl%3S#S^Omwf_}60MVa{CN519&Xwn1jjsMDu~$b zgJmd9PTswaunLr%vjmQEPG~6!3TU2cep|AR0BIV6Yo{ww^5SGYuNS}tQ5#%)cY7@% zXvmkj00Pu6_9bV6p_dV-p!|^E{MW3Vv?Zt=4l~8#J7@-KUk~yf%UHt0IT<_KuJ@aC z@pYU=?vdO%cvCiUN3tzYk&Un=)>vViZ|~SLDrp)rRviw|x7x6Z6*R zFV7F-bqDeUo$oYs z^-f@djo*KDHe-#RGnTLfYRv_%aMo6!MUoNb_f@UWRI*QZS~!j#HF5k=&pfb~i1DIi zM1ORo4vq=G>4~8}lF}0eDqF-nW%=Tw#9YsYFFIr190-Do4S`M$cLs=GhvU2hi!A;C6%7LRCSdIPwt!al=D-7VT9#WV=G z?XBx5#h49!8W&Mh|AJ-jGYxIAbfp5OI8s-DnZ{w#HNT>b`+63q@$v}O9sfp-i4P$M zG20+-tF)I@2V8C!6qUrz?*@Q^1AA1Sg+aj!r;CGrrdc|~!JPck=y?;Pm*5Sv(8O61 zQ)8r;uGf(z4nAzC4NJv&pL6U{qRFM{ES(JN+@$`+w?MRCx)(Ah<|?{Q#S1Qbnaph- zxIs(e#5rB-cd?{HIqt=7jR(>s=h`IX2trKYgKsIyM_Dk?(i{9DdDHCc`HR_Dx zTZEDX0i&tQ9;tEs!gg%XC4T|2_7AY`;9a82kL{3t?|v@WI>#H};vIytS^@rF9aevmKzIAz$93NYt74y z*lN-5=a_)*4GL_#L}W@eeY*gm)?imi6k!QsIE{)1>t1Ng94J^1I$;)!@vqUhC%J+F zpcspp4-9z2L@{5%N&NW{=uP*^+vM`W*`n34K0n_EOMu%0Jrm_+YT2@Rk4bErk@UOB zfwY`@`JRK5rv#ZKhv}GktTEwqkx4q*44tqmu-Jev^POxD^z+Q17s*vXcm6aZPU^a% z7nTVLI;8tI=U_+$#S1`I?fP*yihQED;C2YN>FBM>_Qr!ekk~W;F z0=#XhNErf~uG+7~_=taLA2_Hx7i6}j7V1Dw8|7!0av-;dyt-nZ2mNb&G&>DfVoND% zk2iW6&&E*odtK$wa(C>cwFSbhblg5Squb3&ikF0mc&b}?oNhY*Q1M)F-yQ9ek!S9J zIJ!bPfhli3T^7%qWWDwh0ow;kC(CX7;v_c{7@YC9$P;z+J9TRvElyOeKHEB^|5q*N z84Xw4?(sw!L`^~Tk`P3h(MyD2l<0{BQAT)(?vaQV>>&Dx7HvdC7eSP0L$pz%6B8tY z7$po*XJoW-wsTJ2=XsxX)_Omk{c+Y_*V^}9_uSXMuHXOq?~-yR-@toE|)R{GDU%`5UiLxKx>PXaN=F7H;LW6i5N2szJ(%GP{Jjpk4@6PSjf@Su-4J&;+bi>km+> zIfEag%qKQBK42eju-zvL!=$pt;tTqj*og^`0w9=(pE$7vYWt32@^$Y1*7dQMkDeCa z{Q0g`1W4Ls+M+dwL;}sWqh>-5Jbslnop8LZS6zDMy@hiDUqTYFX>f`^ls|Q?Q~#2T z_ia9DjJJ73ayan=4g6p63=;IYCtQbil};yBJ~dSzxa~LFD-u@&9y-*%K?d)xXOtQt z!~V#I-HVvCoZFO(irIT|5h2G-bcBsB;~9a zzl@Zkp5pdUd`Hk#BQBvKKih40Xz)*`MJ-{KcwnH92%2*o2(h9OLs{6A@a@$Mc{i84OOSCnT(w3cTGPgHEX9BlUX6+doWHE)3 z_JGth5>$$`6DlweE_W8Ur_>A|@7PpkP*BE1aEz@yXCU6ZT7F@kKQ*#)CMiXxd7U|h zc}#8G!U68YM?9Z#AM3bj^=CV^gIPwkP@09{%^6drr&UqbVX=LEmJhO7$8DedTt*O? zW3CPGmWn6xa3jsP^?N^G?laR!G^zPm84kqXPu3IGOL^QkUE-b|8D6~dwuhP}*_w^L zXqG3_qsIXxxUr6Tl9V-B8w?=Yyf&AtFG=^E&ZD|lxZS5Yzv&9O^u4H5x=ko}5=cn5 zIf`9z3EHRD7cc7P4f0zA=hOA_Wql5Yy1-p>e(jqE#^2?XfvT(acjx^r?|RxeKz&U* z=6Ct$!y0NNgSTZtFR!jCw|(Pt6?HIHbY<8-#02&>PVweA<6){U&1f?emk;>A@WG)} znC&`8;-WlNfAv-pB3s{TYYhtn4)JOa83rGJZI!8IcSbi}iJfpF{#4lV^4NdfV(3XZ zQ+Nc+X9M!%RqZX@(gAAB<6>1dUzzi%x`bRr2)R(w@v1dm&~FE#CvM=ge_}jiwS)#o zz~C#SlL@1W23+jLRt`;^^0ZRQR2hH4tdqwj)JrYKdGN5SxZ1PB4W{AM>KmUaU9Jvr zC=b1-iHxVFdS)i)Ayved?Lt7l0t-KpP@TRtycD#<1n^8tmzK=}h3f-Dv@4bcRnWOIE!K`^!3RwIY^tE%!?ypA5|=8>A5q5h>-7HlvqO1z z|GRT`QvbhxOY4(qTHNM+g-4Z_m&fhU6elovKaxv1$OSzsx)&@{or%R=N}~Z|!^sF#$g&9GzxJNSkEUCfq!g{K_ z(lC!zaJ$KX{P|cke%8jgruu9qf-o!9Lc)F2x;yv?T-0;Zq{xoR+~n_6^Qb!Zl?OM) zX)XAzLCPONR+_7bTUVid1+fVfQFHevjOysHUNEQLCfPKiB?DV@H@o8MPm9Xb0qaVK z&UXXDGSiTO#1e+uWc6aA{b|Uay$GXQdXsCG2y?b~h8-K<+9#igO3~qfm{KjJ3Npsu zZuD;IBoEF2qJ@b5+xE?ELM8f0p&VJo(;HY{bWLO`ee++MpUbar6=zCq*Sn8 zSk`R*lSNFnKSwwYFhABT4I!{iM1tjvfwIGyzf*am90kPXzeH0fAzd3jn8c z$3hcXa!aG7E$59@e67Ws$!j`WBgJnAUpEas58I%UElu1IW+Bgh@Jv7-Hf0~AXN((I z%ip|9Qs^j*9x^r=ndcTBS8KJcRf2(3DZPvn89>s!2%#gk`0YXfk>=B)^Gi zhxyWSr%vR^I|ZvB@M~ticI0W-n4a>7kBf%gOD5ntAE-qY@w!U66gv~GWVzD&m6ae? zg+m?^^MIj(a?X0!tjP#iP5#!r2Dg!~{X{qq+rz{=sw@hLkxf znisz%(aB9Mz5*KN2tD>B>wbefq&a}JTrL%RkTnz}h?MT8<;mFEGo;}Kn93!qJR7O@ zYnUM=ymq}pg-U4|($ZhjEqJo;w-B4J%(s6gl52F4_X=A&kMK1c9jd+~tYKe3NF*f&M-}0M&lMu;RtRj!H(Hf)<+<&Ej0h>r{%zfm;`BU0E_!Qpf2^?N>`L4MB52iBOVDSKgUt zRNPm&JZG4@vAsg=wrUStP0v5gRmGR&<3dd?I+}lI@4RGi4$-k-Cx-53Nw&YJhu~j0 z=P5PTns;VCTT=1bwO0GO5#Y0(2s0~-8GmB;qsl}$-ToAJ^y6AuHqPbN{26sct$St? zBagI+p+Sk87!x#SeEM|h>@{Ab;+~!4$Q1p|iH=5OtZ7O7l=7Ikd}T=ret1V{1pFqi zC!w$GwAcrQR_gvo;o=N3n z)92p5%FLd__$=X0GEiSV_y_}aNvQl?W|`n5e}(YYt(zNUv_ zE5&?4Hd?^mX1SDWQ9h5FU}vg-P%pNuK`QNe|8s zv3KY{oNj8!ry4T%A^{#c`&|F9eiv)=Zel$1@FNh7Q8mAgSQVF?(Qaw`I3eS4sxybj zw(P()ANELYEJd;-ekB@VOdvR#-R@^$0m%sPsV1B)MeOm`0)eG zv@2bru|7PbO8i&^2|JdP>1W@OzRC^f2%^Ilq|2pPOc)sEI}O;xE1O*3n{4k!t#s(r zO8}g%-B5M}lh82eW%RxzN98bF*=1U_Z0(HXi#xmeY89H6LC(<$^BHD}b(#AbpiIi$ zC2)HoU$dKv9bS*G7sPonrQjw8N)5%i%0#jfSPMI8!j^$P>Bwen)XQB^SZJYhT#~@U zFM(%BL#7fabzjX~{EhPWJGkiZHkQkE>H0z%sKe1I#52<#PFMrB!BoQsW5e7m&U7wd0N{E^%PNN^8uqDqm zor>(Wh&#Grr z$z331RbNa%5*4m*WIL%JB?u{N3l=}B_5uWRQM+<1R4?Q-)|W( zFNLR%nHm25`QP6Hr1~=1&d~pH*tZc5XIved1VJAT=Vjn$;(x!%WMBMOdH?+W|Lchm zD>?vTH;Q`pv2sjsZ0LNf*X6;yAASDfv^J7@_+o_I7BDoFU%Rn_eQoX_)`1 z+4FMNbTKDZZWl|(Nr8QcV=b<1;4{>#1todjjn?~9>AF8(5E^GqzWqjG9)#qsS^kvg zf0#fb$xUZ5#&UQGbBY&h$>(501&Rh1U_pY9#6#^w1p#_+t) zN8ed!MOayvNW9e}0`Bdn+W?@h2hXy;FfXNu<71c(|IIqa)c`}A@;Xx=S#14E5K(K# zB7Q&Vh?Es@cfjsCZ(YZ^G~Mcb*I3bmSFGIb^KhYdb<#^Tjl?}Fg3+)N_-@GixE+8E zFmOKfdV088Oa~q{aKs3x=aY;gcld6%NyQt&%uf3#U5R|I4aUdtJhr8hV%Ld63T{n$#dhAZ}qaFZR~a09&h*ZCJL1~ITo7mtQ&Xo z@b7ib>Q4J<)WVj?uPD>Km@qzmIMwyOm~4u(X%cC@n9>qn%)v13IKJ7!R8=RAMb@cG z8XK)7Pt<1Wn)Y-{%hB4-;bn3S)Y|uTfND%>7n_fKvLy~> z|Hxi>o9@ig+@ZGQQ2@5?uTO&Mm=Wdv1|>m7H8?xed?mCV2xlf<7{nq<Z3;$#e z^Gtj+gU%bD9`D2pO_FI9x#Nd;uNE-I3|i&RsLv2>7+H(u>{BKWby!iHvCEoK@pa$a zYsz{aHkcw$Ed68&bs|%v)Tl9OI%@T_O$#~6_!CU2Gw_87yT&YyD@=!Jr{)4HAkJm#@tccnG<52~8xx$zD-L@qN1 z7o;^*U21yX@iYk5OEP)UA@SV?@|h+Hb(k|9FLrQ>)NsC4iX?A>V=I(JVsG=CZJEU8 zdaOLMfrdcq!{t0N5H7J&nd&k}eQoJpf7rNJM*My0XSfB9KT;IWsl+9CMV?0{dAT<` ztOVli(%5U7gk|q+>NCX(>viqBF*El~)4g4K)Ztl~OIeg~X#1VFGlM+JRn&A?;|$6g zHv6f1!QM;?Ss!7(5e>(`cpIWmzf(Vy0i!SmcM@DDD^yQVOL8qnWRoMiw|m)Aw3!5+A9hD zlI}8Rt`cKac6Ly=oHo!j+E>?*Yc6NOebnq&CYQrC%yS0K4)fJQL_Cyx7@qR`*G1=P z9Zq&G>+H9HE7usq0L!KS-; z@;$OrnF(VApG+IHlr zePcJXr?cEkaGj;j$}Y7-%_jn}~(e~$Hs6_7LeK;TYvC;udl8rwk#i2^r=-Y*oxt?Hu&Ae^^G~8Cva6l=(E7%xB2NljSTO?Gth6&zB7ei`#Ec3<8Z~VTj2G- zwW*m@-HmG+n=d^ z=rVt-y(dP7-@0IkznbQG$Z1Q?eYN1&gsG-I8nO-%>BiH}30~2#y}j7mvfcXbC}&HY zf4CFR)juUaq@l_ac@3yfq4DR56M0XB5QchgCYyeTC*`#PtfymXaCJIetjo4HrWwe-4Q~c=m~aya}8a6A6DYgba+!3U2PJ~ z#bfxu`R_zMpV>A_63G*z5X~TV#ig%y?Xsp7))B|-CQ z=0%$RZ4r997o}ofoJK`Chgu~VLh5SH`{M$Lr8rbB;2PHv-u^F|7~)il}x8hkxxcyy6Rb50piPG(mvio7?F>g^iZR1I0U7 zI*f$dh_yMBzc>cJqfV>YN92ly6PzRo07({FMeF4;l(ICfk4L~0sf4*u9o<1&3Sp6K zTq!LpX{p7UnKsPvG}CT8*S6$)oXkIwX!5)ks+tyIP_~LuZt2wv9=Z1gKUstdTV9%X zJ31y;uLtLaR(_SEd(5C(1ea>rCiD663Q;vgE-vv7_bq`8Mnm*I*`ajo+lpR7;H(!h zhe5ud#So_arqD-lp>GE=cbrpbaDgW;S0pc3&jjs=#az>mSoIr`Im7+OB3uQpfhyz52%S|ozZ*T$+}w9?dTl6>LVRQZ-39udhsGb4zd zthp>LuFKwPEj*+-=Jv)ind#dkai3Eq)v&L_OZ(49ube81Wk_up9di$X?Uxn}hWlfE ztp&}Q4zJm%v5YIMo$x6vSIr2{@t7kjV?VAia1+1IeVnHz)MJ$`JE=c*-JWwLnJxX5 zUac?pF1IXCg(VUzK2t)XOFbx;!m+(txiYzqAS zM&D|z+m1ZhZ{w^W2prQGz!HiWB;^0)o5^J1`<)VV86zv%98A@rBtd%P$GGV1k2||D z%rD|cfRXe19vRdKw7c%H95+aWGu4D9bz~5IC+F7pwei*0G}1}Ey*pi8G&|8Why|#K z*7!kpVpPACa8+bq=_GP^M?!s+BJcT*S*RRljvsT7%|)dNOF>xD4sd!1Ehb$Ld_`Vu z_Ln-be{=HXC4gxBOE_69QhtP|p+|;vW$pE=$CLH1Oc9J2U3aJqAA-X~b7E>L&IVIg z6%InXcx=+$zD5lTk|QF#J>^+J;RW^^dKnKo3K$1K%;$Sx-`twC<` z4=|Y{`0tN}yT7X(;eT8LjDHz~fs&33pGk(V&#B5Gd6N-(LiykAOBUx;QPuHIq z;NG}xx@0QTtAl+gYOhpRme<`>W{rAv!a|f5=Jv8@Ps~p2V;8WhcQ#?k8^}s0&m|s< zjDw}2ns2|hWiml6$Ix2NbE+dXOS?MZ2KVsOJU^8!vBXOB+Ehbh>zp{(al=Xc_IxKc z44eO#$UFWJH`4>(}aUx6_CmV1Eq0Kh2B{tt7-|(t8RRo+W556J@&&H3Sg-Rlu zMi6PaFev`ko>Ky37)9L@py(DH{p7@W@x~5f6JT5Los_9*8U?pa`=>ndN>()}*j#6g z6Wy=Wml0jb$G#Z^;x_XYA)@LIizZ32U(Og=eodLyybUw&s^>rD%eht9=1L&9T&t9L zAV6-ukXG!Vy+0s!-F}lmtl_o!L8E_w%up(X=@qG(zm2<42wsYm`Oy{?&c{ge3G^T@@$I8XP4XuQZ(Z6FHBFRtj*4o)n zz+4O750zZkDlUYv;A0GC9tL#`xJiGQC_IcG<^;OmGKbgqied9)+wqY}sH!wrtzxs% zVIZpit20cO+_#YAb~;Gi-Y)2ud|_gH`%bq{iWfTtLeg^Czu;vdMOMc7q{^Fv@v-P! zn?}!aO>=wV%F=%}N~zQA14U9vCE2aY(hH>K0YA?3Z6Pl3U$@ZBi)dPY)Mq-^nvB;CgJ$yysGcgnv#g)L8v z0vfBzFnkjSR(~vM!tDt!`FgGSqLq1vFBu`#DiHo{S!DI*R{mC~kX06^=7SCBGWLKXI&Mj`(0dAp~Wi zfoI=^<|sIetcLOQUA+AjQYsU2NdhNf@Ij|XSMR2g6z;gP;YXJNX1S`-yClJhdi$WT zI+IEMc)p@0-KJu1mZu@@_OwJ;P19884>1B+zXX~=kVECVBEs)&`j*gnnj`!tR3Vv68V+#1HnnOL_EYG*vW2_1W`YDO z5OSga(nx|}hn>;r%kB&GJ(3#wUaNOmj2|um+x1pv^THw&t??a3rxbQ^ z;tC0=5bGT`SGk913I}Brg#08oJ;OlVha~Y(I2F!F`_N>IO?9j|>Xn4ed;@(e1DUnd z3X@66vjP|ELOG)v8KbN7*Q?*NE91)SL{FNnCWLfeor{XxG%4fluyL7|b^H4W$?WWs zU0P1@A{rI}G`%&#`6J3nc!(de0kH(|} zKKQM|JYz(G;^H-}$}QfOQl5Y0ryAF9RSE`DYdZY>12+|nXn7K+5;&Yz3+c-#L%)9) zC}{@*7FUZyEP0JO`0oj%yVR*wvYmbkz;iB^39v9sSw7lX%EH2Df9KwY^;2f*+sz(S zC7!Q7$#@fsE-ksS2bYrCR!PoBM)g}9De@P0>lWwaQ72vQp;>x0Vgz3mD|CfyN9Gpe zN{~HLS1pf+wjt)OvnDG(4I5QFM?OYO;L#u}!0INDUrVd;E>X0oVP)0tR+4j3v2=<* z33XM{_fl9kOChuudqRvWmfqOJGMSEtf}VnREEEMvbtk*%){K+Wj)VTyVZp_=f6>Z0 zUJ!YdCKxTXIFXs<-tPz+a`fUmDo^O_b-Ivxq&mC|dV2A>v6Cl9ktUqITS(VWS-Tw_ z5N4kvogx-xN1-;68RVR-RfM0Od`1WCImhnY9#z}Ka!&DrEVIoSdn?}isTW|Pd8C{O zuH#WGx+}Z7!KrAZm}P}b%mUCPlcM5ZYe3*C5ORW4i;1({)g(o`8rng6LW@WGCuoE} z&=96p!#Q+7>q1xcqjShB9dh4=s7^rJNGi(VeX~;>s)%l@wpV1!#~2mcA$e8xr7-5M zbsxeRe#_m#2248^Yl?w2S^bv-t z-Xcxg*a1$Vo$7L&?fuWMRz%TpoBi!*&=IHyCEg@_Uqr|M)D!MXE(c520z$3$D^QrJ z-Bbu~j>PYA7!G$_r3wKLZF#Rp0_VuP&ovcU>UaIPzV`3B#4U5ZQy)va?DgzUAB?T~VPNIqdhgK=hJj+*+f0gC;EN zJ$4+v;~H=-vr!lb2hepClyGgl(O%2kB=&_^pW+sW zCWcA9BxB$*RZ=iZ4F`-~R5K6kvMDP+7`Qm0N)F4e<*tUIP&4P-pN#i$^G3;bZpSWf2kTn26?cGN(Im=u=jg6PwF#Yae0}Xe5e7!=L{W>zf-x5ND;)7L z;nm4>nvKwOjbq?Aa&M*pA%g?)>igbovXjwfcsKkqi;2~UqlKi_9mFjMA*mWaODWzc z!6Lqnur;+^b=>L`K7|}w?65da+zAG2f_Br#qa<_x_Harsf82WWrf692=m42e+~IIc zh%YK4CW0@A_S547^X%p2<2@rk#HY*PJdICe)~wegmmE;6_^LL_f%gV$@#jPiVsipo z$*MhTsH)riY_;;WU>hAtG)16?`E30uy?sxG#LEZ6;PUI7Bi6N6^l&}=bkAG9t%%Fm z@{o4RAL==Jd0$ZZTweC=kYk#EMdtZZKzYrE=^TWp3%|)pKof^%tD0JEb5X{c+>=x+ z3pHl?LJ|ciP~k6wq@Fl>G4y9;dNm6mR#xz%H(r0%P~Cu_dA%k#xm8!elwJ4V7_l!1 zW?U!#lo5OueiWwanfY@RDSc)fW|C&-hv7&_=pQc@UOE1j=}IxlAT5}$EME76E7Oea z1wBb-4fzW5Ui4v zjaL6$#VNqTV$p;$V8FyuD&oKpNhFK)lQ!xb{0XL1X%%>TWn{VLV2#*nzZ`tDQziO} z%fJnGtn+BCUSet&4WsI^g5waX(lK+5{2{nDmzEawE8bggOaw8ckkm`kv237gEHq1$ zF&26P+yknNsZTuh@$1T15JFnHe7c+?u^(FnPSuK=@$%+sm}h#-;|YRr->rFcU7a8! zoCV$bCNlZgtae@t?Fea&OO`_v3%~J=W5xfaZE6m5*rSm(P> zHjZuWDVUB6apbs@@=#@2e4&a^+k1lnk13f6SawT4teX*}BkQ?KKH#}ey_RF%T}Mzp zwWaEi6rBG84Q-qAxfNU0<7FU*SbrbFHb*%_OUk zS4{Q)MFYrSf6&9hE)VO4gjuJr@DXnt(a^7p3oeDD*R_=TKS zWE=mVni%-MYU559wVuh-2T(mgz@MuiPB0k(Vo$f*kJoD`SCeXb0b6KdIPDRPJvb|~ zHkv^Kb8IQ&cAZFDpj?608-I2BsP%3L6hLsSbMOD?Yy?qPZk}HDw)@ktcPxO}$Nlvl zd`B5}keJ?D*yX*y*-AeIWete!!SY-qM{Q4sgdW?(?05TBO(2;->A$iGGP-uUIw|%PB>T$Wfykrx+tDmn> z>P4}2ChGL)7UoMe^#I_Z=7`e1A&}6+Y}vM<_T5P&@6CFwO|&SMeR#v~4AV6C9>!Ks zz;XUtVrUGUtG>j(0#hbVAX96scy#;?WaleEc?5CXY0b3mfF0=|sEJ5Ob?B#VAGXm9 z`&nAIg!f~bEybd!uoC3n>T_`ipud5t3s{vij_nM;%9CDzi2oe7pJGtM;9AFZkri8x zqJ2MJO_v%`Yry{JXQ;bR?_+3qH#yk(y|y85H4VZJ5rgPQ4~NZw0z zfPc|xeB;&?PJ1=5m1z2ICqJDYt$6Eobbw*)xE!~8Gf33KJ`D{s5A!@eg$XmY5<09&n}g{uIpM*mhcH|ZlC$LI=u@>Y%>{3#^{0k7(L8gEjxMNygU_cdz_wC zJ=?f&^)9rVbae~pFmGWnDKw(@*F)b_IZSDoZY=C)Ucfb9EqON7x`QeitrTjR=@@DH z<=AbpXecY=QLOVfq7d|uyUthP5|+6lKH}?SlKU45~kj7 zH2F@+2oK|Yi4by!&@U#OJZQ$<+FxT|a}k2nJQM{)+aLh5xrr%4v_1&V+O5-2dfw(Y z>-0JK+dli+tu#-VEAVEk=B0ORDcg;lH5T)!=XW&L6OSovYY z79nhFEI?~9QI$J$guvfU3`d(veLoO>l!QKSSjG(#lWmx?`RDbKc?HMZ|Mr;fo=DqrE3X?o& z^4-2iTfJY5b6bgIr~l^U_?euZRXx82!i&D$ffSpk>-}g;PI?vwXFpXoNAw?#Xw+Me z6%<}D#4TB`hETdl(^iNCv{HF^Jxjob{hU0wM{VCH6~a>_{j_e_NYD+kPtY*2&rkCx zIzkbx&p+}GS4!ve7MdU9tLCnrP?UDp8Ryvl+G@Hm3r0q7{`W@&t_!0K0zf@>U(*p_ z+j^5n^YD%$?gT|l3tMV||N4*LVQ!-CsdER$^ZtquD zxOVPp^Ss$e^o}Xgg8HTGCL0q{_9K@uzP)J>Fd(#Zp z3a&ju8b91x#M6woy04IKgNp=?AOBFI&;00)KxXK=qmW4lq8vMs*!_KI5mSknG4&uM z-}&^sPP%6sw$k3EU8IK8jqGGMW6vEmWOZJHl(_$C452+PJOAx=wv0=tP)Zz>2PkD? zcJh+k3#BPLHmJv@&cBK}A<>?D3b?sg*&ha%cW^=0+GY7LE>VL^l}zS@YLtOmm~Q$$ z&ivEMpF}{<%a}ot4F&*tqvT_UfkmEQQB z%9C}6ThiqzW&J$9mdWLv6X9@B7y?Kbdn^{I$A0Gi+uwLFut|6$iV9v8bu{U6iduiMX)c;G?~wIB6AP&57393 zW&rW#7{xy6hzNv9Vz))~tEl0TU2-v1_{8pZ{UTfS19nC!d48zCbcjbztRe2f^q}8m6If& z235;(xULV(Ck+tbS?&C6G9`9|5S?HG4D331JXrnzpe_DD%D=Ub!(D|VrYi>2skx)= zxEXIxe4cKn+nOz2a+%{%OqtUP}sY^fx~#LhZm0s4t*`hy_C_ z+8Jq`FS;%x9fWX+x{gVZM#KRFx>&fG4+@2uzz-E9Mm{FOtQaVGj4k?>bH!%?MiZD} z_1IbuWcA;fKP7ZmEWdmj1|HWIRWO4dX%iL+ZI9J$TP~V`+x!)KY_YLoO-SCDDryli zsG^YPsQra&No=715N~Z8d|>KCL~Lzj03k4P=F#mH*T4CBX$pq}yp$EsI7k+=Ky-8k z)2YDz>^Qplkrmuj5g1uc&p26F{lx)KhHB;m0QH&h}RcD<~bH9uk&-L+g&ec z)8R%HutlK=f``sSC7_|89x^g|`MFx>@oxFaMI;6k)-EpxDTpz-vACUq`UdW$nN*bleH3RtZYR9^ zVQf-c)!s7m9Z4;^GWZ*eEGd2hTJJi#R78li zYKgvES@sM)60SkRc#;W96o38DMKcl#hs3e{LhLOk_wOgX;KwM$&`-#PiQh}No>e{! z#L)*@K>2-S7b67@PZET}Vbu8)gD{*-acKx$Cf;hmJsW%Mv%P;vshEUKoMA}y_Mp4z zeKZrzPFL~p??$E*<`pHM(Ve1{5~d}USLaM9WEv6}(Z@!mz@&;=Lj>F0FXEE`!$7`L zE_5!=c}9+Or!mPi>M!lV^QrcvtR8Qo8&?ATfWdeOMF_v$i_YFJB;>c8&L1VY!%*Z> z)%AT}3w;b%5P4$bVZVcEjyLq0ES+OTE)LhfA97~L&?{Zow1K-HN8Lt?5wm_vTYxK| z^o4P>RJc}ktTc+ni*(q_S%`Yc!N3yEk}sR*rGrVD=|Y8-@s}w&6*5zZFg4=|DYZe2 z2oQr%;r#njOGf5YhQuNo|8A$nX8L4Mfm{l5C#n}bDjBloQEhcHZFsJG-tM57sQzX` z21c$9SrTI>1wBTNRz@Y%uSv?j8X2#aUALWd$vh*H7CjnI$m0T~kLW?Z7Y2)dUrIkD zibbJLFvMXi)ip3lqH+8Yn2#v0A4IuMy2WtdQZZ~kN| z9_XqNqLZRd5(M3p+|_}yD#2p>$l`sPyNpHGc^aPFLeIsR zPpG&wlDEHi0vE|Uy8Qaaink$j*t~fG5XgH8>QzX;n_aknOydrwE3b)AIH?x^N$LSr ze80<2Ii+KaNFU9+64Yh-1==S*qlkAwwmV-G$`P(aBX^rxj}cZgd&tOyN;pMBI$i{? zL@%?2<|!3<~*7%}!kGaAsO zWT>4=o*2pgG7{^@$p_}6_sjs0Q~V(5yl;msOq`y!t5rx(e<+oLB3?{I?3>Hr$d-nK z>9*xYlg@ltz^qj(1xaj)9~oYwA|pQzx)Ih>y*N}{*a6OQ$eUBNdYef{%!d{ik$$Z$`ZU6Dx#VdJcp)7@d8cqOT~-`_0ltbUnV#SO+yJ+z zXX&{jlnL40>hFm;1N0*`;`%Zd9&ust`e?ZP&)rYIR0CO0@1zkupUV-Cmu@+2 zq)T`!n5JC*NK@tXq6|o0G=cK|S8*f?Z&oUxyV2RmOC(tV*;Azf!#Q=js1VX3q=r^e z{5f?BNX5qmnd<<1UwcSuSm_V8adc?I>Zfkb+7!y z3XNiKC({0AjiKh&Y$sgs@rGS zL+LA8bEy_VgMhw##JME7SpNlMYD3EWDZa|ldzJ

NFC4BS-~Dm+{l8o{W|N#aav zyY12rrpqgxtv+PXar}NU;7a9f%0WTLI~0lm6OIT42xmCMA&QQjR$g%k^?wN{Mx<@s zmuOTcuiEhULDY(uAek8pBUjZaq$)N$6)X>BP{f%Of6*d9)}xMBtinC44rjC&dPrjJ zKY>P#iY}D=S*XKF{vIOL{v&g;%Z0!j0X{^=1XIgOg8SoKJ}rhUOLW~(uiN{@&^;;6 z8pLf6V7I-;-eYb$`se$_92wF1Bm3`ZUQwSyU6FTU4uwY8wm-A^eJwF~>&Hif)pTGJ zAa_R!AGM6R#O%~mp)a;83#_+27X2u!A3gs6wk)Xr1%|~5JR<=#9 z^kpz|#GNE2>k3Ec_;l-9!n{$%@f4 z-fr4H@6AJ=)7M%`Wz`S=ItvQ*V~g-1s9?Fw!04-r27$kGU1*uH8PEAC3zKI!O<)}D z=L}*t-2_S8j7ub4>*V4;;prbF)U_smvga?%3-C>NuRlLd=R1l1A2{YeNa&G;d~w9z zz#^R>hB`M$-BU06+nwOf{{bv@4$Ny9|3-AlpRrqXbvlLr)g1r4c{~H5mt~jLbAMZ=z|1q;+zP%V zq2Rwu#d+EC46u5td1_iekH3FAu~5z6!6Is*Ri!8XY9HUFfS*5%i^uD1mn#M7zW=)z zNEaFoD7s&K)}?fv)gU7I0&iaCCX4Czobkh%yn-R<;^5 zSt!E$ME|x$P4rL2nVfemX8ErFyi%h`_i$$X*XVdGgku?GX}#Fbw+Wk-M``Y-iR!yF?Nzw6VBx+|%~#)v%bzLwjrAYwT+8$Qn?E!X`` zc1gZEH8SB=(?b^^ao)z)aiR7seGOUT3QzpTut-H0>1yHOsL}h$IeFeb`pip{J@qli zgLPw0MSBT&^8ROA0Oky(KX{%IUv3ajTp$UfRovEF2R9!KjiBRU={L{W;Nw4_am&?G z6O~!Hz31IQ-6DO{{whV9n<4u%4$V5H-suev;@*baXI<{IZNm{n1`D*n)NcWlmByY0 zaW)=zAS^0M(``Ik1E=%liJrbAoKFk4xJa*k2kl)pA>;|i?&bBXp^=qcB1@n{u=$(E zZsDTC0NvJRs;jOm=yK^1d~>wydBj@>;)PA%>==&VKCFLy1ifQra0sM;_j9reVqi<& zV8(>`D2jY>tJ}hbgvauWz^%x zrJCE#Lj9J@O~7@*KFC)3;@OwC0OlXNhs|el%UHW*YPa2wcSlE{2Z9n|6rDyPJ?sJZ zIrfYQqkaW^ONqIhS}d6i;Hr2NT7ErUYCo>-`%S~%viRISCRz<}$_*nQda7LZ9swr& zPsf(EH&*$^9U>l2?fefZsdYy!vvU|kvAo{H^GF+7Z?8{D3Hx2tueH;cLbR3@(bl(R zZGW+SzwFu%67;L$f4XMowus&HzHtlOjZcIewcY~_@=v$1glnVI@u$o7k5!KiQ_J(i zfaU=Awe#ylqexB)43n?3)bj>G1VSkujv#hr`s@R`O1Ej;nP`|$$x)eowjr*)aa8^w zCy&Hp_vfe8GgQL{BH~u*-j{tnM1Ysxk^5#cB{%?A0dIEEs30kKQRk}?9&`*sGATsQ&$x-aojs8hy!GbJ2gJBB_7XRy_<1p`R;5(b`HGELf)TJRlI`M1c^#4`v>HSbAReN39^9)!Tls z^iK#PjCxVLGIl=ex%}y!Jm7G7XA8XTX;}Rg#Orr)8%qSde$pE9=x^fnTz+xnaz#FT zw`Br}Aft*AO4sdYX}z5ZT6!olx(%vqewq_Fa#?}8@~mUp2%y$=tz4^Y=YPZ-^R*AO zzu#%n`=aIHxK=3tiPF~9a##DJfu#n;urJbI`sIYrK4X@P$4ZJ6(qtC1f7<Nu-J>@Mm%7<_zxN*2y%DUhV+6IU8s2F}BMP>PSF3)&O1%Z*YZVP-6fOTGAv zlf)vFCa(G?cKRsO&+FS~K2VeT^x&EnD0Ktz$LG#Tw_+05Y9PYn8SK(eo$q$94tX2R zltRLaymbD0?hBC{-oNIH7OI(<=j|K@p(9DEqbIFrMbr8p*oCO&2a>hbXtAXLRhI0Np;(MRFTw!)#uUF8o zW!_cq;cbTl^Zs$?dfJ6=rNCJ!ODb1Gp9#)%1V_JJ*>XICXPw7}vFpx*8f;@S*7lkh zj(h#rp(na&h5SeM7OaKUZ{oH-i#jgX!iPb&uy>6Qr{lKm?*`NAALu9*B`C*c{tzy| zT|vLgzv@J~a-CJ=_cm#&VP+WwAOzRDD&qpChoj;|7yJfU6SH~&g7&t}CxLBGAi1^x z(kK{C&|tEq)kXj^_c8^#z2TlduV4^=<#xV0nI-W8_mm&*?MRP6O0K^mbtv9i$)~ZU zKcT2`ax8>&0rcGBp5mD2M1P0)KPJ;Dy01LA{D9aX^OM*5W`cMtFxdk7ObQ$oToH;mw_|c+$Gy&7t&KOOdgJ-2KMRQWb=xC*(7xP14TKQ?+M0%7NYq&M!MP= z!^3AnP@eZ9&edcdq&L!$wDRMkN}b3dhSC}XrH38_KG!Q3wdXYjNAZOi^H(5PSpUvr zyDf;?I+lb7s1fKmcjqa1wF@KqAl2EUw%9n;>a$yz&lPRg`1o+wglqbp5gG3=Y?lpO zXSfU|ym4;JpL0$yje70dCccKw>T#8e%$tX@d_A`4Fg0#7z&T_mzj)yi%=xE#c)>09 zk=J0A(I=g;`H6cN!0NDfe8OaCK+5CnBlyu1uM5Z@hwB`jKI-}0w@ec)SR67h3R>!I z@cA?e*IE1wOK-B?;+{`!+2U<*D&lRvaq89QZU1Dn`Q;-a&hk;))X9+bzL4`@)|Gt}X@xPRoI#J%WU(@2$DhfVCB0e?-fU*j zT~-ecrSZL$%peM_)_>>*{@L4ez<}}a>w!snE7J$!*li@F#fBFzP*R>BMqmO4x2tgV z_4OW{?ChfsNX8W`GkHudNANepQ0n_LFv4z`BpJY9GrPF`x@^5lw=8S8uVDtB>6s`t zki*@|lIPrrd&CXIHNls{CA0I>@6QAS=5*bnb=~4-dc49aRABTakb_()d-R}np_V7N z{XQ!lH+Gd;YqG-eP`5Vpfq9V8JCLRNX#J(CyFOF}h=;rL>SKwWPl^SLB8T6Cq?Yoi z9-d@x$3NSF5gRy`E*3)1Ws1X?Iv1y>rlN8`d!aXO_X+Gx<$MoI(ohwT1m+XXWQN-Y zTyKZl2(FFF4uPZA^8oJSIp>9T_L40zLTl*g$Ch=%h7)J@J{F^<#UZW9AOiPuw1v0K zt|c@1Dc(Zt=Mdqh19@G>M7f$E_VgWnylC1UmQ^Z(4ZrAOHQjKZ_x)(GQzrvaZDFYo z4^-$bb&lNqQE_J(d(+O$L+pi~r;&B|FswAze05sR-vD==1snht8y%>4v;KrWCcn?&n7avyFCac~HRe+KXThR@Dyf~f<;DpbJtyDH^OBE|MRejDzED4d(SCX~HvJQYRC;sY zDvSN=s0Kk#()L&XFv7gp11BPAAcWMr8j;w(#|}}rAkMrLD_yh~cswJr_3~3d3a_(z zfW0s7W_JD89t-^?5@UpNxb>U{cG_U-@t-^!-}wBin0nZ9_wEoMz26?4$KB`Y!U;~} zL#)x?43;DpOoHe(S`M*$&{8yUx^_0xW{6Y0j^Y`b=d;;c4mYE%R{*zfGoH#f@9w7) zAn2`0fq0F6ArB7>VTUset@+LT$h!rx+iShL9nCH|3f7i9*w^K-yA2(Ajt#Te=^aLw z5mxQOM6D;2dc<7xHFkZk_w$0lr#%dWq@~Tftwngl{~!x+;>H^=O2VbB5rrLLb}cTc z3`Oy)M(%jSMsD0XK>!3|z&L^A=t(%GRu{y?%6zRBOnL7zAaKtv(nN_$I=EQ)!cQUA z8(?bu3)C}sEVb+t#Yo|x)P5so^f*=<3zMv43ahn_UT8lKRS5iMRoS;{hP0?Qpq1=AoPxC>_z!296?0wP&~pU1LVQ_ifvLNx!|Y>XghU;7)9m+Oq>h zOi@FlIyFqAkn>*fN^LJm>)8!hpI%+&x|W=<4~b-ZEGxsFn$D&Knk2a@0;MWTC%5-1 z`NL<Lnv}YDw4y$a4I9=c*DbK-ldEIadoJ;j#C=vzw+pEjq z(*zXQC&5P@^hqNQ$( zn(faLOJ`T>yo~xF5^e;{Y|=?%$jxR3B{2nTGsO&38HNAZjaPHVZhI^-a+qp{^`14j zW}FRQD^yxx^*oG%Fd(g zC5kk<6PgjS@9X11g&|@HnZ|^WvW%@s$TpTIWgCOx+)h{Lbgpxq7w5Xp+xxm-e1HEJ z_x-=VpYQKGxA~Qj^k@9PoyT{_PBx6@>i2LfJMXD)nS1Q|QU(M^u$1 z%4FXm+*Z2k6w9RLY4r@PrLy#Wjk@5mfSq?P4&f?3PMN8r+G0_&^44scuRl!ogc!KN z(u3%cX)l^xr+3X!{vP)vO+KoS;l z!{C(v5fdzF5>pOzlv{t`blUrP_2_GGx~kTV`->l6CAUK%Dr;>6p2z+~l1ykXB_Z&E zzy#}=+Ff+Rx3Zw%n=I6qAF%aX=a*fUTRwJ$HZShITzLbm=W6HZgbCoe%qglu@4gH_ zar~q=oV+49Ms*XUup`3)gaB7lDXg407Acll{~4Hxrnh}r)XT0}n(eJK87XC^`ZvcO zb8VYZzM3glZti)3`$+r(yW;B57HByW4`V!h~IQTu#yQhE)pj zD-%&bDmFx9LT}E|*zC*PI4@p0-ZNnsuiwqO}61Fnet+?Pp~uB3SxdFoClMnPshNrdXvxF%ei1{ zN9547?>G7yfHflvu6J{h?A4EIyt9wf)MaFT9XFn{|d$ux7ex}@hM4T)QmTA=6;>bDQoc-604~2JYZHqXtyxrB|s_`>VSNKq?2*Y&?gobH8_z{=@LuHC$*> zNdDXF5A`k3SYJx{fvA>V9VYet8vd5Du4XYUBOJf_42=@l)EJQyL8~_JU+}tRwoZQ;m<{G(FUKHJ-Drl&RX{`KZf+4-Ej9XQq zSk(y#oF}*(#j`%;FjrVNb)UnbPvk45c-Y4TpNeH-uR!FE*snjny*w5GZZa)6(udEQ z4pNZnv^wiP4QV@zLa3Et6le^6OgvVc?UncMO3o7*gi7fH^OCg^ zJ;KQKl;FNS^eJFu*KI<35TZP_$*erxjXSLGQxw9@dwx`L5*LnQtO_WbZaM84O}wSz z&)aCWQjxU~+xhC`8QIu(5~C!GOMEn0od=A~CmfCE@LT7v#@#4v?<+ibc8z_H*rdPLjLLk#_yXp%(Yrd*z2Tnr z#L;btn^EM@CztMs?|2;kMGv4NNqZ!bVCsB|+Nr4RJ#6F)U?Blsil^Y^i3G}Y6M2aL zijv-@8}1-OHbQuJYUlFkX~oN??I?nyWtq`#v4GoLdzN^jiA>~8g1gLx!;zCo?qQjK zd5F46`5vsubg+SuG46iC`#ISbp94!-R7lL#+yNrLNd%uHcX9SFz&-gyUl~*EPQ&Xw z^*I8+lW}Ewyp&QpmSiQ6l&G4bzl5^E5WrvFvWxw%fvReIb zewQ0D6z~3~vc|BAAO@p-9 zIc8u|t&#Il=!JT2p+Ce%%oR#zznW0Zpyy6P%mvcSb@>oPiJoZt_w}jgl_3U(_M-lxN+EQ>=ji&978aYAJ`?Owv| zwj&FB7y30jp6bk-HZsIZ(jk6+)yd8px&RamK^&QZHt`qyD(cpxx{+PM(S0MLYYTX| zTD5Xoaoa}Y6u16i=3fP<-dg4tTLn^t&L?^dfwX#u!~>y4u3HH7v%89X9Y6g3okek* zJU?N(0kQs%zAfp7w+?l$oU4Nush=k&iQBM8OFq)-0O%$-epfIbCk(iYoY{NY?(dh^ z`hzWau!f1*M7cNTbAE;@CSY}+PvP85+w;8GyfxL0VQ+7N$ulbYVd%J~b7PnC_A5k# zgwW9d9;^)t3_-GffzD;s9zSE0{TMD^9rIcJW`*kIpPz%TcfkQ%(6AHXXZYYNdc;hM zanI8Jjilg?s0+^8n_2Cjj{R|?lBgqY6ZN6P!t47fey2HP2@w5|HIjJW{y3B)7_fZ= zkGB1wXXF1hDVRSAN&tiHL1Z{m`DfVifdynVZzJ@R+l$FFYkIz7VueC4EZ(w>u1lNAxnr0%RhVe!VCO8 z4gMAQ(~`Q@xtxp?5!ovp<{2#b#`C9-qGS=VVSnB#7*u>gBz*polbw~HDaslND=HO9 z{RIb4)D4m}fJNy1{n<0YXA;666rG;$r@Txi6uIc4BZkMv@Pb2TMDW)rYtusEzFIvv z__notaNytI^XIqL3(sesq@IGoK2EHUm&NNG6IB9i0u2>O3j?lcLspJ)Q>6uk+VAsC zmC)=&{l1gw!sEX9`(=#iMR9Hev4%hLbD>r=W&jrHuUA<97oG64!p#5kC-|-|uShHI z@9nRDis%086y#s*!GHYsfu7rMo*TgCj(OHt5&X}WJRKjLKJc$I|NR9y3b4WG$+B>Z z^_Ks!#t-<;;>8K~_aT41#LJ?8(I{AIxjP9W;Ih)przjEfN3lwE-k?dyVT>4;x@Q^S zUJWkMsIjuOsc?HZ9eTeYM&iB(x%vacwd8hMS2d+QH^9PG;Z6Ux7$E&T5nE@ z>p`7X!?*qUKfk@%#(ohyZjWZ08I@N9kN1~WJLCBY=cc6h$(GeK^I`8FwpvMp>KzXc z)N4u@T!$S`Ngl8KHEcVuDt=Mv*$?vM?vCe20mLl&J_I~=h8VeA75%J1rPOq?V>T(y z8wmo+^A!)}D4npAMFT7m7bid;)1`s!&310X`-iKx5PS|=S>=m_YCm(2@;bGWbhNl~ zgRYxv6~<45B$p0XN6Sjv3YwqtO**45O{TOQ8DffPZ96)Xp$UD?iwpX5)5!!yeDZ@FFMT#idW8530t!mwL9(m`&gs8C$-b|(hF1pB@Ra=jh z0P0_nRC)uZZiWW5-;v$j)*{3rA%+X{wp~Ot^si6|<70|GwE57yJKGu|pFkl>dsXbj zOXC(MaM9ha4C&no0R(G}e|fN6o(z9~4#PT^wK^Dc0(rPv^>Vwy`n9MwTIQixK|iZq z#Ar!y42D<5VUF7uqYLlLPY^fr59rNFg(dZ7#*1aCCaKO;G^;nerJokfW!law$7Ja= zo!4V1`+f?59~sNXx&@nD*oSaTijVqmsc1RGFai15u_wO)IHIqgdv2f?zL%986xT;Z z3OqyaAKm1UTm@Cd7H?}yCw3G@%UHQeTOR7uRyWfCkA&!nfadc_ji8mvWj}(Ag0;Ph zAx8C1kK28fFTuRinejup$##;SO__{qy+)BtDIX`-4CUUIWNcrphJAXzfPwoVf;;<$ zBJw-5yo)#q@OcV8O1+mb?scdO z;zvER!Iey!Tyqu`6b!1VV8H2p&0Ktce>Rq1J)CNWkwi&?KHyt0v#8hRHvz3~4kncj zX^wO8{oO59nUBkM1{yI%yAeoNBEB=H=TGz+&;jlpB6AXr0@ku@+R?=*988yLiQUge zBvM>pDhf=mT8+HZ_?>@==9U~cs9CHwuE%P~U!Q%Ti1CF%;clJkGQQeKvNltEzK1YY zz2#7|@J(v8o$GSZ)jIOQkHG5PXAkV(>~h4x&nhRBH7KuL&UchZ52@4K!UVQoarqQK4o2LwZk^{`O>OM_55$z-)4*?@Z--a26+pQF6OLp zgw-{!z|`_u$L&}IgCusM#(GInqhKOeF++ryv3~3A5|xd)GzNmS(_X)h=zm*^%5WP`Ghovp@*` z6Cp;N0L9c*RyWP22o|ZqJ$}23YtwW#Dup;CaH&ZfERrezTF0e#pbwtY|9GUsJ~!nX zEh3`47uu4OExm>y7A7ek${f_a>c>FB$#h;442Q65n*73ghu}i^HbN$#iuU$b6MF3)k^6GvKq|aM)ax)yh1o zhT(BB5gt#cZiT(7Aw7!Nh8O7pB(t?t#J{-7Ip5+iw=5D(gMPzm0Jbvx-CQ3|(;nYQ zjixUjwmf<;@VBlH3!yToS!3IOg5Vy<<7c8kVDm}x-O5o*Yqf5Y%}qw)gs73HikK7` zDif!!M63{kbsN>bPq|+%Y_@2tsjlMiwV!t2=t#?`E4~qb=6ZRs-xkIP9Mm#9AGlJy zUcTBef=#-)@+ruLBXKLECwA*e`8q10%?eYkkRwf#l$(K37T{+9b>|S@OTsYD)}9QAQR;qj^zag1!1VM6IXRsJJHh-n_Ia_RZ;{>)~~s=j+1go4Q1!-_MLBravP4 zO|Ltj6Zg|^OUacxvPlVj03P4p?p3IFn5s{<=#TRC$E{qxGlo{`j*f~!YJ)~NrZiXk zyz3O+q7JDUJcrocGn*b87Y7q#4ASctt=j#Xbd5J24M27-){hsm#p5>#QR28!EPO8a zERudFzdiUZ?a#$9ed}G83}hD0{!M+KwNHEB&DVQwNKJ!zzia}Jw}8%4>hNBM>TTPR zCZE)#zJ9PSyupJjcA?WR(*q~H;iDt`z_a<*%K&!EMsLUfzs9>zkfN9bAK|dOk^&+GuT= z{{5CR0`{m#Z{u!B1Jv+=Sd_XX$prU0{!JK|8r0i9+>? zMXB(UZQU}PYlh@HD*sU{bAjApjD$kN5L%iIfK1RKM!p@t3Fx^TNU?Y!tHJG$eH%fM z9zKzmDAl8($8o=2kCWB@si@AM@omxB#2+rl4Uex}&IXab>^o*aDkhc?Z87A}#PwEY z*i9uOLuELoYq2yFn+%C|e(t9<$RZ3**c1SUucOuz5b^Yvu34h)#F;+BUSy%SKI@0H(C4WSDTcD!aVpfoG+m~-O14%Jt`H)odgXcI9lUND)J@O za#^ndSqr!M+%%7!Ch6ZY<5G(iDUQx54wEBUkW|MM9)?I_xY)PHN2dw`j&!Eh@;03z zZlraO_lJON7TIEwoi{S4&75_MyJ+w12j?Zr>u#I=w4&>v&Gg7=*E(<&PP!Tdu64CU z;p1pCH6DGWCi%07sM4mECS1REw6T_YsT~hI4`FmEHdZUzUa?sxcKYL~=nk&OFieip zhW+ZfajH zek#>~gXI>l>ZJ&DM>XsV`K?&>7K;iH$JckI3mIDZRnhDuPQq8v zUZA(f^id4acAaVOS07_D(GM4vaNjZ^LD*u;;X-`1C9v+uFU_V;N~TXo%P;IsJt;p# z?C6-5FX;8EHf>T<*-H;ZkNjT5kMBn-iJ(+Ii;5m51Al>%7+czLWnP;M2$yAa!6`}a z<$Z8Xa6PUidl(cF7r!>kA>8BXMHgKQjL<&c4YduP`%Tzm#^y|Ci_uyOkz&Z;NfD%8{9wvkvLmNuS{KcPq~(RGbKMwa_W9Ec?lPU8VCiCZ zaG1qafZ;R#2@g7J*Q)sqUB^lS&O>Wy9GT+=c=aU%Y2Nt z40Mfe5ej|4jq#hwaYMRg+a6_@@!O#Wywsxf^zO3Una`{9 zhczvF5i_PC50y}-v`WS%d6O;i&m~AJ21$0k@`NXA)FF~hl|(KKK-H#Uj8-v3?%TV3 zlqJj++|r`UfOFX?yT#`z{!m?KYTOfL$v|Xdm3aGioGu-&-cwAzS@9V;`JE_RT5sax z3~qzPOzghhd{d)*L9uVN`dGXuPz5^PpuIwHrMpg)@RV{AcVagsE|GrL*uott2&cYk zAj-^6uRIrM6S3^81Teo-=o>NbR($t8V^eK<@QT=v&QLMtw@3#;!P2&vKW0>0&}-7pC(@wyHC(1$q-{~ ztFtbu(MA`s>Wk94QqzVAK}t$ z$^#R|z^;LEW(7IIxi_bPgpQ&}s_uqN{5B@SU~oq)g4N5%7qQVVb9T%F+)GWZN9%%< za1lGhF9}wiia(W-k*jVu$V{w_{5r$8vUz)-hZPoQLGyY^jUmcBzv`!|^8~$AR$bN2 z#|B;Q)6a{J3cy5}-cnEDViR+aGmRAUbC^8Q3$=@HwlnpRWZObqS?-2yuA6FpUwby+x$eQ_=9z~j5HR$%XJhj-jZTUO4Psitn_U zwVlPengRN`pZeDkv1S6AZE?J@pGiYLqgo=(w7FICp@RLHbH1D2(#o~WHJR_ZZ;+Ck zpZXSsH#=j0+{7R=1-7J;u-5*rf*&e>+sS#8?sJqjhs?gQMYO1+)N^WjwmrlP z`SP!fhOn!b^g|6Y3$_Vl!R4S4GA@t)o;;ZMpElg2J3R;T)QY~ z)^(oNsxGjsqNTp<(nRiCgJc~UP~X*uB@~C7cDJc2sULu4!{@@3#-vJ;kCItPa&E8) zNa^xFd<(QrDpgLqrfiU(@h$d*2l51spD*O>m7$L3-%q;msQXrhbBNvM*>(UWYKcg_ zoqd(^1;%P_{aqwim2~FhNONl24f#2cXrhPX2sxTW36Yr!J7ib=_R=Fu1^I*7vl1gIgS+CqV)kww zmg*rXC~j7y{$5ao0D!wlrx#t&KGdu`Oc+T&Da-lN`YWMdT>WNl^vB4M_ijb&@!y}j z8-%;%=M+Tv@;Zm6P};nu5epH3*t#lfJFioG0pW+!K{yG)00Kg9W&qwlkcT>^TiaG~ zt~%ycE5LdN`B0o`I~mAF|l+6GR3wL zzGDMm$tBxd(ggDZzqRY|ISTE~px7!d5X+p6L;vQi9NyhTi608sy!Thv`U^Y#g-!H2sag$%I zNF3ifp;*U1SJz-kLklcawNMKgEam4^kq^+aIz*P;xl4hUaLkFoU?YGqc}y z*0OX`TYS>;0QeH_b|T)5GCaD5=_SalYd2$(W%Ppmpt5> zsDW&j#|uEi+H4+~W;Wi3hr@XKl4+)_*5$IGMi5z z<;QmWUp^T30hSFC3OiG*tyf=pX5MCj{-%$*y&?))UEQ2k#%NHRRNc^br%|3WbDn!nKGt7!7WU26_FYj1w3NU3vU)#$H=Vqf}>d@)_6}~yJ6Q14JF&n2asLkmhcjBsjKW|M9J3*LZ z0@rL=auz!WdFX1)M(^RQdM(Cw;oF=gW)F2hLd!yfBU3`DrlD3ZRuVqYOsQ&60{q5l zppVZl;C2ix9lH|vq12HteeR~JA3!Nj`GZYxr*2wEsAVphgEmRvRa!^lA6&RWZ}0GK z(w-Hh0o82(thbX+?Y_zt^t~JX{8`$0k_m0ectHq)(v=*YjdhJlwmRJY7#P;4+_cW3 z`t43W6i{D7!Zw?O!{%Enk~bwVdjPqbJR{G-&#i`mnH1>v`p>IEu1a3{=aE!WAI14M z+1@4uPu-fGN$jGQCbQsU%ofSNF+Wl@iND+j*JKPbR>|}VZ&vMjMo|Sp#4F3N+j%iQdhgc)yrq>c*|CA~~K148#pcNa_ck8Xn*W{;# zfyY9Fv*c3J*9*m?vc;B2)G#}!==MhxSPd1B$$m4a*8glro{?6ypGDPaCV0C09UXt3 z^8Nw$5j%y26_gXrKf%u`QV&o+XsaBT3*>I($Wm1z1R70e{oHDJ!torq{mo0ups({E zzT9ihkd{TYqtMPL>%qr!k(F-;jME@VE~r${cZ!4_9T^0<=VE-be_J;$1{lfsM8M~( z9~#$+`K)i&4fPTfjIFS$a$R?OU0wCQ-6NK(>ulGK64{d?R7M&QxCG`UB8!C5(47c9 zu9hVdk6zt6M!@s#<;Gkw_enCn4~eT2EgSs7mN`Vsnmw@-RSmWx-v!r8VNaClKjYW0ixDH=5Z~@B6M9pYQs^B*L zLrt36RyAI-4l(g?D&S4cDvenDgReGMIdujxOq;V|o(MatO8b;#@~U?pXYgh+wW8IJ zU?YXoMdu$*auR|kQUs8uz*f2Kj-BMju8{O(Kmk*;tzIZbp8nMoOL~7NVVdg8??8PyhJLbxtYN@bUksM( zJ3lehjw|1;h;}o_PEyp4L+n#g2>?Pz$Tn^hPHaKYl_43TluGFPv;7YK)P@Nu=2G`1 zSH#9dVD6y&dO@N;n}868C@D?QCGJNo!l|%XTn@ZN7k&7H)pNHYyM2c91DkFKRh%6P z872Z(+R(2Vb%2x>*zz?TEo4e_cxD{t1+-n*q;a%d_b{z|DJ{`hQ|ZMg-#z)v7Ky~27k#uKpL#LGq2hq zm}Q+v;jr#5)KRp}jv>uK-)J{%i|tvMgZPR0j~Ft$CQ#Z4s9B1o5T;_J=SnLUep1_? zWJa&2*zaF`KE#`GpE02JChbk@qDJT&u|fV&2|y*$g~ydWr|3SJLm;O!PiVKjUB-6w zTy`~3Yiv_aIA6D2SX0y7HWP!26_Meav^4(U`?22h0>ozr3-~!IxVV;vZm$TK1X@ne z*uEsjaRUN^tkoT3>@L|U(>%kXq42{;HY349NaP`RZZbZi7Y0{|#COwN=bx(?HNg&n zu6pSwVQ0sx`fYO8d@cF~e<$7C*HIhudc3!{qe zj{fAy084BF)^8&C*gC7_KEDkXw{CNS?UKVXWp?DPx^0d#pofF3OVCxOuME$^t_-NG z;4TWtU=$0Q){n6AKfI=fyjevhC{DA5U%Cam`q_vMO}l3$7b5&e3YHm*;pjf5NL;oM z7XV?*HWOvz=WL;A!+Eb*bgH$=z%SiM;n^57SCI94{nIW-B2~4etbDxMM`eTj{>`=$ zM%v}V+UoZT`L>g@R8)b*zQ*{@>l5kI3HG1gPgpunmyM}!7hiz`*@>$0Ft z?e5`DF<^vq1G1^t4Axd2k1H_Y&UgSb5Hvo;6|!1Y`Dr=EU2Oc?gRp%(z4ZZ6h_Do^ z7fl6>2Pae2D~J!tJU)pIdgYfi=p~gj4x85>#bDiFwNos|E99~bB8As@0-f>!jg1D$ zdU=+kbOyby-T2es!cVG&R+#eXR-VdBh%(>F4?|b!1iRp-{Y-T!+BT0MUk`w1wN64l zNt;Gf{1&0iukI2>ySvL-UNq;7(V6lrq4nxtsMf0Hak#q;-0Tdzh)vTO!;vhF?of$m zeZ96y|GPHyNNp}r2`f*j7fovPK%#8dBh z%B7NlS;r!Ed0IjFol^IWlm4>*j(mwE?COBT9Ja^TFGS3mywn^-Q%5i(T;dNgW~Dha z-;7u$dC({A>v0u#Z#wxrSMgQ}(|8gt7dG4x`%n*swt3AXJ3ZrAowHAXw*HUDa0#)P zI#MjBF|pYw$hJiMY^Bk^I#~xZ4?%>WB7RI*laYjY{f9H!W}nN(JpuxmSI#OqTTmbZ zJ-bdmT`*ZRB?Ds&Q)rZdz8en<`I}t2%OVnniLEH>&k2A&WaU`oN2LE1T8`z-V!N}h zPg`C0r#@_QifT2d@b?tJq~@f~q$p)2oSJhR^M|wSW?9SFs=HIMZ7N1j5D=%YA#s8}ki;SRYTw&S-S&JS^RSI&e9`Pye9rYnCDZ)Q_< zz;9SqX0!cAtFRIFeR}`aUWkX32=z;5J=u4_KJ1IkJzNmst+E{1*t~h|&q;OtCt<|D z{^tt|d=ZGIFE-$yIPZD@=xq?h=JuSuDpf#WpM~3_-x`VZS@y)LML6bj1>{ZG-blQEC@zLa(U?WcvP1ID zyZ*;~&?L`fcwF*$%y&}8|4>~BN(q6|j#$p0lK{zoE|wYA?yYXHu53De!ao%w49K9q zLu2-bI`B`KMt1a9w-j>xGF_%;)f-LIoH$vm(ZHxxpH*UcG( zY%qndq1uw5QBA^~VVkO*cWlWH8j@kQo7K8ceas!2P7Pxps7MT>@i6_BL z{vSM(TNSaM7iC>0hslAG`4XMu_utBVgL5tRhNa+^F^Yr z#Yq3sKe3{}V<(kKWX~@fm88SUT3I_RX*iIaaO$PWrLZ+GDi>V_RTr`yb3zc zB3`E0Uo)*A1T{%>k1Y-7&BEBoKRWIr!5O#)32LlZb+dfMhE%tcp8hk?C?{I2Fk*q% zYo&->g_ex`3gBN}yqKWgZ(=w$)37=Fh8JYs15=3E?suRcz;p{CkYD4Cd&c@uQaqhLOUQara z(J=6azn4}&cQtA?ol;WKTJOV=aD>JWfyx|TkNX>?rbS7_HweZj8FChoINB&jFI23E zXH-4EHjFV$|GqY~7sw^vB1F|75y?06)d;XG%^2XE>jxDlVpa!rn-|af7#oT-!2X+c z!Z)$2#3;8OtQ2h&RsrAYjhZSM;wa>$p3ReA4)gAe85w0l=X0 z4;T8Xs2O2BYQ_O6ybhATOz$NGF>9Sci`?lxjv>7W<=uJbxZp5F$pDKPu~HB6BRmnj z)XT`WLRw5McfXj_kA;evBBiJ!jot&*3*R&@CscHtCme(-$v!0n=Zd>8jD32)50coM z{A!Wx@$`G3IH`w!>PJyszU?=F<0(<6XP|wt5&cZi3<_uXOP0F zuR4MvHh99oxoApw%V?!R+~-H7nD(F4bOW*Vk%!&nVP8S9UnMHIC-0_+_e8RD&D(hM zt%VEmWB&BCz3s4QZ93_tYliHogL<)fB)0zao_j1RudrZnqh8c=#Kt%ctvy(1xYe~- zDN{0Je1{C+v9_NdE-lR!vmWZA=?RWEqw@8G!ad%-ov$Un`xF9}TVOtvqpvJ6byp&g zyaA4W!|_FUxe#bGuWNP0Vm3&05^D%RScPTwu(iI*U3L;!bML)3@N$O1$s<&@U6dbk z6F&MtBQH55sP%P334y#C1lA~!*C2=?V;nDScXWb8#oQi>zrujx1u})HG!`9VwaO{~aXV%}pai}W+Qx7$W@aciw1x*7XkWUi*9ol10Zs!yyC+3yJ0CxS|;U?KXPit0r@RAoP-Y5GCK4 zEXno*r)uc~@~Ir0gDGx-+>eJ5m0soZCn?kJ>Q$HK1phSHsA)dwx`6r|I~b~2?WUg? zIGt3cZ_Q4dE@tT@90z$euefiz;*U!#D+V|v_zy7^QRvd%t?F?5{lSYY*fpi2HH5zv zsS}@ka*ZoW8Kf>hN(E<#lgf z4dr`1ExA*{9M;Salnpa)Agm6>Fqcfj>VGQSr%Z0_&nT*s$S4Ylr~W(%ZWX(mOqbK? zAI(-J)Utv$o0uyhDffSswCT7nkgdqYJ9^>m!a^b#V>L@_S6;VFmvhsWhhxHJzKYN} z-FiVX3)LGA8*%6+TnQ&7&Xa(|ONw)uV;lbvN&BTor~$lCg2KnP;Zb9jd9pz9UP6gq zi=vInlq83wvwE$e$C$HHspzG=eeu;J0Jh8D_whYk`g!c*{Pb=sZGfT>vkuPh<>?TaJ<~# zTt)fmUI|0Z9WdNd-YoteTYOlkJToaPMF|_f~d_} zcZ7rst`VUI)VUnY>{o8}uOhfA(StXsPIY#(JbnOmGLRkY{7O{BDS?f;64P0$wC;z$ zs_M_Mp0g%S-8&=U6a=-HObuDD^@ba1$q~W{6a~Wzjub^c;+l!SX&KXjKPE|din$$*^5nt<-z*eN{-T%^wC#HV_+AAMYWo`2_+G`9esd%fT~gye<*$@ z4=G)tVn(dcu&V3OQ?{M6tf4%_J|r`n0qj&Mv6P%Jlb*w8ZhoC-VOe$;l|kz9_f)yv zArx(5qQ-Wb8}dfs?G9*XE!XJmf5r>y{*uM37gAjhw$rMXbje9*7kzZYrzNl}5XR38 zu{VY@sNl#a$;Go~AJq4B;?vmka(`otI!Ifp;*m!L-*WdGkm}(tfC|v9N4M{HKv0;$ zs7(X@Qy-HK&(2HUz2WVS8x)ICtw&bArQgyqW4>G3(p*D?xEi!zk|KkXi?!{C>c#SA zBEJDtU^gvucFvWLxFy$Q^9Myk9kU4s%}eTxe?9;G&LHV4C?ll0uygng(?!ZD0iVPO zK`IMV|B~Swi0#kjAIYk-JmUp0Kb!XYXfa#QVwP&AsrM|}CQJ7~_qL;P*;{1X13zW@ z7u9o_vkh-l$Ps6xKZ!@JBP=?NdO}|Rx{WLH=jfZ`N z+8FCe$I}HP$U^!fz;b8E#FNaYhnytq)Yo+Xto% zT+#LVsW9yLnk(r6`Iko?@3xbb;qam5Mgu=>h~7LGRK`I3Yu%H6NVl|``oV>}xLQ74 zVKStiYR?XX5S?e*_?u1JnLD<Py)Wl0mIMBUZ{XrgjgaW_TrU&bNBkxlwCP8WLBF?j%k@e$&19n?f zdIt1&=_BIqZ*)-_izGje4!U5He#}r+@gdu0G_1n7o#W6yXo`XDvZF1TS?Eb}LuhSI z(QO5a3E3;cB5A^_&vHm!!b01R<#5J85A42Y|LS)24Diug*R`O`@?Pg(;;_^zS$2cp zw5$6Ct?ZpA3sHW(+2l7f@zl+FN}_*Z4@mcLFo%&#FwB<>@1?PauJpTeNK6PaBRNsh zE#n3GWM)TAH(w(s9l!HuOO4H2JlYZ&eaopt{FgfWWt5fdGLvZ*Mz>IRx?Jfr$2En% z`3Ar_t?SKQ2Pm7q3NDO{#hmngt(wi4rH~&~EpRqEszx`e13HZJkI&Y@Or=pBu;1Ee9eP&EdS0Z5Fye=DOB|3Avqe3fhl!2+klCL zOrlYqM^BjWdtP(A!PS89GvA^m$&oNr`vJ`pbiW87`V~ieAz-FU{#T>E6C?;Gy}W}fnw+D)ct7%hRx4@>Y(1^Ba8WW(D6zzL z8|wSBpbbziUhtu(v-EoUhXbZEfuIrLXK$MF<&4+7hmEI%7cvZm`nP3Y=tGEx@Q@~D2jNU!=Ztz!la?s zpI3-CVE829MbYTP1@%z7BiI2#gr7cV(V13?N|M}NQHGWmi6-h=rgVqjG43~~DaDhi z?l|rQA^JjrNMPefM zoWR~jnGlv3#ZbJyPVX@n;euB?ru;-XH{{wxajjoygXLRczq9z}OIl#dY5xVami5^V z+xLhKLbt`3Ikm=ZwKupxXQJedS7BKo9JW#gB8$QRmR&5)>DmlvT+L@W3!T@7Jv%urrW9ww6eERx!KZpW{w6_wkrPh;VJfOl*I=Oa)4vRONFv$fUC*^1ni- zoluQKuYQElzJQ^Oh|W3L{*)sJx%TlK&yrc8jZ|A=WPYx}Z&L>c!-F_d6XA$^_l9Ie z=uivl47^dfL5OMVrGEXeY+I<^sE(tu0ZUx#G#~hN*7f$)%f-f7J zOmTqV_p9R0ZwG)gkv|gs$KrbKCvsc6f7-p6lOcZQDq9QEyt0g~QmUuJ#vMLoM>uM0 z zTp~m^^W>?k$_;wZpawrlzkWgqPl@=<>;4H7o_}mrY&-WFV*Te7*svQqh)G_Drc}-U z6O(iS$HR1+{#5jX^I!it2fNM_df04W!oc+p^e|l;1Ta;ZuIMZNb0qGI0UQvNOgfxa z>jN$NH(Z;C10s!+h>$*ue}bK-PjhS=eBUPR{&OTV?2a!8V4eg2TZ8{U+v{2Fob*)s zZ^)Q!{|%%j!oL8XMz$Ju3JMd&&5Q8BB4~IVt$P6eSNvCyV9&2wnl<>xm3$}rEC#EN z<+Bq%_3NL=uP!{h0c^i$86IjX$HG7Td_xNQ*?qX8@t+_qcr57VPHz(#(9QqApCM0v zCOM-o{VzY`KKc3hkJ{LO`5E=e&u{{L=KruxkWCcyvoXKZV#JJwfJGO#OOT-wKSN@mt5Mc3r(eKt9A`b2zU8nC|m!%fSRhqS` zSGiJ;8u^U+8Wy(a<~L98ZtlAyorxQ6h#ddCFL8t`weu!88vK-C3gC11r_{^|Sxt#B z+J|WZ$I8Zu^{~X{7YrqjN$GJ>g1@Y|4%w3h{cBV+x=Q;=3|~@BKm?-g|KLqx(g(P{?5V*@lIHjiZNyf!L?7 z6%Gj_VqK=x!AWb)yN^jprq!X^FYr1Qs5!r%ng4+knqQ$CNMiOV6}a=AJCCjcR%!QV z{<&m~Jq1OA|1jh>{?YxTJlE*oa1g&aZ)fi4=$caDS#!I4*^Ohh05GhgG#VOQPTUI3 zTjn-xiH6})(=I-qMhb|mCWQgV7jA8S!51G8=|@BqTlt)^=$zk?oHICl=vw?$Mew*O z>^S53$%V=$fddDyv3`FalVlW%w{TozwYm{^*8%_P{LJ#eLBPj3Nej0w?(ngQ%CvP` zz`hUeAYga#RC@oD4yq#}16wZN{sWPKz3Y&3Y27EC+eOQ=UAu!AI0z2bxnWITfO%ap zs#*tIL(AhCI3MaQ5~yUo{9lo53TwB^4(QFNHk}LAjMIc}?m?wf>n8-PMzrl4GOedafqjRI z8>>DTm5YJTkfSynxVlHJqxy!mY=MRB(!KR>kse4l7I_UyxTDTEDQ&;AJ~DfXE!Ae% z*lRXleuNY)Wx~w=f<}LfFHZrgf&aQZE?7G@uYJC8c+^tvHJYJD2c3e#xi3Sh%`KLX zfrxx**(g-RRCYiIGLG#j(_#ma`%Rq6{y}XsR z9HNRO5ex$Uj{4qfIWqDY@9V(r+acieL!OtAx>fOkwq+13Q&6+pp$W;nK}TKG*2Rj? zi>ezbiXTm6gT!-0&1R8)1Twk7oAjfX;oSac94E9!{4s6(->l);*A+*%&zJR5o8KF2=?_loOMdVnkW-KvxO@*-6`LAV?KJv6djs6rE zQXe|M>f%30Bu{fFgKdX>A#?Pj@xB7?^1-gGQ;Ljz5usD zdnx(|&YrDl+u5O!1e4S{<%S$eS>>94ZBwmF@N34f02R_zTw` z$hz1WE)Rcvtry^!Ja(05okkKVWG(*UWMV zKwZe=FCDk#gHLV(cW3k7aZT%EZf7T?JdDP9b35tyz-B}gXNv~E|B}Pd!o^E3&Fp8- zu!Wu;Mv%d08q}~5NjJ1x-_Iu{KY93m!ihU7!A|$P%EF!XoRCi5zfj1(i?8$qvf&M+ z41UWEF0ZuuZFhfzuCNn(OOmtuNmsN&3QN&rVyeB6u9i0ryWBN7-153hT$?W{c!4aJ z3df|L`?~>~%Rx36fr@T9MsA~Ix66wMCp-`8t6wmwHt&%M50cnEEB;RL9oeP2ju6Cn zZ0)eF)qx8;YvE_r#1#!q*=9F~yF8O2sfA;5Z<9%7pK!&o?kwAInpA*UAb<)& zJoXz>_3n*^#cRsMAF)+`7Mg5a6$85CgWQ3wnk#Qk4x=V7-0PNWA*nPX`ag&ah8%G0 zN(Ih*y%UtESW&;l{vH2+U9V@(55YpOVHtSPhr;Tz;rpLTjG6k{slZBluGQ00+K zb!a8CLl%uO4sH(eVLndM`}jVdjRc-xTIWyhDVvEsxgXpIBDlza5vxJMz3NsAqdE4q zIdrIv?6LNc9v-+x|HJnS+6z4eP0L10ZQXhC$gJz*r!IDNE#-gJ_J1e6iI1c;!e#1M z>OB9lp z;{SbecDAQN(ip~g;4c#Ce*!`Msg7izC?@>p_a)+i6(eWG^NNjsmVuuc0N6x!@~bYA zmt6Y77pQ4yTuw*hX3N8W)}wEq$Bkq@J_xM7vX*8m)m>>1Fmh#fFuSgc$W8u*Wkm5# zfI}4iBSBE; zKGOoRd&-7S1C|5eY~#ZXIQjXiE@&y$WhdVX92RwyVrr^<)Bx_!7eJg)m0t+)hco#0 zLd)a5HK^8s3!4I;NwC=&Mv}HOGr+a9Hwg|5)?5Iy=6(D#<4rd^h51)+vEOxO8wYYFx3tuz|0fI#Oj_{wJOWGNlS2C@c^8o z3=AqbpD)#UUjxF94PdgIVW{8wdC+(|SQukGUu%1kT9n)mxAD?h-RQp4AiQB5xbr!l z2cI*sfFVV{3}Rt0VY^)<0O6YK5(Fe(At8ejO{~~Yn2YOncAO+@^7iOn&)+VULklmx z58N92M6D9tthgeVnS=cbH8oO_Rkzg;{W%Jg+ohI_)8j)Fq7(y1!O`6*raSL-(<)So zN#?{B+jCO&~H|X+5tXOFS-5~pT2Xf&zCJ~`noHK@)Vb_3~Ns5xHRo!}|9Vo!{&q@YS z_@Xk92EDz>TLoE8iIrbgC(*aykSP}UoPlPjw(GtU_jXp)ovOlxBwev$JiDp9aydjg z{&U(!{cZ_ex0qK8^%4kKr}o-|$m$!Hv#sS@F_LR?cN$u*_aIv-q_fl=LGI2{nRI>q zM6ly9T%Zb{k3t^6aq%WE=KAL1C;DGcjM&+5q!C;1Uuj_Y3jWA9H3@R_#V$joRIY9; zPO7$#(cpC$zkJ&t)7dtZTLsRoQHQ&uwl_11emFaA@K7Jd#-ncVX%0{|Wx*KzXWqYa zaM`>p9zYD&IuN0D>t~SLtctcXC7=={Ji9(tv0d|ZRzHa0HHrI~+~~m5LsKq^I$>XY zvRMQP{z!J8-)_Hs1VQXjRu5bR(kuhfQqACeEoVv0DTj_!O6sO(B& zPVCJy({vI58AL>|&alT2)~du2RSpPy7T*ezyJnvc%*(H7MjO=6e*nT`{`;VKjj|6U zD1F^|BRHwHuWBlZ&N$w9Gco5u~|2bc&Q5273o*19)DcoV%+Jv zJS^8*Gy;)^qJj^8GAgPdK(2)@Z{m$@D^p6oE;19spr`oQ7@l_YikCG~$e;JRgKkm+ zcULe#jK6Wjw`j*iMj(f=iRyL@s|+SE;2zK>n>VA{POv(HCDyEEh?6~nN+#sebtCOV zUm6a`M;%K0B0a^b7QtKau9l5GXeKzxCGoCc`~_)ICyoO%JCL!n`gQq9r&zFX$jb39WO^rL0N{gi<7jevOOHk$rXA3Ft_M>nx_&L&oNVD1 zJeqz%qs5RWZXvkzv-9NoQ}F>7g5_J;P7OZ*AqRH?2=~bB*WRgtR3Y9e{iFLB2%H|g zM~Tg6T6%2jck#?}+|7sUMu?I!uzeahk_Yh7@D=htqT_LEB`=ohrW^ZNOh z{X2gZj5FFFb_gWnv&$lHDv){*YyG!HDDqT)5htv?@S}#kg ztb02*UDOa}QriVUq@czw!#mr{ws2>{+o+nTr2{gGks_f)xL5AUmU0`j+a(_WA zGbnH>)7_H92;YP;AI#W}58tz=Jtcu|!btl~pzYMIg7Qs>!0?gHs+{bjEkS!ZOSdgXJXyf9J)2B?8m_!1szJ+dVc%Rp3G(VzSFj;WbJH{-q7fc>%5R< zV|ksBlV3>WR;NfkwPejl2{8Icj~CW_a7eGbr#in?@BwYLtf9aLsnvhyIAm(Rjkc`; z+EeS4Nj)-KUv3QcNpR@@)=p_cg~L>Z&UNxCX&1m-TTw==+rtzUoVW_^qdg+mp|a_! z^3JFx<0g%i!EyKNjdpGV`&ttaR-Pn=Fj7u8qfM?xuB#+HvK7;aCT%DX|}8^BF_ zt?E)yyVKpg@MU(RC5jO!iL77;@i|s`5w{b)u?(bq*9kH&(k#eQF;SG}< zE_=!GW`jc-a&@z;^*6WNs>UAaojS<9sN&$!2gu2}94-CXfrn^aUU=bX%NJ$1Kh91F zBVWBI+lr#!^~WiuZn##`Kw&KJ4V(i6QOMy>;5xrbn*W7b<(yLixhiF995TCYQ@|7tf-j4$9yQaE@EYaMsDc+3q6N7l@)FE( zq6K&7?y@aexv`wa=fJ;QIs0?Tb#ReG9Rh@aPQGa4_2c8fW}lUN$&NX}$3I2qR;xXvl2W0?UdR3Jg)J?p^hb~9 z_+pF0qs2!O-Zbnfi<@V{3h1+fm~8hZ4wkC{@5aCpGw(LX!39p%SMsVg;GQXtc|J{R zX|C=VFrV%eP<|hd!#?0QJ-JdGj#vJ2`+U96^1Kiddv8BnVd>+($buc~A{armCKOY@ z{-NIJl_(K}lKNGMW8-g~i~ytWiH5wu(PqMjnN1XWtd|Mr<`x;cyb@z+!WI#g{+lHl^&C+b*%l2X-*gEqe5r%HuS);-SRb8k^b9&;w zRl-J$wNsYQ#A&2Ox4PEp()$8{i=~PU%={y6;$l@BYckO)*;lB8xRD+F03xU#+L&XX zVCLSqJzh(D!)?L6{z-1&k5_sviak?1V?{fWR^KI4Ty=!zc9({<2NQX;Ocg$`4Or;i z^dHD2)As4g@nbm5(|WTXhA$a?0LFCb!A7eP6Er%XSn^J(N+1udJRY4|>~!hP%?SOm zr#?&Y)}whP?}V)c`56k%0aOEyY5O|M@L8GYak)nyHPvSy`WkEHwSe%|p`+mnkwilK zVdRDTW{#%u;Wzx}$=YdC-sv_l4K1Fd7M~i3Lb{8VUa#&t<0=S^86AX~1rN4ZWLuM- zAA(6ZH6RxTOrp^^lpIhY22R%@>0tFC1?SI~flPx;^1{vMCWeeMvAaa2 zBwD93ElhEGsE*}xJ#yV@?%}e22B(ReZUu=&g#89FPUZleWA*IbmETAIP%^E*?X=(A z$>5^oDOI=c*bkYkuV#zZugjtkqX?W&PmeeYTv~rZQL(Ra7fclZWD*vwd;&#B5};3F zQn~dn=n1KFAwsYfe0W1dEA7`h_sRaS0fz7yA#>hvHKksmfadZ8MRr-&Oe$5zpV!TXDFO6+`d8J&7vL4+Xf{45v?h0HY@nF_EWJ-;33JLzLC zA{tOV6-I4lCBoDTW|y_wldrW=P`YNzP{q!PG`NnGS#x1s5?j=DeZ+*)RMG-gIBX$* zbf1@G=NJ#oKEUk8p9@;}0%Opb2}oy2#O~y-hu(Kn^E;l@5`*J`Y`ZIC?L0^;otui) zjbEMfzN}Stw-<4!DF!d!o*SYg+C<-CQut6@m&{JNR(QRql^PXU8#!-YKfYP* zFm&mj+ew4N>dEo?Nz&uQe*QbN8^k)}Y%k30bVE2!?q|DJrGDeXbQE5AAo@WbH9@I| zAe8~pmVW_{l?T$eU3jzmymoA}VoY703}=>^gueK5!}vx4$5Z$&4|jn~mV?Lny=HQg z3F|-~2K1NBZ}rHI#IVt;?_bz>rHOyby9i@MCJ|x6@+;(0O+nf)+xKkkxU5u){-v@9 zuK8|8DoT0{VV1E9Z!Y7Cs*;d6Zkc(H;=E%#ka=85;g6y%vDKC#BSGdv+ zmU0)vdGeHG!VSF>7R*7@xl~fVCZvdU!aRQ{g&Kdu@e0~z$rh=nBoDhLw%C_p8~Y5% zV&4g4uzsui_fITS0uFY!ld)eDp5jL<;1gaPalbWzF0ACw zysY0=zaa--mnIbllW+8l;B;Btl4!a{;4_%kIxM{3=HP!kB)*qqVzqGb# zy!8UX%ZuE1qy)zVzF5o60&U<~SPC5F#s++UY=Ow%sQwuWP0jV1Yk?2=&Yw*(K7Po1 zbGQ&VIgTzBSn^7f^mGvCAQz0E!4f}_$p>5%=1Wn~3^@pMU^cyxbc5rm0j=@()I2+q zOb?+UBDA1kv{2fBpu~3#yUMSKhS3rw;<$1)Bd>OM{BwPbzI-_Y<_^>?;Wk@Br==Ad zHm1VmzBPeg>PsM<>zO`J-#d8`<} zt-7-Ntz^@)Fq4Sp11x_GIOY1;Cos?xRFZbqtG;sB(7Tk?7S4<4Rzj|lU2fX>D5cY3 ze5is*yXyIul%PSJpU|%U%tpJQb-rs`3~9RMhtU28Z_20Ik5ZObDM`e}zlxKu?_Z?q zpJojLvm@CeI!QiEg-y=6tF0x;2b)a8PyN_qd9rq|v!ytwWdatnBlp@5HC}e27D`6Z z`!Ssa#}sy`XJsV5MEtZMBmlQpjmy1$crWyp9_6sLr@x({o)+?(S(p;?;3BXFD%x^s z1(*o%&=8lrY+mzNuqA2s;2vdG6s}g9$H0qyfqL2C3RgC=w*D7XIzus^II65i&zq@d zQbsv=f=9iQlz1?;Z8(22N*C1&pd|5uLcckh=|J$omk;aFQ#J_wPAVCDTiKgu=*G4X9E2J_oB z{=ei^C~>a#$%Xjo^s4hiOpgf(w3gEOIY$jIr9&lb;idHdsBxb-Pm6o6<-c!qx%^kE zg>Mig-4ZPIOd(HYsvitZnPSt&A<7phm=YM=$wrEwK23ZS1+2L#d71b=Dpr}^0 zvibhof8Ai?(>&H#b$iA?RZlJJcW$Rz-5Ale@!xJJ(8|9GfEC!DwO{!k4RGvK1FR&B z1UUUyg@l0>#Hr%7{uL%cx4QDw$8|)PZ}yFEm7dR90fqZv76Us kIqN?LGv_%i#y#LA-#zWg(bhOHH1~|2mZ4^u`opmQ11q8^mjD0& delta 21230 zcmc$`byQSg`}ZrMl(ck*gVHDsk|N>I-QCjC1Dg(!P(Zqrk`(Fg?nY|p&Y}BkpXc|) zS!b=Y&U)ATuJg~to;~;6`_5~w&-c5_x{=D7km$md6r`~+$uS>2dW0=2^H$~2BV-8p z`WN~W@KMMLBy+#$G>#0>Mv?4<0=ddnEf-T+IV%KMgHay%NzaNu-STg`O9csK0!roR3x5TQmLo z)ZtJ+qfvCX{9^H@{Nj^IYl#+~l(I4_EtR<7`~$3f=+FFKtzB*ZqPe9=#)6B2!Yf(d zURlWN*m(#UouwH%DRPJ(ozdSXvB$I{707)?Qkge*K=l!6o;m(d6NjXZ^8+?KnSc?Gz+vJH?zgf_t*B5>YA>D$viBAjqfN$U^OX%t_333 zETn<1>^4^WE^wcVTJgXqwQSY%V|I1hxwQrB?Qr11XSx{xs{7gNYYrBG`prmP5{^H$ z53A9lQ$`EfvV_&AWACTYtNq6CF88b9^A*9Xn6M(E3xD4_%u?-1qQeJe^ z05uqdkd9GI5!ETjf#FF$EG|r^Bw^3rhd}UH$Kh&S6qQgFt1Iy7aO1xK0TFwK3jmi2ZwrPpQnO~Qn{p!&NY%!h`vDP7C#z)iOF zj|~GBDR$qor@gO6UPTKdWELx|IeJCyH!U-Kc-d?SrebELpY%PQQR-&OfW7)DIXl>R zv6vFv8AB&I_I(ntPK2r?QNQsri(zHiEI zNxi+zEN2FZA&hI;sh_Qy5nkD~>~;Rf(z2xfpsu0e#jq!^nU-+$wbR(<4d0AWAftkZ z%iYOkzOVlwNqj z(QqW8q5VQpAiR|#t13%LpkK&hAd!4#=%-N|V<9}pAnEAS^S(G+XWxw^OPgP{=d-r4 zw{GAp7;uOCBy_xF;uwXPMF}0$Fme=AOcN>!bM*qM4tVL@I9Mg_i=IrcT{fUTX|9pY zgJAQp$QBCMEP8IHX>Xs3a|3il<^L_a@4RCfOWe|(rztws$~i0(Fm2!N7H+-!9>bW?jj3kp_O3v;+FBQ{g&Lbn=f z`|W7RZt~lYhEQ5Ev+jk%<)D9ODkfy9hY`OjDUm`hAVKO~a!A0WEtDzMFFiIMx&B}6P-^!Iyu4{c;7y?_kfriga;#93gT@!p%M>~q(PY_4ROL_>Ae&RIn9uCH zSJsWWLgzL)u$`B2pjl!hWII(9V7sI8!=q3D1svp#!ax}qXY>{q)5ZU?8gc)W2k?XO)w zVZc^y@ySitzG6~a`Hh8DMO|e_UZ6X55Ylvzf7p`b!c8BaEueHQQ1&B0K}302r_G_p z65!vn2+}CdbH%C7D+4^j4IwsymzOGaD`h@guetC6SsWn|t% zPc!65pgX@C!1r;50F`qOoow^uaX>ik!>jAZ7ts5%m$YmT&IqrgRy5nV2!t`U-vqEZ z$OnhA$Z2RbIh<6@*%y8p23MWr`1H{7A;JD`N!kG^E1qY7WMhDS(cq|*MsgMXT5&|n z!|kT12?a_)Wibf^IM^0FhN2jsfQ(Zb~*N zO++L$`Q#s==%~tA_a$?M8>`w{1lOQobG+FJ+n;9g2u$lUOl1mUSQONUm3%q za}v!>UKlu=P!+MSwqHn{xlwJb;qmo&o}p>r*?H(_EZr-5e~im?e$v5`zQ7Vk7x`As zZDJYhOnHNHl$9QaXlVM$LQlbLf-a@R0yxlj8jFtE;ox47`Z0*{W&P*36kmmR<(qx) zZuiM;i%@AFtqrmr z$CKXEpGxYUj3Y5^O5509>=*Juk z2dHKn!L2MRjp)?yT9NL%erMcfnehc`z0izo8KRHnMxSmuNby-vCga)ilx6v( zhHkBSg8mddp-r>@MIk#=l`Eg)l2~&D%KTfZJ@=?%HizH65c0|9P2D`jez_U{oFa6e zhlw<tPvC!3WMjO^G7*{n!Y+VmvG-(^d~I`dUpq2 z51+!V8Da~!R z*U;K!RPRzB=5%WsB>=%e@aOD{m}t(dCm0?i-bk2KE0WVVDm>c?iPi-JXI<2hozTu z6)%WSAy|$db>{$k7LbNyc64J3Pu?4T@QinuB?=Y46KSurX-021QB`;e=Us(r8Lm^5 z_-;o#32nY6+rrfG4A$g+oTDV*xm=A0VCnD zSL7f@SX@V_V))AlhN=^zC*WVNO7M=~SQ~gfR~+cfQQC_|2RJ&jv_yV4HoVhbigg&0 zUDYoU%sE!TN-@IiOf)Is9;u-36`i}|Q+A#Fmm4Qy%T8X_CjOWD-vg<1LCahMH zO%ADd+@g&k0kS{gyxOtvJhyqCLEqZyuD+>PLMUGjGAou`b$!yi?dyGCq0h4;QH$sI zDa)i;o+l;9m8tk3UEJg4w=v>AbjCf403f;f-GFX)C>T2P`Np3WHE|f%*5Y=@7NH7I zYb|vMdjQ1pDpAk2TN7JNM6M`d<27-G3f2uS!RFB(%0&|7gx^D{J2Cy_OrLr!FZFv9 z5zA1NTJbYTVt8jDR>>7AxD_th^@u_PZLMCbMXz&GxrtR;BU*{xe`v?N3OYA9dp3RX z^cS8)Y{+)y%xxIS=FhemL`zj6toHW;^re0G(GOsVDRE%%JRFT4LhnXa#-_X*Osn*C zTP)QgWhkE7?(0T752efs5oICI0#dN4L1L6Mi6j_LJ89JYvUhq20c4DQp)GR9WH?#l z9J8M|^>qZ=FFKDRgF_uBUv50l$iSpsB123cBKtpoI_+Pbh_Go~Wm#41d0m8cey-(> zj|o7I-8BZPOLkweQ;&??ED^1)K7ab3K zv9`giRtAT-h-*06FFi{BwOhcM^p58jYh$w6MCA1xVzfSu#Jm# zh($;ieRmCrs!pYkc@eA?ChCXFp4*Tt(iCF)Xy#ZxB(DFevI@@I&@LF0H>G=G&cbF& zXBsu73}3Iywf0&bWRFt$ncASG*9BlH^buj6Sg*63&8dwGH?)feEvm?pWYU52`LP~* zdh)lbe(4p{cy)!rh$9u5lkro%ZNqDQqV|Xyr;H%n&7wF7a20>)N#GupYMAr=u`Jar z_xH7J2dTj#@%v9FMrDkI>KbaP38yt)sXK9w$m%PJu9Ovy-_^dhw)fJmrvdrgMI-S) z5+uKdzkn4jTZKz5&C`BaEC2n)pTRRKP1NZV+gq^hRuDyv=d-dS&i!l&^CWexqhx4& zFA~aK>8NuW92Of+6=l>Ow1*qM5pf)3ZWHXgB4O z6{>R8*Rq+WCxxz{6%idy2pG?Y8*NkJp5+m&96k!w%{fYG(@s0|R6y@Oig9e>2x)W1 zYfgoq;|T37U9tqfO_Yg@imgQVk#fNElLePx*r#y^=!*AxTD9psx4}<11gB($C8zgJ zy`skVwUmYtKCZPwC8MhJ6pYnTA=FIBe#i=DJ0S`g<@E?X(?UJq{^KF!#c<`z+am`SRB8Qw{Vf z^Io6U$5E+vM(qnOpv&clPaE=95OD~RODXlk?`!wvk}#98p<;gPW_N;%Ntn4fI!T^| zG+YADXenFXMIQaiBlys!T@#w%OQO^h`umq^AQjiG@6$Vu^iwJCn5PxXd@LIgG3vz! ze~<`O%22$X`^=$JLzSQ(_Chd7O!r<9hzd7ju@+gD+&Cjlfcd340_?npI|#A2Tt$7N z%{&UOaHhESv`Vx5sL~}I3Clg!oQrkY|1oMkXQ!Et1oP$$Po9wC2?iG{rd2}BCG{}> z>SxXuwU(P-5}RRnU&U&(8e6#-M@O#H3XDlFzWXEU-DXA!&z(@E%aA)p?+&snoYbL{ z_tnR5z0>1!0X^3~6P-Ot5u*13P=T^pHM^c_xUcuOS(bC&AByhop)DR{4n!>rONQA)4H*eP*VuDdN8GYhDbB5CDR}I9UJE}e zBpsKryHv673g@omGH=bJJ!ha3-WLwV3pDAY=)yfTeT`!AIc9}R)(2{HW7jTJ0(|Jh zIN|s>tbn=G!q-}Ze-rIbb0{^#(!s2%U~sv)#38>D<{5y=@E!dG)UCcJEWWe#+QyzDdEPC~#GLfQVaX)6JT%WbGkeN*Dll+cmqLbNB zAl{7?W$HC$N-Do7J>$?ix-9t{SW&EZw$a2#I-pF{5Mx|iCV$EFMBy_1=GrEshB8OE zzvGqWAb(CA+h>Y&uJ6nApSSE5S@!!%X?AyFdrsjdJifJw(;*zqE6h>3i)LM)(p>~i zim^8hG8-eB17|8?3-u~742^?{et!*jF`aTL39Y-Y64;f2&`^dr6SY3fvTZ7h6=`X2 zn*zR3>PJNdSZUR7Q@%xztS=AXR3KYzZMCOrKd(>oh?k1v>1xP=rDzj_Q z^CVJDcroFJLNZ`iat2A<@>kHY!Z{}RsRPis+z0XEuOPimox4^1$P~W*B>o5ggQsgk zh-upw7lGTQOtif%A+=%3@2X*()7V1)~ zf+;1i>wnd>4$UlkBzit$8q2?oeINUs%PH%D3M;tc(mBp|hqRSHP?Yo;rYtxVwHCh~ zp8V8YS>R|BZeNzB>(qS`PFT>rqA##~NZpZsUALcbf~REK_WnVM@1+xeRKXr__U2|l z`O6RcIPod(=5l#FBKquRi=WIaYDZ<)L}wmcK}q5#>wN+Ya+FOuhxzqS^Y4kwryLu1 zgyd5W$sHtd+>2wPP*Tj6E|~0*(XP-Df@&D(e1)Yn4?3pOqlyP zGOHAakI<`=yDx+16ir`~*(p5*I^o23?_{IraONIY3S^_b={)50>40EXH8GEhwsFt* zqaM14`K4G1*)%W>!(fxA_zyw#P$&jRlPUVRe5$Ag^~CJnhr0JeXG@Lg9BTOuS4~tn zK3FFvbH!bo^9e^(7t0Fmv#`0(?#tC^iagq??gN5O>DG6;08b)fr$!3!%BkR`H(c;4 zKCbx|t6B4d$xiyIuGQ-G@c{qR!QN`JRZQN>oJ)$l&-5?W%<_D`deWzkb0b;sZsgP) z)Vg^h8jT6YF8k1+Jgg0f8!Kzn5^|1tx*oa=@)FIa=8E5@_ZaTW+w4QmlP`B|>)J@>3fUEB?$DT{-TJ znjYrMt?t^a$Hkq?9bLs zYWH}lp{OWOMw?~Wg~gA<70j<7zWdA|$-U!tV@@0zNVZ{!TH#TogjB5OD%V~Z_e-sn zF)q7JF683TGn%buP13(GGTLuOwBvD^sWV5pN3OWB19=FE^3p;A9;5JJ7agC<=?!W6 zj|Jh1WM4`j1Ke!W73$*T`IOTo*hQLS3^_X;Qi4OTFs`~UbfzkVNl{lWmtUjB1P5_G zGx@$QF*6=`$nD$1)UX++ol|F@!M)7Gigg%4_XhRFgvGPiD1Kw#TU+@(z_&W87)qX17gjOW@}^jvc=PM*G1vb!wk zVx(rR6v@IX@}zxal`-L=*Zw$yBj&bW(;zZ-_mf|?5BYn^5*mprb>?RS?_@*k_Pz8dR1G}HaHNtm%f z*v)cUB=J#TXAO%x^dULM08_t|F_V9Yx@Bl6ni7Lw0b={WV3=yz8lAFEPo{LIp$${} z^{4oEAG7w5SnXImi{2AbLwhn|y_@s>#s}F@94er%vuO(UYd=D18Q{i-I!3LbVcJT9 z7~-<*4J>Gj<2agpDtE}`HTGPnbzx3|!K{OKUfQLxTvCgQU3(n%rUd27ZI?@jEjN1+ z%hXnL^PT3Lndy*^4V(4LXs0F*aVnQ4ftyfDUk|)Iha|JR==EipZrrQ&*^ZDO67z)EvaIdGr|HB4ok)^%vq0V0BT zbi)rRR;pjlS{=5FH^<2~D_3ehKh64t2W3V=X9@f#;`~cGCq$9Ys#x;w3+P}++ z4>R@uznAsDQWWsNv8?}gURcE z`0QpUT(?H(SPDh&FWcqQ1bHxQ;Z2RUiw#4cV%J`CECy+Xp6Oa;1<%7y21%X_R^G_Q zLTYGxw2Jx-RQ#L#d;rn;A^xs1x~zGuBm86N5V z9L+F9dP>r3%evC0>%Kb>JeQbuN7*Nte{+gp{Vk)BYC^L$eror8M@*mj%ezSO^`^@e zEItqz8(KPEEr-%j-dXfMPdJsL=KYX(17>IEy>^Q?lfg{OxlMNE8z8B|YP9Vy!8RW* zrF?1)#}&ODv8xz;XMWSyDZPo3T^ncu$c)5H5p6JaBBJ+;~=O9&QDl#`Y zU;i+kE5FglQaW(DGuhu=4>I4c{5kUt*E@yUJ`V+jCCtyS^dh`L;CbbBzCF(5vKq-} zQDX!5JROzx5dakkUxF6E+PSJ!foR<9FBoM6XxLCz(d^o096@Yu_I9K=_SUxjBPc64 zTVL=#Y_yFPKDoPqTfVK49oyJRkwvZNLM10dE-Bf$*~|d0#ih{5OK5yfl{MTz@*PG4 zdt3gM$R0O`!Eweo^m*V7$Xc}o0|`JXx9FK9admsWTPmAm^#JM-UhyO>HhNoj0)+cg zHVq~y47~=}~7a)Cpx^och+e0&=-M5^*{RVT|5F!U0 zkE`{Bg{9U&pA=h(Bu1(Y60)L&o-YhOM_>vkuhiBxKHlxC!XPg29p4e`R2RFsD%ai8 z<|aTLlNQ7_c^$!jZOpuH~mp#-XATiRGvebc@K%37R6?@nVkCx4s2ukMAU@Q=26 zc$enGs0d&Avi<(;4D|qczP<^|Mfe*0=CJwjilb@E729LR9vOw+h{L^5qB$YB^Uy}wL)N`%7=i<^!h&ZS6g*-3)!_=xLaFz z661C~dhRJznD_NWi$97_GcBd~Rv@m3qlp2+>4>lG@QFC3u-k&Grl_0JL;4%k1f(9J&(zk)-+6pM=3kFfef3v=qG3?b*R7Y`MEXkE zqQhyhP|f`N&qy_w@K34)PcaB4|0V+vAXwMRzbAc-&Jzj>+CrJ!}Dm$xoV-W4b3b z4bwUb<0v>s5j#YL-=Gz;_*^bjjx9*C4d5&pY5w6`K?a{45;{z6*z^v-G+t-Oxy72p z6P8ptAvf<_Ls@?K?)w20xqNY;>GQ)Nt6TZR0{g?>c&cKkG4{!E4?gPt_ScbhItpBG8y?^wtOur)CnQ{_b)hL`?E+<`4|}Nb)h^L7DPycmLU}ZRr^e z4NV~d3f3VWCGD6kIhCV?AtvQBCwi=4B1%^CQ0cEYbZBF}f&Kup30?qSz(9jo4mu>* zvCEqHLt?&DN${^chD0&JEn)S_p+bZI6(Cy6^S&y@BV~QcOrz^rxal zpN3dPkv}I{I7hftmXo!-dkoBppTg;TGpa%p<0^n&%<0hJyJxL$E3kSqO9>&dnA8P?*v8>^r{;=0r{;70c^jL4O2wbDD(KOL&E?NGszr z;uj1RhfO!vi1Hb~7c}cFKdH0iZU{}0g5dZ87_NDSgbAEhpa$oBc-9ws+OUb6CG!|_ zmgxJ_2Gv%x+sk8F7%Cs7YrAjRw1M~4N5Lmwcu%>)Zve=h;Rz!U)+8^8ATCbB#p-@@ zT^nVsV^7n2Plk+*=b!A&m!moTxHo8}BjSxQ8*-rL8wiqfU>~tce0>2oM*D|H!MdN; z121fJCN5&0zB7pI)li%m#+##~`!I=TqX?CFm3V16@h3l>f{>_9+vm*@E&)tr_sJ&!W1Ih@)9MMiy3)ad0w-?Ev(Fzcl?-*`JSQUg;)r^1R?8 z3AuxKCOMB6X_pr%L4!YJ1)Hgy_(%^v&Ob7zS6pE!VJU5JCLm7s{@UWzlm(4`bDoF- zByD{iGkSu@7RD!wpBVB7l(TJf1}^$sEc8})x&vWkId~ef@18GqSP0<@r&b9;jGHV6 z1Y^((7NI|UNL^C*&`+Brr74|)2qP6%E<&{N1VPC}n$`V0D9kv_s9ZE+2q@>f!X^?; z(Gz)2p3y;H#6O^8)Zu{|J~eFUa)&ieh~`3|B_SB~6Ze@NW3uD}jteiN#AjXbqVFPkZuGIFVQvtBarU?G3>3}C2>#(t>0ZnISnry=5pwb*EH;!;l}%pQd0vE zK^K>TQXTlCQ6EKF@&D*l@N=5=T-VlcYGMv{Ty$*5jzz+6#hyDKm8MC%)SY;OOPNMi zK?g&|Q!6WV-<@J(5=vE#S0hCq<~5c8+7i%B2uA&N#^YA|g0h25U-+QGk*=|kEAqF` zTvxE-QPs8v{g6-^q6fj^U91VOG3^4zU-_)XNa7sJ;wj+?%Ai>T%hb*IR$xr#W|Dh+&nU7~3Zw$-0V6G! zzd^}mXz8aqO*(R8dXaG^8k8p7=lkP3cu@W(CF^HVfxl%GB20|IHCZh4YM%9ZRL~x} zmT4r9jYqC3j>gt^jWZlQkH*y(6j&UUY#f0^fzJpNXFk~W=eFjWI(8W2E(Z~5i=;Yv zbX+C&Zh~5z6W6XQ$51M_o-x2H9<$4-n`)DaSvC}f6(`^-Z1UGv8|u>$k+$}D?z2Ly zv2dhCG#c2i2dtCUGR>E)DIw}BMB+icl-*KbzIgb+UEA zzLA_sz&gfUE%@bMrSNj^g4{w=(#Sv}bBG60$E-Y} zt!%Q1!P4TH*5Tf!NLwLIm-?sIuOOA`bHFI>><%O6udFWah_No(cX2`_f#7)4I zhHm5CYfD!;k~#@gTbu9CIB32()C-WK9a|!fIn~2Z^4co`t_tupw8@CCT5w1O9u0Np zie_RMKUE$}dvQIWK%r15H6nr61!WPDzOTG2xOBGBtGcr>WGDjWlT{T4f;*xIZS`;4 z4Ig?E?#ZL08#KnbFRbsV%Ok(gadI#y^o-^hluMO}_rA^EP9*)PqC`WG)}y4$!u9A!T_dPHq&h$L^p`MxFVAb#Du+Lg zT`2XW!o<;nB6|-ALIeuVJ^w5A@D8i`V?hDfO_!uKfA!J2~yEs?(@t< zH;>=~8gltZ3>{+Bib(|!RMD0{R>RmQ&`p`F5a*v$9tR*a@~VcKeSLhHZ}0wC;D!_C zH-{kCuY4y*rw=>HzsTdi*(>*Sm0ZG*D_24c{O7Fe$}=`tzqWW$oQ-2-`K?9ceSuwQ%+wa;&njn}2K+pVwLV z{dMTOL|-7fGy)TH`AaNI;7O|^C`3950Ot9)oJ@QJNJZ)PyDta)FyOqp({y$e`_n-= z@sW-Ul9}SPw1E9qBI3{)gQlo=mxbW%ChMg z3H}h<8){>Zss7A(qG&1>rJBj;nzhl9YOHc0WZ=UyrpDjpVgnl5%XJjFctd*xMw%$w z=6I*|=`*=6Q8K%aLCw?qOXpO6Av_ZTbuW$kx_UphpGWbnGfc)LGBx#nh?DgF7(88} zKT_hJW|eyOAn)#H$v_btTWneI8Qum$lJ^+(})xyI?a+{ zY(U!a)JIMw0(lh;!8ayU_t}L0?|x`>sB>#J=@}tc?Kv(}KV4%bHhkA)e54BVpgD-5 zOFfi!%1H-9Sy=WGjgZdAgwGqD@gl7_ByJU9=R=3N7+=lbwhRT`C2~(lLe`$v|9n9L zREX}Qi;kXq5bsSG3&9dISXE%ud+1Nwe*&ibZo&1TVckPwZp&8WeMO0Q#OaAF5e3<$ zs?=`VVUBS`8N83YJ!}k2Bu;(e)2!%OuLSXOJ)PGp^nPKsZ(V?aDj5G@1b-XSFBMr>Dw)EJx?_5UegW_6VTBE5)w)xg4%b&Y~;(?b< z5G$b>s^AmDFj2IxqrR_NhKSI$-i+gpEIE4FLQe@bYEB6!9*SOQvn9l8A;se65e3T2 z%$9P7FENN|ySi(WmJ%FnKkS`n-W|b>ItP*TDPL%?_l2t0BK&aok3or^*Xyt7sOdbp z&nYdqu8h3C!%YxaK1N|-79kZ@i@-+q!BU>WpS@f0n>K%M*lQ=loDW7g_b}67^g#|0 zQJAIISLZD5^F=(XE%b>(qp^|Jas#M3oa5OkV=kkIoyA} z44KBXnZC4f-xh*)evEO-IeFkAoUDGUVT7{lh-t|1JWwMnqO^7q9%= zWwB0Y6d=1^86MyBudJd~1$v^<*nbxVp#SeH>wl%FfBHZEpY{4%)*u$qPsI$8;O{fu z|1^d8V@hVK<|!P2X^hfeD%ws@{oBx=J%et|Qwh1T4)mJ*J5>rc0cR*WcQ5)N9q#*o zx7uIli8dZ5P3-R$E9n1tNdEqQCMDP^Dmc#~L;sVzje86xTlKRP91Firas>vLDfznYj6wa*{yiFL72Fmd&-6aW5fYc?w_FjC&_R zbiSiZ74a^E$EAuw$V6q4Kd0f=rA>{`4)Iy#tCrq>Ro_o64HLrPOL@qC)yy>CTxYnR z-f$3zEah}irE2JEd2r+>WVspYlMDyQGx!F_`CwZ(4>xQFJ+AhyfugrOJO`zcT{RKF zh3EW!t@EWlcKxc zT0flM#l`-Z(Db`h=;9ct>-=VYXaP-$=lgRWTD(5BXi?kXRQJ*0Fie^(nU_P+H58kA z9}=0dqcLZ;-QYR*{vX%!&++#H+kYcRii6GVH9qRMd)@Bu1U^#^`6&+f#3k0{kgvJ1 z^($w0n7IrGU*8C>MM|d*>Cav94<^S;b9!$6EM3@XtP|MzZ->*mWTsWCSzCVZLH%$@ zeU8T73*@&hd=CL^RJr8&gKa1EZfV4hi3YifX~1?n@qX1W+tJ@i&A1O)c8PCnt9F@T z$=r4;7SN3KuJFTd_Y53&(8b`!5GsE(D#TtJV>_13yiQ{ZsW)t*T1A({BK+Eg(O%On zZN8PzT;EHWSMNrpb2IHUZ>?%kLxUFSj8$d3&cnUzjpzss^1>$LoM&sES_!8`oD&+> z1C7J;{6k~u|4Nc0NgCnczmVgsJsFs$MDFyOJ9FfFxG}HZ>C#_7BC%h-dv`5*RXKN> z!BXE|I^XAW)9dJd@u##td^I;rcH^O8L$d$kZj+%wv?p66<}1T3gHN?CfDFbm->ZZOkrRTSptc zR>N^mg1RzF?`o$IT8*W{{S20NvoGJ2t8Q(q(9IVSgBh=*mFk6-YQR}{ud%@UWVEy+ zn|plUb@rWo-jAu{w&^Xy2S;1`GHA&j+x&lKsZF?;xGrh{(DB~gXXl*XQ?h#9*S6+0 z&FsRBFYAb#NcHAR3Jdf4QFfL(Y&RV`7AQ^A-5Q_!NT;p`p?owC$w@%Zo6U2V2=;}c zG_OPU`&k=*;`?;V0Y*(~#OG-Zy_+HbgPy|b^Lr*!-;}MP*=;&}oiXH72G)dx#BK#h@0b^EgIjY+;9VtZF|;ep+!TmO#(;69mxR`VvCTC$mP`+xw+`q-Sr z30)pfqBp#Hf9cLlV|Q{e^bD~1r7IPbeWg=&qm0UJOgRz$2t)AAdM!Q{pS!0A0Zyfw|^CQ1Zj7W`NB#N6n?*kHu^SW^U@ zi>0_aJJB*XeB>yBn9<~D<453p&}i&J-5h&HgZMb%SRhIB`r`~;MbA#H9JfHUgu4+HpFr`2&WRn-Zh3kYC7Gp1CaZ!$IVJ z5bZ*%{5_&BhksH+{$3?%sWjkZr=+_irDOTs>QCAUjU8%&%}k?n?YtI+OwPy!3S`&F z7LW{VfO@)LMIVOvfz_wgLxQ_z!GrsuO+GhuU^11rx5|1v_h8Zc(A93v;wJn#{$AHV znpQkB(+FAaCZjoN_xPf7KC-)cZP9Cw<>CkW#f81Tot@CW~2$l5SIdzK$pc53R+OlfTuKzqc6?kkf-A7@XS$SKXZQ=-8w*r-Z$mAfdJ9 z#_^f!U)7zT%X<%i{;Nd;u&VxI-BDBvAe+^p(cT;+jFk7WwL=h(7?6Y8trWe|NVnO> zXYGSpU6J!SuT6*XF}vEs9RuCboSQb^ZY%(j6m595m?huRyqZ%ldO~!!l|!eERv39( zYeT`w0r#=^1(0AO@&7TIGQQ#L_`|Sk*W3WdKN%Vi^dWt4)YH#?pVBdG8ek=oDZ5%2 zl?~tFcu1J0zHHlUS3Zl@ziWy<`9mRT?tLUvy^60s^efNd!f$Wjs;3clk#}&qPXjoz zlr+}t{_>a$0vC!xH@q*TBIstj_L}?kD@n7W|Fz#yAP)`^^>s(WYdr7I6)B?%ntP59 zP_FNn+_&;`vnuz7==K&YoA6J1o>gb@hpfZCL9Fy3WX z$h8#yl>%27IGoq>nE}cVrs+(5xI)6Va~sjA)1JQ}Ctnox>a0JwFkZdu{befUXPk13 zj5WAvP~+q0y;tDAFhEv*cR)47-?~7}dhJa8I3y}@N*y8Yi0;0bW zhuh!KY5o1DaMP+=Cr9#F0rKEP{`}7uR*EBDzv4BjH$t57Uxmf?mok2)Sd8TmVL%s( zV4Z@%;xE!~XF2LJqXTQh*@ozp?T3W78`SCp%*pTD`SJZMuJonoaO?U06Hy`P0C0S) zWxO{~ZkpfVQi#62KJ+Ylr$P0vcF*+1f057i6TYH-@Sl}qBoXh$*6nO!%da|cjQz_~ zd`AyPXLlOSXSch#hSo}8BT%X zT27qo-tGkV%9jpW#Q$unOf^(yInM1l_mDZ8LfL~L!rYqk{qFk8>GZ7-+){F*n~RKX zFmvz~guTZH=W}`SI>WWp51=8+8O+Sr+*~w|D~5vX&U*n&AaBi9+3l7pzByAY!q{+GW1>!K96>j#g*s7wE*nEx{7LP7U=r_i~N zMCO#qsWL1@dEK^$C#4%{RJY9a}dHw4K3?oHxDfvf4 zBhG2dLA#ju0O(S`9RP`Q^3fG2$tM&6t=#4tEhxCBS6{lr@EN|kTk~}uQSsQQS%g3T z)e#+@1R2K`j2|`syM0|9U8FX}y2;uZ7d?_PBc! zYRmDDr34+{=(%c)$jt|fT+3mM$E!pX#-)uA^K{s6xup^AZpxE9DCyM+aCx<<{i-K& za3I_$K9uEXSlJsGF7w3j%!yWC$RI29%ke&9S4o$8CM`G`)cD0^7f_!XdIqLzzK-J6Fcah8vDuOUmvyOzV^93 z?Il=QdS91fU-wcr58z{dL_`4nwCoMK_RR1Y;~|Raspuli{M+?MUbEJD<{+=HW(;e1 z!4VKoAn{V77bKzF`CGOn_b+E7$*;n?x+z%HQx(V%|1h~d2&L+#J2xZHe`f)3_|f4ABK# zQML6qCLJG`PJ+Oh_NUkIQrOu{J?X?avtiX13?)`*A3ifbeABf*X7-7sMX4g(4+Z5+ zXWYPx!%FM%0*i55QnCyX?s`fc-L?&;3cQA`gHn^&L_D*zT{DUIi>6gAYb`VzPm{!A zH<#N(nT~7x4?*g``w&JF3#}1BH%NTCR173y&_cP$A!*~Goew?I+=`1Ia}MwCFPept z9yL1P9HQY;?-_0J zmJSkDT!FQipkYvTa>(s4ASTBk9W`JR{TB(kZjVVB-kdKP=cT#HwHbIUKmNwx={ zj77Cm5Z^N%VlJ%hWY`dQ=;fm>CdR&f{T%Mu7kaPx_Sy}K#a@$nI2RNFWD+%3S^i_Y zv7uE`;%v=8@kd=ohDTdw6q6^4T$z2-bzcZSh*Qn2QP;ea+#4mF&lFdW^0~aYTcwGe z*LRn9G4vI9|0MUg!~vcZ#XT%)7=OR$o#*Xq+BM)NO)c=JnL^m(o6t&F@2eO^x7|`v zWk@E*K^MizP3#(rv>q_Wl#xnzPpTe!=D8{Fq^Zv%4`D}StPtye3>a1={@JA0&vp?? zlDYH+D=-l#8_v3;K@#VN9Dv%KkF}5plgn4Ne~r5K!*4E536H^*^}q+)G2j+tL!IuH zEYn$%XiO{~p}=F`N55y?zE553;nVZZ7@<_pJNFAH_un)F6b=oeqM^R8g1)3<9E(0& zMeCZ!tEXBK3*+wiMRqXup2PJoMEm*<3c2@f@k1I*C`Gug2{L8d`H$qh z)t=IK*MA@l*}Z{*a{{JJOvm5@muTSvQ*dD1LW^9h_R`oRnws!8RKT=*h}iX%zf3^$ zDh+n;84rXdo22`C)v@|_TUgw$e{-I*Q+*u`2)`OEU+b&iOz-DikEzlu75)0*n1^oh zXP(AQ>@DJb)yq630RoHGL1pQzI>c~t!U5y;xWKz zr6mEhQhRB}qJQbop6(_Saw}3~h>E}0R=*5tm%MxIdwVSX@L3TpNEWG?WF^)6pq30K z+OfkT|6NQm1J#77mzGh79^B|U;-&zqN9VscClEJ9H6ValapLbvCFu>DoeCZd# zdm)hjmT4TG6s7UKG|rf&_TKvvZx7zf;&gc_BZBk)s^!eXp=|d!uEr8s<|RwAL=3Mb zgpkQvBn%n5ye5XBys|YYc`#!cjBRknG`6C$G_vPKMI(`|7umNFz4k#QlJivOT%AA8 zb*^)+>-7Kcx_-~|+|PaAzx#WCeqY5jeXpv$tx=V=#AaWZZcLzTmH)TNR0##^wzR1G z?Bm!`jq8E+n{RzqwrAvSbsYkIHIgK90vde@CRcXU)c3FSR2`^D+X z^o!`jr`J;IJGB~>Y?D*n!X?);yCiJl*uszeVlXy$r_R6W=76nv8+`8F(afjX!GD0I z&Y3qdk4Fn@@r0SijqZ&G8`S_5EUN!94hG@h7b;htAFHn#^hNDV^Ia|Cd7np3v=jxG zkjH~hmH==O(~G5qf^Emg_B*@NG~YVDm_%w-Yk@u*3HNAAFlouo15m~5)sQ9mUt@ad zsrz6P{Q{|AJgU%PxAY5=2+<95H`0|sw5@=h$|VI|@iSTLm$}^*taibsa5ks1*5>|I z_JkTm)HX;35I8GS2I2Wd&G5qhGAH%PTn3;>LMgV8pPu}bz<#)kct{=*?m^wI-L6cS zTI|wUQc^g#bY=uivK%|JST+-O#in7oc66N*#kE%b5v)UdL!l1`mUchAuR=ffnFE;{ z70z?aKX=ay`7ln8mc=ZNH^emTuFS8jwP9YNPe0TBE1x1jr?y;vC|1N=9j?kI^8n5S zACId%QYA<9T3@#M;)-$l!v!;S=?rz2cA9tyxVOQ|m_*bqHJ8MoOCYuT4fR0_$K}0Z zIz_F2VgoHSkDf#g}Tc@ElnWKfrE8X#Us$jLpdTTJgeLl@pB26zK z)v8B-BGCjECWNt<(?zFa06(aVsuaQGxY`Y+ntmI-_zNa=B*{54MIQ69hMs7TEsROoJLN8e;rGsTY zsIw;sSx_`f|3pO|5sl~;o*$_xF4L~6@K7@yN>n*1f3V$l6&Kxeh?3EVC>26iA_rEA zt#v4xV2#{QhMEgiFgl%Pf#^K*)rl_G_ua!>MNr%r)f~X#j(6X=Nn;n##auXYw>O(a zNV8i2$+eZ-usPu9M^Mdf0B7|BXkz-LWZ1j32*IvlL2o1GE&14oC!z%eBuI-O!nDk$ zYMHnn#f8Kd@)yM=O0G+iMT8-qeVLfsifm*1_$oB z$G6)D11VPN3h9twHKLq~gHutq_O)&%jEdudK&pMcIXO653is$pjJVW)u>YfF;cT577*?)52w^P@XkpxHfdU7BZ6G)C6~fL zXO(2}zvq_D7*IsytS_g^6py`EGk-l*`b4-2kjP&+yAmFDUaQkq%+JW|D z@zQFjN~5YVc3eYn^R|g?YnSY@ua=JuNW7*L4wBr=biFJx{R>yQGI1Kd`{UmC2&c?6 z|3)adk-l%P(v@mfN)R>~3JC7f>_d+RxeU@=1?mQWQ{GFa#z|b25^ysgirtWj~E*mR~B^;Gu~6ak>C00A=C{I??C7qAnB4V+@oX; z9?^{L4%CFvAr{k-Xsm(2iR8A4&Nh1&Ji-D6oi9F2k+eX1!=NH+Jnnvq=jwMbvVk1S#sbI5jRZSikOthD4DSq3m=B|7V5q0vs0 zP{!rbOFZ9o7+%TX{4K_Nq}s)KQf7fUasewF-b+34Bf+yKZT*==yk=-+SZJ|Gt`?R~bZi!X>dkHAQN`acOr$*!L zqwYyM*Oa`yisGGyR&Ooe+1B@LIcmWeo!1m&)Xi%O>?A9G5t9b!uNG=A-r|2++I5>2 zD6gWSOrCYAqbW2bjaM$voK6z+7AdiYv?rfh)QD3-k>xC3WQw4e^n4v3aDoxh;A@s6 zD!ql6mhmAs`!9&FO_fUtBw8iLp3gyRiYhWx0WmSB3Sr|V3B$uSW!EpN!8LhDM2}nuk%Ot121Lo@9VB!ci+>5m_3n8?8zlYIEZgPQIM{vRqFsEH9lfo;HXDCb9Lzxr1(EePVc^)J%d zKf-|jGVW)E0f+w6)BUWhKav9f=9~QdtUr NameLast ModifiedSizeDescription
- subfolder/ - -   + subfolder/ - -   testfile.txt13-January-2021 22:3519 B  From 75e6e35e3c3be9c77a25cfd90534408717340e25 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Feb 2021 10:48:29 -0500 Subject: [PATCH 146/274] Changed the RestAssured tests to reflect the API changes. #7084 --- src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 9 ++++----- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 953989369e7..8d4369fa85b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -1470,8 +1470,7 @@ public void test_ProcessShapeFilePackage() { } /* - First test for the new "crawlable file access" API - (#7084; work in progress) + First test for the new "crawlable file access" API (#7084) */ @Test public void test_CrawlableAccessToDatasetFiles() { @@ -1514,16 +1513,16 @@ public void test_CrawlableAccessToDatasetFiles() { // maybe upload a few more files, in more folders, // and try an actual recursive crawl of a full tree - ? - // Make some calls to the "/fileaccess API: + // Make some calls to the "/dirindex API: // (note that this API outputs HTML!) // Expected values in the output: String expectedTitleTopFolder = "Index of folder /"; String expectedLinkTopFolder = folderName + "/"; - String expectedLinkAhrefTopFolder = "/api/datasets/"+datasetId+"/versions/:draft/fileaccess?folder=subfolder"; + String expectedLinkAhrefTopFolder = "/api/datasets/"+datasetId+"/dirindex/?version=:draft&folder=subfolder"; String expectedTitleSubFolder = "Index of folder /" + folderName; - String expectedLinkAhrefSubFolder = "/api/access/datafile/" + dataFileId; + String expectedLinkAhrefSubFolder = "/api/access/datafile/" + folderName + "/" + dataFileId; // ... with no folder specified: // (with just the one file above, this should show one folder only - "subfolder", and no files) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9919a1c107e..2b6bbaf0caa 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -646,9 +646,9 @@ static Response uploadFileViaNative(String datasetId, String pathToFile, String static Response getCrawlableFileAccess(String datasetId, String folderName, String apiToken) { RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken); - String apiPath = "/api/datasets/" + datasetId + "/versions/:draft/fileaccess"; + String apiPath = "/api/datasets/" + datasetId + "/dirindex?version=:draft"; if (StringUtil.nonEmpty(folderName)) { - apiPath = apiPath.concat("?folder="+folderName); + apiPath = apiPath.concat("&folder="+folderName); } return requestSpecification.get(apiPath); } From 9c07284cf7b0a870b9eedf2d839cbebc75e5dbf7 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Thu, 4 Feb 2021 11:49:39 -0500 Subject: [PATCH 147/274] Added js+css workarounds to get the checksum to display in Bootstrap tooltips [ref #7081, #6685, #5210] --- src/main/webapp/file-info-fragment.xhtml | 24 ++++++++++++------- .../resources/js/dv_rebind_bootstrap_ui.js | 3 +++ src/main/webapp/search-include-fragment.xhtml | 20 +++++++++------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 9a0b286d9e8..9a2add76147 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -71,24 +71,32 @@

- #{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} - + title="#{bundle['file.metaData.checksum.copy']}<br/>#{fileMetadata.dataFile.checksumValue}">#{fileMetadata.dataFile.checksumType}: #{fileMetadata.dataFile.checksumValue} +
- #{fileMetadata.dataFile.unf} - + title="#{bundle['file.metaData.checksum.copy']}<br/>#{fileMetadata.dataFile.unf}">#{fileMetadata.dataFile.unf} +
diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 5b8986111f6..b027d75cef5 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -186,6 +186,9 @@ function checksumTruncate(){ } } }); + $('span.checksum-tooltip').on('inserted.bs.tooltip', function () { + $("body div.tooltip-inner").css("word-break", "break-all"); + }); } function clickCopyClipboard(){ diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index 1b875fea375..0167a53ca48 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -614,25 +614,29 @@ - #{result.fileChecksumType}: #{result.fileChecksumValue} - #{result.fileChecksumType}: #{result.fileChecksumValue} + + data-clipboard-text="#{result.fileChecksumValue}" title="#{bundle['file.metaData.checksum.copy']}<br/>#{result.fileChecksumValue}"> - #{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)} - #{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)} + + data-clipboard-text="#{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)}" title="#{bundle['file.metaData.checksum.copy']}<br/>#{SearchIncludeFragment.tabularDataUnfDisplay(result.entity)}">
From 638c30dccbffeed0c85c40da3dcaec7d7bb4d1a7 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 4 Feb 2021 11:49:51 -0500 Subject: [PATCH 148/274] update naming --- doc/release-notes/4259-java-11.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/4259-java-11.md b/doc/release-notes/4259-java-11.md index eada46f3751..1e5ee19230a 100644 --- a/doc/release-notes/4259-java-11.md +++ b/doc/release-notes/4259-java-11.md @@ -19,13 +19,13 @@ There are some things to note and keep in mind regarding the move to Java 11: The `remove` command may provide an error message if -headless isn't installed. - We targeted and tested Java 11, but 11+ will likely work. Java 11 was targeted because of its long term support. -- If you're moving from a Dataverse installation that was previously running Glassfish 4.x (typically this would be Dataverse 4.x), you will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if your first installed Dataverse version was running Payara 5.x (typically Dataverse 5.x). +- If you're moving from a Dataverse installation that was previously running Glassfish 4.x (typically this would be Dataverse Software 4.x), you will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if your first installed Dataverse version was running Payara 5.x (typically Dataverse Software 5.x). ### New JVM and DB Options #### New JVM Options -For installations that were previously running on Dataverse 4.x, the following JVM Options have been added in support of Java 11: +For installations that were previously running on Dataverse Software 4.x, the following JVM Options have been added in support of Java 11: - `[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED` - `[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED` @@ -57,7 +57,7 @@ For installations that were previously running on Dataverse 4.x, the following J - remove the generated directory: `rm -rf /payara/domains/domain1/generated` -3\. (only required for installations previously running on Dataverse 4.x) +3\. (only required for installations previously running on Dataverse Software 4.x) In domain.xml: From e1b691b550bfcb011ae8ce38c38fa61ae703d9f8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 4 Feb 2021 16:10:01 -0500 Subject: [PATCH 149/274] test passing FacesContext --- .../impl/FinalizeDatasetPublicationCommand.java | 2 +- .../command/impl/PublishDatasetCommand.java | 5 ++++- .../dataverse/workflow/WorkflowServiceBean.java | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 7a32f515d36..df389e74f1a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -183,7 +183,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.workflows().getDefaultWorkflow(TriggerType.PostPublishDataset).ifPresent(wf -> { try { - ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased)); + ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased), null); } catch (CommandException ex) { logger.log(Level.SEVERE, "Error invoking post-publish workflow: " + ex.getMessage(), ex); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index ae555faee58..5141325cdcc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -20,6 +20,9 @@ import java.util.List; import java.util.Optional; import java.util.logging.Logger; + +import javax.faces.context.FacesContext; + import static java.util.stream.Collectors.joining; import static edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetResult.Status; @@ -91,7 +94,7 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException // We start a workflow theDataset = ctxt.em().merge(theDataset); ctxt.em().flush(); - ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased)); + ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased), FacesContext.getCurrentInstance()); return new PublishDatasetResult(theDataset, Status.Workflow); } else{ diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index bf9eb68612b..a8ce879a3e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -31,6 +31,8 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; +import javax.faces.application.FacesMessage; +import javax.faces.context.FacesContext; import javax.inject.Inject; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -97,9 +99,21 @@ public WorkflowServiceBean() { * @throws CommandException If the dataset could not be locked. */ @Asynchronous - public void start(Workflow wf, WorkflowContext ctxt) throws CommandException { + public void start(Workflow wf, WorkflowContext ctxt, FacesContext facesCtxt) throws CommandException { ctxt = refresh(ctxt, retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(ctxt.getRequest().getAuthenticatedUser())); lockDataset(ctxt); + if (facesCtxt != null) { + try { + facesCtxt.addMessage(null, new FacesMessage("Starting Workflow")); + } catch (Throwable t) { + logger.warning(t.getLocalizedMessage()); + logger.warning(t.getClass().toString()); + + t.printStackTrace(); + } + } else { + logger.warning("FacesCtxt was null"); + } forward(wf, ctxt); } From c3d8bbfbd4552298bf380c6c12af69f31bafb8b7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 10:50:28 -0500 Subject: [PATCH 150/274] Revert "test passing FacesContext" This reverts commit e1b691b550bfcb011ae8ce38c38fa61ae703d9f8. --- .../impl/FinalizeDatasetPublicationCommand.java | 2 +- .../command/impl/PublishDatasetCommand.java | 5 +---- .../dataverse/workflow/WorkflowServiceBean.java | 16 +--------------- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index df389e74f1a..7a32f515d36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -183,7 +183,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.workflows().getDefaultWorkflow(TriggerType.PostPublishDataset).ifPresent(wf -> { try { - ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased), null); + ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased)); } catch (CommandException ex) { logger.log(Level.SEVERE, "Error invoking post-publish workflow: " + ex.getMessage(), ex); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index 5141325cdcc..ae555faee58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -20,9 +20,6 @@ import java.util.List; import java.util.Optional; import java.util.logging.Logger; - -import javax.faces.context.FacesContext; - import static java.util.stream.Collectors.joining; import static edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetResult.Status; @@ -94,7 +91,7 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException // We start a workflow theDataset = ctxt.em().merge(theDataset); ctxt.em().flush(); - ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased), FacesContext.getCurrentInstance()); + ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased)); return new PublishDatasetResult(theDataset, Status.Workflow); } else{ diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index a8ce879a3e7..bf9eb68612b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -31,8 +31,6 @@ import javax.ejb.Stateless; import javax.ejb.TransactionAttribute; import javax.ejb.TransactionAttributeType; -import javax.faces.application.FacesMessage; -import javax.faces.context.FacesContext; import javax.inject.Inject; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -99,21 +97,9 @@ public WorkflowServiceBean() { * @throws CommandException If the dataset could not be locked. */ @Asynchronous - public void start(Workflow wf, WorkflowContext ctxt, FacesContext facesCtxt) throws CommandException { + public void start(Workflow wf, WorkflowContext ctxt) throws CommandException { ctxt = refresh(ctxt, retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(ctxt.getRequest().getAuthenticatedUser())); lockDataset(ctxt); - if (facesCtxt != null) { - try { - facesCtxt.addMessage(null, new FacesMessage("Starting Workflow")); - } catch (Throwable t) { - logger.warning(t.getLocalizedMessage()); - logger.warning(t.getClass().toString()); - - t.printStackTrace(); - } - } else { - logger.warning("FacesCtxt was null"); - } forward(wf, ctxt); } From edee04e3d98804adf86dc93b1d7966d3442e00d5 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Feb 2021 10:52:38 -0500 Subject: [PATCH 151/274] #7205 fix dataset page load --- .../java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 69e35b8e8c8..12a2cf58feb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -469,8 +469,8 @@ public static String getDownloadSize(DatasetVersion dsv, boolean original) { long bytes = 0l; for (FileMetadata fileMetadata : dsv.getFileMetadatas()) { DataFile dataFile = fileMetadata.getDataFile(); - if (original && dataFile.isTabularData()) { - bytes += dataFile.getOriginalFileSize(); + if (original && dataFile.isTabularData()) { + bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); } else { bytes += dataFile.getFilesize(); } From dba965ccc5caf4046a2fcf9e3aa0f30dd10971f1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 11:39:23 -0500 Subject: [PATCH 152/274] update to use a foreign-key reference to datasetlock, add flyway --- .../iq/dataverse/PermissionServiceBean.java | 2 +- .../workflow/PendingWorkflowInvocation.java | 16 +++++++++------- .../iq/dataverse/workflow/WorkflowContext.java | 17 +++++++++-------- .../dataverse/workflow/WorkflowServiceBean.java | 6 +++--- .../db/migration/V5.4.0.1__7563-workflow.sql | 4 ++++ 5 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index aaf38af1b36..f38efa7b66b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -816,7 +816,7 @@ public boolean isMatchingWorkflowLock(Dataset d, String userId, String invocatio if (invocationId != null) { PendingWorkflowInvocation pwfi = workflowService.getPendingWorkflow(invocationId); for (DatasetLock dl : d.getLocks()) { - if (dl.getId().equals(pwfi.getLockId()) && pwfi.getUserId().equals(userId)) { + if (dl.equals(pwfi.getLock()) && pwfi.getUserId().equals(userId)) { return true; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java index 6efccf4de03..227176dc4b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.workflow; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.User; @@ -52,7 +53,8 @@ public class PendingWorkflowInvocation implements Serializable { int typeOrdinal; boolean datasetExternallyReleased; - long lockId; + @OneToOne + DatasetLock lock; /** Empty constructor for JPA */ public PendingWorkflowInvocation(){ @@ -70,13 +72,13 @@ public PendingWorkflowInvocation(Workflow wf, WorkflowContext ctxt, Pending resu localData = new HashMap<>(result.getData()); typeOrdinal = ctxt.getType().ordinal(); datasetExternallyReleased=ctxt.getDatasetExternallyReleased(); - lockId=ctxt.getLockId(); + lock=ctxt.getLock(); } public WorkflowContext reCreateContext(RoleAssigneeServiceBean roleAssignees) { DataverseRequest aRequest = new DataverseRequest((User)roleAssignees.getRoleAssignee(userId), IpAddress.valueOf(ipAddress)); final WorkflowContext workflowContext = new WorkflowContext(aRequest, dataset, nextVersionNumber, - nextMinorVersionNumber, WorkflowContext.TriggerType.values()[typeOrdinal], null, null, datasetExternallyReleased, invocationId, lockId); + nextMinorVersionNumber, WorkflowContext.TriggerType.values()[typeOrdinal], null, null, datasetExternallyReleased, invocationId, lock); return workflowContext; } @@ -160,10 +162,10 @@ public void setTypeOrdinal(int typeOrdinal) { this.typeOrdinal = typeOrdinal; } - public long getLockId() { - return lockId; + public DatasetLock getLock() { + return lock; } - public void setLockId(long lockId) { - this.lockId = lockId; + public void setLockId(DatasetLock lock) { + this.lock = lock; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java index 76ca212976f..6208acb0999 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.workflow; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; @@ -31,7 +32,7 @@ public enum TriggerType { private final ApiToken apiToken; private final boolean datasetExternallyReleased; private Map settings; - private long lockId = -1; + private DatasetLock lock = null; private String invocationId = UUID.randomUUID().toString(); @@ -47,7 +48,7 @@ public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersi } public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersionNumber, - long nextMinorVersionNumber, TriggerType type, Map settings, ApiToken apiToken, boolean datasetExternallyReleased, String invocationId, Long lockId) { + long nextMinorVersionNumber, TriggerType type, Map settings, ApiToken apiToken, boolean datasetExternallyReleased, String invocationId, DatasetLock lock) { this.request = request; this.dataset = dataset; this.nextVersionNumber = nextVersionNumber; @@ -60,8 +61,8 @@ public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersi if(invocationId!=null) { setInvocationId(invocationId); } - if(lockId != null) { - this.setLockId(lockId); + if(lock != null) { + this.setLock(lock); } } @@ -108,11 +109,11 @@ public ApiToken getApiToken() { public boolean getDatasetExternallyReleased() { return datasetExternallyReleased; } - public long getLockId() { - return lockId; + public DatasetLock getLock() { + return lock; } - public void setLockId(long lockId) { - this.lockId = lockId; + public void setLock(DatasetLock lock) { + this.lock = lock; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index bf9eb68612b..2c20bd54151 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -282,7 +282,7 @@ void lockDataset( WorkflowContext ctxt ) throws CommandException { em.persist(datasetLock); //flush creates the id em.flush(); - ctxt.setLockId(datasetLock.getId()); + ctxt.setLock(datasetLock); } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) @@ -295,7 +295,7 @@ void unlockDataset( WorkflowContext ctxt ) throws CommandException { lockCounter.setParameter("datasetId", ctxt.getDataset().getId()); List locks = lockCounter.getResultList(); for(DatasetLock lock: locks) { - if(lock.getReason() == DatasetLock.Reason.Workflow && lock.getId()==ctxt.getLockId()) { + if(lock.getReason() == DatasetLock.Reason.Workflow && lock==ctxt.getLock()) { logger.fine("Removing lock"); em.remove(lock); } @@ -458,7 +458,7 @@ private WorkflowContext refresh( WorkflowContext ctxt, Map setti */ WorkflowContext newCtxt =new WorkflowContext( ctxt.getRequest(), em.merge(ctxt.getDataset()), ctxt.getNextVersionNumber(), - ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLockId()); + ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLock()); return newCtxt; } diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql new file mode 100644 index 00000000000..a7ee9b8478b --- /dev/null +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -0,0 +1,4 @@ +ALTER TABLE pendingworkflowinvocation +ADD COLUMN IFNOTEXISTS datasetlock_id bigint, +ADD CONSTRAINT fk_pendingworkflowinvocation_datasetlock_id FOREIGN KEY (datasetlock_id) + REFERENCES datasetlock (id); \ No newline at end of file From 80f886e48c6655a7bd87ceed530e25ec668e0b5f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 11:58:18 -0500 Subject: [PATCH 153/274] fix name --- src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index a7ee9b8478b..6a50e89c1ea 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,4 +1,4 @@ ALTER TABLE pendingworkflowinvocation -ADD COLUMN IFNOTEXISTS datasetlock_id bigint, -ADD CONSTRAINT fk_pendingworkflowinvocation_datasetlock_id FOREIGN KEY (datasetlock_id) +ADD COLUMN IFNOTEXISTS lock_id bigint, +ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (datasetlock_id) REFERENCES datasetlock (id); \ No newline at end of file From 77677625ed83b0ccb1bd0eb5fc75369cc7c1b394 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 11:59:25 -0500 Subject: [PATCH 154/274] one more --- src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index 6a50e89c1ea..8b852f840bb 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,4 +1,4 @@ ALTER TABLE pendingworkflowinvocation ADD COLUMN IFNOTEXISTS lock_id bigint, -ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (datasetlock_id) +ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) REFERENCES datasetlock (id); \ No newline at end of file From 7527eb461de91f5eb0708ef243ac64f57546754a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Feb 2021 12:14:36 -0500 Subject: [PATCH 155/274] #7205 suppress orig button if size 0 B --- src/main/webapp/dataset.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 34b49a8658c..4f23d4c7b44 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -158,7 +158,7 @@ -
  • +
  • #{bundle.downloadOriginal} From dc50df56060f56b5f166462dcf065e5f06ee8627 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 5 Feb 2021 12:27:31 -0500 Subject: [PATCH 156/274] adding release note --- doc/release-notes/7205-orig-file-size.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/release-notes/7205-orig-file-size.md diff --git a/doc/release-notes/7205-orig-file-size.md b/doc/release-notes/7205-orig-file-size.md new file mode 100644 index 00000000000..2f7af23b6f3 --- /dev/null +++ b/doc/release-notes/7205-orig-file-size.md @@ -0,0 +1,11 @@ +## Notes to Admins + +Beginning in Dataverse Software 4.10, the size of the saved original file (for an ingested tabular datafile) was stored in the database. For files added before this change, we provide an API that retrieves and permanently stores the sizes for any already existing saved originals. See [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) for more information. + +This was documented as a step in previous release notes, but we are noting it in these release notes to give it more visibility. + +## Upgrade Instructions + +X./ Retroactively store original file size + +Use the [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) to ensure that the sizes of all original files are stored in the database. \ No newline at end of file From 87e89a2fc222fda152a9e0e440ff80c344bd8a68 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 12:30:41 -0500 Subject: [PATCH 157/274] spaces --- src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index 8b852f840bb..fb3714bb573 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,4 +1,4 @@ ALTER TABLE pendingworkflowinvocation -ADD COLUMN IFNOTEXISTS lock_id bigint, +ADD COLUMN IF NOT EXISTS lock_id bigint, ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) REFERENCES datasetlock (id); \ No newline at end of file From 911de9a4817aa66670414409202d78611126e278 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 12:51:03 -0500 Subject: [PATCH 158/274] make idempotent/fix syntax --- .../resources/db/migration/V5.4.0.1__7563-workflow.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index fb3714bb573..246acca3964 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,4 +1,10 @@ ALTER TABLE pendingworkflowinvocation ADD COLUMN IF NOT EXISTS lock_id bigint, -ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) - REFERENCES datasetlock (id); \ No newline at end of file + +DO $$ +BEGIN +IF NOT EXISTS (SELECT 1 FROM information_schema.constraint_column_usage WHERE constraint_name='fk_pendingworkflowinvocation_lock_id') +THEN +ALTER TABLE pendingworkflowinvocation ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) REFERENCES datasetlock (id); +END IF; +END$$; \ No newline at end of file From 127977e6bfd7311e748c3f98338c7dc0018a4f95 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 13:11:34 -0500 Subject: [PATCH 159/274] one more typo --- src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index 246acca3964..57ecec383a7 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,5 +1,5 @@ ALTER TABLE pendingworkflowinvocation -ADD COLUMN IF NOT EXISTS lock_id bigint, +ADD COLUMN IF NOT EXISTS lock_id bigint; DO $$ BEGIN From 5fe37496231a16d3178850dedcea8f3e44fd4058 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 5 Feb 2021 13:50:33 -0500 Subject: [PATCH 160/274] Had to remove a negative z-index and add a position:relative to the content block to fix footer overlap bug [ref #7558] --- src/main/webapp/resources/css/structure.css | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 605342db8d7..e70bb2f7b93 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -14,8 +14,8 @@ body .ui-widget {font-size: inherit;} .ui-widget-content a:hover, .ui-widget-content a:focus {color: #23527C;} /* -------- TEMPLATE -------- */ -/* TODO FIX DROPDOWN MENU LAYERED UNDER FOOTER */ -#content {padding-bottom:0px;} +/* STICKY FOOTER VS DROPDOWN MENU POSITION + Z-INDEX */ +#content {position:relative; z-index:50;} .panelgridHeaderTable {width:100%;} .panelgridHeaderTable .ui-widget-content, .panelgridHeaderTable .ui-widget-content > td {border:0; background:none;} .panelgridHeaderTable .ui-widget-content > td:last-child table {margin-left:auto;} @@ -63,7 +63,7 @@ body .ui-widget {font-size: inherit;} #status-alert {margin-top:0; margin-bottom:0;} #status-alert div.alert {border:0; box-shadow:none;} -#footer {position:absolute; z-index:-1; bottom:0; width:100%; height:60px; padding-bottom:100px; color:#808080;} +#footer {position:absolute; bottom:0; width:100%; height:60px; padding-bottom:100px; color:#808080;} #footer.widget-view {position:fixed; left:0; bottom:0; margin:0; padding:4px 0 0 0; min-height:44px; height:auto; background:#fff;} #footer .poweredbylogo {text-align:right;} #footer .poweredbylogo span {font-size:.85em;margin-right:.3em;} From 1a2d175f1968f4734b33fc4a669ba85fab5c716c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 15:29:31 -0500 Subject: [PATCH 161/274] fix/cleanup lock deletion removes the pendingworkflowinvocation from the db, but not the context, then, in workflowCompleted, the lock is deleted and we refresh the context to make sure the dataset is up-to-date before passing on to Finalize --- .../workflow/PendingWorkflowInvocation.java | 4 +++- .../workflow/WorkflowServiceBean.java | 20 ++++++------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java index 227176dc4b5..3ec83fa8036 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java @@ -10,6 +10,8 @@ import java.io.Serializable; import java.util.HashMap; import java.util.Map; + +import javax.persistence.CascadeType; import javax.persistence.ElementCollection; import javax.persistence.Entity; import javax.persistence.FetchType; @@ -165,7 +167,7 @@ public void setTypeOrdinal(int typeOrdinal) { public DatasetLock getLock() { return lock; } - public void setLockId(DatasetLock lock) { + public void setLock(DatasetLock lock) { this.lock = lock; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index 2c20bd54151..46b9ac5cfe4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -154,6 +154,7 @@ private Map retrieveRequestedSettings(Map requir @Asynchronous public void resume(PendingWorkflowInvocation pending, String body) { em.remove(em.merge(pending)); + em.flush(); doResume(pending, body); } @@ -287,19 +288,9 @@ void lockDataset( WorkflowContext ctxt ) throws CommandException { @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) void unlockDataset( WorkflowContext ctxt ) throws CommandException { - /* Since the lockDataset command above directly persists a lock to the database, - * the ctxt.getDataset() is not updated and its list of locks can't be used. Using the named query below will find the workflow - * lock and remove it (actually all workflow locks for this Dataset but only one workflow should be active). - */ - TypedQuery lockCounter = em.createNamedQuery("DatasetLock.getLocksByDatasetId", DatasetLock.class); - lockCounter.setParameter("datasetId", ctxt.getDataset().getId()); - List locks = lockCounter.getResultList(); - for(DatasetLock lock: locks) { - if(lock.getReason() == DatasetLock.Reason.Workflow && lock==ctxt.getLock()) { - logger.fine("Removing lock"); - em.remove(lock); - } - } + + em.remove(em.merge(ctxt.getLock())); + em.flush(); } @@ -319,7 +310,8 @@ private void workflowCompleted(Workflow wf, WorkflowContext ctxt) { try { if ( ctxt.getType() == TriggerType.PrePublishDataset ) { unlockDataset(ctxt); - + ctxt.setLock(null); + ctxt = refresh(ctxt); //Now lock for FinalizePublication - this block mirrors that in PublishDatasetCommand AuthenticatedUser user = ctxt.getRequest().getAuthenticatedUser(); DatasetLock lock = new DatasetLock(DatasetLock.Reason.finalizePublication, user); From 9ee84f482d042c839efd1d6c7eec9d12651aee5b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 16:55:31 -0500 Subject: [PATCH 162/274] fix editMetadata wrappedResponse handling --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 655cdafe04c..9826ecfe795 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -771,8 +771,12 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) throws WrappedResponse{ Boolean replaceData = replace != null; - + try { DataverseRequest req = createDataverseRequest(findUserOrDie()); + } catch (WrappedResponse ex) { + logger.log(Level.SEVERE, "Edit metdata error: " + ex.getMessage(), ex); + return ex.getResponse(); + } return processDatasetUpdate(jsonBody, id, req, replaceData); } From 308bae13d425b1faafaac37864bbe37941495194 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 16:56:57 -0500 Subject: [PATCH 163/274] log the stack trace for Throwable/500 errors --- .../harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java index 7b7f1f08fad..d0e1637243a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/ThrowableHandler.java @@ -27,6 +27,7 @@ public class ThrowableHandler implements ExceptionMapper{ @Override public Response toResponse(Throwable ex){ + ex.printStackTrace(); return JsonResponseBuilder.error(Response.Status.INTERNAL_SERVER_ERROR) .randomIncidentId() .internalError(ex) From 07dff1a2a9e24ce3bca970593bdbaa00f095549d Mon Sep 17 00:00:00 2001 From: Stefan Kasberger Date: Fri, 5 Feb 2021 23:05:14 +0100 Subject: [PATCH 164/274] Update pyDataverse section --- doc/sphinx-guides/source/api/client-libraries.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index 64726c77f58..e611f6b2565 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -13,7 +13,8 @@ Python There are two Python modules for interacting with Dataverse APIs. -`pyDataverse `_ had its initial release in 2019 and can be installed with ``pip install pyDataverse``. The module is developed by `Stefan Kasberger `_ from `AUSSDA - The Austrian Social Science Data Archive `_. +`pyDataverse `_ helps with the Dataverse API's and data types (Dataverse, Dataset, Datafile). It's intention is to help with data migrations, DevOps activities like testing and configuration and to build services upon. +The module is developed by `Stefan Kasberger `_ from `AUSSDA - The Austrian Social Science Data Archive `_. `dataverse-client-python `_ had its initial release in 2015. `Robert Liebowitz `_ created this library while at the `Center for Open Science (COS) `_ and the COS uses it to integrate the `Open Science Framework (OSF) `_ with Dataverse via an add-on which itself is open source and listed on the :doc:`/api/apps` page. From 3190fd971d841bdf8a26b2b48ad18d8987b16580 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 5 Feb 2021 17:44:46 -0500 Subject: [PATCH 165/274] var outside try{} --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 9826ecfe795..4021969bf6e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -768,11 +768,12 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ @PUT @Path("{id}/editMetadata") - public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) throws WrappedResponse{ + public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) { Boolean replaceData = replace != null; + DataverseRequest req = null; try { - DataverseRequest req = createDataverseRequest(findUserOrDie()); + req = createDataverseRequest(findUserOrDie()); } catch (WrappedResponse ex) { logger.log(Level.SEVERE, "Edit metdata error: " + ex.getMessage(), ex); return ex.getResponse(); From 9e1d6f543f2d02d6daa7ff5b66eebb1c38e0236a Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Mon, 8 Feb 2021 09:15:11 -0500 Subject: [PATCH 166/274] #7582 tidy 5.0 release note upgrade instruction formatting --- doc/release-notes/5.0-release-notes.md | 41 +++++++++++++++++--------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/doc/release-notes/5.0-release-notes.md b/doc/release-notes/5.0-release-notes.md index e3d13eaca7c..c65900846b1 100644 --- a/doc/release-notes/5.0-release-notes.md +++ b/doc/release-notes/5.0-release-notes.md @@ -250,33 +250,46 @@ The instructions below describe the upgrade procedure based on moving an existin `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/` `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/` -8. In domain.xml: +8. Make the following changes in **domain.xml**: - Replace the -XX:PermSize and -XX:MaxPermSize JVM options with -XX:MetaspaceSize and -XX:MaxMetaspaceSize. + Replace the `-XX:PermSize` and `-XX:MaxPermSize` JVM options with `-XX:MetaspaceSize` and `-XX:MaxMetaspaceSize` + ``` -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m + ``` -Add the below JVM options beneath the -Ddataverse settings: + Add the below JVM options beneath the -Ddataverse settings: + ``` -Dfish.payara.classloading.delegate=false -XX:+UseG1GC -XX:+UseStringDeduplication -XX:+DisableExplicitGC + ``` -Also in domain.xml, replace the follow element: + Replace the following element: - ` ` + ``` + + + + +``` -with + with - ` ` + ``` + + + + ``` -9. Change any full pathnames /usr/local/glassfish4/... to /usr/local/payara5/... or whatever it is in your case. (Specifically check the -Ddataverse.files.directory and -Ddataverse.files.file.directory JVM options) +9. Change any full pathnames matching `/usr/local/glassfish4/...` to `/usr/local/payara5/...` or whatever it is in your case. Specifically check the `-Ddataverse.files.directory` and `-Ddataverse.files.file.directory` JVM options. 10. In domain1/config/jhove.conf, change the hard-coded /usr/local/glassfish4 path, as above. -(Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`: + (Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`: `sudo chown -R dataverse /usr/local/payara5/glassfish/domains/domain1` `sudo chown -R dataverse /usr/local/payara5/glassfish/lib` @@ -309,20 +322,20 @@ with 3. (Required for installations using DataCite) Add the JVM option doi.dataciterestapiurlstring -For production environments: + For production environments: `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.datacite.org"` -For test environments: + For test environments: `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.test.datacite.org"` -The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example: + The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example: `/usr/local/payara5/bin/asadmin delete-jvm-options "\-Ddoi.mdcbaseurlstring=https\://api.test.datacite.org"` 4. (Recommended for installations using DataCite) Pre-register DOIs -Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. + Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. -Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. + Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. From 71a46e92bc29966f0339f847e5a91de9982dad7b Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Mon, 8 Feb 2021 12:53:00 -0500 Subject: [PATCH 167/274] #7582 reformat the reformat to correct indentation --- doc/release-notes/5.0-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/5.0-release-notes.md b/doc/release-notes/5.0-release-notes.md index c65900846b1..3bbd33db3ae 100644 --- a/doc/release-notes/5.0-release-notes.md +++ b/doc/release-notes/5.0-release-notes.md @@ -275,7 +275,7 @@ The instructions below describe the upgrade procedure based on moving an existin -``` + ``` with From fbe12d2922ea5308d21472de7f756a319ee1578e Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Feb 2021 14:25:54 -0500 Subject: [PATCH 168/274] Added position CSS for tabView component to fix dropdown z-index layer, cleaned up other CSS on structure stylesheet [ref #7558] --- src/main/webapp/resources/css/structure.css | 113 +++++--------------- 1 file changed, 28 insertions(+), 85 deletions(-) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index e70bb2f7b93..3269cf7e12f 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -15,27 +15,15 @@ body .ui-widget {font-size: inherit;} /* -------- TEMPLATE -------- */ /* STICKY FOOTER VS DROPDOWN MENU POSITION + Z-INDEX */ -#content {position:relative; z-index:50;} +/*#content {position:relative;}*/ .panelgridHeaderTable {width:100%;} .panelgridHeaderTable .ui-widget-content, .panelgridHeaderTable .ui-widget-content > td {border:0; background:none;} .panelgridHeaderTable .ui-widget-content > td:last-child table {margin-left:auto;} -.bg-dataverse, bg-brand {background:#C55B28;} -.bg-dataset {background:#337AB7;} -.bg-file {background:#777777;} -.bg-muted {background:#f5f5f5;} -.text-dataverse, .text-brand {color:#C55B28;} -.text-dataset {color:#31708F;} -.text-file {color:#777777;} - -#navbarFixed {margin-bottom:0;} -.navbar-fixed-top, .navbar-fixed-bottom { - z-index: 930; -} +#navbarFixed.navbar-fixed-top {margin-bottom:0; z-index:9999;} #navbarFixed .navbar-brand {color:#C55B28; padding-left:32px;} #navbarFixed .navbar-brand.custom-logo {height:auto; padding:0;} #navbarFixed .icon-dataverse {color:#C55B28; font-size:28px; margin: -4px 0 0 -27px; position: absolute;} - #navbarFixed .dropdown.accountName a:focus {outline: -webkit-focus-ring-color 5px auto;} #navbarFixed form.navbar-form.navbar-form-link {margin:0; padding:0;} #navbarFixed form.navbar-form-link a.ui-commandlink, #navbarFixed ul.navbar-nav li.dropdown span.dropdown-toggle {display:block; padding:15px; color: #777;} @@ -70,40 +58,8 @@ body .ui-widget {font-size: inherit;} #footer .version {vertical-align:bottom;white-space:nowrap;} #footer.widget-view .widgetBrandMsg {line-height:42px;} -.panelLayoutBlock, .panelLayoutTitleBlock, .panelLayoutButtonBlock {border:0;} -.panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content, .panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content > td {border:0;} .form-group table.noBorders tr.ui-widget-content, .form-group table.noBorders td {border:0; padding-left:0;} -.panelgridLayoutTable, .panelgridFormTable {width:100%;} -.panelgridFormTable > tbody > tr.ui-widget-content > td:first-child {width:20%;} -.panelgridFormTable.ui-panelgrid td table.ui-picklist td {border: 0; padding:0;} -.panelgridFormTable.ui-panelgrid td table.ui-picklist td ul.ui-widget-content.ui-corner-all {border:1px solid #DDDDDD; border-radius: 4px;} - -.panelLayoutButtonBlock button {/*margin-right:2em;*/} - -.panelLayoutBlock > .ui-widget-content {border-bottom:1px solid #DDDDDD;} -.panelLayoutBlock.ui-panel.padding-none > .ui-widget-content.ui-panel-content {padding:0;} - -.panelLayoutBlock.breadcrumbNavBlock > .ui-widget-content {border-bottom:0;} -.panelLayoutTitleBlock > .ui-widget-content {border:0;} - -.panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content .ui-panelgrid {margin-bottom: 1em;} -.panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content .ui-panelgrid .ui-widget-content .ui-panelgrid {margin-bottom: 0} -.panelLayoutBlock .ui-panelgrid.panelgridLayoutTable .ui-widget-content .ui-panelgrid .ui-widget-content .ui-panelgrid td:first-child {padding-left:0;} - -.btn span.glyphicon {margin-right:.3em;} -.btn.btn-link.bootstrap-button-tooltip span.glyphicon {margin:0;} /* OR JUST USE NO-TEXT... */ -.btn.btn-link.bootstrap-button-tooltip span.caret {margin-left:6px;} -.btn span.glyphicon.no-text {margin:0;} -.btn-group .btn span.glyphicon.no-text {margin-left:10px;margin-right:10px;} -.col-manage-action .btn-group .btn span.glyphicon.no-text {margin:0;} -.icon-inline-action {margin-left:.3em;} -.icon-inline-action:first-of-type {margin-left:1em;} - -.btn-group .text-button {display:inline-block;padding:6px;} - -.text-icon-inline {font-size:1.3em;line-height:1.1em;margin-right:4px;} - #messagePanel div.messagePanel {margin-top:1em;} .messagePanel .glyphicon {font-size: 1.5em; vertical-align: bottom;} @@ -128,6 +84,14 @@ td.col-select-width, th.col-select-width {width:36px;} #ajaxStatusPanel_start {width:31px;height:31px;position:fixed;right:50%;bottom:50%;z-index:111111;} #ajaxStatusPanel_complete {position:absolute;top:-9999px;left:-9999px;} +.bg-dataverse, bg-brand {background:#C55B28;} +.bg-dataset {background:#337AB7;} +.bg-file {background:#777777;} +.bg-muted {background:#f5f5f5;} +.text-dataverse, .text-brand {color:#C55B28;} +.text-dataset {color:#31708F;} +.text-file {color:#777777;} + .italic {font-style: italic;} .margin-top {margin-top:1em;} .margin-top-half {margin-top:.5em;} @@ -148,6 +112,16 @@ table.selectManyCheckboxBlock label {font-weight:normal;} hr {border-color: #ddd;} +.btn span.glyphicon {margin-right:.3em;} +.btn.btn-link.bootstrap-button-tooltip span.glyphicon {margin:0;} /* OR JUST USE NO-TEXT... */ +.btn.btn-link.bootstrap-button-tooltip span.caret {margin-left:6px;} +.btn span.glyphicon.no-text {margin:0;} +.btn-group .btn span.glyphicon.no-text {margin-left:10px;margin-right:10px;} +.col-manage-action .btn-group .btn span.glyphicon.no-text {margin:0;} +.icon-inline-action {margin-left:.3em;} +.icon-inline-action:first-of-type {margin-left:1em;} +.btn-group .text-button {display:inline-block;padding:6px;} +.text-icon-inline {font-size:1.3em;line-height:1.1em;margin-right:4px;} span.glyphicon[data-toggle='tooltip'] {cursor: default;} span.glyphicon-question-sign.tooltip-icon {color: #99bcdb !important;} span.glyphicon-question-sign.tooltip-icon:hover {color: #337AB7 !important;} @@ -261,17 +235,11 @@ body.widget-view #dataverse-header-block {padding-top:0px;} #breadcrumbNavBlock .breadcrumbActive {font-weight:bold;} #breadcrumbNavBlock a[id^='breadcrumbLnkTree'] {cursor: pointer; cursor: hand;} #breadcrumbNavBlock a[id^='breadcrumbLnkTree'] span.glyphicon-chevron-down {vertical-align:middle;padding-bottom:2px;} - #breadcrumbNavBlock > .ui-panel-content {overflow:hidden;} - #breadcrumbNavBlock.panelLayoutBlock > .ui-widget-content {border-bottom: 0;} - #breadcrumbNavBlock .breadcrumbCarrot {float:left; margin:0 .4em; color:#B3B3B3;} - #breadcrumbNavBlock div.dropdown-menu {padding-right:.5em; min-width:280px;} - #breadcrumbNavBlock .ui-tree {width: auto;} - #breadcrumbNavBlock div.ui-tree.ui-widget-content {border:0;} #breadcrumbNavBlock div.ui-tree.ui-widget-content span.ui-treenode-content span.ui-icon.ui-tree-toggler {background-size: auto 284px; height:20px; width:20px;} #breadcrumbNavBlock div.ui-tree.ui-widget-content span.ui-treenode-content span.ui-icon.ui-tree-toggler.ui-icon-triangle-1-s {background-position: -72px -16px;} @@ -511,9 +479,7 @@ form[id$='facetType'] > div.clearfix:last-child {margin-bottom:0;} form[id$='facetType'] > div.clearfix .facetTypeChBox {display:block; float:left;} form[id$='facetType'] > div.clearfix .facetTypeChBox > div.ui-chkbox {padding-top:2px;padding-right:4px;} form[id$='facetType'] > div.clearfix a.facetTypeLink {display:block; float:left; text-decoration:none;} - form[id$='facetType'] > div.clearfix a.facetTypeLink span {padding:4px 0px;} - form[id$='facetCategoryForm'] {margin-top:1em;} div[id$='facetCategoryList'] ul, div[id$='facetCategoryList'] li, div[id$='facetCategoryList'] div {margin:0; padding:0; list-style: none; border:0;} div[id$='facetCategoryList'] > div > ul > li {border-bottom:1px solid #DDD; margin-bottom:.5em;} @@ -522,7 +488,6 @@ div[id$='facetCategoryList'] div li div {padding:4px 0px;} div[id$='facetCategoryList'] div li div.ui-datalist-empty-message {padding:4px 0px; font-style:italic; color:#AAA;} div[id$='facetCategoryList'] div.ui-datalist, div[id$='facetCategoryList'] div.ui-datalist div.ui-datalist-content {padding-top:0;} .facetCategoryName {font-weight:bold;} - .facetCategoryName, .facetLink { -ms-word-break: break-all; @@ -572,7 +537,7 @@ div.ui-dialog.smallPopUp {max-width:600px;} div.ui-dialog.largePopUp {min-width:60%; max-width:80%; max-height:80%; overflow-y:scroll;} div.ui-dialog .ui-datatable-scrollable table {table-layout: auto !important;} div.ui-dialog .ui-datatable-scrollable table colgroup col {max-width:1000px;} - +.ui-dialog.ui-widget.ui-widget-content {min-width:600px;} div.ui-dialog div.highlight { background-color: #f7f7f9; border: 1px solid #e1e1e8; @@ -583,8 +548,6 @@ div.ui-dialog div.highlight { .ui-dialog.dialog-submenu, .ui-dialog.dialog-submenu .ui-dialog-content, .ui-dialog.dialog-submenu .form-inline, .ui-dialog.dialog-submenu .form-group {overflow:visible !important;} .ui-dialog.dialog-submenu .ui-menu-list.ui-menu-child {z-index: 9999 !important; width: auto; min-width: 12.5em;} -.ui-dialog.ui-widget.ui-widget-content {min-width:600px;} - /* -------- SUBTABS -------- */ div.popupSubTabParent > div.ui-tabs-panels > div.ui-tabs-panel.ui-widget-content {padding-left:0; padding-right:0;} div.ui-tabs-panels .ui-tabs .ui-tabs-nav.ui-widget-header li {border:0;} @@ -742,15 +705,10 @@ div[id$="filesHeaderCount"] {padding:6px 12px;} div[id$="filesHeaderCount"] span.highlightBold {display:block; padding:0; line-height:1.65;} div[id$="filesTable"] .ui-selection-column {width:20px !important;} div[id$="filesTable"] .col-file-metadata {padding-left:0; padding-right:0; border-left:0; border-right:0; text-align:left; font-weight:normal;} - -/*div[id$="filesTable"] .col-file-thumb {padding:4px 10px; width:84px;}*/ div[id$="filesTable"] .col-file-thumb div.thumbnail-block {position:relative;width:64px;height:64px;margin:0 auto;} div[id$="filesTable"] .col-file-thumb div.thumbnail-block span.file-thumbnail-icon {font-size:62px;line-height:1.05;} div[id$="filesTable"] .col-file-thumb div.thumbnail-block span.file-thumbnail-preview-img {display:block;} div[id$="filesTable"] .col-file-thumb div.thumbnail-block div.file-icon-restricted-block {display:block;position:absolute;bottom:-4px;right:-1px;} -/*div[id$="filesTable"] .file-metadata-block {-webkit-flex: 1 100%; - -moz-flex: 1 100%; - -ms-flex: 1 100%; flex: 1 100%;}*/ div[id$="filesTable"] div.file-tags-block {padding-bottom:2px;} div[id$="filesTable"] div.file-tags-block a.ui-commandlink {font-size:12px;} div[id$="filesTable"] div.file-tags-block span.label {display:inline-block;} @@ -837,8 +795,6 @@ div[id$="filesTable"] .ui-paginator .ui-paginator-pages .ui-state-active { /* FILES TREE */ div[id$="filesTree"] span.ui-treenode-label span[class^='icon'] {margin-left: -2px;} -/* span.ingest-warning {font-size:2em !important; line-height:1; vertical-align: middle;} */ - /* DOWNLOAD POPUP */ div.panel-body.read-terms{max-height:220px; overflow-y:scroll; width:100%; background-color:#eeeeee;} @@ -848,11 +804,9 @@ div.panel-body.read-terms{max-height:220px; overflow-y:scroll; width:100%; backg /* VERSIONS */ div[id$="versionsTable"] th.col-select-width * {display:none;} - #version-details-block > div {width:35%; padding:4px 10px; border: 1px solid #DDDDDD;} #version-details-block > div:first-of-type {width:30%; border:0;} #version-details-block > div:last-of-type {border-left-width:0;} - div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > thead {display:none;} div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table, div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody {border:0;} div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr:first-child, div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr:first-child > td {border-top-width:0;} @@ -861,22 +815,14 @@ div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr, div div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr, div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr > td:last-child {border-right-width:0;} div.dvnDifferanceTable div.ui-datatable-tablewrapper table td.versionValue {width:30%;} div.dvnDifferanceTable div.ui-datatable-tablewrapper table td.versionDetails {width:35%;} - div.dvnDifferanceTable .diffDetailBlock {display:block;} -/* MANAGE TABLE + DROPDOWN BUTTON */ -div.manageTable.ui-datatable div.ui-datatable-tablewrapper, div.manageTable.ui-datatable td.col-manage-action {overflow:visible;} -div[id$="filesTable"].ui-datatable thead th.col-file-action, div[id$="filesTable"].ui-datatable tbody td.col-file-action {overflow:visible !important;} +/* DATATABLE + DROPDOWN BUTTON + OVERFLOW VISIBLE */ thead.ui-datatable-scrollable-theadclone {display:none} - -/* DATATABE + OVERFLOW VISIBLE */ -.ui-datatable thead th, .ui-datatable tbody td, .ui-datatable tfoot td, .ui-datatable tfoot th {overflow:visible;} +.ui-datatable thead th, .ui-datatable tbody td, .ui-datatable tfoot td, .ui-datatable tfoot th {overflow:visible !important;} /* FILE */ -#file-preview-icon-block > #border-block {/*border:1px solid #DDD;border-radius:4px; padding: 10px;*/} -#file-preview-icon-block > #border-block > #preview-square {/*position: relative; overflow:hidden; padding-bottom:100%;*/} -#file-preview-icon-block > #border-block > #preview-square > span.text-muted {font-size:6.5em; line-height:1 !important; /*text-align: center; position: absolute; width: 100%; height: 100%; margin:0;*/} -#file-preview-icon-block > #border-block > #preview-square > img.img-responsive {/*position: absolute; max-width: 100%; max-height: 100%; top: 50%; left: 50%; transform: translateX(-50%) translateY(-50%);*/} +#file-preview-icon-block > #border-block > #preview-square > span.text-muted {font-size:6.5em; line-height:1 !important;} @media(min-width:768px){ #file-preview-icon-block > #border-block > #preview-square > span.text-muted { font-size: 9.5em; @@ -905,10 +851,11 @@ thead.ui-datatable-scrollable-theadclone {display:none} div.ui-dialog.ui-widget-content {z-index:100001 !important;} /* POPUP DROPDOWN MENUS + TOOLTIPS + BLOCK UI */ -div.ui-selectcheckboxmenu-panel, div.ui-selectonemenu-panel, div.ui-autocomplete-panel, -.ui-selectonemenu-list, .ui-selectcheckboxmenu-list, div.tooltip, div.popover {z-index:100002 !important;} +.ui-selectcheckboxmenu-panel, .ui-selectonemenu-panel, .ui-autocomplete-panel, +.ui-selectonemenu-list, .ui-selectcheckboxmenu-list, .tooltip, .popover {z-index:100002 !important;} /* Overwrite Primefaces */ +.ui-tabs.ui-widget.ui-widget-content {position:relative; z-index:999;} .ui-tabs.ui-widget-content .ui-tabs-nav li.ui-tabs-selected a {color: #555555;} button.btn-default.ui-button-text-only .ui-button-text {padding:0; font-size:14px;} button.btn-default span.ui-button-text {line-height:inherit !important;} @@ -922,12 +869,8 @@ div.ui-widget-overlay {background:#ccc; opacity:.6;} .ui-button.btn-link:hover, .ui-button.btn-link:focus {color: #23527c; text-decoration: underline !important; background-color: transparent;} .ui-button.btn-link[disabled]:hover, fieldset[disabled] .ui-button.btn-link:hover, .ui-button.btn-link[disabled]:focus, fieldset[disabled] .ui-button.btn-link:focus {color: #777;text-decoration: none;} .ui-widget-header .form-group {font-weight:normal;} -/* .ui-messages-info-detail, .ui-messages-warn-detail, .ui-messages-error-detail, .ui-messages-fatal-detail, .ui-message-info-detail, .ui-message-warn-detail, .ui-message-error-detail, .ui-message-fatal-detail { - margin-left: 0px; -} */ + .ui-state-disabled, .ui-widget-content .ui-state-disabled, .ui-widget-header .ui-state-disabled {background: #e0e0e0; border-color: #ccc; opacity: 0.65;} -/* .ui-autocomplete-input-token input, .ui-autocomplete-input-token {width:100%;} */ -.ui-autocomplete-panel {z-index:100002 !important;} div.ui-autocomplete {width:100%;} div.ui-autocomplete > ul, div.ui-autocomplete > ul > li > input {width:100%;} /* FIX WIDTH FOR AUTOCOMPLETE WITHOUT P:FRAGMENT */ span.ui-autocomplete {width:100%;} /* FIX WIDTH FOR AUTOCOMPLETE IN P:FRAGMENT */ From 754b3f9d71aa0b9ab51a3028180e6e95bf6115c3 Mon Sep 17 00:00:00 2001 From: Michael Heppler <687227+mheppler@users.noreply.github.com> Date: Mon, 8 Feb 2021 14:31:37 -0500 Subject: [PATCH 169/274] Removing commented out styles not in use [ref #7758] --- src/main/webapp/resources/css/structure.css | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 3269cf7e12f..2603fd6ba0b 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -14,8 +14,6 @@ body .ui-widget {font-size: inherit;} .ui-widget-content a:hover, .ui-widget-content a:focus {color: #23527C;} /* -------- TEMPLATE -------- */ -/* STICKY FOOTER VS DROPDOWN MENU POSITION + Z-INDEX */ -/*#content {position:relative;}*/ .panelgridHeaderTable {width:100%;} .panelgridHeaderTable .ui-widget-content, .panelgridHeaderTable .ui-widget-content > td {border:0; background:none;} .panelgridHeaderTable .ui-widget-content > td:last-child table {margin-left:auto;} From 51707f5ad01b5a59b28225ebc20fe289c8d6b5d1 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Feb 2021 16:28:23 -0500 Subject: [PATCH 170/274] Removed extra link from Feature Dataverse carousel [ref #7476] --- src/main/webapp/dataverse.xhtml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index fbbe7563baf..14dc7ee22cb 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -580,17 +580,13 @@
    • +
    +

    VIEW ALL DATA

    +
    From ba623ad830b66c66c836c651e96d63d8c0216ee7 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 9 Feb 2021 15:17:18 -0500 Subject: [PATCH 172/274] #6972 allow pdf download if "original" is selected. --- src/main/java/edu/harvard/iq/dataverse/api/Access.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index df9e39547b9..e98bc8aa1fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -331,6 +331,14 @@ public DownloadInstance datafile(@PathParam("fileId") String fileId, @QueryParam // So we need to identify when a service is being called and then let checkIfServiceSupportedAndSetConverter see if the required one exists if (key.equals("imageThumb") || key.equals("format") || key.equals("variables") || key.equals("noVarHeader")) { serviceRequested = true; + //In the dataset file table context a user is allowed to select original as the format + //for download + // if the dataset has tabular files - it should not be applied to instances + // where the file selected is not tabular see #6972 + if("format".equals(key) && "original".equals(value) && !df.isTabularData()) { + serviceRequested = false; + break; + } //Only need to check if this key is associated with a service if (downloadInstance.checkIfServiceSupportedAndSetConverter(key, value)) { // this automatically sets the conversion parameters in From d44c6d8d8794ce20beb6a74606df7f6052b7f08e Mon Sep 17 00:00:00 2001 From: gdurand Date: Tue, 9 Feb 2021 17:10:51 -0500 Subject: [PATCH 173/274] n ew logic to handle the conditally required validation --- .../harvard/iq/dataverse/DatasetField.java | 74 +++++++++----- .../iq/dataverse/DatasetFieldType.java | 2 + .../iq/dataverse/DatasetFieldValidator.java | 47 +++++++-- src/main/java/propertyFiles/Bundle.properties | 3 +- src/main/webapp/dataverse.xhtml | 4 +- src/main/webapp/metadataFragment.xhtml | 8 +- .../dataverse/DatasetFieldValidatorTest.java | 99 +++++++++++++------ 7 files changed, 169 insertions(+), 68 deletions(-) mode change 100755 => 100644 src/main/java/propertyFiles/Bundle.properties diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index 6d4b56cf6cf..0bca1b50319 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -407,20 +407,23 @@ public void setValidationMessage(String validationMessage) { this.validationMessage = validationMessage; } + + // these two booleans are used to wrap around the field type isRequired and isHasChildren methods to also check + // if the dataverse has overriden the default values @Transient private Boolean required; + @Transient + private Boolean hasRequiredChildren; public boolean isRequired() { if (required == null) { required = false; - if (this.datasetFieldType.isPrimitive() && this.datasetFieldType.isRequired()) { - required = true; - } - - if (this.datasetFieldType.isHasRequiredChildren()) { + + if (this.datasetFieldType.isRequired()) { required = true; } - + + // otherwise check if overridden at the dataverse level Dataverse dv = getDataverse(); while (!dv.isMetadataBlockRoot()) { if (dv.getOwner() == null) { @@ -430,29 +433,54 @@ public boolean isRequired() { } List dftilListFirst = dv.getDataverseFieldTypeInputLevels(); - if (!getDatasetFieldType().isHasChildren()) { - for (DataverseFieldTypeInputLevel dsftil : dftilListFirst) { - if (dsftil.getDatasetFieldType().equals(this.datasetFieldType)) { - required = dsftil.isRequired(); - } + for (DataverseFieldTypeInputLevel dsftil : dftilListFirst) { + if (dsftil.getDatasetFieldType().equals(this.datasetFieldType)) { + required = dsftil.isRequired(); } } - - if (getDatasetFieldType().isHasChildren() && (!dftilListFirst.isEmpty())) { - for (DatasetFieldType child : getDatasetFieldType().getChildDatasetFieldTypes()) { - for (DataverseFieldTypeInputLevel dftilTest : dftilListFirst) { - if (child.equals(dftilTest.getDatasetFieldType())) { - if (dftilTest.isRequired()) { - required = true; - } + + // since we don't currently support the use case where the parent is required + // but no specific children are (the "true/false" case), we override this as false + if (required && this.datasetFieldType.isCompound() && !isHasRequiredChildren()) + { + required = false; + } + } + + return required; + } + + public boolean isHasRequiredChildren() { + if (hasRequiredChildren == null) { + hasRequiredChildren = false; + } + + if (this.datasetFieldType.isHasRequiredChildren()) { + hasRequiredChildren = true; + } + + Dataverse dv = getDataverse(); + while (!dv.isMetadataBlockRoot()) { + if (dv.getOwner() == null) { + break; // we are at the root; which by defintion is metadata blcok root, regarldess of the value + } + dv = dv.getOwner(); + } + + List dftilListFirst = dv.getDataverseFieldTypeInputLevels(); + + if (getDatasetFieldType().isHasChildren() && (!dftilListFirst.isEmpty())) { + for (DatasetFieldType child : getDatasetFieldType().getChildDatasetFieldTypes()) { + for (DataverseFieldTypeInputLevel dftilTest : dftilListFirst) { + if (child.equals(dftilTest.getDatasetFieldType())) { + if (dftilTest.isRequired()) { + hasRequiredChildren = true; } } } } - - } - // logger.fine("at return " + this.datasetFieldType.getDisplayName() + " " + required); - return required; + } + return hasRequiredChildren; } public Dataverse getDataverse() { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java index 04901264455..842b9490784 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java @@ -141,7 +141,9 @@ public void setOptionSelectItems(List optionSelectItems) { public DatasetFieldType() {} public DatasetFieldType(String name, FieldType fieldType, boolean allowMultiples) { + // use the name for both default name and title this.name = name; + this.title = name; this.fieldType = fieldType; this.allowMultiples = allowMultiples; childDatasetFieldTypes = new LinkedList<>(); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java index 425ed0b22ef..00b7be97b83 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValidator.java @@ -9,6 +9,8 @@ import javax.validation.ConstraintValidatorContext; import edu.harvard.iq.dataverse.util.BundleUtil; +import java.util.Collections; +import java.util.List; import org.apache.commons.lang.StringUtils; @@ -25,22 +27,38 @@ public void initialize(ValidateDatasetFieldType constraintAnnotation) { @Override public boolean isValid(DatasetField value, ConstraintValidatorContext context) { context.disableDefaultConstraintViolation(); // we do this so we can have different messages depending on the different issue - + DatasetFieldType dsfType = value.getDatasetFieldType(); //SEK Additional logic turns off validation for templates if (isTemplateDatasetField(value)){ return true; } - if (((dsfType.isPrimitive() && dsfType.isRequired()) || (dsfType.isPrimitive() && value.isRequired())) - && StringUtils.isBlank(value.getValue())) { - try{ - context.buildConstraintViolationWithTemplate(dsfType.getDisplayName() + " " + BundleUtil.getStringFromBundle("isrequired")).addConstraintViolation(); - } catch (NullPointerException npe){ - //if there's no context for the error we can't put it anywhere.... + + // if value is not primitive or not empty + if (!dsfType.isPrimitive() || !StringUtils.isBlank(value.getValue())) { + return true; + } + + if (value.isRequired()) { + String errorMessage = null; + DatasetFieldCompoundValue parent = value.getParentDatasetFieldCompoundValue(); + if (parent == null || parent.getParentDatasetField().isRequired()) { + errorMessage = BundleUtil.getStringFromBundle("isrequired", List.of(dsfType.getDisplayName())); + } else if (areSiblingsPopulated(value)) { + errorMessage = BundleUtil.getStringFromBundle("isrequired.conditional", List.of(dsfType.getDisplayName(),parent.getParentDatasetField().getDatasetFieldType().getDisplayName())); } - return false; + if (errorMessage != null) { + try { + context.buildConstraintViolationWithTemplate(errorMessage).addConstraintViolation(); + } catch (NullPointerException npe){ + //if there's no context for the error we can't put it anywhere.... + } + + return false; + } } + return true; } @@ -51,4 +69,17 @@ private boolean isTemplateDatasetField(DatasetField dsf) { return dsf.getTemplate() != null; } } + + private boolean areSiblingsPopulated(DatasetField dsf) { + if (dsf.getParentDatasetFieldCompoundValue() != null) { + DatasetFieldCompoundValue compound = dsf.getParentDatasetFieldCompoundValue(); + for (DatasetField sibling : compound.getChildDatasetFields()) { + if (!StringUtils.isBlank(sibling.getValue())) { + return true; + } + } + } + + return false; + } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties old mode 100755 new mode 100644 index 2a73ea97307..fd46e16b0fe --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2446,7 +2446,8 @@ editfilesfragment.label4.item2=Unicode (UTF-16LE) editfilesfragment.label4.item3=Unicode (UTF-16BE) editfilesfragment.label5=US-ASCII -isrequired=is required. +isrequired={0} is required. +isrequired.conditional={0} is required if you choose to enter text in any of the {1} fields. draftversion=DRAFT VERSION deaccessionedversion=DEACCESSIONED VERSION diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index fbbe7563baf..1921eb324e8 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -331,10 +331,10 @@ + rendered="#{!dsft.required}"> - + #{bundle['dataverse.field.requiredByDataverse']} diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 908a216fb12..231e59d133e 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -105,14 +105,14 @@
    #{dsf.datasetFieldType.localeTitle} - + @@ -185,7 +185,9 @@
    #{dsf.datasetFieldType.localeTitle} - + + +
    diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValidatorTest.java index 9369c1b6930..e82fc02e37a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValidatorTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValidatorTest.java @@ -47,43 +47,80 @@ public void tearDown() { */ @Test public void testIsValid() { - DatasetField dataSetField = new DatasetField(); + + Dataverse dataverse = new Dataverse(); + Dataset dataset = new Dataset(); + dataset.setOwner(dataverse); + DatasetVersion datasetVersion = new DatasetVersion(); + datasetVersion.setDataset(dataset); + + testPrimitiveDatasetField("test isValid() on template field", true, "", null, true); + testPrimitiveDatasetField("test isValid() for required primitive with empty value", true, "", datasetVersion, false); + testPrimitiveDatasetField("test isValid() for required primitive with nonempty value", true, "test", datasetVersion, true); + testPrimitiveDatasetField("test isValid() for not required primitive", false, "", datasetVersion, true); + + // the compound tests are defined by parent/child1 required and child1/child2 values + testCompoundDatasetField("test isValid() for false/false compound with empty/empty children", false, false, "", "", datasetVersion, true); + testCompoundDatasetField("test isValid() for true/true compound with empty/empty children", true, true, "", "", datasetVersion, false); + testCompoundDatasetField("test isValid() for true/true compound with empty/nonempty children", true, true, "", "test", datasetVersion, false); + testCompoundDatasetField("test isValid() for true/true compound with nonempty/empty children", true, true, "test", "", datasetVersion, true); + testCompoundDatasetField("test isValid() for false/true compound with empty/empty children", false, true, "", "", datasetVersion, true); + testCompoundDatasetField("test isValid() for false/true compound with empty/nonempty children", false, true, "", "test", datasetVersion, false); + testCompoundDatasetField("test isValid() for false/true compound with nonempty/empty children", false, true, "test", "", datasetVersion, true); + } + + private void testPrimitiveDatasetField(String test, boolean required, String value, + DatasetVersion dsv, boolean expectedOutcome) { + DatasetFieldType primitiveDSFType = new DatasetFieldType("primitive", DatasetFieldType.FieldType.TEXT, false); + primitiveDSFType.setRequired(required); + + DatasetField testDatasetField = new DatasetField(); + if (dsv != null) { + testDatasetField.setDatasetVersion(dsv); + } else { + testDatasetField.setTemplate(new Template()); + } + testDatasetField.setDatasetFieldType(primitiveDSFType); + testDatasetField.setSingleValue(value); + DatasetFieldValidator datasetFieldValidator = new DatasetFieldValidator(); + assertEquals( test, expectedOutcome, datasetFieldValidator.isValid(testDatasetField, constraintValidatorContext)); + + } + + private void testCompoundDatasetField(String test, boolean requiredParent, boolean requiredChild1, + String valueChild1, String valueChild2, DatasetVersion dsv, boolean expectedOutcome) { - // if it is a template field it is always valid - dataSetField.setTemplate(new Template()); - boolean expResult = true; - boolean result = datasetFieldValidator.isValid(dataSetField, constraintValidatorContext); - assertEquals("test isValid() on template field", expResult, result); + DatasetFieldType parentDSFType = new DatasetFieldType("parent", DatasetFieldType.FieldType.TEXT, false); + DatasetFieldType child1DSFType = new DatasetFieldType("child1", DatasetFieldType.FieldType.TEXT, false); + DatasetFieldType child2DSFType = new DatasetFieldType("child2", DatasetFieldType.FieldType.TEXT, false); - // if not a template field and required - dataSetField.setTemplate(null); - DatasetVersion datasetVersion = new DatasetVersion(); - Dataset dataset = new Dataset(); - Dataverse dataverse = new Dataverse(); - dataset.setOwner(dataverse); - datasetVersion.setDataset(dataset); - dataSetField.setDatasetVersion(datasetVersion); + parentDSFType.setRequired(requiredParent); + child1DSFType.setRequired(requiredChild1); + child2DSFType.setRequired(false); + + DatasetField parentDatasetField = new DatasetField(); + parentDatasetField.setDatasetVersion(dsv); + parentDatasetField.setDatasetFieldType(parentDSFType); + + DatasetFieldCompoundValue compound = new DatasetFieldCompoundValue(); + compound.setParentDatasetField(parentDatasetField); - DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); - DatasetFieldType datasetFieldType = new DatasetFieldType("test", DatasetFieldType.FieldType.TEXT, false); - datasetFieldType.setRequired(true); - dataSetField.setDatasetFieldType(datasetFieldType); - dataSetField.setSingleValue(""); - datasetFieldValue.setValue(""); - result = datasetFieldValidator.isValid(dataSetField, constraintValidatorContext); - assertEquals("test isValid() if not template field and required with empty DataSetField", false, result); + DatasetField child1DatasetField = new DatasetField(); + child1DatasetField.setParentDatasetFieldCompoundValue(compound); + compound.getChildDatasetFields().add(child1DatasetField); + child1DatasetField.setDatasetFieldType(child1DSFType); + child1DatasetField.setSingleValue(valueChild1); - // fill in a value - the required constraint is satisfied now - dataSetField.setSingleValue("value"); - result = datasetFieldValidator.isValid(dataSetField, constraintValidatorContext); - assertEquals("test isValid() if not template field and required with non-empty DataSetField", true, result); + DatasetField child2DatasetField = new DatasetField(); + child2DatasetField.setParentDatasetFieldCompoundValue(compound); + compound.getChildDatasetFields().add(child2DatasetField); + child2DatasetField.setDatasetFieldType(child2DSFType); + child2DatasetField.setSingleValue(valueChild2); + - // if not required the field can be blank - datasetFieldType.setRequired(false); - dataSetField.setSingleValue(""); - result = datasetFieldValidator.isValid(dataSetField, constraintValidatorContext); - assertEquals("test isValid() if not template field and not required", true, result); + DatasetFieldValidator datasetFieldValidator = new DatasetFieldValidator(); + assertEquals( test, expectedOutcome, datasetFieldValidator.isValid(child1DatasetField, constraintValidatorContext)); } } From 14138e47e32060bee658f5302183e772f39378da Mon Sep 17 00:00:00 2001 From: gdurand Date: Wed, 10 Feb 2021 11:38:26 -0500 Subject: [PATCH 174/274] tweaks for conditionally required validation --- src/main/java/propertyFiles/Bundle.properties | 4 +++- src/main/webapp/metadataFragment.xhtml | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index fd46e16b0fe..cd6c542c62b 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -68,6 +68,8 @@ selectedFiles=Selected Files htmlAllowedTitle=Allowed HTML Tags htmlAllowedMsg=This field supports only certain HTML tags. htmlAllowedTags=<a>, <b>, <blockquote>, <br>, <code>, <del>, <dd>, <dl>, <dt>, <em>, <hr>, <h1>-<h3>, <i>, <img>, <kbd>, <li>, <ol>, <p>, <pre>, <s>, <sup>, <sub>, <strong>, <strike>, <u>, <ul> +conditionalRequiredMsg=One or more of these fields may become required if you add to one or more of these optional fields. +conditionalRequiredMsg.tooltip=This field will become required if you choose to enter values in one or more of these optional fields. toggleNavigation=Toggle navigation defaultBody=Default Body loading=Loading... @@ -2447,7 +2449,7 @@ editfilesfragment.label4.item3=Unicode (UTF-16BE) editfilesfragment.label5=US-ASCII isrequired={0} is required. -isrequired.conditional={0} is required if you choose to enter text in any of the {1} fields. +isrequired.conditional={0} is required if you choose to enter a value in any of the optional {1} fields. draftversion=DRAFT VERSION deaccessionedversion=DEACCESSIONED VERSION diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 231e59d133e..bb04f173bdb 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -185,13 +185,16 @@
    #{dsf.datasetFieldType.localeTitle} - - - +
    + +

    + +

    +
    @@ -204,9 +207,10 @@ From adc5236d7839d3b5a6e794a2323021cfe5efcc10 Mon Sep 17 00:00:00 2001 From: gdurand Date: Wed, 10 Feb 2021 13:34:58 -0500 Subject: [PATCH 175/274] minor formatting change --- src/main/webapp/metadataFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index bb04f173bdb..74e1f531234 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -210,7 +210,7 @@ + data-original-title="#{subdsf.datasetFieldType.localeDescription}#{subdsf.required and !dsf.required?' – '.concat(bundle['conditionalRequiredMsg.tooltip']) : ''}"> From a9725b0149a3e3bb09d0b5e62026edc4899ed890 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 10 Feb 2021 13:49:01 -0500 Subject: [PATCH 176/274] Fixed accessible input labels on add dv form [ref #7566] --- src/main/java/propertyFiles/Bundle.properties | 1 + src/main/webapp/contactFormFragment.xhtml | 3 +- src/main/webapp/dataverse.xhtml | 222 +++++++++--------- 3 files changed, 114 insertions(+), 112 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 1fd7c300ab4..1d2f6cee9cc 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -140,6 +140,7 @@ contact.msg=Message contact.msg.required=Message text is required. contact.send=Send Message contact.question=Please fill this out to prove you are not a robot. +contact.sum.title=Human Access Validation Answer contact.sum.required=Value is required. contact.sum.invalid=Incorrect sum, please try again. contact.sum.converterMessage=Please enter a number. diff --git a/src/main/webapp/contactFormFragment.xhtml b/src/main/webapp/contactFormFragment.xhtml index 1a28562df1c..284d84eb885 100644 --- a/src/main/webapp/contactFormFragment.xhtml +++ b/src/main/webapp/contactFormFragment.xhtml @@ -63,7 +63,8 @@ + validatorMessage="#{bundle['contact.sum.invalid']}" validator="#{sendFeedbackDialog.validateUserSum}" + title="#{bundle['contact.sum.title']}"> diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index fbbe7563baf..67ecd4017e7 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -61,14 +61,14 @@
    - + #{bundle.hostDataverse} - +
    -

    #{DataversePage.dataverse.getOwner().name}

    +

    #{DataversePage.dataverse.getOwner().name}

     - + #{bundle['dataverse.email']} - +
    - + +
    @@ -674,121 +674,121 @@ - - -

    #{bundle['dataverse.share.dataverseShare.tip']}

    -
    -
    - -
    -     -     - - - - - - -

    #{bundle['dataverse.link.dataverse.choose']}

    -     - -

    #{bundle['dataverse.savedsearch.dataverse.choose']}

    -     -
    - -
    - - - - - - -
    - #{u.id}
    - #{u.affiliation} -
    -     -     -     -
    -
    - -
    + + +

    #{bundle['dataverse.share.dataverseShare.tip']}

    +
    +
    + +
    +     +     + + + + + + +

    #{bundle['dataverse.link.dataverse.choose']}

    +     + +

    #{bundle['dataverse.savedsearch.dataverse.choose']}

    +     +
    -

    - #{SearchIncludeFragment.query.toString()} -

    + + + + + + +
    + #{u.id}
    + #{u.affiliation} +
    +     +     +
    -
    - -
    - -

    -     + +
    + +
    +

    + #{SearchIncludeFragment.query.toString()} +

    +
    +
    +
    + +
    + +

    +     +
    +     +
    + + +
    -
    - - - -
    - - -

    #{bundle['dataverse.link.no.choice']}

    -
    -
    - -
    -

    #{DataversePage.linkingDataverse.displayName}

    + +

    #{bundle['dataverse.link.no.choice']}

    +
    +
    + +
    +

    #{DataversePage.linkingDataverse.displayName}

    +
    -
    -
    - - -
    - - -

    - #{bundle['dataverse.link.no.linkable']} -

    -
    - -
    -     - - +
    + + +
    + + +

    + #{bundle['dataverse.link.no.linkable']} +

    +
    + +
    +     + + From 3fdbb45277b4a128ad63855479b278b1db5053e7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 10 Feb 2021 14:54:49 -0500 Subject: [PATCH 177/274] support different checksum algorithms in direct upload --- .../datasetutility/AddReplaceFileHelper.java | 14 +++++--- .../datasetutility/OptionalFileParams.java | 36 ++++++++++++++----- .../harvard/iq/dataverse/util/FileUtil.java | 19 ++++++---- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index ea1cfc38cfa..a8c52414e56 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -6,6 +6,7 @@ package edu.harvard.iq.dataverse.datasetutility; import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; @@ -117,11 +118,12 @@ public class AddReplaceFileHelper{ // ----------------------------------- private Dataset dataset; // constructor (for add, not replace) private DataverseRequest dvRequest; // constructor - private InputStream newFileInputStream; // step 20 - private String newFileName; // step 20 - private String newFileContentType; // step 20 - private String newStorageIdentifier; // step 20 - private String newCheckSum; // step 20 + private InputStream newFileInputStream; // step 30 + private String newFileName; // step 30 + private String newFileContentType; // step 30 + private String newStorageIdentifier; // step 30 + private String newCheckSum; // step 30 + private ChecksumType newCheckSumType; //step 30 // -- Optional private DataFile fileToReplace; // step 25 @@ -552,6 +554,7 @@ private boolean runAddReplacePhase1(Dataset owner, if(optionalFileParams != null) { if(optionalFileParams.hasCheckSum()) { newCheckSum = optionalFileParams.getCheckSum(); + newCheckSumType = optionalFileParams.getCheckSumType(); } } @@ -1131,6 +1134,7 @@ private boolean step_030_createNewFilesViaIngest(){ this.newFileContentType, this.newStorageIdentifier, this.newCheckSum, + this.newCheckSumType, this.systemConfig); } catch (IOException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index e48d96e355d..7fb8134630c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -10,7 +10,9 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.internal.LinkedTreeMap; import com.google.gson.reflect.TypeToken; + import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.DataFileTag; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.api.Util; @@ -68,8 +70,12 @@ public class OptionalFileParams { public static final String FILE_NAME_ATTR_NAME = "fileName"; private String mimeType; public static final String MIME_TYPE_ATTR_NAME = "mimeType"; - private String checkSum; - public static final String CHECKSUM_ATTR_NAME = "md5Hash"; + private String checkSumValue; + private ChecksumType checkSumType; + public static final String LEGACY_CHECKSUM_ATTR_NAME = "md5Hash"; + public static final String CHECKSUM_OBJECT_NAME = "checksum"; + public static final String CHECKSUM_OBJECT_TYPE = "@type"; + public static final String CHECKSUM_OBJECT_VALUE = "@value"; public OptionalFileParams(String jsonData) throws DataFileTagException{ @@ -218,16 +224,20 @@ public String getMimeType() { } public void setCheckSum(String checkSum) { - this.checkSum = checkSum; + this.checkSumValue = checkSum; } public boolean hasCheckSum() { - return ((checkSum!=null)&&(!checkSum.isEmpty())); + return ((checkSumValue!=null)&&(!checkSumValue.isEmpty())); } public String getCheckSum() { - return checkSum; + return checkSumValue; } + + public ChecksumType getCheckSumType() { + return checkSumType; + } /** * Set tags @@ -350,11 +360,21 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{ } // ------------------------------- - // get checkSum as string + // get md5 checkSum as string + // ------------------------------- + if ((jsonObj.has(LEGACY_CHECKSUM_ATTR_NAME)) && (!jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).isJsonNull())){ + + this.checkSumValue = jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).getAsString(); + this.checkSumType= ChecksumType.MD5; + } // ------------------------------- - if ((jsonObj.has(CHECKSUM_ATTR_NAME)) && (!jsonObj.get(CHECKSUM_ATTR_NAME).isJsonNull())){ + // get checkSum type and value + // ------------------------------- + else if ((jsonObj.has(CHECKSUM_OBJECT_NAME)) && (!jsonObj.get(CHECKSUM_OBJECT_NAME).isJsonNull())){ + + this.checkSumValue = ((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_VALUE).getAsString(); + this.checkSumType = ChecksumType.fromString(((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_TYPE).getAsString()); - this.checkSum = jsonObj.get(CHECKSUM_ATTR_NAME).getAsString(); } // ------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index c30927281f6..ad70b128ea0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -719,7 +719,17 @@ public static String generateOriginalExtension(String fileType) { return ""; } - public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, SystemConfig systemConfig) throws IOException { + public static List createDataFiles(DatasetVersion version, InputStream inputStream, + String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, + SystemConfig systemConfig) throws IOException { + ChecksumType checkSumType = DataFile.ChecksumType.MD5; + if (newStorageIdentifier == null) { + checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); + } + return createDataFiles(version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, checkSumType, systemConfig); + } + + public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, ChecksumType newCheckSumType, SystemConfig systemConfig) throws IOException { List datafiles = new ArrayList<>(); String warningMessage = null; @@ -1106,12 +1116,9 @@ public static List createDataFiles(DatasetVersion version, InputStream if (tempFile != null) { newFile = tempFile.toFile(); } - ChecksumType checkSumType = DataFile.ChecksumType.MD5; - if (newStorageIdentifier == null) { - checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); - } + - DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, checkSumType, newCheckSum); + DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum); File f = null; if (tempFile != null) { f = tempFile.toFile(); From d8b4fb3d853dcff1f885e09addc02fe96ab91435 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 10 Feb 2021 14:54:49 -0500 Subject: [PATCH 178/274] support different checksum algorithms in direct upload --- .../datasetutility/AddReplaceFileHelper.java | 14 +++++--- .../datasetutility/OptionalFileParams.java | 36 ++++++++++++++----- .../harvard/iq/dataverse/util/FileUtil.java | 19 ++++++---- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index ea1cfc38cfa..a8c52414e56 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -6,6 +6,7 @@ package edu.harvard.iq.dataverse.datasetutility; import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; @@ -117,11 +118,12 @@ public class AddReplaceFileHelper{ // ----------------------------------- private Dataset dataset; // constructor (for add, not replace) private DataverseRequest dvRequest; // constructor - private InputStream newFileInputStream; // step 20 - private String newFileName; // step 20 - private String newFileContentType; // step 20 - private String newStorageIdentifier; // step 20 - private String newCheckSum; // step 20 + private InputStream newFileInputStream; // step 30 + private String newFileName; // step 30 + private String newFileContentType; // step 30 + private String newStorageIdentifier; // step 30 + private String newCheckSum; // step 30 + private ChecksumType newCheckSumType; //step 30 // -- Optional private DataFile fileToReplace; // step 25 @@ -552,6 +554,7 @@ private boolean runAddReplacePhase1(Dataset owner, if(optionalFileParams != null) { if(optionalFileParams.hasCheckSum()) { newCheckSum = optionalFileParams.getCheckSum(); + newCheckSumType = optionalFileParams.getCheckSumType(); } } @@ -1131,6 +1134,7 @@ private boolean step_030_createNewFilesViaIngest(){ this.newFileContentType, this.newStorageIdentifier, this.newCheckSum, + this.newCheckSumType, this.systemConfig); } catch (IOException ex) { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index e48d96e355d..7fb8134630c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -10,7 +10,9 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.internal.LinkedTreeMap; import com.google.gson.reflect.TypeToken; + import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.DataFileTag; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.api.Util; @@ -68,8 +70,12 @@ public class OptionalFileParams { public static final String FILE_NAME_ATTR_NAME = "fileName"; private String mimeType; public static final String MIME_TYPE_ATTR_NAME = "mimeType"; - private String checkSum; - public static final String CHECKSUM_ATTR_NAME = "md5Hash"; + private String checkSumValue; + private ChecksumType checkSumType; + public static final String LEGACY_CHECKSUM_ATTR_NAME = "md5Hash"; + public static final String CHECKSUM_OBJECT_NAME = "checksum"; + public static final String CHECKSUM_OBJECT_TYPE = "@type"; + public static final String CHECKSUM_OBJECT_VALUE = "@value"; public OptionalFileParams(String jsonData) throws DataFileTagException{ @@ -218,16 +224,20 @@ public String getMimeType() { } public void setCheckSum(String checkSum) { - this.checkSum = checkSum; + this.checkSumValue = checkSum; } public boolean hasCheckSum() { - return ((checkSum!=null)&&(!checkSum.isEmpty())); + return ((checkSumValue!=null)&&(!checkSumValue.isEmpty())); } public String getCheckSum() { - return checkSum; + return checkSumValue; } + + public ChecksumType getCheckSumType() { + return checkSumType; + } /** * Set tags @@ -350,11 +360,21 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{ } // ------------------------------- - // get checkSum as string + // get md5 checkSum as string + // ------------------------------- + if ((jsonObj.has(LEGACY_CHECKSUM_ATTR_NAME)) && (!jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).isJsonNull())){ + + this.checkSumValue = jsonObj.get(LEGACY_CHECKSUM_ATTR_NAME).getAsString(); + this.checkSumType= ChecksumType.MD5; + } // ------------------------------- - if ((jsonObj.has(CHECKSUM_ATTR_NAME)) && (!jsonObj.get(CHECKSUM_ATTR_NAME).isJsonNull())){ + // get checkSum type and value + // ------------------------------- + else if ((jsonObj.has(CHECKSUM_OBJECT_NAME)) && (!jsonObj.get(CHECKSUM_OBJECT_NAME).isJsonNull())){ + + this.checkSumValue = ((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_VALUE).getAsString(); + this.checkSumType = ChecksumType.fromString(((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_TYPE).getAsString()); - this.checkSum = jsonObj.get(CHECKSUM_ATTR_NAME).getAsString(); } // ------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index c30927281f6..ad70b128ea0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -719,7 +719,17 @@ public static String generateOriginalExtension(String fileType) { return ""; } - public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, SystemConfig systemConfig) throws IOException { + public static List createDataFiles(DatasetVersion version, InputStream inputStream, + String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, + SystemConfig systemConfig) throws IOException { + ChecksumType checkSumType = DataFile.ChecksumType.MD5; + if (newStorageIdentifier == null) { + checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); + } + return createDataFiles(version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, checkSumType, systemConfig); + } + + public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, ChecksumType newCheckSumType, SystemConfig systemConfig) throws IOException { List datafiles = new ArrayList<>(); String warningMessage = null; @@ -1106,12 +1116,9 @@ public static List createDataFiles(DatasetVersion version, InputStream if (tempFile != null) { newFile = tempFile.toFile(); } - ChecksumType checkSumType = DataFile.ChecksumType.MD5; - if (newStorageIdentifier == null) { - checkSumType = systemConfig.getFileFixityChecksumAlgorithm(); - } + - DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, checkSumType, newCheckSum); + DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum); File f = null; if (tempFile != null) { f = tempFile.toFile(); From 9ddbbead1e1a813d87b16482d9e30b06905df48d Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 10 Feb 2021 15:18:19 -0500 Subject: [PATCH 179/274] Fixed extra returns copied with api token on account pg [ref #7601] --- src/main/webapp/dataverseuser.xhtml | 2 +- src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index ba25798e081..5de0154f49c 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -521,7 +521,7 @@ //]]> diff --git a/src/main/webapp/provenance-popups-fragment.xhtml b/src/main/webapp/provenance-popups-fragment.xhtml index 6e1663fbe05..df993f03950 100644 --- a/src/main/webapp/provenance-popups-fragment.xhtml +++ b/src/main/webapp/provenance-popups-fragment.xhtml @@ -94,9 +94,7 @@ forceSelection="true" converter="provEntityFileDataConverter" styleClass="DropdownPopup" panelStyleClass="DropdownPopupPanel" - onkeypress="if (event.keyCode == 13) { - return false; - }"> + onkeypress="if (event.keyCode == 13) {return false;}"> @@ -107,9 +105,6 @@ -
    diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 44e86831c69..8b6a7bff5d2 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -41,6 +41,9 @@ function bind_bsui_components(){ // clipboard.js click to copy clickCopyClipboard(); + // Scrolling autoComplete dropdown in popups + handle_dropdown_popup_scroll(); + // Dialog Listener For Calling handleResizeDialog PrimeFaces.widget.Dialog.prototype.postShow = function() { var dialog_id = this.jq.attr('id').split(/[:]+/).pop(); diff --git a/src/main/webapp/roles-assign.xhtml b/src/main/webapp/roles-assign.xhtml index a3f73793d7b..672086a7882 100644 --- a/src/main/webapp/roles-assign.xhtml +++ b/src/main/webapp/roles-assign.xhtml @@ -40,9 +40,6 @@ -
    From c5121a676150bbd32428c832a06e50f374ee9a14 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 18 Feb 2021 10:42:06 -0500 Subject: [PATCH 208/274] #3175 fix input level editing --- src/main/webapp/dataverse.xhtml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index a069d87e9c7..96d523c335d 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -330,8 +330,9 @@
    - + @@ -345,6 +346,7 @@
    From cb964bd4379fdf1532bddb9eb9983a4f92309285 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 18 Feb 2021 15:44:09 -0500 Subject: [PATCH 209/274] #3175 fixes 1198 --- src/main/webapp/dataverse.xhtml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 96d523c335d..62f03b64be1 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -287,6 +287,7 @@ From 9b0cab63b60aff0ebc8a902e9bc34bd161bbb000 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 19 Feb 2021 17:13:25 -0500 Subject: [PATCH 210/274] adding note to dataverse management about conditionally required --- doc/sphinx-guides/source/user/dataverse-management.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index 29685936b4d..a5bfbf7989b 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -28,7 +28,7 @@ Creating a Dataverse collection is easy but first you must be a registered user #. **Choose the sets of Metadata Fields for datasets in this Dataverse collection**: * By default the metadata elements will be from the host Dataverse collection that this new Dataverse collection is created in. * The Dataverse Software offers metadata standards for multiple domains. To learn more about the metadata standards in the Dataverse Software please check out the :doc:`/user/appendix`. - * Metadata fields can be hidden, or selected as required or optional. + * Metadata fields can be hidden, or selected as required, optional, or conditionally required. * Selected metadata elements are also used to pick which metadata fields you would like to use for creating :ref:`Dataset Templates `: after you finish creating your Dataverse collection. #. **Choose which metadata fields will be used as browse/search facets on your Dataverse collection**: * These facets will allow users browsing or searching your Dataverse collection to filter its contents according to the fields you have selected. For example, if you select “Subject” as a facet, users will be able to filter your Dataverse collection’s contents by subject area. From c90befec367479701183d8e1f699dcfff7dec54a Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 22 Feb 2021 11:33:21 -0500 Subject: [PATCH 211/274] #3175 remove unused method/onclick --- .../java/edu/harvard/iq/dataverse/DataversePage.java | 12 ------------ src/main/webapp/dataverse.xhtml | 1 - 2 files changed, 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 2bc7e7eaf27..d4aede8591c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -511,18 +511,6 @@ public void hideDatasetFieldTypes(Long mdbId) { } setEditInputLevel(false); } - - public void toggleInputLevel( Long mdbId, long dsftId){ - for (MetadataBlock mdb : allMetadataBlocks) { - if (mdb.getId().equals(mdbId)) { - for (DatasetFieldType dsftTest : mdb.getDatasetFieldTypes()) { - if (dsftTest.getId().equals(dsftId)) { - dsftTest.setRequiredDV(!dsftTest.isRequiredDV()); - } - } - } - } - } public void updateInclude(Long mdbId, long dsftId) { List childDSFT = new ArrayList<>(); diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 62f03b64be1..284b7135f09 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -332,7 +332,6 @@ From d7c8611a5c58b41715297f2c4f9d4d81c78f4d18 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 22 Feb 2021 11:51:20 -0500 Subject: [PATCH 212/274] Update dataverse-management.rst Revising sentence about controlling which fields are shown, hidden, required and conditionally required in Dataverse collections. --- doc/sphinx-guides/source/user/dataverse-management.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index a5bfbf7989b..5cc58a970ac 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -28,8 +28,8 @@ Creating a Dataverse collection is easy but first you must be a registered user #. **Choose the sets of Metadata Fields for datasets in this Dataverse collection**: * By default the metadata elements will be from the host Dataverse collection that this new Dataverse collection is created in. * The Dataverse Software offers metadata standards for multiple domains. To learn more about the metadata standards in the Dataverse Software please check out the :doc:`/user/appendix`. - * Metadata fields can be hidden, or selected as required, optional, or conditionally required. - * Selected metadata elements are also used to pick which metadata fields you would like to use for creating :ref:`Dataset Templates `: after you finish creating your Dataverse collection. + * Most metadata fields can be hidden or can be selected as required or optional. Some fields may be selected as conditionally required, depending on metadata options chosen for the Dataverse installation. + * Selected metadata fields are also used to pick which metadata fields you would like to use for creating :ref:`Dataset Templates `: after you finish creating your Dataverse collection. #. **Choose which metadata fields will be used as browse/search facets on your Dataverse collection**: * These facets will allow users browsing or searching your Dataverse collection to filter its contents according to the fields you have selected. For example, if you select “Subject” as a facet, users will be able to filter your Dataverse collection’s contents by subject area. * By default, the facets that will appear on your Dataverse collection's landing page will be from the host Dataverse collection that this new Dataverse collection was created in, but you can add or remove facets from this default. From 444a73cd94e8d2b7c46751dd0855121ca5763f57 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 22 Feb 2021 12:01:34 -0500 Subject: [PATCH 213/274] Added conditionally required metadata field info Added conditionally required metadata field info to Introduction section --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index bc0b18112ab..f35d46e514e 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -9,7 +9,7 @@ The Dataverse Software has a flexible data-driven metadata system powered by "me Introduction ------------ -Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make field required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. +Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make field required or conditionally required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. Much more customization of metadata is possible, but this is an advanced topic so feedback on what is written below is very welcome. The possibilities for customization include: From f0fa6d21bb79af1561645af5faf4dca800990db8 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 22 Feb 2021 13:34:28 -0500 Subject: [PATCH 214/274] Update metadatacustomization.rst Added info about conditionally required fields to "required" row of the "datasetField (field) properties" table --- .../source/admin/metadatacustomization.rst | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index f35d46e514e..2ac0ec0347b 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -289,16 +289,37 @@ Each of the three main sections own sets of properties: | | the dataset has been | | | | saved. | | +-----------------------+-----------------------+------------------------+ -| required | Specify whether or | TRUE (required) or | -| | not the field is | FALSE (optional) | -| | required. This means | | -| | that at least one | | -| | instance of the field | | -| | must be present. More | | -| | than one field may be | | -| | allowed, depending on | | -| | the value of | | -| | allowmultiples. | | +| required | For primitive | For primitive | +| | fields, specify | fields, TRUE | +| | whether or not the | (required) or FALSE | +| | field is required. | (optional). | +| | For compound | | +| | fields, also | For compound fields: | +| | specify if one or | | +| | more subfields are | \• To make one or more | +| | required or | subfields optional, | +| | conditionally | the parent field and | +| | required. At least | subfield(s) must be | +| | one instance of a | FALSE (optional). | +| | required field must | | +| | be present. More | \• To make one or more | +| | than one instance | subfields required, | +| | of a field may be | the parent field and | +| | allowed, depending | the required | +| | on the value of | subfield(s) must be | +| | allowmultiples. | TRUE (required). | +| | | | +| | | \• To make one or more | +| | | subfields | +| | | conditionally | +| | | required, make the | +| | | parent field FALSE | +| | | (optional) and make | +| | | TRUE (required) any | +| | | subfield or subfields | +| | | that are required if | +| | | any other subfields | +| | | are filled. | +-----------------------+-----------------------+------------------------+ | parent | For subfields, | \• Must not result in | | | specify the name of | a cyclical | From e81a8770e1e39e6582da75a75f5c6123b8f5b8f9 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 22 Feb 2021 15:44:22 -0500 Subject: [PATCH 215/274] Update metadatacustomization.rst Fixing small typo --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 2ac0ec0347b..fdbac996108 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -9,7 +9,7 @@ The Dataverse Software has a flexible data-driven metadata system powered by "me Introduction ------------ -Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make field required or conditionally required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. +Before you embark on customizing metadata in your Dataverse installation you should make sure you are aware of the modest amount of customization that is available with your Dataverse installation's web interface. It's possible to hide fields and make fields required or conditionally required by clicking "Edit" at the Dataverse collection level, clicking "General Information" and making adjustments under "Metadata Fields" as described in the :ref:`create-dataverse` section of the Dataverse Collection Management page in the User Guide. Much more customization of metadata is possible, but this is an advanced topic so feedback on what is written below is very welcome. The possibilities for customization include: From acd317cc25b238dbc64c2c346cebddf9624c9910 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Mon, 22 Feb 2021 18:24:31 -0500 Subject: [PATCH 216/274] Update V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql --- ...V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql b/src/main/resources/db/migration/V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql index 56b699fa366..f9cb9d0e526 100644 --- a/src/main/resources/db/migration/V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql +++ b/src/main/resources/db/migration/V5.3.0.3__7551-expanded-compound-datasetfield-validation.sql @@ -10,7 +10,7 @@ DELETE FROM dataversefieldtypeinputlevel WHERE datasetfieldtype_id in ( SELECT parentdatasetfieldtype_id FROM datasetfieldtype dsft); INSERT INTO dataversefieldtypeinputlevel (include,required,datasetfieldtype_id,dataverse_id) -SELECT true, true, parentdatasetfieldtype_id, dataverse_id FROM +SELECT DISTINCT true, true, parentdatasetfieldtype_id, dataverse_id FROM dataversefieldtypeinputlevel dsftil, datasetfieldtype dsft WHERE dsftil.datasetfieldtype_id = dsft.id AND parentdatasetfieldtype_id IS NOT null; From 78e253418d1b9cbdab12985b4b748f43be07bd9a Mon Sep 17 00:00:00 2001 From: gdurand Date: Mon, 22 Feb 2021 18:50:09 -0500 Subject: [PATCH 217/274] added an else --- .../harvard/iq/dataverse/DatasetField.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java index eee2684bfe1..79f8916deb9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetField.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetField.java @@ -421,21 +421,21 @@ public boolean isRequired() { if (this.datasetFieldType.isRequired()) { required = true; - } - - // otherwise check if overridden at the dataverse level - Dataverse dv = getDataverse(); - while (!dv.isMetadataBlockRoot()) { - if (dv.getOwner() == null) { - break; // we are at the root; which by defintion is metadata blcok root, regarldess of the value + } else { + // otherwise check if overridden at the dataverse level + Dataverse dv = getDataverse(); + while (!dv.isMetadataBlockRoot()) { + if (dv.getOwner() == null) { + break; // we are at the root; which by defintion is metadata blcok root, regarldess of the value + } + dv = dv.getOwner(); } - dv = dv.getOwner(); - } - List dftilListFirst = dv.getDataverseFieldTypeInputLevels(); - for (DataverseFieldTypeInputLevel dsftil : dftilListFirst) { - if (dsftil.getDatasetFieldType().equals(this.datasetFieldType)) { - required = dsftil.isRequired(); + List dftilListFirst = dv.getDataverseFieldTypeInputLevels(); + for (DataverseFieldTypeInputLevel dsftil : dftilListFirst) { + if (dsftil.getDatasetFieldType().equals(this.datasetFieldType)) { + required = dsftil.isRequired(); + } } } From 6daa2b4fffd8664f581280dde2eb087557d591e6 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Mon, 22 Feb 2021 19:15:15 -0500 Subject: [PATCH 218/274] Update 7551-expanded-compound-datasetfield-validation.md --- ...551-expanded-compound-datasetfield-validation.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index 662a5d31fa1..5541f6deb93 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -14,12 +14,19 @@ Going forward: - For required, the parent compound field would be required = true and at least one child would be required = true. - For conditionally required, the parent compound field would be required = false and at least one child would be required = true. -This release updates the citation .tsv file that us distributed with the software (no other distributed .tsv file has any required values). **If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. +This release updates the citation .tsv file that us distributed with the software for the required parent compound fields (e.g. author), as well as sets Producer Name to be conditionally required. No other distributed .tsv files were updated, as they did not have any required compound values. + +**If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. -Note: The metadata .tsv files do not have to be reloaded via API, as there is an accompanying Flyway script that updates the values directly in the database. +Note: With the exception of citation.tsv (due to the conditionally required Producer Name), the metadata .tsv files do not have to be reloaded via API, as there is an accompanying Flyway script that updates the values directly in the database. ### Additional Upgrade Steps -1. Update any custom metadata blocks (if used) \ No newline at end of file +1. Update any custom metadata blocks (if used) + +2. Reload Citation Metadata Block: + + `wget https://github.com/IQSS/dataverse/releases/download/v5.4/citation.tsv` + `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` From 56929541616d6fbf27a82ba7ab883c7138652397 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Mon, 22 Feb 2021 19:16:57 -0500 Subject: [PATCH 219/274] Update 7551-expanded-compound-datasetfield-validation.md --- .../7551-expanded-compound-datasetfield-validation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index 5541f6deb93..7275a827633 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -14,7 +14,7 @@ Going forward: - For required, the parent compound field would be required = true and at least one child would be required = true. - For conditionally required, the parent compound field would be required = false and at least one child would be required = true. -This release updates the citation .tsv file that us distributed with the software for the required parent compound fields (e.g. author), as well as sets Producer Name to be conditionally required. No other distributed .tsv files were updated, as they did not have any required compound values. +This release updates the citation .tsv file that is distributed with the software for the required parent compound fields (e.g. author), as well as sets Producer Name to be conditionally required. No other distributed .tsv files were updated, as they did not have any required compound values. **If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. From fdfc3ba7685d1fc4f8bb02c523c10f30606a3561 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 23 Feb 2021 09:56:39 -0500 Subject: [PATCH 220/274] Update 7551-expanded-compound-datasetfield-validation.md --- ...xpanded-compound-datasetfield-validation.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index 7275a827633..b1ad275c2f2 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -1,6 +1,6 @@ ## Notes for Dataverse Administrators -Prior to this release, when defining metadata for compound fields, fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. +Prior to this release, when defining metadata for compound fields (via their datasrt field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. In order to support more robust metadata (and specifically to resolve #7551), we need to allow a third case: Conditionally Required, that is, the field is required if and only if any of its "sibling" fields are required. For example, a user does not have to enter a Producer, but if they do, they have to enter a Producer Name. @@ -18,15 +18,17 @@ This release updates the citation .tsv file that is distributed with the softwar **If you have created any custom metadata .tsv files**, you will need to make the same (type of) changes there. -For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. - -Note: With the exception of citation.tsv (due to the conditionally required Producer Name), the metadata .tsv files do not have to be reloaded via API, as there is an accompanying Flyway script that updates the values directly in the database. - ### Additional Upgrade Steps -1. Update any custom metadata blocks (if used) - -2. Reload Citation Metadata Block: +1. Reload Citation Metadata Block: `wget https://github.com/IQSS/dataverse/releases/download/v5.4/citation.tsv` `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` + +2. Update any custom metadata blocks (if used): + +For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. + +Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. + + From cfb1fbee13daee5a6aca9678bf7f54a901b2fd1f Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 23 Feb 2021 10:04:59 -0500 Subject: [PATCH 221/274] Update 7551-expanded-compound-datasetfield-validation.md --- .../7551-expanded-compound-datasetfield-validation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index b1ad275c2f2..14b8e4d1aab 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -27,8 +27,8 @@ This release updates the citation .tsv file that is distributed with the softwar 2. Update any custom metadata blocks (if used): -For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. + For any subfield that has a required value of TRUE, find the corresponding parent field and change its required value to TRUE. -Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. + Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. From 01e9666b67bf02b54bdedbf0c27c808189c1375c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 23 Feb 2021 11:18:43 -0500 Subject: [PATCH 222/274] typo, review Looks good @scolapasta. I will add a use case at release time, or you can do so now. --- .../7551-expanded-compound-datasetfield-validation.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index 14b8e4d1aab..cf8b9a7ba33 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -1,6 +1,6 @@ ## Notes for Dataverse Administrators -Prior to this release, when defining metadata for compound fields (via their datasrt field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. +Prior to this release, when defining metadata for compound fields (via their dataset field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. In order to support more robust metadata (and specifically to resolve #7551), we need to allow a third case: Conditionally Required, that is, the field is required if and only if any of its "sibling" fields are required. For example, a user does not have to enter a Producer, but if they do, they have to enter a Producer Name. @@ -31,4 +31,3 @@ This release updates the citation .tsv file that is distributed with the softwar Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. - From 3bf665e31a6c89f6ab56e1e5a1521603d46f39d6 Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 23 Feb 2021 12:48:52 -0500 Subject: [PATCH 223/274] Added a use case --- .../7551-expanded-compound-datasetfield-validation.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md index cf8b9a7ba33..19d6d573699 100644 --- a/doc/release-notes/7551-expanded-compound-datasetfield-validation.md +++ b/doc/release-notes/7551-expanded-compound-datasetfield-validation.md @@ -2,7 +2,7 @@ Prior to this release, when defining metadata for compound fields (via their dataset field types), fields could be either be optional or required, i.e. if required you must always have (at least one) value for that field. For example, Author Name being required means you must have at least one Author with an nonempty Author name. -In order to support more robust metadata (and specifically to resolve #7551), we need to allow a third case: Conditionally Required, that is, the field is required if and only if any of its "sibling" fields are required. For example, a user does not have to enter a Producer, but if they do, they have to enter a Producer Name. +In order to support more robust metadata (and specifically to resolve #7551), we need to allow a third case: Conditionally Required, that is, the field is required if and only if any of its "sibling" fields are entered. For example, Producer Name is now conditionally required in the citation metadata block. A user does not have to enter a Producer, but if they do, they have to enter a Producer Name. This change required some modifications to how "required" is defined in the metadata .tsv files (for compound fields). @@ -31,3 +31,6 @@ This release updates the citation .tsv file that is distributed with the softwar Note: As there is an accompanying Flyway script that updates the values directly in the database, you do not need to reload these metadata .tsv files via API, unless you make additional changes, e.g set some compound fields to be conditionally required. +### Use Case + +Metadata designers can now set subfields of compound fields as **conditionally required**, that is, the field is required if and only if any of its "sibling" fields are entered. For example, Producer Name is now conditionally required in the citation metadata block. A user does not have to enter a Producer, but if they do, they have to enter a Producer Name. From 67a28b4406de54219034bed00defadc217036a8e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 09:33:25 -0500 Subject: [PATCH 224/274] sync with 7564 changes sans messaging --- .../edu/harvard/iq/dataverse/DatasetPage.java | 10 --------- .../iq/dataverse/PermissionServiceBean.java | 2 +- .../harvard/iq/dataverse/api/Datasets.java | 6 ++++-- .../FinalizeDatasetPublicationCommand.java | 21 +++++++++---------- .../command/impl/PublishDatasetCommand.java | 11 +++++----- .../harvard/iq/dataverse/util/MailUtil.java | 4 ---- .../workflow/PendingWorkflowInvocation.java | 18 +++++++--------- .../internalspi/InternalWorkflowStepSP.java | 2 ++ .../db/migration/V5.4.0.1__7563-workflow.sql | 10 +-------- 9 files changed, 31 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 04a50bd712d..112c57b7f41 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1858,16 +1858,6 @@ private String init(boolean initFull) { MakeDataCountEntry entry = new MakeDataCountEntry(FacesContext.getCurrentInstance(), dvRequestService, workingVersion); mdcLogService.logEntry(entry); } - - List comments = workingVersion.getWorkflowComments(); - for(WorkflowComment wfc: comments) { - if(wfc.getType()==WorkflowComment.Type.WORKFLOW_SUCCESS) { - JsfHelper.addSuccessMessage(wfc.getMessage()); - - } else if (wfc.getType()==WorkflowComment.Type.WORKFLOW_FAILURE) { - JsfHelper.addWarningMessage(wfc.getMessage()); - } - } if (initFull) { // init the list of FileMetadatas diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index f38efa7b66b..aaf38af1b36 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -816,7 +816,7 @@ public boolean isMatchingWorkflowLock(Dataset d, String userId, String invocatio if (invocationId != null) { PendingWorkflowInvocation pwfi = workflowService.getPendingWorkflow(invocationId); for (DatasetLock dl : d.getLocks()) { - if (dl.equals(pwfi.getLock()) && pwfi.getUserId().equals(userId)) { + if (dl.getId().equals(pwfi.getLockId()) && pwfi.getUserId().equals(userId)) { return true; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 67f52c9cbe0..5cd87adae82 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -977,7 +977,9 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S logger.fine("MT: " + ds.getModificationTime()); logger.fine("PIT: " + ds.getPermissionIndexTime()); logger.fine("PMT: " + ds.getPermissionModificationTime()); - logger.fine("ITMT: " + (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)); + if (ds.getIndexTime() != null && ds.getModificationTime() != null) { + logger.fine("ITMT: " + (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)); + } /* * Some calls, such as the /datasets/actions/:import* commands do not set the * modification or permission modification times. The checks here are trying to @@ -989,7 +991,7 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S * */ if ((ds.getModificationTime()!=null && (ds.getIndexTime() == null || (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0))) || - (ds.getPermissionModificationTime()!=null && (ds.getIndexTime() == null || (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)))) { + (ds.getPermissionModificationTime()!=null && (ds.getPermissionIndexTime() == null || (ds.getPermissionIndexTime().compareTo(ds.getPermissionModificationTime()) <= 0)))) { return error(Response.Status.CONFLICT, "Dataset is awaiting indexing"); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 0da5ee7535d..da9c4515752 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -183,28 +183,27 @@ public Dataset execute(CommandContext ctxt) throws CommandException { final Dataset ds = ctxt.em().merge(theDataset); //Remove any pre-pub workflow lock (not needed as WorkflowServiceBean.workflowComplete() should already have removed it after setting the finalizePublication lock?) ctxt.datasets().removeDatasetLocks(ds, DatasetLock.Reason.Workflow); + + //Should this be in onSuccess()? ctxt.workflows().getDefaultWorkflow(TriggerType.PostPublishDataset).ifPresent(wf -> { try { - WorkflowContext wfc = buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased); - // Set workflow lock before returning and before releasing the - // finalizePublication lock - ctxt.workflows().lockDataset(wfc); - ctxt.workflows().start(wf, wfc); + ctxt.workflows().start(wf, buildContext(ds, TriggerType.PostPublishDataset, datasetExternallyReleased)); } catch (CommandException ex) { + ctxt.datasets().removeDatasetLocks(ds, DatasetLock.Reason.Workflow); logger.log(Level.SEVERE, "Error invoking post-publish workflow: " + ex.getMessage(), ex); } }); - Dataset readyDataset = ctxt.em().merge(theDataset); + Dataset readyDataset = ctxt.em().merge(ds); // Finally, unlock the dataset (leaving any post-publish workflow lock in place) - ctxt.datasets().removeDatasetLocks(theDataset, DatasetLock.Reason.finalizePublication); - if ( theDataset.isLockedFor(DatasetLock.Reason.InReview) ) { - ctxt.datasets().removeDatasetLocks(theDataset, DatasetLock.Reason.InReview); + ctxt.datasets().removeDatasetLocks(readyDataset, DatasetLock.Reason.finalizePublication); + if (readyDataset.isLockedFor(DatasetLock.Reason.InReview) ) { + ctxt.datasets().removeDatasetLocks(readyDataset, DatasetLock.Reason.InReview); } - logger.info("Successfully published the dataset "+theDataset.getGlobalId().asString()); - + logger.info("Successfully published the dataset "+readyDataset.getGlobalId().asString()); + readyDataset = ctxt.em().merge(readyDataset); return readyDataset; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index a47d2a39d87..a2d206ead0d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -70,6 +70,10 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException Dataset theDataset = getDataset(); + + //ToDo - any reason to set the version in publish versus finalize? Failure in a prepub workflow or finalize will leave draft versions with an assigned version number as is. + //Changing the dataset in this transaction also potentially makes a race condition with a prepub workflow, possibly resulting in an OptimisticLockException there. + // Set the version numbers: if (theDataset.getPublicationDate() == null) { @@ -87,16 +91,13 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException theDataset.getLatestVersion().setMinorVersionNumber(new Long(0)); } + //ToDo - should this be in onSuccess()? May relate to todo above Optional prePubWf = ctxt.workflows().getDefaultWorkflow(TriggerType.PrePublishDataset); if ( prePubWf.isPresent() ) { // We start a workflow theDataset = ctxt.em().merge(theDataset); ctxt.em().flush(); - - WorkflowContext wfc = buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased); - //add workflow lock before returning - ctxt.workflows().lockDataset(wfc); - ctxt.workflows().start(prePubWf.get(), wfc); + ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased)); return new PublishDatasetResult(theDataset, Status.Workflow); } else{ diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index 2ffec60516c..37667d16b55 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -54,10 +54,6 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti return BundleUtil.getStringFromBundle("notification.email.publishFailure.dataset.subject", rootDvNameAsList); case RETURNEDDS: return BundleUtil.getStringFromBundle("notification.email.returned.dataset.subject", rootDvNameAsList); - case WORKFLOW_SUCCESS: - return BundleUtil.getStringFromBundle("notification.email.workflow.success.dataset.subject", rootDvNameAsList); - case WORKFLOW_FAILURE: - return BundleUtil.getStringFromBundle("notification.email.workflow.failure.dataset.subject", rootDvNameAsList); case CREATEACC: return BundleUtil.getStringFromBundle("notification.email.create.account.subject", rootDvNameAsList); case CHECKSUMFAIL: diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java index b211347568b..577e0f756de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/PendingWorkflowInvocation.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.workflow; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.User; @@ -10,8 +9,6 @@ import java.io.Serializable; import java.util.HashMap; import java.util.Map; - -import javax.persistence.CascadeType; import javax.persistence.ElementCollection; import javax.persistence.Entity; import javax.persistence.FetchType; @@ -55,8 +52,7 @@ public class PendingWorkflowInvocation implements Serializable { int typeOrdinal; boolean datasetExternallyReleased; - @OneToOne - DatasetLock lock; + long lockid; public static final String AUTHORIZED= "authorized"; @@ -76,13 +72,13 @@ public PendingWorkflowInvocation(Workflow wf, WorkflowContext ctxt, Pending resu localData = new HashMap<>(result.getData()); typeOrdinal = ctxt.getType().ordinal(); datasetExternallyReleased=ctxt.getDatasetExternallyReleased(); - lock=ctxt.getLock(); + lockid=ctxt.getLockId(); } public WorkflowContext reCreateContext(RoleAssigneeServiceBean roleAssignees) { DataverseRequest aRequest = new DataverseRequest((User)roleAssignees.getRoleAssignee(userId), IpAddress.valueOf(ipAddress)); final WorkflowContext workflowContext = new WorkflowContext(aRequest, dataset, nextVersionNumber, - nextMinorVersionNumber, WorkflowContext.TriggerType.values()[typeOrdinal], null, null, datasetExternallyReleased, invocationId, lock); + nextMinorVersionNumber, WorkflowContext.TriggerType.values()[typeOrdinal], null, null, datasetExternallyReleased, invocationId, lockid); return workflowContext; } @@ -166,10 +162,10 @@ public void setTypeOrdinal(int typeOrdinal) { this.typeOrdinal = typeOrdinal; } - public DatasetLock getLock() { - return lock; + public long getLockId() { + return lockid; } - public void setLock(DatasetLock lock) { - this.lock = lock; + public void setLockId(long lockId) { + this.lockid = lockId; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java index 76b7bcd7d44..df5f2de9058 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/InternalWorkflowStepSP.java @@ -19,6 +19,8 @@ public WorkflowStep getStep(String stepType, Map stepParameters) return new PauseStep(stepParameters); case "http/sr": return new HttpSendReceiveClientStep(stepParameters); + case "http/authExt": + return new AuthorizedExternalStep(stepParameters); case "archiver": return new ArchivalSubmissionWorkflowStep(stepParameters); default: diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql index 57ecec383a7..085a2e4bba3 100644 --- a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql +++ b/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql @@ -1,10 +1,2 @@ ALTER TABLE pendingworkflowinvocation -ADD COLUMN IF NOT EXISTS lock_id bigint; - -DO $$ -BEGIN -IF NOT EXISTS (SELECT 1 FROM information_schema.constraint_column_usage WHERE constraint_name='fk_pendingworkflowinvocation_lock_id') -THEN -ALTER TABLE pendingworkflowinvocation ADD CONSTRAINT fk_pendingworkflowinvocation_lock_id FOREIGN KEY (lock_id) REFERENCES datasetlock (id); -END IF; -END$$; \ No newline at end of file +ADD COLUMN IF NOT EXISTS lockid bigint; From dcb0537b3ced4e0dc8bbfdf433dde0e536f45b36 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 10:00:03 -0500 Subject: [PATCH 225/274] more to split from 7564 --- pom.xml | 12 -- .../edu/harvard/iq/dataverse/DatasetPage.java | 4 +- .../iq/dataverse/DatasetServiceBean.java | 6 - .../harvard/iq/dataverse/MailServiceBean.java | 15 --- .../iq/dataverse/UserNotification.java | 3 +- .../harvard/iq/dataverse/api/Datasets.java | 114 ++---------------- .../FinalizeDatasetPublicationCommand.java | 1 - .../command/impl/PublishDatasetCommand.java | 1 - .../dataverse/workflow/WorkflowContext.java | 19 ++- .../workflow/WorkflowServiceBean.java | 81 +++++++------ .../iq/dataverse/workflow/step/Failure.java | 2 +- .../iq/dataverse/workflow/step/Success.java | 47 -------- .../dataverse/workflows/WorkflowComment.java | 20 +-- src/main/java/propertyFiles/Bundle.properties | 4 - 14 files changed, 69 insertions(+), 260 deletions(-) delete mode 100644 src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java diff --git a/pom.xml b/pom.xml index 29ae1c49747..de8ff91863a 100644 --- a/pom.xml +++ b/pom.xml @@ -205,12 +205,6 @@ org.apache.abdera abdera-core 1.1.3 - - - org.apache.geronimo.specs - geronimo-stax-api_1.0_spec - - @@ -504,12 +498,6 @@ com.lyncode xoai-common 4.1.0-header-patch - - - javax.xml.stream - stax-api - - com.lyncode diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 112c57b7f41..5f596d1b556 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -54,8 +54,6 @@ import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.workflows.WorkflowComment; - import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -1858,7 +1856,7 @@ private String init(boolean initFull) { MakeDataCountEntry entry = new MakeDataCountEntry(FacesContext.getCurrentInstance(), dvRequestService, workingVersion); mdcLogService.logEntry(entry); } - + if (initFull) { // init the list of FileMetadatas if (workingVersion.isDraft() && canUpdateDataset()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 71eef6d6c5f..c1efe119fd2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -787,12 +787,6 @@ public WorkflowComment addWorkflowComment(WorkflowComment workflowComment) { return workflowComment; } - public void markWorkflowCommentAsRead(WorkflowComment workflowComment) { - workflowComment.setToBeShown(false); - em.merge(workflowComment); - } - - /** * This method used to throw CommandException, which was pretty pointless * seeing how it's called asynchronously. As of v5.0 any CommanExceptiom diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java index 42feaf6d2c2..13a92c9cd27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java @@ -496,19 +496,6 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio version.getDataset().getOwner().getDisplayName(), getDataverseLink(version.getDataset().getOwner()), optionalReturnReason}; messageText += MessageFormat.format(pattern, paramArrayReturnedDataset); return messageText; - - case WORKFLOW_SUCCESS: - version = (DatasetVersion) targetObject; - pattern = BundleUtil.getStringFromBundle("notification.email.workflow.success"); - String[] paramArrayWorkflowSuccess = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset()), comment}; - messageText += MessageFormat.format(pattern, paramArrayWorkflowSuccess); - return messageText; - case WORKFLOW_FAILURE: - version = (DatasetVersion) targetObject; - pattern = BundleUtil.getStringFromBundle("notification.email.workflow.failure"); - String[] paramArrayWorkflowFailure = {version.getDataset().getDisplayName(), getDatasetLink(version.getDataset())}; - messageText += MessageFormat.format(pattern, paramArrayWorkflowFailure); - return messageText; case CREATEACC: String rootDataverseName = dataverseService.findRootDataverse().getName(); InternetAddress systemAddress = getSystemAddress(); @@ -607,8 +594,6 @@ private Object getObjectOfNotification (UserNotification userNotification){ case PUBLISHEDDS: case PUBLISHFAILED_PIDREG: case RETURNEDDS: - case WORKFLOW_SUCCESS: - case WORKFLOW_FAILURE: return versionService.find(userNotification.getObjectId()); case CREATEACC: return userNotification.getUser(); diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java index 445e12f9ef8..e44c5f6333e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java +++ b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java @@ -30,7 +30,7 @@ public enum Type { ASSIGNROLE, REVOKEROLE, CREATEDV, CREATEDS, CREATEACC, SUBMITTEDDS, RETURNEDDS, PUBLISHEDDS, REQUESTFILEACCESS, GRANTFILEACCESS, REJECTFILEACCESS, FILESYSTEMIMPORT, CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS, - PUBLISHFAILED_PIDREG, WORKFLOW_SUCCESS, WORKFLOW_FAILURE + PUBLISHFAILED_PIDREG }; private static final long serialVersionUID = 1L; @@ -156,5 +156,4 @@ public void setRoleString(String roleString) { public String getLocaleSendDate() { return DateUtil.formatDate(sendDate); } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 5cd87adae82..7341e3f66ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -112,8 +112,6 @@ import java.io.StringReader; import java.sql.Timestamp; import java.text.MessageFormat; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -158,6 +156,7 @@ import org.glassfish.jersey.media.multipart.FormDataParam; import com.amazonaws.services.s3.model.PartETag; +import edu.harvard.iq.dataverse.FileMetadata; import java.util.Map.Entry; @Path("datasets") @@ -770,16 +769,11 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ @PUT @Path("{id}/editMetadata") - public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) { + public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) throws WrappedResponse{ Boolean replaceData = replace != null; - DataverseRequest req = null; - try { - req = createDataverseRequest(findUserOrDie()); - } catch (WrappedResponse ex) { - logger.log(Level.SEVERE, "Edit metdata error: " + ex.getMessage(), ex); - return ex.getResponse(); - } + + DataverseRequest req = createDataverseRequest(findUserOrDie()); return processDatasetUpdate(jsonBody, id, req, replaceData); } @@ -939,12 +933,12 @@ private String validateDatasetFieldValues(List fields) { @Deprecated public Response publishDataseUsingGetDeprecated( @PathParam("id") String id, @QueryParam("type") String type ) { logger.info("publishDataseUsingGetDeprecated called on id " + id + ". Encourage use of POST rather than GET, which is deprecated."); - return publishDataset(id, type, false); + return publishDataset(id, type); } @POST @Path("{id}/actions/:publish") - public Response publishDataset(@PathParam("id") String id, @QueryParam("type") String type, @QueryParam("assureIsIndexed") boolean mustBeIndexed) { + public Response publishDataset(@PathParam("id") String id, @QueryParam("type") String type) { try { if (type == null) { return error(Response.Status.BAD_REQUEST, "Missing 'type' parameter (either 'major','minor', or 'updatecurrent')."); @@ -972,29 +966,6 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S } Dataset ds = findDatasetOrDie(id); - if (mustBeIndexed) { - logger.fine("IT: " + ds.getIndexTime()); - logger.fine("MT: " + ds.getModificationTime()); - logger.fine("PIT: " + ds.getPermissionIndexTime()); - logger.fine("PMT: " + ds.getPermissionModificationTime()); - if (ds.getIndexTime() != null && ds.getModificationTime() != null) { - logger.fine("ITMT: " + (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)); - } - /* - * Some calls, such as the /datasets/actions/:import* commands do not set the - * modification or permission modification times. The checks here are trying to - * see if indexing or permissionindexing could be pending, so they check to see - * if the relevant modification time is set and if so, whether the index is also - * set and if so, if it after the modification time. If the modification time is - * set and the index time is null or is before the mod time, the 409/conflict - * error is returned. - * - */ - if ((ds.getModificationTime()!=null && (ds.getIndexTime() == null || (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0))) || - (ds.getPermissionModificationTime()!=null && (ds.getPermissionIndexTime() == null || (ds.getPermissionIndexTime().compareTo(ds.getPermissionModificationTime()) <= 0)))) { - return error(Response.Status.CONFLICT, "Dataset is awaiting indexing"); - } - } if (updateCurrent) { /* * Note: The code here mirrors that in the @@ -2317,76 +2288,5 @@ public Response resetFileStore(@PathParam("identifier") String dvIdtf, datasetService.merge(dataset); return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER); } - - @GET - @Path("{identifier}/timestamps") - @Produces(MediaType.APPLICATION_JSON) - public Response getTimestamps(@PathParam("identifier") String id) { - - Dataset dataset = null; - DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; - try { - dataset = findDatasetOrDie(id); - User u = findUserOrDie(); - Set perms = new HashSet(); - perms.add(Permission.ViewUnpublishedDataset); - boolean canSeeDraft = permissionSvc.hasPermissionsFor(u, dataset, perms); - JsonObjectBuilder timestamps = Json.createObjectBuilder(); - logger.fine("CSD: " + canSeeDraft); - logger.fine("IT: " + dataset.getIndexTime()); - logger.fine("MT: " + dataset.getModificationTime()); - logger.fine("PIT: " + dataset.getPermissionIndexTime()); - logger.fine("PMT: " + dataset.getPermissionModificationTime()); - // Basic info if it's released - if (dataset.isReleased() || canSeeDraft) { - timestamps.add("createTime", formatter.format(dataset.getCreateDate().toLocalDateTime())); - if (dataset.getPublicationDate() != null) { - timestamps.add("publicationTime", formatter.format(dataset.getPublicationDate().toLocalDateTime())); - } - - if (dataset.getLastExportTime() != null) { - timestamps.add("lastMetadataExportTime", - formatter.format(dataset.getLastExportTime().toInstant().atZone(ZoneId.systemDefault()))); - } - - if (dataset.getMostRecentMajorVersionReleaseDate() != null) { - timestamps.add("lastMajorVersionReleaseTime", formatter.format( - dataset.getMostRecentMajorVersionReleaseDate().toInstant().atZone(ZoneId.systemDefault()))); - } - // If the modification/permissionmodification time is - // set and the index time is null or is before the mod time, the relevant index is stale - timestamps.add("hasStaleIndex", - (dataset.getModificationTime() != null && (dataset.getIndexTime() == null - || (dataset.getIndexTime().compareTo(dataset.getModificationTime()) <= 0))) ? true - : false); - timestamps.add("hasStalePermissionIndex", - (dataset.getPermissionModificationTime() != null && (dataset.getIndexTime() == null - || (dataset.getIndexTime().compareTo(dataset.getModificationTime()) <= 0))) ? true - : false); - } - // More detail if you can see a draft - if (canSeeDraft) { - timestamps.add("lastUpdateTime", formatter.format(dataset.getModificationTime().toLocalDateTime())); - if (dataset.getIndexTime() != null) { - timestamps.add("lastIndexTime", formatter.format(dataset.getIndexTime().toLocalDateTime())); - } - if (dataset.getPermissionModificationTime() != null) { - timestamps.add("lastPermissionUpdateTime", - formatter.format(dataset.getPermissionModificationTime().toLocalDateTime())); - } - if (dataset.getPermissionIndexTime() != null) { - timestamps.add("lastPermissionIndexTime", - formatter.format(dataset.getPermissionIndexTime().toLocalDateTime())); - } - if (dataset.getGlobalIdCreateTime() != null) { - timestamps.add("globalIdCreateTime", formatter - .format(dataset.getGlobalIdCreateTime().toInstant().atZone(ZoneId.systemDefault()))); - } - - } - return ok(timestamps); - } catch (WrappedResponse wr) { - return wr.getResponse(); - } - } } + diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index da9c4515752..c2f186f1e8c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -21,7 +21,6 @@ import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.workflow.WorkflowContext; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; import java.io.IOException; import java.sql.Timestamp; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index a2d206ead0d..41645078a25 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -15,7 +15,6 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.workflow.Workflow; -import edu.harvard.iq.dataverse.workflow.WorkflowContext; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; import java.util.Date; import java.util.List; diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java index 6208acb0999..a0aee4cb4c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowContext.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.workflow; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; @@ -32,7 +31,7 @@ public enum TriggerType { private final ApiToken apiToken; private final boolean datasetExternallyReleased; private Map settings; - private DatasetLock lock = null; + private Long lockId = null; private String invocationId = UUID.randomUUID().toString(); @@ -48,7 +47,7 @@ public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersi } public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersionNumber, - long nextMinorVersionNumber, TriggerType type, Map settings, ApiToken apiToken, boolean datasetExternallyReleased, String invocationId, DatasetLock lock) { + long nextMinorVersionNumber, TriggerType type, Map settings, ApiToken apiToken, boolean datasetExternallyReleased, String invocationId, Long lockId) { this.request = request; this.dataset = dataset; this.nextVersionNumber = nextVersionNumber; @@ -57,12 +56,12 @@ public WorkflowContext(DataverseRequest request, Dataset dataset, long nextVersi this.settings = settings; this.apiToken = apiToken; this.datasetExternallyReleased = datasetExternallyReleased; - //If null, we'll keel the randomly generated one + //If null, we'll keep the randomly generated one if(invocationId!=null) { setInvocationId(invocationId); } - if(lock != null) { - this.setLock(lock); + if(lockId != null) { + this.setLockId(lockId); } } @@ -109,11 +108,11 @@ public ApiToken getApiToken() { public boolean getDatasetExternallyReleased() { return datasetExternallyReleased; } - public DatasetLock getLock() { - return lock; + public Long getLockId() { + return lockId; } - public void setLock(DatasetLock lock) { - this.lock = lock; + public void setLockId(Long lockId) { + this.lockId = lockId; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java index e7358cdbcba..b6aa35ede7b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java @@ -5,9 +5,6 @@ import edu.harvard.iq.dataverse.DataverseRequestServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; -import edu.harvard.iq.dataverse.UserNotification; -import edu.harvard.iq.dataverse.UserNotification.Type; -import edu.harvard.iq.dataverse.UserNotificationServiceBean; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.CommandContext; @@ -19,14 +16,11 @@ import edu.harvard.iq.dataverse.workflow.internalspi.InternalWorkflowStepSP; import edu.harvard.iq.dataverse.workflow.step.Failure; import edu.harvard.iq.dataverse.workflow.step.Pending; -import edu.harvard.iq.dataverse.workflow.step.Success; import edu.harvard.iq.dataverse.workflow.step.WorkflowStep; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; -import edu.harvard.iq.dataverse.workflows.WorkflowComment; -import java.sql.Timestamp; -import java.time.Instant; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -69,9 +63,6 @@ public class WorkflowServiceBean { @EJB SystemConfig systemConfig; - @EJB - UserNotificationServiceBean userNotificationService; - @EJB EjbDataverseEngine engine; @@ -106,9 +97,21 @@ public WorkflowServiceBean() { * @param ctxt the context in which the workflow is executed. * @throws CommandException If the dataset could not be locked. */ + //ToDo - should this be @Async? or just the forward() method? @Asynchronous public void start(Workflow wf, WorkflowContext ctxt) throws CommandException { + + // Since we are calling this asynchronously anyway - sleep here + // for a few seconds, just in case, to make sure the database update of + // the dataset initiated by the PublishDatasetCommand has finished, + // to avoid any concurrency/optimistic lock issues. + try { + Thread.sleep(1000); + } catch (Exception ex) { + logger.warning("Failed to sleep for a second."); + } ctxt = refresh(ctxt, retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(ctxt.getRequest().getAuthenticatedUser())); + lockDataset(ctxt, new DatasetLock(DatasetLock.Reason.Workflow, ctxt.getRequest().getAuthenticatedUser())); forward(wf, ctxt); } @@ -163,7 +166,6 @@ private Map retrieveRequestedSettings(Map requir @Asynchronous public void resume(PendingWorkflowInvocation pending, String body) { em.remove(em.merge(pending)); - em.flush(); doResume(pending, body); } @@ -182,17 +184,10 @@ private void doResume(PendingWorkflowInvocation pending, String body) { final WorkflowContext ctxt = refresh(newCtxt,retrieveRequestedSettings( wf.getRequiredSettings()), getCurrentApiToken(newCtxt.getRequest().getAuthenticatedUser())); WorkflowStepResult res = pendingStep.resume(ctxt, pending.getLocalData(), body); if (res instanceof Failure) { - userNotificationService.sendNotification(ctxt.getRequest().getAuthenticatedUser(), Timestamp.from(Instant.now()), UserNotification.Type.WORKFLOW_FAILURE, ctxt.getDataset().getLatestVersion().getId(), ((Failure) res).getMessage()); - //UserNotification isn't meant to be a long-term record and doesn't store the comment, so we'll also keep it as a workflow comment - WorkflowComment wfc = new WorkflowComment(ctxt.getDataset().getLatestVersion(), WorkflowComment.Type.WORKFLOW_FAILURE, ((Failure) res).getMessage(), ctxt.getRequest().getAuthenticatedUser()); - datasets.addWorkflowComment(wfc); rollback(wf, ctxt, (Failure) res, pending.getPendingStepIdx() - 1); } else if (res instanceof Pending) { pauseAndAwait(wf, ctxt, (Pending) res, pending.getPendingStepIdx()); } else { - if (res instanceof Success) { - - } executeSteps(wf, ctxt, pending.getPendingStepIdx() + 1); } } @@ -286,27 +281,41 @@ void rollbackStep( WorkflowStep step, WorkflowContext ctxt, Failure reason ) { } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void lockDataset( WorkflowContext ctxt ) throws CommandException { - DatasetLock datasetLock = new DatasetLock(DatasetLock.Reason.Workflow, ctxt.getRequest().getAuthenticatedUser()); - /* Note that this method directly adds a lock to the database rather than adding it via - * engine.submit(new AddLockCommand(ctxt.getRequest(), ctxt.getDataset(), datasetLock)); - * which would update the dataset's list of locks, etc. - * An em.find() for the dataset would get a Dataset that has an updated list of locks, but this copy would not have any changes - * made in a calling command (e.g. for a PostPublication workflow, the fact that the latest version is 'released' is not yet in the - * database. + void lockDataset(WorkflowContext ctxt, DatasetLock datasetLock) throws CommandException { + /* + * Note that this method directly adds a lock to the database rather than adding + * it via engine.submit(new AddLockCommand(ctxt.getRequest(), ctxt.getDataset(), + * datasetLock)); which would update the dataset's list of locks, etc. An + * em.find() for the dataset would get a Dataset that has an updated list of + * locks, but this copy would not have any changes made in a calling command + * (e.g. for a PostPublication workflow, the fact that the latest version is + * 'released' is not yet in the database. */ datasetLock.setDataset(ctxt.getDataset()); em.persist(datasetLock); //flush creates the id em.flush(); - ctxt.setLock(datasetLock); + ctxt.setLockId(datasetLock.getId()); } - + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - void unlockDataset( WorkflowContext ctxt ) throws CommandException { - - em.remove(em.merge(ctxt.getLock())); - + void unlockDataset(WorkflowContext ctxt) throws CommandException { + /* + * Since the lockDataset command above directly persists a lock to the database, + * the ctxt.getDataset() is not updated and its list of locks can't be used. + * Using the named query below will find the workflow lock and remove it + * (actually all workflow locks for this Dataset but only one workflow should be + * active). + */ + TypedQuery lockCounter = em.createNamedQuery("DatasetLock.getLocksByDatasetId", DatasetLock.class); + lockCounter.setParameter("datasetId", ctxt.getDataset().getId()); + List locks = lockCounter.getResultList(); + for (DatasetLock lock : locks) { + if (lock.getReason() == DatasetLock.Reason.Workflow) { + ctxt.getDataset().removeLock(lock); + em.remove(lock); + } + } em.flush(); } @@ -345,9 +354,11 @@ private void workflowCompleted(Workflow wf, WorkflowContext ctxt) { info += registerGlobalIdsForFiles ? "Registering PIDs for Datafiles; " : ""; info += validatePhysicalFiles ? "Validating Datafiles Asynchronously" : ""; lock.setInfo(info); - datasets.addDatasetLock(ctxt.getDataset(), lock); + lockDataset(ctxt, lock); + ctxt.getDataset().addLock(lock); + unlockDataset(ctxt); - ctxt.setLock(null); //the workflow lock + ctxt.setLockId(null); //the workflow lock //Refreshing merges the dataset ctxt = refresh(ctxt); //Then call Finalize @@ -466,7 +477,7 @@ private WorkflowContext refresh( WorkflowContext ctxt, Map setti */ WorkflowContext newCtxt =new WorkflowContext( ctxt.getRequest(), em.merge(ctxt.getDataset()), ctxt.getNextVersionNumber(), - ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLock()); + ctxt.getNextMinorVersionNumber(), ctxt.getType(), settings, apiToken, ctxt.getDatasetExternallyReleased(), ctxt.getInvocationId(), ctxt.getLockId()); return newCtxt; } diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java index 950eed52245..0487313fd0d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Failure.java @@ -11,7 +11,7 @@ public class Failure implements WorkflowStepResult { private final String message; public Failure( String reason ) { - this(reason, null); + this(reason, reason); } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java deleted file mode 100644 index 621afdc61f6..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java +++ /dev/null @@ -1,47 +0,0 @@ -package edu.harvard.iq.dataverse.workflow.step; - -/** - * Result returned when step execution succeeds. - */ -public class Success implements WorkflowStepResult { - - private final String reason; - private final String message; - - public Success( String reason ) { - this(reason, null); - } - - /** - * Constructs a new success message. - * @param reason Technical reason (for logs etc.). - * @param message Human readable reason. - */ - public Success(String reason, String message) { - this.reason = reason; - this.message = message; - } - - /** - * Holds the technical reason for the success, useful for debugging the problem. - * @return the technical reason for the problem. - */ - public String getReason() { - return reason; - } - - /** - * Holds the user-friendly message explaining the failure. - * @return user-friendly message for the success. - */ - public String getMessage() { - return message; - } - - @Override - public String toString() { - return "WorkflowStepResult.Success{" + "reason=" + reason + ", message=" + message + '}'; - } - - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java index 0da8ed223b4..26ddb9b1573 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowComment.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.workflows; import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import java.io.Serializable; import java.sql.Timestamp; @@ -23,9 +22,7 @@ public class WorkflowComment implements Serializable { More may be added in future releases, */ public enum Type { - RETURN_TO_AUTHOR, //, SUBMIT_FOR_REVIEW not available in this release but may be added in the future - WORKFLOW_SUCCESS, - WORKFLOW_FAILURE + RETURN_TO_AUTHOR //, SUBMIT_FOR_REVIEW not available in this release but may be added in the future }; @Id @@ -64,8 +61,6 @@ public enum Type { @Column(nullable = false) private Timestamp created; - private boolean toBeShown; - // TODO: Consider support editing in the GUI some day, like GitHub issue comments (show "Edited" in the UI). We won't send a second email, however. You only get one shot to prevent spam. // @Transient // private Timestamp modified; @@ -74,11 +69,12 @@ public enum Type { // private List notifications; public WorkflowComment(DatasetVersion version, WorkflowComment.Type type, String message, AuthenticatedUser authenticatedUser) { this.type = type; - this.datasetVersion = version; + if (this.type.equals(WorkflowComment.Type.RETURN_TO_AUTHOR)) { + this.datasetVersion = version; + } this.message = message; this.authenticatedUser = authenticatedUser; this.created = new Timestamp(new Date().getTime()); - this.setToBeShown(true); } /** @@ -127,12 +123,4 @@ public void setDatasetVersion(DatasetVersion dv) { datasetVersion=dv; } - public boolean isToBeShown() { - return toBeShown; - } - - public void setToBeShown(boolean toBeShown) { - this.toBeShown = toBeShown; - } - } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index cb62106a491..1fd7c300ab4 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -649,10 +649,6 @@ notification.email.submit.dataset.subject={0}: Your dataset has been submitted f notification.email.publish.dataset.subject={0}: Your dataset has been published notification.email.publishFailure.dataset.subject={0}: Failed to publish your dataset notification.email.returned.dataset.subject={0}: Your dataset has been returned -notification.email.workflow.success.subject={0}: Your dataset has been processed -notification.email.workflow.success=A workflow running on {0} (view at {1}) succeeded: {2} -notification.email.workflow.failure.subject={0}: Failed to process your dataset -notification.email.workflow.failure=A workflow running on {0} (view at {1}) failed: {2} notification.email.create.account.subject={0}: Your account has been created notification.email.assign.role.subject={0}: You have been assigned a role notification.email.revoke.role.subject={0}: Your role has been revoked From a2a3b513020972cc44adc135ee1c0555d37ddfc9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:15:39 -0500 Subject: [PATCH 226/274] docs for new step with authorization --- .../source/developers/workflows.rst | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 78e87556a3f..43f3dfae598 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -1,5 +1,5 @@ Workflows -================ +========= The Dataverse Software has a flexible workflow mechanism that can be used to trigger actions before and after Dataset publication. @@ -12,7 +12,7 @@ Introduction The Dataverse Software can perform two sequences of actions when datasets are published: one prior to publishing (marked by a ``PrePublishDataset`` trigger), and one after the publication has succeeded (``PostPublishDataset``). The pre-publish workflow is useful for having an external system prepare a dataset for being publicly accessed (a possibly lengthy activity that requires moving files around, uploading videos to a streaming server, etc.), or to start an approval process. A post-publish workflow might be used for sending notifications about the newly published dataset. -Workflow steps are created using *step providers*. The Dataverse Software ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers. This allows installations to implement functionality they need without changing the Dataverse Software source code. +Workflow steps are created using *step providers*. The Dataverse Software ships with an internal step provider that offers some basic functionality, and with the ability to load 3rd party step providers (currently disabled). This allows installations to implement functionality they need without changing the Dataverse Software source code. Steps can be internal (say, writing some data to the log) or external. External steps involve the Dataverse Software sending a request to an external system, and waiting for the system to reply. The wait period is arbitrary, and so allows the external system unbounded operation time. This is useful, e.g., for steps that require human intervention, such as manual approval of a dataset publication. @@ -22,8 +22,8 @@ If a step in a workflow fails, the Dataverse installation makes an effort to rol .. tip:: For invoking external systems using a REST api, the Dataverse Software's internal step - provider offers a step for sending and receiving customizable HTTP requests. - It's called *http/sr*, and is detailed below. + provider offers two steps for sending and receiving customizable HTTP requests. + *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. Administration ~~~~~~~~~~~~~~ @@ -104,6 +104,50 @@ Available variables are: * ``majorVersion`` * ``releaseStatus`` +http/authext +++++++++++++ + +Similar to the *http/sr* step. A step that sends a HTTP request to an external system, and then waits for a response. The receiver can use the invocationId of the workflow in lieu of an api key to perform work on behlaf of the user launching the workflow. +The invocationId must be sent as an 'X-Dataverse-invocationId' HTTP Header or as an ?invocationId= query parameter. Note that any external process started using this step then has the ability to access a Dataverse instance via the API as the user. +Once this step completes and responds, the invocationId is invalidated and will not allow further access. + + + +The url, content type, and message body can use data from the workflow context, using a simple markup language. This step has specific parameters for rollback. +The workflow is restarted when the external system replies with a POST request to ``/api/workflows/{invocation-id}``. +The response has is expected to be a json object with three keys: +- "Status" - can be "Success" or "Failure" +- "Reason" - a message that will be logged +- "Message" - a message to send to the user (message sending is not yet implemented). + +.. code:: json + + { + "provider":":internal", + "stepType":"http/authext", + "parameters": { + "url":"http://localhost:5050/dump/${invocationId}", + "method":"POST", + "contentType":"text/plain", + "body":"START RELEASE ${dataset.id} as ${dataset.displayName}", + "rollbackUrl":"http://localhost:5050/dump/${invocationId}", + "rollbackMethod":"DELETE ${dataset.id}" + } + } + +Available variables are: + +* ``invocationId`` +* ``dataset.id`` +* ``dataset.identifier`` +* ``dataset.globalId`` +* ``dataset.displayName`` +* ``dataset.citation`` +* ``minorVersion`` +* ``majorVersion`` +* ``releaseStatus`` + + archiver ++++++++ From 55eb60fd103ff223922ae658c60102a9806f5753 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:17:59 -0500 Subject: [PATCH 227/274] typo/style --- doc/sphinx-guides/source/developers/workflows.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 43f3dfae598..d55ee6015b7 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -107,8 +107,8 @@ Available variables are: http/authext ++++++++++++ -Similar to the *http/sr* step. A step that sends a HTTP request to an external system, and then waits for a response. The receiver can use the invocationId of the workflow in lieu of an api key to perform work on behlaf of the user launching the workflow. -The invocationId must be sent as an 'X-Dataverse-invocationId' HTTP Header or as an ?invocationId= query parameter. Note that any external process started using this step then has the ability to access a Dataverse instance via the API as the user. +Similar to the *http/sr* step. A step that sends a HTTP request to an external system, and then waits for a response. The receiver can use the invocationId of the workflow in lieu of an api key to perform work on behalf of the user launching the workflow. +The invocationId must be sent as an 'X-Dataverse-invocationId' HTTP Header or as an ?invocationId= query parameter. *Note that any external process started using this step then has the ability to access a Dataverse instance via the API as the user.* Once this step completes and responds, the invocationId is invalidated and will not allow further access. From eb611e1d8a1adcfca543980af863d8deb83de82f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:20:48 -0500 Subject: [PATCH 228/274] add note about locking. --- doc/sphinx-guides/source/developers/workflows.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index d55ee6015b7..9d519a52e5d 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -23,7 +23,7 @@ If a step in a workflow fails, the Dataverse installation makes an effort to rol .. tip:: For invoking external systems using a REST api, the Dataverse Software's internal step provider offers two steps for sending and receiving customizable HTTP requests. - *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. + *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. (Both lock the dataset to prevent other processes from changing the dataset between the time the step is launched to when the external process responds to the Dataverse instance.) Administration ~~~~~~~~~~~~~~ @@ -111,8 +111,6 @@ Similar to the *http/sr* step. A step that sends a HTTP request to an external s The invocationId must be sent as an 'X-Dataverse-invocationId' HTTP Header or as an ?invocationId= query parameter. *Note that any external process started using this step then has the ability to access a Dataverse instance via the API as the user.* Once this step completes and responds, the invocationId is invalidated and will not allow further access. - - The url, content type, and message body can use data from the workflow context, using a simple markup language. This step has specific parameters for rollback. The workflow is restarted when the external system replies with a POST request to ``/api/workflows/{invocation-id}``. The response has is expected to be a json object with three keys: From a849c8afe8a0dfd8a6f48c4de5e482aea5f46c2c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:24:46 -0500 Subject: [PATCH 229/274] missing class --- .../iq/dataverse/workflow/step/Success.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java new file mode 100644 index 00000000000..621afdc61f6 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/step/Success.java @@ -0,0 +1,47 @@ +package edu.harvard.iq.dataverse.workflow.step; + +/** + * Result returned when step execution succeeds. + */ +public class Success implements WorkflowStepResult { + + private final String reason; + private final String message; + + public Success( String reason ) { + this(reason, null); + } + + /** + * Constructs a new success message. + * @param reason Technical reason (for logs etc.). + * @param message Human readable reason. + */ + public Success(String reason, String message) { + this.reason = reason; + this.message = message; + } + + /** + * Holds the technical reason for the success, useful for debugging the problem. + * @return the technical reason for the problem. + */ + public String getReason() { + return reason; + } + + /** + * Holds the user-friendly message explaining the failure. + * @return user-friendly message for the success. + */ + public String getMessage() { + return message; + } + + @Override + public String toString() { + return "WorkflowStepResult.Success{" + "reason=" + reason + ", message=" + message + '}'; + } + + +} From c5007b293299e87992ed9353cb1135d00253b185 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 14:31:49 -0500 Subject: [PATCH 230/274] remove messaging related enums --- .../authorization/providers/builtin/DataverseUserPage.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 9199558765f..204d93b5b8f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -485,8 +485,6 @@ public void displayNotification() { case PUBLISHEDDS: case PUBLISHFAILED_PIDREG: case RETURNEDDS: - case WORKFLOW_SUCCESS: - case WORKFLOW_FAILURE: userNotification.setTheObject(datasetVersionService.find(userNotification.getObjectId())); break; From 65b19931976efbaa348a9acb587d61372a175dea Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 24 Feb 2021 16:40:22 -0500 Subject: [PATCH 231/274] rename flyway script --- .../{V5.4.0.1__7563-workflow.sql => V5.5.0.1__7563-workflow.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.4.0.1__7563-workflow.sql => V5.5.0.1__7563-workflow.sql} (100%) diff --git a/src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.5.0.1__7563-workflow.sql similarity index 100% rename from src/main/resources/db/migration/V5.4.0.1__7563-workflow.sql rename to src/main/resources/db/migration/V5.5.0.1__7563-workflow.sql From f7c35fe67595ed45a90d427359d07a42351280a7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 25 Feb 2021 12:17:21 -0500 Subject: [PATCH 232/274] allow publish conditional on indexing being done, add /timestamps --- .../harvard/iq/dataverse/api/Datasets.java | 102 +++++++++++++++++- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4ea4144fbf9..69769b8172c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -112,6 +112,8 @@ import java.io.StringReader; import java.sql.Timestamp; import java.text.MessageFormat; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -969,12 +971,12 @@ private String validateDatasetFieldValues(List fields) { @Deprecated public Response publishDataseUsingGetDeprecated( @PathParam("id") String id, @QueryParam("type") String type ) { logger.info("publishDataseUsingGetDeprecated called on id " + id + ". Encourage use of POST rather than GET, which is deprecated."); - return publishDataset(id, type); + return publishDataset(id, type, false); } @POST @Path("{id}/actions/:publish") - public Response publishDataset(@PathParam("id") String id, @QueryParam("type") String type) { + public Response publishDataset(@PathParam("id") String id, @QueryParam("type") String type, @QueryParam("assureIsIndexed") boolean mustBeIndexed) { try { if (type == null) { return error(Response.Status.BAD_REQUEST, "Missing 'type' parameter (either 'major','minor', or 'updatecurrent')."); @@ -1002,6 +1004,29 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S } Dataset ds = findDatasetOrDie(id); + if (mustBeIndexed) { + logger.fine("IT: " + ds.getIndexTime()); + logger.fine("MT: " + ds.getModificationTime()); + logger.fine("PIT: " + ds.getPermissionIndexTime()); + logger.fine("PMT: " + ds.getPermissionModificationTime()); + if (ds.getIndexTime() != null && ds.getModificationTime() != null) { + logger.fine("ITMT: " + (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0)); + } + /* + * Some calls, such as the /datasets/actions/:import* commands do not set the + * modification or permission modification times. The checks here are trying to + * see if indexing or permissionindexing could be pending, so they check to see + * if the relevant modification time is set and if so, whether the index is also + * set and if so, if it after the modification time. If the modification time is + * set and the index time is null or is before the mod time, the 409/conflict + * error is returned. + * + */ + if ((ds.getModificationTime()!=null && (ds.getIndexTime() == null || (ds.getIndexTime().compareTo(ds.getModificationTime()) <= 0))) || + (ds.getPermissionModificationTime()!=null && (ds.getPermissionIndexTime() == null || (ds.getPermissionIndexTime().compareTo(ds.getPermissionModificationTime()) <= 0)))) { + return error(Response.Status.CONFLICT, "Dataset is awaiting indexing"); + } + } if (updateCurrent) { /* * Note: The code here mirrors that in the @@ -2324,5 +2349,76 @@ public Response resetFileStore(@PathParam("identifier") String dvIdtf, datasetService.merge(dataset); return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER); } -} + @GET + @Path("{identifier}/timestamps") + @Produces(MediaType.APPLICATION_JSON) + public Response getTimestamps(@PathParam("identifier") String id) { + + Dataset dataset = null; + DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; + try { + dataset = findDatasetOrDie(id); + User u = findUserOrDie(); + Set perms = new HashSet(); + perms.add(Permission.ViewUnpublishedDataset); + boolean canSeeDraft = permissionSvc.hasPermissionsFor(u, dataset, perms); + JsonObjectBuilder timestamps = Json.createObjectBuilder(); + logger.fine("CSD: " + canSeeDraft); + logger.fine("IT: " + dataset.getIndexTime()); + logger.fine("MT: " + dataset.getModificationTime()); + logger.fine("PIT: " + dataset.getPermissionIndexTime()); + logger.fine("PMT: " + dataset.getPermissionModificationTime()); + // Basic info if it's released + if (dataset.isReleased() || canSeeDraft) { + timestamps.add("createTime", formatter.format(dataset.getCreateDate().toLocalDateTime())); + if (dataset.getPublicationDate() != null) { + timestamps.add("publicationTime", formatter.format(dataset.getPublicationDate().toLocalDateTime())); + } + + if (dataset.getLastExportTime() != null) { + timestamps.add("lastMetadataExportTime", + formatter.format(dataset.getLastExportTime().toInstant().atZone(ZoneId.systemDefault()))); + } + + if (dataset.getMostRecentMajorVersionReleaseDate() != null) { + timestamps.add("lastMajorVersionReleaseTime", formatter.format( + dataset.getMostRecentMajorVersionReleaseDate().toInstant().atZone(ZoneId.systemDefault()))); + } + // If the modification/permissionmodification time is + // set and the index time is null or is before the mod time, the relevant index is stale + timestamps.add("hasStaleIndex", + (dataset.getModificationTime() != null && (dataset.getIndexTime() == null + || (dataset.getIndexTime().compareTo(dataset.getModificationTime()) <= 0))) ? true + : false); + timestamps.add("hasStalePermissionIndex", + (dataset.getPermissionModificationTime() != null && (dataset.getIndexTime() == null + || (dataset.getIndexTime().compareTo(dataset.getModificationTime()) <= 0))) ? true + : false); + } + // More detail if you can see a draft + if (canSeeDraft) { + timestamps.add("lastUpdateTime", formatter.format(dataset.getModificationTime().toLocalDateTime())); + if (dataset.getIndexTime() != null) { + timestamps.add("lastIndexTime", formatter.format(dataset.getIndexTime().toLocalDateTime())); + } + if (dataset.getPermissionModificationTime() != null) { + timestamps.add("lastPermissionUpdateTime", + formatter.format(dataset.getPermissionModificationTime().toLocalDateTime())); + } + if (dataset.getPermissionIndexTime() != null) { + timestamps.add("lastPermissionIndexTime", + formatter.format(dataset.getPermissionIndexTime().toLocalDateTime())); + } + if (dataset.getGlobalIdCreateTime() != null) { + timestamps.add("globalIdCreateTime", formatter + .format(dataset.getGlobalIdCreateTime().toInstant().atZone(ZoneId.systemDefault()))); + } + + } + return ok(timestamps); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } +} From 89c4a26f14821575d06fc1877343bd61d0c5c192 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Thu, 25 Feb 2021 14:17:22 -0500 Subject: [PATCH 233/274] minor text edit --- doc/sphinx-guides/source/developers/workflows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 9d519a52e5d..36711488bf3 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -23,7 +23,7 @@ If a step in a workflow fails, the Dataverse installation makes an effort to rol .. tip:: For invoking external systems using a REST api, the Dataverse Software's internal step provider offers two steps for sending and receiving customizable HTTP requests. - *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. (Both lock the dataset to prevent other processes from changing the dataset between the time the step is launched to when the external process responds to the Dataverse instance.) + *http/sr* and *http/authExt*, detailed below, with the latter able to use the API to make changes to the dataset being processed. Both lock the dataset to prevent other processes from changing the dataset between the time the step is launched to when the external process responds to the Dataverse instance. Administration ~~~~~~~~~~~~~~ From 86dfba1a41633620bcf2ee33b11e42e888b20d76 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 26 Feb 2021 10:20:53 -0500 Subject: [PATCH 234/274] use constant, add comment --- .../workflow/internalspi/AuthorizedExternalStep.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java index 414641ada5f..244abdb017c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java +++ b/src/main/java/edu/harvard/iq/dataverse/workflow/internalspi/AuthorizedExternalStep.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.workflow.internalspi; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.workflow.PendingWorkflowInvocation; import edu.harvard.iq.dataverse.workflow.WorkflowContext; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; import edu.harvard.iq.dataverse.workflow.step.Failure; @@ -60,7 +61,8 @@ public WorkflowStepResult run(WorkflowContext context) { if (responseStatus>=200 && responseStatus<300 ) { // HTTP OK range Map data = new HashMap (); - data.put("authorized", "true"); + //Allow external client to use invocationId as a key to act on the user's behalf + data.put(PendingWorkflowInvocation.AUTHORIZED, "true"); return new Pending(data); } else { String responseBody = mtd.getResponseBodyAsString(); From afdfe8df47d0dd14ff326311e3d0cfc1ae236be1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 26 Feb 2021 10:32:16 -0500 Subject: [PATCH 235/274] note pause step is primarily for testing --- doc/sphinx-guides/source/developers/workflows.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/workflows.rst b/doc/sphinx-guides/source/developers/workflows.rst index 9d519a52e5d..3f866a08b99 100644 --- a/doc/sphinx-guides/source/developers/workflows.rst +++ b/doc/sphinx-guides/source/developers/workflows.rst @@ -60,7 +60,8 @@ A step that writes data about the current workflow invocation to the instance lo pause +++++ -A step that pauses the workflow. The workflow is paused until a POST request is sent to ``/api/workflows/{invocation-id}``. Sending 'fail' in the POST body (Content-type:text/plain) will trigger a failure and workflow rollback. All other responses are considered as successes. +A step that pauses the workflow. The workflow is paused until a POST request is sent to ``/api/workflows/{invocation-id}``. Sending 'fail' in the POST body (Content-type:text/plain) will trigger a failure and workflow rollback. All other responses are considered as successes. +The pause step is intended for testing - the invocationId required to end the pause is only available in the log (and database). Adding a parameter (see log step) with the key/value "authorized":"true" will allow the invocationId to be used as a credential as with the http/authext step below. .. code:: json From 80552f7f99bf6aff3ca0f01893084a2a36437f2e Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 26 Feb 2021 11:03:55 -0500 Subject: [PATCH 236/274] #7373 copy files to current solr directory --- conf/solr/8.8.1/readme.md | 1 + conf/solr/8.8.1/schema.xml | 1198 ++++++++++++++++++ conf/solr/8.8.1/schema_dv_mdb_copies.xml | 161 +++ conf/solr/8.8.1/schema_dv_mdb_fields.xml | 161 +++ conf/solr/8.8.1/solrconfig.xml | 1410 ++++++++++++++++++++++ conf/solr/8.8.1/updateSchemaMDB.sh | 85 ++ 6 files changed, 3016 insertions(+) create mode 100644 conf/solr/8.8.1/readme.md create mode 100644 conf/solr/8.8.1/schema.xml create mode 100644 conf/solr/8.8.1/schema_dv_mdb_copies.xml create mode 100644 conf/solr/8.8.1/schema_dv_mdb_fields.xml create mode 100644 conf/solr/8.8.1/solrconfig.xml create mode 100644 conf/solr/8.8.1/updateSchemaMDB.sh diff --git a/conf/solr/8.8.1/readme.md b/conf/solr/8.8.1/readme.md new file mode 100644 index 00000000000..4457cf9a7df --- /dev/null +++ b/conf/solr/8.8.1/readme.md @@ -0,0 +1 @@ +Please see the dev guide for what to do with Solr config files. \ No newline at end of file diff --git a/conf/solr/8.8.1/schema.xml b/conf/solr/8.8.1/schema.xml new file mode 100644 index 00000000000..da40a8e99fa --- /dev/null +++ b/conf/solr/8.8.1/schema.xml @@ -0,0 +1,1198 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/solr/8.8.1/schema_dv_mdb_copies.xml b/conf/solr/8.8.1/schema_dv_mdb_copies.xml new file mode 100644 index 00000000000..080cc71ef50 --- /dev/null +++ b/conf/solr/8.8.1/schema_dv_mdb_copies.xml @@ -0,0 +1,161 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/solr/8.8.1/schema_dv_mdb_fields.xml b/conf/solr/8.8.1/schema_dv_mdb_fields.xml new file mode 100644 index 00000000000..3f844c6183c --- /dev/null +++ b/conf/solr/8.8.1/schema_dv_mdb_fields.xml @@ -0,0 +1,161 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/solr/8.8.1/solrconfig.xml b/conf/solr/8.8.1/solrconfig.xml new file mode 100644 index 00000000000..51100af7d6e --- /dev/null +++ b/conf/solr/8.8.1/solrconfig.xml @@ -0,0 +1,1410 @@ + + + + + + + + + 7.3.0 + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + edismax + 0.075 + + dvName^400 + authorName^180 + dvSubject^190 + dvDescription^180 + dvAffiliation^170 + title^130 + subject^120 + keyword^110 + topicClassValue^100 + dsDescriptionValue^90 + authorAffiliation^80 + publicationCitation^60 + producerName^50 + fileName^30 + fileDescription^30 + variableLabel^20 + variableName^10 + _text_^1.0 + + + dvName^200 + authorName^100 + dvSubject^100 + dvDescription^100 + dvAffiliation^100 + title^75 + subject^75 + keyword^75 + topicClassValue^75 + dsDescriptionValue^75 + authorAffiliation^75 + publicationCitation^75 + producerName^75 + + + + isHarvested:false^25000 + + + + + + + + + + + + + + + + + + explicit + json + true + + + + + + + + explicit + + + + + + _text_ + + + + + + + true + ignored_ + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + + + tvComponent + + + + + + + + + + + + true + false + + + terms + + + + + + + + string + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + ${velocity.template.base.dir:} + ${velocity.solr.resource.loader.enabled:true} + ${velocity.params.resource.loader.enabled:false} + + + + + 5 + + + + + + + + + + + + + + diff --git a/conf/solr/8.8.1/updateSchemaMDB.sh b/conf/solr/8.8.1/updateSchemaMDB.sh new file mode 100644 index 00000000000..0044f15c7cd --- /dev/null +++ b/conf/solr/8.8.1/updateSchemaMDB.sh @@ -0,0 +1,85 @@ +#!/bin/bash +set -euo pipefail + +# This script updates the and schema configuration necessary to properly +# index custom metadata fields in Solr. +# 1. Retrieve from Dataverse API endpoint +# 2. Parse and write Solr schema files (which might replace the included files) +# 3. Reload Solr +# +# List of variables: +# ${DATAVERSE_URL}: URL to Dataverse. Defaults to http://localhost:8080 +# ${SOLR_URL}: URL to Solr. Defaults to http://localhost:8983 +# ${UNBLOCK_KEY}: File path to secret or unblock key as string. Only necessary on k8s or when you secured your installation. +# ${TARGET}: Directory where to write the XML files. Defaults to /tmp +# +# Programs used (need to be available on your PATH): +# coreutils: mktemp, csplit +# curl + +usage() { + echo "usage: updateSchemaMDB.sh [options]" + echo "options:" + echo " -d Dataverse URL, defaults to http://localhost:8080" + echo " -h Show this help text" + echo " -s Solr URL, defaults to http://localhost:8983" + echo " -t Directory where to write the XML files. Defaults to /tmp" + echo " -u Dataverse unblock key either as key string or path to keyfile" +} + +### Init (with sane defaults) +DATAVERSE_URL=${DATAVERSE_URL:-"http://localhost:8080"} +SOLR_URL=${SOLR_URL:-"http://localhost:8983"} +TARGET=${TARGET:-"/tmp"} +UNBLOCK_KEY=${UNBLOCK_KEY:-""} + +# if cmdline args are given, override any env var setting (or defaults) +while getopts ":d:hs:t:u:" opt +do + case $opt in + d) DATAVERSE_URL=${OPTARG};; + h) usage; exit 0;; + s) SOLR_URL=${OPTARG};; + t) TARGET=${OPTARG};; + u) UNBLOCK_KEY=${OPTARG};; + :) echo "Missing option argument for -${OPTARG}. Use -h for help." >&2; exit 1;; + \?) echo "Unknown option -${OPTARG}." >&2; usage; exit 1;; + esac +done + +# Special handling of unblock key depending on referencing a secret file or key in var +if [ ! -z "${UNBLOCK_KEY}" ]; then + if [ -f "${UNBLOCK_KEY}" ]; then + UNBLOCK_KEY="?unblock-key=$(cat ${UNBLOCK_KEY})" + else + UNBLOCK_KEY="?unblock-key=${UNBLOCK_KEY}" + fi +fi + +### Retrieval +echo "Retrieve schema data from ${DATAVERSE_URL}/api/admin/index/solr/schema" +TMPFILE=`mktemp` +curl -f -sS "${DATAVERSE_URL}/api/admin/index/solr/schema${UNBLOCK_KEY}" > $TMPFILE + +### Fail gracefull if Dataverse is not ready yet. +if [[ "`wc -l ${TMPFILE}`" < "3" ]]; then + echo "Dataverse responded with empty file. When running on K8s: did you bootstrap yet?" + exit 123 +fi + +### Processing +echo "Writing ${TARGET}/schema_dv_mdb_fields.xml" +echo "" > ${TARGET}/schema_dv_mdb_fields.xml +cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_fields.xml +echo "" >> ${TARGET}/schema_dv_mdb_fields.xml + +echo "Writing ${TARGET}/schema_dv_mdb_copies.xml" +echo "" > ${TARGET}/schema_dv_mdb_copies.xml +cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_copies.xml +echo "" >> ${TARGET}/schema_dv_mdb_copies.xml + +rm ${TMPFILE}* + +### Reloading +echo "Triggering Solr RELOAD at ${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1" +curl -f -sS "${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1" From be5ff941b73e0780ccf7c1613608c13a0b80dae3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 Feb 2021 11:42:22 -0500 Subject: [PATCH 237/274] restricted summary stats #7619 - At the dataset level, DDI exports no longer show "dataDscr" information for restricted files. There is only one version of this export and it is the version that's suitable for public consumption with the "dataDscr" information hidden for restricted files. - Similarly, at the dataset level, the DDI HTML Codebook no longer shows "dataDscr" information for restricted files. - At the file level, "dataDscr" information is no longer publicly available for restricted files. In practice, it was only possible to get this publicly via API (the download/access button was hidden). - At the file level, "dataDscr" (variable metadata) information can still be downloaded for restricted files if you have access to download the file. --- .../7619-restricted-summary-starts.md | 14 ++ .../source/user/dataset-management.rst | 15 --- .../edu/harvard/iq/dataverse/api/Access.java | 18 +++ .../WebApplicationExceptionHandler.java | 3 + .../dataverse/export/ddi/DdiExportUtil.java | 11 ++ .../harvard/iq/dataverse/api/DatasetsIT.java | 123 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 28 +++- 7 files changed, 195 insertions(+), 17 deletions(-) create mode 100644 doc/release-notes/7619-restricted-summary-starts.md diff --git a/doc/release-notes/7619-restricted-summary-starts.md b/doc/release-notes/7619-restricted-summary-starts.md new file mode 100644 index 00000000000..e1c20f3bde2 --- /dev/null +++ b/doc/release-notes/7619-restricted-summary-starts.md @@ -0,0 +1,14 @@ +Restricted Files and DDI "dataDscr" Information (Summary Statistics, Variable Names, Variable Labels) + +In previous releases, DDI "dataDscr" information (summary statistics, variable names, and variable labels, sometimes known as "variable metadata") for tabular files that were ingested successfully were available even if files were restricted. This has been changed in the following ways: + +- At the dataset level, DDI exports no longer show "dataDscr" information for restricted files. There is only one version of this export and it is the version that's suitable for public consumption with the "dataDscr" information hidden for restricted files. +- Similarly, at the dataset level, the DDI HTML Codebook no longer shows "dataDscr" information for restricted files. +- At the file level, "dataDscr" information is no longer publicly available for restricted files. In practice, it was only possible to get this publicly via API (the download/access button was hidden). +- At the file level, "dataDscr" (variable metadata) information can still be downloaded for restricted files if you have access to download the file. + +After upgrading, you should re-export to replace cached DDI exports with restricted summary stats with DDI exports fit for public consumption: + + curl http://localhost:8080/api/admin/metadata/reExportAll + +For details on this operation, see https://guides.dataverse.org/en/5.4/admin/metadataexport.html diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 8a853449488..1d5f1344c6c 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -179,8 +179,6 @@ Additional download options available for tabular data (found in the same drop-d - Data File Citation (currently in either RIS, EndNote XML, or BibTeX format); - All of the above, as a zipped bundle. -See also :ref:`restricted-tabular-files`. - Astronomy (FITS) ---------------- @@ -214,19 +212,6 @@ When you restrict a file it cannot be downloaded unless permission has been gran See also :ref:`terms-of-access` and :ref:`permissions`. -.. _restricted-tabular-files: - -Restricted Tabular Files ------------------------- - -Restricted tabular files are treated differently than other file types in that the following information is exposed when the files are published: - -- The name of columns (variables). -- The "label" (description) of columns. -- Summary statistics (max, min, mean, etc.) of columns. - -This information can been seen from the file landing page if you click "Export Metada" and then "DDI". Depending on your installation, the information above may also be available from :doc:`/admin/external-tools`. - Edit Files ========== diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 281f0a76ce1..2cd7e0dd122 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -450,6 +450,24 @@ public String tabularDatafileMetadataDDI(@PathParam("fileId") String fileId, @Q throw new BadRequestException("tabular data required"); } + if (dataFile.isRestricted()) { + boolean hasPermissionToDownloadFile = false; + DataverseRequest dataverseRequest; + try { + dataverseRequest = createDataverseRequest(findUserOrDie()); + } catch (WrappedResponse ex) { + throw new BadRequestException("cannot find user"); + } + if (dataverseRequest != null && dataverseRequest.getUser() instanceof GuestUser) { + // We must be in the UI. Try to get a non-GuestUser from the session. + dataverseRequest = dvRequestService.getDataverseRequest(); + } + hasPermissionToDownloadFile = permissionService.requestOn(dataverseRequest, dataFile).has(Permission.DownloadFile); + if (!hasPermissionToDownloadFile) { + throw new BadRequestException("no permission to download file"); + } + } + response.setHeader("Content-disposition", "attachment; filename=\"dataverse_files.zip\""); FileMetadata fm = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index b6229e58192..aa8bb2b3772 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -43,8 +43,11 @@ public Response toResponse(WebApplicationException ex) { switch (ex.getResponse().getStatus()) { // BadRequest case 400: + // It's strange to have these "startsWith" conditionals here. They both come from Access.java. if ( (ex.getMessage()+"").toLowerCase().startsWith("tabular data required")) { jrb.message(BundleUtil.getStringFromBundle("access.api.exception.metadata.not.available.for.nontabular.file")); + } else if ((ex.getMessage() + "").toLowerCase().startsWith("no permission to download file")) { + jrb.message("You do not have permission to download this file."); } else { jrb.message("Bad Request. The API request cannot be completed with the parameters supplied. Please check your code for typos, or consult our API guide at http://guides.dataverse.org."); jrb.request(request); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index dd797a55539..1ce1a2119a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1388,6 +1388,17 @@ public static void createDataDscr(XMLStreamWriter xmlw, DatasetVersion datasetVe for (FileMetadata fileMetadata : datasetVersion.getFileMetadatas()) { DataFile dataFile = fileMetadata.getDataFile(); + /** + * Previously (in Dataverse 5.3 and below) the dataDscr section was + * included for restricted files but that meant that summary + * statistics were exposed. (To get at these statistics, API users + * should instead use the "Data Variable Metadata Access" endpoint.) + * These days we return early to avoid this exposure. + */ + if (dataFile.isRestricted()) { + return; + } + if (dataFile != null && dataFile.isTabularData()) { if (!tabularData) { xmlw.writeStartElement("dataDscr"); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index fb7914561bc..04d7c77d4fb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -36,7 +36,10 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.builtin.AuthenticatedUsers; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import javax.json.Json; @@ -53,6 +56,7 @@ import static javax.ws.rs.core.Response.Status.OK; import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; import static junit.framework.Assert.assertEquals; +import org.hamcrest.CoreMatchers; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; import org.junit.After; @@ -2041,4 +2045,123 @@ public void testLinkingDatasets() { */ } + /** + * In this test we are restricting a file and testing "export DDI" at the + * dataset level as well as getting the DDI at the file level. + * + * Export at the dataset level is always the public version. + * + * At the file level, you can still get summary statistics (and friends, + * "dataDscr") if you have access to download the file. If you don't have + * access, you get an error. + */ + @Test + public void testRestrictFileExportDdi() throws IOException { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String authorUsername = UtilIT.getUsernameFromResponse(createUser); + String authorApiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverse = UtilIT.createRandomDataverse(authorApiToken); + createDataverse.prettyPrint(); + createDataverse.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, authorApiToken); + createDataset.prettyPrint(); + createDataset.then().assertThat() + .statusCode(CREATED.getStatusCode()); + + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + String datasetPid = JsonPath.from(createDataset.asString()).getString("data.persistentId"); + + Path pathToFile = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.csv"); + String contentOfCsv = "" + + "name,pounds,species\n" + + "Marshall,40,dog\n" + + "Tiger,17,cat\n" + + "Panther,21,cat\n"; + java.nio.file.Files.write(pathToFile, contentOfCsv.getBytes()); + + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile.toString(), authorApiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.files[0].label", equalTo("data.csv")); + + String fileId = JsonPath.from(uploadFile.body().asString()).getString("data.files[0].dataFile.id"); + + assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetId.longValue(), "Ingest", authorApiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); + + Response restrictFile = UtilIT.restrictFile(fileId, true, authorApiToken); + restrictFile.prettyPrint(); + restrictFile.then().assertThat().statusCode(OK.getStatusCode()); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, authorApiToken); + publishDataverse.then().assertThat().statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", authorApiToken); + publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + + // We're testing export here, which is at dataset level. + // Guest/public version + Response exportByGuest = UtilIT.exportDataset(datasetPid, "ddi"); + exportByGuest.prettyPrint(); + exportByGuest.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("codeBook.fileDscr.fileTxt.fileName", equalTo("data.tab")); + + // Here we are asserting that dataDscr is empty. TODO: Do this in REST Assured. + String dataDscrForGuest = XmlPath.from(exportByGuest.asString()).getString("codeBook.dataDscr"); + Assert.assertEquals("", dataDscrForGuest); + + // Author export (has access) + Response exportByAuthor = UtilIT.exportDataset(datasetPid, "ddi", authorApiToken); + exportByAuthor.prettyPrint(); + exportByAuthor.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("codeBook.fileDscr.fileTxt.fileName", equalTo("data.tab")); + + // Here we are asserting that dataDscr is empty. TODO: Do this in REST Assured. + String dataDscrForAuthor = XmlPath.from(exportByAuthor.asString()).getString("codeBook.dataDscr"); + Assert.assertEquals("", dataDscrForAuthor); + + // Now we are testing file-level retrieval. + // The author has access to a restricted file and gets all the metadata. + Response fileMetadataDdiAuthor = UtilIT.getFileMetadata(fileId, "ddi", authorApiToken); + fileMetadataDdiAuthor.prettyPrint(); + fileMetadataDdiAuthor.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("codeBook.fileDscr.fileTxt.fileName", equalTo("data.tab")) + // .body("codeBook", containsString("dataDscr")) + // The names of all these variables (name, pounds, species) should be visible. + .body("codeBook.dataDscr", CoreMatchers.not(equalTo(null))) + // .body("codeBook.dataDscr", equalTo(null)) + .body("codeBook.dataDscr.var[0].@name", equalTo("name")) + .body("codeBook.dataDscr.var[1].@name", equalTo("pounds")) + // This is an example of a summary stat (max) that should be visible. + .body("codeBook.dataDscr.var[1].sumStat.find { it.@type == 'max' }", equalTo("40.0")) + .body("codeBook.dataDscr.var[2].@name", equalTo("species")); + + Response createUserNoAuth = UtilIT.createRandomUser(); + createUserNoAuth.prettyPrint(); + String usernameNoAuth = UtilIT.getUsernameFromResponse(createUserNoAuth); + String apiTokenNoAuth = UtilIT.getApiTokenFromResponse(createUserNoAuth); + + // Users with no access to the restricted file are blocked. + Response fileMetadataDdiUserNoAuth = UtilIT.getFileMetadata(fileId, "ddi", apiTokenNoAuth); + fileMetadataDdiUserNoAuth.prettyPrint(); + fileMetadataDdiUserNoAuth.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("You do not have permission to download this file.")); + + // Guest users (not logged in) are also blocked. + Response fileMetadataDdiGuest = UtilIT.getFileMetadata(fileId, "ddi"); + fileMetadataDdiGuest.prettyPrint(); + fileMetadataDdiGuest.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", equalTo("You do not have permission to download this file.")); + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 2b6bbaf0caa..f3ff8f8fae4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -834,6 +834,22 @@ static Response getFileMetadata(String fileIdOrPersistentId, String optionalForm .get("/api/access/datafile/" + idInPath + "/metadata" + optionalFormatInPath + "?key=" + apiToken + optionalQueryParam); } + static Response getFileMetadata(String fileIdOrPersistentId, String optionalFormat) { + String idInPath = fileIdOrPersistentId; // Assume it's a number. + String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. + if (!NumberUtils.isNumber(fileIdOrPersistentId)) { + idInPath = ":persistentId"; + optionalQueryParam = "?persistentId=" + fileIdOrPersistentId; + } + String optionalFormatInPath = ""; + if (optionalFormat != null) { + optionalFormatInPath = "/" + optionalFormat; + } + return given() + .urlEncodingEnabled(false) + .get("/api/access/datafile/" + idInPath + "/metadata" + optionalFormatInPath + optionalQueryParam); + } + static Response testIngest(String fileName, String fileType) { return given() .get("/api/ingest/test/file?fileName=" + fileName + "&fileType=" + fileType); @@ -1634,11 +1650,19 @@ static Response uploadProvFreeForm(String idOrPersistentId, JsonObject jsonObjec // } // return requestSpecification.delete("/api/files/" + idInPath + "/prov-freeform" + optionalQueryParam); // } + static Response exportDataset(String datasetPersistentId, String exporter) { + return exportDataset(datasetPersistentId, exporter, null); + } static Response exportDataset(String datasetPersistentId, String exporter, String apiToken) { // http://localhost:8080/api/datasets/export?exporter=dataverse_json&persistentId=doi%3A10.5072/FK2/W6WIMQ - return given() - .header(API_TOKEN_HTTP_HEADER, apiToken) + RequestSpecification requestSpecification = given(); + if (apiToken != null) { + requestSpecification = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); + } + return requestSpecification + // .header(API_TOKEN_HTTP_HEADER, apiToken) // .get("/api/datasets/:persistentId/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); .get("/api/datasets/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); } From 26a83c4f539e524169b8159500493a8fe40fe7e6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 26 Feb 2021 14:02:31 -0500 Subject: [PATCH 238/274] change flyway number --- .../{V5.5.0.1__7563-workflow.sql => V5.3.0.4__7563-workflow.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.5.0.1__7563-workflow.sql => V5.3.0.4__7563-workflow.sql} (100%) diff --git a/src/main/resources/db/migration/V5.5.0.1__7563-workflow.sql b/src/main/resources/db/migration/V5.3.0.4__7563-workflow.sql similarity index 100% rename from src/main/resources/db/migration/V5.5.0.1__7563-workflow.sql rename to src/main/resources/db/migration/V5.3.0.4__7563-workflow.sql From 9813c77eecbe4b2e3d62a5cd29a4e522d6a6ffad Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 26 Feb 2021 14:28:27 -0500 Subject: [PATCH 239/274] put message in bundle #7619 --- .../api/errorhandlers/WebApplicationExceptionHandler.java | 2 +- src/main/java/propertyFiles/Bundle.properties | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java index aa8bb2b3772..5f28bfd0afc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/errorhandlers/WebApplicationExceptionHandler.java @@ -47,7 +47,7 @@ public Response toResponse(WebApplicationException ex) { if ( (ex.getMessage()+"").toLowerCase().startsWith("tabular data required")) { jrb.message(BundleUtil.getStringFromBundle("access.api.exception.metadata.not.available.for.nontabular.file")); } else if ((ex.getMessage() + "").toLowerCase().startsWith("no permission to download file")) { - jrb.message("You do not have permission to download this file."); + jrb.message(BundleUtil.getStringFromBundle("access.api.exception.metadata.restricted.no.permission")); } else { jrb.message("Bad Request. The API request cannot be completed with the parameters supplied. Please check your code for typos, or consult our API guide at http://guides.dataverse.org."); jrb.request(request); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 4d95322caf2..8352e66c92d 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2354,6 +2354,7 @@ access.api.requestList.fileNotFound=Could not find datafile with id {0}. access.api.requestList.noKey=You must provide a key to get list of access requests for a file. access.api.requestList.noRequestsFound=There are no access requests for this file {0}. access.api.exception.metadata.not.available.for.nontabular.file=This type of metadata is only available for tabular files. +access.api.exception.metadata.restricted.no.permission=You do not have permission to download this file. access.api.exception.version.not.found=Could not find requested dataset version. #permission From 698c9b718f0d48394c4320c4e03d8ad54c8805fc Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 26 Feb 2021 16:43:57 -0500 Subject: [PATCH 240/274] Add truncation plugin to apply to html+text desc blocks [ref #6685] --- src/main/webapp/dataverse_template.xhtml | 3 +- src/main/webapp/resources/css/structure.css | 26 ++++- .../resources/js/dv_rebind_bootstrap_ui.js | 64 ++++++++++- .../webapp/resources/js/jquery.truncate.js | 103 ++++++++++++++++++ 4 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 src/main/webapp/resources/js/jquery.truncate.js diff --git a/src/main/webapp/dataverse_template.xhtml b/src/main/webapp/dataverse_template.xhtml index e461e853955..61487c0d1c0 100644 --- a/src/main/webapp/dataverse_template.xhtml +++ b/src/main/webapp/dataverse_template.xhtml @@ -84,7 +84,8 @@ - + + diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 2603fd6ba0b..f4dea7efe7c 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -627,13 +627,37 @@ div.edit-field div.ui-message {margin:6px 0;} .metadata-container table.metadata {border-collapse:separate;margin:0 4px;width:100%;} .metadata-container table.metadata th {width:25%;vertical-align:top;padding:6px 12px;} .metadata-container table.metadata td {padding:6px 12px;vertical-align:top;} -@media(max-width:767px){ + +/* TO-DO FIX max-width:1199px FOR SUMMARY BLOCK ONLY, NOT OTHER PAGES */ +@media(max-width:1199px){ .metadata-container table.metadata th, .metadata-container table.metadata td {display:block;width:100%;} .metadata-container table.metadata th {padding-bottom:0;} .metadata-container table.metadata td {padding-left:30px;} } .panel-body.metadata-container {padding:15px 0;} .metadata-logo {max-width:180px;max-height:40px;} +.metadata-container .truncate-more-link, .metadata-container .collapse-less-link {color: #337AB7; cursor: pointer; cursor: hand;} + +/* HTML METADATA OVERWRITE */ +.metadata-container table.metadata td > div { + /* https://stackoverflow.com/a/24158056 */ + /*-moz-white-space: pre-wrap; + white-space: pre-wrap; + hyphens: auto; + -ms-word-break: break-all; + -ms-word-wrap: break-all; + -webkit-word-break: break-word; + -webkit-word-wrap: break-word; + word-break: break-word; + word-wrap: break-word; + -webkit-hyphens: auto; + -moz-hyphens: auto; + -ms-hyphens: auto; + hyphens: auto;*/ +} +.metadata-container pre { + white-space: pre-wrap !important; +} /* Primefaces vs Bootstrap selectOneMenu fix */ div.ui-selectonemenu.form-control {padding:0px;} diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 44e86831c69..608b61ce45f 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -22,6 +22,12 @@ function bind_bsui_components(){ // Hide open tooltips + popovers $('.bootstrap-button-tooltip, [data-toggle="tooltip"]').tooltip("hide"); $("[data-toggle='popover']").popover("hide"); + + // Truncate content + contentTruncate(); + + // Truncate checksums + checksumTruncate(); // Tooltips + popovers bind_tooltip_popover(); @@ -29,9 +35,6 @@ function bind_bsui_components(){ // Disabled pagination links disabledLinks(); - // Truncate checksums - checksumTruncate(); - // Sharrre sharrre(); @@ -157,7 +160,62 @@ function sharrre(){ } }); } + +/* + * Truncate any metadata content + */ +function contentTruncate(){ + // jquery.truncate.js + // HTML-safe truncation + // https://github.com/pathable/truncate + + // WHAT ARE WE TRUNCATING + // TO-DO IS THERE A MORE SCALABLE SELECTOR METHOD THAN THIS? ADD TRUNCATE CLASS TO OBJECTS? e.g. span.checksum-truncate + $('#dsDescription td div').each(function () { + + // GET THE DESC TEXT AND HTML + var truncateThis = $(this).html(); + + // TO-DO TRUNCATE BTN LABEL TEXT FROM BUNDLE + var readMoreBtn = '... '; + var readLessBtn = ''; + + if ($(this).hasClass('thisWasTruncated')) { + // DO NOTHING + } + else { + + // TO-DO PUT THE ORIGINAL FULL DESCRIPTION SOMEWHERE + // $('#dsDescription td div').attr({'data-html':'true', 'data-original-text':truncateThis}); + + var thisWasTruncated = jQuery.truncate(truncateThis, { + length: 500, + words: true, + keepFirstWord: true, + ellipsis: readMoreBtn, + finishBlock: true + }); + + // ... add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + $(this).toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); + + // Return full description on Read More link click ++ add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + $(document).on('click', 'button.truncate-more-link', function() { + $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(truncateThis).append(readLessBtn).find('img').attr('class', 'img-responsive'); + $('button.collapse-less-link').tooltip(); + }); + // ... add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + $(document).on('click', 'button.collapse-less-link', function() { + $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); + $('button.truncate-more-link').tooltip(); + }); + + } + + }); +} + /* * Truncate file checksums */ diff --git a/src/main/webapp/resources/js/jquery.truncate.js b/src/main/webapp/resources/js/jquery.truncate.js new file mode 100644 index 00000000000..e43d4134ae1 --- /dev/null +++ b/src/main/webapp/resources/js/jquery.truncate.js @@ -0,0 +1,103 @@ +(function($) { + + // Matches trailing non-space characters. + var chop = /(\s*\S+|\s)$/; + + // Matches the first word in the string. + var start = /^(\S*)/; + + // Return a truncated html string. Delegates to $.fn.truncate. + $.truncate = function(html, options) { + return $('
    ').append(html).truncate(options).html(); + }; + + // Truncate the contents of an element in place. + $.fn.truncate = function(options) { + if ($.isNumeric(options)) options = {length: options}; + var o = $.extend({}, $.truncate.defaults, options); + + return this.each(function() { + var self = $(this); + + if (o.noBreaks) self.find('br').replaceWith(' '); + + var text = self.text(); + var excess = text.length - o.length; + + if (o.stripTags) self.text(text); + + // Chop off any partial words if appropriate. + if (o.words && excess > 0) { + var truncated = text.slice(0, o.length).replace(chop, '').length; + + if (o.keepFirstWord && truncated === 0) { + excess = text.length - start.exec(text)[0].length - 1; + } else { + excess = text.length - truncated - 1; + } + } + + if (excess < 0 || !excess && !o.truncated) return; + + // Iterate over each child node in reverse, removing excess text. + $.each(self.contents().get().reverse(), function(i, el) { + var $el = $(el); + var text = $el.text(); + var length = text.length; + + // If the text is longer than the excess, remove the node and continue. + if (length <= excess) { + o.truncated = true; + excess -= length; + $el.remove(); + return; + } + + // Remove the excess text and append the ellipsis. + if (el.nodeType === 3) { + // should we finish the block anyway? + if (o.finishBlock) { + $(el.splitText(length)).replaceWith(o.ellipsis); + } else { + $(el.splitText(length - excess - 1)).replaceWith(o.ellipsis); + } + return false; + } + + // Recursively truncate child nodes. + $el.truncate($.extend(o, {length: length - excess})); + return false; + }); + }); + }; + + $.truncate.defaults = { + + // Strip all html elements, leaving only plain text. + stripTags: false, + + // Only truncate at word boundaries. + words: false, + + // When 'words' is active, keeps the first word in the string + // even if it's longer than a target length. + keepFirstWord: false, + + // Replace instances of
    with a single space. + noBreaks: false, + + // if true always truncate the content at the end of the block. + finishBlock: false, + + // The maximum length of the truncated html. + length: Infinity, + + // The character to use as the ellipsis. The word joiner (U+2060) can be + // used to prevent a hanging ellipsis, but displays incorrectly in Chrome + // on Windows 7. + // http://code.google.com/p/chromium/issues/detail?id=68323 + ellipsis: '\u2026' // '\u2060\u2026' + + }; + +})(jQuery); From 0f7b3aa433dbf5179e16b7e47bcd4de21f6813d2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 10:07:10 -0500 Subject: [PATCH 241/274] #7373 update scripts --- conf/docker-aio/1prep.sh | 6 +++--- conf/docker-aio/c8.dockerfile | 8 ++++---- conf/docker-aio/entrypoint.bash | 2 +- .../installation/files/etc/systemd/solr.service | 6 +++--- downloads/download.sh | 2 +- scripts/installer/README.txt | 2 +- scripts/vagrant/setup-solr.sh | 12 ++++++------ scripts/vagrant/setup.sh | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/conf/docker-aio/1prep.sh b/conf/docker-aio/1prep.sh index c5b2595925e..98cbc2eec93 100755 --- a/conf/docker-aio/1prep.sh +++ b/conf/docker-aio/1prep.sh @@ -4,9 +4,9 @@ # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world mkdir -p testdata/doc/sphinx-guides/source/_static/util/ -cp ../solr/7.7.2/schema*.xml testdata/ -cp ../solr/7.7.2/solrconfig.xml testdata/ -cp ../solr/7.7.2/updateSchemaMDB.sh testdata/ +cp ../solr/8.8.1/schema*.xml testdata/ +cp ../solr/8.8.1/solrconfig.xml testdata/ +cp ../solr/8.8.1/updateSchemaMDB.sh testdata/ cp ../jhove/jhove.conf testdata/ cp ../jhove/jhoveConfig.xsd testdata/ cd ../../ diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile index c505211cb75..92c3dc24d56 100644 --- a/conf/docker-aio/c8.dockerfile +++ b/conf/docker-aio/c8.dockerfile @@ -17,7 +17,7 @@ COPY testdata/sushi_sample_logs.json /tmp/ COPY disableipv6.conf /etc/sysctl.d/ RUN rm /etc/httpd/conf/* COPY httpd.conf /etc/httpd/conf -RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.7.2dv.tgz +RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.6.zip ; ln -s /opt/payara5 /opt/glassfish4 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm` @@ -28,9 +28,9 @@ RUN sudo -u postgres /usr/bin/initdb /var/lib/pgsql/data # copy configuration related files RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ -RUN cp -r /opt/solr-7.7.2/server/solr/configsets/_default /opt/solr-7.7.2/server/solr/collection1 -RUN cp /tmp/dv/schema*.xml /opt/solr-7.7.2/server/solr/collection1/conf/ -RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.7.2/server/solr/collection1/conf/solrconfig.xml +RUN cp -r /opt/solr-8.8.1/server/solr/configsets/_default /opt/solr-8.8.1/server/solr/collection1 +RUN cp /tmp/dv/schema*.xml /opt/solr-8.8.1/server/solr/collection1/conf/ +RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml # skipping payara user and solr user (run both as root) diff --git a/conf/docker-aio/entrypoint.bash b/conf/docker-aio/entrypoint.bash index 7ff0c527a28..6cb5463cd1a 100755 --- a/conf/docker-aio/entrypoint.bash +++ b/conf/docker-aio/entrypoint.bash @@ -1,7 +1,7 @@ #!/usr/bin/env bash export LANG=en_US.UTF-8 sudo -u postgres /usr/bin/pg_ctl start -D /var/lib/pgsql/data & -cd /opt/solr-7.7.2/ +cd /opt/solr-8.8.1/ # TODO: Run Solr as non-root and remove "-force". bin/solr start -force bin/solr create_core -c collection1 -d server/solr/collection1/conf -force diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 06eacc68ca2..96960793938 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = forking -WorkingDirectory = /usr/local/solr/solr-7.7.2 -ExecStart = /usr/local/solr/solr-7.7.2/bin/solr start -m 1g -j "jetty.host=127.0.0.1" -ExecStop = /usr/local/solr/solr-7.7.2/bin/solr stop +WorkingDirectory = /usr/local/solr/solr-8.8.1 +ExecStart = /usr/local/solr/solr-8.8.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1" +ExecStop = /usr/local/solr/solr-8.8.1/bin/solr stop LimitNOFILE=65000 LimitNPROC=65000 Restart=on-failure diff --git a/downloads/download.sh b/downloads/download.sh index 59bbcef9dc8..8c2b51dd4c7 100755 --- a/downloads/download.sh +++ b/downloads/download.sh @@ -1,5 +1,5 @@ #!/bin/sh curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip -curl -L -O https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz +curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar diff --git a/scripts/installer/README.txt b/scripts/installer/README.txt index 35dbaadc3bf..70f08550a75 100644 --- a/scripts/installer/README.txt +++ b/scripts/installer/README.txt @@ -35,7 +35,7 @@ from conf/jhove: jhove.conf -SOLR schema and config files, from conf/solr/7.7.2: +SOLR schema and config files, from conf/solr/8.8.1: schema.xml schema_dv_cmb_copies.xml diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index 02654dec1d0..9c1ba50ee49 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -5,12 +5,12 @@ SOLR_USER=solr SOLR_HOME=/usr/local/solr mkdir $SOLR_HOME chown $SOLR_USER:$SOLR_USER $SOLR_HOME -su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-7.7.2.tgz $SOLR_HOME" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-7.7.2.tgz" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.7.2/server/solr && cp -r configsets/_default . && mv _default collection1" -su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/7.7.2/schema*.xml $SOLR_HOME/solr-7.7.2/server/solr/collection1/conf/" -su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/7.7.2/solrconfig.xml $SOLR_HOME/solr-7.7.2/server/solr/collection1/conf/solrconfig.xml" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-7.7.2 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" +su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.8.1.tgz $SOLR_HOME" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-78.8.1.tgz" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1/server/solr && cp -r configsets/_default . && mv _default collection1" +su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/schema*.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/" +su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/solrconfig.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" cp /dataverse/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr /etc/init.d/solr chmod 755 /etc/init.d/solr /etc/init.d/solr stop diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index 7e62fefd0ae..14f12cea692 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -54,7 +54,7 @@ echo "Ensuring Unix user '$SOLR_USER' exists" useradd $SOLR_USER || : DOWNLOAD_DIR='/dataverse/downloads' PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2020.6.zip" -SOLR_TGZ="$DOWNLOAD_DIR/solr-7.7.2.tgz" +SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz" if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...." cd $DOWNLOAD_DIR && ./download.sh && cd From 45b4cc092e9ed2988875c7d3419d3f687a7989c1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 10:40:07 -0500 Subject: [PATCH 242/274] #7373 add release notes --- doc/release-notes/7373-solr-upgrade.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/release-notes/7373-solr-upgrade.md diff --git a/doc/release-notes/7373-solr-upgrade.md b/doc/release-notes/7373-solr-upgrade.md new file mode 100644 index 00000000000..c6c084961bf --- /dev/null +++ b/doc/release-notes/7373-solr-upgrade.md @@ -0,0 +1,11 @@ +### Solr Update + +With this release we upgrade to the latest available stable release in the Solr 8.x branch. We recommend a fresh installation of Solr 8.8.1 (the index will be empty) +followed by an "index all". + +Before you start the "index all", Dataverse will appear to be empty because +the search results come from Solr. As indexing progresses, partial results will +appear until indexing is complete. + + +See http://guides.dataverse.org/installation/prerequisites.html#installing-solr From 663652d6f472e81ac666a4aa9f5eaeae111b03f6 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 10:49:20 -0500 Subject: [PATCH 243/274] #7373 update script --- conf/docker-aio/0prep_deps.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 71398f03a42..50294dd3772 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -10,12 +10,11 @@ if [ ! -e dv/deps/payara-5.2020.6.zip ]; then wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip -O dv/deps/payara-5.2020.6.zip fi -if [ ! -e dv/deps/solr-7.7.2dv.tgz ]; then +if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then echo "solr dependency prep" # schema changes *should* be the only ones... - cd dv/deps/ - #wget https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz -O solr-7.3.0dv.tgz - wget https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz -O solr-7.7.2dv.tgz + cd dv/deps/ + wget https://archive.apache.org/dist/lucene/solr/8.8.2/solr-8.8.1.tgz -O solr-8.8.1dv.tgz cd ../../ fi From 2064fec9c8264ecb1f36fea325e7065cddcab66a Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 1 Mar 2021 15:03:22 -0500 Subject: [PATCH 244/274] Truncation clean up, moved text to bundle [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 6 ++++ src/main/webapp/dataset.xhtml | 3 ++ src/main/webapp/resources/css/structure.css | 18 ++--------- .../resources/js/dv_rebind_bootstrap_ui.js | 30 ++++++++----------- 4 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 29ba794821b..fdaa68d3650 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -59,6 +59,12 @@ apply=Apply add=Add delete=Delete copyClipboard=Copy to Clipboard + +truncateMoreBtn=Read More +truncateMoreTip=Click to read the full +truncateLessBtn=Read Less +truncateLessTip=Click to collapse the + yes=Yes no=No previous=Previous diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 4f23d4c7b44..3480e83ba57 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1655,6 +1655,9 @@ // div { - /* https://stackoverflow.com/a/24158056 */ - /*-moz-white-space: pre-wrap; - white-space: pre-wrap; - hyphens: auto; - -ms-word-break: break-all; - -ms-word-wrap: break-all; - -webkit-word-break: break-word; - -webkit-word-wrap: break-word; - word-break: break-word; - word-wrap: break-word; - -webkit-hyphens: auto; - -moz-hyphens: auto; - -ms-hyphens: auto; - hyphens: auto;*/ + /* TO-DO #6685 #6578 #1634 #1249 */ } +/* METADATA HTML */ .metadata-container pre { white-space: pre-wrap !important; } diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 608b61ce45f..16a5584ef01 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -164,55 +164,49 @@ function sharrre(){ /* * Truncate any metadata content */ -function contentTruncate(){ +function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip){ // jquery.truncate.js // HTML-safe truncation // https://github.com/pathable/truncate // WHAT ARE WE TRUNCATING - // TO-DO IS THERE A MORE SCALABLE SELECTOR METHOD THAN THIS? ADD TRUNCATE CLASS TO OBJECTS? e.g. span.checksum-truncate - $('#dsDescription td div').each(function () { - + // SELECTOR ID FROM PARAMETERS + $('#' + truncSelector + ' td div').each(function () { // GET THE DESC TEXT AND HTML var truncateThis = $(this).html(); - // TO-DO TRUNCATE BTN LABEL TEXT FROM BUNDLE - var readMoreBtn = '... '; - var readLessBtn = ''; + // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS + var readMoreBtn = '... '; + var readLessBtn = ''; if ($(this).hasClass('thisWasTruncated')) { - // DO NOTHING + // ALREADY TRUNCATED, DO NOTHING } else { - - // TO-DO PUT THE ORIGINAL FULL DESCRIPTION SOMEWHERE - // $('#dsDescription td div').attr({'data-html':'true', 'data-original-text':truncateThis}); - var thisWasTruncated = jQuery.truncate(truncateThis, { length: 500, words: true, keepFirstWord: true, ellipsis: readMoreBtn, - finishBlock: true + // REMOVED FINISH BLOCK TO TRUNCATE MID CONENT + // finishBlock: true }); - // ... add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + // ... add responsive img class ++ link target $(this).toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); - // Return full description on Read More link click ++ add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + // ... add full description to summary block on Read More link click ++ add responsive img class ++ link target $(document).on('click', 'button.truncate-more-link', function() { $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(truncateThis).append(readLessBtn).find('img').attr('class', 'img-responsive'); $('button.collapse-less-link').tooltip(); }); - // ... add responsive img class ++ link target ... DO THIS ANYWAY SOMEWHERE ELSE!! + // ... add responsive img class ++ link target $(document).on('click', 'button.collapse-less-link', function() { $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); $('button.truncate-more-link').tooltip(); }); - } - }); } From 6633943a2988ddce4322d495b168b6a47bc9116f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 15:09:11 -0500 Subject: [PATCH 245/274] #7373 update dev environment doc --- .../source/developers/dev-environment.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 63f482b86ee..b6177e2cd9f 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -117,7 +117,7 @@ On Linux, you should just install PostgreSQL from your package manager without w Install Solr ~~~~~~~~~~~~ -`Solr `_ 7.7.2 is required. +`Solr `_ 8.8.1 is required. To install Solr, execute the following commands: @@ -127,27 +127,27 @@ To install Solr, execute the following commands: ``cd /usr/local/solr`` -``curl -O http://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz`` +``curl -O http://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz`` -``tar xvfz solr-7.7.2.tgz`` +``tar xvfz solr-8.8.1.tgz`` -``cd solr-7.7.2/server/solr`` +``cd solr-8.8.1/server/solr`` ``cp -r configsets/_default collection1`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/schema.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema.xml`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/schema_dv_mdb_fields.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema_dv_mdb_fields.xml`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/schema_dv_mdb_copies.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema_dv_mdb_copies.xml`` ``mv schema*.xml collection1/conf`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/7.7.2/solrconfig.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/solrconfig.xml`` ``mv solrconfig.xml collection1/conf/solrconfig.xml`` -``cd /usr/local/solr/solr-7.7.2`` +``cd /usr/local/solr/solr-8.8.1`` (Please note that the extra jetty argument below is a security measure to limit connections to Solr to only your computer. For extra security, run a firewall.) From 924087a719ee5b75d7af01fc48b07098c13bf861 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 15:31:15 -0500 Subject: [PATCH 246/274] #7373 update prerequisites doc --- .../source/installation/prerequisites.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index b1643fabf40..bc96680b8de 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -166,7 +166,7 @@ The Dataverse Software search index is powered by Solr. Supported Versions ================== -The Dataverse Software has been tested with Solr version 7.7.2. Future releases in the 7.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 7.x (e.g. 8.x) are not supported. +The Dataverse Software has been tested with Solr version 8.8.1. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported. Installing Solr =============== @@ -181,19 +181,19 @@ Become the ``solr`` user and then download and configure Solr:: su - solr cd /usr/local/solr - wget https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz - tar xvzf solr-7.7.2.tgz - cd solr-7.7.2 + wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz + tar xvzf solr-8.8.1.tgz + cd solr-8.8.1 cp -r server/solr/configsets/_default server/solr/collection1 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place:: - cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-7.7.2/server/solr/collection1/conf - cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-7.7.2/server/solr/collection1/conf + cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf + cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev . -A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-7.7.2/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` +A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-8.8.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``:: @@ -212,7 +212,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo Finally, you need to tell Solr to create the core "collection1" on startup:: - echo "name=collection1" > /usr/local/solr/solr-7.7.2/server/solr/collection1/core.properties + echo "name=collection1" > /usr/local/solr/solr-8.8.1/server/solr/collection1/core.properties Solr Init Script ================ From 273133fc7890e9dcc288e5586d7be74f2765a7a2 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 1 Mar 2021 15:54:48 -0500 Subject: [PATCH 247/274] #7373 update metadata customization --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index fdbac996108..923f9ff6e3d 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -644,7 +644,7 @@ configuration, including any enabled metadata schemas: ``curl http://localhost:8080/api/admin/index/solr/schema`` -For convenience and automation you can download and consider running :download:`updateSchemaMDB.sh <../../../../conf/solr/7.7.2/updateSchemaMDB.sh>`. It uses the API endpoint above and writes schema files to the filesystem (so be sure to run it on the Solr server itself as the Unix user who owns the Solr files) and then triggers a Solr reload. +For convenience and automation you can download and consider running :download:`updateSchemaMDB.sh <../../../../conf/solr/8.8.1/updateSchemaMDB.sh>`. It uses the API endpoint above and writes schema files to the filesystem (so be sure to run it on the Solr server itself as the Unix user who owns the Solr files) and then triggers a Solr reload. By default, it will download from your Dataverse installation at `http://localhost:8080` and reload Solr at `http://localhost:8983`. You may use the following environment variables with this script or mix'n'match with options: @@ -657,13 +657,13 @@ Environment variable Option Description E `UNBLOCK_KEY` `-u` If your installation has a blocked admin API *xyz* or */secrets/unblock.key* endpoint, you can provide either the key itself or a path to a keyfile -`TARGET` `-t` Provide the config directory of your Solr core */usr/local/solr/solr-7.7.2/server/solr/collection1/conf* +`TARGET` `-t` Provide the config directory of your Solr core */usr/local/solr/solr-8.8.1/server/solr/collection1/conf* "collection1" ==================== ====== =============================================== ========================================================= See the :doc:`/installation/prerequisites/` section of the Installation Guide for a suggested location on disk for the Solr schema file. -Please note that if you are going to make a pull request updating ``conf/solr/7.7.2/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields. +Please note that if you are going to make a pull request updating ``conf/solr/8.8.1/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields. Reloading a Metadata Block -------------------------- From c498f6a47ffe4352aa374911a62e5deb196cb851 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 2 Mar 2021 09:46:00 -0500 Subject: [PATCH 248/274] #7373 fix directory typo --- conf/docker-aio/0prep_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 50294dd3772..5bf44bfbe82 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -14,7 +14,7 @@ if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then echo "solr dependency prep" # schema changes *should* be the only ones... cd dv/deps/ - wget https://archive.apache.org/dist/lucene/solr/8.8.2/solr-8.8.1.tgz -O solr-8.8.1dv.tgz + wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz -O solr-8.8.1dv.tgz cd ../../ fi From cb74b81012d634acb1b1b36e586d2c4fd8a5dd9f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 2 Mar 2021 10:10:24 -0500 Subject: [PATCH 249/274] #7373 another typo --- scripts/vagrant/setup-solr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index 9c1ba50ee49..41635994bc5 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -6,7 +6,7 @@ SOLR_HOME=/usr/local/solr mkdir $SOLR_HOME chown $SOLR_USER:$SOLR_USER $SOLR_HOME su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.8.1.tgz $SOLR_HOME" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-78.8.1.tgz" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-8.8.1.tgz" su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1/server/solr && cp -r configsets/_default . && mv _default collection1" su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/schema*.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/" su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/solrconfig.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml" From d3cbab31d8fbb2bdb5861a95c9371b09e368864f Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 2 Mar 2021 10:15:24 -0500 Subject: [PATCH 250/274] #7373 update pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index de8ff91863a..717c0f6afa1 100644 --- a/pom.xml +++ b/pom.xml @@ -325,7 +325,7 @@ org.apache.solr solr-solrj - 7.7.2 + 8.8.1 org.apache.commons From d46f6a783ae8990c76b0d92160f7663fb32b3f06 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 2 Mar 2021 15:54:40 -0500 Subject: [PATCH 251/274] Removed truncation plugin and attempted hacky css solution WIP [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 4 +- .../resources/js/dv_rebind_bootstrap_ui.js | 51 +++------ .../webapp/resources/js/jquery.truncate.js | 103 ------------------ 3 files changed, 18 insertions(+), 140 deletions(-) delete mode 100644 src/main/webapp/resources/js/jquery.truncate.js diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index fdaa68d3650..ce45c768d9b 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -59,12 +59,10 @@ apply=Apply add=Add delete=Delete copyClipboard=Copy to Clipboard - truncateMoreBtn=Read More truncateMoreTip=Click to read the full truncateLessBtn=Read Less -truncateLessTip=Click to collapse the - +truncateLessTip=Click to collapse the yes=Yes no=No previous=Previous diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 16a5584ef01..1e1c6d5d6e3 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -166,47 +166,30 @@ function sharrre(){ */ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip){ - // jquery.truncate.js - // HTML-safe truncation - // https://github.com/pathable/truncate + // I MADE IT MYSELF... // WHAT ARE WE TRUNCATING // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td div').each(function () { - // GET THE DESC TEXT AND HTML - var truncateThis = $(this).html(); - + + // ADD A MAX-HEIGHT TO CONTAINER + $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); + // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS - var readMoreBtn = '... '; - var readLessBtn = ''; + var readMoreBtn = ''; + var moreFade = '
    ' + readMoreBtn + '
    '; - if ($(this).hasClass('thisWasTruncated')) { - // ALREADY TRUNCATED, DO NOTHING - } - else { - var thisWasTruncated = jQuery.truncate(truncateThis, { - length: 500, - words: true, - keepFirstWord: true, - ellipsis: readMoreBtn, - // REMOVED FINISH BLOCK TO TRUNCATE MID CONENT - // finishBlock: true - }); - - // ... add responsive img class ++ link target - $(this).toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); + // add responsive img class + $(this).find('img').attr('class', 'img-responsive'); + + // add Read More + button background fade + $(this).append(moreFade); - // ... add full description to summary block on Read More link click ++ add responsive img class ++ link target - $(document).on('click', 'button.truncate-more-link', function() { - $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(truncateThis).append(readLessBtn).find('img').attr('class', 'img-responsive'); - $('button.collapse-less-link').tooltip(); - }); - // ... add responsive img class ++ link target - $(document).on('click', 'button.collapse-less-link', function() { - $(this).tooltip('hide').parent('div').toggleClass('thisWasTruncated').html(thisWasTruncated).find('img').attr('class', 'img-responsive'); - $('button.truncate-more-link').tooltip(); - }); - } + // ... add full description to summary block on Read More link click ++ add responsive img class + $(document).on('click', 'button.desc-more-link', function() { + $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); + $('#more-fade-block').remove(); + }); }); } diff --git a/src/main/webapp/resources/js/jquery.truncate.js b/src/main/webapp/resources/js/jquery.truncate.js deleted file mode 100644 index e43d4134ae1..00000000000 --- a/src/main/webapp/resources/js/jquery.truncate.js +++ /dev/null @@ -1,103 +0,0 @@ -(function($) { - - // Matches trailing non-space characters. - var chop = /(\s*\S+|\s)$/; - - // Matches the first word in the string. - var start = /^(\S*)/; - - // Return a truncated html string. Delegates to $.fn.truncate. - $.truncate = function(html, options) { - return $('
    ').append(html).truncate(options).html(); - }; - - // Truncate the contents of an element in place. - $.fn.truncate = function(options) { - if ($.isNumeric(options)) options = {length: options}; - var o = $.extend({}, $.truncate.defaults, options); - - return this.each(function() { - var self = $(this); - - if (o.noBreaks) self.find('br').replaceWith(' '); - - var text = self.text(); - var excess = text.length - o.length; - - if (o.stripTags) self.text(text); - - // Chop off any partial words if appropriate. - if (o.words && excess > 0) { - var truncated = text.slice(0, o.length).replace(chop, '').length; - - if (o.keepFirstWord && truncated === 0) { - excess = text.length - start.exec(text)[0].length - 1; - } else { - excess = text.length - truncated - 1; - } - } - - if (excess < 0 || !excess && !o.truncated) return; - - // Iterate over each child node in reverse, removing excess text. - $.each(self.contents().get().reverse(), function(i, el) { - var $el = $(el); - var text = $el.text(); - var length = text.length; - - // If the text is longer than the excess, remove the node and continue. - if (length <= excess) { - o.truncated = true; - excess -= length; - $el.remove(); - return; - } - - // Remove the excess text and append the ellipsis. - if (el.nodeType === 3) { - // should we finish the block anyway? - if (o.finishBlock) { - $(el.splitText(length)).replaceWith(o.ellipsis); - } else { - $(el.splitText(length - excess - 1)).replaceWith(o.ellipsis); - } - return false; - } - - // Recursively truncate child nodes. - $el.truncate($.extend(o, {length: length - excess})); - return false; - }); - }); - }; - - $.truncate.defaults = { - - // Strip all html elements, leaving only plain text. - stripTags: false, - - // Only truncate at word boundaries. - words: false, - - // When 'words' is active, keeps the first word in the string - // even if it's longer than a target length. - keepFirstWord: false, - - // Replace instances of
    with a single space. - noBreaks: false, - - // if true always truncate the content at the end of the block. - finishBlock: false, - - // The maximum length of the truncated html. - length: Infinity, - - // The character to use as the ellipsis. The word joiner (U+2060) can be - // used to prevent a hanging ellipsis, but displays incorrectly in Chrome - // on Windows 7. - // http://code.google.com/p/chromium/issues/detail?id=68323 - ellipsis: '\u2026' // '\u2060\u2026' - - }; - -})(jQuery); From e3e5cbe1468221b26180d11f3ce89df6111926b1 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Tue, 2 Mar 2021 16:22:12 -0500 Subject: [PATCH 252/274] Few minor code clean up edits for desc truncation [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 2 -- src/main/webapp/dataset.xhtml | 6 ++++-- src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js | 10 ++-------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index ce45c768d9b..7ab054e18fc 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -61,8 +61,6 @@ delete=Delete copyClipboard=Copy to Clipboard truncateMoreBtn=Read More truncateMoreTip=Click to read the full -truncateLessBtn=Read Less -truncateLessTip=Click to collapse the yes=Yes no=No previous=Previous diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 3480e83ba57..1da3c2a69eb 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1656,8 +1656,10 @@ $(document).ready(function () { popoverHTML('#{bundle.htmlAllowedTitle}', '#{bundle.htmlAllowedTags}'); - // TO-DO MAKE DYNAMIC - contentTruncate('dsDescription', 'Description', '#{bundle.truncateMoreBtn}', '#{bundle.truncateMoreTip}', '#{bundle.truncateLessBtn}', '#{bundle.truncateLessTip}'); + // truncate summary description, contentTruncate function in dv_rebind_bootstrap_ui.js + var = 'dsDescription' + var descLabel = $('#' + descId + ' th').text(); + contentTruncate(descId, descLabel, '#{bundle.truncateMoreBtn}', '#{bundle.truncateMoreTip}'); }); function openDialog() { PF('details').show(); diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 1e1c6d5d6e3..e0acdc77d4c 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -23,9 +23,6 @@ function bind_bsui_components(){ $('.bootstrap-button-tooltip, [data-toggle="tooltip"]').tooltip("hide"); $("[data-toggle='popover']").popover("hide"); - // Truncate content - contentTruncate(); - // Truncate checksums checksumTruncate(); @@ -162,13 +159,10 @@ function sharrre(){ } /* - * Truncate any metadata content + * Truncate dataset description content */ -function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip){ - - // I MADE IT MYSELF... +function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip){ - // WHAT ARE WE TRUNCATING // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td div').each(function () { From f9e94835939a94898bb2595b84a12ce362be8503 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 3 Mar 2021 09:58:14 -0500 Subject: [PATCH 253/274] Fixed broken var in truncation code on dataset pg, other code clean up [ref #6685] --- src/main/webapp/dataset.xhtml | 2 +- src/main/webapp/resources/css/structure.css | 17 +++++++++++++---- .../resources/js/dv_rebind_bootstrap_ui.js | 12 ++++++------ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 1da3c2a69eb..2ff088c5220 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1657,7 +1657,7 @@ popoverHTML('#{bundle.htmlAllowedTitle}', '#{bundle.htmlAllowedTags}'); // truncate summary description, contentTruncate function in dv_rebind_bootstrap_ui.js - var = 'dsDescription' + var descId = 'dsDescription'; var descLabel = $('#' + descId + ' th').text(); contentTruncate(descId, descLabel, '#{bundle.truncateMoreBtn}', '#{bundle.truncateMoreTip}'); }); diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index c531950d19a..cd6785679ae 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -628,11 +628,17 @@ div.edit-field div.ui-message {margin:6px 0;} .metadata-container table.metadata th {width:25%;vertical-align:top;padding:6px 12px;} .metadata-container table.metadata td {padding:6px 12px;vertical-align:top;} -/* TO-DO FIX max-width:1199px FOR SUMMARY BLOCK ONLY, NOT OTHER PAGES */ +/* STACK METADATA TABLE LABELS + VALUES FOR SM, XS */ +@media(max-width:767px){ + .metadata-container:not(#dataset-summary-metadata) table.metadata th, .metadata-container:not(#dataset-summary-metadata) table.metadata td {display:block;width:100%;} + .metadata-container:not(#dataset-summary-metadata) table.metadata th {padding-bottom:0;} + .metadata-container:not(#dataset-summary-metadata) table.metadata td {padding-left:30px;} +} +/* STACK METADATA TABLE LABELS + VALUES FOR MD, SM, XS ... DATASET-SUMMARY-BLOCK ONLY */ @media(max-width:1199px){ - .metadata-container table.metadata th, .metadata-container table.metadata td {display:block;width:100%;} - .metadata-container table.metadata th {padding-bottom:0;} - .metadata-container table.metadata td {padding-left:30px;} + #dataset-summary-metadata.metadata-container table.metadata th, #dataset-summary-metadata.metadata-container table.metadata td {display:block;width:100%;} + #dataset-summary-metadata.metadata-container table.metadata th {padding-bottom:0;} + #dataset-summary-metadata.metadata-container table.metadata td {padding-left:30px;} } .panel-body.metadata-container {padding:15px 0;} .metadata-logo {max-width:180px;max-height:40px;} @@ -646,6 +652,9 @@ div.edit-field div.ui-message {margin:6px 0;} .metadata-container pre { white-space: pre-wrap !important; } +/* TRUNCATION */ +div.more-fade-block {text-align:center; padding-top:250px; width:100%; position:absolute; bottom:0; background:linear-gradient(180deg,hsla(0,0%,100%,0),#fff 80%);} +button.desc-more-link {margin:0; padding:0;} /* Primefaces vs Bootstrap selectOneMenu fix */ div.ui-selectonemenu.form-control {padding:0px;} diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index e0acdc77d4c..6d9d42c276c 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -22,9 +22,6 @@ function bind_bsui_components(){ // Hide open tooltips + popovers $('.bootstrap-button-tooltip, [data-toggle="tooltip"]').tooltip("hide"); $("[data-toggle='popover']").popover("hide"); - - // Truncate checksums - checksumTruncate(); // Tooltips + popovers bind_tooltip_popover(); @@ -32,6 +29,9 @@ function bind_bsui_components(){ // Disabled pagination links disabledLinks(); + // Truncate checksums + checksumTruncate(); + // Sharrre sharrre(); @@ -170,8 +170,8 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS - var readMoreBtn = ''; - var moreFade = '
    ' + readMoreBtn + '
    '; + var readMoreBtn = ''; + var moreFade = '
    ' + readMoreBtn + '
    '; // add responsive img class $(this).find('img').attr('class', 'img-responsive'); @@ -182,7 +182,7 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT // ... add full description to summary block on Read More link click ++ add responsive img class $(document).on('click', 'button.desc-more-link', function() { $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); - $('#more-fade-block').remove(); + $(this).parent('div.more-fade-block').remove(); }); }); } From fb406a726fbf961ed2c1cbacbf6ee0c635556e63 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 3 Mar 2021 10:14:06 -0500 Subject: [PATCH 254/274] Fixed truncation function to only apply read more button for desc greater than 250px in height [ref #6685] --- .../resources/js/dv_rebind_bootstrap_ui.js | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 6d9d42c276c..2b32be6de0b 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -166,24 +166,30 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td div').each(function () { - // ADD A MAX-HEIGHT TO CONTAINER - $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); - - // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS - var readMoreBtn = ''; - var moreFade = '
    ' + readMoreBtn + '
    '; - - // add responsive img class + // add responsive img class to limit width to that of container $(this).find('img').attr('class', 'img-responsive'); - // add Read More + button background fade - $(this).append(moreFade); + // find container height + var containerHeight = $(this).outerHeight(); + + if (containerHeight > 250) { + // ADD A MAX-HEIGHT TO CONTAINER + $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); - // ... add full description to summary block on Read More link click ++ add responsive img class - $(document).on('click', 'button.desc-more-link', function() { - $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); - $(this).parent('div.more-fade-block').remove(); - }); + // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS + var readMoreBtn = ''; + var moreFade = '
    ' + readMoreBtn + '
    '; + + // add Read More + button background fade + $(this).append(moreFade); + + // ... add full description to summary block on Read More link click ++ add responsive img class + $(document).on('click', 'button.desc-more-link', function() { + $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); + $(this).parent('div.more-fade-block').remove(); + }); + } + }); } From 2a2f7bfc004f46da88287d046a409c72f87f0656 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Wed, 3 Mar 2021 11:35:35 -0500 Subject: [PATCH 255/274] Improved truncation btn label text and tooltip by passing dynamic label text [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 4 ++-- src/main/webapp/dataset.xhtml | 11 ++++++++--- .../webapp/resources/js/dv_rebind_bootstrap_ui.js | 7 ++----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 7ab054e18fc..b386e8084f4 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -59,8 +59,8 @@ apply=Apply add=Add delete=Delete copyClipboard=Copy to Clipboard -truncateMoreBtn=Read More -truncateMoreTip=Click to read the full +truncateMoreBtn=Read full {0} [+] +truncateMoreTip=Click to read the full {0}. yes=Yes no=No previous=Previous diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 2ff088c5220..949c97ceb4d 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -9,7 +9,8 @@ xmlns:jsf="http://xmlns.jcp.org/jsf" xmlns:pt="http://java.sun.com/jsf/passthrough" xmlns:cc="http://java.sun.com/jsf/composite" - xmlns:o="http://omnifaces.org/ui"> + xmlns:o="http://omnifaces.org/ui" + xmlns:of="http://omnifaces.org/functions"> @@ -1657,9 +1658,13 @@ popoverHTML('#{bundle.htmlAllowedTitle}', '#{bundle.htmlAllowedTags}'); // truncate summary description, contentTruncate function in dv_rebind_bootstrap_ui.js + // add summary desc row id to pass as function selector var descId = 'dsDescription'; - var descLabel = $('#' + descId + ' th').text(); - contentTruncate(descId, descLabel, '#{bundle.truncateMoreBtn}', '#{bundle.truncateMoreTip}'); + // add metadata dynamic label text to button and tooltip text from bundle + var truncMoreBtn = '#{of:format1(bundle.truncateMoreBtn, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; + var truncMoreTip = '#{of:format1(bundle.truncateMoreTip, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; + // pass text params to function + contentTruncate(descId, truncMoreBtn, truncMoreTip); }); function openDialog() { PF('details').show(); diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 2b32be6de0b..a5a77009782 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -161,11 +161,9 @@ function sharrre(){ /* * Truncate dataset description content */ -function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreTip){ - +function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip){ // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td div').each(function () { - // add responsive img class to limit width to that of container $(this).find('img').attr('class', 'img-responsive'); @@ -177,7 +175,7 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS - var readMoreBtn = ''; + var readMoreBtn = ''; var moreFade = '
    ' + readMoreBtn + '
    '; // add Read More + button background fade @@ -189,7 +187,6 @@ function contentTruncate(truncSelector, truncMetaLabel, truncMoreBtn, truncMoreT $(this).parent('div.more-fade-block').remove(); }); } - }); } From ed6f57583f10879659ab8ab9079b1d345d2957f8 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 3 Mar 2021 16:08:12 -0500 Subject: [PATCH 256/274] #7373 remove old version --- conf/solr/7.7.2/readme.md | 1 - conf/solr/7.7.2/schema.xml | 1198 ------------------ conf/solr/7.7.2/schema_dv_mdb_copies.xml | 161 --- conf/solr/7.7.2/schema_dv_mdb_fields.xml | 161 --- conf/solr/7.7.2/solrconfig.xml | 1410 ---------------------- conf/solr/7.7.2/updateSchemaMDB.sh | 85 -- 6 files changed, 3016 deletions(-) delete mode 100644 conf/solr/7.7.2/readme.md delete mode 100644 conf/solr/7.7.2/schema.xml delete mode 100644 conf/solr/7.7.2/schema_dv_mdb_copies.xml delete mode 100644 conf/solr/7.7.2/schema_dv_mdb_fields.xml delete mode 100644 conf/solr/7.7.2/solrconfig.xml delete mode 100755 conf/solr/7.7.2/updateSchemaMDB.sh diff --git a/conf/solr/7.7.2/readme.md b/conf/solr/7.7.2/readme.md deleted file mode 100644 index 4457cf9a7df..00000000000 --- a/conf/solr/7.7.2/readme.md +++ /dev/null @@ -1 +0,0 @@ -Please see the dev guide for what to do with Solr config files. \ No newline at end of file diff --git a/conf/solr/7.7.2/schema.xml b/conf/solr/7.7.2/schema.xml deleted file mode 100644 index da40a8e99fa..00000000000 --- a/conf/solr/7.7.2/schema.xml +++ /dev/null @@ -1,1198 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/conf/solr/7.7.2/schema_dv_mdb_copies.xml b/conf/solr/7.7.2/schema_dv_mdb_copies.xml deleted file mode 100644 index 080cc71ef50..00000000000 --- a/conf/solr/7.7.2/schema_dv_mdb_copies.xml +++ /dev/null @@ -1,161 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/conf/solr/7.7.2/schema_dv_mdb_fields.xml b/conf/solr/7.7.2/schema_dv_mdb_fields.xml deleted file mode 100644 index 3f844c6183c..00000000000 --- a/conf/solr/7.7.2/schema_dv_mdb_fields.xml +++ /dev/null @@ -1,161 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/conf/solr/7.7.2/solrconfig.xml b/conf/solr/7.7.2/solrconfig.xml deleted file mode 100644 index 51100af7d6e..00000000000 --- a/conf/solr/7.7.2/solrconfig.xml +++ /dev/null @@ -1,1410 +0,0 @@ - - - - - - - - - 7.3.0 - - - - - - - - - - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - edismax - 0.075 - - dvName^400 - authorName^180 - dvSubject^190 - dvDescription^180 - dvAffiliation^170 - title^130 - subject^120 - keyword^110 - topicClassValue^100 - dsDescriptionValue^90 - authorAffiliation^80 - publicationCitation^60 - producerName^50 - fileName^30 - fileDescription^30 - variableLabel^20 - variableName^10 - _text_^1.0 - - - dvName^200 - authorName^100 - dvSubject^100 - dvDescription^100 - dvAffiliation^100 - title^75 - subject^75 - keyword^75 - topicClassValue^75 - dsDescriptionValue^75 - authorAffiliation^75 - publicationCitation^75 - producerName^75 - - - - isHarvested:false^25000 - - - - - - - - - - - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - _text_ - - - - - - - true - ignored_ - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - - - true - false - - - terms - - - - - - - - string - - - - - - explicit - - - elevator - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - - 5 - - - - - - - - - - - - - - diff --git a/conf/solr/7.7.2/updateSchemaMDB.sh b/conf/solr/7.7.2/updateSchemaMDB.sh deleted file mode 100755 index 0044f15c7cd..00000000000 --- a/conf/solr/7.7.2/updateSchemaMDB.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# This script updates the and schema configuration necessary to properly -# index custom metadata fields in Solr. -# 1. Retrieve from Dataverse API endpoint -# 2. Parse and write Solr schema files (which might replace the included files) -# 3. Reload Solr -# -# List of variables: -# ${DATAVERSE_URL}: URL to Dataverse. Defaults to http://localhost:8080 -# ${SOLR_URL}: URL to Solr. Defaults to http://localhost:8983 -# ${UNBLOCK_KEY}: File path to secret or unblock key as string. Only necessary on k8s or when you secured your installation. -# ${TARGET}: Directory where to write the XML files. Defaults to /tmp -# -# Programs used (need to be available on your PATH): -# coreutils: mktemp, csplit -# curl - -usage() { - echo "usage: updateSchemaMDB.sh [options]" - echo "options:" - echo " -d Dataverse URL, defaults to http://localhost:8080" - echo " -h Show this help text" - echo " -s Solr URL, defaults to http://localhost:8983" - echo " -t Directory where to write the XML files. Defaults to /tmp" - echo " -u Dataverse unblock key either as key string or path to keyfile" -} - -### Init (with sane defaults) -DATAVERSE_URL=${DATAVERSE_URL:-"http://localhost:8080"} -SOLR_URL=${SOLR_URL:-"http://localhost:8983"} -TARGET=${TARGET:-"/tmp"} -UNBLOCK_KEY=${UNBLOCK_KEY:-""} - -# if cmdline args are given, override any env var setting (or defaults) -while getopts ":d:hs:t:u:" opt -do - case $opt in - d) DATAVERSE_URL=${OPTARG};; - h) usage; exit 0;; - s) SOLR_URL=${OPTARG};; - t) TARGET=${OPTARG};; - u) UNBLOCK_KEY=${OPTARG};; - :) echo "Missing option argument for -${OPTARG}. Use -h for help." >&2; exit 1;; - \?) echo "Unknown option -${OPTARG}." >&2; usage; exit 1;; - esac -done - -# Special handling of unblock key depending on referencing a secret file or key in var -if [ ! -z "${UNBLOCK_KEY}" ]; then - if [ -f "${UNBLOCK_KEY}" ]; then - UNBLOCK_KEY="?unblock-key=$(cat ${UNBLOCK_KEY})" - else - UNBLOCK_KEY="?unblock-key=${UNBLOCK_KEY}" - fi -fi - -### Retrieval -echo "Retrieve schema data from ${DATAVERSE_URL}/api/admin/index/solr/schema" -TMPFILE=`mktemp` -curl -f -sS "${DATAVERSE_URL}/api/admin/index/solr/schema${UNBLOCK_KEY}" > $TMPFILE - -### Fail gracefull if Dataverse is not ready yet. -if [[ "`wc -l ${TMPFILE}`" < "3" ]]; then - echo "Dataverse responded with empty file. When running on K8s: did you bootstrap yet?" - exit 123 -fi - -### Processing -echo "Writing ${TARGET}/schema_dv_mdb_fields.xml" -echo "" > ${TARGET}/schema_dv_mdb_fields.xml -cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_fields.xml -echo "" >> ${TARGET}/schema_dv_mdb_fields.xml - -echo "Writing ${TARGET}/schema_dv_mdb_copies.xml" -echo "" > ${TARGET}/schema_dv_mdb_copies.xml -cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_copies.xml -echo "" >> ${TARGET}/schema_dv_mdb_copies.xml - -rm ${TMPFILE}* - -### Reloading -echo "Triggering Solr RELOAD at ${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1" -curl -f -sS "${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1" From 17b85ea3755cc73325fc3c17c6b2fa1b87f3cfb6 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Mar 2021 11:17:22 -0500 Subject: [PATCH 257/274] Changing the language on required postgres versions/announcing support for later versions. (#7374) --- doc/release-notes/7374-postgresql-update | 9 ++++++++ .../source/installation/prerequisites.rst | 23 ++++++------------- pom.xml | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) create mode 100644 doc/release-notes/7374-postgresql-update diff --git a/doc/release-notes/7374-postgresql-update b/doc/release-notes/7374-postgresql-update new file mode 100644 index 00000000000..332ffda0060 --- /dev/null +++ b/doc/release-notes/7374-postgresql-update @@ -0,0 +1,9 @@ +## Notes for Dataverse Administrators + +Up until this release our installation guide "strongly recommended" to install PostgreSQL v. 9.6. While that version is known to be very stable, it is nearing its end-of-life (in Nov. 2021). Dataverse Software has now been tested with versions up to 13. If you decide to upgrade PostgreSQL, the tested and recommended way of doing that is as follows: + +- Export your current database with ``pg_dumpall``; +- Install the new version of PostgreSQL; (make sure it's running on the same port, etc. so that no changes are needed in the Payara configuration) +- Re-import the database with ``psql``, as the postgres user. + +Consult the PostgreSQL upgrde documentation for more information, for example https://www.postgresql.org/docs/13/upgrading.html#UPGRADING-VIA-PGDUMPALL. diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index b1643fabf40..2606ad5fbb5 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -98,25 +98,16 @@ PostgreSQL Installing PostgreSQL ======================= -Version 9.6 is strongly recommended because it is the version developers and QA test with:: +The application has been tested with PostgreSQL versions up to 13. We recommend installing the latest version that is available for your OS distribution. *For example*, to install PostgreSQL 13 under RHEL/CentOS 7:: # yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm # yum makecache fast - # yum install -y postgresql96-server - # /usr/pgsql-9.6/bin/postgresql96-setup initdb - # /usr/bin/systemctl start postgresql-9.6 - # /usr/bin/systemctl enable postgresql-9.6 + # yum install -y postgresql13-server + # /usr/pgsql-13/bin/postgresql-13-setup initdb + # /usr/bin/systemctl start postgresql-13 + # /usr/bin/systemctl enable postgresql-13 -The above steps are specific to RHEL/CentOS 7. For RHEL/CentOS 8 you must install Postgres 10 or higher:: - - # yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm - # yum makecache fast - # yum install -y postgresql10-server - # /usr/pgsql-10/bin/postgresql-10-setup initdb - # systemctl start postgresql-10 - # systemctl enable postgresql-10 - -Note that the Dataverse Software installer includes its own Postgres JDBC driver. If you choose to install the newest version of Postgres (12 as of this writing), you may need to grab a current JDBC driver from https://jdbc.postgresql.org/download.html before launching the install script. +For RHEL/CentOS 8 the process would be identical, except for the very first command - you would need to install the "EL-8" yum repository configuration instead. Configuring Database Access for the Dataverse Installation (and the Dataverse Software Installer) ================================================================================================= @@ -152,7 +143,7 @@ Configuring Database Access for the Dataverse Installation (and the Dataverse So - **Important: PostgreSQL must be restarted** for the configuration changes to take effect! On RHEL/CentOS 7 and similar (provided you installed Postgres as instructed above):: - # systemctl restart postgresql-9.6 + # systemctl restart postgresql-13 On MacOS X a "Reload Configuration" icon is usually supplied in the PostgreSQL application folder. Or you could look up the process id of the PostgreSQL postmaster process, and send it the SIGHUP signal:: diff --git a/pom.xml b/pom.xml index de8ff91863a..790e0649d39 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 8.0.0 5.2020.6 - 42.2.18 + 42.2.19 1.11.762 1.2 4.5.5 From f00b6378a1d995c29a3d6d01db1b909fd57644c3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Mar 2021 11:23:46 -0500 Subject: [PATCH 258/274] typo. (#7374) --- doc/release-notes/7374-postgresql-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/7374-postgresql-update b/doc/release-notes/7374-postgresql-update index 332ffda0060..3ac74ad70a1 100644 --- a/doc/release-notes/7374-postgresql-update +++ b/doc/release-notes/7374-postgresql-update @@ -6,4 +6,4 @@ Up until this release our installation guide "strongly recommended" to install P - Install the new version of PostgreSQL; (make sure it's running on the same port, etc. so that no changes are needed in the Payara configuration) - Re-import the database with ``psql``, as the postgres user. -Consult the PostgreSQL upgrde documentation for more information, for example https://www.postgresql.org/docs/13/upgrading.html#UPGRADING-VIA-PGDUMPALL. +Consult the PostgreSQL upgrade documentation for more information, for example https://www.postgresql.org/docs/13/upgrading.html#UPGRADING-VIA-PGDUMPALL. From 0565ff90bb7c3912219caab3b2db98f5effe688e Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Mar 2021 12:49:38 -0500 Subject: [PATCH 259/274] similar changes for the dev. guide. (#7374) --- doc/sphinx-guides/source/developers/dev-environment.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 63f482b86ee..e5cab7df0d1 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -94,9 +94,9 @@ To install Payara, run the following commands: Install PostgreSQL ~~~~~~~~~~~~~~~~~~ -PostgreSQL 9.6 is recommended to match the version in the Installation Guide. +For the past few release cycles much of the development has been done under PostgreSQL 9.6. While that version is known to be very stable, it is nearing its end-of-life (in Nov. 2021). The Dataverse Software has now been tested with versions up to 13 (13.2 is the latest released version as of writing this). -On Mac, go to https://www.postgresql.org/download/macosx/ and choose "Interactive installer by EDB" option. We've tested version 9.6.19. When prompted to set a password for the "database superuser (postgres)" just enter "password". +On Mac, go to https://www.postgresql.org/download/macosx/ and choose "Interactive installer by EDB" option. Note that version 9.6 is used in the command line examples below, but the process will be identical for any version up to 13. When prompted to set a password for the "database superuser (postgres)" just enter "password". After installation is complete, make a backup of the ``pg_hba.conf`` file like this: @@ -112,7 +112,7 @@ In the Finder, click "Applications" then "PostgreSQL 9.6" and launch the "Reload Next, to confirm the edit worked, launch the "pgAdmin" application from the same folder. Under "Browser", expand "Servers" and double click "PostgreSQL 9.6". When you are prompted for a password, leave it blank and click "OK". If you have successfully edited "pg_hba.conf", you can get in without a password. -On Linux, you should just install PostgreSQL from your package manager without worrying about the version as long as it's 9.x. Find ``pg_hba.conf`` and set the authentication method to "trust" and restart PostgreSQL. +On Linux, you should just install PostgreSQL using your favorite package manager, such as ``yum``. (Consult the PostgreSQL section of :doc:`/installation/prerequisites` in the main Installation guide for more info and command line examples). Find ``pg_hba.conf`` and set the authentication method to "trust" and restart PostgreSQL. Install Solr ~~~~~~~~~~~~ From 34453f533e2bc911f0532aa4b9f67e3bb7f6f05f Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 4 Mar 2021 13:12:40 -0500 Subject: [PATCH 260/274] renamed the release note (#7374) --- .../{7374-postgresql-update => 7374-postgresql-update.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/release-notes/{7374-postgresql-update => 7374-postgresql-update.md} (100%) diff --git a/doc/release-notes/7374-postgresql-update b/doc/release-notes/7374-postgresql-update.md similarity index 100% rename from doc/release-notes/7374-postgresql-update rename to doc/release-notes/7374-postgresql-update.md From 4ed6e8d9910c2c7c81a4ba11cd95f4b2b70babf7 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 4 Mar 2021 15:24:11 -0500 Subject: [PATCH 261/274] #7373 add additional upgrade steps --- doc/release-notes/7373-solr-upgrade.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/release-notes/7373-solr-upgrade.md b/doc/release-notes/7373-solr-upgrade.md index c6c084961bf..06e7bc62e5a 100644 --- a/doc/release-notes/7373-solr-upgrade.md +++ b/doc/release-notes/7373-solr-upgrade.md @@ -9,3 +9,14 @@ appear until indexing is complete. See http://guides.dataverse.org/installation/prerequisites.html#installing-solr + +[for the additional upgrade steps section] + +Run the script updateSchemaMDB.sh to generate updated solr schema files and preserve any other custom fields in your Solr configuration. +For example: (modify the path names as needed) +cd /usr/local/solr-8.8.1/server/solr/collection1/conf +wget https://github.com/IQSS/dataverse/releases/download/v5.4/updateSchemaMDB.sh +chmod +x updateSchemaMDB.sh +./updateSchemaMDB.sh -t . + +See http://guides.dataverse.org/en/5.4/admin/metadatacustomization.html?highlight=updateschemamdb for more information. From f28511e7cc3ec989a4942a8e7339b81242cc103a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 5 Mar 2021 11:49:43 -0500 Subject: [PATCH 262/274] changed the path where the solr files live, for the installer build. #7373 --- scripts/installer/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile index 4ef35a01471..180e2cb03d5 100644 --- a/scripts/installer/Makefile +++ b/scripts/installer/Makefile @@ -56,13 +56,13 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} @echo copying jhove schema file /bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} -${SOLR_SCHEMA}: ../../conf/solr/7.7.2/schema.xml ../../conf/solr/7.7.2/schema_dv_mdb_fields.xml ../../conf/solr/7.7.2/schema_dv_mdb_copies.xml ../../conf/solr/7.7.2/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR} +${SOLR_SCHEMA}: ../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/schema_dv_mdb_fields.xml ../../conf/solr/8.8.1/schema_dv_mdb_copies.xml ../../conf/solr/8.8.1/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR} @echo copying Solr schema file - /bin/cp ../../conf/solr/7.7.2/schema*.xml ../../conf/solr/7.7.2/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/8.8.1/schema*.xml ../../conf/solr/8.8.1/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR} -${SOLR_CONFIG}: ../../conf/solr/7.7.2/solrconfig.xml ${INSTALLER_ZIP_DIR} +${SOLR_CONFIG}: ../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR} @echo copying Solr config file - /bin/cp ../../conf/solr/7.7.2/solrconfig.xml ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR} ${PYTHON_FILES}: README_python.txt install.py installConfig.py installAppServer.py installUtils.py requirements.txt default.config interactive.config ${INSTALLER_ZIP_DIR} @echo copying Python installer files From 8093d2f63b58a9804ab6d4fbbb8430cd7b76c478 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Fri, 5 Mar 2021 12:18:05 -0500 Subject: [PATCH 263/274] Fixed truncate and ajax update issues [ref #6685, #7565] --- src/main/webapp/dataset.xhtml | 25 +++++++++++++------ src/main/webapp/dataverse_template.xhtml | 9 +++---- .../resources/js/dv_rebind_bootstrap_ui.js | 3 ++- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 949c97ceb4d..4d43a25ad59 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1653,19 +1653,29 @@ + summaryDescTruncation();     diff --git a/src/main/webapp/dataverse_template.xhtml b/src/main/webapp/dataverse_template.xhtml index 61487c0d1c0..8594cd0b579 100644 --- a/src/main/webapp/dataverse_template.xhtml +++ b/src/main/webapp/dataverse_template.xhtml @@ -79,14 +79,13 @@ + - - - - + + @@ -105,7 +104,7 @@ } ; }); - + // Rebind bootstrap UI components bind_bsui_components(); }); diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index a5a77009782..25a9d05e0fd 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -163,7 +163,8 @@ function sharrre(){ */ function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip){ // SELECTOR ID FROM PARAMETERS - $('#' + truncSelector + ' td div').each(function () { + $('#' + truncSelector + ' td > div:first-child').each(function () { + // add responsive img class to limit width to that of container $(this).find('img').attr('class', 'img-responsive'); From 883141c0aaa8beec91a19ed895dc6709ef68fafd Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 5 Mar 2021 13:21:10 -0500 Subject: [PATCH 264/274] #7373 update solr installation file --- .../source/_static/installation/files/etc/init.d/solr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index d351c544a65..16d364fc9cf 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -5,7 +5,7 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr/solr-7.7.2" +SOLR_DIR="/usr/local/solr/solr-8.8.1" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g -j jetty.host=127.0.0.1" SOLR_USER=solr From 36f0677859aa57a3bff818a3059e9d9da7f6321f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 14:24:21 -0500 Subject: [PATCH 265/274] doc for the new publish parameter and /timestamps call --- doc/sphinx-guides/source/api/native-api.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c02733629f4..741cc5bf3fb 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1020,6 +1020,8 @@ When publishing a dataset it's good to be aware of the Dataverse Software's vers If this is the first version of the dataset, its version number will be set to ``1.0``. Otherwise, the new dataset version number is determined by the most recent version number and the ``type`` parameter. Passing ``type=minor`` increases the minor version number (2.3 is updated to 2.4). Passing ``type=major`` increases the major version number (2.3 is updated to 3.0). (Superusers can pass ``type=updatecurrent`` to update metadata without changing the version number.) +This call also supports an optional boolean query parameter: ``assureIsIndexed``. If true, the call will fail, with a 409 ("CONFLICT") response, if the dataset is awaiting re-indexing. If indexing occurs during publishing, it could cause the publish request to fail, after a 202 response has been received. Using this parameter allows the caller to wait for indexing to occur and avoid this possibility. It is most useful in situations where edits are made immediately before publication. + .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. .. code-block:: bash @@ -1039,7 +1041,7 @@ The fully expanded example above (without environment variables) looks like this The quotes around the URL are required because there is more than one query parameter separated by an ampersand (``&``), which has special meaning to Unix shells such as Bash. Putting the ``&`` in quotes ensures that "type" is interpreted as one of the query parameters. -You should expect JSON output and a 200 ("OK") response in most cases. If you receive a 202 ("ACCEPTED") response, this is normal for installations that have workflows configured. Workflows are described in the :doc:`/developers/workflows` section of the Developer Guide. +You should expect JSON output and a 200 ("OK") response in most cases. If you receive a 202 ("ACCEPTED") response, this is normal for installations that have workflows configured. Workflows are described in the :doc:`/developers/workflows` section of the Developer Guide. A 409 ("CONFLICT") response is also possible if you set ``assureIsIndexed`=true. (In this case, one could then repeat the call until a 200/202 response is sent.) .. note:: POST should be used to publish a dataset. GET is supported for backward compatibility but is deprecated and may be removed: https://github.com/IQSS/dataverse/issues/2431 @@ -1742,6 +1744,15 @@ Configure a Dataset to Use a Specific File Store ``/api/datasets/$dataset-id/storageDriver`` can be used to check, configure or reset the designated file store (storage driver) for a dataset. Please see the :doc:`/admin/dataverses-datasets` section of the guide for more information on this API. +View the Timestamps on a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``/api/datasets/$dataset-id/timestamps`` can be used to view timestamps associated with various events in the dataset's lifecycle. For published datasets, this API call provides the ``createTime``, ``publicationTime``, ``lastMetadataExportTime`` and ``lastMajorVersionReleaseTime``, as well as two booleans - ``hasStaleIndex`` and ``hasStalePermissionIndex`` - which, if true, indicate the Dataverse displays for the dataset are up-to-date. The reponses is ``application/json`` with the timestamps included in the returned ``data`` object. + +When called by a user who can view the draft version of the dataset, additional timestamps are reported: ``lastUpdateTime``, ``lastIndexTime``, ``lastPermissionUpdateTime``, and ``globalIdCreateTime``. + +One use case where this API call could be useful is in allowing an external application to poll and wait for changes being made by the Dataverse software or other external tool to complete prior to continuing its own processing. + Files ----- From c550a949c2ed9b441e244b1d7abee3f9f32ffa5e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Sat, 6 Mar 2021 14:28:44 -0500 Subject: [PATCH 266/274] typos --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 741cc5bf3fb..d50ef05d87e 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1020,7 +1020,7 @@ When publishing a dataset it's good to be aware of the Dataverse Software's vers If this is the first version of the dataset, its version number will be set to ``1.0``. Otherwise, the new dataset version number is determined by the most recent version number and the ``type`` parameter. Passing ``type=minor`` increases the minor version number (2.3 is updated to 2.4). Passing ``type=major`` increases the major version number (2.3 is updated to 3.0). (Superusers can pass ``type=updatecurrent`` to update metadata without changing the version number.) -This call also supports an optional boolean query parameter: ``assureIsIndexed``. If true, the call will fail, with a 409 ("CONFLICT") response, if the dataset is awaiting re-indexing. If indexing occurs during publishing, it could cause the publish request to fail, after a 202 response has been received. Using this parameter allows the caller to wait for indexing to occur and avoid this possibility. It is most useful in situations where edits are made immediately before publication. +This call also supports an optional boolean query parameter: ``assureIsIndexed``. If true, the call will fail with a 409 ("CONFLICT") response if the dataset is awaiting re-indexing. If indexing occurs during publishing it could cause the publish request to fail, after a 202 response has been received. Using this parameter allows the caller to wait for indexing to occur and avoid this possibility. It is most useful in situations where edits are made immediately before publication. .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. @@ -1747,7 +1747,7 @@ Configure a Dataset to Use a Specific File Store View the Timestamps on a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``/api/datasets/$dataset-id/timestamps`` can be used to view timestamps associated with various events in the dataset's lifecycle. For published datasets, this API call provides the ``createTime``, ``publicationTime``, ``lastMetadataExportTime`` and ``lastMajorVersionReleaseTime``, as well as two booleans - ``hasStaleIndex`` and ``hasStalePermissionIndex`` - which, if true, indicate the Dataverse displays for the dataset are up-to-date. The reponses is ``application/json`` with the timestamps included in the returned ``data`` object. +``/api/datasets/$dataset-id/timestamps`` can be used to view timestamps associated with various events in the dataset's lifecycle. For published datasets, this API call provides the ``createTime``, ``publicationTime``, ``lastMetadataExportTime`` and ``lastMajorVersionReleaseTime``, as well as two booleans - ``hasStaleIndex`` and ``hasStalePermissionIndex`` - which, if false, indicate the Dataverse displays for the dataset are up-to-date. The response is ``application/json`` with the timestamps included in the returned ``data`` object. When called by a user who can view the draft version of the dataset, additional timestamps are reported: ``lastUpdateTime``, ``lastIndexTime``, ``lastPermissionUpdateTime``, and ``globalIdCreateTime``. From 810e917d058c4021b96fc2520349969e5798fe7b Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Mar 2021 11:06:37 -0500 Subject: [PATCH 267/274] Add read less btn to dataset desc truncate, plus code clean up [ref #6685] --- src/main/java/propertyFiles/Bundle.properties | 2 ++ src/main/webapp/dataset.xhtml | 10 ++++---- src/main/webapp/resources/css/structure.css | 3 ++- .../resources/js/dv_rebind_bootstrap_ui.js | 24 +++++++++++++------ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b386e8084f4..8855793f7be 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -61,6 +61,8 @@ delete=Delete copyClipboard=Copy to Clipboard truncateMoreBtn=Read full {0} [+] truncateMoreTip=Click to read the full {0}. +truncateLessBtn=Collapse {0} [+] +truncateLessTip=Click to collapse the {0}. yes=Yes no=No previous=Previous diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 4d43a25ad59..fbe9a6c656b 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1663,15 +1663,17 @@ }); function summaryDescTruncation() { // truncate summary description, contentTruncate function in dv_rebind_bootstrap_ui.js - // add summary desc row id to pass as function selector + // add summary desc row id to pass as selector param var descId = 'dsDescription'; - // add metadata dynamic label text to button and tooltip text from bundle + // add metadata dynamic label text, tooltip text from bundle as param var truncMoreBtn = '#{of:format1(bundle.truncateMoreBtn, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; var truncMoreTip = '#{of:format1(bundle.truncateMoreTip, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; + var truncLessBtn = '#{of:format1(bundle.truncateLessBtn, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; + var truncLessTip = '#{of:format1(bundle.truncateLessTip, DatasetPage.datasetVersionUI.description.datasetFieldType.localeTitle)}'; if (!truncactionHappened) { - // pass text params to function - contentTruncate(descId, truncMoreBtn, truncMoreTip); + // pass bundle text variabls as params to function + contentTruncate(descId, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip); truncactionHappened = true; } diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index cd6785679ae..f1cc793df5f 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -653,8 +653,9 @@ div.edit-field div.ui-message {margin:6px 0;} white-space: pre-wrap !important; } /* TRUNCATION */ -div.more-fade-block {text-align:center; padding-top:250px; width:100%; position:absolute; bottom:0; background:linear-gradient(180deg,hsla(0,0%,100%,0),#fff 80%);} +div.more-block {text-align:center; padding-top:250px; width:100%; position:absolute; bottom:0; background:linear-gradient(180deg,hsla(0,0%,100%,0),#fff 80%);} button.desc-more-link {margin:0; padding:0;} +div.less-block {text-align:center;}; /* Primefaces vs Bootstrap selectOneMenu fix */ div.ui-selectonemenu.form-control {padding:0px;} diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index 25a9d05e0fd..fb4e3a5b971 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -161,7 +161,7 @@ function sharrre(){ /* * Truncate dataset description content */ -function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip){ +function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip, truncLessBtn, truncLessTip){ // SELECTOR ID FROM PARAMETERS $('#' + truncSelector + ' td > div:first-child').each(function () { @@ -175,17 +175,27 @@ function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip){ // ADD A MAX-HEIGHT TO CONTAINER $(this).css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); - // BTN LABEL TEXT, DYNAMIC ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS + // BTN LABEL TEXT, ARIA ATTR'S, FROM BUNDLE VIA PARAMETERS var readMoreBtn = ''; - var moreFade = '
    ' + readMoreBtn + '
    '; + var moreBlock = '
    ' + readMoreBtn + '
    '; + var readLessBtn = ''; + var lessBlock = '
    ' + readLessBtn + '
    '; - // add Read More + button background fade - $(this).append(moreFade); + // add "Read full desc [+]" btn, background fade + $(this).append(moreBlock); - // ... add full description to summary block on Read More link click ++ add responsive img class + // show full description in summary block on "Read full desc [+]" btn click $(document).on('click', 'button.desc-more-link', function() { $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'none','overflow-y':'visible','position':'relative'}); - $(this).parent('div.more-fade-block').remove(); + $(this).parent('div.more-block').replaceWith(lessBlock); + $('.less-block button').tooltip(); + }); + + // trucnate description in summary block on "Collapse desc [-]" btn click + $(document).on('click', 'button.desc-less-link', function() { + $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); + $(this).parent('div.less-block').replaceWith(moreBlock); + $('.more-block button').tooltip(); }); } }); From fe6b13a535e49c110ea56bcb6be1df9a6d44c9b4 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Mar 2021 11:25:10 -0500 Subject: [PATCH 268/274] Added animation and scroll target to desc collapse [ref #6685] --- src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js index fb4e3a5b971..9f7e377dea6 100644 --- a/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js +++ b/src/main/webapp/resources/js/dv_rebind_bootstrap_ui.js @@ -195,6 +195,7 @@ function contentTruncate(truncSelector, truncMoreBtn, truncMoreTip, truncLessBtn $(document).on('click', 'button.desc-less-link', function() { $(this).tooltip('hide').parent('div').parent('div').css({'max-height':'250px','overflow-y':'hidden','position':'relative'}); $(this).parent('div.less-block').replaceWith(moreBlock); + $('html, body').animate({scrollTop: $('#' + truncSelector).offset().top - 60}, 500); $('.more-block button').tooltip(); }); } From 7eb7f34bb0ee230eab2f1cabab7b75f64d6ce823 Mon Sep 17 00:00:00 2001 From: Michael Heppler Date: Mon, 8 Mar 2021 15:26:20 -0500 Subject: [PATCH 269/274] Removed unused code from truncation [ref #6685] --- src/main/webapp/dataset.xhtml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index fbe9a6c656b..0198e303b06 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1654,12 +1654,9 @@ - summaryDescTruncation(); From 768ed3c7b5d879f6dc8792bf4d56b887d6a0a1eb Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 9 Mar 2021 16:06:29 -0500 Subject: [PATCH 270/274] flagging release note --- doc/release-notes/820-non-ascii-chars-in-search.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release-notes/820-non-ascii-chars-in-search.md b/doc/release-notes/820-non-ascii-chars-in-search.md index c449a68cb11..9e21dd83694 100644 --- a/doc/release-notes/820-non-ascii-chars-in-search.md +++ b/doc/release-notes/820-non-ascii-chars-in-search.md @@ -1,3 +1,5 @@ +(review these notes if this gets into the same release as #7645 as the steps are included there - we expect to include this in the same release) + ### Search with non-ascii characters Many languages include characters that have close analogs in ascii, e.g. (á, à, â, ç, é, è, ê, ë, í, ó, ö, ú, ù, û, ü…). This release changes the default Solr configuration to allow search to match words based on these associations, e.g. a search for Mercè would match the word Merce in a Dataset, and vice versa. This should generally be helpful, but can result in false positives.,e.g. "canon" will be found searching for "cañon". From c702dfc071e164725f7628a7e065be1351156b94 Mon Sep 17 00:00:00 2001 From: Peter Kiraly Date: Wed, 10 Mar 2021 15:31:13 +0100 Subject: [PATCH 271/274] #7666: use the production citation.tsv in test, remove the copy stored in test resources. --- .../edu/harvard/iq/dataverse/api/AdminIT.java | 3 +- src/test/resources/tsv/citation.tsv | 324 ------------------ 2 files changed, 2 insertions(+), 325 deletions(-) delete mode 100644 src/test/resources/tsv/citation.tsv diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index 84da33cd3ee..f83e9d9c839 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -689,11 +689,12 @@ public void testLoadMetadataBlock_NoErrorPath() { byte[] updatedContent = null; try { - updatedContent = Files.readAllBytes(Paths.get("src/test/resources/tsv/citation.tsv")); + updatedContent = Files.readAllBytes(Paths.get("scripts/api/data/metadatablocks/citation.tsv")); } catch (IOException e) { logger.warning(e.getMessage()); assertEquals(0,1); } + Response response = UtilIT.loadMetadataBlock(apiToken, updatedContent); assertEquals(200, response.getStatusCode()); response.then().assertThat().statusCode(OK.getStatusCode()); diff --git a/src/test/resources/tsv/citation.tsv b/src/test/resources/tsv/citation.tsv deleted file mode 100644 index d63390e9866..00000000000 --- a/src/test/resources/tsv/citation.tsv +++ /dev/null @@ -1,324 +0,0 @@ -#metadataBlock name dataverseAlias displayName blockURI - citation Citation Metadata https://dataverse.org/schema/citation/ -#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI - title Title Full title by which the Dataset is known. Enter title... text 0 TRUE FALSE FALSE FALSE TRUE TRUE citation http://purl.org/dc/terms/title - subtitle Subtitle A secondary title used to amplify or state certain limitations on the main title. text 1 FALSE FALSE FALSE FALSE FALSE FALSE citation - alternativeTitle Alternative Title A title by which the work is commonly referred, or an abbreviation of the title. text 2 FALSE FALSE FALSE FALSE FALSE FALSE citation http://purl.org/dc/terms/alternative - alternativeURL Alternative URL A URL where the dataset can be viewed, such as a personal or project website. Enter full URL, starting with http:// url 3 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE citation https://schema.org/distribution - otherId Other ID Another unique identifier that identifies this Dataset (e.g., producer's or another repository's number). none 4 : FALSE FALSE TRUE FALSE FALSE FALSE citation - otherIdAgency Agency Name of agency which generated this identifier. text 5 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE otherId citation - otherIdValue Identifier Other identifier that corresponds to this Dataset. text 6 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE otherId citation - author Author The person(s), corporate body(ies), or agency(ies) responsible for creating the work. none 7 FALSE FALSE TRUE FALSE TRUE TRUE citation http://purl.org/dc/terms/creator - authorName Name The author's Family Name, Given Name or the name of the organization responsible for this Dataset. FamilyName, GivenName or Organization text 8 #VALUE TRUE FALSE FALSE TRUE TRUE TRUE author citation - authorAffiliation Affiliation The organization with which the author is affiliated. text 9 (#VALUE) TRUE FALSE FALSE TRUE TRUE FALSE author citation - authorIdentifierScheme Identifier Scheme Name of the identifier scheme (ORCID, ISNI). text 10 - #VALUE: FALSE TRUE FALSE FALSE TRUE FALSE author citation http://purl.org/spar/datacite/AgentIdentifierScheme - authorIdentifier Identifier Uniquely identifies an individual author or organization, according to various schemes. text 11 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE author citation http://purl.org/spar/datacite/AgentIdentifier - datasetContact Contact The contact(s) for this Dataset. none 12 FALSE FALSE TRUE FALSE TRUE TRUE citation - datasetContactName Name The contact's Family Name, Given Name or the name of the organization. FamilyName, GivenName or Organization text 13 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE datasetContact citation - datasetContactAffiliation Affiliation The organization with which the contact is affiliated. text 14 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE datasetContact citation - datasetContactEmail E-mail The e-mail address(es) of the contact(s) for the Dataset. This will not be displayed. email 15 #EMAIL FALSE FALSE FALSE FALSE TRUE TRUE datasetContact citation - dsDescription Description A summary describing the purpose, nature, and scope of the Dataset. none 16 FALSE FALSE TRUE FALSE TRUE TRUE citation - dsDescriptionValue Text A summary describing the purpose, nature, and scope of the Dataset. textbox 17 #VALUE TRUE FALSE FALSE FALSE TRUE TRUE dsDescription citation - dsDescriptionDate Date In cases where a Dataset contains more than one description (for example, one might be supplied by the data producer and another prepared by the data repository where the data are deposited), the date attribute is used to distinguish between the two descriptions. The date attribute follows the ISO convention of YYYY-MM-DD. YYYY-MM-DD date 18 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE dsDescription citation - subject Subject Domain-specific Subject Categories that are topically relevant to the Dataset. text 19 TRUE TRUE TRUE TRUE TRUE TRUE citation http://purl.org/dc/terms/subject - keyword Keyword Key terms that describe important aspects of the Dataset. none 20 FALSE FALSE TRUE FALSE TRUE FALSE citation - keywordValue Term Key terms that describe important aspects of the Dataset. Can be used for building keyword indexes and for classification and retrieval purposes. A controlled vocabulary can be employed. The vocab attribute is provided for specification of the controlled vocabulary in use, such as LCSH, MeSH, or others. The vocabURI attribute specifies the location for the full controlled vocabulary. text 21 #VALUE TRUE FALSE FALSE TRUE TRUE FALSE keyword citation - keywordVocabulary Vocabulary For the specification of the keyword controlled vocabulary in use, such as LCSH, MeSH, or others. text 22 (#VALUE) FALSE FALSE FALSE FALSE TRUE FALSE keyword citation - keywordVocabularyURI Vocabulary URL Keyword vocabulary URL points to the web presence that describes the keyword vocabulary, if appropriate. Enter an absolute URL where the keyword vocabulary web site is found, such as http://www.my.org. Enter full URL, starting with http:// url 23 #VALUE FALSE FALSE FALSE FALSE TRUE FALSE keyword citation - topicClassification Topic Classification The classification field indicates the broad important topic(s) and subjects that the data cover. Library of Congress subject terms may be used here. none 24 FALSE FALSE TRUE FALSE FALSE FALSE citation - topicClassValue Term Topic or Subject term that is relevant to this Dataset. text 25 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE topicClassification citation - topicClassVocab Vocabulary Provided for specification of the controlled vocabulary in use, e.g., LCSH, MeSH, etc. text 26 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation - topicClassVocabURI Vocabulary URL Specifies the URL location for the full controlled vocabulary. Enter full URL, starting with http:// url 27 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE topicClassification citation - publication Related Publication Publications that use the data from this Dataset. The full list of Related Publications will be displayed on the metadata tab. none 28 FALSE FALSE TRUE FALSE TRUE FALSE citation http://purl.org/dc/terms/isReferencedBy - publicationCitation Citation The full bibliographic citation for this related publication. textbox 29 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/dc/terms/bibliographicCitation - publicationIDType ID Type The type of digital identifier used for this publication (e.g., Digital Object Identifier (DOI)). text 30 #VALUE: TRUE TRUE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifierScheme - publicationIDNumber ID Number The identifier for the selected ID type. text 31 #VALUE TRUE FALSE FALSE FALSE TRUE FALSE publication citation http://purl.org/spar/datacite/ResourceIdentifier - publicationURL URL Link to the publication web page (e.g., journal article page, archive record page, or other). Enter full URL, starting with http:// url 32 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE publication citation https://schema.org/distribution - notesText Notes Additional important information about the Dataset. textbox 33 FALSE FALSE FALSE FALSE TRUE FALSE citation - language Language Language of the Dataset text 34 TRUE TRUE TRUE TRUE FALSE FALSE citation http://purl.org/dc/terms/language - producer Producer Person or organization with the financial or administrative responsibility over this Dataset none 35 FALSE FALSE TRUE FALSE FALSE FALSE citation - producerName Name Producer name FamilyName, GivenName or Organization text 36 #VALUE TRUE FALSE FALSE TRUE FALSE TRUE producer citation - producerAffiliation Affiliation The organization with which the producer is affiliated. text 37 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerAbbreviation Abbreviation The abbreviation by which the producer is commonly known. (ex. IQSS, ICPSR) text 38 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerURL URL Producer URL points to the producer's web presence, if appropriate. Enter an absolute URL where the producer's web site is found, such as http://www.my.org. Enter full URL, starting with http:// url 39 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE producer citation - producerLogoURL Logo URL URL for the producer's logo, which points to this producer's web-accessible logo image. Enter an absolute URL where the producer's logo image is found, such as http://www.my.org/images/logo.gif. Enter full URL for image, starting with http:// url 40
    FALSE FALSE FALSE FALSE FALSE FALSE producer citation - productionDate Production Date Date when the data collection or other materials were produced (not distributed, published or archived). YYYY-MM-DD date 41 TRUE FALSE FALSE TRUE FALSE FALSE citation - productionPlace Production Place The location where the data collection and any other related materials were produced. text 42 FALSE FALSE FALSE FALSE FALSE FALSE citation - contributor Contributor The organization or person responsible for either collecting, managing, or otherwise contributing in some form to the development of the resource. none 43 : FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/contributor - contributorType Type The type of contributor of the resource. text 44 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE contributor citation - contributorName Name The Family Name, Given Name or organization name of the contributor. FamilyName, GivenName or Organization text 45 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE contributor citation - grantNumber Grant Information Grant Information none 46 : FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/sponsor - grantNumberAgency Grant Agency Grant Number Agency text 47 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - grantNumberValue Grant Number The grant or contract number of the project that sponsored the effort. text 48 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE grantNumber citation - distributor Distributor The organization designated by the author or producer to generate copies of the particular work including any necessary editions or revisions. none 49 FALSE FALSE TRUE FALSE FALSE FALSE citation - distributorName Name Distributor name FamilyName, GivenName or Organization text 50 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE distributor citation - distributorAffiliation Affiliation The organization with which the distributor contact is affiliated. text 51 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorAbbreviation Abbreviation The abbreviation by which this distributor is commonly known (e.g., IQSS, ICPSR). text 52 (#VALUE) FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorURL URL Distributor URL points to the distributor's web presence, if appropriate. Enter an absolute URL where the distributor's web site is found, such as http://www.my.org. Enter full URL, starting with http:// url 53 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributorLogoURL Logo URL URL of the distributor's logo, which points to this distributor's web-accessible logo image. Enter an absolute URL where the distributor's logo image is found, such as http://www.my.org/images/logo.gif. Enter full URL for image, starting with http:// url 54
    FALSE FALSE FALSE FALSE FALSE FALSE distributor citation - distributionDate Distribution Date Date that the work was made available for distribution/presentation. YYYY-MM-DD date 55 TRUE FALSE FALSE TRUE FALSE FALSE citation - depositor Depositor The person (Family Name, Given Name) or the name of the organization that deposited this Dataset to the repository. text 56 FALSE FALSE FALSE FALSE FALSE FALSE citation - dateOfDeposit Deposit Date Date that the Dataset was deposited into the repository. YYYY-MM-DD date 57 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted - timePeriodCovered Time Period Covered Time period to which the data refer. This item reflects the time period covered by the data, not the dates of coding or making documents machine-readable or the dates the data were collected. Also known as span. none 58 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage - timePeriodCoveredStart Start Start date which reflects the time period covered by the data, not the dates of coding or making documents machine-readable or the dates the data were collected. YYYY-MM-DD date 59 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - timePeriodCoveredEnd End End date which reflects the time period covered by the data, not the dates of coding or making documents machine-readable or the dates the data were collected. YYYY-MM-DD date 60 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation - dateOfCollection Date of Collection Contains the date(s) when the data were collected. none 61 ; FALSE FALSE TRUE FALSE FALSE FALSE citation - dateOfCollectionStart Start Date when the data collection started. YYYY-MM-DD date 62 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - dateOfCollectionEnd End Date when the data collection ended. YYYY-MM-DD date 63 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE dateOfCollection citation - kindOfData Kind of Data Type of data included in the file: survey data, census/enumeration data, aggregate data, clinical data, event/transaction data, program source code, machine-readable text, administrative records data, experimental data, psychological test, textual data, coded textual, coded documents, time budget diaries, observation data/ratings, process-produced data, or other. text 64 TRUE FALSE TRUE TRUE FALSE FALSE citation http://rdf-vocabulary.ddialliance.org/discovery#kindOfData - series Series Information about the Dataset series. none 65 : FALSE FALSE FALSE FALSE FALSE FALSE citation - seriesName Name Name of the dataset series to which the Dataset belongs. text 66 #VALUE TRUE FALSE FALSE TRUE FALSE FALSE series citation - seriesInformation Information History of the series and summary of those features that apply to the series as a whole. textbox 67 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE series citation - software Software Information about the software used to generate the Dataset. none 68 , FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasGeneratedBy - softwareName Name Name of software used to generate the Dataset. text 69 #VALUE FALSE TRUE FALSE FALSE FALSE FALSE software citation - softwareVersion Version Version of the software used to generate the Dataset. text 70 #NAME: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE software citation - relatedMaterial Related Material Any material related to this Dataset. textbox 71 FALSE FALSE TRUE FALSE FALSE FALSE citation - relatedDatasets Related Datasets Any Datasets that are related to this Dataset, such as previous research on this subject. textbox 72 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/relation - otherReferences Other References Any references that would serve as background or supporting material to this Dataset. text 73 FALSE FALSE TRUE FALSE FALSE FALSE citation http://purl.org/dc/terms/references - dataSources Data Sources List of books, articles, serials, or machine-readable data files that served as the sources of the data collection. textbox 74 FALSE FALSE TRUE FALSE FALSE FALSE citation https://www.w3.org/TR/prov-o/#wasDerivedFrom - originOfSources Origin of Sources For historical materials, information about the origin of the sources and the rules followed in establishing the sources should be specified. textbox 75 FALSE FALSE FALSE FALSE FALSE FALSE citation - characteristicOfSources Characteristic of Sources Noted Assessment of characteristics and source material. textbox 76 FALSE FALSE FALSE FALSE FALSE FALSE citation - accessToSources Documentation and Access to Sources Level of documentation of the original sources. textbox 77 FALSE FALSE FALSE FALSE FALSE FALSE citation -#controlledVocabulary DatasetField Value identifier displayOrder - subject Agricultural Sciences D01 0 - subject Arts and Humanities D0 1 - subject Astronomy and Astrophysics D1 2 - subject Business and Management D2 3 - subject Chemistry D3 4 - subject Computer and Information Science D7 5 - subject Earth and Environmental Sciences D4 6 - subject Engineering D5 7 - subject Law D8 8 - subject Mathematical Sciences D9 9 - subject Medicine, Health and Life Sciences D6 10 - subject Physics D10 11 - subject Social Sciences D11 12 - subject Other D12 13 - publicationIDType ark 0 - publicationIDType arXiv 1 - publicationIDType bibcode 2 - publicationIDType doi 3 - publicationIDType ean13 4 - publicationIDType eissn 5 - publicationIDType handle 6 - publicationIDType isbn 7 - publicationIDType issn 8 - publicationIDType istc 9 - publicationIDType lissn 10 - publicationIDType lsid 11 - publicationIDType pmid 12 - publicationIDType purl 13 - publicationIDType upc 14 - publicationIDType url 15 - publicationIDType urn 16 - contributorType Data Collector 0 - contributorType Data Curator 1 - contributorType Data Manager 2 - contributorType Editor 3 - contributorType Funder 4 - contributorType Hosting Institution 5 - contributorType Project Leader 6 - contributorType Project Manager 7 - contributorType Project Member 8 - contributorType Related Person 9 - contributorType Researcher 10 - contributorType Research Group 11 - contributorType Rights Holder 12 - contributorType Sponsor 13 - contributorType Supervisor 14 - contributorType Work Package Leader 15 - contributorType Other 16 - authorIdentifierScheme ORCID 0 - authorIdentifierScheme ISNI 1 - authorIdentifierScheme LCNA 2 - authorIdentifierScheme VIAF 3 - authorIdentifierScheme GND 4 - authorIdentifierScheme DAI 5 - authorIdentifierScheme ResearcherID 6 - authorIdentifierScheme ScopusID 7 - language Abkhaz 0 - language Afar 1 - language Afrikaans 2 - language Akan 3 - language Albanian 4 - language Amharic 5 - language Arabic 6 - language Aragonese 7 - language Armenian 8 - language Assamese 9 - language Avaric 10 - language Avestan 11 - language Aymara 12 - language Azerbaijani 13 - language Bambara 14 - language Bashkir 15 - language Basque 16 - language Belarusian 17 - language Bengali, Bangla 18 - language Bihari 19 - language Bislama 20 - language Bosnian 21 - language Breton 22 - language Bulgarian 23 - language Burmese 24 - language Catalan,Valencian 25 - language Chamorro 26 - language Chechen 27 - language Chichewa, Chewa, Nyanja 28 - language Chinese 29 - language Chuvash 30 - language Cornish 31 - language Corsican 32 - language Cree 33 - language Croatian 34 - language Czech 35 - language Danish 36 - language Divehi, Dhivehi, Maldivian 37 - language Dutch 38 - language Dzongkha 39 - language English 40 - language Esperanto 41 - language Estonian 42 - language Ewe 43 - language Faroese 44 - language Fijian 45 - language Finnish 46 - language French 47 - language Fula, Fulah, Pulaar, Pular 48 - language Galician 49 - language Georgian 50 - language German 51 - language Greek (modern) 52 - language Guaraní 53 - language Gujarati 54 - language Haitian, Haitian Creole 55 - language Hausa 56 - language Hebrew (modern) 57 - language Herero 58 - language Hindi 59 - language Hiri Motu 60 - language Hungarian 61 - language Interlingua 62 - language Indonesian 63 - language Interlingue 64 - language Irish 65 - language Igbo 66 - language Inupiaq 67 - language Ido 68 - language Icelandic 69 - language Italian 70 - language Inuktitut 71 - language Japanese 72 - language Javanese 73 - language Kalaallisut, Greenlandic 74 - language Kannada 75 - language Kanuri 76 - language Kashmiri 77 - language Kazakh 78 - language Khmer 79 - language Kikuyu, Gikuyu 80 - language Kinyarwanda 81 - language Kyrgyz 82 - language Komi 83 - language Kongo 84 - language Korean 85 - language Kurdish 86 - language Kwanyama, Kuanyama 87 - language Latin 88 - language Luxembourgish, Letzeburgesch 89 - language Ganda 90 - language Limburgish, Limburgan, Limburger 91 - language Lingala 92 - language Lao 93 - language Lithuanian 94 - language Luba-Katanga 95 - language Latvian 96 - language Manx 97 - language Macedonian 98 - language Malagasy 99 - language Malay 100 - language Malayalam 101 - language Maltese 102 - language Māori 103 - language Marathi (Marāṭhī) 104 - language Marshallese 105 - language Mixtepec Mixtec 106 - language Mongolian 107 - language Nauru 108 - language Navajo, Navaho 109 - language Northern Ndebele 110 - language Nepali 111 - language Ndonga 112 - language Norwegian Bokmål 113 - language Norwegian Nynorsk 114 - language Norwegian 115 - language Nuosu 116 - language Southern Ndebele 117 - language Occitan 118 - language Ojibwe, Ojibwa 119 - language Old Church Slavonic,Church Slavonic,Old Bulgarian 120 - language Oromo 121 - language Oriya 122 - language Ossetian, Ossetic 123 - language Panjabi, Punjabi 124 - language Pāli 125 - language Persian (Farsi) 126 - language Polish 127 - language Pashto, Pushto 128 - language Portuguese 129 - language Quechua 130 - language Romansh 131 - language Kirundi 132 - language Romanian 133 - language Russian 134 - language Sanskrit (Saṁskṛta) 135 - language Sardinian 136 - language Sindhi 137 - language Northern Sami 138 - language Samoan 139 - language Sango 140 - language Serbian 141 - language Scottish Gaelic, Gaelic 142 - language Shona 143 - language Sinhala, Sinhalese 144 - language Slovak 145 - language Slovene 146 - language Somali 147 - language Southern Sotho 148 - language Spanish, Castilian 149 - language Sundanese 150 - language Swahili 151 - language Swati 152 - language Swedish 153 - language Tamil 154 - language Telugu 155 - language Tajik 156 - language Thai 157 - language Tigrinya 158 - language Tibetan Standard, Tibetan, Central 159 - language Turkmen 160 - language Tagalog 161 - language Tswana 162 - language Tonga (Tonga Islands) 163 - language Turkish 164 - language Tsonga 165 - language Tatar 166 - language Twi 167 - language Tahitian 168 - language Uyghur, Uighur 169 - language Ukrainian 170 - language Urdu 171 - language Uzbek 172 - language Venda 173 - language Vietnamese 174 - language Volapük 175 - language Walloon 176 - language Welsh 177 - language Wolof 178 - language Western Frisian 179 - language Xhosa 180 - language Yiddish 181 - language Yoruba 182 - language Zhuang, Chuang 183 - language Zulu 184 - language Not applicable 185 From a3bc5ee71e1f9761c392eb7cd34274ef209dcc07 Mon Sep 17 00:00:00 2001 From: Peter Kiraly Date: Wed, 10 Mar 2021 17:40:50 +0100 Subject: [PATCH 272/274] #7672: changing outdated reference from readme.txt to readme.md in testing guide and fixing an unclosed quote issue in native API guide. --- doc/sphinx-guides/source/api/native-api.rst | 2 +- doc/sphinx-guides/source/developers/testing.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d50ef05d87e..94cb6c720c6 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1041,7 +1041,7 @@ The fully expanded example above (without environment variables) looks like this The quotes around the URL are required because there is more than one query parameter separated by an ampersand (``&``), which has special meaning to Unix shells such as Bash. Putting the ``&`` in quotes ensures that "type" is interpreted as one of the query parameters. -You should expect JSON output and a 200 ("OK") response in most cases. If you receive a 202 ("ACCEPTED") response, this is normal for installations that have workflows configured. Workflows are described in the :doc:`/developers/workflows` section of the Developer Guide. A 409 ("CONFLICT") response is also possible if you set ``assureIsIndexed`=true. (In this case, one could then repeat the call until a 200/202 response is sent.) +You should expect JSON output and a 200 ("OK") response in most cases. If you receive a 202 ("ACCEPTED") response, this is normal for installations that have workflows configured. Workflows are described in the :doc:`/developers/workflows` section of the Developer Guide. A 409 ("CONFLICT") response is also possible if you set ``assureIsIndexed=true``. (In this case, one could then repeat the call until a 200/202 response is sent.) .. note:: POST should be used to publish a dataset. GET is supported for backward compatibility but is deprecated and may be removed: https://github.com/IQSS/dataverse/issues/2431 diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 7f59d31410f..79850357edf 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -166,7 +166,7 @@ Near the beginning and at the end of the ec2-create-instance.sh output you will Running the full API test suite using Docker ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To run the full suite of integration tests on your laptop, we recommend using the "all in one" Docker configuration described in ``conf/docker-aio/readme.txt`` in the root of the repo. +To run the full suite of integration tests on your laptop, we recommend using the "all in one" Docker configuration described in ``conf/docker-aio/readme.md`` in the root of the repo. Alternatively, you can run tests against the app server running on your laptop by following the "getting set up" steps below. @@ -446,7 +446,7 @@ How to Run the Phoenix Tests List of Tests Run Against the Phoenix Server ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -We haven't thought much about a good way to publicly list the "IT" classes that are executed against the phoenix server. (Currently your best bet is to look at the ``Executing Maven`` line at the top of the "Full Log" of "Console Output" of ``phoenix.dataverse.org-apitest-develop`` Jenkins job mentioned above.) We endeavor to keep the list of tests in the "all-in-one" Docker environment described above in sync with the list of tests configured in Jenkins. That is to say, refer to :download:`run-test-suite.sh <../../../../conf/docker-aio/run-test-suite.sh>` mentioned in ``conf/docker-aio/readme.txt`` for the current list of IT tests that are expected to pass. Here's a dump of that file: +We haven't thought much about a good way to publicly list the "IT" classes that are executed against the phoenix server. (Currently your best bet is to look at the ``Executing Maven`` line at the top of the "Full Log" of "Console Output" of ``phoenix.dataverse.org-apitest-develop`` Jenkins job mentioned above.) We endeavor to keep the list of tests in the "all-in-one" Docker environment described above in sync with the list of tests configured in Jenkins. That is to say, refer to :download:`run-test-suite.sh <../../../../conf/docker-aio/run-test-suite.sh>` mentioned in ``conf/docker-aio/readme.md`` for the current list of IT tests that are expected to pass. Here's a dump of that file: .. literalinclude:: ../../../../conf/docker-aio/run-test-suite.sh From 25ec376ae7110cd07222d21bfd4eb77ed7f1a3e6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 15 Mar 2021 13:06:59 -0400 Subject: [PATCH 273/274] bug fixes, per update --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 373454bf694..a308e6637f3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -61,7 +61,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // final DatasetVersion editVersion = getDataset().getEditVersion(); tidyUpFields(updateVersion); - // Merge the new version into out JPA context + // Merge the new version into our JPA context ctxt.em().merge(updateVersion); @@ -71,7 +71,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException { updateVersion.setTermsOfUseAndAccess(newTerms); //Put old terms on version that will be deleted.... getDataset().getEditVersion().setTermsOfUseAndAccess(oldTerms); - + //Also set the fileaccessrequest boolean on the dataset to match the new terms + getDataset().setFileAccessRequest(updateVersion.getTermsOfUseAndAccess().isFileAccessRequest()); List newComments = getDataset().getEditVersion().getWorkflowComments(); if (newComments!=null && newComments.size() >0) { for(WorkflowComment wfc: newComments) { @@ -162,7 +163,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // Update modification time on the published version and the dataset updateVersion.setLastUpdateTime(getTimestamp()); tempDataset.setModificationTime(getTimestamp()); - + ctxt.em().merge(updateVersion); Dataset savedDataset = ctxt.em().merge(tempDataset); // Flush before calling DeleteDatasetVersion which calls From 99a92cb9156e0a28277b15771c5211c6be6f3a06 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 15 Mar 2021 13:07:44 -0400 Subject: [PATCH 274/274] Revert "bug fixes, per update" This reverts commit 25ec376ae7110cd07222d21bfd4eb77ed7f1a3e6. --- .../command/impl/CuratePublishedDatasetVersionCommand.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index a308e6637f3..373454bf694 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -61,7 +61,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // final DatasetVersion editVersion = getDataset().getEditVersion(); tidyUpFields(updateVersion); - // Merge the new version into our JPA context + // Merge the new version into out JPA context ctxt.em().merge(updateVersion); @@ -71,8 +71,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { updateVersion.setTermsOfUseAndAccess(newTerms); //Put old terms on version that will be deleted.... getDataset().getEditVersion().setTermsOfUseAndAccess(oldTerms); - //Also set the fileaccessrequest boolean on the dataset to match the new terms - getDataset().setFileAccessRequest(updateVersion.getTermsOfUseAndAccess().isFileAccessRequest()); + List newComments = getDataset().getEditVersion().getWorkflowComments(); if (newComments!=null && newComments.size() >0) { for(WorkflowComment wfc: newComments) { @@ -163,7 +162,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // Update modification time on the published version and the dataset updateVersion.setLastUpdateTime(getTimestamp()); tempDataset.setModificationTime(getTimestamp()); - ctxt.em().merge(updateVersion); + Dataset savedDataset = ctxt.em().merge(tempDataset); // Flush before calling DeleteDatasetVersion which calls