From 385174065ed4b095bbcc6ab36873c9d036bf56b7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 31 Jan 2019 18:31:18 -0500 Subject: [PATCH 001/427] IQSS-5505 --- pom.xml | 5 +++ .../AbstractGlobalIdServiceBean.java | 4 +++ .../dataverse/DOIDataCiteRegisterService.java | 35 +++++++++++++++++++ .../iq/dataverse/GlobalIdServiceBean.java | 2 ++ .../harvard/iq/dataverse/api/Datasets.java | 12 +++++-- .../UpdateDvObjectPIDMetadataCommand.java | 4 +-- 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 380fbbbe402..224ab957fd7 100644 --- a/pom.xml +++ b/pom.xml @@ -585,6 +585,11 @@ tika-parsers 1.19 + + org.xmlunit + xmlunit-core + 2.6.2 + diff --git a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java index 98310a136b5..4e632dce27f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AbstractGlobalIdServiceBean.java @@ -428,5 +428,9 @@ public String getMetadataFromDvObject(String identifier, Map met logger.log(Level.FINE, "XML to send to DataCite: {0}", xmlMetadata); return xmlMetadata; } + + public boolean updateIdentifier(DvObject dvObject) { + return publicizeIdentifier(dvObject); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index 50f92f81fb5..99cf4f3694d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -23,10 +23,17 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.persistence.TypedQuery; +import javax.xml.transform.Source; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.xmlunit.builder.DiffBuilder; +import org.xmlunit.builder.Input; +import org.xmlunit.builder.Input.Builder; +import org.xmlunit.diff.Diff; +import org.xmlunit.diff.Difference; /** * @@ -114,6 +121,34 @@ public String registerIdentifier(String identifier, Map metadata } return retString; } + + public String reRegisterIdentifier(String identifier, Map metadata, DvObject dvObject) throws IOException { + String retString = ""; + String numericIdentifier = identifier.substring(identifier.indexOf(":") + 1); + String xmlMetadata = getMetadataFromDvObject(identifier, metadata, dvObject); + String target = metadata.get("_target"); + DataCiteRESTfullClient client = getClient(); + String currentMetadata = client.getMetadata(numericIdentifier); + Diff myDiff = DiffBuilder.compare(xmlMetadata) + .withTest(currentMetadata).ignoreWhitespace().checkForSimilar() + .build(); + + if (myDiff.hasDifferences()) { + for(Difference d : myDiff.getDifferences()) { + + logger.fine(d.toString()); + } + retString = "metadata:\\r" + client.postMetadata(xmlMetadata) + "\\r"; + } + if (!target.equals(client.getUrl(numericIdentifier))) { + logger.fine("updating target URl to " + target); + client.postUrl(numericIdentifier, target); + retString = retString + "url:\\r" + target; + + } + + return retString; + } public String deactivateIdentifier(String identifier, HashMap metadata, DvObject dvObject) { String retString = ""; diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java index 0d64c1050b8..cd0c2c04c73 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/GlobalIdServiceBean.java @@ -50,6 +50,8 @@ public interface GlobalIdServiceBean { boolean publicizeIdentifier(DvObject studyIn); + boolean updateIdentifier(DvObject dvObject); + static GlobalIdServiceBean getBean(String protocol, CommandContext ctxt) { final Function protocolHandler = BeanDispatcher.DISPATCHER.get(protocol); if ( protocolHandler != null ) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4f868d90ae7..42f01f67ace 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -386,8 +386,9 @@ public Response updateDatasetPIDMetadata(@PathParam("id") String id) { } return response(req -> { - execCommand(new UpdateDvObjectPIDMetadataCommand(findDatasetOrDie(id), req)); - List args = Arrays.asList(id); + Dataset dataset = findDatasetOrDie(id); + execCommand(new UpdateDvObjectPIDMetadataCommand(dataset, req)); + List args = Arrays.asList(dataset.getIdentifier()); return ok(BundleUtil.getStringFromBundle("datasets.api.updatePIDMetadata.success.for.single.dataset", args)); }); } @@ -398,7 +399,14 @@ public Response updateDatasetPIDMetadataAll() { return response( req -> { datasetService.findAll().forEach( ds -> { try { + logger.fine("ReRegistering: " + ds.getId() + " : " + ds.getIdentifier()); + if (!ds.isReleased() || (!ds.isIdentifierRegistered() || (ds.getIdentifier() == null))) { + if (ds.isReleased()) { + logger.warning("Dataset id=" + ds.getId() + " is in an inconsistent state (publicationdate but no identifier/identifier not registered"); + } + } else { execCommand(new UpdateDvObjectPIDMetadataCommand(findDatasetOrDie(ds.getId().toString()), req)); + } } catch (WrappedResponse ex) { Logger.getLogger(Datasets.class.getName()).log(Level.SEVERE, null, ex); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java index e36fe06b863..99d0a183c9d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDvObjectPIDMetadataCommand.java @@ -47,13 +47,13 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { } GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(target.getProtocol(), ctxt); try { - Boolean doiRetString = idServiceBean.publicizeIdentifier(target); + Boolean doiRetString = idServiceBean.updateIdentifier(target); if (doiRetString) { target.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); ctxt.em().merge(target); ctxt.em().flush(); for (DataFile df : target.getFiles()) { - doiRetString = idServiceBean.publicizeIdentifier(df); + doiRetString = idServiceBean.updateIdentifier(df); if (doiRetString) { df.setGlobalIdCreateTime(new Timestamp(new Date().getTime())); ctxt.em().merge(df); From 8195b1ed98e4312b93e28c7577c22b61aac13991 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 11 Mar 2019 13:55:17 -0400 Subject: [PATCH 002/427] use dataset thumbnail if available --- src/main/webapp/dataset.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 0cb0f9d80e4..133bb263b7f 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -43,7 +43,7 @@ - + From d5dec010424435bdcea65c76dd2b1e524473d8a9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 22 Jun 2020 17:56:28 -0400 Subject: [PATCH 003/427] xml escape dataset description --- .../edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java index f9107d8565b..fa04b710819 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java +++ b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteRegisterService.java @@ -228,7 +228,8 @@ public static String getMetadataFromDvObject(String identifier, Map from HTML, it leaves '&' (at least so we need to xml escape as well + String description = StringEscapeUtils.escapeXml(dataset.getLatestVersion().getDescriptionPlainText()); if (description.isEmpty() || description.equals(DatasetField.NA_VALUE)) { description = AbstractGlobalIdServiceBean.UNAVAILABLE; } From 694e510301286b3a30fa9234288c7c6795a013a4 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 26 Sep 2022 18:03:37 +0200 Subject: [PATCH 004/427] allow slash in check permissions api request --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index ef08444af69..5e80f88d6eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -58,6 +58,7 @@ import java.io.InputStream; import java.io.StringReader; +import java.net.URLDecoder; import java.util.Map; import java.util.Map.Entry; import java.util.logging.Level; @@ -1317,7 +1318,7 @@ public Response convertUserFromBcryptToSha1(String json) { } - @Path("permissions/{dvo}") + @Path("permissions/{dvo:.+}") @GET public Response findPermissonsOn(@PathParam("dvo") String dvo) { try { From d57e749ed7ba945175f0dab263d8b469f8a5586f Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 27 Sep 2022 10:07:07 +0200 Subject: [PATCH 005/427] added findDatasetOrDie lookup method in find permissions api --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 5e80f88d6eb..137403e0e89 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1318,13 +1318,16 @@ public Response convertUserFromBcryptToSha1(String json) { } - @Path("permissions/{dvo:.+}") + @Path("permissions/{dvo}") @GET public Response findPermissonsOn(@PathParam("dvo") String dvo) { try { DvObject dvObj = findDvo(dvo); if (dvObj == null) { - return notFound("DvObject " + dvo + " not found"); + dvObj = findDatasetOrDie(dvo); + if (dvObj == null) { + return notFound("DvObject " + dvo + " not found"); + } } try { User aUser = findUserOrDie(); From 850f4c67c8937166beacca96d8eb5644de14a01f Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 28 Sep 2022 11:10:23 +0200 Subject: [PATCH 006/427] added a note on listing permissions on a dataset using a persistentId --- doc/sphinx-guides/source/api/native-api.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 93e1c36f179..be9e625fa7c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3808,6 +3808,17 @@ List permissions a user (based on API Token used) has on a Dataverse collection The ``$identifier`` can be a Dataverse collection alias or database id or a dataset persistent ID or database id. +.. note:: Datasets can be selected using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the dataset is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. + +Example: List permissions a user (based on API Token used) has on a dataset whose DOI is *10.5072/FK2/J8SJZB*: + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB + + curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/admin/permissions/:persistentId?persistentId=$PERSISTENT_IDENTIFIER + Show Role Assignee ~~~~~~~~~~~~~~~~~~ From 53535b4c3830c1cecd3def374c3d8d353e9a46f0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 28 Feb 2023 08:58:14 +0100 Subject: [PATCH 007/427] Add JProfile acknowledgement for free license --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index d40e5f228f7..36fa2de67bf 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ We love contributors! Please see our [Contributing Guide][] for ways you can hel Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. +We thank EJ Technologies for granting us a free open source project license for their Java profiler [JProfiler][]. + [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org) [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/) @@ -38,3 +40,4 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi [chat.dataverse.org]: http://chat.dataverse.org [Dataverse Community Meeting]: https://dataverse.org/events [open source]: LICENSE.md +[JProfiler]: https://www.ej-technologies.com/products/jprofiler/overview.html From 3a337e9405dee42ece3868c75930f695fb0125fe Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 28 Feb 2023 09:04:58 +0100 Subject: [PATCH 008/427] Add JProfiler acknowledgement to dev guide intro --- doc/sphinx-guides/source/developers/intro.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index 7f4e8c1ba34..4eed89a28ef 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -32,6 +32,8 @@ We make use of a variety of Jakarta EE technologies such as JPA, JAX-RS, JMS, an In addition, we start to adopt parts of Eclipse MicroProfile, namely `MicroProfile Config `_. +We thank EJ Technologies for granting us a free open source project license for their Java profiler `JProfiler `_. + Roadmap ------- From fc4cd59dfc1aecc51a319a9034360aa7d7b05166 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 28 Feb 2023 09:12:18 +0100 Subject: [PATCH 009/427] Add JProfiler and acknowledgement to tools page of dev guide --- doc/sphinx-guides/source/developers/tools.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst index cbd27d6e8d2..4ef52f25f43 100755 --- a/doc/sphinx-guides/source/developers/tools.rst +++ b/doc/sphinx-guides/source/developers/tools.rst @@ -147,6 +147,14 @@ For example... would be consistent with a file descriptor leak on the dataset page. +JProfiler ++++++++++ + +Tracking down resource drainage, bottlenecks etc gets easier using a profiler. + +We thank EJ Technologies for granting us a free open source project license for their Java profiler +`JProfiler `_. + jmap and jstat ++++++++++++++ From 305739fb73360118997d0514325e109d10c4db53 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 1 Mar 2023 14:30:14 +0100 Subject: [PATCH 010/427] Revert "Add JProfile acknowledgement for free license" This reverts commit 53535b4c3830c1cecd3def374c3d8d353e9a46f0. --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 36fa2de67bf..d40e5f228f7 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,6 @@ We love contributors! Please see our [Contributing Guide][] for ways you can hel Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. -We thank EJ Technologies for granting us a free open source project license for their Java profiler [JProfiler][]. - [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org) [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/) @@ -40,4 +38,3 @@ We thank EJ Technologies for granting us a free open source project license for [chat.dataverse.org]: http://chat.dataverse.org [Dataverse Community Meeting]: https://dataverse.org/events [open source]: LICENSE.md -[JProfiler]: https://www.ej-technologies.com/products/jprofiler/overview.html From e4d12a14ef631beefdc2d7287f8b9ef49227e9df Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 1 Mar 2023 14:30:17 +0100 Subject: [PATCH 011/427] Revert "Add JProfiler acknowledgement to dev guide intro" This reverts commit 3a337e9405dee42ece3868c75930f695fb0125fe. --- doc/sphinx-guides/source/developers/intro.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index 4eed89a28ef..7f4e8c1ba34 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -32,8 +32,6 @@ We make use of a variety of Jakarta EE technologies such as JPA, JAX-RS, JMS, an In addition, we start to adopt parts of Eclipse MicroProfile, namely `MicroProfile Config `_. -We thank EJ Technologies for granting us a free open source project license for their Java profiler `JProfiler `_. - Roadmap ------- From 72fca2c7e30cce95c8c4446399d32c6a418630f0 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 27 Mar 2023 13:26:21 -0400 Subject: [PATCH 012/427] Pdf export function --- pom.xml | 6 +++ .../dataverse/export/ddi/DdiExportUtil.java | 46 ++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e028771d24b..780bb7bdc0c 100644 --- a/pom.xml +++ b/pom.xml @@ -429,6 +429,12 @@ commons-compress + + + org.apache.xmlgraphics + fop + 2.8 + org.duracloud common diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index c78bb02d5c5..ee9231a9d53 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -74,7 +74,19 @@ import javax.xml.transform.stream.StreamResult; import java.io.File; import java.io.InputStream; -import java.io.InputStreamReader; + +import java.io.OutputStream; +import javax.xml.transform.Result; +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.sax.SAXResult; +import javax.xml.transform.stream.StreamSource; + +import org.apache.fop.apps.FOUserAgent; +import org.apache.fop.apps.Fop; +import org.apache.fop.apps.FopFactory; +import org.apache.fop.apps.MimeConstants; public class DdiExportUtil { @@ -1948,6 +1960,38 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa return true; } + public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { + try { + File xsltfile = new File("ddi-to-fo.xsl"); + + final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI)); + FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); + + try { + Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); + // Setup XSLT + TransformerFactory factory = TransformerFactory.newInstance(); + Transformer transformer = factory.newTransformer(new StreamSource(xsltfile)); + + // Set the value of a in the stylesheet + transformer.setParameter("versionParam", "2.0"); + + // Setup input for XSLT transformation + Source src = new StreamSource(datafile); + + // Resulting SAX events (the generated FO) must be piped through to FOP + Result res = new SAXResult(fop.getDefaultHandler()); + + // Start XSLT transformation and FOP processing + transformer.transform(src, res); + } finally { + outputStream.close(); + } + } catch (Exception e) { + logger.severe(e.getMessage()); + } + } + public static void datasetHtmlDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); From a5821b9fd0d53137918c329fb79590bc6431dbe1 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 3 Apr 2023 17:35:28 -0400 Subject: [PATCH 013/427] pdf codebook --- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../dataverse/export/PdfCodeBookExporter.java | 91 + .../dataverse/export/ddi/DdiExportUtil.java | 7 +- src/main/java/propertyFiles/Bundle.properties | 1 + .../from-ddi-2.5/ddi-pdf/i18n.inc.xslt | 5 + .../ddi-pdf/messages_en.properties.xml | 174 + .../ddi-pdf/messages_es.properties.xml | 170 + .../ddi-pdf/messages_fr.properties.xml | 173 + .../ddi-pdf/messages_ja.properties.xml | 161 + .../ddi-pdf/messages_nn.properties.xml | 174 + .../ddi-pdf/messages_no.properties.xml | 174 + .../ddi-pdf/messages_ru.properties.xml | 169 + .../iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl | 4473 +++++++++++++++++ 13 files changed, 5771 insertions(+), 3 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index d40bc153141..3b3326611dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -272,7 +272,7 @@ public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id" @GET @Path("/export") - @Produces({"application/xml", "application/json", "application/html" }) + @Produces({"application/xml", "application/json", "application/html", "*/*" }) public Response exportDataset(@QueryParam("persistentId") String persistentId, @QueryParam("exporter") String exporter, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java new file mode 100644 index 00000000000..8f3bb78f8d3 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -0,0 +1,91 @@ +package edu.harvard.iq.dataverse.export; + +import com.google.auto.service.AutoService; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; +import edu.harvard.iq.dataverse.export.spi.Exporter; +import edu.harvard.iq.dataverse.util.BundleUtil; + +import javax.json.JsonObject; +import javax.ws.rs.core.MediaType; +import javax.xml.stream.XMLStreamException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; + +@AutoService(Exporter.class) +public class PdfCodeBookExporter implements Exporter { + + @Override + public String getProviderName() { + return "pdf"; + } + + @Override + public String getDisplayName() { + return BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") != null ? BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") : "DDI PDF Codebook"; + } + + @Override + public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { + try { + InputStream ddiInputStream; + try { + ddiInputStream = ExportService.getInstance().getExport(version.getDataset(), "ddi"); + } catch(ExportException | IOException e) { + throw new ExportException ("Cannot open export_ddi cached file"); + } + DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); + } catch (XMLStreamException xse) { + throw new ExportException ("Caught XMLStreamException performing DDI export"); + } + } + + @Override + public Boolean isXMLFormat() { + return false; + } + + @Override + public Boolean isHarvestable() { + // No, we don't want this format to be harvested! + // For datasets with tabular data the portions of the DDIs + // become huge and expensive to parse; even as they don't contain any + // metadata useful to remote harvesters. -- L.A. 4.5 + return false; + } + + @Override + public Boolean isAvailableToUsers() { + return true; + } + + @Override + public String getXMLNameSpace() throws ExportException { + return null; + } + + @Override + public String getXMLSchemaLocation() throws ExportException { + return null; + } + + @Override + public String getXMLSchemaVersion() throws ExportException { + return null; + } + + @Override + public void setParam(String name, Object value) { + // this exporter does not uses or supports any parameters as of now. + } + + @Override + public String getMediaType() { + return MediaType.WILDCARD; + }; +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index ee9231a9d53..3912b28a886 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -1962,9 +1962,12 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { - File xsltfile = new File("ddi-to-fo.xsl"); + File xsltfile = new File("/home/victoria/ddi-to-fo.xsl"); + logger.info("start datasetPdfDDI"); + //InputStream xsltfile = DdiExportUtil.class.getClassLoader().getResourceAsStream( + // "edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl"); - final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI)); + final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); try { diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 45807dc7cde..3ba8d3fc8dd 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1395,6 +1395,7 @@ dataset.exportBtn.itemLabel.json=JSON dataset.exportBtn.itemLabel.oai_ore=OAI_ORE dataset.exportBtn.itemLabel.dataciteOpenAIRE=OpenAIRE dataset.exportBtn.itemLabel.html=DDI HTML Codebook +dataset.exportBtn.itemLabel.pdf=DDI PDF Codebook license.custom=Custom Dataset Terms license.custom.description=Custom terms specific to this dataset metrics.title=Metrics diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt new file mode 100644 index 00000000000..edf876f3b04 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt @@ -0,0 +1,5 @@ + + + + + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml new file mode 100644 index 00000000000..d8e98dfd3c6 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Valid +Frequency table not shown +Derivation +discrete +Data Collection Mode +Other Processing +Other Acknowledgment(s) +Untitled +Identification +Click here to access/export data files from Nesstar format +Value +Percentage +Weighting +Primary Investigator(s) +This document was generated using the +Sampling +Cases +Access Conditions +Source +Modalities +Rights & Disclaimer +Definition +Estimates of Sampling Error +Data Files +Geographic Coverage +April +Mean +Metadata Production +Data Collection +Missing Data +Scripts and programs +Variable(s) +Interviewer instructions +Funding Agency/ies +November +Missing +Version +Universe +Contributor(s) +Access Authority +Data Processing & Appraisal +Scope +Administrative documents +StdDev +Contact(s) +Label +Technical documents +Decimal(s) +Type +Literal question +Concepts +Range +Abstract +June +Supervision +Other Forms of Data Appraisal +References +Accessibility +Data Collection Dates +Data Editing +Questionnaires +Valid case(s) +Reports and analytical documents +Copyright +Documentation +Deviations from Sample Design +Publisher(s) +February +Dataset contains +Acknowledgment(s) +Continuous +Standard deviation +Variables Description +Producer +Production Date + +The Explorer allows you to view data files and export them to common statistical formats +Discrete +Group +July +Filename +Cases +Name +Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. +Statistical tables +December +Subjects +Processing Checks +software +Interviewer's instructions +Table of Contents +Document Information +Subgroup(s) +Keywords +group(s) +W +Weight +Files Description +Notes +Data Collection Notes +file(s) +continuous +Disclaimer +Content +variable(s) +Other Producer(s) +Producers & Sponsors +Data Cleaning Notes +Distributor(s) +Overview +Citation Requirements +September +Category +Confidentiality +Statistics +May +Undetermined +Structure +file +Pre-question +Response Rate +Width +Recoding and Derivation +Series +October +Unit of Analysis +Data Processing Notes +Kind of Data +File +Time Period(s) +File Content +Invalid +Vars +cont. +Key(s) +Question +Source of information +Imputation +Security +To open this file, you will need the free +Other resources +Data Dictionnary +Information +January +Other documents +Minimum +Scope & Coverage +Metadata Producer(s) +Show more info +Data Collector(s) +Post-question +Topics +Sampling Procedure +File Structure +Variables List +Format +Sampling Notes +Variables Group(s) +Description +Categories +Maximum +Depositor(s) +August +NW +Cover Page +Weighted +March + total - showing a subset of +Countries +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml new file mode 100644 index 00000000000..9cfcdaf6e7e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml @@ -0,0 +1,170 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Válido +No se presentan las tablas de frecuencias +Derivación +discreta +Método de Recolección +Otros relacionados al procesamiento +Otros Reconocimientos +Sin título +Identificación +Presione aquí para acceder/exportar al archivo(s) de datos +Valor +Porcentaje +Ponderando +Investigadores Principales +Este documento fue producido utilizando el +Muestreo +Casos +Condiciones de uso +Fuente +Modalidades +Derechos y Notas Legales +Definición +Estimaciones del Error Muestral +Archivo de Datos +Cobertura Geográfica +Abril +Media +Producción de los Metadatos +Recolección de Datos +Datos perdidos +Programas informáticos +Variable(s) +Manual del encuestador +Agencias Auspiciadoras +Noviembre +Valores perdidos +Versión +Universo +Contribuidor(es) +Institución Propietaria +Tratamiento y Validación de Datos +Dominio Temático +Documentos Administrativos +Desviación +Contacto(s) +Etiqueta +Documentos Técnicos +Decimal(es) +Tipo +Pregunta textual +Conceptos +Rango +Resumen +Junio +Supervisión +Otras Formas de Validación de los Datos +Referencias +Accesibilidad +Fechas de Recolección de Datos +Procesamiento de Datos +Cuestionarios +Casos válidos +Reportes y documentos analíticos +Derechos de Autor +Documentación +Modificaciones al Diseño Muestral +Editor(es) +Febrero +Contenido de la Base de Datos +Reconocimiento(s) +Contínua +Desviación estándar +Descripción de la variable +Productor +Fecha de Producción +El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos +Discreta +Grupo +Julio +Nombre del Archivo +Casos +Nombre +Cuadros estadísticos +Diciembre +Temas +Controles de Tratamiento +software +Manual del encuestador +Indice +Información acerca de la Documentación +Subgrupo(s) +Palabra Clave +grupo(s) +P +Ponderador +Descripción de los Archivos +Notas +Notas sobre la Recolección de Datos +archivo(s) +continua +Nota Legal +Contenido +variable(s) +Otros Productores +Productores y Auspiciadores +Notas acerca de la Depuración de los Datos +Distribuidor(es) +Resumen General +Forma de citar +Septiembre +Categoría +Confidencialidad +Estadística + +Mayo +Indeterminado +Estructura +archivo +Pre-pregunta +Tasa de Respuesta +Ancho +Recodificación y Derivación +Series +Octubre +Unidad de Análisis +Notas sobre el Procesamiento de Datos +Tipo de Datos +Archivo +Periodo de Referencia +Contenido del Archivo +Inválido +Vars. +cont. +Clave(s) +Pregunta +Fuente de información +Imputación +Seguridad +Para abrir este archivo se necesita el software gratuito +Otros recursos +Diccionario de Datos +Información +Enero +Otros documentos +Mínimo +Cobertura y Dominio Temático +Productor de los Metadatos +Mostrar más información +Entrevistador(es) +Pos-pregunta +Temas +Procedimiento de Muestreo +Estructura del Archivo +Lista de variables +Formato +Notas sobre el Muestreo +Grupo(s) de Variables +Descripción +Categorías +Máximo +Depositante(s) +Agosto +NP +Carátula +Ponderado +Marzo + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml new file mode 100644 index 00000000000..9fa4d2178b1 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml @@ -0,0 +1,173 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Valide +Tableau de fréquences non-affiché +Mode de calcul +discrète +Méthode de collecte +Autre traitement +Autre(s) remerciement(s) +Sans titre +Identification +Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar +Valeur +Pourcentage +Pondération +Enquêteur(s) principal/aux +Ce document a été généré à l'aide du +Echantillonage +Enregistrements +Conditions d'accès +Source +Modalités +Responsabilité et droits d'auteurs +Définition +Estimation des erreurs d'échantillonage +Fichiers de données +Couverture géographique +Avril +Moyenne +Production des méta-données +Collecte des données +Valeures manquantes +Programmes informatiques +Variable(s) +Instructions aux enquêteurs +Financement +Novembre +Manquant +Version +Univers +Contributeur(s) +Agence propriétaire +Traitement et évaluation des données +Domaines thématiques +Documents administratifs +Ecart type +Contact(s) +Libellé +Documents techniques +Décimale(s) +Type +Formulation de la question +Concepts +Gamme +Résumé +Juin +Supervision +Autres formes d'évaluation des données +Références +Accessibilité +Dates de collecte +Edition des données +Questionnaires +Cas valide(s) +Rapports et documents analytiques +Droits d'auteurs +Documentation +Déviations par rapport à l'échantillon initial +Editeur(s) +Février +Le jeu de données contient +Remerciement(s) +Continue +Ecart type +Description des variables +Producteur +Date de production + +L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants +Discrète +Groupe +Juillet +Nom du fichier +Enreg. +Nom +Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. +Tableaux statistiques +Décembre +Sujets +Contrôles de traitement + +Instructions aux enquêteurs +Table des matières +Informations sur le document +Sous-groupe(s) +Mots-clé +groupe(s) +P +Pondération +Description des fichiers +Notes +Notes sur la collecte +fichier(s) +continue +Responsabilité(s) +Contenu +variable(s) +Autre(s) producteur(s) +Producteurs et sponsors +Notes sur l'apurement des données +Distributeur(s) +Aperçu +Citation +Septembre +Catégorie +Confidentialité +Statistiques +Mai +Indéterminé +Structure +fichier +Pré-question +Taux de réponse +Taille +Formulation de la question +Recodage et dérivation +Série +Octobre +Unité d'analyse +Notes sur le traitement des données +Type d'étude +Fichier +Période(s) de référence +Contenu du fichier +Non-valide +Vars +suite +Clé(s) +Question +Source d'information +Imputation +Sécurité +Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit +Autres resources +Dictionnaire des variables +Information +Janvier +Autres documents +Minimum +Domaines thématiques et couverture +Producteur(s) des méta-données +Information complémentaire +Enquêteurs +Post-question +Thèmes +Procédure d'échantillonage +Structure du fichier +Liste des variables +Format +Notes sur l'échantillonage +Groupe(s) de variables +Description +Catégories +Maximum +Dépositaire(s) +Août +NP +Couverture +Pondéré +Mars +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml new file mode 100644 index 00000000000..bc5dbb06154 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml @@ -0,0 +1,161 @@ + + +Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 +有効な +度数表(Frequency table)は表示されません +由来 +不連続な +データ収集モード +その他の確認事項 +識別番号 +データファイルにアクセスするにはここをクリックしてください +無題 +値 +割合 +ウェイティング +第一次調査官 +この文書はToolkitを使用して作られました +サンプリング +ケース +アクセス条件 +情報源 +様相 +権利及び声明文 +定義 +サンプルエラーの見積もり +データファイル +地理的な適用範囲 +４月 +平均 +メタデータ製作 +データ収集 +損失データ +スクリプトおよびプログラム +可変的 +面接者の指示 +出資機関 +１１月 +バージョン +共通の +貢献者 +アクセス権限 +データ処理、評価 +範囲, 領域 +管理用文章 +連絡先 +ラベル +技術的な文書 +小数点 +タイプ +文字の質問 +概念 +範囲 +要約 +６月 +監督 +その他ファーマットのデータ評価 +参照 +アクセス、入手法 +データ収集日 +データ編集 +質問 +レポートおよび分析的な文書 +有効な場合 +コピーライト +書類 +サンプルデザインによる偏差 +発行者 +２月 +データセットに含まれる +確認事項 +連続的な +標準偏差 +変数の記述 +製作者 +製作日 +” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です +不連続性 +グループ +７月 +ファイルの名前 +ケース +名前 +統計表 +１２月 +主題, 内容 +工程監査 +ソフト +面接者への指示 +目録 +書類の情報 +サブグループ +キーワード + +グループ +ウェイト +ファイルの詳細 +メモ +データ収集メモ +ファイル +継続的な +声明文 +内容 +変数 +その他の製作者 +製作者とスポンサー +データクリーニングメモ +分配者 +概略 +引用する場合の必要条件 +９月 +カテゴリー +機密性、コンフィデンシャリティー +５月 +未定 +構造 +ファイル +調査前の質問 +回答比率 +幅 +記録と誘導 +シリー +１０月 +分析の単位 +データ処理メモ +データの種類 + +ファイル +期間 +ファイルの内容 +無効 +キー +情報源 +非難 +セキュリティー +このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 +その他の資料 +データ辞典 +情報 +１月 +その他の書類 +最小値 +規模及び適用範囲 +メタデータ製作者 +さらにインフォメーションを表示 +データ収集者 +調査後の質問 +サンプリングの手順 +ファイルの構造 +変数のリスト +フォーマット +サンプリングメモ +変数のグループ +詳細 +カテゴリー +最大値 +デポジター、提供者、供託者 +８月 +表紙 +ウェイトされた +３月 + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml new file mode 100644 index 00000000000..06fde85af5e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml @@ -0,0 +1,169 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Валидный +Частотная таблица не выводится +Расчет +дискретная +Способ сбора данных +Прочая обработка +Другие участники +Безымянный +Индентификация +Щелкните здесь, чтобы получить доступ к файлам или экспортировать их +Значение +Процент +Взвешивание +Первичный(е) исследователь(и) +Документ был сгенерирован с использованием +Выборка +Наблюдения +Условия доступа +Источник +Модальности +Авторские права и ограничения ответственности +Определение +Оценки ошибок выборки +Файлы данных +Географический охват +Апрель +Среднее +Разработка метаданных +Сбор данных +Пропущенные данные +Скрипты и программы +Переменная(ые) +Инструкции интервьюеру +Кто финансировал +Ноябрь +Пропущенные +Версия +Генеральная совокупность +Участник(и) +Права доступа +Обработка и инспекция данных +Охват +Административные документы +СтдОткл +Контак(ы) +Метка +Технические документы +Десятичные +Тип +Текст вопроса +Концепции +Диапазон +Резюме +Июнь +Контроль +Другие формы инспекции данных +Установки +Доступность +Даты сбора данных +Редактирование данных +Вопросники +Валидное(ые) наблюдение(я) +Отчеты и аналитические документы +Авторские права +Документация +Отклонения от дизайна выборки +Издатель(и) +Февраль +Набор данных содержит +Участник(и) +Непрерывная +Стандартное отклонение +Описание переменных +Разработчик +Дата разработки +Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы +Дикретная +Группа +Июль +Имя файла +Наблюдения +Имя +Статистичсекие таблицы +Декабрь +Темы +Контроль обработки +программное обеспечение +Инструкции интервьюеру +Оглавление +Информация о документе +Подгруппа(ы) +Ключевые слова +группа(ы) +B +Вес +Описание файла +Примечания +Примечания по сбору данных +файл(ы) +непрерывная +Ограничения ответственности +Содержание +переменная(ые) +Другие разработчики +Разработчики и спонсоры +Примечания по чистке данных +Дистрибьютор(ы) +Обзор +Требования по цитированию +Сентябрь +Категория +Конфиденциальность +Статистики +Май +Неопределенный +Структура +файл +Текст, предваряющий вопрос +Доля ответов +Ширина +Перекодировка и расчеты +Серия +Октябрь +Единица анализа +Примечания по обработке данных +Тип данных +Файл +Период(ы) времени +Содержание файла +Некорректный +Переменные +непр. +Ключ(и) +Вопрос +Источник информации +Импутация +Безопасность +Чтобы открыть этот файл, необходимо иметь свободным +Прочие источники +Словарь данных +Информация +Январь +Прочие документы +Минимум +Охват и покрытие +Разработчик(и) метаданных +Показать дополнительную информацию +Кто собирал данные +Текст после вопроса +Разделы +Процедура выборки +Структура файла +Список переменных +Формат +Примечания по выборке +Группа(ы) переменных +Описание +Категории +Максимум +Депозитор(ы) +Август +HB +Титульный лист +взвешенные +Март + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl new file mode 100644 index 00000000000..64849846481 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl @@ -0,0 +1,4473 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + default-page + + + + + + + + + + + + + + + + + + + + + + Times + + + + + + + + + + + + + + + + + + + + + + + + + 50 + + + + + + + 0 + 0 + 0 + 1 + + + + + + + 1 + 0 + + + + + + + 0 + 1 + + + + + + ddp + toolkit + toolkit + toolkit + + + + + + + + () + + + + + + , + + + + + + + + + + + + + + + + - + + + + + + + + + + + + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 1 + 1 + 1 + 1 + 0 + + + + + 1 + 1 + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + *** + + **** + + + + + + + + + + + + + + + , + + + + + + + + : + + + + + + + + + + + + + () + , + + + + , + + + + + + + + + + + () + , + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # + + + + + + + + + + + # + + + + + + + + + + + + + + + + + : + + + + + + :  + + + + + ,  + + +  () + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + () + , + + + + + + + + + + , + + , + + + + + + + + + + () + , + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + .. + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + - + + + + - + + + + . + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + +   + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + # + + + + + : + + + + + + + + + + + + + + + + + + + + + [= + + + + + + ] + + + + [= + + - + + + + . + + + ] + + + + [= - + ] + + + [ + + =* + + / + + + ] + + + + + + + + + + + + + + [ + + / + + ] + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + = + + + / + + + + + + - + + + ] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + () + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ( + ) + + + + + + + + + + + + + + + + + + + + : + + + + + + + + + + + + + + + + + + *** + *** + + + + + , + + + + + + + , + + + + + + , + + + + + + + + + + + + + , + + + + + , + + + + + , " + + " + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <h2> + + + </h2> + + + <br/> + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Z + + + + + + + + + + + + + + + + + + + - + + + + + + + + NaN + + + + + + + + + + + From 84caa2a85e2f86d1c6f82725673bb4a60ec19f3d Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 12 Jun 2023 17:52:04 -0400 Subject: [PATCH 014/427] q --- .../dataverse/export/PdfCodeBookExporter.java | 88 +++++++++---------- 1 file changed, 40 insertions(+), 48 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java index 8f3bb78f8d3..c4f1d3ff2cc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -1,55 +1,58 @@ + + package edu.harvard.iq.dataverse.export; import com.google.auto.service.AutoService; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; -import edu.harvard.iq.dataverse.export.spi.Exporter; -import edu.harvard.iq.dataverse.util.BundleUtil; + import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; + import io.gdcc.spi.export.ExportDataProvider; + import io.gdcc.spi.export.ExportException; + import io.gdcc.spi.export.Exporter; + import edu.harvard.iq.dataverse.util.BundleUtil; -import javax.json.JsonObject; -import javax.ws.rs.core.MediaType; -import javax.xml.stream.XMLStreamException; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Path; -import java.nio.file.Paths; + import javax.json.JsonObject; + import javax.ws.rs.core.MediaType; + import javax.xml.stream.XMLStreamException; + import java.io.File; + import java.io.IOException; + import java.io.InputStream; + import java.io.OutputStream; + import java.nio.file.Path; + import java.nio.file.Paths; + import java.util.Locale; + import java.util.Optional; @AutoService(Exporter.class) public class PdfCodeBookExporter implements Exporter { @Override - public String getProviderName() { + public String getFormatName() { return "pdf"; } @Override - public String getDisplayName() { - return BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") != null ? BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf") : "DDI PDF Codebook"; + public String getDisplayName(Locale locale) { + String displayName = BundleUtil.getStringFromBundle("dataset.exportBtn.itemLabel.pdf", locale); + return Optional.ofNullable(displayName).orElse("DDI pdf codebook"); } @Override - public void exportDataset(DatasetVersion version, JsonObject json, OutputStream outputStream) throws ExportException { - try { - InputStream ddiInputStream; - try { - ddiInputStream = ExportService.getInstance().getExport(version.getDataset(), "ddi"); - } catch(ExportException | IOException e) { - throw new ExportException ("Cannot open export_ddi cached file"); + public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException { + Optional ddiInputStreamOptional = dataProvider.getPrerequisiteInputStream(); + if (ddiInputStreamOptional.isPresent()) { + try (InputStream ddiInputStream = ddiInputStreamOptional.get()) { + DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); + } catch (IOException e) { + throw new ExportException("Cannot open export_ddi cached file"); + } catch (XMLStreamException xse) { + throw new ExportException("Caught XMLStreamException performing DDI export"); } - DdiExportUtil.datasetPdfDDI(ddiInputStream, outputStream); - } catch (XMLStreamException xse) { - throw new ExportException ("Caught XMLStreamException performing DDI export"); + } else { + throw new ExportException("No prerequisite input stream found"); } } - @Override - public Boolean isXMLFormat() { - return false; - } - @Override public Boolean isHarvestable() { // No, we don't want this format to be harvested! @@ -65,27 +68,16 @@ public Boolean isAvailableToUsers() { } @Override - public String getXMLNameSpace() throws ExportException { - return null; - } - - @Override - public String getXMLSchemaLocation() throws ExportException { - return null; - } - - @Override - public String getXMLSchemaVersion() throws ExportException { - return null; - } - - @Override - public void setParam(String name, Object value) { - // this exporter does not uses or supports any parameters as of now. + public Optional getPrerequisiteFormatName() { + //This exporter relies on being able to get the output of the ddi exporter + return Optional.of("ddi"); } @Override public String getMediaType() { - return MediaType.WILDCARD; + return MediaType.TEXT_HTML; }; } + + + From cb85fd678945ed206cbcca7fc378bbb7d1771394 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 15 Jun 2023 17:54:09 -0400 Subject: [PATCH 015/427] english --- .../dataverse/export/PdfCodeBookExporter.java | 2 +- .../dataverse/export/ddi/DdiExportUtil.java | 15 +- .../{from-ddi-2.5 => }/ddi-to-fo.xsl | 444 +++++++++--------- .../from-ddi-2.5/ddi-pdf/i18n.inc.xslt | 5 - .../ddi-pdf/messages_en.properties.xml | 174 ------- .../ddi-pdf/messages_es.properties.xml | 170 ------- .../ddi-pdf/messages_fr.properties.xml | 173 ------- .../ddi-pdf/messages_ja.properties.xml | 161 ------- .../ddi-pdf/messages_nn.properties.xml | 174 ------- .../ddi-pdf/messages_no.properties.xml | 174 ------- .../ddi-pdf/messages_ru.properties.xml | 169 ------- 11 files changed, 234 insertions(+), 1427 deletions(-) rename src/main/resources/edu/harvard/iq/dataverse/{from-ddi-2.5 => }/ddi-to-fo.xsl (91%) delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java index c4f1d3ff2cc..e0d5171e30c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -75,7 +75,7 @@ public Optional getPrerequisiteFormatName() { @Override public String getMediaType() { - return MediaType.TEXT_HTML; + return MediaType.WILDCARD; }; } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 00ec3c4d5c9..c77be3515f7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -43,6 +43,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -2110,7 +2111,10 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { - File xsltfile = new File("/home/victoria/ddi-to-fo.xsl"); + //File xsltfile = new File("/home/victoria/fop-2.8/fop/ddi-to-fo.xsl"); + //URL resource = DdiExportUtil.class.getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); + //File xsltfile = new File(resource.toURI()); + InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); logger.info("start datasetPdfDDI"); //InputStream xsltfile = DdiExportUtil.class.getClassLoader().getResourceAsStream( // "edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl"); @@ -2122,7 +2126,7 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); // Setup XSLT TransformerFactory factory = TransformerFactory.newInstance(); - Transformer transformer = factory.newTransformer(new StreamSource(xsltfile)); + Transformer transformer = factory.newTransformer(new StreamSource(styleSheetInput)); // Set the value of a in the stylesheet transformer.setParameter("versionParam", "2.0"); @@ -2135,10 +2139,13 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream // Start XSLT transformation and FOP processing transformer.transform(src, res); - } finally { - outputStream.close(); + + } catch (Exception e) { + logger.info("First try"); + logger.severe(e.getMessage()); } } catch (Exception e) { + logger.info("Second try"); logger.severe(e.getMessage()); } } diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl similarity index 91% rename from src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl rename to src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index 64849846481..1c03d5caf34 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -66,7 +66,8 @@ Report optional text --> - + + @@ -123,7 +124,6 @@ Times - - + - + - + - + - + @@ -437,21 +437,21 @@ + select="'Data_Processing_and_Appraisal'"/> - + - + @@ -461,7 +461,7 @@ - + @@ -486,7 +486,7 @@ - + @@ -503,7 +503,7 @@ - + @@ -528,7 +528,7 @@ - + @@ -569,7 +569,7 @@ - + @@ -642,7 +642,7 @@ - + @@ -657,7 +657,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Metadata_Producers'"/> + select="'Production_Date'"/> + select="'Version'"/> + select="'Identification'"/> - + @@ -745,7 +745,7 @@ - + @@ -764,14 +764,14 @@ - + - + @@ -782,7 +782,7 @@ + select="'Scope_and_Coverage'"/> @@ -793,7 +793,7 @@ + select="'Producers_and_Sponsors'"/> @@ -803,7 +803,7 @@ - + @@ -813,7 +813,7 @@ - + @@ -825,7 +825,7 @@ internal-destination="data-processing-and-appraisal" text-decoration="underline" color="blue"> + select="'Data_Processing_and_Appraisal'"/> @@ -836,7 +836,7 @@ - + @@ -847,7 +847,7 @@ + select="'Rights_and_Disclaimer'"/> @@ -858,7 +858,7 @@ + select="'Files_Description'"/> @@ -891,7 +891,7 @@ - + @@ -924,7 +924,7 @@ - + @@ -949,7 +949,7 @@ + select="'Variables_Description'"/> @@ -982,7 +982,7 @@ - + @@ -1002,7 +1002,7 @@ - + @@ -1065,7 +1065,7 @@ - + @@ -1078,7 +1078,7 @@ - + + select="'Identification'"/> - @@ -1130,7 +1130,7 @@ select="/ddi:codeBook/ddi:stdyDscr/ddi:citation/ddi:verStmt/ddi:version"> : + select="'Production_Date'"/>: @@ -1139,7 +1139,7 @@ - @@ -1156,7 +1156,7 @@ - + - + select="'Kind_of_Data'"/> + select="'Unit_of_Analysis'"/> + select="'Scope_and_Coverage'"/> @@ -1257,7 +1257,7 @@ - + - @@ -1299,7 +1299,7 @@ - + - + @@ -1344,7 +1344,7 @@ - + @@ -1361,7 +1361,7 @@ border="{$default-border}" padding="{$cell-padding}"> - @@ -1422,7 +1422,7 @@ padding="{$cell-padding}"> @@ -1444,7 +1444,7 @@ padding="{$cell-padding}"> + select="'Other_Producers'"/> + select="'Funding_Agencies'"/> @@ -1514,7 +1514,7 @@ - @@ -1534,7 +1534,7 @@ padding="{$cell-padding}"> + select="'Sampling'"/> + select="'Sampling_Notes'"/> + select="'Sampling_Procedure'"/> + select="'Response_Rate'"/> + select="'Weighting'"/> + select="'Data_Collection'"/> @@ -1672,7 +1672,7 @@ padding="{$cell-padding}"> @@ -1694,7 +1694,7 @@ padding="{$cell-padding}"> + select="'Time_Periods'"/> @@ -1738,7 +1738,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Questionnaires'"/> + select="'Data_Collectors'"/> + select="'Supervision'"/> - + @@ -1885,7 +1885,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Data_Editing'"/> + select="'Other_Processing'"/> + select="'Accessibility'"/> @@ -1980,7 +1980,7 @@ padding="{$cell-padding}"> + select="'Access_Authority'"/> - @@ -2022,7 +2022,7 @@ padding="{$cell-padding}"> + select="'Distributors'"/> + select="'Depositors'"/> + select="'Confidentiality'"/> + select="'Access_Conditions'"/> @@ -2142,7 +2142,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="'Disclaimer'"/> + select="'Copyright'"/> - + @@ -2194,15 +2194,15 @@ - + - + - + @@ -2218,7 +2218,7 @@ - + @@ -2227,16 +2227,16 @@ - + - + - - + + - + @@ -2262,17 +2262,17 @@ - + - + - + - + @@ -2290,7 +2290,7 @@ - + @@ -2299,17 +2299,17 @@ - + - + - + - + @@ -2327,7 +2327,7 @@ - + @@ -2336,7 +2336,7 @@ - + @@ -2372,7 +2372,7 @@ + select="'Reports_and_analytical_documents'"/> @@ -2389,7 +2389,7 @@ space-after="0.03in"> - + @@ -2407,7 +2407,7 @@ + select="'Technical_documents'"/> @@ -2425,7 +2425,7 @@ + select="'Administrative_documents'"/> @@ -2442,7 +2442,7 @@ space-after="0.03in"> - + @@ -2459,7 +2459,7 @@ space-after="0.03in"> - + @@ -2477,7 +2477,7 @@ + select="'Statistical_tables'"/> @@ -2495,7 +2495,7 @@ + select="'Scripts_and_programs'"/> @@ -2512,7 +2512,7 @@ space-after="0.03in"> - + @@ -2532,7 +2532,7 @@ + select="'Reports_and_analytical_documents'"/> - + - + - - + - + - + @@ -2602,7 +2602,7 @@ - + @@ -2611,7 +2611,7 @@ - + *** - + **** @@ -2749,7 +2749,7 @@ - # # @@ -2762,7 +2762,7 @@ - # # @@ -2776,18 +2776,18 @@ - + - : + : - :  + :  @@ -2811,7 +2811,7 @@ - + @@ -2823,7 +2823,7 @@ - + @@ -2835,7 +2835,7 @@ - + @@ -2847,7 +2847,7 @@ - + @@ -2859,7 +2859,7 @@ - + @@ -2871,7 +2871,7 @@ - + @@ -2905,10 +2905,10 @@ - + - + @@ -2955,7 +2955,7 @@ - + : @@ -3059,7 +3059,7 @@ - + @@ -3080,7 +3080,7 @@ - + @@ -3092,7 +3092,7 @@ - + @@ -3104,7 +3104,7 @@ - + @@ -3116,7 +3116,7 @@ - + @@ -3239,13 +3239,13 @@ - + - + - + @@ -3322,7 +3322,7 @@   - + @@ -3369,7 +3369,7 @@ : - + @@ -3382,7 +3382,7 @@ - + [= + select="'Type'"/>= + select="'discrete'"/> + select="'continuous'"/> ] @@ -3404,7 +3404,7 @@ [== - @@ -3419,13 +3419,13 @@ [= = - ] [ - + =* / @@ -3437,7 +3437,7 @@ @@ -3452,12 +3452,12 @@ - + [ + select="'Abbrev_NotWeighted'"/> / - + ] @@ -3470,18 +3470,18 @@ - + select="'Invalid'"/> - + - @@ -3517,7 +3517,7 @@ - + - + - + - + - + - + @@ -3638,7 +3638,7 @@ - + + select="'Recoding_and_Derivation'"/> - + - + - + - - + + - @@ -3798,7 +3798,7 @@ padding="{$cell-padding}" text-align="center"> + select="'Cases_Abbreviation'"/> @@ -3806,7 +3806,7 @@ padding="{$cell-padding}" text-align="center"> + select="'Weighted'"/> @@ -3814,9 +3814,9 @@ padding="{$cell-padding}" text-align="center"> + select="'Percentage'"/> () + select="'Weighted'"/>) @@ -3950,16 +3950,16 @@ + select="'SumStat_Warning'"/> - + ( - ) + ) @@ -4003,7 +4003,7 @@ - *** + *** *** @@ -4055,7 +4055,7 @@ - + @@ -4070,7 +4070,7 @@ - + @@ -4085,7 +4085,7 @@ - + @@ -4100,7 +4100,7 @@ - + @@ -4162,38 +4162,38 @@ - + - + - + - + - + - + - + @@ -4257,40 +4257,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt deleted file mode 100644 index edf876f3b04..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/i18n.inc.xslt +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml deleted file mode 100644 index d8e98dfd3c6..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_en.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Valid -Frequency table not shown -Derivation -discrete -Data Collection Mode -Other Processing -Other Acknowledgment(s) -Untitled -Identification -Click here to access/export data files from Nesstar format -Value -Percentage -Weighting -Primary Investigator(s) -This document was generated using the -Sampling -Cases -Access Conditions -Source -Modalities -Rights & Disclaimer -Definition -Estimates of Sampling Error -Data Files -Geographic Coverage -April -Mean -Metadata Production -Data Collection -Missing Data -Scripts and programs -Variable(s) -Interviewer instructions -Funding Agency/ies -November -Missing -Version -Universe -Contributor(s) -Access Authority -Data Processing & Appraisal -Scope -Administrative documents -StdDev -Contact(s) -Label -Technical documents -Decimal(s) -Type -Literal question -Concepts -Range -Abstract -June -Supervision -Other Forms of Data Appraisal -References -Accessibility -Data Collection Dates -Data Editing -Questionnaires -Valid case(s) -Reports and analytical documents -Copyright -Documentation -Deviations from Sample Design -Publisher(s) -February -Dataset contains -Acknowledgment(s) -Continuous -Standard deviation -Variables Description -Producer -Production Date - -The Explorer allows you to view data files and export them to common statistical formats -Discrete -Group -July -Filename -Cases -Name -Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. -Statistical tables -December -Subjects -Processing Checks -software -Interviewer's instructions -Table of Contents -Document Information -Subgroup(s) -Keywords -group(s) -W -Weight -Files Description -Notes -Data Collection Notes -file(s) -continuous -Disclaimer -Content -variable(s) -Other Producer(s) -Producers & Sponsors -Data Cleaning Notes -Distributor(s) -Overview -Citation Requirements -September -Category -Confidentiality -Statistics -May -Undetermined -Structure -file -Pre-question -Response Rate -Width -Recoding and Derivation -Series -October -Unit of Analysis -Data Processing Notes -Kind of Data -File -Time Period(s) -File Content -Invalid -Vars -cont. -Key(s) -Question -Source of information -Imputation -Security -To open this file, you will need the free -Other resources -Data Dictionnary -Information -January -Other documents -Minimum -Scope & Coverage -Metadata Producer(s) -Show more info -Data Collector(s) -Post-question -Topics -Sampling Procedure -File Structure -Variables List -Format -Sampling Notes -Variables Group(s) -Description -Categories -Maximum -Depositor(s) -August -NW -Cover Page -Weighted -March - total - showing a subset of -Countries -question details - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml deleted file mode 100644 index 9cfcdaf6e7e..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_es.properties.xml +++ /dev/null @@ -1,170 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Válido -No se presentan las tablas de frecuencias -Derivación -discreta -Método de Recolección -Otros relacionados al procesamiento -Otros Reconocimientos -Sin título -Identificación -Presione aquí para acceder/exportar al archivo(s) de datos -Valor -Porcentaje -Ponderando -Investigadores Principales -Este documento fue producido utilizando el -Muestreo -Casos -Condiciones de uso -Fuente -Modalidades -Derechos y Notas Legales -Definición -Estimaciones del Error Muestral -Archivo de Datos -Cobertura Geográfica -Abril -Media -Producción de los Metadatos -Recolección de Datos -Datos perdidos -Programas informáticos -Variable(s) -Manual del encuestador -Agencias Auspiciadoras -Noviembre -Valores perdidos -Versión -Universo -Contribuidor(es) -Institución Propietaria -Tratamiento y Validación de Datos -Dominio Temático -Documentos Administrativos -Desviación -Contacto(s) -Etiqueta -Documentos Técnicos -Decimal(es) -Tipo -Pregunta textual -Conceptos -Rango -Resumen -Junio -Supervisión -Otras Formas de Validación de los Datos -Referencias -Accesibilidad -Fechas de Recolección de Datos -Procesamiento de Datos -Cuestionarios -Casos válidos -Reportes y documentos analíticos -Derechos de Autor -Documentación -Modificaciones al Diseño Muestral -Editor(es) -Febrero -Contenido de la Base de Datos -Reconocimiento(s) -Contínua -Desviación estándar -Descripción de la variable -Productor -Fecha de Producción -El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos -Discreta -Grupo -Julio -Nombre del Archivo -Casos -Nombre -Cuadros estadísticos -Diciembre -Temas -Controles de Tratamiento -software -Manual del encuestador -Indice -Información acerca de la Documentación -Subgrupo(s) -Palabra Clave -grupo(s) -P -Ponderador -Descripción de los Archivos -Notas -Notas sobre la Recolección de Datos -archivo(s) -continua -Nota Legal -Contenido -variable(s) -Otros Productores -Productores y Auspiciadores -Notas acerca de la Depuración de los Datos -Distribuidor(es) -Resumen General -Forma de citar -Septiembre -Categoría -Confidencialidad -Estadística - -Mayo -Indeterminado -Estructura -archivo -Pre-pregunta -Tasa de Respuesta -Ancho -Recodificación y Derivación -Series -Octubre -Unidad de Análisis -Notas sobre el Procesamiento de Datos -Tipo de Datos -Archivo -Periodo de Referencia -Contenido del Archivo -Inválido -Vars. -cont. -Clave(s) -Pregunta -Fuente de información -Imputación -Seguridad -Para abrir este archivo se necesita el software gratuito -Otros recursos -Diccionario de Datos -Información -Enero -Otros documentos -Mínimo -Cobertura y Dominio Temático -Productor de los Metadatos -Mostrar más información -Entrevistador(es) -Pos-pregunta -Temas -Procedimiento de Muestreo -Estructura del Archivo -Lista de variables -Formato -Notas sobre el Muestreo -Grupo(s) de Variables -Descripción -Categorías -Máximo -Depositante(s) -Agosto -NP -Carátula -Ponderado -Marzo - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml deleted file mode 100644 index 9fa4d2178b1..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_fr.properties.xml +++ /dev/null @@ -1,173 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Valide -Tableau de fréquences non-affiché -Mode de calcul -discrète -Méthode de collecte -Autre traitement -Autre(s) remerciement(s) -Sans titre -Identification -Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar -Valeur -Pourcentage -Pondération -Enquêteur(s) principal/aux -Ce document a été généré à l'aide du -Echantillonage -Enregistrements -Conditions d'accès -Source -Modalités -Responsabilité et droits d'auteurs -Définition -Estimation des erreurs d'échantillonage -Fichiers de données -Couverture géographique -Avril -Moyenne -Production des méta-données -Collecte des données -Valeures manquantes -Programmes informatiques -Variable(s) -Instructions aux enquêteurs -Financement -Novembre -Manquant -Version -Univers -Contributeur(s) -Agence propriétaire -Traitement et évaluation des données -Domaines thématiques -Documents administratifs -Ecart type -Contact(s) -Libellé -Documents techniques -Décimale(s) -Type -Formulation de la question -Concepts -Gamme -Résumé -Juin -Supervision -Autres formes d'évaluation des données -Références -Accessibilité -Dates de collecte -Edition des données -Questionnaires -Cas valide(s) -Rapports et documents analytiques -Droits d'auteurs -Documentation -Déviations par rapport à l'échantillon initial -Editeur(s) -Février -Le jeu de données contient -Remerciement(s) -Continue -Ecart type -Description des variables -Producteur -Date de production - -L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants -Discrète -Groupe -Juillet -Nom du fichier -Enreg. -Nom -Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. -Tableaux statistiques -Décembre -Sujets -Contrôles de traitement - -Instructions aux enquêteurs -Table des matières -Informations sur le document -Sous-groupe(s) -Mots-clé -groupe(s) -P -Pondération -Description des fichiers -Notes -Notes sur la collecte -fichier(s) -continue -Responsabilité(s) -Contenu -variable(s) -Autre(s) producteur(s) -Producteurs et sponsors -Notes sur l'apurement des données -Distributeur(s) -Aperçu -Citation -Septembre -Catégorie -Confidentialité -Statistiques -Mai -Indéterminé -Structure -fichier -Pré-question -Taux de réponse -Taille -Formulation de la question -Recodage et dérivation -Série -Octobre -Unité d'analyse -Notes sur le traitement des données -Type d'étude -Fichier -Période(s) de référence -Contenu du fichier -Non-valide -Vars -suite -Clé(s) -Question -Source d'information -Imputation -Sécurité -Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit -Autres resources -Dictionnaire des variables -Information -Janvier -Autres documents -Minimum -Domaines thématiques et couverture -Producteur(s) des méta-données -Information complémentaire -Enquêteurs -Post-question -Thèmes -Procédure d'échantillonage -Structure du fichier -Liste des variables -Format -Notes sur l'échantillonage -Groupe(s) de variables -Description -Catégories -Maximum -Dépositaire(s) -Août -NP -Couverture -Pondéré -Mars -question details - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml deleted file mode 100644 index bc5dbb06154..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ja.properties.xml +++ /dev/null @@ -1,161 +0,0 @@ - - -Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 -有効な -度数表(Frequency table)は表示されません -由来 -不連続な -データ収集モード -その他の確認事項 -識別番号 -データファイルにアクセスするにはここをクリックしてください -無題 -値 -割合 -ウェイティング -第一次調査官 -この文書はToolkitを使用して作られました -サンプリング -ケース -アクセス条件 -情報源 -様相 -権利及び声明文 -定義 -サンプルエラーの見積もり -データファイル -地理的な適用範囲 -４月 -平均 -メタデータ製作 -データ収集 -損失データ -スクリプトおよびプログラム -可変的 -面接者の指示 -出資機関 -１１月 -バージョン -共通の -貢献者 -アクセス権限 -データ処理、評価 -範囲, 領域 -管理用文章 -連絡先 -ラベル -技術的な文書 -小数点 -タイプ -文字の質問 -概念 -範囲 -要約 -６月 -監督 -その他ファーマットのデータ評価 -参照 -アクセス、入手法 -データ収集日 -データ編集 -質問 -レポートおよび分析的な文書 -有効な場合 -コピーライト -書類 -サンプルデザインによる偏差 -発行者 -２月 -データセットに含まれる -確認事項 -連続的な -標準偏差 -変数の記述 -製作者 -製作日 -” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です -不連続性 -グループ -７月 -ファイルの名前 -ケース -名前 -統計表 -１２月 -主題, 内容 -工程監査 -ソフト -面接者への指示 -目録 -書類の情報 -サブグループ -キーワード - -グループ -ウェイト -ファイルの詳細 -メモ -データ収集メモ -ファイル -継続的な -声明文 -内容 -変数 -その他の製作者 -製作者とスポンサー -データクリーニングメモ -分配者 -概略 -引用する場合の必要条件 -９月 -カテゴリー -機密性、コンフィデンシャリティー -５月 -未定 -構造 -ファイル -調査前の質問 -回答比率 -幅 -記録と誘導 -シリー -１０月 -分析の単位 -データ処理メモ -データの種類 - -ファイル -期間 -ファイルの内容 -無効 -キー -情報源 -非難 -セキュリティー -このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 -その他の資料 -データ辞典 -情報 -１月 -その他の書類 -最小値 -規模及び適用範囲 -メタデータ製作者 -さらにインフォメーションを表示 -データ収集者 -調査後の質問 -サンプリングの手順 -ファイルの構造 -変数のリスト -フォーマット -サンプリングメモ -変数のグループ -詳細 -カテゴリー -最大値 -デポジター、提供者、供託者 -８月 -表紙 -ウェイトされた -３月 - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml deleted file mode 100644 index fdf14f5dfcd..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_nn.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Gyldige -Frekvenstabell ikke vist -Avledning -diskret -Type datainnsamling -Annen prosessering -Andre identifikatorer og krediteringer -Uten tittel -Identifisering -Click here to access/export data files from Nesstar format -Verdi -Prosent -Vekting -Primary Investigator(s) -Dette dokumentet ble generert av -Utvalg -Enheter -Tilgangsbetingelser -Kilde -Modaliteter -Rights & Disclaimer -Definisjon -Estimert utvalgsfeil -Datafiler -Geografisk omfang -April -Mean -Metadata-produksjon -Datainnsamling -Manglende data -Script og programmer -Variable(r) -Instruksjoner til intervjueren -Sponsor/finansierende institusjon(er) -November -Missing -Versjon -Univers -Bidragsyter(e) -Tilgangskontrollør -Dataprosessering og -evaluering -Omfang -Administrative dokumenter -Standardavvik -Kontaktperson(er) -Merkelapp -Tekniske dokumenter -Desimal(er) -Type -Spørsmålstekst -Begrep(er) -Rekkevidde -Sammendrag -Juni -Supervision -Andre former for dataevaluering -Referanser -Tilgjengelighet -Datainnsamlingsdatoer -Dataredigering -Spørreskjema -Gyldige enheter -Rapporter og analysedokumenter -Copyright -Dokumentasjon -Avvik fra utvalgsdesign -Utgiver(e) -Februar -Datasettet inneholder -Krediteringer -Kontinuerlig -Standardavvik -Variabelbeskrivelse -Produsent -Produksjonsdato - -The Explorer allows you to view data files and export them to common statistical formats -Diskret -Gruppe -Juli -Filnavn -Enheter -Navn -Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. -Statistiske tabeller -Desember -Emner -Prosesseringssjekk -programvare -Instruksjoner til intervjueren -Innholdsfortegnelse -Dokumentinformasjon -Undergruppe(r) -Nøkkelord -gruppe(r) -W -Vekt -Filbeskrivelse -Kommentarer -Datainnsamlingskommentarer -file(r) -kontinuerlig -Fraskrivelse -Innhold -variable(r) -Andre produsenter -Produsenter og sponsorer -Kommentarer om datarensing -Distributør(er) -Oversikt -Sitatkrav -September -Kategori -Konfidensialitet -Statistikk -Mai -Uavklart -Struktur -fil -Tekst før spørsmål -Responsrate -Bredde -Omkodinger og utledninger -Serie -Oktober -Analyseenhet -Dataprosesseringskommentarer -Datatype -Fil -Tidsperiode(r) -Filinnhold -Ugyldig -Variabler -kont. -Nøkler -Spørsmål -Kilde for informasjon -Imputasjon -Sikkerhet -For å åpne denne filen trenger du følgende gratisverktøy -Andre ressurser -Dataordbok -Informasjon -Januar -Andre dokumenter -Minimum -Omfang -Metadataprodusenter -Vis mer informasjon -Datainnsamler(e) -Tekst etter spørsmål -Emner -Utvalgsprosedyre -Filstruktur -Variabelliste -Format -Utvalgskommentarer -Variabelgrupper -Beskrivelse -Kategorier -Maksimum -Utgiver(e) -August -NW -Forside -Vektet -Mars - total - viser et utvalg av -Land -spørsmålsdetaljer - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml deleted file mode 100644 index fdf14f5dfcd..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_no.properties.xml +++ /dev/null @@ -1,174 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 -Gyldige -Frekvenstabell ikke vist -Avledning -diskret -Type datainnsamling -Annen prosessering -Andre identifikatorer og krediteringer -Uten tittel -Identifisering -Click here to access/export data files from Nesstar format -Verdi -Prosent -Vekting -Primary Investigator(s) -Dette dokumentet ble generert av -Utvalg -Enheter -Tilgangsbetingelser -Kilde -Modaliteter -Rights & Disclaimer -Definisjon -Estimert utvalgsfeil -Datafiler -Geografisk omfang -April -Mean -Metadata-produksjon -Datainnsamling -Manglende data -Script og programmer -Variable(r) -Instruksjoner til intervjueren -Sponsor/finansierende institusjon(er) -November -Missing -Versjon -Univers -Bidragsyter(e) -Tilgangskontrollør -Dataprosessering og -evaluering -Omfang -Administrative dokumenter -Standardavvik -Kontaktperson(er) -Merkelapp -Tekniske dokumenter -Desimal(er) -Type -Spørsmålstekst -Begrep(er) -Rekkevidde -Sammendrag -Juni -Supervision -Andre former for dataevaluering -Referanser -Tilgjengelighet -Datainnsamlingsdatoer -Dataredigering -Spørreskjema -Gyldige enheter -Rapporter og analysedokumenter -Copyright -Dokumentasjon -Avvik fra utvalgsdesign -Utgiver(e) -Februar -Datasettet inneholder -Krediteringer -Kontinuerlig -Standardavvik -Variabelbeskrivelse -Produsent -Produksjonsdato - -The Explorer allows you to view data files and export them to common statistical formats -Diskret -Gruppe -Juli -Filnavn -Enheter -Navn -Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. -Statistiske tabeller -Desember -Emner -Prosesseringssjekk -programvare -Instruksjoner til intervjueren -Innholdsfortegnelse -Dokumentinformasjon -Undergruppe(r) -Nøkkelord -gruppe(r) -W -Vekt -Filbeskrivelse -Kommentarer -Datainnsamlingskommentarer -file(r) -kontinuerlig -Fraskrivelse -Innhold -variable(r) -Andre produsenter -Produsenter og sponsorer -Kommentarer om datarensing -Distributør(er) -Oversikt -Sitatkrav -September -Kategori -Konfidensialitet -Statistikk -Mai -Uavklart -Struktur -fil -Tekst før spørsmål -Responsrate -Bredde -Omkodinger og utledninger -Serie -Oktober -Analyseenhet -Dataprosesseringskommentarer -Datatype -Fil -Tidsperiode(r) -Filinnhold -Ugyldig -Variabler -kont. -Nøkler -Spørsmål -Kilde for informasjon -Imputasjon -Sikkerhet -For å åpne denne filen trenger du følgende gratisverktøy -Andre ressurser -Dataordbok -Informasjon -Januar -Andre dokumenter -Minimum -Omfang -Metadataprodusenter -Vis mer informasjon -Datainnsamler(e) -Tekst etter spørsmål -Emner -Utvalgsprosedyre -Filstruktur -Variabelliste -Format -Utvalgskommentarer -Variabelgrupper -Beskrivelse -Kategorier -Maksimum -Utgiver(e) -August -NW -Forside -Vektet -Mars - total - viser et utvalg av -Land -spørsmålsdetaljer - diff --git a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml deleted file mode 100644 index 06fde85af5e..00000000000 --- a/src/main/resources/edu/harvard/iq/dataverse/from-ddi-2.5/ddi-pdf/messages_ru.properties.xml +++ /dev/null @@ -1,169 +0,0 @@ - - -Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 -Валидный -Частотная таблица не выводится -Расчет -дискретная -Способ сбора данных -Прочая обработка -Другие участники -Безымянный -Индентификация -Щелкните здесь, чтобы получить доступ к файлам или экспортировать их -Значение -Процент -Взвешивание -Первичный(е) исследователь(и) -Документ был сгенерирован с использованием -Выборка -Наблюдения -Условия доступа -Источник -Модальности -Авторские права и ограничения ответственности -Определение -Оценки ошибок выборки -Файлы данных -Географический охват -Апрель -Среднее -Разработка метаданных -Сбор данных -Пропущенные данные -Скрипты и программы -Переменная(ые) -Инструкции интервьюеру -Кто финансировал -Ноябрь -Пропущенные -Версия -Генеральная совокупность -Участник(и) -Права доступа -Обработка и инспекция данных -Охват -Административные документы -СтдОткл -Контак(ы) -Метка -Технические документы -Десятичные -Тип -Текст вопроса -Концепции -Диапазон -Резюме -Июнь -Контроль -Другие формы инспекции данных -Установки -Доступность -Даты сбора данных -Редактирование данных -Вопросники -Валидное(ые) наблюдение(я) -Отчеты и аналитические документы -Авторские права -Документация -Отклонения от дизайна выборки -Издатель(и) -Февраль -Набор данных содержит -Участник(и) -Непрерывная -Стандартное отклонение -Описание переменных -Разработчик -Дата разработки -Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы -Дикретная -Группа -Июль -Имя файла -Наблюдения -Имя -Статистичсекие таблицы -Декабрь -Темы -Контроль обработки -программное обеспечение -Инструкции интервьюеру -Оглавление -Информация о документе -Подгруппа(ы) -Ключевые слова -группа(ы) -B -Вес -Описание файла -Примечания -Примечания по сбору данных -файл(ы) -непрерывная -Ограничения ответственности -Содержание -переменная(ые) -Другие разработчики -Разработчики и спонсоры -Примечания по чистке данных -Дистрибьютор(ы) -Обзор -Требования по цитированию -Сентябрь -Категория -Конфиденциальность -Статистики -Май -Неопределенный -Структура -файл -Текст, предваряющий вопрос -Доля ответов -Ширина -Перекодировка и расчеты -Серия -Октябрь -Единица анализа -Примечания по обработке данных -Тип данных -Файл -Период(ы) времени -Содержание файла -Некорректный -Переменные -непр. -Ключ(и) -Вопрос -Источник информации -Импутация -Безопасность -Чтобы открыть этот файл, необходимо иметь свободным -Прочие источники -Словарь данных -Информация -Январь -Прочие документы -Минимум -Охват и покрытие -Разработчик(и) метаданных -Показать дополнительную информацию -Кто собирал данные -Текст после вопроса -Разделы -Процедура выборки -Структура файла -Список переменных -Формат -Примечания по выборке -Группа(ы) переменных -Описание -Категории -Максимум -Депозитор(ы) -Август -HB -Титульный лист -взвешенные -Март - From 2968ba2d9a1f84773939a41b6867bab686d7a604 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 28 Jun 2023 10:34:22 +0200 Subject: [PATCH 016/427] combined query for retrieving datasets with API --- .../iq/dataverse/DatasetServiceBean.java | 1 + .../iq/dataverse/api/AbstractApiBean.java | 35 +++++++++++-------- .../harvard/iq/dataverse/api/Datasets.java | 2 +- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 305afd2ed30..9bf506e6c1f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -127,6 +127,7 @@ public Dataset findDeep(Object pk) { .setHint("eclipselink.left-join-fetch", "o.files.dataFileTags") .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas") .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas.fileCategories") + .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas.varGroups") .setHint("eclipselink.left-join-fetch", "o.files.guestbookResponses") .setHint("eclipselink.left-join-fetch", "o.files.embargo") .setHint("eclipselink.left-join-fetch", "o.files.fileAccessRequests") diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 6b3e27becb7..fa281a531e8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -41,6 +41,7 @@ import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; import edu.harvard.iq.dataverse.license.LicenseServiceBean; import edu.harvard.iq.dataverse.metrics.MetricsServiceBean; +import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.locality.StorageSiteServiceBean; import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -390,6 +391,11 @@ protected DataverseLinkingDataverse findDataverseLinkingDataverseOrDie(String da } protected Dataset findDatasetOrDie(String id) throws WrappedResponse { + return findDatasetOrDie(id, false); + } + + protected Dataset findDatasetOrDie(String id, boolean deep) throws WrappedResponse { + Long datasetId; Dataset dataset; if (id.equals(PERSISTENT_ID_KEY)) { String persistentId = getRequestParameter(PERSISTENT_ID_KEY.substring(1)); @@ -397,23 +403,24 @@ protected Dataset findDatasetOrDie(String id) throws WrappedResponse { throw new WrappedResponse( badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset_id_is_null", Collections.singletonList(PERSISTENT_ID_KEY.substring(1))))); } - dataset = datasetSvc.findByGlobalId(persistentId); + datasetId = dvObjSvc.findIdByGlobalId(PidUtil.parseAsGlobalID(persistentId), DvObject.DType.Dataset); + } else { + datasetId = Long.parseLong(id); + } + + try { + if (deep) { + dataset = datasetSvc.findDeep(datasetId); + } else { + dataset = datasetSvc.find(datasetId); + } if (dataset == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.persistentId", Collections.singletonList(persistentId)))); + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); } return dataset; - - } else { - try { - dataset = datasetSvc.find(Long.parseLong(id)); - if (dataset == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); - } - return dataset; - } catch (NumberFormatException nfe) { - throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(id)))); - } + } catch (NumberFormatException nfe) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(id)))); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 8c1390b597e..7a527881b2b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -257,7 +257,7 @@ public interface DsVersionHandler { @Path("{id}") public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) { return response( req -> { - final Dataset retrieved = execCommand(new GetDatasetCommand(req, findDatasetOrDie(id))); + final Dataset retrieved = execCommand(new GetDatasetCommand(req, findDatasetOrDie(id, true))); final DatasetVersion latest = execCommand(new GetLatestAccessibleDatasetVersionCommand(req, retrieved)); final JsonObjectBuilder jsonbuilder = json(retrieved); //Report MDC if this is a released version (could be draft if user has access, or user may not have access at all and is not getting metadata beyond the minimum) From 6083ead476ef195501fd44276db8702fb455cc9a Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Thu, 29 Jun 2023 11:54:39 +0200 Subject: [PATCH 017/427] better error handling - should fix failed integration test --- .../iq/dataverse/api/AbstractApiBean.java | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index fa281a531e8..5a9720639d1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -17,6 +17,7 @@ import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.EjbDataverseEngine; +import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; @@ -403,25 +404,36 @@ protected Dataset findDatasetOrDie(String id, boolean deep) throws WrappedRespon throw new WrappedResponse( badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset_id_is_null", Collections.singletonList(PERSISTENT_ID_KEY.substring(1))))); } - datasetId = dvObjSvc.findIdByGlobalId(PidUtil.parseAsGlobalID(persistentId), DvObject.DType.Dataset); + GlobalId globalId; + try { + globalId = PidUtil.parseAsGlobalID(persistentId); + } catch (IllegalArgumentException e) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(persistentId)))); + } + datasetId = dvObjSvc.findIdByGlobalId(globalId, DvObject.DType.Dataset); + if (datasetId == null) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset_id_is_null", Collections.singletonList(PERSISTENT_ID_KEY.substring(1))))); + } } else { - datasetId = Long.parseLong(id); + try { + datasetId = Long.parseLong(id); + } catch (NumberFormatException nfe) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(id)))); + } } - try { - if (deep) { - dataset = datasetSvc.findDeep(datasetId); - } else { - dataset = datasetSvc.find(datasetId); - } - if (dataset == null) { - throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); - } - return dataset; - } catch (NumberFormatException nfe) { - throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.bad.id", Collections.singletonList(id)))); + if (deep) { + dataset = datasetSvc.findDeep(datasetId); + } else { + dataset = datasetSvc.find(datasetId); + } + if (dataset == null) { + throw new WrappedResponse(notFound(BundleUtil.getStringFromBundle("find.dataset.error.dataset.not.found.id", Collections.singletonList(id)))); } + return dataset; } protected DataFile findDataFileOrDie(String id) throws WrappedResponse { From 8e75241e6fe3c2c4595f1d11fe658d354f7b2f4a Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 4 Jul 2023 14:01:47 +0200 Subject: [PATCH 018/427] find deep in file listing --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 7a527881b2b..7229444e088 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -495,7 +495,7 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id" @Path("{id}/versions/{versionId}/files") public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) { return response( req -> ok( jsonFileMetadatas( - getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getFileMetadatas())), getRequestUser(crc)); + getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId, true), uriInfo, headers).getFileMetadatas())), getRequestUser(crc)); } @GET From d2d7f4df4ef0770d2948a8027cf91c16fda1b1e8 Mon Sep 17 00:00:00 2001 From: okaradeniz Date: Fri, 14 Jul 2023 12:07:52 +0200 Subject: [PATCH 019/427] cite selected dataset version in citation downloads --- .../iq/dataverse/FileDownloadServiceBean.java | 74 +++++++++++++++---- src/main/webapp/dataset-citation.xhtml | 6 +- src/main/webapp/file.xhtml | 6 +- 3 files changed, 64 insertions(+), 22 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index a90489be29a..ff904c41cb8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -349,6 +349,16 @@ public void downloadDatasetCitationXML(Dataset dataset) { downloadCitationXML(null, dataset, false); } + public void downloadDatasetCitationXML(DatasetVersion version) { + // DatasetVersion-level citation: + DataCitation citation=null; + citation = new DataCitation(version); + + String fileNameString; + fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + ".xml"; + downloadXML(citation, fileNameString); + } + public void downloadDatafileCitationXML(FileMetadata fileMetadata) { downloadCitationXML(fileMetadata, null, false); } @@ -364,9 +374,6 @@ public void downloadCitationXML(FileMetadata fileMetadata, Dataset dataset, bool } else { citation= new DataCitation(fileMetadata, direct); } - FacesContext ctx = FacesContext.getCurrentInstance(); - HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); - response.setContentType("text/xml"); String fileNameString; if (fileMetadata == null || fileMetadata.getLabel() == null) { // Dataset-level citation: @@ -375,14 +382,21 @@ public void downloadCitationXML(FileMetadata fileMetadata, Dataset dataset, bool // Datafile-level citation: fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + "-" + FileUtil.getCiteDataFileFilename(citation.getFileTitle(), FileUtil.FileCitationExtension.ENDNOTE); } + downloadXML(citation, fileNameString); + } + + public void downloadXML(DataCitation citation, String fileNameString) { + FacesContext ctx = FacesContext.getCurrentInstance(); + HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); + response.setContentType("text/xml"); response.setHeader("Content-Disposition", fileNameString); + try { ServletOutputStream out = response.getOutputStream(); citation.writeAsEndNoteCitation(out); out.flush(); ctx.responseComplete(); } catch (IOException e) { - } } @@ -392,6 +406,16 @@ public void downloadDatasetCitationRIS(Dataset dataset) { } + public void downloadDatasetCitationRIS(DatasetVersion version) { + // DatasetVersion-level citation: + DataCitation citation=null; + citation = new DataCitation(version); + + String fileNameString; + fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + ".ris"; + downloadRIS(citation, fileNameString); + } + public void downloadDatafileCitationRIS(FileMetadata fileMetadata) { downloadCitationRIS(fileMetadata, null, false); } @@ -408,10 +432,6 @@ public void downloadCitationRIS(FileMetadata fileMetadata, Dataset dataset, bool citation= new DataCitation(fileMetadata, direct); } - FacesContext ctx = FacesContext.getCurrentInstance(); - HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); - response.setContentType("application/download"); - String fileNameString; if (fileMetadata == null || fileMetadata.getLabel() == null) { // Dataset-level citation: @@ -420,6 +440,14 @@ public void downloadCitationRIS(FileMetadata fileMetadata, Dataset dataset, bool // Datafile-level citation: fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + "-" + FileUtil.getCiteDataFileFilename(citation.getFileTitle(), FileUtil.FileCitationExtension.RIS); } + downloadRIS(citation, fileNameString); + } + + public void downloadRIS(DataCitation citation, String fileNameString) { + //SEK 12/3/2018 changing this to open the json in a new tab. + FacesContext ctx = FacesContext.getCurrentInstance(); + HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); + response.setContentType("application/download"); response.setHeader("Content-Disposition", fileNameString); try { @@ -431,7 +459,7 @@ public void downloadCitationRIS(FileMetadata fileMetadata, Dataset dataset, bool } } - + private String getFileNameFromPid(GlobalId id) { return id.asString(); } @@ -442,6 +470,16 @@ public void downloadDatasetCitationBibtex(Dataset dataset) { } + public void downloadDatasetCitationBibtex(DatasetVersion version) { + // DatasetVersion-level citation: + DataCitation citation=null; + citation = new DataCitation(version); + + String fileNameString; + fileNameString = "inline;filename=" + getFileNameFromPid(citation.getPersistentId()) + ".bib"; + downloadBibtex(citation, fileNameString); + } + public void downloadDatafileCitationBibtex(FileMetadata fileMetadata) { downloadCitationBibtex(fileMetadata, null, false); } @@ -457,13 +495,7 @@ public void downloadCitationBibtex(FileMetadata fileMetadata, Dataset dataset, b } else { citation= new DataCitation(fileMetadata, direct); } - //SEK 12/3/2018 changing this to open the json in a new tab. - FacesContext ctx = FacesContext.getCurrentInstance(); - HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); - - //Fix for 6029 FireFox was failing to parse it when content type was set to json - response.setContentType("text/plain"); - + String fileNameString; if (fileMetadata == null || fileMetadata.getLabel() == null) { // Dataset-level citation: @@ -472,6 +504,16 @@ public void downloadCitationBibtex(FileMetadata fileMetadata, Dataset dataset, b // Datafile-level citation: fileNameString = "inline;filename=" + getFileNameFromPid(citation.getPersistentId()) + "-" + FileUtil.getCiteDataFileFilename(citation.getFileTitle(), FileUtil.FileCitationExtension.BIBTEX); } + downloadBibtex(citation, fileNameString); + } + + public void downloadBibtex(DataCitation citation, String fileNameString) { + //SEK 12/3/2018 changing this to open the json in a new tab. + FacesContext ctx = FacesContext.getCurrentInstance(); + HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); + + //Fix for 6029 FireFox was failing to parse it when content type was set to json + response.setContentType("text/plain"); response.setHeader("Content-Disposition", fileNameString); try { diff --git a/src/main/webapp/dataset-citation.xhtml b/src/main/webapp/dataset-citation.xhtml index 9baced25be0..4162bfd92e4 100644 --- a/src/main/webapp/dataset-citation.xhtml +++ b/src/main/webapp/dataset-citation.xhtml @@ -33,17 +33,17 @@
  • - + #{bundle['dataset.cite.downloadBtn.xml']}
  • - + #{bundle['dataset.cite.downloadBtn.ris']}
  • - + #{bundle['dataset.cite.downloadBtn.bib']}
    • diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 39525568797..703050315d9 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -155,17 +155,17 @@
    • - + #{bundle['dataset.cite.downloadBtn.xml']}
    • - + #{bundle['dataset.cite.downloadBtn.ris']}
    • - + #{bundle['dataset.cite.downloadBtn.bib']}
      • From e48800d700ec6db0f6407b3b4cb29b00f86f8599 Mon Sep 17 00:00:00 2001 From: okaradeniz Date: Fri, 14 Jul 2023 12:07:52 +0200 Subject: [PATCH 020/427] cite selected dataset version in citation downloads --- .../iq/dataverse/FileDownloadServiceBean.java | 86 ++++++++++++++----- src/main/webapp/dataset-citation.xhtml | 6 +- src/main/webapp/file.xhtml | 6 +- 3 files changed, 70 insertions(+), 28 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index a90489be29a..229a007ac2e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -349,6 +349,16 @@ public void downloadDatasetCitationXML(Dataset dataset) { downloadCitationXML(null, dataset, false); } + public void downloadDatasetCitationXML(DatasetVersion version) { + // DatasetVersion-level citation: + DataCitation citation=null; + citation = new DataCitation(version); + + String fileNameString; + fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + ".xml"; + downloadXML(citation, fileNameString); + } + public void downloadDatafileCitationXML(FileMetadata fileMetadata) { downloadCitationXML(fileMetadata, null, false); } @@ -358,15 +368,12 @@ public void downloadDirectDatafileCitationXML(FileMetadata fileMetadata) { } public void downloadCitationXML(FileMetadata fileMetadata, Dataset dataset, boolean direct) { - DataCitation citation=null; + DataCitation citation=null; if (dataset != null){ - citation = new DataCitation(dataset.getLatestVersion()); + citation = new DataCitation(dataset.getLatestVersion()); } else { citation= new DataCitation(fileMetadata, direct); } - FacesContext ctx = FacesContext.getCurrentInstance(); - HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); - response.setContentType("text/xml"); String fileNameString; if (fileMetadata == null || fileMetadata.getLabel() == null) { // Dataset-level citation: @@ -375,14 +382,21 @@ public void downloadCitationXML(FileMetadata fileMetadata, Dataset dataset, bool // Datafile-level citation: fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + "-" + FileUtil.getCiteDataFileFilename(citation.getFileTitle(), FileUtil.FileCitationExtension.ENDNOTE); } + downloadXML(citation, fileNameString); + } + + public void downloadXML(DataCitation citation, String fileNameString) { + FacesContext ctx = FacesContext.getCurrentInstance(); + HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); + response.setContentType("text/xml"); response.setHeader("Content-Disposition", fileNameString); + try { ServletOutputStream out = response.getOutputStream(); citation.writeAsEndNoteCitation(out); out.flush(); ctx.responseComplete(); } catch (IOException e) { - } } @@ -392,6 +406,16 @@ public void downloadDatasetCitationRIS(Dataset dataset) { } + public void downloadDatasetCitationRIS(DatasetVersion version) { + // DatasetVersion-level citation: + DataCitation citation=null; + citation = new DataCitation(version); + + String fileNameString; + fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + ".ris"; + downloadRIS(citation, fileNameString); + } + public void downloadDatafileCitationRIS(FileMetadata fileMetadata) { downloadCitationRIS(fileMetadata, null, false); } @@ -401,17 +425,13 @@ public void downloadDirectDatafileCitationRIS(FileMetadata fileMetadata) { } public void downloadCitationRIS(FileMetadata fileMetadata, Dataset dataset, boolean direct) { - DataCitation citation=null; + DataCitation citation=null; if (dataset != null){ - citation = new DataCitation(dataset.getLatestVersion()); + citation = new DataCitation(dataset.getLatestVersion()); } else { citation= new DataCitation(fileMetadata, direct); } - FacesContext ctx = FacesContext.getCurrentInstance(); - HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); - response.setContentType("application/download"); - String fileNameString; if (fileMetadata == null || fileMetadata.getLabel() == null) { // Dataset-level citation: @@ -420,6 +440,14 @@ public void downloadCitationRIS(FileMetadata fileMetadata, Dataset dataset, bool // Datafile-level citation: fileNameString = "attachment;filename=" + getFileNameFromPid(citation.getPersistentId()) + "-" + FileUtil.getCiteDataFileFilename(citation.getFileTitle(), FileUtil.FileCitationExtension.RIS); } + downloadRIS(citation, fileNameString); + } + + public void downloadRIS(DataCitation citation, String fileNameString) { + //SEK 12/3/2018 changing this to open the json in a new tab. + FacesContext ctx = FacesContext.getCurrentInstance(); + HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); + response.setContentType("application/download"); response.setHeader("Content-Disposition", fileNameString); try { @@ -431,7 +459,7 @@ public void downloadCitationRIS(FileMetadata fileMetadata, Dataset dataset, bool } } - + private String getFileNameFromPid(GlobalId id) { return id.asString(); } @@ -442,6 +470,16 @@ public void downloadDatasetCitationBibtex(Dataset dataset) { } + public void downloadDatasetCitationBibtex(DatasetVersion version) { + // DatasetVersion-level citation: + DataCitation citation=null; + citation = new DataCitation(version); + + String fileNameString; + fileNameString = "inline;filename=" + getFileNameFromPid(citation.getPersistentId()) + ".bib"; + downloadBibtex(citation, fileNameString); + } + public void downloadDatafileCitationBibtex(FileMetadata fileMetadata) { downloadCitationBibtex(fileMetadata, null, false); } @@ -451,19 +489,13 @@ public void downloadDirectDatafileCitationBibtex(FileMetadata fileMetadata) { } public void downloadCitationBibtex(FileMetadata fileMetadata, Dataset dataset, boolean direct) { - DataCitation citation=null; + DataCitation citation=null; if (dataset != null){ - citation = new DataCitation(dataset.getLatestVersion()); + citation = new DataCitation(dataset.getLatestVersion()); } else { citation= new DataCitation(fileMetadata, direct); } - //SEK 12/3/2018 changing this to open the json in a new tab. - FacesContext ctx = FacesContext.getCurrentInstance(); - HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); - - //Fix for 6029 FireFox was failing to parse it when content type was set to json - response.setContentType("text/plain"); - + String fileNameString; if (fileMetadata == null || fileMetadata.getLabel() == null) { // Dataset-level citation: @@ -472,6 +504,16 @@ public void downloadCitationBibtex(FileMetadata fileMetadata, Dataset dataset, b // Datafile-level citation: fileNameString = "inline;filename=" + getFileNameFromPid(citation.getPersistentId()) + "-" + FileUtil.getCiteDataFileFilename(citation.getFileTitle(), FileUtil.FileCitationExtension.BIBTEX); } + downloadBibtex(citation, fileNameString); + } + + public void downloadBibtex(DataCitation citation, String fileNameString) { + //SEK 12/3/2018 changing this to open the json in a new tab. + FacesContext ctx = FacesContext.getCurrentInstance(); + HttpServletResponse response = (HttpServletResponse) ctx.getExternalContext().getResponse(); + + //Fix for 6029 FireFox was failing to parse it when content type was set to json + response.setContentType("text/plain"); response.setHeader("Content-Disposition", fileNameString); try { diff --git a/src/main/webapp/dataset-citation.xhtml b/src/main/webapp/dataset-citation.xhtml index 9baced25be0..4162bfd92e4 100644 --- a/src/main/webapp/dataset-citation.xhtml +++ b/src/main/webapp/dataset-citation.xhtml @@ -33,17 +33,17 @@
      • - + #{bundle['dataset.cite.downloadBtn.xml']}
      • - + #{bundle['dataset.cite.downloadBtn.ris']}
      • - + #{bundle['dataset.cite.downloadBtn.bib']}
        • diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 39525568797..703050315d9 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -155,17 +155,17 @@
        • - + #{bundle['dataset.cite.downloadBtn.xml']}
        • - + #{bundle['dataset.cite.downloadBtn.ris']}
        • - + #{bundle['dataset.cite.downloadBtn.bib']}
          • From 15e200295818ca3c136ac3a0c57411191038083d Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 26 Jul 2023 16:42:36 +0200 Subject: [PATCH 021/427] #8655 Re-add value 'cell counting' to Life Science metadatablock's Measurement Type field vocabulary --- scripts/api/data/metadatablocks/biomedical.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/api/data/metadatablocks/biomedical.tsv b/scripts/api/data/metadatablocks/biomedical.tsv index 28d59130c34..f26a25752e6 100644 --- a/scripts/api/data/metadatablocks/biomedical.tsv +++ b/scripts/api/data/metadatablocks/biomedical.tsv @@ -45,6 +45,7 @@ studyFactorType Treatment Compound EFO_0000369 17 studyFactorType Treatment Type EFO_0000727 18 studyFactorType Other OTHER_FACTOR 19 + studyAssayMeasurementType cell counting ERO_0001899 0 studyAssayMeasurementType cell sorting CHMO_0001085 1 studyAssayMeasurementType clinical chemistry analysis OBI_0000520 2 studyAssayMeasurementType copy number variation profiling OBI_0000537 3 From d9bb9a343de06cf0c849ee728e2d0b6ee57ccae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 2 Aug 2023 17:45:46 +0200 Subject: [PATCH 022/427] i18n into MyData page --- .../iq/dataverse/mydata/DataRetrieverAPI.java | 29 +++++++-------- .../dataverse/mydata/MyDataFilterParams.java | 24 ++++++------- .../iq/dataverse/mydata/MyDataFinder.java | 35 ++++++++++--------- src/main/java/propertyFiles/Bundle.properties | 15 ++++++++ .../iq/dataverse/api/DataRetrieverApiIT.java | 20 +++++++++-- 5 files changed, 78 insertions(+), 45 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java index d244021d01a..51e1cb70bc7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java @@ -25,13 +25,10 @@ import edu.harvard.iq.dataverse.search.SearchException; import edu.harvard.iq.dataverse.search.SearchFields; import edu.harvard.iq.dataverse.search.SortBy; -import java.math.BigDecimal; import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.logging.Logger; -import java.util.Locale; import javax.ejb.EJB; import javax.inject.Inject; import javax.json.Json; @@ -41,7 +38,6 @@ import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; -import javax.ws.rs.DefaultValue; import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.core.Context; @@ -272,9 +268,7 @@ public String retrieveMyDataAsJsonString( @QueryParam("dataset_valid") List datasetValidities) { boolean OTHER_USER = false; - String localeCode = session.getLocaleCode(); - String noMsgResultsFound = BundleUtil.getStringFromPropertyFile("dataretrieverAPI.noMsgResultsFound", - "Bundle", new Locale(localeCode)); + String noMsgResultsFound = BundleUtil.getStringFromBundle("dataretrieverAPI.noMsgResultsFound"); if ((session.getUser() != null) && (session.getUser().isAuthenticated())) { authUser = (AuthenticatedUser) session.getUser(); @@ -282,7 +276,10 @@ public String retrieveMyDataAsJsonString( try { authUser = getRequestAuthenticatedUserOrDie(crc); } catch (WrappedResponse e) { - return this.getJSONErrorString("Requires authentication. Please login.", "retrieveMyDataAsJsonString. User not found! Shouldn't be using this anyway"); + return this.getJSONErrorString( + BundleUtil.getStringFromBundle("dataretrieverAPI.authentication.required"), + BundleUtil.getStringFromBundle("dataretrieverAPI.authentication.required.opt") + ); } } @@ -295,7 +292,9 @@ public String retrieveMyDataAsJsonString( authUser = searchUser; OTHER_USER = true; } else { - return this.getJSONErrorString("No user found for: \"" + userIdentifier + "\"", null); + return this.getJSONErrorString( + BundleUtil.getStringFromBundle("dataretrieverAPI.user.not.found", Arrays.asList(userIdentifier)), + null); } } @@ -335,8 +334,7 @@ public String retrieveMyDataAsJsonString( myDataFinder = new MyDataFinder(rolePermissionHelper, roleAssigneeService, dvObjectServiceBean, - groupService, - noMsgResultsFound); + groupService); this.myDataFinder.runFindDataSteps(filterParams); if (myDataFinder.hasError()){ return this.getJSONErrorString(myDataFinder.getErrorMessage(), myDataFinder.getErrorMessage()); @@ -391,11 +389,14 @@ public String retrieveMyDataAsJsonString( } catch (SearchException ex) { solrQueryResponse = null; - this.logger.severe("Solr SearchException: " + ex.getMessage()); + logger.severe("Solr SearchException: " + ex.getMessage()); } - if (solrQueryResponse==null){ - return this.getJSONErrorString("Sorry! There was an error with the search service.", "Sorry! There was a SOLR Error"); + if (solrQueryResponse == null) { + return this.getJSONErrorString( + BundleUtil.getStringFromBundle("dataretrieverAPI.solr.error"), + BundleUtil.getStringFromBundle("dataretrieverAPI.solr.error.opt") + ); } // --------------------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java index c378034f951..15d83fe4296 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.search.SearchConstants; import edu.harvard.iq.dataverse.search.SearchFields; +import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -178,26 +179,25 @@ public List getRoleIds(){ } - - private void checkParams(){ - - if ((this.userIdentifier == null)||(this.userIdentifier.isEmpty())){ - this.addError("Sorry! No user was found!"); + private void checkParams() { + if ((this.userIdentifier == null) || (this.userIdentifier.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("myDataFilterParams.error.no.user")); return; } - if ((this.roleIds == null)||(this.roleIds.isEmpty())){ - this.addError("No results. Please select at least one Role."); + if ((this.roleIds == null) || (this.roleIds.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("myDataFilterParams.error.result.no.role")); return; } - if ((this.dvObjectTypes == null)||(this.dvObjectTypes.isEmpty())){ - this.addError("No results. Please select one of Dataverses, Datasets, Files."); + if ((this.dvObjectTypes == null) || (this.dvObjectTypes.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("myDataFilterParams.error.result.no.dvobject")); return; } - - if ((this.publicationStatuses == null)||(this.publicationStatuses.isEmpty())){ - this.addError("No results. Please select one of " + StringUtils.join(MyDataFilterParams.defaultPublishedStates, ", ") + "."); + + if ((this.publicationStatuses == null) || (this.publicationStatuses.isEmpty())) { + this.addError(BundleUtil.getStringFromBundle("dataretrieverAPI.user.not.found", + Arrays.asList(StringUtils.join(MyDataFilterParams.defaultPublishedStates, ", ")))); return; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java index 6acdfd9cdde..8ed2ef830e5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java @@ -11,7 +11,9 @@ import edu.harvard.iq.dataverse.authorization.DataverseRolePermissionHelper; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.search.SearchFields; +import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -47,7 +49,6 @@ public class MyDataFinder { private RoleAssigneeServiceBean roleAssigneeService; private DvObjectServiceBean dvObjectServiceBean; private GroupServiceBean groupService; - private String noMsgResultsFound; //private RoleAssigneeServiceBean roleService = new RoleAssigneeServiceBean(); //private MyDataQueryHelperServiceBean myDataQueryHelperService; // -------------------- @@ -86,12 +87,11 @@ public class MyDataFinder { private List fileGrandparentFileIds = new ArrayList<>(); // dataverse has file permissions - public MyDataFinder(DataverseRolePermissionHelper rolePermissionHelper, RoleAssigneeServiceBean roleAssigneeService, DvObjectServiceBean dvObjectServiceBean, GroupServiceBean groupService, String _noMsgResultsFound) { + public MyDataFinder(DataverseRolePermissionHelper rolePermissionHelper, RoleAssigneeServiceBean roleAssigneeService, DvObjectServiceBean dvObjectServiceBean, GroupServiceBean groupService) { this.rolePermissionHelper = rolePermissionHelper; this.roleAssigneeService = roleAssigneeService; this.dvObjectServiceBean = dvObjectServiceBean; this.groupService = groupService; - this.noMsgResultsFound = _noMsgResultsFound; this.loadHarvestedDataverseIds(); } @@ -213,7 +213,7 @@ private List getSolrFilterQueries(boolean totalCountsOnly){ // ----------------------------------------------------------------- String dvObjectFQ = this.getSolrDvObjectFilterQuery(); if (dvObjectFQ ==null){ - this.addErrorMessage(noMsgResultsFound); + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.empty")); return null; } filterQueries.add(dvObjectFQ); @@ -286,7 +286,7 @@ public String getSolrDvObjectFilterQuery(){ if ((distinctEntityIds.isEmpty()) && (distinctParentIds.isEmpty())) { - this.addErrorMessage(noMsgResultsFound); + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.empty")); return null; } @@ -430,24 +430,25 @@ public JsonArrayBuilder getListofSelectedRoles(){ } - private boolean runStep1RoleAssignments(){ + private boolean runStep1RoleAssignments() { List results = this.roleAssigneeService.getAssigneeAndRoleIdListFor(filterParams); //logger.info("runStep1RoleAssignments results: " + results.toString()); - if (results == null){ - this.addErrorMessage("Sorry, the EntityManager isn't working (still)."); + if (results == null) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.null")); return false; - }else if (results.isEmpty()){ + } else if (results.isEmpty()) { List roleNames = this.rolePermissionHelper.getRoleNamesByIdList(this.filterParams.getRoleIds()); - if ((roleNames == null)||(roleNames.isEmpty())){ - this.addErrorMessage("Sorry, you have no assigned roles."); - }else{ - if (roleNames.size()==1){ - this.addErrorMessage("Sorry, nothing was found for this role: " + StringUtils.join(roleNames, ", ")); - }else{ - this.addErrorMessage("Sorry, nothing was found for these roles: " + StringUtils.join(roleNames, ", ")); + if ((roleNames == null) || (roleNames.isEmpty())) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.no.role")); + } else { + final List args = Arrays.asList(StringUtils.join(roleNames, ", ")); + if (roleNames.size() == 1) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.role.empty", args)); + } else { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.roles.empty", args)); } } return false; @@ -497,7 +498,7 @@ private boolean runStep2DirectAssignments(){ List results = this.dvObjectServiceBean.getDvObjectInfoForMyData(directDvObjectIds); //List results = this.roleAssigneeService.getAssignmentsFor(this.userIdentifier); if (results.isEmpty()){ - this.addErrorMessage("Sorry, you have no assigned Dataverses, Datasets, or Files."); + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.no.dvobject")); return false; } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 64a1c626f13..a01d175d903 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2781,7 +2781,22 @@ passwdVal.passwdReq.lowercase=lowercase passwdVal.passwdReq.letter=letter passwdVal.passwdReq.numeral=numeral passwdVal.passwdReq.special=special +#mydata API (DataRetriverAPI.java and MyDataFinder.java) dataretrieverAPI.noMsgResultsFound=Sorry, no results were found. +dataretrieverAPI.authentication.required=Requires authentication. Please login. +dataretrieverAPI.authentication.required.opt=retrieveMyDataAsJsonString. User not found! Shouldn't be using this anyway. +dataretrieverAPI.user.not.found=No user found for: "{0}" +dataretrieverAPI.solr.error=Sorry! There was an error with the search service. +dataretrieverAPI.solr.error.opt=Sorry! There was a SOLR Error. +myDataFilterParams.error.no.user=Sorry! No user was found! +myDataFilterParams.error.result.no.role=No results. Please select at least one Role. +myDataFilterParams.error.result.no.dvobject=No results. Please select one of Dataverses, Datasets, Files. +myDataFilterParams.error.result.no.publicationStatus=No results. Please select one of {0}. +myDataFinder.error.result.null=Sorry, the EntityManager isn't working (still). +myDataFinder.error.result.no.role=Sorry, you have no assigned roles. +myDataFinder.error.result.role.empty=Sorry, nothing was found for this role: {0} +myDataFinder.error.result.roles.empty=Sorry, nothing was found for these roles: {0} +myDataFinder.error.result.no.dvobject=Sorry, you have no assigned Dataverses, Datasets, or Files. #xlsxfilereader.java xlsxfilereader.ioexception.parse=Could not parse Excel/XLSX spreadsheet. {0} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java index 69d5db08744..fd0952fd336 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java @@ -3,10 +3,14 @@ import com.jayway.restassured.RestAssured; import com.jayway.restassured.response.Response; import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism; +import edu.harvard.iq.dataverse.util.BundleUtil; + import org.junit.BeforeClass; import org.junit.Test; import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import static javax.ws.rs.core.Response.Status.OK; import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; @@ -15,6 +19,8 @@ public class DataRetrieverApiIT { + private static final String ERR_MSG_FORMAT = "{\"success\":false,\"error_message\":\"%s\"}"; + @BeforeClass public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); @@ -35,14 +41,24 @@ public void testRetrieveMyDataAsJsonString() { String badUserIdentifier = "bad-identifier"; Response invalidUserIdentifierResponse = UtilIT.retrieveMyDataAsJsonString(superUserApiToken, badUserIdentifier, emptyRoleIdsList); - assertEquals("{\"success\":false,\"error_message\":\"No user found for: \\\"" + badUserIdentifier + "\\\"\"}", invalidUserIdentifierResponse.prettyPrint()); + assertEquals(prettyPrintError("dataretrieverAPI.user.not.found", Arrays.asList(badUserIdentifier)), invalidUserIdentifierResponse.prettyPrint()); assertEquals(OK.getStatusCode(), invalidUserIdentifierResponse.getStatusCode()); // Call as superuser with valid user identifier Response createSecondUserResponse = UtilIT.createRandomUser(); String userIdentifier = UtilIT.getUsernameFromResponse(createSecondUserResponse); Response validUserIdentifierResponse = UtilIT.retrieveMyDataAsJsonString(superUserApiToken, userIdentifier, emptyRoleIdsList); - assertEquals("{\"success\":false,\"error_message\":\"Sorry, you have no assigned roles.\"}", validUserIdentifierResponse.prettyPrint()); + assertEquals(prettyPrintError("myDataFinder.error.result.no.role", null), validUserIdentifierResponse.prettyPrint()); assertEquals(OK.getStatusCode(), validUserIdentifierResponse.getStatusCode()); } + + private static String prettyPrintError(String resourceBundleKey, List params) { + final String errorMessage; + if (params == null || params.isEmpty()) { + errorMessage = BundleUtil.getStringFromBundle(resourceBundleKey); + } else { + errorMessage = BundleUtil.getStringFromBundle(resourceBundleKey, params); + } + return String.format(ERR_MSG_FORMAT, errorMessage.replaceAll("\"", "\\\\\"")); + } } From 697f081c7dc896344199818bf0ea09e4cc30558d Mon Sep 17 00:00:00 2001 From: Steven Ferey Date: Wed, 23 Aug 2023 16:39:44 +0200 Subject: [PATCH 023/427] Update src/main/java/propertyFiles/Bundle.properties Co-authored-by: Philip Durbin --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b9651d0c36e..892dbd361da 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2800,7 +2800,7 @@ dataretrieverAPI.authentication.required=Requires authentication. Please login. dataretrieverAPI.authentication.required.opt=retrieveMyDataAsJsonString. User not found! Shouldn't be using this anyway. dataretrieverAPI.user.not.found=No user found for: "{0}" dataretrieverAPI.solr.error=Sorry! There was an error with the search service. -dataretrieverAPI.solr.error.opt=Sorry! There was a SOLR Error. +dataretrieverAPI.solr.error.opt=Sorry! There was a Solr Error. myDataFilterParams.error.no.user=Sorry! No user was found! myDataFilterParams.error.result.no.role=No results. Please select at least one Role. myDataFilterParams.error.result.no.dvobject=No results. Please select one of Dataverses, Datasets, Files. From f146f08d5cbea067572eb25350b10dcca4c0ee8d Mon Sep 17 00:00:00 2001 From: sferey Date: Wed, 23 Aug 2023 17:01:03 +0200 Subject: [PATCH 024/427] changed translation myDataFinder.error.result.null --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 892dbd361da..bdebc4a20f0 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2805,7 +2805,7 @@ myDataFilterParams.error.no.user=Sorry! No user was found! myDataFilterParams.error.result.no.role=No results. Please select at least one Role. myDataFilterParams.error.result.no.dvobject=No results. Please select one of Dataverses, Datasets, Files. myDataFilterParams.error.result.no.publicationStatus=No results. Please select one of {0}. -myDataFinder.error.result.null=Sorry, the EntityManager isn't working (still). +myDataFinder.error.result.null=Sorry, the authenticated user ID could not be retrieved. myDataFinder.error.result.no.role=Sorry, you have no assigned roles. myDataFinder.error.result.role.empty=Sorry, nothing was found for this role: {0} myDataFinder.error.result.roles.empty=Sorry, nothing was found for these roles: {0} From 685bc8be429e6f65765e66668de372ee2c8bb2b5 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Fri, 1 Sep 2023 14:09:06 -0400 Subject: [PATCH 025/427] xslt --- .../edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 7739408e408..9076680754e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -2123,13 +2123,7 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { - //File xsltfile = new File("/home/victoria/fop-2.8/fop/ddi-to-fo.xsl"); - //URL resource = DdiExportUtil.class.getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); - //File xsltfile = new File(resource.toURI()); InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); - logger.info("start datasetPdfDDI"); - //InputStream xsltfile = DdiExportUtil.class.getClassLoader().getResourceAsStream( - // "edu/harvard/iq/dataverse/from-ddi-2.5/ddi-to-fo.xsl"); final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); FOUserAgent foUserAgent = fopFactory.newFOUserAgent(); @@ -2153,7 +2147,6 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream transformer.transform(src, res); } catch (Exception e) { - logger.info("First try"); logger.severe(e.getMessage()); } } catch (Exception e) { From 12c463a8fa41c44ee3a94926299ceffaa7d6a114 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Tue, 5 Sep 2023 12:19:17 -0400 Subject: [PATCH 026/427] inc --- .../dataverse/export/PdfCodeBookExporter.java | 36 +- .../dataverse/export/ddi/DdiExportUtil.java | 7 +- .../harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl | 5 + .../ddi-pdf/messages_en.properties.xml | 174 +++++++ .../ddi-pdf/messages_es.properties.xml | 170 +++++++ .../ddi-pdf/messages_fr.properties.xml | 173 +++++++ .../ddi-pdf/messages_ja.properties.xml | 161 +++++++ .../ddi-pdf/messages_nn.properties.xml | 174 +++++++ .../ddi-pdf/messages_no.properties.xml | 174 +++++++ .../ddi-pdf/messages_ru.properties.xml | 169 +++++++ .../edu/harvard/iq/dataverse/ddi-to-fo.xsl | 448 +++++++++--------- 11 files changed, 1446 insertions(+), 245 deletions(-) create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml create mode 100644 src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml diff --git a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java index e0d5171e30c..4772c09ffd5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/PdfCodeBookExporter.java @@ -3,25 +3,23 @@ package edu.harvard.iq.dataverse.export; import com.google.auto.service.AutoService; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; - import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; - import io.gdcc.spi.export.ExportDataProvider; - import io.gdcc.spi.export.ExportException; - import io.gdcc.spi.export.Exporter; - import edu.harvard.iq.dataverse.util.BundleUtil; - - import javax.json.JsonObject; - import javax.ws.rs.core.MediaType; - import javax.xml.stream.XMLStreamException; - import java.io.File; - import java.io.IOException; - import java.io.InputStream; - import java.io.OutputStream; - import java.nio.file.Path; - import java.nio.file.Paths; - import java.util.Locale; - import java.util.Optional; + +import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; +import io.gdcc.spi.export.ExportDataProvider; +import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.Exporter; +import edu.harvard.iq.dataverse.util.BundleUtil; +import jakarta.ws.rs.core.MediaType; + +import javax.xml.stream.XMLStreamException; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Locale; +import java.util.Optional; @AutoService(Exporter.class) public class PdfCodeBookExporter implements Exporter { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 9ed89a6fe94..aab728c7773 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -76,6 +76,7 @@ import org.apache.fop.apps.Fop; import org.apache.fop.apps.FopFactory; import org.apache.fop.apps.MimeConstants; +import java.io.File; public class DdiExportUtil { @@ -2108,6 +2109,7 @@ private static boolean checkParentElement(XMLStreamWriter xmlw, String elementNa public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream) throws XMLStreamException { try { + String sysId = DdiExportUtil.class.getClassLoader().getResource("edu/harvard/iq/dataverse/ddi-to-fo.xsl").toURI().toString(); InputStream styleSheetInput = DdiExportUtil.class.getClassLoader().getResourceAsStream("edu/harvard/iq/dataverse/ddi-to-fo.xsl"); final FopFactory fopFactory = FopFactory.newInstance(new File(".").toURI()); @@ -2117,7 +2119,9 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream Fop fop = fopFactory.newFop(MimeConstants.MIME_PDF, foUserAgent, outputStream); // Setup XSLT TransformerFactory factory = TransformerFactory.newInstance(); - Transformer transformer = factory.newTransformer(new StreamSource(styleSheetInput)); + Source mySrc = new StreamSource(styleSheetInput); + mySrc.setSystemId(sysId); + Transformer transformer = factory.newTransformer(mySrc); // Set the value of a in the stylesheet transformer.setParameter("versionParam", "2.0"); @@ -2135,7 +2139,6 @@ public static void datasetPdfDDI(InputStream datafile, OutputStream outputStream logger.severe(e.getMessage()); } } catch (Exception e) { - logger.info("Second try"); logger.severe(e.getMessage()); } } diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl new file mode 100644 index 00000000000..edf876f3b04 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/i18n.inc.xsl @@ -0,0 +1,5 @@ + + + + + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml new file mode 100644 index 00000000000..d8e98dfd3c6 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_en.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Valid +Frequency table not shown +Derivation +discrete +Data Collection Mode +Other Processing +Other Acknowledgment(s) +Untitled +Identification +Click here to access/export data files from Nesstar format +Value +Percentage +Weighting +Primary Investigator(s) +This document was generated using the +Sampling +Cases +Access Conditions +Source +Modalities +Rights & Disclaimer +Definition +Estimates of Sampling Error +Data Files +Geographic Coverage +April +Mean +Metadata Production +Data Collection +Missing Data +Scripts and programs +Variable(s) +Interviewer instructions +Funding Agency/ies +November +Missing +Version +Universe +Contributor(s) +Access Authority +Data Processing & Appraisal +Scope +Administrative documents +StdDev +Contact(s) +Label +Technical documents +Decimal(s) +Type +Literal question +Concepts +Range +Abstract +June +Supervision +Other Forms of Data Appraisal +References +Accessibility +Data Collection Dates +Data Editing +Questionnaires +Valid case(s) +Reports and analytical documents +Copyright +Documentation +Deviations from Sample Design +Publisher(s) +February +Dataset contains +Acknowledgment(s) +Continuous +Standard deviation +Variables Description +Producer +Production Date + +The Explorer allows you to view data files and export them to common statistical formats +Discrete +Group +July +Filename +Cases +Name +Warning: these figures indicate the number of cases found in the data file. They cannot be interpreted as summary statistics of the population of interest. +Statistical tables +December +Subjects +Processing Checks +software +Interviewer's instructions +Table of Contents +Document Information +Subgroup(s) +Keywords +group(s) +W +Weight +Files Description +Notes +Data Collection Notes +file(s) +continuous +Disclaimer +Content +variable(s) +Other Producer(s) +Producers & Sponsors +Data Cleaning Notes +Distributor(s) +Overview +Citation Requirements +September +Category +Confidentiality +Statistics +May +Undetermined +Structure +file +Pre-question +Response Rate +Width +Recoding and Derivation +Series +October +Unit of Analysis +Data Processing Notes +Kind of Data +File +Time Period(s) +File Content +Invalid +Vars +cont. +Key(s) +Question +Source of information +Imputation +Security +To open this file, you will need the free +Other resources +Data Dictionnary +Information +January +Other documents +Minimum +Scope & Coverage +Metadata Producer(s) +Show more info +Data Collector(s) +Post-question +Topics +Sampling Procedure +File Structure +Variables List +Format +Sampling Notes +Variables Group(s) +Description +Categories +Maximum +Depositor(s) +August +NW +Cover Page +Weighted +March + total - showing a subset of +Countries +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml new file mode 100644 index 00000000000..9cfcdaf6e7e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_es.properties.xml @@ -0,0 +1,170 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Válido +No se presentan las tablas de frecuencias +Derivación +discreta +Método de Recolección +Otros relacionados al procesamiento +Otros Reconocimientos +Sin título +Identificación +Presione aquí para acceder/exportar al archivo(s) de datos +Valor +Porcentaje +Ponderando +Investigadores Principales +Este documento fue producido utilizando el +Muestreo +Casos +Condiciones de uso +Fuente +Modalidades +Derechos y Notas Legales +Definición +Estimaciones del Error Muestral +Archivo de Datos +Cobertura Geográfica +Abril +Media +Producción de los Metadatos +Recolección de Datos +Datos perdidos +Programas informáticos +Variable(s) +Manual del encuestador +Agencias Auspiciadoras +Noviembre +Valores perdidos +Versión +Universo +Contribuidor(es) +Institución Propietaria +Tratamiento y Validación de Datos +Dominio Temático +Documentos Administrativos +Desviación +Contacto(s) +Etiqueta +Documentos Técnicos +Decimal(es) +Tipo +Pregunta textual +Conceptos +Rango +Resumen +Junio +Supervisión +Otras Formas de Validación de los Datos +Referencias +Accesibilidad +Fechas de Recolección de Datos +Procesamiento de Datos +Cuestionarios +Casos válidos +Reportes y documentos analíticos +Derechos de Autor +Documentación +Modificaciones al Diseño Muestral +Editor(es) +Febrero +Contenido de la Base de Datos +Reconocimiento(s) +Contínua +Desviación estándar +Descripción de la variable +Productor +Fecha de Producción +El Explorador NESSTAR permite visualizar los archivos de datos y exportarlos a diferentes formatos estadísticos +Discreta +Grupo +Julio +Nombre del Archivo +Casos +Nombre +Cuadros estadísticos +Diciembre +Temas +Controles de Tratamiento +software +Manual del encuestador +Indice +Información acerca de la Documentación +Subgrupo(s) +Palabra Clave +grupo(s) +P +Ponderador +Descripción de los Archivos +Notas +Notas sobre la Recolección de Datos +archivo(s) +continua +Nota Legal +Contenido +variable(s) +Otros Productores +Productores y Auspiciadores +Notas acerca de la Depuración de los Datos +Distribuidor(es) +Resumen General +Forma de citar +Septiembre +Categoría +Confidencialidad +Estadística + +Mayo +Indeterminado +Estructura +archivo +Pre-pregunta +Tasa de Respuesta +Ancho +Recodificación y Derivación +Series +Octubre +Unidad de Análisis +Notas sobre el Procesamiento de Datos +Tipo de Datos +Archivo +Periodo de Referencia +Contenido del Archivo +Inválido +Vars. +cont. +Clave(s) +Pregunta +Fuente de información +Imputación +Seguridad +Para abrir este archivo se necesita el software gratuito +Otros recursos +Diccionario de Datos +Información +Enero +Otros documentos +Mínimo +Cobertura y Dominio Temático +Productor de los Metadatos +Mostrar más información +Entrevistador(es) +Pos-pregunta +Temas +Procedimiento de Muestreo +Estructura del Archivo +Lista de variables +Formato +Notas sobre el Muestreo +Grupo(s) de Variables +Descripción +Categorías +Máximo +Depositante(s) +Agosto +NP +Carátula +Ponderado +Marzo + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml new file mode 100644 index 00000000000..9fa4d2178b1 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_fr.properties.xml @@ -0,0 +1,173 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Valide +Tableau de fréquences non-affiché +Mode de calcul +discrète +Méthode de collecte +Autre traitement +Autre(s) remerciement(s) +Sans titre +Identification +Cliquer ici pour accéder/exporter les fichiers de données du format Nesstar +Valeur +Pourcentage +Pondération +Enquêteur(s) principal/aux +Ce document a été généré à l'aide du +Echantillonage +Enregistrements +Conditions d'accès +Source +Modalités +Responsabilité et droits d'auteurs +Définition +Estimation des erreurs d'échantillonage +Fichiers de données +Couverture géographique +Avril +Moyenne +Production des méta-données +Collecte des données +Valeures manquantes +Programmes informatiques +Variable(s) +Instructions aux enquêteurs +Financement +Novembre +Manquant +Version +Univers +Contributeur(s) +Agence propriétaire +Traitement et évaluation des données +Domaines thématiques +Documents administratifs +Ecart type +Contact(s) +Libellé +Documents techniques +Décimale(s) +Type +Formulation de la question +Concepts +Gamme +Résumé +Juin +Supervision +Autres formes d'évaluation des données +Références +Accessibilité +Dates de collecte +Edition des données +Questionnaires +Cas valide(s) +Rapports et documents analytiques +Droits d'auteurs +Documentation +Déviations par rapport à l'échantillon initial +Editeur(s) +Février +Le jeu de données contient +Remerciement(s) +Continue +Ecart type +Description des variables +Producteur +Date de production + +L'Explorer vous permet d'accéder aux données et de les exporter vers les formats statistiques les plus courants +Discrète +Groupe +Juillet +Nom du fichier +Enreg. +Nom +Avertissement: Ces chiffres indiquent le nombre de cas identifiés dans le fichier de données. Ils ne peuvent pas être interpretés comme étant représentatifs de la population concernée. +Tableaux statistiques +Décembre +Sujets +Contrôles de traitement + +Instructions aux enquêteurs +Table des matières +Informations sur le document +Sous-groupe(s) +Mots-clé +groupe(s) +P +Pondération +Description des fichiers +Notes +Notes sur la collecte +fichier(s) +continue +Responsabilité(s) +Contenu +variable(s) +Autre(s) producteur(s) +Producteurs et sponsors +Notes sur l'apurement des données +Distributeur(s) +Aperçu +Citation +Septembre +Catégorie +Confidentialité +Statistiques +Mai +Indéterminé +Structure +fichier +Pré-question +Taux de réponse +Taille +Formulation de la question +Recodage et dérivation +Série +Octobre +Unité d'analyse +Notes sur le traitement des données +Type d'étude +Fichier +Période(s) de référence +Contenu du fichier +Non-valide +Vars +suite +Clé(s) +Question +Source d'information +Imputation +Sécurité +Pour ouvrir ce fichier, vous avez besoin du logiciel gratuit +Autres resources +Dictionnaire des variables +Information +Janvier +Autres documents +Minimum +Domaines thématiques et couverture +Producteur(s) des méta-données +Information complémentaire +Enquêteurs +Post-question +Thèmes +Procédure d'échantillonage +Structure du fichier +Liste des variables +Format +Notes sur l'échantillonage +Groupe(s) de variables +Description +Catégories +Maximum +Dépositaire(s) +Août +NP +Couverture +Pondéré +Mars +question details + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml new file mode 100644 index 00000000000..bc5dbb06154 --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ja.properties.xml @@ -0,0 +1,161 @@ + + +Generated by Properties2Xml on Tue Feb 13 13:55:43 EST 2007 +有効な +度数表(Frequency table)は表示されません +由来 +不連続な +データ収集モード +その他の確認事項 +識別番号 +データファイルにアクセスするにはここをクリックしてください +無題 +値 +割合 +ウェイティング +第一次調査官 +この文書はToolkitを使用して作られました +サンプリング +ケース +アクセス条件 +情報源 +様相 +権利及び声明文 +定義 +サンプルエラーの見積もり +データファイル +地理的な適用範囲 +４月 +平均 +メタデータ製作 +データ収集 +損失データ +スクリプトおよびプログラム +可変的 +面接者の指示 +出資機関 +１１月 +バージョン +共通の +貢献者 +アクセス権限 +データ処理、評価 +範囲, 領域 +管理用文章 +連絡先 +ラベル +技術的な文書 +小数点 +タイプ +文字の質問 +概念 +範囲 +要約 +６月 +監督 +その他ファーマットのデータ評価 +参照 +アクセス、入手法 +データ収集日 +データ編集 +質問 +レポートおよび分析的な文書 +有効な場合 +コピーライト +書類 +サンプルデザインによる偏差 +発行者 +２月 +データセットに含まれる +確認事項 +連続的な +標準偏差 +変数の記述 +製作者 +製作日 +” Explorer”によってデータファイルを参照することも一般的に使えわれている統計データフォーマットに変換。抽出することも可能です +不連続性 +グループ +７月 +ファイルの名前 +ケース +名前 +統計表 +１２月 +主題, 内容 +工程監査 +ソフト +面接者への指示 +目録 +書類の情報 +サブグループ +キーワード + +グループ +ウェイト +ファイルの詳細 +メモ +データ収集メモ +ファイル +継続的な +声明文 +内容 +変数 +その他の製作者 +製作者とスポンサー +データクリーニングメモ +分配者 +概略 +引用する場合の必要条件 +９月 +カテゴリー +機密性、コンフィデンシャリティー +５月 +未定 +構造 +ファイル +調査前の質問 +回答比率 +幅 +記録と誘導 +シリー +１０月 +分析の単位 +データ処理メモ +データの種類 + +ファイル +期間 +ファイルの内容 +無効 +キー +情報源 +非難 +セキュリティー +このファイルを開けるには、無料で配布されているNesstar Explorer が必要です。 +その他の資料 +データ辞典 +情報 +１月 +その他の書類 +最小値 +規模及び適用範囲 +メタデータ製作者 +さらにインフォメーションを表示 +データ収集者 +調査後の質問 +サンプリングの手順 +ファイルの構造 +変数のリスト +フォーマット +サンプリングメモ +変数のグループ +詳細 +カテゴリー +最大値 +デポジター、提供者、供託者 +８月 +表紙 +ウェイトされた +３月 + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_nn.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml new file mode 100644 index 00000000000..fdf14f5dfcd --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_no.properties.xml @@ -0,0 +1,174 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:39 EDT 2008 +Gyldige +Frekvenstabell ikke vist +Avledning +diskret +Type datainnsamling +Annen prosessering +Andre identifikatorer og krediteringer +Uten tittel +Identifisering +Click here to access/export data files from Nesstar format +Verdi +Prosent +Vekting +Primary Investigator(s) +Dette dokumentet ble generert av +Utvalg +Enheter +Tilgangsbetingelser +Kilde +Modaliteter +Rights & Disclaimer +Definisjon +Estimert utvalgsfeil +Datafiler +Geografisk omfang +April +Mean +Metadata-produksjon +Datainnsamling +Manglende data +Script og programmer +Variable(r) +Instruksjoner til intervjueren +Sponsor/finansierende institusjon(er) +November +Missing +Versjon +Univers +Bidragsyter(e) +Tilgangskontrollør +Dataprosessering og -evaluering +Omfang +Administrative dokumenter +Standardavvik +Kontaktperson(er) +Merkelapp +Tekniske dokumenter +Desimal(er) +Type +Spørsmålstekst +Begrep(er) +Rekkevidde +Sammendrag +Juni +Supervision +Andre former for dataevaluering +Referanser +Tilgjengelighet +Datainnsamlingsdatoer +Dataredigering +Spørreskjema +Gyldige enheter +Rapporter og analysedokumenter +Copyright +Dokumentasjon +Avvik fra utvalgsdesign +Utgiver(e) +Februar +Datasettet inneholder +Krediteringer +Kontinuerlig +Standardavvik +Variabelbeskrivelse +Produsent +Produksjonsdato + +The Explorer allows you to view data files and export them to common statistical formats +Diskret +Gruppe +Juli +Filnavn +Enheter +Navn +Advarsel: disse tallene indikerer antall enheter (cases) i datafilen. De kan ikke tolkes som oppsummert statistikk for populasjonen. +Statistiske tabeller +Desember +Emner +Prosesseringssjekk +programvare +Instruksjoner til intervjueren +Innholdsfortegnelse +Dokumentinformasjon +Undergruppe(r) +Nøkkelord +gruppe(r) +W +Vekt +Filbeskrivelse +Kommentarer +Datainnsamlingskommentarer +file(r) +kontinuerlig +Fraskrivelse +Innhold +variable(r) +Andre produsenter +Produsenter og sponsorer +Kommentarer om datarensing +Distributør(er) +Oversikt +Sitatkrav +September +Kategori +Konfidensialitet +Statistikk +Mai +Uavklart +Struktur +fil +Tekst før spørsmål +Responsrate +Bredde +Omkodinger og utledninger +Serie +Oktober +Analyseenhet +Dataprosesseringskommentarer +Datatype +Fil +Tidsperiode(r) +Filinnhold +Ugyldig +Variabler +kont. +Nøkler +Spørsmål +Kilde for informasjon +Imputasjon +Sikkerhet +For å åpne denne filen trenger du følgende gratisverktøy +Andre ressurser +Dataordbok +Informasjon +Januar +Andre dokumenter +Minimum +Omfang +Metadataprodusenter +Vis mer informasjon +Datainnsamler(e) +Tekst etter spørsmål +Emner +Utvalgsprosedyre +Filstruktur +Variabelliste +Format +Utvalgskommentarer +Variabelgrupper +Beskrivelse +Kategorier +Maksimum +Utgiver(e) +August +NW +Forside +Vektet +Mars + total - viser et utvalg av +Land +spørsmålsdetaljer + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml new file mode 100644 index 00000000000..06fde85af5e --- /dev/null +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-pdf/messages_ru.properties.xml @@ -0,0 +1,169 @@ + + +Generated by Properties2Xml on Fri Apr 11 09:45:40 EDT 2008 +Валидный +Частотная таблица не выводится +Расчет +дискретная +Способ сбора данных +Прочая обработка +Другие участники +Безымянный +Индентификация +Щелкните здесь, чтобы получить доступ к файлам или экспортировать их +Значение +Процент +Взвешивание +Первичный(е) исследователь(и) +Документ был сгенерирован с использованием +Выборка +Наблюдения +Условия доступа +Источник +Модальности +Авторские права и ограничения ответственности +Определение +Оценки ошибок выборки +Файлы данных +Географический охват +Апрель +Среднее +Разработка метаданных +Сбор данных +Пропущенные данные +Скрипты и программы +Переменная(ые) +Инструкции интервьюеру +Кто финансировал +Ноябрь +Пропущенные +Версия +Генеральная совокупность +Участник(и) +Права доступа +Обработка и инспекция данных +Охват +Административные документы +СтдОткл +Контак(ы) +Метка +Технические документы +Десятичные +Тип +Текст вопроса +Концепции +Диапазон +Резюме +Июнь +Контроль +Другие формы инспекции данных +Установки +Доступность +Даты сбора данных +Редактирование данных +Вопросники +Валидное(ые) наблюдение(я) +Отчеты и аналитические документы +Авторские права +Документация +Отклонения от дизайна выборки +Издатель(и) +Февраль +Набор данных содержит +Участник(и) +Непрерывная +Стандартное отклонение +Описание переменных +Разработчик +Дата разработки +Проводник позволяет просматривать файлы данных и экспортировать их в распространенные статистические форматы +Дикретная +Группа +Июль +Имя файла +Наблюдения +Имя +Статистичсекие таблицы +Декабрь +Темы +Контроль обработки +программное обеспечение +Инструкции интервьюеру +Оглавление +Информация о документе +Подгруппа(ы) +Ключевые слова +группа(ы) +B +Вес +Описание файла +Примечания +Примечания по сбору данных +файл(ы) +непрерывная +Ограничения ответственности +Содержание +переменная(ые) +Другие разработчики +Разработчики и спонсоры +Примечания по чистке данных +Дистрибьютор(ы) +Обзор +Требования по цитированию +Сентябрь +Категория +Конфиденциальность +Статистики +Май +Неопределенный +Структура +файл +Текст, предваряющий вопрос +Доля ответов +Ширина +Перекодировка и расчеты +Серия +Октябрь +Единица анализа +Примечания по обработке данных +Тип данных +Файл +Период(ы) времени +Содержание файла +Некорректный +Переменные +непр. +Ключ(и) +Вопрос +Источник информации +Импутация +Безопасность +Чтобы открыть этот файл, необходимо иметь свободным +Прочие источники +Словарь данных +Информация +Январь +Прочие документы +Минимум +Охват и покрытие +Разработчик(и) метаданных +Показать дополнительную информацию +Кто собирал данные +Текст после вопроса +Разделы +Процедура выборки +Структура файла +Список переменных +Формат +Примечания по выборке +Группа(ы) переменных +Описание +Категории +Максимум +Депозитор(ы) +Август +HB +Титульный лист +взвешенные +Март + diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index 1c03d5caf34..c25c76246d6 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -66,8 +66,7 @@ Report optional text --> - - + @@ -110,7 +109,7 @@ - + @@ -124,6 +123,7 @@ Times + - 1 + 1 0 @@ -381,21 +381,21 @@ - + - + - + @@ -403,33 +403,33 @@ - + - + - + - + - + @@ -437,21 +437,21 @@ + select="$msg/*/entry[@key='Data_Processing_and_Appraisal']"/> - + - + @@ -461,7 +461,7 @@ - + @@ -486,7 +486,7 @@ - + @@ -503,7 +503,7 @@ - + @@ -528,7 +528,7 @@ - + @@ -569,7 +569,7 @@ - + @@ -642,7 +642,7 @@ - + @@ -657,7 +657,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Metadata_Producers']"/> + select="$msg/*/entry[@key='Production_Date']"/> + select="$msg/*/entry[@key='Version']"/> + select="$msg/*/entry[@key='Identification']"/> - + @@ -745,7 +745,7 @@ - + @@ -764,14 +764,14 @@ - + - + @@ -782,7 +782,7 @@ + select="$msg/*/entry[@key='Scope_and_Coverage']"/> @@ -793,7 +793,7 @@ + select="$msg/*/entry[@key='Producers_and_Sponsors']"/> @@ -803,7 +803,7 @@ - + @@ -813,7 +813,7 @@ - + @@ -825,7 +825,7 @@ internal-destination="data-processing-and-appraisal" text-decoration="underline" color="blue"> + select="$msg/*/entry[@key='Data_Processing_and_Appraisal']"/> @@ -836,7 +836,7 @@ - + @@ -847,7 +847,7 @@ + select="$msg/*/entry[@key='Rights_and_Disclaimer']"/> @@ -858,7 +858,7 @@ + select="$msg/*/entry[@key='Files_Description']"/> @@ -891,7 +891,7 @@ - + @@ -924,7 +924,7 @@ - + @@ -949,7 +949,7 @@ + select="$msg/*/entry[@key='Variables_Description']"/> @@ -982,7 +982,7 @@ - + @@ -1002,7 +1002,7 @@ - + @@ -1065,7 +1065,7 @@ - + @@ -1078,7 +1078,7 @@ - + + select="$msg/*/entry[@key='Identification']"/> - @@ -1130,7 +1130,7 @@ select="/ddi:codeBook/ddi:stdyDscr/ddi:citation/ddi:verStmt/ddi:version"> : + select="$msg/*/entry[@key='Production_Date']"/>: @@ -1139,7 +1139,7 @@ - @@ -1156,7 +1156,7 @@ - + - + select="$msg/*/entry[@key='Kind_of_Data']"/> + select="$msg/*/entry[@key='Unit_of_Analysis']"/> + select="$msg/*/entry[@key='Scope_and_Coverage']"/> @@ -1257,7 +1257,7 @@ - + - @@ -1299,7 +1299,7 @@ - + - + @@ -1344,7 +1344,7 @@ - + @@ -1361,7 +1361,7 @@ border="{$default-border}" padding="{$cell-padding}"> - @@ -1422,7 +1422,7 @@ padding="{$cell-padding}"> @@ -1444,7 +1444,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Other_Producers']"/> + select="$msg/*/entry[@key='Funding_Agencies']"/> @@ -1514,7 +1514,7 @@ - @@ -1534,7 +1534,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Sampling']"/> + select="$msg/*/entry[@key='Sampling_Notes']"/> + select="$msg/*/entry[@key='Sampling_Procedure']"/> + select="$msg/*/entry[@key='Response_Rate']"/> + select="$msg/*/entry[@key='Weighting']"/> + select="$msg/*/entry[@key='Data_Collection']"/> @@ -1672,7 +1672,7 @@ padding="{$cell-padding}"> @@ -1694,7 +1694,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Time_Periods']"/> @@ -1738,7 +1738,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Questionnaires']"/> + select="$msg/*/entry[@key='Data_Collectors']"/> + select="$msg/*/entry[@key='Supervision']"/> - + @@ -1885,7 +1885,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Data_Editing']"/> + select="$msg/*/entry[@key='Other_Processing']"/> + select="$msg/*/entry[@key='Accessibility']"/> @@ -1980,7 +1980,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Access_Authority']"/> - @@ -2022,7 +2022,7 @@ padding="{$cell-padding}"> + select="$msg/*/entry[@key='Distributors']"/> + select="$msg/*/entry[@key='Depositors']"/> + select="$msg/*/entry[@key='Confidentiality']"/> + select="$msg/*/entry[@key='Access_Conditions']"/> @@ -2142,7 +2142,7 @@ border="{$default-border}" padding="{$cell-padding}"> + select="$msg/*/entry[@key='Disclaimer']"/> + select="$msg/*/entry[@key='Copyright']"/> - + @@ -2194,15 +2194,15 @@ - + - + - + @@ -2218,7 +2218,7 @@ - + @@ -2227,16 +2227,16 @@ - + - + - - + + - + @@ -2262,17 +2262,17 @@ - + - + - + - + @@ -2290,7 +2290,7 @@ - + @@ -2299,17 +2299,17 @@ - + - + - + - + @@ -2327,7 +2327,7 @@ - + @@ -2336,7 +2336,7 @@ - + @@ -2372,7 +2372,7 @@ + select="$msg/*/entry[@key='Reports_and_analytical_documents']"/> @@ -2389,7 +2389,7 @@ space-after="0.03in"> - + @@ -2407,7 +2407,7 @@ + select="$msg/*/entry[@key='Technical_documents']"/> @@ -2425,7 +2425,7 @@ + select="$msg/*/entry[@key='Administrative_documents']"/> @@ -2442,7 +2442,7 @@ space-after="0.03in"> - + @@ -2459,7 +2459,7 @@ space-after="0.03in"> - + @@ -2477,7 +2477,7 @@ + select="$msg/*/entry[@key='Statistical_tables']"/> @@ -2495,7 +2495,7 @@ + select="$msg/*/entry[@key='Scripts_and_programs']"/> @@ -2512,7 +2512,7 @@ space-after="0.03in"> - + @@ -2532,7 +2532,7 @@ + select="$msg/*/entry[@key='Reports_and_analytical_documents']"/> - + - + - - + - + - + @@ -2602,7 +2602,7 @@ - + @@ -2611,7 +2611,7 @@ - + *** - + **** @@ -2749,7 +2749,7 @@ - # # @@ -2762,7 +2762,7 @@ - # # @@ -2776,18 +2776,18 @@ - + - : + : - :  + :  @@ -2811,7 +2811,7 @@ - + @@ -2823,7 +2823,7 @@ - + @@ -2835,7 +2835,7 @@ - + @@ -2847,7 +2847,7 @@ - + @@ -2859,7 +2859,7 @@ - + @@ -2871,7 +2871,7 @@ - + @@ -2905,10 +2905,10 @@ - + - + @@ -2955,7 +2955,7 @@ - + : @@ -3059,7 +3059,7 @@ - + @@ -3080,7 +3080,7 @@ - + @@ -3092,7 +3092,7 @@ - + @@ -3104,7 +3104,7 @@ - + @@ -3116,7 +3116,7 @@ - + @@ -3239,13 +3239,13 @@ - + - + - + @@ -3322,7 +3322,7 @@   - + @@ -3369,7 +3369,7 @@ : - + @@ -3382,7 +3382,7 @@ - + [= + select="$msg/*/entry[@key='Type']"/>= + select="$msg/*/entry[@key='discrete']"/> + select="$msg/*/entry[@key='continuous']"/> ] @@ -3404,7 +3404,7 @@ [== - @@ -3419,13 +3419,13 @@ [= = - ] [ - + =* / @@ -3437,7 +3437,7 @@ @@ -3452,12 +3452,12 @@ - + [ + select="$msg/*/entry[@key='Abbrev_NotWeighted']"/> / - + ] @@ -3470,18 +3470,18 @@ - + select="$msg/*/entry[@key='Invalid']"/> - + - @@ -3517,7 +3517,7 @@ - + - + - + - + - + - + @@ -3638,7 +3638,7 @@ - + + select="$msg/*/entry[@key='Recoding_and_Derivation']"/> - + - + - + - - + + - @@ -3798,7 +3798,7 @@ padding="{$cell-padding}" text-align="center"> + select="$msg/*/entry[@key='Cases_Abbreviation']"/> @@ -3806,7 +3806,7 @@ padding="{$cell-padding}" text-align="center"> + select="$msg/*/entry[@key='Weighted']"/> @@ -3814,9 +3814,9 @@ padding="{$cell-padding}" text-align="center"> + select="$msg/*/entry[@key='Percentage']"/> () + select="$msg/*/entry[@key='Weighted']"/>) @@ -3950,16 +3950,16 @@ + select="$msg/*/entry[@key='SumStat_Warning']"/> - + ( - ) + ) @@ -4003,7 +4003,7 @@ - *** + *** *** @@ -4055,7 +4055,7 @@ - + @@ -4070,7 +4070,7 @@ - + @@ -4085,7 +4085,7 @@ - + @@ -4100,7 +4100,7 @@ - + @@ -4162,38 +4162,38 @@ - + - + - + - + - + - + - + @@ -4257,40 +4257,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + From 9430e27d5973f281103caace950188a6784730f3 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 19 Sep 2023 10:55:03 +0200 Subject: [PATCH 027/427] findPermissionsOn improvements --- .../iq/dataverse/api/AbstractApiBean.java | 19 +++++++++++------- .../edu/harvard/iq/dataverse/api/Admin.java | 20 +++++++------------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 0a0861fa1c9..4b27d479552 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -71,6 +71,7 @@ import jakarta.persistence.NoResultException; import jakarta.persistence.PersistenceContext; import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.*; import jakarta.ws.rs.core.Response.ResponseBuilder; @@ -526,16 +527,20 @@ protected DvObject findDvo( Long id ) { * with that alias. If that fails, tries to get a {@link Dataset} with that global id. * @param id a value identifying the DvObject, either numeric of textual. * @return A DvObject, or {@code null} + * @throws WrappedResponse */ - protected DvObject findDvo( String id ) { - if ( isNumeric(id) ) { - return findDvo( Long.valueOf(id)) ; + @NotNull + protected DvObject findDvo(@NotNull final String id) throws WrappedResponse { + DvObject d = null; + if (isNumeric(id)) { + d = findDvo(Long.valueOf(id)); } else { - Dataverse d = dataverseSvc.findByAlias(id); - return ( d != null ) ? - d : datasetSvc.findByGlobalId(id); - + d = dataverseSvc.findByAlias(id); + } + if (d == null) { + return findDatasetOrDie(id); } + return d; } protected T failIfNull( T t, String errorMessage ) throws WrappedResponse { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 968e6f197e8..199410bf547 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -64,7 +64,6 @@ import java.io.InputStream; import java.io.StringReader; -import java.net.URLDecoder; import java.util.Map; import java.util.Map.Entry; import java.util.logging.Level; @@ -1331,22 +1330,17 @@ public Response convertUserFromBcryptToSha1(String json) { @Path("permissions/{dvo}") @AuthRequired @GET - public Response findPermissonsOn(@Context ContainerRequestContext crc, @PathParam("dvo") String dvo) { + public Response findPermissonsOn(@Context final ContainerRequestContext crc, @PathParam("dvo") final String dvo) { try { - DvObject dvObj = findDvo(dvo); - if (dvObj == null) { - dvObj = findDatasetOrDie(dvo); - if (dvObj == null) { - return notFound("DvObject " + dvo + " not found"); - } - } - User aUser = getRequestUser(crc); - JsonObjectBuilder bld = Json.createObjectBuilder(); + final DvObject dvObj = findDvo(dvo); + final User aUser = getRequestUser(crc); + final JsonObjectBuilder bld = Json.createObjectBuilder(); bld.add("user", aUser.getIdentifier()); bld.add("permissions", json(permissionSvc.permissionsFor(createDataverseRequest(aUser), dvObj))); return ok(bld); - - } catch (Exception e) { + } catch (WrappedResponse r) { + return r.getResponse(); + } catch (Exception e) { logger.log(Level.SEVERE, "Error while testing permissions", e); return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); } From 9adfe84a8dd0b5674b2efe7406c7755ebbfb4b49 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 19 Sep 2023 10:58:47 +0200 Subject: [PATCH 028/427] code format -> removed tabs, etc. --- .../iq/dataverse/api/AbstractApiBean.java | 11 ++++--- .../edu/harvard/iq/dataverse/api/Admin.java | 32 +++++++++---------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 4b27d479552..4e0b8fccae8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -523,14 +523,17 @@ protected DvObject findDvo( Long id ) { /** * Tries to find a DvObject. If the passed id can be interpreted as a number, - * it tries to get the DvObject by its id. Else, it tries to get a {@link Dataverse} - * with that alias. If that fails, tries to get a {@link Dataset} with that global id. + * it tries to get the DvObject by its id. Else, it tries to get a + * {@link Dataverse} + * with that alias. If that fails, tries to get a {@link Dataset} with that + * global id. + * * @param id a value identifying the DvObject, either numeric of textual. * @return A DvObject, or {@code null} * @throws WrappedResponse */ @NotNull - protected DvObject findDvo(@NotNull final String id) throws WrappedResponse { + protected DvObject findDvo(@NotNull final String id) throws WrappedResponse { DvObject d = null; if (isNumeric(id)) { d = findDvo(Long.valueOf(id)); @@ -541,7 +544,7 @@ protected DvObject findDvo(@NotNull final String id) throws WrappedResponse { return findDatasetOrDie(id); } return d; - } + } protected T failIfNull( T t, String errorMessage ) throws WrappedResponse { if ( t != null ) return t; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 199410bf547..848cb950b7d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1327,24 +1327,24 @@ public Response convertUserFromBcryptToSha1(String json) { } - @Path("permissions/{dvo}") - @AuthRequired - @GET - public Response findPermissonsOn(@Context final ContainerRequestContext crc, @PathParam("dvo") final String dvo) { - try { - final DvObject dvObj = findDvo(dvo); - final User aUser = getRequestUser(crc); - final JsonObjectBuilder bld = Json.createObjectBuilder(); - bld.add("user", aUser.getIdentifier()); - bld.add("permissions", json(permissionSvc.permissionsFor(createDataverseRequest(aUser), dvObj))); - return ok(bld); - } catch (WrappedResponse r) { + @Path("permissions/{dvo}") + @AuthRequired + @GET + public Response findPermissonsOn(@Context final ContainerRequestContext crc, @PathParam("dvo") final String dvo) { + try { + final DvObject dvObj = findDvo(dvo); + final User aUser = getRequestUser(crc); + final JsonObjectBuilder bld = Json.createObjectBuilder(); + bld.add("user", aUser.getIdentifier()); + bld.add("permissions", json(permissionSvc.permissionsFor(createDataverseRequest(aUser), dvObj))); + return ok(bld); + } catch (WrappedResponse r) { return r.getResponse(); } catch (Exception e) { - logger.log(Level.SEVERE, "Error while testing permissions", e); - return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } - } + logger.log(Level.SEVERE, "Error while testing permissions", e); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } @Path("assignee/{idtf}") @GET From d31596addb40e5245083350879c940f6e48921fb Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 19 Sep 2023 11:01:20 +0200 Subject: [PATCH 029/427] removed newline --- .../java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 4e0b8fccae8..105efd6b740 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -523,8 +523,7 @@ protected DvObject findDvo( Long id ) { /** * Tries to find a DvObject. If the passed id can be interpreted as a number, - * it tries to get the DvObject by its id. Else, it tries to get a - * {@link Dataverse} + * it tries to get the DvObject by its id. Else, it tries to get a {@link Dataverse} * with that alias. If that fails, tries to get a {@link Dataset} with that * global id. * From fad1c825fc2d1305c16437b4cb366dedef79c455 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 19 Sep 2023 11:02:01 +0200 Subject: [PATCH 030/427] removed added newlines --- .../java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 105efd6b740..b548a7168fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -524,9 +524,7 @@ protected DvObject findDvo( Long id ) { /** * Tries to find a DvObject. If the passed id can be interpreted as a number, * it tries to get the DvObject by its id. Else, it tries to get a {@link Dataverse} - * with that alias. If that fails, tries to get a {@link Dataset} with that - * global id. - * + * with that alias. If that fails, tries to get a {@link Dataset} with that global id. * @param id a value identifying the DvObject, either numeric of textual. * @return A DvObject, or {@code null} * @throws WrappedResponse From fbf1f9cbd05e18031170f206b3352c9a424cab16 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 16 Oct 2023 15:36:27 +0200 Subject: [PATCH 031/427] RO-Crate metadata mime type detection --- .../edu/harvard/iq/dataverse/util/FileUtil.java | 15 ++++++++++++++- .../MimeTypeDetectionByFileName.properties | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 1ad389fb0e2..a7c3488e57f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -86,6 +86,7 @@ import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.ResourceBundle; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -182,6 +183,7 @@ public class FileUtil implements java.io.Serializable { public static final String MIME_TYPE_NETCDF = "application/netcdf"; public static final String MIME_TYPE_XNETCDF = "application/x-netcdf"; public static final String MIME_TYPE_HDF5 = "application/x-hdf5"; + public static final String MIME_TYPE_RO_CRATE = "application/ld+json; profile=\"http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate\""; // File type "thumbnail classes" tags: @@ -421,6 +423,11 @@ public static String retestIngestableFileType(File file, String fileType) { } public static String determineFileType(File f, String fileName) throws IOException{ + final ResourceBundle bundle = BundleUtil.getResourceBundle("MimeTypeDetectionByFileName"); + if (bundle.keySet().contains(fileName)) { + return bundle.getString(fileName); + } + String fileType = null; String fileExtension = getFileExtension(fileName); @@ -545,6 +552,11 @@ public static String determineFileType(File f, String fileName) throws IOExcepti } public static String determineFileTypeByNameAndExtension(String fileName) { + final ResourceBundle bundle = BundleUtil.getResourceBundle("MimeTypeDetectionByFileName"); + if (bundle.keySet().contains(fileName)) { + return bundle.getString(fileName); + } + String mimetypesFileTypeMapResult = MIME_TYPE_MAP.getContentType(fileName); logger.fine("MimetypesFileTypeMap type by extension, for " + fileName + ": " + mimetypesFileTypeMapResult); if (mimetypesFileTypeMapResult != null) { @@ -825,7 +837,8 @@ public static boolean useRecognizedType(String suppliedContentType, String recog || canIngestAsTabular(recognizedType) || recognizedType.equals("application/fits-gzipped") || recognizedType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE) || recognizedType.equalsIgnoreCase(BagItFileHandler.FILE_TYPE) - || recognizedType.equals(MIME_TYPE_ZIP)) { + || recognizedType.equals(MIME_TYPE_ZIP) + || recognizedType.equals(MIME_TYPE_RO_CRATE)) { return true; } return false; diff --git a/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties b/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties index 70b0c4e371e..5c1a22bfd5f 100644 --- a/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties +++ b/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties @@ -2,3 +2,5 @@ Makefile=text/x-makefile Snakemake=text/x-snakemake Dockerfile=application/x-docker-file Vagrantfile=application/x-vagrant-file +ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" From fbc6a54ba067e24f135d11cc7f66a950838c45a0 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 16 Oct 2023 19:34:46 +0200 Subject: [PATCH 032/427] fixed display and facet string for mime types with profile + test + reslease notes --- .../10015-RO-Crate-metadata-file.md | 10 +++++++ .../harvard/iq/dataverse/util/FileUtil.java | 10 +++++++ .../propertyFiles/MimeTypeDisplay.properties | 1 + .../propertyFiles/MimeTypeFacets.properties | 1 + .../iq/dataverse/util/FileUtilTest.java | 29 +++++++++++++++++++ .../resources/fileutil/ro-crate-metadata.json | 1 + 6 files changed, 52 insertions(+) create mode 100644 doc/release-notes/10015-RO-Crate-metadata-file.md create mode 100644 src/test/resources/fileutil/ro-crate-metadata.json diff --git a/doc/release-notes/10015-RO-Crate-metadata-file.md b/doc/release-notes/10015-RO-Crate-metadata-file.md new file mode 100644 index 00000000000..4b018a634f7 --- /dev/null +++ b/doc/release-notes/10015-RO-Crate-metadata-file.md @@ -0,0 +1,10 @@ +Detection of mime-types based on a filename with extension and detection of the RO-Crate metadata files. + +From now on, filenames with extensions can be added into `MimeTypeDetectionByFileName.properties` file. Filenames added there will take precedence over simply recognizing files by extensions. For example, two new filenames are added into that file: +``` +ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +``` + +Therefore, files named `ro-crate-metadata.json` will be then detected as RO-Crated metadata files from now on, instead as generic `JSON` files. +For more information on the RO-Crate specifications, see https://www.researchobject.org/ro-crate diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index a7c3488e57f..baff17af601 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -280,6 +280,11 @@ public static String getUserFriendlyFileType(DataFile dataFile) { if (fileType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)){ return ShapefileHandler.SHAPEFILE_FILE_TYPE_FRIENDLY_NAME; } + try { + return BundleUtil.getStringFromPropertyFile(fileType,"MimeTypeDisplay" ); + } catch (MissingResourceException e) { + //NOOP: we will try again after trimming ";" + } if (fileType.contains(";")) { fileType = fileType.substring(0, fileType.indexOf(";")); } @@ -294,6 +299,11 @@ public static String getUserFriendlyFileType(DataFile dataFile) { } public static String getIndexableFacetFileType(DataFile dataFile) { + try { + return BundleUtil.getStringFromDefaultPropertyFile(dataFile.getContentType(),"MimeTypeFacets" ); + } catch (MissingResourceException e) { + //NOOP: we will try again after trimming ";" + } String fileType = getFileType(dataFile); try { return BundleUtil.getStringFromDefaultPropertyFile(fileType,"MimeTypeFacets" ); diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 295ac226fa1..8486a113116 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -222,5 +222,6 @@ text/xml-graphml=GraphML Network Data application/octet-stream=Unknown application/x-docker-file=Docker Image File application/x-vagrant-file=Vagrant Image File +application/ld+json;\u0020profile\u003d\u0022http\u003a//www.w3.org/ns/json-ld#flattened\u0020http\u003a//www.w3.org/ns/json-ld#compacted\u0020https\u003a//w3id.org/ro/crate\u0022=RO-Crate metadata # Dataverse-specific application/vnd.dataverse.file-package=Dataverse Package diff --git a/src/main/java/propertyFiles/MimeTypeFacets.properties b/src/main/java/propertyFiles/MimeTypeFacets.properties index aaab66f20ae..831c509b860 100644 --- a/src/main/java/propertyFiles/MimeTypeFacets.properties +++ b/src/main/java/propertyFiles/MimeTypeFacets.properties @@ -224,5 +224,6 @@ video/webm=Video text/xml-graphml=Network Data # Other application/octet-stream=Unknown +application/ld+json;\u0020profile\u003d\u0022http\u003a//www.w3.org/ns/json-ld#flattened\u0020http\u003a//www.w3.org/ns/json-ld#compacted\u0020https\u003a//w3id.org/ro/crate\u0022=Metadata # Dataverse-specific application/vnd.dataverse.file-package=Data diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index 2cfe9f25d7e..396d613e768 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -371,4 +371,33 @@ public void testHdf4File() throws IOException { assertEquals("application/octet-stream", contentType); } + @Test + public void testDetermineFileTypeROCrate() { + final String roCrateContentType = "application/ld+json; profile=\"http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate\""; + final DataFile rocrate = new DataFile(roCrateContentType); + + assertEquals(roCrateContentType, rocrate.getContentType()); + assertEquals("RO-Crate metadata", FileUtil.getUserFriendlyFileType(rocrate)); + assertEquals("Metadata", FileUtil.getIndexableFacetFileType(rocrate)); + + final File roCrateFile = new File("src/test/resources/fileutil/ro-crate-metadata.json"); + if (roCrateFile.exists()) { + try { + assertEquals(roCrateContentType, FileUtil.determineFileType(roCrateFile, "ro-crate-metadata.json")); + } catch (IOException ex) { + Logger.getLogger(FileUtilTest.class.getName()).log(Level.SEVERE, null, ex); + } + } else { + fail("File does not exist: " + roCrateFile.toPath().toString()); + } + + // test ";" removal + final String dockerFileWithProfile = "application/x-docker-file; profile=\"http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate\""; + final DataFile dockerDataFile = new DataFile(dockerFileWithProfile); + + assertEquals(dockerFileWithProfile, dockerDataFile.getContentType()); + assertEquals("Docker Image File", FileUtil.getUserFriendlyFileType(dockerDataFile)); + assertEquals("Code", FileUtil.getIndexableFacetFileType(dockerDataFile)); + } + } diff --git a/src/test/resources/fileutil/ro-crate-metadata.json b/src/test/resources/fileutil/ro-crate-metadata.json new file mode 100644 index 00000000000..5ac4b24ad2b --- /dev/null +++ b/src/test/resources/fileutil/ro-crate-metadata.json @@ -0,0 +1 @@ +{"@context":["https://w3id.org/ro/crate/1.1/context",{"metadataContext":"https://language-archives.services/metadata-definitions#metadataContext","orthographicNotes":"https://language-archives.services/metadata-definitions#orthographicNotes","media":"https://language-archives.services/metadata-definitions#media","comments":"https://language-archives.services/metadata-definitions#comments","private":"https://language-archives.services/metadata-definitions#private","depositFormReceived":"https://language-archives.services/metadata-definitions#depositFormReceived","subjectLanguages":"https://language-archives.services/metadata-definitions#subjectLanguages","originatedOn":"https://language-archives.services/metadata-definitions#originatedOn","languageAsGiven":"https://language-archives.services/metadata-definitions#languageAsGiven","contentLanguages":"https://language-archives.services/metadata-definitions#contentLanguages","dialect":"https://language-archives.services/metadata-definitions#dialect","discourseType":"https://language-archives.services/metadata-definitions#discourseType","bornDigital":"https://language-archives.services/metadata-definitions#bornDigital","accessNarrative":"https://language-archives.services/metadata-definitions#accessNarrative","receivedOn":"https://language-archives.services/metadata-definitions#receivedOn","digitisedOn":"https://language-archives.services/metadata-definitions#digitisedOn","ingestNotes":"https://language-archives.services/metadata-definitions#ingestNotes","adminComment":"https://language-archives.services/metadata-definitions#adminComment","external":"https://language-archives.services/metadata-definitions#external","dataType":"https://language-archives.services/metadata-definitions#dataType","citeAs":"https://language-archives.services/metadata-definitions#citeAs","originatedOnNarrative":"https://language-archives.services/metadata-definitions#originatedOnNarrative","countries":"https://language-archives.services/metadata-definitions#countries","tapesReturned":"https://language-archives.services/metadata-definitions#tapesReturned","originalMedia":"https://language-archives.services/metadata-definitions#originalMedia","metadataExportable":"https://language-archives.services/metadata-definitions#metadataExportable","metadataImportedOn":"https://language-archives.services/metadata-definitions#metadataImportedOn","metadataExportedOn":"https://language-archives.services/metadata-definitions#metadataExportedOn","tracking":"https://language-archives.services/metadata-definitions#tracking","fieldsOfResearch":"https://language-archives.services/metadata-definitions#fieldsOfResearch","role":"https://language-archives.services/metadata-definitions#role","doi":"https://language-archives.services/metadata-definitions#doi","sampleRate":"https://language-archives.services/metadata-definitions#sampleRate","channels":"https://language-archives.services/metadata-definitions#channels","fps":"https://language-archives.services/metadata-definitions#fps","essenceId":"https://language-archives.services/metadata-definitions#essenceId"}],"@graph":[{"@id":"#Erakor village","@type":"Place","name":"Erakor village"},{"@id":"#country_Australia","@type":"Country","name":"Australia"},{"@id":"#country_Vanuatu","@type":"Country","code":"VU","name":"Vanuatu"},{"@id":"#country_null","@type":"Country"},{"@id":"#geo_166.427,-22.283,166.467,-22.241","@type":"GeoShape","box":"166.427,-22.283 166.467,-22.241"},{"@id":"#geo_168.159,-17.83,168.594,-17.585","@type":"GeoShape","box":"168.159,-17.83 168.594,-17.585"},{"@id":"#geo_168.217,-17.8235,168.317,-17.7235","@type":"GeoShape","box":"168.217,-17.8235 168.317,-17.7235"},{"@id":"#identifier_collectionId","@type":"PropertyValue","name":"collectionIdentifier","value":"NT1"},{"@id":"#identifier_doi","@type":"PropertyValue","name":"doi","value":"10.4225/72/56F94A61DA9EC"},{"@id":"#identifier_domain","@type":"PropertyValue","name":"domain","value":"paradisec.org.au"},{"@id":"#identifier_hashid","@type":"PropertyValue","name":"hashId","value":"72b3dc1401c8ff06aacba0990a128fc113cf9ad5275f494b05c1142177356561bd7f4c0e8800bade2cbbbed75f6d9d019894735ad7e40762684d243a442d658a"},{"@id":"#identifier_id","@type":"PropertyValue","name":"id","value":"/paradisec.org.au/NT1/98007"},{"@id":"#identifier_itemId","@type":"PropertyValue","name":"itemIdentifier","value":"98007"},{"@id":"#language_bis","@type":"Language","code":"bis","location":{"@id":"#geo_166.427,-22.283,166.467,-22.241"},"name":"Bislama"},{"@id":"#language_erk","@type":"Language","code":"erk","location":{"@id":"#geo_168.159,-17.83,168.594,-17.585"},"name":"Efate, South"},{"@id":"#place_geo_168.217,-17.8235,168.317,-17.7235","@type":"Place","geo":{"@id":"#geo_168.217,-17.8235,168.317,-17.7235"}},{"@id":"./","@type":["Dataset","RepositoryObject"],"additionalType":"item","contentLocation":[{"@id":"#Erakor village"},{"@id":"#place_geo_168.217,-17.8235,168.317,-17.7235"}],"contributor":[{"@id":"http://nla.gov.au/nla.party-479603"},{"@id":"Kalsarap Namaf"},{"@id":"Iokopeth null"},{"@id":"John Maklen"},{"@id":"Waia Tenene"}],"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2018-05-17T04:13:04.000Z","description":"NT1-98007. Text #047 (speaker is John Maklen. Text title: History of villages before Erakor); Text #048 (speaker is John Maklen. Text title: Mantu the flying fox and Erromango); Text #049. Text title: Asaraf (speaker is John Maklen);Text #050. Text title: Mumu and Kotkot (speaker is John Maklen); Text #051. Text title: Natopu ni Erakor—the spirit who lives at Erakor (speaker is John Maklen);Text #038. Text title: The need for respect (speaker is Iokopeth) Stories can be seen at NT8-TEXT. There are time-aligned transcripts of this item and handwritten transcripts by Manuel Wayane scanned as jpg files.","hasPart":[{"@id":"NT1-98007-001.jpg"},{"@id":"NT1-98007-002.jpg"},{"@id":"NT1-98007-003.jpg"},{"@id":"NT1-98007-004.jpg"},{"@id":"NT1-98007-005.jpg"},{"@id":"NT1-98007-006.jpg"},{"@id":"NT1-98007-007.jpg"},{"@id":"NT1-98007-008.jpg"},{"@id":"NT1-98007-009.jpg"},{"@id":"NT1-98007-010.jpg"},{"@id":"NT1-98007-011.jpg"},{"@id":"NT1-98007-012.jpg"},{"@id":"NT1-98007-013.jpg"},{"@id":"NT1-98007-014.jpg"},{"@id":"NT1-98007-015.jpg"},{"@id":"NT1-98007-016.jpg"},{"@id":"NT1-98007-017.jpg"},{"@id":"NT1-98007-018.jpg"},{"@id":"NT1-98007-019.jpg"},{"@id":"NT1-98007-020.jpg"},{"@id":"NT1-98007-021.jpg"},{"@id":"NT1-98007-022.jpg"},{"@id":"NT1-98007-023.jpg"},{"@id":"NT1-98007-024.jpg"},{"@id":"NT1-98007-025.jpg"},{"@id":"NT1-98007-026.jpg"},{"@id":"NT1-98007-027.jpg"},{"@id":"NT1-98007-028.jpg"},{"@id":"NT1-98007-029.jpg"},{"@id":"NT1-98007-030.jpg"},{"@id":"NT1-98007-031.jpg"},{"@id":"NT1-98007-98007A.mp3"},{"@id":"NT1-98007-98007A.wav"},{"@id":"NT1-98007-98007B.mp3"},{"@id":"NT1-98007-98007B.wav"},{"@id":"NT1-98007-98007az.xml"},{"@id":"NT1-98007-A.tab"},{"@id":"NT1-98007-A.xml"},{"@id":"NT1-98007-B.tab"},{"@id":"NT1-98007-B.xml"},{"@id":"NT1-98007-98007A.ixt"},{"@id":"NT1-98007-98007A.trs"},{"@id":"NT1-98007-98007A.flextext"},{"@id":"NT1-98007-98007A.eaf"},{"@id":"NT1-98007-98007B.eaf"}],"identifier":[{"@id":"#identifier_domain"},{"@id":"#identifier_id"},{"@id":"#identifier_hashid"},{"@id":"#identifier_itemId"},{"@id":"#identifier_collectionId"},{"@id":"#identifier_doi"}],"license":{"@id":"_:b0"},"memberOf":{"@id":"/paradisec.org.au/NT1"},"name":"Recordings in South Efate","publisher":{"@id":"http://nla.gov.au/nla.party-593909"},"bornDigital":0,"contentLanguages":[{"@id":"#language_bis"},{"@id":"#language_erk"}],"countries":{"@id":"#country_Vanuatu"},"digitisedOn":"Sun Dec 31 2000 13:00:00 GMT+0000 (Coordinated Universal Time)","external":0,"languageAsGiven":"Nafsan","metadataExportable":1,"originalMedia":"audiocassette","originatedOn":"1998-10-03","private":0,"subjectLanguages":{"@id":"#language_erk"},"tapesReturned":0},{"@id":"Iokopeth null","@type":"Person","givenName":"Iokopeth","homeLocation":{"@id":"#country_null"},"name":"Iokopeth","role":{"@id":"role_speaker"}},{"@id":"John Maklen","@type":"Person","familyName":"Maklen","givenName":"John","homeLocation":{"@id":"#country_null"},"name":"John Maklen","role":{"@id":"role_speaker"}},{"@id":"Kalsarap Namaf","@type":"Person","familyName":"Namaf","givenName":"Kalsarap","homeLocation":{"@id":"#country_null"},"name":"Kalsarap Namaf","role":{"@id":"role_speaker"}},{"@id":"NT1-98007-001.jpg","@type":"File","contentSize":1658368,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:31.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-001.jpg","doi":"10.4225/72/575C8A369D680","essenceId":1010709},{"@id":"NT1-98007-002.jpg","@type":"File","contentSize":1816576,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:37.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-002.jpg","doi":"10.4225/72/575C8A3C15C98","essenceId":1010710},{"@id":"NT1-98007-003.jpg","@type":"File","contentSize":1811968,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:43.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-003.jpg","doi":"10.4225/72/575C8A41DD973","essenceId":1010711},{"@id":"NT1-98007-004.jpg","@type":"File","contentSize":1827840,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:48.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-004.jpg","doi":"10.4225/72/575C8A4767685","essenceId":1010712},{"@id":"NT1-98007-005.jpg","@type":"File","contentSize":1853440,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:54.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-005.jpg","doi":"10.4225/72/575C8A4CD8971","essenceId":1010713},{"@id":"NT1-98007-006.jpg","@type":"File","contentSize":1796608,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:01:59.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-006.jpg","doi":"10.4225/72/575C8A525C618","essenceId":1010714},{"@id":"NT1-98007-007.jpg","@type":"File","contentSize":1780224,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:05.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-007.jpg","doi":"10.4225/72/575C8A5810189","essenceId":1010715},{"@id":"NT1-98007-008.jpg","@type":"File","contentSize":1737728,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:11.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-008.jpg","doi":"10.4225/72/575C8A5DB1113","essenceId":1010716},{"@id":"NT1-98007-009.jpg","@type":"File","contentSize":1781760,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:16.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-009.jpg","doi":"10.4225/72/575C8A63479C1","essenceId":1010717},{"@id":"NT1-98007-010.jpg","@type":"File","contentSize":1797632,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:22.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-010.jpg","doi":"10.4225/72/575C8A68B23D2","essenceId":1010718},{"@id":"NT1-98007-011.jpg","@type":"File","contentSize":1800704,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:28.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-011.jpg","doi":"10.4225/72/575C8A6E73D01","essenceId":1010719},{"@id":"NT1-98007-012.jpg","@type":"File","contentSize":1822720,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:33.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-012.jpg","doi":"10.4225/72/575C8A742DE00","essenceId":1010720},{"@id":"NT1-98007-013.jpg","@type":"File","contentSize":1809920,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:39.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-013.jpg","doi":"10.4225/72/575C8A79B1B0F","essenceId":1010721},{"@id":"NT1-98007-014.jpg","@type":"File","contentSize":1821696,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:44.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-014.jpg","doi":"10.4225/72/575C8A7F3F253","essenceId":1010722},{"@id":"NT1-98007-015.jpg","@type":"File","contentSize":1626624,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:50.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-015.jpg","doi":"10.4225/72/575C8A84C0022","essenceId":1010723},{"@id":"NT1-98007-016.jpg","@type":"File","contentSize":1633792,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:02:56.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-016.jpg","doi":"10.4225/72/575C8A8A9A944","essenceId":1010724},{"@id":"NT1-98007-017.jpg","@type":"File","contentSize":1870336,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:01.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-017.jpg","doi":"10.4225/72/575C8A90506E6","essenceId":1010725},{"@id":"NT1-98007-018.jpg","@type":"File","contentSize":1858560,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:07.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-018.jpg","doi":"10.4225/72/575C8A95DF462","essenceId":1010726},{"@id":"NT1-98007-019.jpg","@type":"File","contentSize":1852416,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:12.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-019.jpg","doi":"10.4225/72/575C8A9B56F2F","essenceId":1010727},{"@id":"NT1-98007-020.jpg","@type":"File","contentSize":1838080,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:18.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-020.jpg","doi":"10.4225/72/575C8AA0F09B8","essenceId":1010728},{"@id":"NT1-98007-021.jpg","@type":"File","contentSize":1861120,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:24.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-021.jpg","doi":"10.4225/72/575C8AA6B23AF","essenceId":1010729},{"@id":"NT1-98007-022.jpg","@type":"File","contentSize":1835008,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:29.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-022.jpg","doi":"10.4225/72/575C8AAC3A545","essenceId":1010730},{"@id":"NT1-98007-023.jpg","@type":"File","contentSize":1827328,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:35.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-023.jpg","doi":"10.4225/72/575C8AB1C991E","essenceId":1010731},{"@id":"NT1-98007-024.jpg","@type":"File","contentSize":1805312,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:40.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-024.jpg","doi":"10.4225/72/575C8AB74847D","essenceId":1010732},{"@id":"NT1-98007-025.jpg","@type":"File","contentSize":1912832,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:46.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-025.jpg","doi":"10.4225/72/575C8ABCD2B44","essenceId":1010733},{"@id":"NT1-98007-026.jpg","@type":"File","contentSize":1889792,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:51.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-026.jpg","doi":"10.4225/72/575C8AC26D69E","essenceId":1010734},{"@id":"NT1-98007-027.jpg","@type":"File","contentSize":1878528,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:03:57.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-027.jpg","doi":"10.4225/72/575C8AC7F3886","essenceId":1010735},{"@id":"NT1-98007-028.jpg","@type":"File","contentSize":1868288,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:02.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-028.jpg","doi":"10.4225/72/575C8ACD72196","essenceId":1010736},{"@id":"NT1-98007-029.jpg","@type":"File","contentSize":1859584,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:08.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-029.jpg","doi":"10.4225/72/575C8AD2E8E82","essenceId":1010737},{"@id":"NT1-98007-030.jpg","@type":"File","contentSize":1859072,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:13.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-030.jpg","doi":"10.4225/72/575C8AD8775E6","essenceId":1010738},{"@id":"NT1-98007-031.jpg","@type":"File","contentSize":1708544,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:19.000Z","encodingFormat":"image/jpeg","name":"NT1-98007-031.jpg","doi":"10.4225/72/575C8ADDE64B8","essenceId":1010739},{"@id":"NT1-98007-98007A.eaf","@type":"File","contentSize":165674,"dateCreated":"2016-08-01T05:00:06.000Z","dateModified":"2016-08-01T16:01:41.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007A.eaf","doi":"10.4225/72/579F725FDD059","essenceId":1100487},{"@id":"NT1-98007-98007A.flextext","@type":"File","contentSize":141244,"dateCreated":"2016-05-20T04:00:06.000Z","dateModified":"2016-06-24T12:41:36.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007A.flextext","doi":"10.4225/72/576D2A7B75614","essenceId":1086277},{"@id":"NT1-98007-98007A.ixt","@type":"File","contentSize":40299,"dateCreated":"2016-04-18T07:00:07.000Z","dateModified":"2016-06-24T08:25:06.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007A.ixt","doi":"10.4225/72/576CEE5CED1FB","essenceId":1085095},{"@id":"NT1-98007-98007A.mp3","@type":"File","bitrate":128009,"contentSize":43667584,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2019-11-29T09:17:34.000Z","duration":2729.02,"encodingFormat":"audio/mpeg","name":"NT1-98007-98007A.mp3","channels":2,"doi":"10.4225/72/575C8AE370B93","essenceId":1010740,"sampleRate":44100},{"@id":"NT1-98007-98007A.trs","@type":"File","contentSize":28292,"dateCreated":"2016-04-26T10:00:06.000Z","dateModified":"2016-06-24T08:45:27.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007A.trs","doi":"10.4225/72/576CF32101764","essenceId":1085293},{"@id":"NT1-98007-98007A.wav","@type":"File","bitrate":4608000,"contentSize":1571894006,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2019-11-29T09:16:52.000Z","duration":2728.98,"encodingFormat":"audio/x-wav","name":"NT1-98007-98007A.wav","channels":2,"doi":"10.4225/72/575C8AE8E6E6B","essenceId":1010741,"sampleRate":96000},{"@id":"NT1-98007-98007B.eaf","@type":"File","contentSize":118748,"dateCreated":"2016-08-01T05:00:07.000Z","dateModified":"2016-08-01T16:01:47.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007B.eaf","doi":"10.4225/72/579F7265746C0","essenceId":1100493},{"@id":"NT1-98007-98007B.mp3","@type":"File","bitrate":128007,"contentSize":35305600,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2019-11-29T09:12:58.000Z","duration":2206.47,"encodingFormat":"audio/mpeg","name":"NT1-98007-98007B.mp3","channels":2,"doi":"10.4225/72/575C8AEE64BCA","essenceId":1010742,"sampleRate":44100},{"@id":"NT1-98007-98007B.wav","@type":"File","bitrate":4608000,"contentSize":1270917002,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2019-11-29T09:15:22.000Z","duration":2206.45,"encodingFormat":"audio/x-wav","name":"NT1-98007-98007B.wav","channels":2,"doi":"10.4225/72/575C8AF3D2DA0","essenceId":1010743,"sampleRate":96000},{"@id":"NT1-98007-98007az.xml","@type":"File","contentSize":48755,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:46.000Z","encodingFormat":"application/xml","name":"NT1-98007-98007az.xml","doi":"10.4225/72/575C8AF94896E","essenceId":1010744},{"@id":"NT1-98007-A.tab","@type":"File","contentSize":27810,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:04:57.000Z","encodingFormat":"text/plain","name":"NT1-98007-A.tab","doi":"10.4225/72/575C8B043764B","essenceId":1010746},{"@id":"NT1-98007-A.xml","@type":"File","contentSize":48788,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:05:03.000Z","encodingFormat":"application/xml","name":"NT1-98007-A.xml","doi":"10.4225/72/575C8B09BDC88","essenceId":1010747},{"@id":"NT1-98007-B.tab","@type":"File","contentSize":20239,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:05:08.000Z","encodingFormat":"text/plain","name":"NT1-98007-B.tab","doi":"10.4225/72/575C8B0F4F8F1","essenceId":1010748},{"@id":"NT1-98007-B.xml","@type":"File","contentSize":35289,"dateCreated":"2012-09-27T10:08:01.000Z","dateModified":"2016-06-11T22:05:14.000Z","encodingFormat":"application/xml","name":"NT1-98007-B.xml","doi":"10.4225/72/575C8B14C6702","essenceId":1010749},{"@id":"Waia Tenene","@type":"Person","familyName":"Tenene","givenName":"Waia","homeLocation":{"@id":"#country_null"},"name":"Waia Tenene","role":{"@id":"role_speaker"}},{"@id":"_:b0","@type":"CreativeWork","name":"Open (subject to agreeing to PDSC access conditions)"},{"@id":"http://nla.gov.au/nla.party-479603","@type":"Person","email":"thien@unimelb.edu.au","familyName":"Thieberger","givenName":"Nick","homeLocation":{"@id":"#country_Australia"},"name":"Nick Thieberger","role":[{"@id":"role_collector"},{"@id":"role_depositor"},{"@id":"role_recorder"}]},{"@id":"http://nla.gov.au/nla.party-593909","@type":"Organization","name":"University of Melbourne"},{"@id":"ro-crate-metadata.json","@type":"CreativeWork","conformsTo":{"@id":"https://w3id.org/ro/crate/1.1/context"},"about":{"@id":"./"}},{"@id":"role_collector","@type":"Role","name":"collector"},{"@id":"role_depositor","@type":"Role","name":"depositor"},{"@id":"role_recorder","@type":"Role","name":"recorder"},{"@id":"role_speaker","@type":"Role","name":"speaker"}]} From 42b695871c4faab715d96de6413ef6dfb296b57a Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Fri, 20 Oct 2023 18:19:27 +0200 Subject: [PATCH 033/427] disable s3 tagging JVM option --- .../10022_upload_redirect_without_tagging.md | 1 + .../source/installation/config.rst | 8 +++ .../iq/dataverse/dataaccess/S3AccessIO.java | 11 ++- .../iq/dataverse/settings/JvmSettings.java | 2 + src/main/webapp/resources/js/fileupload.js | 70 ++++++++++--------- 5 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 doc/release-notes/10022_upload_redirect_without_tagging.md diff --git a/doc/release-notes/10022_upload_redirect_without_tagging.md b/doc/release-notes/10022_upload_redirect_without_tagging.md new file mode 100644 index 00000000000..379c7c8f1e0 --- /dev/null +++ b/doc/release-notes/10022_upload_redirect_without_tagging.md @@ -0,0 +1 @@ +If your S3 store does not support tagging and gives an error when redirecting uploads, from now on, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. Disabling the tagging can result in leftover files that are not used by your Dataverse instance and should be removed to preserve the storage space. To clean up the leftover files, you can use the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index ce8876b012c..2b5fd75977c 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -792,6 +792,13 @@ Larger installations may want to increase the number of open S3 connections allo ``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"`` +By default, when redirecting an upload to the S3 storage, the Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup when the file is not added to the dataset after upload (e.g., when the user cancels the operation). +If your S3 store does not support tagging and gives an error when redirecting uploads, you can disable that tag by using the ``dataverse.files..disable-tagging`` JVM option. For example: + +``./asadmin create-jvm-options "-Ddataverse.files..disable-tagging=true"`` + +Disabling the ``temp`` tag can result in leftover files that are not used by your Dataverse instance and should be removed to preserve the storage space. To clean up the leftover files, you can use the [Cleanup Storage of a Dataset](https://guides.dataverse.org/en/5.13/api/native-api.html#cleanup-storage-of-a-dataset) API endpoint. + In case you would like to configure Dataverse to use a custom S3 service instead of Amazon S3 services, please add the options for the custom URL and region as documented below. Please read above if your desired combination has been tested already and what other options have been set for a successful integration. @@ -825,6 +832,7 @@ List of S3 Storage Options dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` + dataverse.files..disable-tagging ``true``/``false`` Do not place the ``temp`` tag when redirecting the upload to the S3 server ``false`` =========================================== ================== =================================================================================== ============= .. table:: diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 822ada0b83e..c904651e81f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -39,6 +39,7 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.datavariable.DataVariable; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.FileUtil; import opennlp.tools.util.StringUtil; @@ -985,7 +986,10 @@ private String generateTemporaryS3UploadUrl(String key, Date expiration) throws GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key).withMethod(HttpMethod.PUT).withExpiration(expiration); //Require user to add this header to indicate a temporary file - generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId).orElse(false); + if (!taggingDisabled) { + generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + } URL presignedUrl; try { @@ -1034,7 +1038,10 @@ public JsonObjectBuilder generateTemporaryS3UploadUrls(String globalId, String s } else { JsonObjectBuilder urls = Json.createObjectBuilder(); InitiateMultipartUploadRequest initiationRequest = new InitiateMultipartUploadRequest(bucketName, key); - initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId).orElse(false); + if (!taggingDisabled) { + initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + } InitiateMultipartUploadResult initiationResponse = s3.initiateMultipartUpload(initiationRequest); String uploadId = initiationResponse.getUploadId(); for (int i = 1; i <= (fileSize / minPartSize) + (fileSize % minPartSize > 0 ? 1 : 0); i++) { diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index cc3272413c7..794ffd5e0af 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -50,6 +50,8 @@ public enum JvmSettings { UPLOADS_DIRECTORY(SCOPE_FILES, "uploads"), DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"), GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"), + SCOPE_DRIVER(SCOPE_FILES), + DISABLE_S3_TAGGING(SCOPE_DRIVER, "disable-tagging"), // SOLR INDEX SETTINGS SCOPE_SOLR(PREFIX, "solr"), diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index 08d6956b62c..03ec82f214b 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -192,41 +192,45 @@ var fileUpload = class fileUploadClass { progBar.html(''); progBar.append($('').attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all')); if(this.urls.hasOwnProperty("url")) { - $.ajax({ - url: this.urls.url, - headers: { "x-amz-tagging": "dv-state=temp" }, - type: 'PUT', - data: this.file, - context:this, - cache: false, - processData: false, - success: function() { - //ToDo - cancelling abandons the file. It is marked as temp so can be cleaned up later, but would be good to remove now (requires either sending a presigned delete URL or adding a callback to delete only a temp file - if(!cancelled) { - this.reportUpload(); - } - }, - error: function(jqXHR, textStatus, errorThrown) { - console.log('Failure: ' + jqXHR.status); - console.log('Failure: ' + errorThrown); - uploadFailure(jqXHR, thisFile); - }, - xhr: function() { - var myXhr = $.ajaxSettings.xhr(); - if (myXhr.upload) { - myXhr.upload.addEventListener('progress', function(e) { - if (e.lengthComputable) { - var doublelength = 2 * e.total; - progBar.children('progress').attr({ - value: e.loaded, - max: doublelength - }); - } - }); + const url = this.urls.url; + const request = { + url: url, + type: 'PUT', + data: this.file, + context:this, + cache: false, + processData: false, + success: function() { + //ToDo - cancelling abandons the file. It is marked as temp so can be cleaned up later, but would be good to remove now (requires either sending a presigned delete URL or adding a callback to delete only a temp file + if(!cancelled) { + this.reportUpload(); + } + }, + error: function(jqXHR, textStatus, errorThrown) { + console.log('Failure: ' + jqXHR.status); + console.log('Failure: ' + errorThrown); + uploadFailure(jqXHR, thisFile); + }, + xhr: function() { + var myXhr = $.ajaxSettings.xhr(); + if (myXhr.upload) { + myXhr.upload.addEventListener('progress', function(e) { + if (e.lengthComputable) { + var doublelength = 2 * e.total; + progBar.children('progress').attr({ + value: e.loaded, + max: doublelength + }); + } + }); + } + return myXhr; } - return myXhr; + }; + if (url.includes("x-amz-tagging")) { + request.headers = { "x-amz-tagging": "dv-state=temp" }; } - }); + $.ajax(request); } else { var loaded=[]; this.etags=[]; From 3764bb2358aac756ed7bf97e052441836d5512e8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 8 Nov 2023 14:34:48 -0500 Subject: [PATCH 034/427] fix guestbook preview stop refresh of datasetForm --- src/main/webapp/dataset-license-terms.xhtml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index c54d94442ea..6d5b0a5fe4f 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -482,7 +482,7 @@
          - #{bundle['file.dataFilesTab.terms.list.guestbook']}   + #{bundle['file.dataFilesTab.terms.list.guestbook']}  
          @@ -524,7 +524,7 @@ + update=":datasetForm:previewGuestbook" oncomplete="PF('viewGuestbook').show();"/>
          @@ -563,7 +563,7 @@ + update=":datasetForm:previewGuestbook" oncomplete="PF('viewGuestbook').show();"/> From 0ead2c24ff7855396f83f07e724af7ce4b18f7be Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Fri, 1 Dec 2023 17:24:40 +0100 Subject: [PATCH 035/427] incmoplete metadata label visibility setting --- .../edu/harvard/iq/dataverse/DatasetPage.java | 10 ++++----- .../edu/harvard/iq/dataverse/FilePage.java | 18 ++++++++++----- .../iq/dataverse/mydata/DataRetrieverAPI.java | 17 ++++++++++++-- .../search/SearchIncludeFragment.java | 11 +++++++--- .../iq/dataverse/search/SolrSearchResult.java | 22 +++++++++++++++---- .../iq/dataverse/settings/JvmSettings.java | 1 + src/main/webapp/file.xhtml | 2 +- 7 files changed, 61 insertions(+), 20 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index fc18257196d..7ab24e69152 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -2210,13 +2210,13 @@ private void displayPublishMessage(){ public boolean isValid() { if (valid == null) { - DatasetVersion version = dataset.getLatestVersion(); - if (!version.isDraft()) { + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false))) { + final DatasetVersion newVersion = workingVersion.cloneDatasetVersion(); + newVersion.setDatasetFields(newVersion.initDatasetFields()); + valid = newVersion.isValid(); + } else { valid = true; } - DatasetVersion newVersion = version.cloneDatasetVersion(); - newVersion.setDatasetFields(newVersion.initDatasetFields()); - valid = newVersion.isValid(); } return valid; } diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index bfae80ade27..d9a0fe2d9b0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -31,6 +31,7 @@ import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; @@ -283,13 +284,20 @@ private void displayPublishMessage(){ } } + Boolean valid = null; + public boolean isValid() { - if (!fileMetadata.getDatasetVersion().isDraft()) { - return true; + if (valid == null) { + final DatasetVersion workingVersion = fileMetadata.getDatasetVersion(); + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false))) { + final DatasetVersion newVersion = workingVersion.cloneDatasetVersion(); + newVersion.setDatasetFields(newVersion.initDatasetFields()); + valid = newVersion.isValid(); + } else { + valid = true; + } } - DatasetVersion newVersion = fileMetadata.getDatasetVersion().cloneDatasetVersion(); - newVersion.setDatasetFields(newVersion.initDatasetFields()); - return newVersion.isValid(); + return valid; } private boolean canViewUnpublishedDataset() { diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java index e9898031343..be81de9267d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java @@ -3,11 +3,13 @@ */ package edu.harvard.iq.dataverse.mydata; +import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataverseRoleServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DataverseSession; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.DvObjectServiceBean; +import edu.harvard.iq.dataverse.PermissionsWrapper; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.search.SearchServiceBean; @@ -25,6 +27,7 @@ import edu.harvard.iq.dataverse.search.SearchException; import edu.harvard.iq.dataverse.search.SearchFields; import edu.harvard.iq.dataverse.search.SortBy; +import edu.harvard.iq.dataverse.settings.JvmSettings; import java.util.List; import java.util.Map; @@ -64,7 +67,9 @@ public class DataRetrieverAPI extends AbstractApiBean { private static final String retrieveDataPartialAPIPath = "retrieve"; @Inject - DataverseSession session; + DataverseSession session; + @Inject + PermissionsWrapper permissionsWrapper; @EJB DataverseRoleServiceBean dataverseRoleService; @@ -82,6 +87,8 @@ public class DataRetrieverAPI extends AbstractApiBean { //MyDataQueryHelperServiceBean myDataQueryHelperServiceBean; @EJB GroupServiceBean groupService; + @EJB + DatasetServiceBean datasetService; private List roleList; private DataverseRolePermissionHelper rolePermissionHelper; @@ -482,12 +489,14 @@ private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagR JsonObjectBuilder myDataCardInfo; JsonArrayBuilder rolesForCard; + DataverseRequest dataverseRequest = createDataverseRequest(authUser); for (SolrSearchResult doc : solrQueryResponse.getSolrSearchResults()){ // ------------------------------------------- // (a) Get core card data from solr // ------------------------------------------- - myDataCardInfo = doc.getJsonForMyData(); + + myDataCardInfo = doc.getJsonForMyData(isValid(doc, dataverseRequest)); if (!doc.getEntity().isInstanceofDataFile()){ String parentAlias = dataverseService.getParentAliasString(doc); @@ -510,4 +519,8 @@ private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagR return jsonSolrDocsArrayBuilder; } + + private boolean isValid(SolrSearchResult result, DataverseRequest dataverseRequest) { + return result.isValid(x -> permissionsWrapper.canUpdateDataset(dataverseRequest, datasetService.find(x.getEntityId()))); + } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 2ce06541afa..d0100bd79a4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -22,6 +22,7 @@ import edu.harvard.iq.dataverse.ThumbnailServiceWrapper; import edu.harvard.iq.dataverse.WidgetWrapper; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import java.time.LocalDate; import java.util.ArrayList; @@ -351,8 +352,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused * https://github.com/IQSS/dataverse/issues/84 */ int numRows = 10; - HttpServletRequest httpServletRequest = (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); - DataverseRequest dataverseRequest = new DataverseRequest(session.getUser(), httpServletRequest); + DataverseRequest dataverseRequest = getDataverseRequest(); List dataverses = new ArrayList<>(); dataverses.add(dataverse); solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null); @@ -1395,8 +1395,13 @@ public boolean isActivelyEmbargoed(SolrSearchResult result) { } } + private DataverseRequest getDataverseRequest() { + final HttpServletRequest httpServletRequest = (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); + return new DataverseRequest(session.getUser(), httpServletRequest); + } + public boolean isValid(SolrSearchResult result) { - return result.isValid(); + return result.isValid(x -> permissionsWrapper.canUpdateDataset(getDataverseRequest(), datasetService.find(x.getEntityId()))); } public enum SortOrder { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 6ad7f9dbbf6..72af8b158dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.logging.Logger; import jakarta.json.Json; @@ -26,6 +27,7 @@ import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.DateUtil; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; @@ -417,7 +419,7 @@ public JsonObject toJsonObject(boolean showRelevance, boolean showEntityIds, boo * * @return */ - public JsonObjectBuilder getJsonForMyData() { + public JsonObjectBuilder getJsonForMyData(boolean isValid) { JsonObjectBuilder myDataJson = json(true, true, true);// boolean showRelevance, boolean showEntityIds, boolean showApiUrls) @@ -425,7 +427,7 @@ public JsonObjectBuilder getJsonForMyData() { .add("is_draft_state", this.isDraftState()).add("is_in_review_state", this.isInReviewState()) .add("is_unpublished_state", this.isUnpublishedState()).add("is_published", this.isPublishedState()) .add("is_deaccesioned", this.isDeaccessionedState()) - .add("is_valid", this.isValid()) + .add("is_valid", isValid) .add("date_to_display_on_card", getDateToDisplayOnCard()); // Add is_deaccessioned attribute, even though MyData currently screens any deaccessioned info out @@ -1265,7 +1267,19 @@ public void setDatasetValid(Boolean datasetValid) { this.datasetValid = datasetValid == null || Boolean.valueOf(datasetValid); } - public boolean isValid() { - return datasetValid; + public boolean isValid(Predicate canUpdateDataset) { + if (this.datasetValid) { + return true; + } + if (!this.getType().equals("datasets")) { + return true; + } + if (this.isDraftState()) { + return false; + } + if (!JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false)) { + return true; + } + return !canUpdateDataset.test(this); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index cc3272413c7..9f0371178c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -145,6 +145,7 @@ public enum JvmSettings { SCOPE_UI(PREFIX, "ui"), UI_ALLOW_REVIEW_INCOMPLETE(SCOPE_UI, "allow-review-for-incomplete"), UI_SHOW_VALIDITY_FILTER(SCOPE_UI, "show-validity-filter"), + UI_SHOW_VALIDITY_LABEL(SCOPE_UI, "show-label-for-incomplete-when-published"), // NetCDF SETTINGS SCOPE_NETCDF(PREFIX, "netcdf"), diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index f69b5c35afd..a5218aedaa6 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -76,7 +76,7 @@ - + From 0cd23fbffb0d3c4a75cb3948333d4c71e7e5110e Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 5 Dec 2023 18:28:26 +0100 Subject: [PATCH 036/427] added documentation --- ...10116-incomplete-metadata-label-setting.md | 1 + .../source/installation/config.rst | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 doc/release-notes/10116-incomplete-metadata-label-setting.md diff --git a/doc/release-notes/10116-incomplete-metadata-label-setting.md b/doc/release-notes/10116-incomplete-metadata-label-setting.md new file mode 100644 index 00000000000..5fd2de2f287 --- /dev/null +++ b/doc/release-notes/10116-incomplete-metadata-label-setting.md @@ -0,0 +1 @@ +Bug fixed for the ``incomplete metadata`` label being shown for published dataset with incomplete metadata in certain scenarios. This label will now only be shown for draft versions of such datasets. This label can also be made visible for published datasets with the new option ``dataverse.api.show-label-for-incomplete-when-published``, and will be only shown on datasets with incomplete metadata that the logged-in user can edit. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index c707daaad07..bc1f01576c9 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2425,6 +2425,25 @@ Defaults to ``false``. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_ALLOW_INCOMPLETE_METADATA``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. +.. _dataverse.api.show-label-for-incomplete-when-published: + +dataverse.api.show-label-for-incomplete-when-published +++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Even when you do not allow incomplete metadata to be saved in dataverse, some metadata may end up being incomplete, e.g., after making a metadata field mandatory. Datasets where that field is +not filled out, become incomplete, and therefore can be labeled with the ``incomplete metadat`` label. By default, this label is only shown for draft datasets. However, in some situations, you +may want to show the label also for published datasets. For example, when an admin needs to have an overview of the datasets that are no longer valid after a metadata fields update, +enabling this option will show the ``incomplete metadata`` labels on published datasets, but only to the users that can edit the specific datasets. Note that you need to reindex the datasets +after changing the metadata definitions. Reindexing will update the labels and other dataset information according to the new situation. + +When enabled, published datasets with incomplete metadata will have an ``incomplete metadata`` label attached to them, but only for the datasets that the user can edit. You can list these datasets, +for example, with the validity of metadata filter shown in "My Data" page that can be turned on by enabling the :ref:`dataverse.ui.show-validity-filter` option. + +Defaults to ``false``. + +Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable +``DATAVERSE_API_SHOW_LABEL_FOR_INCOMPLETE_WHEN_PUBLISHED``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. + .. _dataverse.signposting.level1-author-limit: dataverse.signposting.level1-author-limit @@ -2475,6 +2494,8 @@ Defaults to ``false``. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_UI_ALLOW_REVIEW_FOR_INCOMPLETE``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. +.. _dataverse.ui.show-validity-filter: + dataverse.ui.show-validity-filter +++++++++++++++++++++++++++++++++ From 542a19824aaec5a24d4fd000c61c01c9629db08d Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Wed, 6 Dec 2023 17:43:21 +0100 Subject: [PATCH 037/427] Fixed forgotten code to make localeStrValue of CVV depends on :MetadataLanguages of the dataset --- src/main/webapp/metadataFragment.xhtml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 200d2917b9a..442ee5bc287 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -292,7 +292,7 @@ @@ -368,12 +368,12 @@ - + @@ -387,7 +387,7 @@ rendered="#{subdsf.datasetFieldType.allowMultiples}" label="#{bundle.select}" multiple="true" filter="#{(subdsf.datasetFieldType.controlledVocabularyValues.size() lt 10) ? 'false':'true'}" filterMatchMode="contains" showHeader="#{(subdsf.datasetFieldType.controlledVocabularyValues.size() lt 10) ? 'false':'true'}"> - +
          From 02f2edc2fddd3cf0c39ac6bbdb4381ff0b7ae744 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Thu, 7 Dec 2023 15:48:24 +0100 Subject: [PATCH 038/427] typo fix --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index bc1f01576c9..46a83610871 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2431,7 +2431,7 @@ dataverse.api.show-label-for-incomplete-when-published ++++++++++++++++++++++++++++++++++++++++++++++++++++++ Even when you do not allow incomplete metadata to be saved in dataverse, some metadata may end up being incomplete, e.g., after making a metadata field mandatory. Datasets where that field is -not filled out, become incomplete, and therefore can be labeled with the ``incomplete metadat`` label. By default, this label is only shown for draft datasets. However, in some situations, you +not filled out, become incomplete, and therefore can be labeled with the ``incomplete metadata`` label. By default, this label is only shown for draft datasets. However, in some situations, you may want to show the label also for published datasets. For example, when an admin needs to have an overview of the datasets that are no longer valid after a metadata fields update, enabling this option will show the ``incomplete metadata`` labels on published datasets, but only to the users that can edit the specific datasets. Note that you need to reindex the datasets after changing the metadata definitions. Reindexing will update the labels and other dataset information according to the new situation. From a83c5df8e6f768f13f647622dee2d2ed43099ed0 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Tue, 12 Dec 2023 15:06:00 -0500 Subject: [PATCH 039/427] #10175 docs: correct TRS JSON quotes --- doc/sphinx-guides/source/developers/big-data-support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 8d891e63317..ccdaf8940f2 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -116,7 +116,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV - export JSON_DATA="{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'trs://images/dataverse_project_logo.svg', 'fileName':'dataverse_logo.svg', 'mimeType':'image/svg+xml', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}" + export JSON_DATA='{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"trs://images/dataverse_project_logo.svg", "fileName":"dataverse_logo.svg", "mimeType":"image/svg+xml", "checksum": {"@type": "SHA-1", "@value": "123456"}}' curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA" From f6e5db29f0977187128d99f42b32c08c53af5e74 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 19 Dec 2023 16:42:44 +0100 Subject: [PATCH 040/427] option renamed: show-label-for-incomplete-when-published -> show-validity-label-when-published --- .../10116-incomplete-metadata-label-setting.md | 2 +- doc/sphinx-guides/source/installation/config.rst | 6 +++--- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +- src/main/java/edu/harvard/iq/dataverse/FilePage.java | 2 +- .../edu/harvard/iq/dataverse/search/SolrSearchResult.java | 2 +- .../java/edu/harvard/iq/dataverse/settings/JvmSettings.java | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/release-notes/10116-incomplete-metadata-label-setting.md b/doc/release-notes/10116-incomplete-metadata-label-setting.md index 5fd2de2f287..970f2c922ae 100644 --- a/doc/release-notes/10116-incomplete-metadata-label-setting.md +++ b/doc/release-notes/10116-incomplete-metadata-label-setting.md @@ -1 +1 @@ -Bug fixed for the ``incomplete metadata`` label being shown for published dataset with incomplete metadata in certain scenarios. This label will now only be shown for draft versions of such datasets. This label can also be made visible for published datasets with the new option ``dataverse.api.show-label-for-incomplete-when-published``, and will be only shown on datasets with incomplete metadata that the logged-in user can edit. +Bug fixed for the ``incomplete metadata`` label being shown for published dataset with incomplete metadata in certain scenarios. This label will now only be shown for draft versions of such datasets. This label can also be made visible for published datasets with the new option ``dataverse.ui.show-validity-label-when-published``, and will be only shown on datasets with incomplete metadata that the logged-in user can edit. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 29e53e80d68..70f35cc8797 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2444,10 +2444,10 @@ Defaults to ``false``. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_ALLOW_INCOMPLETE_METADATA``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. -.. _dataverse.api.show-label-for-incomplete-when-published: +.. _dataverse.ui.show-validity-label-when-published: -dataverse.api.show-label-for-incomplete-when-published -++++++++++++++++++++++++++++++++++++++++++++++++++++++ +dataverse.ui.show-validity-label-when-published ++++++++++++++++++++++++++++++++++++++++++++++++ Even when you do not allow incomplete metadata to be saved in dataverse, some metadata may end up being incomplete, e.g., after making a metadata field mandatory. Datasets where that field is not filled out, become incomplete, and therefore can be labeled with the ``incomplete metadata`` label. By default, this label is only shown for draft datasets. However, in some situations, you diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 5402ad01cbc..66ade0c621c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -2243,7 +2243,7 @@ private void displayPublishMessage(){ public boolean isValid() { if (valid == null) { - if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false))) { + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(false))) { final DatasetVersion newVersion = workingVersion.cloneDatasetVersion(); newVersion.setDatasetFields(newVersion.initDatasetFields()); valid = newVersion.isValid(); diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index b7afa83aea0..34624154b12 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -289,7 +289,7 @@ private void displayPublishMessage(){ public boolean isValid() { if (valid == null) { final DatasetVersion workingVersion = fileMetadata.getDatasetVersion(); - if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false))) { + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(false))) { final DatasetVersion newVersion = workingVersion.cloneDatasetVersion(); newVersion.setDatasetFields(newVersion.initDatasetFields()); valid = newVersion.isValid(); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 72af8b158dc..496ccf33ee3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -1277,7 +1277,7 @@ public boolean isValid(Predicate canUpdateDataset) { if (this.isDraftState()) { return false; } - if (!JvmSettings.UI_SHOW_VALIDITY_LABEL.lookupOptional(Boolean.class).orElse(false)) { + if (!JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(false)) { return true; } return !canUpdateDataset.test(this); diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 3c1ae689263..7ef2030a616 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -145,7 +145,7 @@ public enum JvmSettings { SCOPE_UI(PREFIX, "ui"), UI_ALLOW_REVIEW_INCOMPLETE(SCOPE_UI, "allow-review-for-incomplete"), UI_SHOW_VALIDITY_FILTER(SCOPE_UI, "show-validity-filter"), - UI_SHOW_VALIDITY_LABEL(SCOPE_UI, "show-label-for-incomplete-when-published"), + UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED(SCOPE_UI, "show-validity-label-when-published"), // NetCDF SETTINGS SCOPE_NETCDF(PREFIX, "netcdf"), From 3b2a9eb367c8af675a5ee67b83bebd43ff4fffec Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Tue, 30 Jan 2024 10:15:26 -0500 Subject: [PATCH 041/427] bump solrj per qqmyers to address zookeeper dependency --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f45e8fd9033..e8e1af188ff 100644 --- a/pom.xml +++ b/pom.xml @@ -283,7 +283,7 @@ org.apache.solr solr-solrj - 9.3.0 + 9.4.1 colt From 9110adee7c739b21803da912913b52442a7e1e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 31 Jan 2024 18:22:03 +0100 Subject: [PATCH 042/427] Sitemap more than 50000 entries IQSS#8936 --- .../iq/dataverse/sitemap/SiteMapUtil.java | 234 +++++++----------- .../iq/dataverse/sitemap/SiteMapUtilTest.java | 58 ++++- 2 files changed, 143 insertions(+), 149 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java index 86ae697f771..3077c41fa14 100644 --- a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java @@ -1,194 +1,135 @@ package edu.harvard.iq.dataverse.sitemap; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DvObjectContainer; -import edu.harvard.iq.dataverse.settings.ConfigCheckService; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.xml.XmlValidator; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.text.SimpleDateFormat; +import java.text.ParseException; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.logging.Logger; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; + +import com.redfin.sitemapgenerator.W3CDateFormat; +import com.redfin.sitemapgenerator.W3CDateFormat.Pattern; +import com.redfin.sitemapgenerator.WebSitemapGenerator; +import com.redfin.sitemapgenerator.WebSitemapUrl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObjectContainer; +import edu.harvard.iq.dataverse.settings.ConfigCheckService; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; public class SiteMapUtil { private static final Logger logger = Logger.getLogger(SiteMapUtil.class.getCanonicalName()); + private static DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - static final String SITEMAP_FILENAME_FINAL = "sitemap.xml"; static final String SITEMAP_FILENAME_STAGED = "sitemap.xml.staged"; + /** @see https://www.sitemaps.org/protocol.html#index */ + static final int SITEMAP_LIMIT = 50000; + - /** - * TODO: Handle more than 50,000 entries in the sitemap. - * - * (As of this writing Harvard Dataverse only has ~3000 dataverses and - * ~30,000 datasets.) - * - * "each Sitemap file that you provide must have no more than 50,000 URLs" - * https://www.sitemaps.org/protocol.html - * - * Consider using a third party library: "One sitemap can contain a maximum - * of 50,000 URLs. (Some sitemaps, like Google News sitemaps, can contain - * only 1,000 URLs.) If you need to put more URLs than that in a sitemap, - * you'll have to use a sitemap index file. Fortunately, WebSitemapGenerator - * can manage the whole thing for you." - * https://github.com/dfabulich/sitemapgen4j - */ public static void updateSiteMap(List dataverses, List datasets) { logger.info("BEGIN updateSiteMap"); - String sitemapPathString = getSitemapPathString(); - String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; - String finalSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_FINAL; - - Path stagedPath = Paths.get(stagedSitemapPathAndFileString); - if (Files.exists(stagedPath)) { - logger.warning("Unable to update sitemap! The staged file from a previous run already existed. Delete " + stagedSitemapPathAndFileString + " and try again."); + final String dataverseSiteUrl = SystemConfig.getDataverseSiteUrlStatic(); + final String msgErrorFormat = "Problem with %s : %s. The exception is %s"; + final String msgErrorW3CFormat = "%s isn't a valid W3C date time for %s. The exception is %s"; + final String sitemapPathString = getSitemapPathString(); + final String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; + final Path stagedSitemapPath = Paths.get(stagedSitemapPathAndFileString); + + if (Files.exists(stagedSitemapPath)) { + logger.warning(String.format( + "Unable to update sitemap! The staged file from a previous run already existed. Delete %s and try again.", + stagedSitemapPathAndFileString)); return; } - DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder documentBuilder = null; + final File directory = new File(sitemapPathString); + if (!directory.exists()) { + directory.mkdir(); + } + + // Use DAY pattern (2024-01-24), local machine timezone + final W3CDateFormat dateFormat = new W3CDateFormat(Pattern.DAY); + WebSitemapGenerator wsg = null; try { - documentBuilder = documentBuilderFactory.newDocumentBuilder(); - } catch (ParserConfigurationException ex) { - logger.warning("Unable to update sitemap! ParserConfigurationException: " + ex.getLocalizedMessage()); + wsg = WebSitemapGenerator.builder(dataverseSiteUrl, directory).autoValidate(true).dateFormat(dateFormat) + .build(); + } catch (MalformedURLException e) { + logger.warning(String.format(msgErrorFormat, "Dataverse site URL", dataverseSiteUrl, e.getLocalizedMessage())); return; } - Document document = documentBuilder.newDocument(); - - Element urlSet = document.createElement("urlset"); - urlSet.setAttribute("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9"); - urlSet.setAttribute("xmlns:xhtml", "http://www.w3.org/1999/xhtml"); - document.appendChild(urlSet); for (Dataverse dataverse : dataverses) { if (!dataverse.isReleased()) { continue; } - Element url = document.createElement("url"); - urlSet.appendChild(url); - - Element loc = document.createElement("loc"); - String dataverseAlias = dataverse.getAlias(); - loc.appendChild(document.createTextNode(SystemConfig.getDataverseSiteUrlStatic() + "/dataverse/" + dataverseAlias)); - url.appendChild(loc); - - Element lastmod = document.createElement("lastmod"); - lastmod.appendChild(document.createTextNode(getLastModDate(dataverse))); - url.appendChild(lastmod); + final String dvAlias = dataverse.getAlias(); + final String dataverseUrl = dataverseSiteUrl + "/dataverse/" + dvAlias; + final String lastModDate = getLastModDate(dataverse); + try { + final WebSitemapUrl url = new WebSitemapUrl.Options(dataverseUrl).lastMod(lastModDate).build(); + wsg.addUrl(url); + } catch (MalformedURLException e) { + logger.fine(String.format(msgErrorFormat, "dataverse URL", dataverseUrl, e.getLocalizedMessage())); + } catch (ParseException e) { + logger.fine(String.format(msgErrorW3CFormat, lastModDate, "dataverse alias " + dvAlias, e.getLocalizedMessage())); + } } for (Dataset dataset : datasets) { - if (!dataset.isReleased()) { - continue; - } - if (dataset.isHarvested()) { - continue; - } // The deaccessioned check is last because it has to iterate through dataset versions. - if (dataset.isDeaccessioned()) { + if (!dataset.isReleased() || dataset.isHarvested() || dataset.isDeaccessioned()) { continue; } - Element url = document.createElement("url"); - urlSet.appendChild(url); - - Element loc = document.createElement("loc"); - String datasetPid = dataset.getGlobalId().asString(); - loc.appendChild(document.createTextNode(SystemConfig.getDataverseSiteUrlStatic() + "/dataset.xhtml?persistentId=" + datasetPid)); - url.appendChild(loc); - - Element lastmod = document.createElement("lastmod"); - lastmod.appendChild(document.createTextNode(getLastModDate(dataset))); - url.appendChild(lastmod); - } - - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = null; - try { - transformer = transformerFactory.newTransformer(); - } catch (TransformerConfigurationException ex) { - logger.warning("Unable to update sitemap! TransformerConfigurationException: " + ex.getLocalizedMessage()); - return; - } - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); - DOMSource source = new DOMSource(document); - File directory = new File(sitemapPathString); - if (!directory.exists()) { - directory.mkdir(); - } - - boolean debug = false; - if (debug) { - logger.info("Writing sitemap to console/logs"); - StreamResult consoleResult = new StreamResult(System.out); + final String datasetPid = dataset.getGlobalId().asString(); + final String datasetUrl = dataverseSiteUrl + "/dataset.xhtml?persistentId=" + datasetPid; + final String lastModDate = getLastModDate(dataset); try { - transformer.transform(source, consoleResult); - } catch (TransformerException ex) { - logger.warning("Unable to print sitemap to the console: " + ex.getLocalizedMessage()); + final WebSitemapUrl url = new WebSitemapUrl.Options(datasetUrl).lastMod(lastModDate).build(); + wsg.addUrl(url); + } catch (MalformedURLException e) { + logger.fine(String.format(msgErrorFormat, "dataset URL", datasetUrl, e.getLocalizedMessage())); + } catch (ParseException e) { + logger.fine(String.format(msgErrorW3CFormat, lastModDate, "dataset " + datasetPid, e.getLocalizedMessage())); } } - logger.info("Writing staged sitemap to " + stagedSitemapPathAndFileString); - StreamResult result = new StreamResult(new File(stagedSitemapPathAndFileString)); + logger.info(String.format("Writing and checking sitemap file into %s", sitemapPathString)); try { - transformer.transform(source, result); - } catch (TransformerException ex) { - logger.warning("Unable to update sitemap! Unable to write staged sitemap to " + stagedSitemapPathAndFileString + ". TransformerException: " + ex.getLocalizedMessage()); - return; - } - - logger.info("Checking staged sitemap for well-formedness. The staged file is " + stagedSitemapPathAndFileString); - try { - XmlValidator.validateXmlWellFormed(stagedSitemapPathAndFileString); + wsg.write(); + if (dataverses.size() + datasets.size() > SITEMAP_LIMIT) { + wsg.writeSitemapsWithIndex(); + } } catch (Exception ex) { - logger.warning("Unable to update sitemap! Staged sitemap file is not well-formed XML! The exception for " + stagedSitemapPathAndFileString + " is " + ex.getLocalizedMessage()); - return; - } - - logger.info("Checking staged sitemap against XML schema. The staged file is " + stagedSitemapPathAndFileString); - URL schemaUrl = null; - try { - schemaUrl = new URL("https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"); - } catch (MalformedURLException ex) { - // This URL is hard coded and it's fine. We should never get MalformedURLException so we just swallow the exception and carry on. - } - try { - XmlValidator.validateXmlSchema(stagedSitemapPathAndFileString, schemaUrl); - } catch (SAXException | IOException ex) { - logger.warning("Unable to update sitemap! Exception caught while checking XML staged file (" + stagedSitemapPathAndFileString + " ) against XML schema: " + ex.getLocalizedMessage()); + final StringBuffer errorMsg = new StringBuffer("Unable to write or validate sitemap ! The exception is "); + errorMsg.append(ex.getLocalizedMessage()); + // Add causes messages exception + Throwable cause = ex.getCause(); + // Fix limit to 5 causes + final int causeLimit = 5; + int cpt = 0; + while (cause != null && cpt < causeLimit) { + errorMsg.append(" with cause ").append(cause.getLocalizedMessage()); + cause = ex.getCause(); + cpt = cpt + 1; + } + logger.warning(errorMsg.toString()); return; } - Path finalPath = Paths.get(finalSitemapPathAndFileString); - logger.info("Copying staged sitemap from " + stagedSitemapPathAndFileString + " to " + finalSitemapPathAndFileString); + logger.info(String.format("Remove staged sitemap %s", stagedSitemapPathAndFileString)); try { - Files.move(stagedPath, finalPath, StandardCopyOption.REPLACE_EXISTING); + Files.deleteIfExists(stagedSitemapPath); } catch (IOException ex) { - logger.warning("Unable to update sitemap! Unable to copy staged sitemap from " + stagedSitemapPathAndFileString + " to " + finalSitemapPathAndFileString + ". IOException: " + ex.getLocalizedMessage()); + logger.warning("Unable to delete sitemap staged file! IOException: " + ex.getLocalizedMessage()); return; } @@ -199,12 +140,11 @@ private static String getLastModDate(DvObjectContainer dvObjectContainer) { // TODO: Decide if YYYY-MM-DD is enough. https://www.sitemaps.org/protocol.html // says "The date of last modification of the file. This date should be in W3C Datetime format. // This format allows you to omit the time portion, if desired, and use YYYY-MM-DD." - return new SimpleDateFormat("yyyy-MM-dd").format(dvObjectContainer.getModificationTime()); + return dvObjectContainer.getModificationTime().toLocalDateTime().format(formatter); } public static boolean stageFileExists() { - String sitemapPathString = getSitemapPathString(); - String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; + String stagedSitemapPathAndFileString = getSitemapPathString() + File.separator + SITEMAP_FILENAME_STAGED; Path stagedPath = Paths.get(stagedSitemapPathAndFileString); if (Files.exists(stagedPath)) { logger.warning("Unable to update sitemap! The staged file from a previous run already existed. Delete " + stagedSitemapPathAndFileString + " and try again."); @@ -212,7 +152,7 @@ public static boolean stageFileExists() { } return false; } - + /** * Lookup the location where to generate the sitemap. * @@ -223,6 +163,6 @@ public static boolean stageFileExists() { */ private static String getSitemapPathString() { return JvmSettings.DOCROOT_DIRECTORY.lookup() + File.separator + "sitemap"; - } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java index 41032ffa811..704859be86f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java @@ -11,18 +11,19 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Date; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -32,6 +33,10 @@ class SiteMapUtilTest { + // see https://www.sitemaps.org/protocol.html#validating + final String xsdSitemap = "https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"; + final String xsdSitemapIndex = "https://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd"; + @TempDir Path tempDir; Path tempDocroot; @@ -105,7 +110,7 @@ void testUpdateSiteMap() throws IOException, ParseException, SAXException { // then String pathToSiteMap = tempDocroot.resolve("sitemap").resolve("sitemap.xml").toString(); assertDoesNotThrow(() -> XmlValidator.validateXmlWellFormed(pathToSiteMap)); - assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap, new URL("https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"))); + assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap, new URL(xsdSitemap))); File sitemapFile = new File(pathToSiteMap); String sitemapString = XmlPrinter.prettyPrintXml(new String(Files.readAllBytes(Paths.get(sitemapFile.getAbsolutePath())))); @@ -116,7 +121,56 @@ void testUpdateSiteMap() throws IOException, ParseException, SAXException { assertFalse(sitemapString.contains(unpublishedPid)); assertFalse(sitemapString.contains(harvestedPid)); assertFalse(sitemapString.contains(deaccessionedPid)); + } + + @Test + void testHugeSiteMap() throws IOException, ParseException, SAXException { + // given + final int nbDataverse = 50; + final int nbDataset = 50000; + + final List dataverses = new ArrayList<>(nbDataverse); + for (int i = 1; i <= nbDataverse; i++) { + final Dataverse publishedDataverse = new Dataverse(); + publishedDataverse.setAlias(String.format("publishedDv%s", i)); + publishedDataverse.setModificationTime(new Timestamp(new Date().getTime())); + publishedDataverse.setPublicationDate(new Timestamp(new Date().getTime())); + dataverses.add(publishedDataverse); + } + + final List datasets = new ArrayList<>(nbDataset); + for (int i = 1; i <= nbDataset; i++) { + final Dataset published = new Dataset(); + published.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, "10.666", String.format("FAKE/published%s", i), null, DOIServiceBean.DOI_RESOLVER_URL, null)); + published.setPublicationDate(new Timestamp(new Date().getTime())); + published.setModificationTime(new Timestamp(new Date().getTime())); + datasets.add(published); + } + // when + SiteMapUtil.updateSiteMap(dataverses, datasets); + + // then + final Path siteMapDir = tempDocroot.resolve("sitemap"); + final String pathToSiteMapIndexFile = siteMapDir.resolve("sitemap_index.xml").toString(); + assertDoesNotThrow(() -> XmlValidator.validateXmlWellFormed(pathToSiteMapIndexFile)); + assertTrue(XmlValidator.validateXmlSchema(pathToSiteMapIndexFile, new URL(xsdSitemapIndex))); + + final File sitemapFile = new File(pathToSiteMapIndexFile); + String sitemapString = XmlPrinter.prettyPrintXml(new String(Files.readAllBytes(Paths.get(sitemapFile.getAbsolutePath())), StandardCharsets.UTF_8)); + // System.out.println("sitemap: " + sitemapString); + + assertTrue(sitemapString.contains("sitemap1.xml")); + assertTrue(sitemapString.contains("sitemap2.xml")); + assertTrue(sitemapString.contains("")); + + final String pathToSiteMap1File = siteMapDir.resolve("sitemap1.xml").toString(); + assertDoesNotThrow(() -> XmlValidator.validateXmlWellFormed(pathToSiteMap1File)); + assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap1File, new URL(xsdSitemap))); + + final String pathToSiteMap2File = siteMapDir.resolve("sitemap2.xml").toString(); + assertDoesNotThrow(() -> XmlValidator.validateXmlWellFormed(pathToSiteMap2File)); + assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap2File, new URL(xsdSitemap))); } } From 528e25a50c74976449ad306c7e61b39906671ea1 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Fri, 2 Feb 2024 14:22:58 -0500 Subject: [PATCH 043/427] xsl --- .../edu/harvard/iq/dataverse/ddi-to-fo.xsl | 52 +++---------------- 1 file changed, 7 insertions(+), 45 deletions(-) diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index c25c76246d6..cb622912960 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -109,7 +109,7 @@ - + @@ -123,7 +123,7 @@ Times - + - 1 + 1 0 @@ -3884,50 +3884,17 @@ select="$catgry-freq div $catgry-sum-freq"/> - - - - - - - - - - - - - - - - - - - - 0 - - - - - + + + test="not(@missing='Y')"> - - - - - - + @@ -3947,11 +3914,6 @@ - - - From 60a2795951aa665819cfecea92c4b9e4e0a63b07 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Fri, 2 Feb 2024 17:09:24 -0500 Subject: [PATCH 044/427] xsl instead xslt --- src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl index cb622912960..26fd7c23479 100644 --- a/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl +++ b/src/main/resources/edu/harvard/iq/dataverse/ddi-to-fo.xsl @@ -123,7 +123,7 @@ Times - + @@ -156,6 +159,7 @@ rendered="${cvocOnDsf and cvPart.key.datasetFieldType.name.equals(cvocConf.get(dsf.datasetFieldType.id).getString('term-uri-field'))}"> + diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index af568170157..2eed454800d 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -213,6 +213,7 @@ + @@ -234,6 +235,7 @@ + @@ -384,6 +386,7 @@ + @@ -621,6 +624,7 @@ + diff --git a/src/main/webapp/search/advanced.xhtml b/src/main/webapp/search/advanced.xhtml index 06e662064e7..24b78ba0dc9 100644 --- a/src/main/webapp/search/advanced.xhtml +++ b/src/main/webapp/search/advanced.xhtml @@ -121,6 +121,7 @@ + From 5d07f7c85d13f1089462167d15210af90639af08 Mon Sep 17 00:00:00 2001 From: paulboon Date: Tue, 20 Feb 2024 18:04:45 +0100 Subject: [PATCH 051/427] Implemented Retention class and it's persistency in the database --- .../edu/harvard/iq/dataverse/DataFile.java | 12 ++ .../edu/harvard/iq/dataverse/Retention.java | 103 ++++++++++++++++++ .../migration/V6.1.0.99__xxxx-retention.sql | 1 + 3 files changed, 116 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/Retention.java create mode 100644 src/main/resources/db/migration/V6.1.0.99__xxxx-retention.sql diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 3d8086b142b..b8bfadddc9b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -242,6 +242,18 @@ public void setEmbargo(Embargo embargo) { this.embargo = embargo; } + @ManyToOne + @JoinColumn(name="retention_id") + private Retention retention; + + public Retention getRetention() { + return retention; + } + + public void setRetention(Retention retention) { + this.retention = retention; + } + public DataFile() { this.fileMetadatas = new ArrayList<>(); initFileReplaceAttributes(); diff --git a/src/main/java/edu/harvard/iq/dataverse/Retention.java b/src/main/java/edu/harvard/iq/dataverse/Retention.java new file mode 100644 index 00000000000..a934db62402 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/Retention.java @@ -0,0 +1,103 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.util.BundleUtil; +import jakarta.persistence.*; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Objects; + +@NamedQueries({ + @NamedQuery( name="Retention.findAll", + query = "SELECT r FROM Retention r"), + @NamedQuery( name="Retention.findById", + query = "SELECT r FROM Retention r WHERE r.id=:id"), + @NamedQuery( name="Retention.findByDateUnavailable", + query = "SELECT r FROM Retention r WHERE r.dateUnavailable=:dateUnavailable"), + @NamedQuery( name="Retention.deleteById", + query = "DELETE FROM Retention r WHERE r.id=:id") +}) +@Entity +public class Retention { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false) + private LocalDate dateUnavailable; + + @Column(columnDefinition="TEXT") + private String reason; + + @OneToMany(mappedBy="retention", cascade={ CascadeType.REMOVE, CascadeType.PERSIST}) + private List dataFiles; + + // persistency demands default constructor! + public Retention(){ + dateUnavailable = LocalDate.now().plusYears(1000); + } + + public Retention(LocalDate dateUnavailable, String reason) { + this.dateUnavailable = dateUnavailable; + this.reason = reason; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public LocalDate getDateUnavailable() { + return dateUnavailable; + } + + public void setDateUnavailable(LocalDate dateUnavailable) { + this.dateUnavailable = dateUnavailable; + } + + public String getFormattedDateUnavailable() { + return getDateUnavailable().format(DateTimeFormatter.ISO_LOCAL_DATE.withLocale(BundleUtil.getCurrentLocale())); + } + + public String getReason() { + return reason; + } + + public void setReason(String reason) { + this.reason = reason; + } + + public List getDataFiles() { + return dataFiles; + } + + public void setDataFiles(List dataFiles) { + this.dataFiles = dataFiles; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Retention retention = (Retention) o; + return id.equals(retention.id) && dateUnavailable.equals(retention.dateUnavailable) && Objects.equals(reason, retention.reason); + } + + @Override + public int hashCode() { + return Objects.hash(id, dateUnavailable, reason); + } + + @Override + public String toString() { + return "Retention{" + + "id=" + id + + ", dateUnavailable=" + dateUnavailable + + ", reason='" + reason + '\'' + + '}'; + } +} diff --git a/src/main/resources/db/migration/V6.1.0.99__xxxx-retention.sql b/src/main/resources/db/migration/V6.1.0.99__xxxx-retention.sql new file mode 100644 index 00000000000..cb23d589542 --- /dev/null +++ b/src/main/resources/db/migration/V6.1.0.99__xxxx-retention.sql @@ -0,0 +1 @@ +ALTER TABLE datafile ADD COLUMN IF NOT EXISTS retention_id BIGINT; \ No newline at end of file From 1cd596880bc13ff024b451634c13e1b2d55b0e2a Mon Sep 17 00:00:00 2001 From: paulboon Date: Wed, 21 Feb 2024 15:02:47 +0100 Subject: [PATCH 052/427] Implemented set and unset retention API --- .../iq/dataverse/RetentionServiceBean.java | 66 +++++ .../harvard/iq/dataverse/api/Datasets.java | 257 ++++++++++++++++++ .../settings/SettingsServiceBean.java | 6 + 3 files changed, 329 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java diff --git a/src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java new file mode 100644 index 00000000000..1421ac61120 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java @@ -0,0 +1,66 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; +import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; +import jakarta.ejb.EJB; +import jakarta.ejb.Stateless; +import jakarta.inject.Named; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import jakarta.persistence.Query; + +import java.util.List; + + +@Stateless +@Named +public class RetentionServiceBean { + + @PersistenceContext + EntityManager em; + + @EJB + ActionLogServiceBean actionLogSvc; + + public List findAllRetentions() { + return em.createNamedQuery("Retention.findAll", Retention.class).getResultList(); + } + + public Retention findByRetentionId(Long id) { + Query query = em.createNamedQuery("Retention.findById", Retention.class); + query.setParameter("id", id); + try { + return (Retention) query.getSingleResult(); + } catch (Exception ex) { + return null; + } + } + + public Retention merge(Retention r) { + return em.merge(r); + } + + public Long save(Retention retention, String userIdentifier) { + if (retention.getId() == null) { + em.persist(retention); + em.flush(); + } + //Not quite from a command, but this action can be done by anyone, so command seems better than Admin or other alternatives + actionLogSvc.log(new ActionLogRecord(ActionLogRecord.ActionType.Command, "retentionCreate") + .setInfo("id: " + retention.getId() + " date unavailable: " + retention.getDateUnavailable() + " reason: " + retention.getReason()).setUserIdentifier(userIdentifier)); + return retention.getId(); + } + + private int deleteById(long id, String userIdentifier) { + //Not quite from a command, but this action can be done by anyone, so command seems better than Admin or other alternatives + actionLogSvc.log(new ActionLogRecord(ActionLogRecord.ActionType.Command, "retentionDelete") + .setInfo(Long.toString(id)) + .setUserIdentifier(userIdentifier)); + return em.createNamedQuery("Retention.deleteById") + .setParameter("id", id) + .executeUpdate(); + } + public int delete(Retention retention, String userIdentifier) { + return deleteById(retention.getId(), userIdentifier); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 2181f189fc0..7d1296ad7e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -151,6 +151,9 @@ public class Datasets extends AbstractApiBean { @EJB EmbargoServiceBean embargoService; + @EJB + RetentionServiceBean retentionService; + @Inject MakeDataCountLoggingServiceBean mdcLogService; @@ -1675,6 +1678,260 @@ public Response removeFileEmbargo(@Context ContainerRequestContext crc, @PathPar } } + @POST + @AuthRequired + @Path("{id}/files/actions/:set-retention") + public Response createFileRetention(@Context ContainerRequestContext crc, @PathParam("id") String id, String jsonBody){ + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + Dataset dataset; + try { + dataset = findDatasetOrDie(id); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + + boolean hasValidTerms = TermsOfUseAndAccessValidator.isTOUAValid(dataset.getLatestVersion().getTermsOfUseAndAccess(), null); + + if (!hasValidTerms){ + return error(Status.CONFLICT, BundleUtil.getStringFromBundle("dataset.message.toua.invalid")); + } + + // client is superadmin or (client has EditDataset permission on these files and files are unreleased) + // check if files are unreleased(DRAFT?) + if ((!authenticatedUser.isSuperuser() && (dataset.getLatestVersion().getVersionState() != DatasetVersion.VersionState.DRAFT) ) || !permissionService.userOn(authenticatedUser, dataset).has(Permission.EditDataset)) { + return error(Status.FORBIDDEN, "Either the files are released and user is not a superuser or user does not have EditDataset permissions"); + } + + // check if retentions are allowed(:MinRetentionDurationInMonths), gets the :MinRetentionDurationInMonths setting variable, if 0 or not set(null) return 400 + long minRetentionDurationInMonths = 0; + try { + minRetentionDurationInMonths = Long.parseLong(settingsService.get(SettingsServiceBean.Key.MinRetentionDurationInMonths.toString())); + } catch (NumberFormatException nfe){ + if (nfe.getMessage().contains("null")) { + return error(Status.BAD_REQUEST, "No Retentions allowed"); + } + } + if (minRetentionDurationInMonths == 0){ + return error(Status.BAD_REQUEST, "No Retentions allowed"); + } + + JsonObject json = JsonUtil.getJsonObject(jsonBody); + + Retention retention = new Retention(); + + + LocalDate currentDateTime = LocalDate.now(); + LocalDate dateUnavailable = LocalDate.parse(json.getString("dateUnavailable")); + // TODO - check if dateUnavailable is specified and a valid date + + // check :MinRetentionDurationInMonths if -1 + LocalDate minRetentionDateTime = minRetentionDurationInMonths != -1 ? LocalDate.now().plusMonths(minRetentionDurationInMonths) : null; + // dateUnavailable is not in the past + if (dateUnavailable.isAfter(currentDateTime)){ + retention.setDateUnavailable(dateUnavailable); + } else { + return error(Status.BAD_REQUEST, "Date unavailable can not be in the past"); + } + + // dateAvailable is within limits + if (minRetentionDateTime != null){ + if (dateUnavailable.isBefore(minRetentionDateTime)){ + return error(Status.BAD_REQUEST, "Date unavailable can not be earlier than MinRetentionDurationInMonths: "+minRetentionDurationInMonths + " from now"); + } + } + + retention.setReason(json.getString("reason")); + + List datasetFiles = dataset.getFiles(); + List filesToRetention = new LinkedList<>(); + + // extract fileIds from json, find datafiles and add to list + if (json.containsKey("fileIds")){ + JsonArray fileIds = json.getJsonArray("fileIds"); + for (JsonValue jsv : fileIds) { + try { + DataFile dataFile = findDataFileOrDie(jsv.toString()); + filesToRetention.add(dataFile); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + } + + List orphanedRetentions = new ArrayList(); + // check if files belong to dataset + if (datasetFiles.containsAll(filesToRetention)) { + JsonArrayBuilder restrictedFiles = Json.createArrayBuilder(); + boolean badFiles = false; + for (DataFile datafile : filesToRetention) { + // superuser can overrule an existing retention, even on released files + if (datafile.isReleased() && !authenticatedUser.isSuperuser()) { + restrictedFiles.add(datafile.getId()); + badFiles = true; + } + } + if (badFiles) { + return Response.status(Status.FORBIDDEN) + .entity(NullSafeJsonBuilder.jsonObjectBuilder().add("status", ApiConstants.STATUS_ERROR) + .add("message", "You do not have permission to retention the following files") + .add("files", restrictedFiles).build()) + .type(MediaType.APPLICATION_JSON_TYPE).build(); + } + retention=retentionService.merge(retention); + // Good request, so add the retention. Track any existing retentions so we can + // delete them if there are no files left that reference them. + for (DataFile datafile : filesToRetention) { + Retention ret = datafile.getRetention(); + if (ret != null) { + ret.getDataFiles().remove(datafile); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + } + // Save merges the datafile with an retention into the context + datafile.setRetention(retention); + fileService.save(datafile); + } + //Call service to get action logged + long retentionId = retentionService.save(retention, authenticatedUser.getIdentifier()); + if (orphanedRetentions.size() > 0) { + for (Retention ret : orphanedRetentions) { + retentionService.delete(ret, authenticatedUser.getIdentifier()); + } + } + //If superuser, report changes to any released files + if (authenticatedUser.isSuperuser()) { + String releasedFiles = filesToRetention.stream().filter(d -> d.isReleased()) + .map(d -> d.getId().toString()).collect(Collectors.joining(",")); + if (!releasedFiles.isBlank()) { + actionLogSvc + .log(new ActionLogRecord(ActionLogRecord.ActionType.Admin, "retentionAddedTo") + .setInfo("Retention id: " + retention.getId() + " added for released file(s), id(s) " + + releasedFiles + ".") + .setUserIdentifier(authenticatedUser.getIdentifier())); + } + } + return ok(Json.createObjectBuilder().add("message", "Files were retentioned")); + } else { + return error(BAD_REQUEST, "Not all files belong to dataset"); + } + } + + @POST + @AuthRequired + @Path("{id}/files/actions/:unset-retention") + public Response removeFileRetention(@Context ContainerRequestContext crc, @PathParam("id") String id, String jsonBody){ + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + Dataset dataset; + try { + dataset = findDatasetOrDie(id); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + + // client is superadmin or (client has EditDataset permission on these files and files are unreleased) + // check if files are unreleased(DRAFT?) + //ToDo - here and below - check the release status of files and not the dataset state (draft dataset version still can have released files) + if ((!authenticatedUser.isSuperuser() && (dataset.getLatestVersion().getVersionState() != DatasetVersion.VersionState.DRAFT) ) || !permissionService.userOn(authenticatedUser, dataset).has(Permission.EditDataset)) { + return error(Status.FORBIDDEN, "Either the files are released and user is not a superuser or user does not have EditDataset permissions"); + } + + // check if retentions are allowed(:MinRetentionDurationInMonths), gets the :MinRetentionDurationInMonths setting variable, if 0 or not set(null) return 400 + int minRetentionDurationInMonths = 0; + try { + minRetentionDurationInMonths = Integer.parseInt(settingsService.get(SettingsServiceBean.Key.MinRetentionDurationInMonths.toString())); + } catch (NumberFormatException nfe){ + if (nfe.getMessage().contains("null")) { + return error(Status.BAD_REQUEST, "No Retentions allowed"); + } + } + if (minRetentionDurationInMonths == 0){ + return error(Status.BAD_REQUEST, "No Retentions allowed"); + } + + JsonObject json = JsonUtil.getJsonObject(jsonBody); + + List datasetFiles = dataset.getFiles(); + List retentionFilesToUnset = new LinkedList<>(); + + // extract fileIds from json, find datafiles and add to list + if (json.containsKey("fileIds")){ + JsonArray fileIds = json.getJsonArray("fileIds"); + for (JsonValue jsv : fileIds) { + try { + DataFile dataFile = findDataFileOrDie(jsv.toString()); + retentionFilesToUnset.add(dataFile); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + } + + List orphanedRetentions = new ArrayList(); + // check if files belong to dataset + if (datasetFiles.containsAll(retentionFilesToUnset)) { + JsonArrayBuilder restrictedFiles = Json.createArrayBuilder(); + boolean badFiles = false; + for (DataFile datafile : retentionFilesToUnset) { + // superuser can overrule an existing retention, even on released files + if (datafile.getRetention()==null || ((datafile.isReleased() && datafile.getRetention() != null) && !authenticatedUser.isSuperuser())) { + restrictedFiles.add(datafile.getId()); + badFiles = true; + } + } + if (badFiles) { + return Response.status(Status.FORBIDDEN) + .entity(NullSafeJsonBuilder.jsonObjectBuilder().add("status", ApiConstants.STATUS_ERROR) + .add("message", "The following files do not have retentions or you do not have permission to remove their retentions") + .add("files", restrictedFiles).build()) + .type(MediaType.APPLICATION_JSON_TYPE).build(); + } + // Good request, so remove the retention from the files. Track any existing retentions so we can + // delete them if there are no files left that reference them. + for (DataFile datafile : retentionFilesToUnset) { + Retention ret = datafile.getRetention(); + if (ret != null) { + ret.getDataFiles().remove(datafile); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + } + // Save merges the datafile with an retention into the context + datafile.setRetention(null); + fileService.save(datafile); + } + if (orphanedRetentions.size() > 0) { + for (Retention ret : orphanedRetentions) { + retentionService.delete(ret, authenticatedUser.getIdentifier()); + } + } + String releasedFiles = retentionFilesToUnset.stream().filter(d -> d.isReleased()).map(d->d.getId().toString()).collect(Collectors.joining(",")); + if(!releasedFiles.isBlank()) { + ActionLogRecord removeRecord = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "retentionRemovedFrom").setInfo("Retention removed from released file(s), id(s) " + releasedFiles + "."); + removeRecord.setUserIdentifier(authenticatedUser.getIdentifier()); + actionLogSvc.log(removeRecord); + } + return ok(Json.createObjectBuilder().add("message", "Retention(s) were removed from files")); + } else { + return error(BAD_REQUEST, "Not all files belong to dataset"); + } + } @PUT @AuthRequired diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 3b7632f3d9e..f56aca1bf32 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -528,6 +528,12 @@ Whether Harvesting (OAI) service is enabled * n: embargo enabled with n months the maximum allowed duration */ MaxEmbargoDurationInMonths, + /** This setting enables Retention capabilities in Dataverse and sets the minimum Retention duration allowed. + * 0 or not set: new retentions disabled + * -1: retention enabled, no time limit + * n: retention enabled with n months the minimum allowed duration + */ + MinRetentionDurationInMonths, /* * Include "Custom Terms" as an item in the license drop-down or not. */ From c113e666c3951d60df7b609d8aad0a99ca95f4fb Mon Sep 17 00:00:00 2001 From: paulboon Date: Wed, 21 Feb 2024 16:57:00 +0100 Subject: [PATCH 053/427] Implemented minimal retention output on the GUI --- .../iq/dataverse/DataFileServiceBean.java | 5 +- .../edu/harvard/iq/dataverse/DatasetPage.java | 4 + .../harvard/iq/dataverse/util/FileUtil.java | 31 ++++++-- src/main/java/propertyFiles/Bundle.properties | 5 ++ src/main/webapp/dataset.xhtml | 1 + src/main/webapp/file-info-fragment.xhtml | 1 + src/main/webapp/file.xhtml | 78 ++++++++++--------- src/main/webapp/resources/css/structure.css | 2 +- 8 files changed, 83 insertions(+), 44 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index c9d50bbed9d..c71d63a3c85 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -1366,7 +1366,10 @@ public Embargo findEmbargo(Long id) { DataFile d = find(id); return d.getEmbargo(); } - + + public boolean isActivelyRetended(FileMetadata fm) { + return FileUtil.isActivelyRetended(fm); + } /** * Checks if the supplied DvObjectContainer (Dataset or Collection; although * only collection-level storage quotas are officially supported as of now) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index b79f387f20b..6558b666718 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -6278,6 +6278,10 @@ private boolean containsOnlyActivelyEmbargoedFiles(List selectedFi return true; } + public boolean isActivelyRetended(List fmdList) { + return FileUtil.isActivelyRetended(fmdList); + } + public String getIngestMessage() { return BundleUtil.getStringFromBundle("file.ingestFailed.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 8decf74fe13..e40464ec80e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -21,14 +21,8 @@ package edu.harvard.iq.dataverse.util; -import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DataFile.ChecksumType; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Embargo; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; @@ -1776,6 +1770,29 @@ public static boolean isActivelyEmbargoed(List fmdList) { return false; } + public static boolean isActivelyRetended(DataFile df) { + Retention e = df.getRetention(); + if (e != null) { + LocalDate endDate = e.getDateUnavailable(); + if (endDate != null && endDate.isBefore(LocalDate.now())) { + return true; + } + } + return false; + } + + public static boolean isActivelyRetended(FileMetadata fileMetadata) { + return isActivelyRetended(fileMetadata.getDataFile()); + } + + public static boolean isActivelyRetended(List fmdList) { + for (FileMetadata fmd : fmdList) { + if (isActivelyRetended(fmd)) { + return true; + } + } + return false; + } public static String getStorageDriver(DataFile dataFile) { String storageIdentifier = dataFile.getStorageIdentifier(); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 3d141554326..c7b3ccde7ba 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -15,6 +15,7 @@ embargoed=Embargoed embargoedaccess=Embargoed with Access embargoedandrestricted=Embargoed and then Restricted embargoedandrestrictedaccess=Embargoed and then Restricted with Access +retended=Retended incomplete=Incomplete metadata valid=Valid find=Find @@ -30,6 +31,9 @@ embargoed.wasthrough=Was embargoed until embargoed.willbeuntil=Draft: will be embargoed until embargo.date.invalid=Date is outside the allowed range: ({0} to {1}) embargo.date.required=An embargo date is required +retention.after=Retended after +retention.isfrom=Is retended from +retention.willbeafter=Draft: will be retended from cancel=Cancel ok=OK saveChanges=Save Changes @@ -2174,6 +2178,7 @@ file.metadataTab.fileMetadata.type.label=Type file.metadataTab.fileMetadata.description.label=Description file.metadataTab.fileMetadata.publicationDate.label=Publication Date file.metadataTab.fileMetadata.embargoReason.label=Embargo Reason +file.metadataTab.fileMetadata.retentionReason.label=Retention Reason file.metadataTab.fileMetadata.metadataReleaseDate.label=Metadata Release Date file.metadataTab.fileMetadata.depositDate.label=Deposit Date file.metadataTab.fileMetadata.hierarchy.label=File Path diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index e50e68ec162..4f632b71a75 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -132,6 +132,7 @@ + diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 72fe279fbf8..be502d211d2 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -64,6 +64,7 @@
          +
          diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index f69b5c35afd..3a49af29329 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -43,7 +43,7 @@
          #{FilePage.fileMetadata.label} - +
          @@ -64,22 +64,23 @@

          - + - + + - + + value="#{bundle['file.DatasetVersion']} #{FilePage.fileMetadata.datasetVersion.versionNumber}.#{FilePage.fileMetadata.datasetVersion.minorVersionNumber}"/>
          @@ -98,9 +99,9 @@ - -
          @@ -145,11 +146,11 @@ - - -
          @@ -192,7 +193,7 @@
          -
            - @@ -227,12 +228,12 @@
            #{bundle['file.editBtn']} - +
            #{bundle['metrics.file.title']} - -
            @@ -317,7 +318,7 @@
            -
            @@ -351,7 +352,7 @@ - @@ -376,7 +377,7 @@ - +
              @@ -393,8 +394,8 @@
              - -
              @@ -443,7 +444,7 @@ #{bundle['file.dataFilesTab.metadata.addBtn']}
            @@ -483,7 +484,7 @@
            -
            @@ -495,7 +496,7 @@ #{bundle['file.metadata.filetags']} -
            +
            @@ -504,7 +505,7 @@
            - + @@ -566,6 +567,13 @@ #{FilePage.file.embargo.reason} + + + #{bundle['file.metadataTab.fileMetadata.retentionReason.label']} + + #{FilePage.file.retention.reason} + + #{bundle['file.metadataTab.fileMetadata.size.label']} @@ -624,8 +632,8 @@ - - + +

            #{bundle['file.share.tip']}

            @@ -634,7 +642,7 @@ #{bundle.close}
            - + @@ -645,7 +653,7 @@ - + @@ -657,7 +665,7 @@ - + @@ -678,7 +686,7 @@ #{bundle.close}
          - +

          #{bundle['file.compute.fileAccessDenied']}

          @@ -690,7 +698,7 @@