Merge pull request #58 from PaulBoon/signposting

Signposting

conf/docker-aio/0prep_deps.sh

@@ -10,12 +10,11 @@ if [ ! -e dv/deps/payara-5.2020.6.zip ]; then
 	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip  -O dv/deps/payara-5.2020.6.zip
 fi
 
-if [ ! -e dv/deps/solr-7.7.2dv.tgz ]; then
+if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then
 	echo "solr dependency prep"
 	# schema changes *should* be the only ones...
-	cd dv/deps/
-	#wget https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz -O solr-7.3.0dv.tgz
-	wget https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz -O solr-7.7.2dv.tgz
+	cd dv/deps/	
+	wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz -O solr-8.8.1dv.tgz
 	cd ../../
 fi
 

conf/docker-aio/1prep.sh

@@ -4,9 +4,9 @@
 # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world
 
 mkdir -p testdata/doc/sphinx-guides/source/_static/util/
-cp ../solr/7.7.2/schema*.xml testdata/
-cp ../solr/7.7.2/solrconfig.xml testdata/
-cp ../solr/7.7.2/updateSchemaMDB.sh testdata/
+cp ../solr/8.8.1/schema*.xml testdata/
+cp ../solr/8.8.1/solrconfig.xml testdata/
+cp ../solr/8.8.1/updateSchemaMDB.sh testdata/
 cp ../jhove/jhove.conf testdata/
 cp ../jhove/jhoveConfig.xsd testdata/
 cd ../../

conf/docker-aio/c8.dockerfile

@@ -17,7 +17,7 @@ COPY testdata/sushi_sample_logs.json /tmp/
 COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
 COPY httpd.conf /etc/httpd/conf 
-RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.7.2dv.tgz 
+RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz 
 RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.6.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
@@ -28,9 +28,9 @@ RUN sudo -u postgres /usr/bin/initdb /var/lib/pgsql/data
 
 # copy configuration related files
 RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/
-RUN cp -r /opt/solr-7.7.2/server/solr/configsets/_default /opt/solr-7.7.2/server/solr/collection1
-RUN cp /tmp/dv/schema*.xml /opt/solr-7.7.2/server/solr/collection1/conf/
-RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.7.2/server/solr/collection1/conf/solrconfig.xml
+RUN cp -r /opt/solr-8.8.1/server/solr/configsets/_default /opt/solr-8.8.1/server/solr/collection1
+RUN cp /tmp/dv/schema*.xml /opt/solr-8.8.1/server/solr/collection1/conf/
+RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml
 
 # skipping payara user and solr user (run both as root)
 

conf/docker-aio/entrypoint.bash

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 export LANG=en_US.UTF-8
 sudo -u postgres /usr/bin/pg_ctl start -D /var/lib/pgsql/data &
-cd /opt/solr-7.7.2/
+cd /opt/solr-8.8.1/
 # TODO: Run Solr as non-root and remove "-force".
 bin/solr start -force
 bin/solr create_core -c collection1 -d server/solr/collection1/conf -force

conf/solr/7.7.2/schema.xml → conf/solr/8.8.1/schema.xml

@@ -38,7 +38,7 @@
     catchall "text" field, and use that for searching.
 -->
 
-<schema name="default-config" version="1.6">
+<schema name="default-config" version="1.7">
     <!-- attribute "name" is the name of this schema and is only used for display purposes.
        version="x.y" is Solr's version number for the schema syntax and 
        semantics.  It should not normally be changed by applications.
@@ -566,6 +566,7 @@
             <filter class="solr.KeywordRepeatFilterFactory" />
             <filter class="solr.PorterStemFilterFactory"/>
             <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+            <filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true" />
         </analyzer>
         <analyzer type="query">
             <tokenizer class="solr.StandardTokenizerFactory"/>
@@ -616,6 +617,7 @@
         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
         <filter class="solr.PorterStemFilterFactory"/>
         <filter class="solr.FlattenGraphFilterFactory" />
+        <filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true" />
       </analyzer>
       <analyzer type="query">
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>

doc/release-notes/4259-java-11.md

@@ -6,34 +6,41 @@ The Dataverse software now runs and requires Java 11. This will provide performa
 
 ## Notes to Admins
 
-### Java 11
+### Java 11 Upgrade
 
 There are some things to note and keep in mind regarding the move to Java 11:
 
-- You should install the JDK/JRE following your usual methods, depending on your operating system.
+- You should install the JDK/JRE following your usual methods, depending on your operating system. An example of this on a RHEL/CentOS 7 or RHEL/CentOS 8 system is:
+
+    `$ sudo yum remove java-1.8.0-openjdk java-1.8.0-openjdk-devel java-1.8.0-openjdk-headless`  
+
+    `$ sudo yum install java-11-openjdk-devel`  
+
+    The `remove` command may provide an error message if -headless isn't installed.
+
 - We targeted and tested Java 11, but 11+ will likely work. Java 11 was targeted because of its long term support.
-- If you're moving from a Dataverse installation that was previously running Dataverse 4.x, wou will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if you first installed Dataverse version was in the 5.x series.
+- If you're moving from a Dataverse installation that was previously running Glassfish 4.x (typically this would be Dataverse Software 4.x), you will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if your first installed Dataverse version was running Payara 5.x (typically Dataverse Software 5.x).
 
 ### New JVM and DB Options
 
 #### New JVM Options
 
-For installations that were previously running on Dataverse 4.x, the following JVM Options have been added in support of Java 11:
-
-<jvm-options>[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.base/java.lang=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.base/java.net=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.base/java.nio=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.base/java.util=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.management/sun.management=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED</jvm-options>
-<jvm-options>[9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED</jvm-options>
+For installations that were previously running on Dataverse Software 4.x, the following JVM Options have been added in support of Java 11:
+
+- `<jvm-options>[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.base/java.lang=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.base/java.net=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.base/java.nio=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.base/java.util=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.management/sun.management=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED</jvm-options>`
+- `<jvm-options>[9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED</jvm-options>`
 
 ## Upgrade Instructions
 
@@ -50,32 +57,45 @@ For installations that were previously running on Dataverse 4.x, the following J
 - remove the generated directory: 
 `rm -rf <payara install path>/payara/domains/domain1/generated`
 
-3\. (only required for installations previously running on Dataverse 4.x)
+3\. (only required for installations previously running on Dataverse Software 4.x)
 
 In domain.xml:
 
-Remove the following JVM options from the domain><configs><config name-"server-config"><java-config> element::
+Remove the following JVM options from the domain><configs><config name-"server-config"><java-config> element:
+
+`<jvm-options>-Djava.endorsed.dirs=/usr/local/payara5/glassfish/modules/endorsed:/usr/local/payara5/glassfish/lib/endorsed</jvm-options>`  
 
-  <jvm-options>-Djava.endorsed.dirs=/usr/local/payara5/glassfish/modules/endorsed:/usr/local/payara5/glassfish/lib/endorsed</jvm-options>
-  459a448
-  <jvm-options>-Djava.ext.dirs=${com.sun.aas.javaRoot}/lib/ext${path.separator}${com.sun.aas.javaRoot}/jre/lib/ext${path.separator}${com.sun.aas.instanceRoot}/lib/ext</jvm-options>
+`<jvm-options>-Djava.ext.dirs=${com.sun.aas.javaRoot}/lib/ext${path.separator}${com.sun.aas.javaRoot}/jre/lib/ext${path.separator}${com.sun.aas.instanceRoot}/lib/ext</jvm-options>`  
 
 Add the following JVM options to the domain><configs><config name-"server-config"><java-config> element:  
 
-  <jvm-options>[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.base/java.lang=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.base/java.net=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.base/java.nio=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.base/java.util=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.management/sun.management=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED</jvm-options>
-  <jvm-options>[9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED</jvm-options>
+`<jvm-options>[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.base/java.lang=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.base/java.net=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.base/java.nio=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.base/java.util=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.management/sun.management=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED</jvm-options>`  
+
+`<jvm-options>[9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED</jvm-options>`  
 
 4\. Start Payara
 

doc/release-notes/5.0-release-notes.md

@@ -250,33 +250,46 @@ The instructions below describe the upgrade procedure based on moving an existin
    `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/`  
    `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/`
 
-8. In domain.xml:
+8. Make the following changes in **domain.xml**:
 
- Replace the -XX:PermSize and -XX:MaxPermSize JVM options with -XX:MetaspaceSize and -XX:MaxMetaspaceSize.
+   Replace the `-XX:PermSize` and `-XX:MaxPermSize` JVM options with `-XX:MetaspaceSize` and `-XX:MaxMetaspaceSize`
 
+   ```
    <jvm-options>-XX:MetaspaceSize=256m</jvm-options>
    <jvm-options>-XX:MaxMetaspaceSize=512m</jvm-options>
+   ```
 
-Add the below JVM options beneath the -Ddataverse settings:  
+   Add the below JVM options beneath the -Ddataverse settings:  
 
+   ```
    <jvm-options>-Dfish.payara.classloading.delegate=false</jvm-options>
    <jvm-options>-XX:+UseG1GC</jvm-options>
    <jvm-options>-XX:+UseStringDeduplication</jvm-options>
    <jvm-options>-XX:+DisableExplicitGC</jvm-options>
+   ```
 
-Also in domain.xml, replace the follow element:
+   Replace the following element:
 
-  `<jdbc-connection-pool datasource-classname="org.apache.derby.jdbc.EmbeddedXADataSource" name="__TimerPool" res-type="javax.sql.XADataSource"> <property name="databaseName" value="${com.sun.aas.instanceRoot}/lib/databases/ejbtimer"></property><property name="connectionAttributes" value=";create=true"></property></jdbc-connection-pool>`
+   ```
+   <jdbc-connection-pool datasource-classname="org.apache.derby.jdbc.EmbeddedXADataSource" name="__TimerPool" res-type="javax.sql.XADataSource">
+      <property name="databaseName" value="${com.sun.aas.instanceRoot}/lib/databases/ejbtimer"></property>
+      <property name="connectionAttributes" value=";create=true"></property>
+   </jdbc-connection-pool>
+   ```
 
-with
+   with
 
-  `<jdbc-connection-pool datasource-classname="org.h2.jdbcx.JdbcDataSource" name="__TimerPool" res-type="javax.sql.XADataSource"> <property name="URL" value="jdbc:h2:${com.sun.aas.instanceRoot}/lib/databases/ejbtimer;AUTO_SERVER=TRUE"></property> </jdbc-connection-pool>`
+  ```
+  <jdbc-connection-pool datasource-classname="org.h2.jdbcx.JdbcDataSource" name="__TimerPool" res-type="javax.sql.XADataSource">
+    <property name="URL" value="jdbc:h2:${com.sun.aas.instanceRoot}/lib/databases/ejbtimer;AUTO_SERVER=TRUE"></property>
+  </jdbc-connection-pool>
+  ```
 
-9. Change any full pathnames /usr/local/glassfish4/... to /usr/local/payara5/... or whatever it is in your case. (Specifically check the -Ddataverse.files.directory and -Ddataverse.files.file.directory JVM options)
+9. Change any full pathnames matching `/usr/local/glassfish4/...` to `/usr/local/payara5/...` or whatever it is in your case. Specifically check the `-Ddataverse.files.directory` and `-Ddataverse.files.file.directory` JVM options.
 
 10.   In domain1/config/jhove.conf, change the hard-coded /usr/local/glassfish4 path, as above.
 
-(Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`:
+   (Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`:
 
    `sudo chown -R dataverse /usr/local/payara5/glassfish/domains/domain1`
    `sudo chown -R dataverse /usr/local/payara5/glassfish/lib`
@@ -309,20 +322,20 @@ with
 
 3. (Required for installations using DataCite) Add the JVM option doi.dataciterestapiurlstring
 
-For production environments:
+   For production environments:
 
    `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.datacite.org"`
 
-For test environments: 
+   For test environments: 
 
    `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.test.datacite.org"`
 
-The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example:
+   The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example:
 
    `/usr/local/payara5/bin/asadmin delete-jvm-options "\-Ddoi.mdcbaseurlstring=https\://api.test.datacite.org"`
 
 4. (Recommended for installations using DataCite) Pre-register DOIs
 
-Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. 
+   Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. 
 
-Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. 
+   Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. 

doc/release-notes/7084-crawlable-file-access.md

@@ -0,0 +1,29 @@
+ ## Release Highlights
+
+### A new file access API
+
+A new api offers *crawlable* access view of the folders and files within a datset:
+
+```
+  /api/datasets/<dataset id>/dirindex/
+```
+
+will output a simple html listing, based on the standard Apache
+directory index, with Access API download links for individual files,
+and recursive calls to the API above for sub-folders. (See the
+documentation entry in the guides for more information).
+
+Using this API, ``wget --recursive`` (or similar crawling client) can
+be used to download all the files in a dataset, preserving the file
+names and folder structure; without having to use the download-as-zip
+API. In addition to being faster (zipping is a relatively
+resource-intensive operation on the server side), this process can be
+restarted if interrupted (with ``wget --continue`` or equivalent) -
+unlike zipped multi-file downloads that always have to start from the
+beginning.
+
+On a system that uses S3 with download redirects, the individual file
+downloads will be handled by S3 directly, without having to be proxied
+through the Dataverse application.
+
+

doc/release-notes/7205-orig-file-size.md

@@ -0,0 +1,11 @@
+## Notes to Admins
+
+Beginning in Dataverse Software 4.10, the size of the saved original file (for an ingested tabular datafile) was stored in the database. For files added before this change, we provide an API that retrieves and permanently stores the sizes for any already existing saved originals. See [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) for more information.
+
+This was documented as a step in previous release notes, but we are noting it in these release notes to give it more visibility.
+
+## Upgrade Instructions
+
+X./ Retroactively store original file size
+
+Use the [Datafile Integrity API](https://guides.dataverse.org/en/5.4/api/native-api.html#datafile-integrity) to ensure that the sizes of all original files are stored in the database.

doc/release-notes/7373-solr-upgrade.md

@@ -0,0 +1,22 @@
+### Solr Update
+
+With this release we upgrade to the latest available stable release in the Solr 8.x branch. We recommend a fresh installation of Solr 8.8.1 (the index will be empty)
+followed by an "index all".
+
+Before you start the "index all", Dataverse will appear to be empty because
+the search results come from Solr. As indexing progresses, partial results will 
+appear until indexing is complete.
+
+
+See http://guides.dataverse.org/installation/prerequisites.html#installing-solr
+
+[for the additional upgrade steps section]
+
+Run the script updateSchemaMDB.sh to generate updated solr schema files and preserve any other custom fields in your Solr configuration.
+For example: (modify the path names as needed)
+cd /usr/local/solr-8.8.1/server/solr/collection1/conf
+wget https://github.com/IQSS/dataverse/releases/download/v5.4/updateSchemaMDB.sh
+chmod +x updateSchemaMDB.sh
+./updateSchemaMDB.sh -t .
+
+See http://guides.dataverse.org/en/5.4/admin/metadatacustomization.html?highlight=updateschemamdb for more information.