Merge pull request IQSS#7514 from OdumInstitute/4259_java_11

IQSS#4259 support Java 11

conf/docker-aio/1prep.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # move things necessary for integration tests into build context.
 # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world
@@ -13,10 +13,20 @@ cd ../../
 cp -r scripts conf/docker-aio/testdata/
 cp doc/sphinx-guides/source/_static/util/createsequence.sql conf/docker-aio/testdata/doc/sphinx-guides/source/_static/util/
 
+wget -q https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
+tar xfz apache-maven-3.6.3-bin.tar.gz
+mkdir maven
+mv apache-maven-3.6.3/* maven/
+echo "export JAVA_HOME=/usr/lib/jvm/jre-openjdk" > maven/maven.sh
+echo "export M2_HOME=../maven" >> maven/maven.sh
+echo "export MAVEN_HOME=../maven" >> maven/maven.sh
+echo "export PATH=../maven/bin:${PATH}" >> maven/maven.sh
+chmod 0755 maven/maven.sh
+
 # not using dvinstall.zip for setupIT.bash; but still used in install.bash for normal ops
-mvn clean
+source maven/maven.sh && mvn clean
 ./scripts/installer/custom-build-number
-mvn package
+source maven/maven.sh && mvn package
 cd scripts/installer
 make clean
 make

conf/docker-aio/c8.dockerfile

@@ -2,7 +2,7 @@ FROM centos:8
 # OS dependencies
 # PG 10 is the default in centos8; keep the repo comment for when we bump to 11+
 #RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
-RUN yum install -y java-1.8.0-openjdk-devel postgresql-server sudo epel-release unzip perl curl httpd
+RUN yum install -y java-11-openjdk-devel postgresql-server sudo epel-release unzip curl httpd
 RUN yum install -y jq lsof awscli
 
 # copy and unpack dependencies (solr, payara)
@@ -21,10 +21,10 @@ RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.7.2dv.tgz
 RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.6.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
-COPY domain-restmonitor.xml /opt/payara5/glassfish/domains/domain1/config/domain.xml
+# this dies under Java 11, do we keep it?
+#COPY domain-restmonitor.xml /opt/payara5/glassfish/domains/domain1/config/domain.xml
 
-#RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data
-RUN sudo -u postgres /usr/pgsql-9.6/bin/initdb -D /var/lib/pgsql/data
+RUN sudo -u postgres /usr/bin/initdb /var/lib/pgsql/data
 
 # copy configuration related files
 RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/

conf/docker-aio/entrypoint.bash

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 export LANG=en_US.UTF-8
-#sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data &
-sudo -u postgres /usr/pgsql-9.6/bin/postgres -D /var/lib/pgsql/data &
+sudo -u postgres /usr/bin/pg_ctl start -D /var/lib/pgsql/data &
 cd /opt/solr-7.7.2/
 # TODO: Run Solr as non-root and remove "-force".
 bin/solr start -force

conf/docker-aio/readme.md

@@ -4,7 +4,7 @@ First pass docker all-in-one image, intended for running integration tests again
 Also usable for normal development and system evaluation; not intended for production.
 
 ### Requirements:
- - java8 compiler, maven, make, wget, docker
+ - java11 compiler, maven, make, wget, docker
 
 ### Quickstart:
  - in the root of the repository, run `./conf/docker-aio/prep_it.bash`

conf/docker-aio/run-test-suite.sh

@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT -Ddataverse.test.baseurl=$dvurl
+source maven/maven.sh mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT -Ddataverse.test.baseurl=$dvurl

conf/docker-aio/testscripts/db.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-psql -U postgres -c "CREATE ROLE dvnapp UNENCRYPTED PASSWORD 'secret' SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN" template1
+psql -U postgres -c "CREATE ROLE dvnapp PASSWORD 'secret' SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN" template1
 psql -U dvnapp -c 'CREATE DATABASE "dvndb" WITH OWNER = "dvnapp"' template1

doc/release-notes/4259-java-11.md

@@ -0,0 +1,91 @@
+## Release Highlights
+
+### Java 11
+
+The Dataverse software now runs and requires Java 11. This will provide performance and security enhancements, allows developers to take advantage of new and updated Java features, and moves the project to a platform with better longer term support. This upgrade requires a few extra steps in the release process, outlined below.
+
+## Notes to Admins
+
+### Java 11
+
+There are some things to note and keep in mind regarding the move to Java 11:
+
+- You should install the JDK/JRE following your usual methods, depending on your operating system.
+- We targeted and tested Java 11, but 11+ will likely work. Java 11 was targeted because of its long term support.
+- If you're moving from a Dataverse installation that was previously running Dataverse 4.x, wou will need to adjust some JVM options in domain.xml as part of the upgrade process. We've provided these optional steps below. These steps are not required if you first installed Dataverse version was in the 5.x series.
+
+### New JVM and DB Options
+
+#### New JVM Options
+
+For installations that were previously running on Dataverse 4.x, the following JVM Options have been added in support of Java 11:
+
+<jvm-options>[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.base/java.lang=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.base/java.net=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.base/java.nio=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.base/java.util=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.management/sun.management=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED</jvm-options>
+<jvm-options>[9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED</jvm-options>
+
+## Upgrade Instructions
+
+1\. Undeploy the previous version.
+
+- `<payara install path>/bin/asadmin list-applications`
+- `<payara install path>/bin/asadmin undeploy dataverse<-version>`
+
+(where `<payara install path>` is where Payara 5 is installed, for example: `/usr/local/payara5`)
+
+2\. Stop Payara and remove the generated directory
+
+- `service payara stop`
+- remove the generated directory: 
+`rm -rf <payara install path>/payara/domains/domain1/generated`
+
+3\. (only required for installations previously running on Dataverse 4.x)
+
+In domain.xml:
+
+Remove the following JVM options from the domain><configs><config name-"server-config"><java-config> element::
+
+  <jvm-options>-Djava.endorsed.dirs=/usr/local/payara5/glassfish/modules/endorsed:/usr/local/payara5/glassfish/lib/endorsed</jvm-options>
+  459a448
+  <jvm-options>-Djava.ext.dirs=${com.sun.aas.javaRoot}/lib/ext${path.separator}${com.sun.aas.javaRoot}/jre/lib/ext${path.separator}${com.sun.aas.instanceRoot}/lib/ext</jvm-options>
+
+Add the following JVM options to the domain><configs><config name-"server-config"><java-config> element:  
+
+  <jvm-options>[9|]--add-opens=java.base/jdk.internal.loader=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.base/java.lang=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.base/java.net=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.base/java.nio=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.base/java.util=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.base/sun.nio.ch=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.management/sun.management=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jrt=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.base/sun.net.www.protocol.jar=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED</jvm-options>
+  <jvm-options>[9|]--add-opens=java.logging/java.util.logging=ALL-UNNAMED</jvm-options>
+
+4\. Start Payara
+
+- `service payara start`
+
+5\. Deploy this version.
+
+- `<payara install path>/bin/asadmin deploy dataverse-5.4.war`
+
+6\. Restart payara
+
+- `service payara stop`
+- `service payara start`

doc/sphinx-guides/source/developers/dev-environment.rst

@@ -25,7 +25,7 @@ Windows is not well supported, unfortunately, but Vagrant and Minishift environm
 Install Java
 ~~~~~~~~~~~~
 
-Dataverse requires Java 8.
+Dataverse requires Java 11.
 
 We suggest downloading OpenJDK from https://adoptopenjdk.net
 

doc/sphinx-guides/source/installation/prerequisites.rst

@@ -19,7 +19,7 @@ We assume you plan to run Dataverse on Linux and we recommend RHEL/CentOS, which
 Java
 ----
 
-Dataverse requires Java SE 8 (8u74/JDK 1.8.0u74 or higher).
+Dataverse requires Java SE 11 (or higher).
 
 Installing Java
 ===============
@@ -30,17 +30,14 @@ The Oracle JDK can be downloaded from http://www.oracle.com/technetwork/java/jav
 
 On a RHEL/CentOS, install OpenJDK (devel version) using yum::
 
-	# yum install java-1.8.0-openjdk-devel
+	# sudo yum install java-11-openjdk
 
-If you have multiple versions of Java installed, Java 8 should be the default when ``java`` is invoked from the command line. You can test this by running ``java -version``.
+If you have multiple versions of Java installed, Java 11 should be the default when ``java`` is invoked from the command line. You can test this by running ``java -version``.
 
-On RHEL/CentOS you can make Java 8 the default with the ``alternatives`` command, having it prompt you to select the version of Java from a list::
+On RHEL/CentOS you can make Java 11 the default with the ``alternatives`` command, having it prompt you to select the version of Java from a list::
 
         # alternatives --config java
 
-If you don't want to be prompted, here is an example of the non-interactive invocation::
-
-        # alternatives --set java /usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java
 
 .. _payara:
 

pom.xml

@@ -39,11 +39,7 @@
         <mockito.version>2.28.2</mockito.version>
         <flyway.version>5.2.4</flyway.version>
         <jhove.version>1.20.1</jhove.version>
-        <!--
-        Jacoco 0.8.2 seems to break Netbeans code coverage integration so we'll use 0.8.1 instead.
-        See https://github.com/jacoco/jacoco/issues/772 for discussion of how the XML changed.
-        -->
-        <jacoco.version>0.8.1</jacoco.version>
+        <jacoco.version>0.8.6</jacoco.version>
     </properties>
     <pluginRepositories>
         <pluginRepository>
@@ -816,27 +812,6 @@
             <id>all-unit-tests</id>
         </profile>
         <!-- TODO: Add a profile to run API tests (integration tests that end in IT.java. See conf/docker-aio/run-test-suite.sh -->
-      <profile>
-        <id>Java8</id>
-        <activation>
-          <jdk>1.8</jdk>
-        </activation>
-        <build>
-          <plugins>
-            <plugin>
-              <groupId>org.apache.maven.plugins</groupId>
-              <artifactId>maven-compiler-plugin</artifactId>
-              <version>3.8.0</version>
-              <configuration>
-                <source>1.8</source>
-                <target>1.8</target>
-                <!-- for use with `mvn -DcompilerArgument=-Xlint:unchecked compile` -->
-                <compilerArgument>${compilerArgument}</compilerArgument>
-              </configuration>
-            </plugin>
-          </plugins>
-        </build>
-      </profile>
       <profile>
         <id>Java9Plus</id>
         <activation>
@@ -849,7 +824,7 @@
               <artifactId>maven-compiler-plugin</artifactId>
               <version>3.8.0</version>
               <configuration>
-                <release>8</release>
+                <release>11</release>
                 <!-- for use with `mvn -DcompilerArgument=-Xlint:unchecked compile` -->
                 <compilerArgument>${compilerArgument}</compilerArgument>
               </configuration>

scripts/vagrant/install-dataverse.sh

@@ -9,7 +9,7 @@ if [ ! -f $WAR ]; then
   echo "no war file found... building"
   #echo "Installing nss on CentOS 6 to avoid java.security.KeyException while building war file: https://github.com/IQSS/dataverse/issues/2744"
   #yum install -y nss
-  su $SUDO_USER -s /bin/sh -c "cd /dataverse && mvn -q package"
+  su $SUDO_USER -s /bin/sh -c "cd /dataverse && source /etc/profile.d/maven.sh && mvn -q package"
 fi
 cd /dataverse/scripts/installer
 

scripts/vagrant/setup.sh

@@ -7,29 +7,36 @@ dnf install -qy wget
 # python3 and psycopg2 for the Dataverse installer
 dnf install -qy python3 python3-psycopg2
 
-# Add JQ (TODO: just install this from EPEL?)
+# JQ
 echo "Installing jq for the setup scripts"
-wget -q http://stedolan.github.io/jq/download/linux64/jq
-chmod +x jq
-# this is where EPEL puts it
-sudo mv jq /usr/bin/jq
+dnf install -qy epel-release
+dnf install -qy jq
 
 echo "Adding Shibboleth yum repo"
 cp /dataverse/conf/vagrant/etc/yum.repos.d/shibboleth.repo /etc/yum.repos.d
 # Uncomment this (and other shib stuff below) if you want
 # to use Vagrant (and maybe PageKite) to test Shibboleth.
 #yum install -y shibboleth shibboleth-embedded-ds
 
-# java configuration et al
-dnf install -qy java-1.8.0-openjdk-headless maven httpd mod_ssl unzip
-alternatives --set java /usr/lib/jvm/jre-1.8.0-openjdk/bin/java
-# do we need javac? the symlink is tied to package version...
-# /etc/alternatives/javac -> /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.262.b10-0.el8_2.x86_64/bin/javac
-#alternatives --set javac /usr/lib/jvm/java-1.8.0-openjdk.x86_64/bin/javac
+# java configuration et alia
+dnf install -qy java-11-openjdk-devel httpd mod_ssl unzip
+alternatives --set java /usr/lib/jvm/jre-11-openjdk/bin/java
 java -version
-#javac -version
+
+# maven included in centos8 requires 1.8.0 - download binary instead
+# current version is 3.6.3 - requires newer jacoco.pom.xml
+wget -q https://www-us.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
+tar xfz apache-maven-3.6.3-bin.tar.gz
+mkdir /opt/maven
+mv apache-maven-3.6.3/* /opt/maven/
+echo "export JAVA_HOME=/usr/lib/jvm/jre-openjdk" > /etc/profile.d/maven.sh
+echo "export M2_HOME=/opt/maven" >> /etc/profile.d/maven.sh
+echo "export MAVEN_HOME=/opt/maven" >> /etc/profile.d/maven.sh
+echo "export PATH=/opt/maven/bin:${PATH}" >> /etc/profile.d/maven.sh
+chmod 0755 /etc/profile.d/maven.sh
 
 # disable centos8 postgresql module and install postgresql10-server
+# note: postgresql10 because 9.6 isn't backwards compatible with python3-psycopg2
 dnf -qy module disable postgresql
 dnf install -qy https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
 dnf install -qy postgresql10-server

src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java

@@ -250,11 +250,12 @@ public Response loadDatasetFields(File file) {
         int lineNumber = 0;
         HeaderType header = null;
         JsonArrayBuilder responseArr = Json.createArrayBuilder();
+        String[] values = null;
         try {
             br = new BufferedReader(new FileReader("/" + file));
             while ((line = br.readLine()) != null) {
                 lineNumber++;
-                String[] values = line.split(splitBy);
+                values = line.split(splitBy);
                 if (values[0].startsWith("#")) { // Header row
                     switch (values[0]) {
                         case "#metadataBlock":
@@ -301,7 +302,7 @@ public Response loadDatasetFields(File file) {
             return error(Status.EXPECTATION_FAILED, "File not found");
 
         } catch (ArrayIndexOutOfBoundsException e) {
-            String message = getArrayIndexOutOfBoundMessage(header, lineNumber, e);
+            String message = getArrayIndexOutOfBoundMessage(header, lineNumber, values.length);
             logger.log(Level.WARNING, message, e);
             alr.setActionResult(ActionLogRecord.Result.InternalError);
             alr.setInfo(alr.getInfo() + "// " + message);
@@ -352,10 +353,9 @@ public String getGeneralErrorMessage(HeaderType header, int lineNumber, String m
      */
     public String getArrayIndexOutOfBoundMessage(HeaderType header,
                                                  int lineNumber,
-                                                 ArrayIndexOutOfBoundsException e) {
+                                                 int wrongIndex) {
 
         List<String> columns = getColumnsByHeader(header);
-        int wrongIndex = Integer.parseInt(e.getMessage());
 
         String column = columns.get(wrongIndex - 1);
         List<String> arguments = new ArrayList<>();

src/test/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApiTest.java

@@ -43,7 +43,7 @@ public void testGeneralErrorMessageBundle() {
     @Test
     public void testGetArrayIndexOutOfBoundMessage() {
         DatasetFieldServiceApi api = new DatasetFieldServiceApi();
-        String message = api.getArrayIndexOutOfBoundMessage(DatasetFieldServiceApi.HeaderType.DATASETFIELD, 5, new ArrayIndexOutOfBoundsException("4"));
+        String message = api.getArrayIndexOutOfBoundMessage(DatasetFieldServiceApi.HeaderType.DATASETFIELD, 5, 4);
         assertEquals(
             "Error parsing metadata block in DATASETFIELD part, line #5: missing 'watermark' column (#5)",
             message

src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java

@@ -62,24 +62,26 @@ public void testGetReadChannel() throws Exception {
     @Test
     public void testGetDvObject() {
         assertEquals(null, instance.getDvObject());
-        instance.setDvObject(new Dataset());
-        assertEquals(new Dataset(), instance.getDataset());
-
-        try {
+        Dataset d= new Dataset();
+        instance.setDvObject(d);
+        //assertSame uses == rather than the .equals() method which would (currently) be true for any two Datasets 
+        assertSame(d, instance.getDataset());        try {
             instance.getDataFile();
             fail("This should have thrown");
         } catch (ClassCastException ex) {
-            assertEquals(ex.getMessage(), "edu.harvard.iq.dataverse.Dataset cannot be cast to edu.harvard.iq.dataverse.DataFile");
+            //Test succeeds
         }
         try {
             instance.getDataverse();
             fail("This should have thrown");
         } catch (ClassCastException ex) {
-            assertEquals(ex.getMessage(), "edu.harvard.iq.dataverse.Dataset cannot be cast to edu.harvard.iq.dataverse.Dataverse");
+            //Test succeeds
         }
         // null driver defaults to 'file'
-        assertEquals(new DataFile(), new FileAccessIO<>(new DataFile(), null, null).getDataFile());
-        assertEquals(new Dataverse(), new FileAccessIO<>(new Dataverse(), null, null).getDataverse());
+        DataFile f= new DataFile();
+        Dataverse dv = new Dataverse();
+        assertSame(f, new FileAccessIO<>(f, null, null).getDataFile());
+        assertSame(dv, new FileAccessIO<>(dv, null, null).getDataverse());
     }
 
     @Test