diff --git a/Dockerfile b/Dockerfile index 5f492ea0594..b0864a0c55f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1 @@ -# See `conf/docker` for Docker images +# See http://guides.dataverse.org/en/latest/developers/containers.html diff --git a/Vagrantfile b/Vagrantfile index b3c6e7b39a9..f846d826726 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,77 +1,41 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.define "standalone", primary: true do |standalone| - config.vm.hostname = "standalone" - # Uncomment this temporarily to get `vagrant destroy` to work - #standalone.vm.box = "puppetlabs/centos-7.2-64-puppet" - - operating_system = "centos" - if ENV['OPERATING_SYSTEM'].nil? - config.vm.box = "puppetlabs/centos-7.2-64-puppet" - config.vm.box_version = '1.0.1' - elsif ENV['OPERATING_SYSTEM'] == 'debian' - puts "WARNING: Debian specified. Here be dragons! https://github.com/IQSS/dataverse/issues/1059" - config.vm.box_url = "http://puppet-vagrant-boxes.puppetlabs.com/debian-73-x64-virtualbox-puppet.box" - config.vm.box = "puppet-vagrant-boxes.puppetlabs.com-debian-73-x64-virtualbox-puppet.box" - else - operating_system = ENV['OPERATING_SYSTEM'] - puts "Not sure what do to with operating system: #{operating_system}" - exit 1 - end - - mailserver = "localhost" - if ENV['MAIL_SERVER'].nil? - puts "MAIL_SERVER environment variable not specified. Using #{mailserver} by default.\nTo specify it in bash: export MAIL_SERVER=localhost" - else - mailserver = ENV['MAIL_SERVER'] - puts "MAIL_SERVER environment variable found, using #{mailserver}" - end - - config.vm.provider "virtualbox" do |v| - v.memory = 2048 - v.cpus = 1 - end - config.vm.provision "shell", path: "scripts/vagrant/setup.sh" - config.vm.provision "shell", path: "scripts/vagrant/setup-solr.sh" - config.vm.provision "shell", path: "scripts/vagrant/install-dataverse.sh", args: mailserver - # FIXME: get tests working and re-enable them! - #config.vm.provision "shell", path: "scripts/vagrant/test.sh" - - config.vm.network "private_network", type: "dhcp" - config.vm.network "forwarded_port", guest: 80, host: 8888 - config.vm.network "forwarded_port", guest: 443, host: 9999 - config.vm.network "forwarded_port", guest: 8983, host: 8993 - config.vm.network "forwarded_port", guest: 8080, host: 8088 - config.vm.network "forwarded_port", guest: 8181, host: 8188 - - # FIXME: use /dataverse/downloads instead - config.vm.synced_folder "downloads", "/downloads" - # FIXME: use /dataverse/conf instead - config.vm.synced_folder "conf", "/conf" - # FIXME: use /dataverse/scripts instead - config.vm.synced_folder "scripts", "/scripts" - config.vm.synced_folder ".", "/dataverse" - end - - config.vm.define "solr", autostart: false do |solr| - config.vm.hostname = "solr" - solr.vm.box = "puppet-vagrant-boxes.puppetlabs.com-centos-65-x64-virtualbox-puppet.box" - config.vm.synced_folder ".", "/dataverse" - config.vm.network "private_network", type: "dhcp" - config.vm.network "forwarded_port", guest: 8983, host: 9001 + config.vm.box = "bento/centos-7.7" + + puts "Sorry, this Vagrant environment is not working." + puts "If you'd like to help get it working, please see" + puts "https://github.com/IQSS/dataverse/issues/6849" + puts + puts "You can also try the Vagrant environment at" + puts "https://github.com/IQSS/dataverse-ansible" + exit 1 + + config.vm.provider "virtualbox" do |vbox| + vbox.cpus = 4 + vbox.memory = 4096 end - config.vm.define "test", autostart: false do |test| - config.vm.hostname = "test" - test.vm.box = "puppet-vagrant-boxes.puppetlabs.com-centos-65-x64-virtualbox-puppet.box" - config.vm.synced_folder ".", "/dataverse" - config.vm.network "private_network", type: "dhcp" - end + #config.vm.provision "shell", path: "scripts/vagrant/setup.sh" + #config.vm.provision "shell", path: "scripts/vagrant/setup-solr.sh" + config.vm.provision "shell", path: "scripts/vagrant/install-dataverse.sh" + + config.vm.network "private_network", type: "dhcp" + config.vm.network "forwarded_port", guest: 80, host: 8888 + config.vm.network "forwarded_port", guest: 443, host: 9999 + config.vm.network "forwarded_port", guest: 8983, host: 8993 + config.vm.network "forwarded_port", guest: 8080, host: 8088 + config.vm.network "forwarded_port", guest: 8181, host: 8188 + + # FIXME: use /dataverse/downloads instead + config.vm.synced_folder "downloads", "/downloads" + # FIXME: use /dataverse/conf instead + config.vm.synced_folder "conf", "/conf" + # FIXME: use /dataverse/scripts instead + config.vm.synced_folder "scripts", "/scripts" + config.vm.synced_folder ".", "/dataverse" end diff --git a/conf/docker/build.sh b/conf/docker/build.sh deleted file mode 100755 index 27145d199e1..00000000000 --- a/conf/docker/build.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# Creates images and pushes them to Docker Hub. -# The "latest" tag under "iqss" should be relatively stable. Don't push breaking changes there. -# None of the tags are suitable for production use. See https://github.com/IQSS/dataverse/issues/4040 -# To iterate on images, push to custom tags or tags based on branch names or a non-iqss Docker Hub org/username. Consider trying "internal" to push to the internal Minishift registry. - -# Docker Hub organization or username -HUBORG=iqss -# The most stable tag we have. -STABLE=latest -#FIXME: Use a real flag/argument parser. download-files.sh uses "getopts" for example. -if [ -z "$1" ]; then - echo "No argument supplied. For experiments, specify \"branch\" or \"custom my-custom-tag\" or \"huborg \" or \"internal\". Specify \"stable\" to push to the \"$STABLE\" tag under \"$HUBORG\" if your change won't break anything." - exit 1 -fi - -if [ "$1" == 'branch' ]; then - echo "We'll push a tag to the branch you're on." - GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) - TAG=$GIT_BRANCH -elif [ "$1" == 'stable' ]; then - echo "We'll push a tag to the most stable tag (which isn't saying much!)." - TAG=$STABLE -elif [ "$1" == 'custom' ]; then - if [ -z "$2" ]; then - echo "You must provide a custom tag as the second argument. Something other than \"$STABLE\"." - exit 1 - else - echo "We'll push a custom tag." - TAG=$2 - fi -elif [ "$1" == 'huborg' ]; then - if [ -z "$2" ]; then - echo "You must provide your Docker Hub organization or username as the second argument. \"$USER\" or whatever." - exit 1 - else - HUBORG=$2 - TAG=$STABLE - echo "We'll push to the Docker Hub organization or username you specified: $HUBORG." - fi -elif [ "$1" == 'internal' ]; then - echo "Building for internal Minishift registry." - TAG=$STABLE -else - echo "Unexpected argument: $1. Exiting. Run with no arguments for help." - TAG=$STABLE - exit 1 -fi -echo Images are being built for registry org/username \"$HUBORG\" with the tag \"$TAG\". -# -# Build dataverse-solr -# -cp ../solr/7.3.0/schema.xml solr/ -# move solr*.tgz to the solr image -cp ../../downloads/solr-7.3.0.tgz solr/ -docker build -t $HUBORG/dataverse-solr:$TAG solr/ -if [ "$1" == 'internal' ]; then - echo "Skipping docker push because we're using the internal Minishift registry." -else - # FIXME: Check the output of "docker build" and only push on success. - docker push $HUBORG/dataverse-solr:$TAG -fi -# -# Build dataverse-glassfish -# -# TODO: Think about if we really need dataverse.war because it's in dvinstall.zip. -cd ../.. -mvn clean -scripts/installer/custom-build-number -mvn package -cd conf/docker -cp ../../target/dataverse*.war dataverse-glassfish/dataverse.war -if [[ "$?" -ne 0 ]]; then - echo "Unable to copy war file into place. Did 'mvn package' work?" - exit 1 -fi -cd ../../scripts/installer -make clean -make -cd ../../conf/docker -cp ../../scripts/installer/dvinstall.zip dataverse-glassfish -if [[ "$?" -ne 0 ]]; then - echo "Unable to copy dvinstall.zip file into place. Did 'make' work?" - exit 1 -fi -cp ../../downloads/glassfish-4.1.zip dataverse-glassfish -if [[ "$?" -ne 0 ]]; then - echo "Unable to copy Glassfish zip file into place. You must run the download script in that directory once. " - exit 1 -fi -# We'll assume at this point that the download script has been run. -cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish -docker build -t $HUBORG/dataverse-glassfish:$TAG dataverse-glassfish -if [ "$1" == 'internal' ]; then - echo "Skipping docker push because we're using the internal Minishift registry." -else - # FIXME: Check the output of "docker build" and only push on success. - docker push $HUBORG/dataverse-glassfish:$TAG -fi -# -# Build init-container -# -cp ../../scripts/installer/install dataverse-glassfish/init-container -docker build -t $HUBORG/init-container:$TAG dataverse-glassfish/init-container -if [ "$1" == 'internal' ]; then - echo "Skipping docker push because we're using the internal Minishift registry." -else - # FIXME: Check the output of "docker build" and only push on success. - docker push $HUBORG/init-container:$TAG -fi diff --git a/conf/docker/dataverse-glassfish/.gitignore b/conf/docker/dataverse-glassfish/.gitignore deleted file mode 100644 index 2084aa8849e..00000000000 --- a/conf/docker/dataverse-glassfish/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -glassfish-4.1.zip -weld-osgi-bundle-2.2.10.Final-glassfish4.jar -dvinstall.zip -dataverse.war -init-container/postgres-setup diff --git a/conf/docker/dataverse-glassfish/Dockerfile b/conf/docker/dataverse-glassfish/Dockerfile deleted file mode 100644 index d92bfbc54f0..00000000000 --- a/conf/docker/dataverse-glassfish/Dockerfile +++ /dev/null @@ -1,101 +0,0 @@ -FROM centos:7 -MAINTAINER Dataverse (support@dataverse.org) - -COPY glassfish-4.1.zip /tmp -COPY weld-osgi-bundle-2.2.10.Final-glassfish4.jar /tmp -COPY default.config /tmp -# Install dependencies -#RUN yum install -y unzip -RUN yum install -y \ - cronie \ - git \ - java-1.8.0-openjdk-devel \ - nc \ - perl \ - postgresql \ - sha1sum \ - unzip \ - wget - -ENV GLASSFISH_DOWNLOAD_SHA1 d1a103d06682eb08722fbc9a93089211befaa080 -ENV GLASSFISH_DIRECTORY "/usr/local/glassfish4" -ENV HOST_DNS_ADDRESS "localhost" -ENV POSTGRES_DB "dvndb" -ENV POSTGRES_USER "dvnapp" -ENV RSERVE_USER "rserve" -ENV RSERVE_PASSWORD "rserve" - -#RUN exitEarlyBeforeJq -RUN yum -y install epel-release \ - jq - -COPY dvinstall.zip /tmp - -#RUN ls /tmp -# -RUN find /tmp -# -#RUN exitEarly - -# Install Glassfish 4.1 -RUN cd /tmp \ - && unzip glassfish-4.1.zip \ - && mv glassfish4 /usr/local \ - && cd /usr/local/glassfish4/glassfish/modules \ - && rm weld-osgi-bundle.jar \ - && cp /tmp/weld-osgi-bundle-2.2.10.Final-glassfish4.jar . \ - && cd /tmp && unzip /tmp/dvinstall.zip \ - && chmod 777 -R /tmp/dvinstall/ \ - #FIXME: Patch Grizzly too! - && echo "Done installing and patching Glassfish" - -RUN chmod g=u /etc/passwd - -RUN mkdir -p /home/glassfish -RUN chgrp -R 0 /home/glassfish && \ - chmod -R g=u /home/glassfish - -RUN mkdir -p /usr/local/glassfish4 -RUN chgrp -R 0 /usr/local/glassfish4 && \ - chmod -R g=u /usr/local/glassfish4 - -#JHOVE -RUN cp /tmp/dvinstall/jhove* /usr/local/glassfish4/glassfish/domains/domain1/config - - -#SETUP JVM OPTIONS -ARG DOCKER_BUILD="true" -RUN echo $DOCKER_BUILD -RUN /tmp/dvinstall/glassfish-setup.sh -###glassfish-setup will handle everything in Dockerbuild - -##install jdbc driver -RUN cp /tmp/dvinstall/pgdriver/postgresql-42.2.12.jar /usr/local/glassfish4/glassfish/domains/domain1/lib - -# Customized persistence xml to avoid database recreation -#RUN mkdir -p /tmp/WEB-INF/classes/META-INF/ -#COPY WEB-INF/classes/META-INF/persistence.xml /tmp/WEB-INF/classes/META-INF/ - -# Install iRods iCommands -#RUN cd /tmp \ -# && yum -y install epel-release \ -# && yum -y install ftp://ftp.renci.org/pub/irods/releases/4.1.6/centos7/irods-icommands-4.1.6-centos7-x86_64.rpm - -#COPY config-glassfish /root/dvinstall -#COPY restart-glassfish /root/dvinstall -#COPY config-dataverse /root/dvinstall - -#RUN cd /root/dvinstall && ./config-dataverse -COPY ./entrypoint.sh / -#COPY ./ddl /root/dvinstall -#COPY ./init-postgres /root/dvinstall -#COPY ./init-glassfish /root/dvinstall -#COPY ./init-dataverse /root/dvinstall -#COPY ./setup-all.sh /root/dvinstall -#COPY ./setup-irods.sh /root/dvinstall -COPY ./Dockerfile / - -EXPOSE 8080 - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["dataverse"] diff --git a/conf/docker/dataverse-glassfish/default.config b/conf/docker/dataverse-glassfish/default.config deleted file mode 100644 index 7af10336f30..00000000000 --- a/conf/docker/dataverse-glassfish/default.config +++ /dev/null @@ -1,16 +0,0 @@ -HOST_DNS_ADDRESS localhost -GLASSFISH_DIRECTORY /usr/local/glassfish4 -ADMIN_EMAIL -MAIL_SERVER mail.hmdc.harvard.edu -POSTGRES_ADMIN_PASSWORD secret -POSTGRES_SERVER dataverse-postgresql-0.dataverse-postgresql-service -POSTGRES_PORT 5432 -POSTGRES_DATABASE dvndb -POSTGRES_USER dvnapp -POSTGRES_PASSWORD secret -SOLR_LOCATION dataverse-solr-service:8983 -TWORAVENS_LOCATION NOT INSTALLED -RSERVE_HOST localhost -RSERVE_PORT 6311 -RSERVE_USER rserve -RSERVE_PASSWORD rserve diff --git a/conf/docker/dataverse-glassfish/entrypoint.sh b/conf/docker/dataverse-glassfish/entrypoint.sh deleted file mode 100755 index 55bbbdedbb7..00000000000 --- a/conf/docker/dataverse-glassfish/entrypoint.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash -x - -# Entrypoint script for Dataverse web application. This script waits -# for dependent services (Rserve, Postgres, Solr) to start before -# initializing Glassfish. - -echo "whoami before..." -whoami -if ! whoami &> /dev/null; then - if [ -w /etc/passwd ]; then - # Make `whoami` return the glassfish user. # See https://docs.openshift.org/3.6/creating_images/guidelines.html#openshift-origin-specific-guidelines - # Fancy bash magic from https://github.com/RHsyseng/container-rhel-examples/blob/1208dcd7d4f431fc6598184dba6341b9465f4197/starter-arbitrary-uid/bin/uid_entrypoint#L4 - echo "${USER_NAME:-glassfish}:x:$(id -u):0:${USER_NAME:-glassfish} user:/home/glassfish:/bin/bash" >> /etc/passwd - fi -fi -echo "whoami after" -whoami - -set -e - -if [ "$1" = 'dataverse' ]; then - - export GLASSFISH_DIRECTORY=/usr/local/glassfish4 - export HOST_DNS_ADDRESS=localhost - - TIMEOUT=30 - - if [ -n "$RSERVE_SERVICE_HOST" ]; then - RSERVE_HOST=$RSERVE_SERVICE_HOST - elif [ -n "$RSERVE_PORT_6311_TCP_ADDR" ]; then - RSERVE_HOST=$RSERVE_PORT_6311_TCP_ADDR - elif [ -z "$RSERVE_HOST" ]; then - RSERVE_HOST="localhost" - fi - export RSERVE_HOST - - if [ -n "$RSERVE_SERVICE_PORT" ]; then - RSERVE_PORT=$RSERVE_SERVICE_PORT - elif [ -n "$RSERVE_PORT_6311_TCP_PORT" ]; then - RSERVE_PORT=$RSERVE_PORT_6311_TCP_PORT - elif [ -z "$RSERVE_PORT" ]; then - RSERVE_PORT="6311" - fi - export RSERVE_PORT - - echo "Using Rserve at $RSERVE_HOST:$RSERVE_PORT" - - if ncat $RSERVE_HOST $RSERVE_PORT -w $TIMEOUT --send-only < /dev/null > /dev/null 2>&1 ; then - echo Rserve running; - else - echo Optional service Rserve not running. - fi - - - # postgres - if [ -n "$POSTGRES_SERVICE_HOST" ]; then - POSTGRES_HOST=$POSTGRES_SERVICE_HOST - elif [ -n "$POSTGRES_PORT_5432_TCP_ADDR" ]; then - POSTGRES_HOST=$POSTGRES_PORT_5432_TCP_ADDR - elif [ -z "$POSTGRES_HOST" ]; then - POSTGRES_HOST="localhost" - fi - export POSTGRES_HOST - - if [ -n "$POSTGRES_SERVICE_PORT" ]; then - POSTGRES_PORT=$POSTGRES_SERVICE_PORT - elif [ -n "$POSTGRES_PORT_5432_TCP_PORT" ]; then - POSTGRES_PORT=$POSTGRES_PORT_5432_TCP_PORT - else - POSTGRES_PORT=5432 - fi - export POSTGRES_PORT - - echo "Using Postgres at $POSTGRES_HOST:$POSTGRES_PORT" - - if ncat $POSTGRES_HOST $POSTGRES_PORT -w $TIMEOUT --send-only < /dev/null > /dev/null 2>&1 ; then - echo Postgres running; - else - echo Required service Postgres not running. Have you started the required services? - exit 1 - fi - - # solr - if [ -n "$SOLR_SERVICE_HOST" ]; then - SOLR_HOST=$SOLR_SERVICE_HOST - elif [ -n "$SOLR_PORT_8983_TCP_ADDR" ]; then - SOLR_HOST=$SOLR_PORT_8983_TCP_ADDR - elif [ -z "$SOLR_HOST" ]; then - SOLR_HOST="localhost" - fi - export SOLR_HOST - - if [ -n "$SOLR_SERVICE_PORT" ]; then - SOLR_PORT=$SOLR_SERVICE_PORT - elif [ -n "$SOLR_PORT_8983_TCP_PORT" ]; then - SOLR_PORT=$SOLR_PORT_8983_TCP_PORT - else - SOLR_PORT=8983 - fi - export SOLR_PORT - - echo "Using Solr at $SOLR_HOST:$SOLR_PORT" - - if ncat $SOLR_HOST $SOLR_PORT -w $TIMEOUT --send-only < /dev/null > /dev/null 2>&1 ; then - echo Solr running; - else - echo Required service Solr not running. Have you started the required services? - exit 1 - fi - - GLASSFISH_INSTALL_DIR="/usr/local/glassfish4" - cd /tmp/dvinstall - echo Copying the non-interactive file into place - cp /tmp/default.config . - echo Looking at first few lines of default.config - head default.config - # non-interactive install - echo Running non-interactive install - #./install -y -f > install.out 2> install.err - ./install -y -f - -# if [ -n "$DVICAT_PORT_1247_TCP_PORT" ]; then -# ./setup-irods.sh -# fi - - # We do change the Solr server in Minishift/OpenShift, which is - # the primary target for all of the work under conf/docker. - # echo -e "\n\nRestarting Dataverse in case Solr host was changed..." - # /usr/local/glassfish4/glassfish/bin/asadmin stop-domain - # sleep 3 - # /usr/local/glassfish4/glassfish/bin/asadmin start-domain - - echo -e "\n\nDataverse started" - - sleep infinity -else - exec "$@" -fi - diff --git a/conf/docker/dataverse-glassfish/init-container/Dockerfile b/conf/docker/dataverse-glassfish/init-container/Dockerfile deleted file mode 100644 index 829ab03bcb2..00000000000 --- a/conf/docker/dataverse-glassfish/init-container/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM centos:7 -MAINTAINER Dataverse (support@dataverse.org) - -### init-container is an Init Container for glassfish service in OpenShift or other Kubernetes environment -# This initContainer will take care of setting up glassfish - -# Install dependencies -RUN yum install -y \ - nc \ - perl \ - postgresql \ - sha1sum - -COPY install / - -ENTRYPOINT ["/install", "--pg_only", "--yes"] diff --git a/conf/docker/dataverse-glassfish/init-container/default.config b/conf/docker/dataverse-glassfish/init-container/default.config deleted file mode 100644 index 7af10336f30..00000000000 --- a/conf/docker/dataverse-glassfish/init-container/default.config +++ /dev/null @@ -1,16 +0,0 @@ -HOST_DNS_ADDRESS localhost -GLASSFISH_DIRECTORY /usr/local/glassfish4 -ADMIN_EMAIL -MAIL_SERVER mail.hmdc.harvard.edu -POSTGRES_ADMIN_PASSWORD secret -POSTGRES_SERVER dataverse-postgresql-0.dataverse-postgresql-service -POSTGRES_PORT 5432 -POSTGRES_DATABASE dvndb -POSTGRES_USER dvnapp -POSTGRES_PASSWORD secret -SOLR_LOCATION dataverse-solr-service:8983 -TWORAVENS_LOCATION NOT INSTALLED -RSERVE_HOST localhost -RSERVE_PORT 6311 -RSERVE_USER rserve -RSERVE_PASSWORD rserve diff --git a/conf/docker/postgresql/Dockerfile b/conf/docker/postgresql/Dockerfile deleted file mode 100644 index 81ecf0fdeb8..00000000000 --- a/conf/docker/postgresql/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -# PostgreSQL for Dataverse (but consider switching to the image from CentOS) -# -# See also conf/docker/dataverse-glassfish/Dockerfile diff --git a/conf/docker/solr/.gitignore b/conf/docker/solr/.gitignore deleted file mode 100644 index a6237a89914..00000000000 --- a/conf/docker/solr/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -solr-7.3.0.tgz -schema.xml diff --git a/conf/docker/solr/Dockerfile b/conf/docker/solr/Dockerfile deleted file mode 100644 index 993c2909f4d..00000000000 --- a/conf/docker/solr/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM centos:7 -MAINTAINER Dataverse (support@dataverse.org) - -RUN yum install -y unzip java-1.8.0-openjdk-devel lsof - -# Install Solr 7.3.0 -# The context of the build is the "conf" directory. -COPY solr-7.3.0.tgz /tmp -RUN cd /tmp \ - && tar xvfz solr-7.3.0.tgz \ - && rm solr-7.3.0.tgz \ - && mkdir /usr/local/solr \ - && mv solr-7.3.0 /usr/local/solr/ - -COPY schema.xml /tmp -COPY solrconfig_master.xml /tmp -COPY solrconfig_slave.xml /tmp - -RUN chmod g=u /etc/passwd - -RUN chgrp -R 0 /usr/local/solr && \ - chmod -R g=u /usr/local/solr - -EXPOSE 8983 - -COPY Dockerfile / -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] -USER 1001 -CMD ["solr"] diff --git a/conf/docker/solr/backup_cron.sh b/conf/docker/solr/backup_cron.sh deleted file mode 100644 index 95f31b4b53a..00000000000 --- a/conf/docker/solr/backup_cron.sh +++ /dev/null @@ -1 +0,0 @@ -0 */6 * * * curl 'http://localhost:8983/solr/collection1/replication?command=backup&location=/home/share' diff --git a/conf/docker/solr/entrypoint.sh b/conf/docker/solr/entrypoint.sh deleted file mode 100755 index 5d003f9c56b..00000000000 --- a/conf/docker/solr/entrypoint.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -if ! whoami &> /dev/null; then - if [ -w /etc/passwd ]; then - echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd - fi -fi - -SOLR_DIR=/usr/local/solr/solr-7.3.0 - -if [ "$1" = 'solr' ]; then - - cp -r $SOLR_DIR/server/solr/configsets/_default $SOLR_DIR/server/solr/collection1 - cp /tmp/schema.xml $SOLR_DIR/server/solr/collection1/conf - - if [ $HOSTNAME = "dataverse-solr-0" ]; then - echo "I am the master" - mv /tmp/solrconfig_master.xml $SOLR_DIR/server/solr/collection1/conf/solrconfig.xml - cp /tmp/solrconfig_slave.xml $SOLR_DIR/server/solr/collection1/conf - - else - echo "I am the slave" - cp /tmp/solrconfig_slave.xml $SOLR_DIR/server/solr/collection1/conf - mv $SOLR_DIR/server/solr/collection1/conf/solrconfig_slave.xml $SOLR_DIR/server/solr/collection1/conf/solrconfig.xml - fi - cd $SOLR_DIR - bin/solr start - bin/solr create_core -c collection1 -d server/solr/collection1/conf - if [ $HOSTNAME = "dataverse-solr-0" ]; then - curl 'http://localhost:8983/solr/collection1/replication?command=restore&location=/home/share' - fi - - sleep infinity -elif [ "$1" = 'usage' ]; then - echo 'docker run -d iqss/dataverse-solr solr' -else - exec "$@" -fi diff --git a/conf/docker/solr/solrconfig_master.xml b/conf/docker/solr/solrconfig_master.xml deleted file mode 100644 index d409f70b5d5..00000000000 --- a/conf/docker/solr/solrconfig_master.xml +++ /dev/null @@ -1,1431 +0,0 @@ - - - - - - - - - 7.3.0 - - - - - - - - - - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - edismax - 0.075 - - dvName^400 - authorName^180 - dvSubject^190 - dvDescription^180 - dvAffiliation^170 - title^130 - subject^120 - keyword^110 - topicClassValue^100 - dsDescriptionValue^90 - authorAffiliation^80 - publicationCitation^60 - producerName^50 - fileName^30 - fileDescription^30 - variableLabel^20 - variableName^10 - _text_^1.0 - - - dvName^200 - authorName^100 - dvSubject^100 - dvDescription^100 - dvAffiliation^100 - title^75 - subject^75 - keyword^75 - topicClassValue^75 - dsDescriptionValue^75 - authorAffiliation^75 - publicationCitation^75 - producerName^75 - - - - isHarvested:false^25000 - - - - - - - - - - - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - _text_ - - - - - - - true - ignored_ - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - - - true - false - - - terms - - - - - - - - - string - - - - - - explicit - - - elevator - - - - - - - - startup - commit - - - - - schema.xml,stopwords.txt,elevate.xml - solrconfig_slave.xml:solrconfig.xml,x.xml,y.xml - - - - - 1 - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - - 5 - - - - - - - - - - - - - - diff --git a/conf/docker/solr/solrconfig_slave.xml b/conf/docker/solr/solrconfig_slave.xml deleted file mode 100644 index c31710ebace..00000000000 --- a/conf/docker/solr/solrconfig_slave.xml +++ /dev/null @@ -1,1442 +0,0 @@ - - - - - - - - - 7.3.0 - - - - - - - - - - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - edismax - 0.075 - - dvName^400 - authorName^180 - dvSubject^190 - dvDescription^180 - dvAffiliation^170 - title^130 - subject^120 - keyword^110 - topicClassValue^100 - dsDescriptionValue^90 - authorAffiliation^80 - publicationCitation^60 - producerName^50 - fileName^30 - fileDescription^30 - variableLabel^20 - variableName^10 - _text_^1.0 - - - dvName^200 - authorName^100 - dvSubject^100 - dvDescription^100 - dvAffiliation^100 - title^75 - subject^75 - keyword^75 - topicClassValue^75 - dsDescriptionValue^75 - authorAffiliation^75 - publicationCitation^75 - producerName^75 - - - - isHarvested:false^25000 - - - - - - - - - - - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - _text_ - - - - - - - true - ignored_ - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - - - true - false - - - terms - - - - - - - - - string - - - - - - explicit - - - elevator - - - - - - - - - http://dataverse-solr-0.dataverse-solr-service:8983/solr/collection1 - - - 00:00:20 - - - internal - - 5000 - 10000 - - - username - password - - - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - - 5 - - - - - - - - - - - - - - diff --git a/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml b/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml index fc47b0f4e9a..c4025ce07fb 100644 --- a/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml +++ b/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml @@ -1,8 +1,8 @@ # currently no other option but to have daily logs and have year-month-day format in the name with # 4-digit year and 2-digit month and day -# /usr/local/glassfish4/glassfish/domains/domain1/logs/counter_2019-01-11.log +# /usr/local/payara5/glassfish/domains/domain1/logs/counter_2019-01-11.log #log_name_pattern: sample_logs/counter_(yyyy-mm-dd).log -log_name_pattern: /usr/local/glassfish4/glassfish/domains/domain1/logs/counter_(yyyy-mm-dd).log +log_name_pattern: /usr/local/payara5/glassfish/domains/domain1/logs/counter_(yyyy-mm-dd).log # path_types regular expressions allow matching to classify page urls as either an investigation or request # based on specific URL structure for your system. @@ -42,7 +42,7 @@ year_month: 2019-01 # Don't put the filename extension, the code will tack on the tsv or json extension for you. # Output formats are either tsv or json currently. TSV is currently broken until anyone accepts reports in that format. # FIXME: "/tmp" is fine for a quick test but pick another directory that -# the "counter" user can write to and that the "glassfish" user can read from. +# the "counter" user can write to and that the "dataverse" user can read from. output_file: /tmp/make-data-count-report output_format: json diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/httpd/conf.d/ssl.conf b/doc/sphinx-guides/source/_static/installation/files/etc/httpd/conf.d/ssl.conf index d17ca127b4a..a6c1c7b419c 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/httpd/conf.d/ssl.conf +++ b/doc/sphinx-guides/source/_static/installation/files/etc/httpd/conf.d/ssl.conf @@ -223,14 +223,14 @@ CustomLog logs/ssl_request_log \ # Require all granted # -# don't pass paths used by rApache and TwoRavens to Glassfish +# don't pass paths used by rApache and TwoRavens to app server ProxyPassMatch ^/RApacheInfo$ ! ProxyPassMatch ^/custom ! ProxyPassMatch ^/dataexplore ! -# don't pass paths used by Shibboleth to Glassfish +# don't pass paths used by Shibboleth to app server ProxyPassMatch ^/Shibboleth.sso ! ProxyPassMatch ^/shibboleth-ds ! -# pass everything else to Glassfish +# pass everything else to app server ProxyPass / ajp://localhost:8009/ diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/glassfish.init.root b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.root similarity index 64% rename from doc/sphinx-guides/source/_static/installation/files/etc/init.d/glassfish.init.root rename to doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.root index 8d74f89ec18..1de94331523 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/glassfish.init.root +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.root @@ -1,24 +1,24 @@ #! /bin/sh # chkconfig: 2345 99 01 -# description: GlassFish App Server +# description: Payara App Server set -e -ASADMIN=/usr/local/glassfish4/bin/asadmin +ASADMIN=/usr/local/payara5/bin/asadmin case "$1" in start) - echo -n "Starting GlassFish server: glassfish" + echo -n "Starting Payara" # Increase file descriptor limit: ulimit -n 32768 # Allow "memory overcommit": # (basically, this allows to run exec() calls from inside the # app, without the Unix fork() call physically hogging 2X - # the amount of memory glassfish is already using) + # the amount of memory Payara is already using) echo 1 > /proc/sys/vm/overcommit_memory #echo - #echo "GLASSFISH IS UNDER MAINTENANCE;" + #echo "PAYARA IS UNDER MAINTENANCE;" #echo "PLEASE DO NOT USE service init script." #echo LANG=en_US.UTF-8; export LANG @@ -26,9 +26,9 @@ case "$1" in echo "." ;; stop) - echo -n "Stopping GlassFish server: glassfish" + echo -n "Stopping Payara" #echo - #echo "GLASSFISH IS UNDER MAINTENANCE;" + #echo "PAYARA IS UNDER MAINTENANCE;" #echo "PLEASE DO NOT USE service init script." #echo @@ -37,7 +37,7 @@ case "$1" in ;; *) - echo "Usage: /etc/init.d/glassfish {start|stop}" + echo "Usage: /etc/init.d/payara {start|stop}" exit 1 esac diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/glassfish.init.service b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.service similarity index 52% rename from doc/sphinx-guides/source/_static/installation/files/etc/init.d/glassfish.init.service rename to doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.service index 32769763f5d..7c457e615d8 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/glassfish.init.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.service @@ -1,33 +1,33 @@ #! /bin/sh # chkconfig: 2345 80 01 -# description: GlassFish App Server +# description: Payara App Server set -e -ASADMIN=/usr/local/glassfish4/bin/asadmin -GF_USER=glassfish +ASADMIN=/usr/local/payara5/bin/asadmin +APP_SERVER_USER=dataverse case "$1" in start) - echo -n "Starting GlassFish server: glassfish" + echo -n "Starting Payara" # Increase file descriptor limit: ulimit -n 32768 # Allow "memory overcommit": # (basically, this allows to run exec() calls from inside the # app, without the Unix fork() call physically hogging 2X - # the amount of memory glassfish is already using) + # the amount of memory Payara is already using) echo 1 > /proc/sys/vm/overcommit_memory LANG=en_US.UTF-8; export LANG - sudo -u $GF_USER $ASADMIN start-domain domain1 + sudo -u $APP_SERVER_USER $ASADMIN start-domain domain1 ;; stop) - echo -n "Stopping GlassFish server: glassfish" + echo -n "Stopping Payara" - sudo -u $GF_USER $ASADMIN stop-domain domain1 + sudo -u $APP_SERVER_USER $ASADMIN stop-domain domain1 echo "." ;; *) - echo "Usage: /etc/init.d/glassfish {start|stop}" + echo "Usage: /etc/init.d/payara {start|stop}" exit 1 esac diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/glassfish.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/glassfish.service deleted file mode 100644 index 37c2107412a..00000000000 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/glassfish.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description = GlassFish Server v4.1 -After = syslog.target network.target - -[Service] -Type = forking -ExecStart = /usr/bin/java -jar /usr/local/glassfish4/glassfish/lib/client/appserver-cli.jar start-domain -ExecStop = /usr/bin/java -jar /usr/local/glassfish4/glassfish/lib/client/appserver-cli.jar stop-domain -ExecReload = /usr/bin/java -jar /usr/local/glassfish4/glassfish/lib/client/appserver-cli.jar restart-domain -User=glassfish -LimitNOFILE=32768 -Environment="LANG=en_US.UTF-8" -TimeoutStartSec=120s - -[Install] -WantedBy = multi-user.target diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/payara.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/payara.service new file mode 100644 index 00000000000..c8c82f6d6b2 --- /dev/null +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/payara.service @@ -0,0 +1,16 @@ +[Unit] +Description = Payara Server +After = syslog.target network.target + +[Service] +Type = forking +ExecStart = /usr/bin/java -jar /usr/local/payara5/glassfish/lib/client/appserver-cli.jar start-domain +ExecStop = /usr/bin/java -jar /usr/local/payara5/glassfish/lib/client/appserver-cli.jar stop-domain +ExecReload = /usr/bin/java -jar /usr/local/payara5/glassfish/lib/client/appserver-cli.jar restart-domain +User=dataverse +LimitNOFILE=32768 +Environment="LANG=en_US.UTF-8" +TimeoutStartSec=120s + +[Install] +WantedBy = multi-user.target diff --git a/doc/sphinx-guides/source/_static/installation/files/issues/2180/grizzly-patch/glassfish-grizzly-extra-all.jar b/doc/sphinx-guides/source/_static/installation/files/issues/2180/grizzly-patch/glassfish-grizzly-extra-all.jar deleted file mode 100644 index 5e2690688c5..00000000000 Binary files a/doc/sphinx-guides/source/_static/installation/files/issues/2180/grizzly-patch/glassfish-grizzly-extra-all.jar and /dev/null differ diff --git a/doc/sphinx-guides/source/_static/installation/files/issues/2180/grizzly-patch/readme.md b/doc/sphinx-guides/source/_static/installation/files/issues/2180/grizzly-patch/readme.md deleted file mode 100644 index 88bb7a039fd..00000000000 --- a/doc/sphinx-guides/source/_static/installation/files/issues/2180/grizzly-patch/readme.md +++ /dev/null @@ -1,9 +0,0 @@ -This patch was created on 2015-06-30 by pdurbin for Glassfish 4.1. The file AjpHttpRequest.class was replaced with the one from https://java.net/jira/secure/attachment/55000/grizzly-http-ajp-2.3.19.jar with the following command: - -`jar uf glassfish-grizzly-extra-all.jar org/glassfish/grizzly/http/ajp/AjpHttpRequest.class` - -For details on this patch, see https://github.com/payara/Payara/issues/350 and https://java.net/jira/browse/GRIZZLY-1787 - -Here is the MD5 for the updated jar: - -547af1855e697bc2e51ca4ec9c45cf00 glassfish-grizzly-extra-all.jar diff --git a/doc/sphinx-guides/source/_static/util/clear_timer.sh b/doc/sphinx-guides/source/_static/util/clear_timer.sh index 73e87e8850c..d51455ba8ed 100755 --- a/doc/sphinx-guides/source/_static/util/clear_timer.sh +++ b/doc/sphinx-guides/source/_static/util/clear_timer.sh @@ -5,14 +5,14 @@ # assumes this script is being run as root, and that the postgres user had passwordless # access to the database (local sockets, or appropriate environmental variables). -# will restart glassfish if it's stopped; comment out the `start-domain` command at the end +# will restart Payara if it's stopped; comment out the `start-domain` command at the end # if you'd like to avoid that. -# directory where glassfish is installed -GLASSFISH_DIR=/usr/local/glassfish4 +# directory where Payara is installed +PAYARA_DIR=/usr/local/payara5 -# directory within glassfish (defaults) -DV_DIR=${GLASSFISH_DIR}/glassfish/domains/domain1 +# directory within Payara (defaults) +DV_DIR=${PAYARA_DIR}/glassfish/domains/domain1 # name of dataverse database DV_DB=dvndb @@ -20,14 +20,13 @@ DV_DB=dvndb # OS user for the database DB_USER=postgres -# stop the glassfish domain (generates a warning if glassfish is stopped) -${GLASSFISH_DIR}/bin/asadmin stop-domain +# stop the domain (generates a warning if app server is stopped) +${PAYARA_DIR}/bin/asadmin stop-domain -rm -rf ${GLASSFISH_DIR}/${DV_DIR}/generated/ -rm -rf ${GLASSFISH_DIR}/${DV_DIR}/osgi-cache/felix +rm -rf ${PAYARA_DIR}/${DV_DIR}/generated/ +rm -rf ${PAYARA_DIR}/${DV_DIR}/osgi-cache/felix sudo -u ${DB_USER} psql ${DV_DB} -c 'delete from "EJB__TIMER__TBL"'; -# restart the domain (also generates a warning if glassfish is stopped) -${GLASSFISH_DIR}/bin/asadmin start-domain - +# restart the domain (also generates a warning if app server is stopped) +${PAYARA_DIR}/bin/asadmin start-domain diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 9dcdb13814e..26af8eee32c 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -24,6 +24,6 @@ The process of creating a new, or editing an existing client, is largely self-ex What if a Run Fails? ~~~~~~~~~~~~~~~~~~~~ -Each harvesting client run logs a separate file per run to Glassfish's default logging directory (``/usr/local/glassfish4/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. +Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara5/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. Note that you'll want to run a minimum Dataverse version of 4.6, optimally 4.18, for the best OAI-PMH interoperability. diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index d6e9828a872..aa19ce5ce8d 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -100,7 +100,7 @@ Soon we will be setting up a cron job to run nightly but we start with a single * If you are running Counter Processor for the first time in the middle of a month, you will need create blank log files for the previous days. e.g.: - * ``cd /usr/local/glassfish4/glassfish/domains/domain1/logs`` + * ``cd /usr/local/payara5/glassfish/domains/domain1/logs`` * ``touch counter_2019-02-01.log`` diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index bf89805007b..a7c7e48cdd6 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -546,7 +546,7 @@ Loading TSV files into Dataverse A number of TSV files are loaded into Dataverse on every new installation, becoming the metadata blocks you see in the UI. For the list of metadata blocks that are included with Dataverse out of the box, see the :doc:`/user/appendix` section of the User Guide. -Along with TSV file, there are corresponding ResourceBundle property files with key=value pair `here `__. To add other language files, see the :doc:`/installation/config` for dataverse.lang.directory JVM Options section, and add a file, for example: "citation_lang.properties" to the path you specified for the ``dataverse.lang.directory`` JVM option, and then restart Glassfish. +Along with TSV file, there are corresponding ResourceBundle property files with key=value pair `here `__. To add other language files, see the :doc:`/installation/config` for dataverse.lang.directory JVM Options section, and add a file, for example: "citation_lang.properties" to the path you specified for the ``dataverse.lang.directory`` JVM option, and then restart the app server. If you are improving an existing metadata block, the Dataverse installation process will load the TSV for you, assuming you edited the TSV file in place. The TSV file for the Citation metadata block, for example, can be found at ``scripts/api/data/metadatablocks/citation.tsv``. If any of the below mentioned property values are changed, corresponsing ResourceBundle property file has to be edited and stored under ``dataverse.lang.directory`` location diff --git a/doc/sphinx-guides/source/admin/metadataexport.rst b/doc/sphinx-guides/source/admin/metadataexport.rst index b9036363cac..60f5c96a4e2 100644 --- a/doc/sphinx-guides/source/admin/metadataexport.rst +++ b/doc/sphinx-guides/source/admin/metadataexport.rst @@ -28,7 +28,7 @@ Note, that creating, modifying, or re-exporting an OAI set will also attempt to Export Failures --------------- -An export batch job, whether started via the API, or by the application timer, will leave a detailed log in your configured logs directory. This is the same location where your main Glassfish server.log is found. The name of the log file is ``export_[timestamp].log`` - for example, *export_2016-08-23T03-35-23.log*. The log will contain the numbers of datasets processed successfully and those for which metadata export failed, with some information on the failures detected. Please attach this log file if you need to contact Dataverse support about metadata export problems. +An export batch job, whether started via the API, or by the application timer, will leave a detailed log in your configured logs directory. This is the same location where your main app server logs are found. The name of the log file is ``export_[timestamp].log`` - for example, *export_2016-08-23T03-35-23.log*. The log will contain the numbers of datasets processed successfully and those for which metadata export failed, with some information on the failures detected. Please attach this log file if you need to contact Dataverse support about metadata export problems. Downloading Metadata via GUI ---------------------------- diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst index a901a357907..74b41c1578f 100644 --- a/doc/sphinx-guides/source/admin/monitoring.rst +++ b/doc/sphinx-guides/source/admin/monitoring.rst @@ -36,7 +36,7 @@ Create a username/password (i.e. "admin" for both): ``htpasswd /etc/munin/munin-htpasswd admin`` -Assuming you are fronting Glassfish with Apache, prevent Apache from proxying "/munin" traffic to Glassfish by adding the following line to your Apache config: +Assuming you are fronting your app server with Apache, prevent Apache from proxying "/munin" traffic to the app server by adding the following line to your Apache config: ``ProxyPassMatch ^/munin !`` @@ -90,12 +90,12 @@ Process the logs: Please note that load balancers (such as Amazon's ELB) might interfer with the ``LogFormat`` mentioned above. To start troubleshooting errors such as ``AWStats did not find any valid log lines that match your LogFormat parameter``, you might need to bump up the value of ``NbOfLinesForCorruptedLog`` in the config files above and re-try while you interate on your Apache and AWStats config. -Please note that the Dataverse team has attempted to parse Glassfish logs using AWStats but it didn't seem to just work and posts have been made at https://stackoverflow.com/questions/49134154/what-logformat-definition-does-awstats-require-to-parse-glassfish-http-access-logs and https://sourceforge.net/p/awstats/discussion/43428/thread/9b1befda/ that can be followed up on some day. +Please note that the Dataverse team has attempted to parse Glassfish/Payara logs using AWStats but it didn't seem to just work and posts have been made at https://stackoverflow.com/questions/49134154/what-logformat-definition-does-awstats-require-to-parse-glassfish-http-access-logs and https://sourceforge.net/p/awstats/discussion/43428/thread/9b1befda/ that can be followed up on some day. -Database Connection Pool used by Glassfish ------------------------------------------- +Database Connection Pool Used by App Server +------------------------------------------- -https://github.com/IQSS/dataverse/issues/2595 contains some information on enabling monitoring of Glassfish, which is disabled by default. It's a TODO to document what to do here if there is sufficient interest. +https://github.com/IQSS/dataverse/issues/2595 contains some information on enabling monitoring of app servers, which is disabled by default. It's a TODO to document what to do here if there is sufficient interest. actionlogrecord diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index d203ae020c8..8773a1c6d38 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -43,22 +43,22 @@ A User Needs Their Account to Be Converted From Institutional (Shibboleth), ORCI See :ref:`converting-shibboleth-users-to-local` and :ref:`converting-oauth-users-to-local`. -.. _troubleshooting-glassfish: +.. _troubleshooting-payara: -Glassfish ---------- +Payara +------ -.. _glassfish-log: +.. _payara-log: -Finding the Glassfish Log File -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Finding the Payara Log File +~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``/usr/local/glassfish4/glassfish/domains/domain1/logs/server.log`` is the main place to look when you encounter problems (assuming you installed Glassfish in the default directory). Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version or commit. Send more of the stack trace (the entire file if possible) to developers who can help (see "Getting Help", below) and be sure to say which version of Dataverse you are running. +``/usr/local/payara5/glassfish/domains/domain1/logs/server.log`` is the main place to look when you encounter problems (assuming you installed Payara in the default directory). Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version or commit. Send more of the stack trace (the entire file if possible) to developers who can help (see "Getting Help", below) and be sure to say which version of Dataverse you are running. -.. _increase-glassfish-logging: +.. _increase-payara-logging: -Increasing Glassfish Logging -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Increasing Payara Logging +~~~~~~~~~~~~~~~~~~~~~~~~~ For debugging purposes, you may find it helpful to temporarily increase logging levels. Here's an example of increasing logging for the Java class behind the "datasets" API endpoints: @@ -66,26 +66,26 @@ For debugging purposes, you may find it helpful to temporarily increase logging For more on setting log levels, see the :doc:`/developers/debugging` section of the Developer Guide. -Our guides focus on using the command line to manage Glassfish but you might be interested in an admin GUI at http://localhost:4848 +Our guides focus on using the command line to manage Payara but you might be interested in an admin GUI at http://localhost:4848 Deployment fails, "EJB Timer Service not available" --------------------------------------------------- -Sometimes the Dataverse application fails to deploy, or Glassfish fails to restart once the application is deployed, with the following error message: :fixedwidthplain:`"remote failure: Error occurred during deployment: Exception while loading the app : EJB Timer Service is not available. Please see server.log for more details."` +Sometimes the Dataverse application fails to deploy, or Payara fails to restart once the application is deployed, with the following error message: :fixedwidthplain:`"remote failure: Error occurred during deployment: Exception while loading the app : EJB Timer Service is not available. Please see server.log for more details."` We don't know what's causing this issue, but here's a known workaround: -- Stop Glassfish; +- Stop Payara; - Remove the ``generated`` and ``osgi-cache`` directories; - Delete all the rows from the ``EJB__TIMER__TBL`` table in the database; -- Start Glassfish +- Start Payara The shell script below performs the steps above. -Note that it may or may not work on your system, so it is provided as an example only, downloadable :download:`here `. Aside from the configuration values that need to be changed to reflect your environment (the Glassfish directory, name of the database, etc.) the script relies on the database being configured in a certain way for access. (See the comments in the script for more information) +Note that it may or may not work on your system, so it is provided as an example only, downloadable :download:`here `. Aside from the configuration values that need to be changed to reflect your environment (the Payara directory, name of the database, etc.) the script relies on the database being configured in a certain way for access. (See the comments in the script for more information) .. literalinclude:: ../_static/util/clear_timer.sh @@ -94,7 +94,7 @@ Note that it may or may not work on your system, so it is provided as an example Timer Not Working ----------------- -Dataverse relies on EJB timers to perform scheduled tasks: harvesting from remote servers, updating the local OAI sets and running metadata exports. (See :doc:`timers` for details.) If these scheduled jobs are not running on your server, this may be the result of the incompatibility between the version of PostgreSQL database you are using, and PostgreSQL JDBC driver in use by your instance of Glassfish. The symptoms: +Dataverse relies on EJB timers to perform scheduled tasks: harvesting from remote servers, updating the local OAI sets and running metadata exports. (See :doc:`timers` for details.) If these scheduled jobs are not running on your server, this may be the result of the incompatibility between the version of PostgreSQL database you are using, and PostgreSQL JDBC driver in use by your instance of Payara. The symptoms: If you are seeing the following in your server.log... @@ -108,14 +108,14 @@ followed by an Exception stack trace with these lines in it: ... it most likely means that it is the JDBC driver incompatibility that's preventing the timer from working correctly. -Make sure you install the correct version of the driver. For example, if you are running the version 9.3 of PostgreSQL, make sure you have the driver postgresql-9.3-1104.jdbc4.jar in your :fixedwidthplain:`/glassfish/lib` directory. Go `here `_ -to download the correct version of the driver. If you have an older driver in glassfish/lib, make sure to remove it, replace it with the new version and restart Glassfish. (You may need to remove the entire contents of :fixedwidthplain:`/glassfish/domains/domain1/generated` before you start Glassfish). +Make sure you install the correct version of the driver. For example, if you are running the version 9.3 of PostgreSQL, make sure you have the driver postgresql-9.3-1104.jdbc4.jar in your :fixedwidthplain:`/glassfish/lib` directory. Go `here `_ +to download the correct version of the driver. If you have an older driver in glassfish/lib, make sure to remove it, replace it with the new version and restart Payara. (You may need to remove the entire contents of :fixedwidthplain:`/glassfish/domains/domain1/generated` before you start Payara). Constraint Violations Issues ---------------------------- -In real life production use, it may be possible to end up in a situation where some values associated with the datasets in your database are no longer valid under the constraints enforced by the latest version of Dataverse. This is not very likely to happen, but if it does, the symptomps will be as follows: Some datasets can no longer be edited, long exception stack traces logged in the Glassfish server log, caused by:: +In real life production use, it may be possible to end up in a situation where some values associated with the datasets in your database are no longer valid under the constraints enforced by the latest version of Dataverse. This is not very likely to happen, but if it does, the symptoms will be as follows: Some datasets can no longer be edited, long exception stack traces logged in the app server log, caused by:: javax.validation.ConstraintViolationException: Bean Validation constraint(s) violated while executing Automatic Bean Validation on callback event:'preUpdate'. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index fa71a163608..be0d16197b9 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -511,7 +511,7 @@ The JSON format is the same as that supported by the native API's :ref:`create d .. literalinclude:: ../../../../scripts/api/data/dataset-package-files.json -Before calling the API, make sure the data files referenced by the ``POST``\ ed JSON are placed in the dataset directory with filenames matching their specified storage identifiers. In installations using POSIX storage, these files must be made readable by GlassFish. +Before calling the API, make sure the data files referenced by the ``POST``\ ed JSON are placed in the dataset directory with filenames matching their specified storage identifiers. In installations using POSIX storage, these files must be made readable by the app server user. .. tip:: If possible, it's best to avoid spaces and special characters in the storage identifier in order to avoid potential portability problems. The storage identifier corresponds with the filesystem name (or bucket identifier) of the data file, so these characters may cause unpredictability with filesystem tools. @@ -2434,7 +2434,7 @@ The fully expanded example above (without environment variables) looks like this Show Dataverse Server Name ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the server name. This is useful when a Dataverse system is composed of multiple Java EE servers behind a load balancer: +Get the server name. This is useful when a Dataverse system is composed of multiple app servers behind a load balancer: .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. @@ -2943,7 +2943,7 @@ This API streams its output in real time, i.e. it will start producing the outpu ] } -Note that if you are attempting to validate a very large number of datasets in your Dataverse, this API may time out - subject to the timeout limit set in your Glassfish configuration. If this is a production Dataverse instance serving large amounts of data, you most likely have that timeout set to some high value already. But if you need to increase it, it can be done with the asadmin command. For example:: +Note that if you are attempting to validate a very large number of datasets in your Dataverse, this API may time out - subject to the timeout limit set in your app server configuration. If this is a production Dataverse instance serving large amounts of data, you most likely have that timeout set to some high value already. But if you need to increase it, it can be done with the asadmin command. For example:: asadmin set server-config.network-config.protocols.protocol.http-listener-1.http.request-timeout-seconds=3600 diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index c1c2969a60a..b0fd7a98d57 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -58,6 +58,8 @@ Data Capture Module (DCM) Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync over ssh. +DCM was developed and tested using Glassfish but these docs have been updated with references to Payara. + Install a DCM ~~~~~~~~~~~~~ @@ -90,7 +92,7 @@ The JSON that a DCM sends to Dataverse on successful checksum validation looks s :language: json - ``status`` - The valid strings to send are ``validation passed`` and ``validation failed``. -- ``uploadFolder`` - This is the directory on disk where Dataverse should attempt to find the files that a DCM has moved into place. There should always be a ``files.sha`` file and a least one data file. ``files.sha`` is a manifest of all the data files and their checksums. The ``uploadFolder`` directory is inside the directory where data is stored for the dataset and may have the same name as the "identifier" of the persistent id (DOI or Handle). For example, you would send ``"uploadFolder": "DNXV2H"`` in the JSON file when the absolute path to this directory is ``/usr/local/glassfish4/glassfish/domains/domain1/files/10.5072/FK2/DNXV2H/DNXV2H``. +- ``uploadFolder`` - This is the directory on disk where Dataverse should attempt to find the files that a DCM has moved into place. There should always be a ``files.sha`` file and a least one data file. ``files.sha`` is a manifest of all the data files and their checksums. The ``uploadFolder`` directory is inside the directory where data is stored for the dataset and may have the same name as the "identifier" of the persistent id (DOI or Handle). For example, you would send ``"uploadFolder": "DNXV2H"`` in the JSON file when the absolute path to this directory is ``/usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/DNXV2H/DNXV2H``. - ``totalSize`` - Dataverse will use this value to represent the total size in bytes of all the files in the "package" that's created. If 360 data files and one ``files.sha`` manifest file are in the ``uploadFolder``, this value is the sum of the 360 data files. @@ -112,9 +114,9 @@ Add Dataverse settings to use mock (same as using DCM, noted above): At this point you should be able to download a placeholder rsync script. Dataverse is then waiting for news from the DCM about if checksum validation has succeeded or not. First, you have to put files in place, which is usually the job of the DCM. You should substitute "X1METO" for the "identifier" of the dataset you create. You must also use the proper path for where you store files in your dev environment. -- ``mkdir /usr/local/glassfish4/glassfish/domains/domain1/files/10.5072/FK2/X1METO`` -- ``mkdir /usr/local/glassfish4/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO`` -- ``cd /usr/local/glassfish4/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO`` +- ``mkdir /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO`` +- ``mkdir /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO`` +- ``cd /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO`` - ``echo "hello" > file1.txt`` - ``shasum file1.txt > files.sha`` @@ -158,7 +160,7 @@ Optional steps for setting up the S3 Docker DCM Variant - Set S3 as the storage driver - - ``cd /opt/glassfish4/bin/`` + - ``cd /opt/payara5/bin/`` - ``./asadmin delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"`` - ``./asadmin create-jvm-options "\-Ddataverse.files.storage-driver-id=s3"`` - ``./asadmin create-jvm-options "\-Ddataverse.files.s3.type=s3"`` @@ -181,11 +183,11 @@ Optional steps for setting up the S3 Docker DCM Variant - S3 bucket for Dataverse - - ``/usr/local/glassfish4/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.s3.bucket-name=iqsstestdcmbucket"`` + - ``/usr/local/payara5/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.s3.bucket-name=iqsstestdcmbucket"`` - S3 bucket for DCM (as Dataverse needs to do the copy over) - - ``/usr/local/glassfish4/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.dcm-s3-bucket-name=test-dcm"`` + - ``/usr/local/payara5/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.dcm-s3-bucket-name=test-dcm"`` - Set download method to be HTTP, as DCM downloads through S3 are over this protocol ``curl -X PUT "http://localhost:8080/api/admin/settings/:DownloadMethods" -d "native/http"`` @@ -218,11 +220,11 @@ Additional DCM docker development tips - There are a few logs to tail - - dvsrv : ``tail -n 2000 -f /opt/glassfish4/glassfish/domains/domain1/logs/server.log`` + - dvsrv : ``tail -n 2000 -f /opt/payara5/glassfish/domains/domain1/logs/server.log`` - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/breakage.log`` - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/access.log`` -- You may have to restart the glassfish domain occasionally to deal with memory filling up. If deployment is getting reallllllly slow, its a good time. +- You may have to restart the app server domain occasionally to deal with memory filling up. If deployment is getting reallllllly slow, its a good time. Repository Storage Abstraction Layer (RSAL) ------------------------------------------- diff --git a/doc/sphinx-guides/source/developers/coding-style.rst b/doc/sphinx-guides/source/developers/coding-style.rst index 1ae11a2f4ca..f0a05991fc0 100755 --- a/doc/sphinx-guides/source/developers/coding-style.rst +++ b/doc/sphinx-guides/source/developers/coding-style.rst @@ -85,7 +85,7 @@ Use this ``logger`` field with varying levels such as ``fine`` or ``info`` like logger.fine("will get thumbnail from dataset logo"); -Generally speaking you should use ``fine`` for everything that you don't want to show up in Glassfish's ``server.log`` file by default. If you use a higher level such as ``info`` for common operations, you will probably hear complaints that your code is too "chatty" in the logs. These logging levels can be controlled at runtime both on your development machine and in production as explained in the :doc:`debugging` section. +Generally speaking you should use ``fine`` for everything that you don't want to show up by default in the app server's log file. If you use a higher level such as ``info`` for common operations, you will probably hear complaints that your code is too "chatty" in the logs. These logging levels can be controlled at runtime both on your development machine and in production as explained in the :doc:`debugging` section. When adding logging, do not simply add ``System.out.println()`` lines because the logging level cannot be controlled. diff --git a/doc/sphinx-guides/source/developers/containers.rst b/doc/sphinx-guides/source/developers/containers.rst index 31d13b38314..ecf82533160 100755 --- a/doc/sphinx-guides/source/developers/containers.rst +++ b/doc/sphinx-guides/source/developers/containers.rst @@ -1,411 +1,16 @@ -================================= -Docker, Kubernetes, and OpenShift -================================= +================================== +Docker, Kubernetes, and Containers +================================== -Dataverse is exploring the use of Docker, Kubernetes, OpenShift and other container-related technologies. +The Dataverse community is exploring the use of Docker, Kubernetes, and other container-related technologies. The primary community-lead projects to watch are: -.. contents:: |toctitle| - :local: - -OpenShift ---------- - -From the Dataverse perspective, we are in the business of providing a "template" for OpenShift that describes how the various components we build our application on (Glassfish, PostgreSQL, Solr, the Dataverse war file itself, etc.) work together. We publish Docker images to DockerHub at https://hub.docker.com/u/iqss/ that are used in this OpenShift template. - -Dataverse's (light) use of Docker is documented below in a separate section. We actually started with Docker in the context of OpenShift, which is why OpenShift is listed first but we can imagine rearranging this in the future. - -The OpenShift template for Dataverse can be found at ``conf/openshift/openshift.json`` and if you need to hack on the template or related files under ``conf/docker`` it is recommended that you iterate on them using Minishift. - -The instructions below will walk you through spinning up Dataverse within Minishift. It is recommended that you do this on the "develop" branch to make sure everything is working before changing anything. - -Install Minishift -~~~~~~~~~~~~~~~~~ - -Minishift requires a hypervisor and since we already use VirtualBox for Vagrant, you should install VirtualBox from http://virtualbox.org . - -Download the Minishift tarball from https://docs.openshift.org/latest/minishift/getting-started/installing.html and put the ``minishift`` binary in ``/usr/local/bin`` or somewhere in your ``$PATH``. This assumes Mac or Linux. These instructions were last tested on version ``v1.14.0+1ec5877`` of Minishift. - -At this point, you might want to consider going through the Minishift quickstart to get oriented: https://docs.openshift.org/latest/minishift/getting-started/quickstart.html - -Start Minishift -~~~~~~~~~~~~~~~ - -``minishift start --vm-driver=virtualbox --memory=8GB`` - -Make the OpenShift Client Binary (oc) Executable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``eval $(minishift oc-env)`` - -Log in to Minishift from the Command Line -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Note that if you just installed and started Minishift, you are probably logged in already. This ``oc login`` step is included in case you aren't logged in anymore. - -``oc login --username developer --password=whatever`` - -Use "developer" as the username and a couple characters as the password. - -Create a Minishift Project -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Calling the project "project1" is fairly arbitrary. We'll probably want to revisit this name in the future. A project is necessary in order to create an OpenShift app. - -``oc new-project project1`` - -Note that ``oc projects`` will return a list of projects. - -Create a Dataverse App within the Minishift Project -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following command operates on the ``conf/openshift/openshift.json`` file that resides in the main Dataverse git repo. It will download images from Docker Hub and use them to spin up Dataverse within Minishift/OpenShift. Later we will cover how to make changes to the images on Docker Hub. - -``oc new-app conf/openshift/openshift.json`` - -Log into Minishift and Visit Dataverse in your Browser -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -After running the ``oc new-app`` command above, deployment of Dataverse within Minishift/OpenShift will begin. You should log into the OpenShift web interface to check on the status of the deployment. If you just created the Minishift VM with the ``minishift start`` command above, the ``oc new-app`` step is expected to take a while because the images need to be downloaded from Docker Hub. Also, the installation of Dataverse takes a while. - -Typing ``minishift console`` should open the OpenShift web interface in your browser. The IP address might not be "192.168.99.100" but it's used below as an example. - -- https://192.168.99.100:8443 (or URL from ``minishift console``) -- username: developer -- password: - -In the OpenShift web interface you should see a link that looks something like http://dataverse-project1.192.168.99.100.nip.io but the IP address will vary and will match the output of ``minishift ip``. Eventually, after deployment is complete, the Dataverse web interface will appear at this URL and you will be able to log in with the username "dataverseAdmin" and the password "admin". - -Another way to verify that Dataverse has been succesfully deployed is to make sure that the Dataverse "info" API endpoint returns a version (note that ``minishift ip`` is used because the IP address will vary): - -``curl http://dataverse-project1.`minishift ip`.nip.io/api/info/version`` - -From the perspective of OpenShift and the ``openshift.json`` config file, the HTTP link to Dataverse in called a route. See also documentation for ``oc expose``. - -Troubleshooting -~~~~~~~~~~~~~~~ - -Here are some tips on troubleshooting your deployment of Dataverse to Minishift. - -Check Status of Dataverse Deployment to Minishift -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``oc status`` - -Once images have been downloaded from Docker Hub, the output below will change from ``Pulling`` to ``Pulled``. - -``oc get events | grep Pull`` - -This is a deep dive: - -``oc get all`` - -Review Logs of Dataverse Deployment to Minishift -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Logs are provided in the web interface to each of the deployment configurations. The URLs should be something like this (but the IP address) will vary and you should click "View Log". The installation of Dataverse is done within the one Glassfish deployment configuration: - -- https://192.168.99.100:8443/console/project/project1/browse/dc/dataverse-glassfish -- https://192.168.99.100:8443/console/project/project1/browse/dc/dataverse-postgresql -- https://192.168.99.100:8443/console/project/project1/browse/dc/dataverse-solr - -You can also see logs from each of the components (Glassfish, PostgreSQL, and Solr) from the command line with ``oc logs`` like this (just change the ``grep`` at the end): - -``oc logs $(oc get po -o json | jq '.items[] | select(.kind=="Pod").metadata.name' -r | grep glassfish)`` - -Get a Shell (ssh/rsh) on Containers Deployed to Minishift -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can get a shell on any of the containers for each of the components (Glassfish, PostgreSQL, and Solr) with ``oc rc`` (just change the ``grep`` at the end): - -``oc rsh $(oc get po -o json | jq '.items[] | select(.kind=="Pod").metadata.name' -r | grep glassfish)`` - -From the ``rsh`` prompt of the Glassfish container you could run something like the following to make sure that Dataverse is running on port 8080: - -``curl http://localhost:8080/api/info/version`` - -Cleaning up -~~~~~~~~~~~ - -If you simply wanted to try out Dataverse on Minishift and want to clean up, you can run ``oc delete project project1`` to delete the project or ``minishift stop`` and ``minishift delete`` to delete the entire Minishift VM and all the Docker containers inside it. - -Making Changes -~~~~~~~~~~~~~~ - -Making Changes to Docker Images -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you're interested in using Minishift for development and want to change the Dataverse code, you will need to get set up to create Docker images based on your changes and make them available within Minishift. - -It is recommended to add experimental images to Minishift's internal registry. Note that despite what https://docs.openshift.org/latest/minishift/openshift/openshift-docker-registry.html says you will not use ``docker push`` because we have seen "unauthorized: authentication required” when trying to push to it as reported at https://github.com/minishift/minishift/issues/817 . Rather you will run ``docker build`` and run ``docker images`` to see that your newly build images are listed in Minishift's internal registry. - -First, set the Docker environment variables so that ``docker build`` and ``docker images`` refer to the internal Minishift registry rather than your normal Docker setup: - -``eval $(minishift docker-env)`` - -When you're ready to build, change to the right directory: - -``cd conf/docker`` - -And then run the build script in "internal" mode: - -``./build.sh internal`` - -Note that ``conf/openshift/openshift.json`` must not have ``imagePullPolicy`` set to ``Always`` or it will pull from "iqss" on Docker Hub. Changing it to ``IfNotPresent`` allow Minishift to use the images shown from ``docker images`` rather than the ones on Docker Hub. - -Using Minishift for day to day Dataverse development might be something we want to investigate in the future. These blog posts talk about developing Java applications using Minishift/OpenShift: - -- https://blog.openshift.com/fast-iterative-java-development-on-openshift-kubernetes-using-rsync/ -- https://blog.openshift.com/debugging-java-applications-on-openshift-kubernetes/ - -Making Changes to the OpenShift Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you are interested in changing the OpenShift config file for Dataverse at ``conf/openshift/openshift.json`` note that in many cases once you have Dataverse running in Minishift you can use ``oc process`` and ``oc apply`` like this (but please note that some errors and warnings are expected): - -``oc process -f conf/openshift/openshift.json | oc apply -f -`` - -The slower way to iterate on the ``openshift.json`` file is to delete the project and re-create it. - -Making Changes to the PostgreSQL Database from the Glassfish Pod -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can access and modify the PostgreSQL database via an interactive terminal called psql. - -To log in to psql from the command line of the Glassfish pod, type the following command: - -``PGPASSWORD=$POSTGRES_PASSWORD; export PGPASSWORD; /usr/bin/psql -h $POSTGRES_SERVER.$POSTGRES_SERVICE_HOST -U $POSTGRES_USER -d $POSTGRES_DATABASE`` +- https://github.com/IQSS/dataverse-kubernetes +- https://github.com/IQSS/dataverse-docker -To log in as an admin, type this command instead: +The :doc:`testing` section mentions using Docker for integration tests. -``PGPASSWORD=$POSTGRESQL_ADMIN_PASSWORD; export PGPASSWORD; /usr/bin/psql -h $POSTGRES_SERVER.$POSTGRES_SERVICE_HOST -U postgres -d $POSTGRES_DATABASE`` - -Scaling Dataverse by Increasing Replicas in a StatefulSet -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Glassfish, Solr and PostgreSQL Pods are in a "StatefulSet" which is a concept from OpenShift and Kubernetes that you can read about at https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ - -As of this writing, the ``openshift.json`` file has a single "replica" for each of these two stateful sets. It's possible to increase the number of replicas from 1 to 3, for example, with this command: - -``oc scale statefulset/dataverse-glassfish --replicas=3`` - -The command above should result in two additional Glassfish pods being spun up. The name of the pods is significant and there is special logic in the "zeroth" pod ("dataverse-glassfish-0" and "dataverse-postgresql-0"). For example, only "dataverse-glassfish-0" makes itself the dedicated timer server as explained in :doc:`/admin/timers` section of the Admin Guide. "dataverse-glassfish-1" and other higher number pods will not be configured as a timer server. - -Once you have multiple Glassfish servers you may notice bugs that will require additional configuration to fix. One such bug has to do with Dataverse logos which are stored at ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/logos`` on each of the Glassfish servers. This means that the logo will look fine when you just uploaded it because you're on the server with the logo on the local file system but when you visit that dataverse in the future and you're on a differernt Glassfish server, you will see a broken image. (You can find some discussion of this logo bug at https://github.com/IQSS/dataverse-aws/issues/10 and http://irclog.iq.harvard.edu/dataverse/2016-10-21 .) This is all "advanced" installation territory (see the :doc:`/installation/advanced` section of the Installation Guide) and OpenShift might be a good environment in which to work on some of these bugs. - -Multiple PostgreSQL servers are possible within the OpenShift environment as well and have been set up with some amount of replication. "dataverse-postgresql-0" is the master and non-zero pods are the slaves. We have just scratched the surface of this configuration but replication from master to slave seems to we working. Future work could include failover and making Dataverse smarter about utilizing multiple PostgreSQL servers for reads. Right now we assume Dataverse is only being used with a single PostgreSQL server and that it's the master. - -Solr supports index distribution and replication for scaling. For OpenShift use, we choose replication. It's possible to scale up Solr using the method method similar to Glassfish, as mentioned aboved -In OpenShift, the first Solr pod, dataverse-solr-0, will be the master node, and the rest will be slave nodes - - -Configuring Persistent Volumes and Solr master node recovery -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Solr requires backing up the search index to persistent storage. For our proof of concept, we configure a hostPath, which allows Solr containers to access the hosts' file system, for our Solr containers backups. To read more about OpenShift/Kubernetes' persistent volumes, please visit: https://kubernetes.io/docs/concepts/storage/persistent-volumes - -To allow containers to use a host's storage, we need to allow access to that directory first. In this example, we expose /tmp/share to the containers:: - -# mkdir /tmp/share -# chcon -R -t svirt_sandbox_file_t -# chgrp root -R /tmp/share -# oc login -u system:admin -# oc edit scc restricted # Update allowHostDirVolumePlugin to true and runAsUser type to RunAsAny - - -To add a persistent volume and persistent volume claim, in conf/docker/openshift/openshift.json, add the following to objects in openshift.json. -Here, we are using hostPath for development purposes. Since OpenShift supports many types of cluster storages, -if the administrator wishes to use any cluster storage like EBS, Google Cloud Storage, etc, they would have to use a different type of Persistent Storage:: - - { - "kind" : "PersistentVolume", - "apiVersion" : "v1", - "metadata":{ - "name" : "solr-index-backup", - "labels":{ - "name" : "solr-index-backup", - "type" : "local" - } - }, - "spec":{ - "capacity":{ - "storage" : "8Gi" - }, - "accessModes":[ - "ReadWriteMany", "ReadWriteOnce", "ReadOnlyMany" - ], - "hostPath": { - "path" : "/tmp/share" - } - } - }, - { - "kind" : "PersistentVolumeClaim", - "apiVersion": "v1", - "metadata": { - "name": "solr-claim" - }, - "spec": { - "accessModes": [ - "ReadWriteMany", "ReadWriteOnce", "ReadOnlyMany" - ], - "resources": { - "requests": { - "storage": "3Gi" - } - }, - "selector":{ - "matchLabels":{ - "name" : "solr-index-backup", - "type" : "local" - } - } - } - } - - -To make solr container mount the hostPath, add the following part under .spec.spec (for Solr StatefulSet):: - - { - "kind": "StatefulSet", - "apiVersion": "apps/v1beta1", - "metadata": { - "name": "dataverse-solr", - .... - - "spec": { - "serviceName" : "dataverse-solr-service", - ..... - - "spec": { - "volumes": [ - { - "name": "solr-index-backup", - "persistentVolumeClaim": { - "claimName": "solr-claim" - } - } - ], - - "containers": [ - .... - - "volumeMounts":[ - { - "mountPath" : "/var/share", - "name" : "solr-index-backup" - } - - - -Solr is now ready for backup and recovery. In order to backup:: - - oc rsh dataverse-solr-0 - curl 'http://localhost:8983/solr/collection1/replication?command=backup&location=/var/share' - - -In solr entrypoint.sh, it's configured so that if dataverse-solr-0 failed, it will get the latest version of the index in the backup and restore. All backups are stored in /tmp/share in the host, or /home/share in solr containers. - -Running Containers to Run as Root in Minishift -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is **not** recommended to run containers as root in Minishift because for security reasons OpenShift doesn't support running containers as root. However, it's good to know how to allow containers to run as root in case you need to work on a Docker image to make it run as non-root. - -For more information on improving Docker images to run as non-root, see "Support Arbitrary User IDs" at https://docs.openshift.org/latest/creating_images/guidelines.html#openshift-origin-specific-guidelines - -Let's say you have a container that you suspect works fine when it runs as root. You want to see it working as-is before you start hacking on the Dockerfile and entrypoint file. You can configure Minishift to allow containers to run as root with this command: - -``oc adm policy add-scc-to-user anyuid -z default --as system:admin`` - -Once you are done testing you can revert Minishift back to not allowing containers to run as root with this command: - -``oc adm policy remove-scc-from-user anyuid -z default --as system:admin`` - -Minishift Resources -~~~~~~~~~~~~~~~~~~~ - -The following resources might be helpful. - -- https://blog.openshift.com/part-1-from-app-to-openshift-runtimes-and-templates/ -- https://blog.openshift.com/part-2-creating-a-template-a-technical-walkthrough/ -- https://docs.openshift.com/enterprise/3.0/architecture/core_concepts/templates.html - -Docker ------- - -From the Dataverse perspective, Docker is important for a few reasons: - -- There is interest from the community in running Dataverse on OpenShift and some initial work has been done to get Dataverse running on Minishift in Docker containers. Minishift makes use of Docker images on Docker Hub. To build new Docker images and push them to Docker Hub, you'll need to install Docker. The main issue to follow is https://github.com/IQSS/dataverse/issues/4040 . -- Docker may aid in testing efforts if we can easily spin up Docker images based on code in pull requests and run the full integration suite against those images. See the :doc:`testing` section for more information on integration tests. - -Installing Docker -~~~~~~~~~~~~~~~~~ - -On Linux, you can probably get Docker from your package manager. - -On Mac, download the ``.dmg`` from https://www.docker.com and install it. As of this writing is it known as Docker Community Edition for Mac. - -On Windows, we have heard reports of success using Docker on a Linux VM running in VirtualBox or similar. There's something called "Docker Community Edition for Windows" but we haven't tried it. See also the :doc:`windows` section. - -As explained above, we use Docker images in two different contexts: - -- Testing using an "all in one" Docker image (ephemeral, unpublished) -- Future production use on Minishift/OpenShift/Kubernetes (published to Docker Hub) - -All In One Docker Images for Testing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The "all in one" Docker files are in ``conf/docker-aio`` and you should follow the readme in that directory for more information on how to use them. - -Future production use on Minishift/OpenShift/Kubernetes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -FIXME: rewrite this section to talk about only pushing stable images to Docker Hub. - -When working with Docker in the context of Minishift, follow the instructions above and make sure you get the Dataverse Docker images running in Minishift before you start messing with them. - -As of this writing, the Dataverse Docker images we publish under https://hub.docker.com/u/iqss/ are highly experimental. They were originally tagged with branch names like ``kick-the-tires`` and as of this writing the ``latest`` tag should be considered highly experimental and not for production use. See https://github.com/IQSS/dataverse/issues/4040 for the latest status and please reach out if you'd like to help! - -Change to the docker directory: - -``cd conf/docker`` - -Edit one of the files: - -``vim dataverse-glassfish/Dockerfile`` - -At this point you want to build the image and run it. We are assuming you want to run it in your Minishift environment. We will be building your image and pushing it to Docker Hub. - -Log in to Docker Hub with an account that has access to push to the ``iqss`` organization: - -``docker login`` - -(If you don't have access to push to the ``iqss`` organization, you can push elsewhere and adjust your ``openshift.json`` file accordingly.) - -Build and push the images to Docker Hub: - -``./build.sh`` - -Note that you will see output such as ``digest: sha256:213b6380e6ee92607db5d02c9e88d7591d81f4b6d713224d47003d5807b93d4b`` that should later be reflected in Minishift to indicate that you are using the latest image you just pushed to Docker Hub. - -You can get a list of all repos under the ``iqss`` organization with this: - -``curl https://hub.docker.com/v2/repositories/iqss/`` - -To see a specific repo: - -``curl https://hub.docker.com/v2/repositories/iqss/dataverse-glassfish/`` - -Known Issues with Dataverse Images on Docker Hub -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Again, Dataverse Docker images on Docker Hub are highly experimental at this point. As of this writing, their purpose is primarily for kicking the tires on Dataverse. Here are some known issues: - -- The Dataverse installer is run in the entrypoint script every time you run the image. Ideally, Dataverse would be installed in the Dockerfile instead. Dataverse is being installed in the entrypoint script because it needs PosgreSQL to be up already so that database tables can be created when the war file is deployed. -- The storage should be abstracted. Storage of data files and PostgreSQL data. Probably Solr data. -- Better tuning of memory by examining ``/sys/fs/cgroup/memory/memory.limit_in_bytes`` and incorporating this into the Dataverse installation script. -- Only a single Glassfish server can be used. See "Dedicated timer server in a Dataverse server cluster" in the :doc:`/admin/timers` section of the Installation Guide. -- Only a single PostgreSQL server can be used. -- Only a single Solr server can be used. +.. contents:: |toctitle| + :local: ---- diff --git a/doc/sphinx-guides/source/developers/debugging.rst b/doc/sphinx-guides/source/developers/debugging.rst index e2cfd0ac9f4..402788cf4e0 100644 --- a/doc/sphinx-guides/source/developers/debugging.rst +++ b/doc/sphinx-guides/source/developers/debugging.rst @@ -8,7 +8,7 @@ Debugging Logging ------- -By default, Glassfish logs at the "INFO" level but logging can be increased to "FINE" on the fly with (for example) ``./asadmin set-log-levels edu.harvard.iq.dataverse.api.Datasets=FINE``. Running ``./asadmin list-log-levels`` will show the current logging levels. +By default, the app server logs at the "INFO" level but logging can be increased to "FINE" on the fly with (for example) ``./asadmin set-log-levels edu.harvard.iq.dataverse.api.Datasets=FINE``. Running ``./asadmin list-log-levels`` will show the current logging levels. ---- diff --git a/doc/sphinx-guides/source/developers/dependencies.rst b/doc/sphinx-guides/source/developers/dependencies.rst index 72fdad03e43..599606e7714 100644 --- a/doc/sphinx-guides/source/developers/dependencies.rst +++ b/doc/sphinx-guides/source/developers/dependencies.rst @@ -5,7 +5,7 @@ Dependency Management .. contents:: |toctitle| :local: -Dataverse is (currently) a Java EE 7 based application, that uses a lot of additional libraries for special purposes. +Dataverse is (currently) a Jakarta EE 8 based application, that uses a lot of additional libraries for special purposes. This includes features like support for SWORD-API, S3 storage and many others. Besides the code that glues together the single pieces, any developer needs to describe used dependencies for the @@ -76,7 +76,7 @@ dependency definition does not change. 2. **Clean up** direct dependencies no longer in use. It will bloat the deployment package otherwise! 3. Care about the **scope**. Do not include "testing only" dependencies in the package - it will hurt you in IDEs and bloat things. [#f1]_ 4. Avoid using different dependencies for the **same purpose**, e. g. different JSON parsing libraries. -5. Refactor your code to **use Java EE** standards as much as possible. +5. Refactor your code to **use Jakarta EE** standards as much as possible. 6. When you rely on big SDKs or similar big cool stuff, try to **include the smallest portion possible**. Complete SDK bundles are typically heavyweight and most of the time unnecessary. 7. **Don't include transitive dependencies.** [#f2]_ @@ -194,13 +194,13 @@ A reduced example, only showing bits relevant to the above cases and usage of an aws-java-sdk-s3 - + com.fasterxml.jackson.core jackson-core - + com.fasterxml.jackson.core jackson-databind diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index e7babf75b34..617149ba4ba 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -34,7 +34,7 @@ On Linux, you are welcome to use the OpenJDK available from package managers. Install Netbeans or Maven ~~~~~~~~~~~~~~~~~~~~~~~~~ -NetBeans IDE (Java EE bundle) is recommended, and can be downloaded from http://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Java EE projects, and includes a required build tool, Maven. +NetBeans IDE is recommended, and can be downloaded from http://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Jakarta EE projects, and includes a required build tool, Maven. Below we describe how to build the Dataverse war file with Netbeans but if you prefer to use only Maven, you can find installation instructions in the :doc:`tools` section. @@ -53,11 +53,11 @@ Fork https://github.com/IQSS/dataverse and then clone your fork like this: Build the Dataverse War File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To build the Dataverse war file using versions of Netbeans newer than 8.2 requires some setup because Java EE support is not enabled by default. An alternative is to build the war file with Maven, which is explained below. +To build the Dataverse war file using versions of Netbeans newer than 8.2 requires some setup because Jakarta EE support is not enabled by default. An alternative is to build the war file with Maven, which is explained below. Launch Netbeans and click "File" and then "Open Project". Navigate to where you put the Dataverse code and double-click "dataverse" to open the project. -If you are using Netbeans 8.2, Java EE support should "just work" but if you are using a newer version of Netbeans, you will see "dataverse (broken)". If you see "broken", click "Tools", "Plugins", and "Installed". Check the box next to "Java Web and EE" and click "Activate". Let Netbeans install all the dependencies. You will observe that the green "Active" checkmark does not appear next to "Java Web and EE". Restart Netbeans. +If you are using Netbeans 8.2, Jakarta EE support should "just work" but if you are using a newer version of Netbeans, you will see "dataverse (broken)". If you see "broken", click "Tools", "Plugins", and "Installed". Check the box next to "Java Web and EE" and click "Activate". Let Netbeans install all the dependencies. You will observe that the green "Active" checkmark does not appear next to "Java Web and EE". Restart Netbeans. In Netbeans, select "dataverse" under Projects and click "Run" in the menu and then "Build Project (dataverse)". The first time you build the war file, it will take a few minutes while dependencies are downloaded from Maven Central. Feel free to move on to other steps but check back for "BUILD SUCCESS" at the end. @@ -77,29 +77,29 @@ On Mac, run this command: On Linux, install ``jq`` from your package manager or download a binary from http://stedolan.github.io/jq/ -Install Glassfish -~~~~~~~~~~~~~~~~~ +Install Payara +~~~~~~~~~~~~~~ -Glassfish 4.1 is required. +Payara 5.201 or higher is required. -To install Glassfish, run the following commands: +To install Payara, run the following commands: ``cd /usr/local`` -``sudo curl -O http://download.oracle.com/glassfish/4.1/release/glassfish-4.1.zip`` +``sudo curl -O -L https://search.maven.org/remotecontent?filepath=fish/payara/distributions/payara/5.201/payara-5.201.zip`` -``sudo unzip glassfish-4.1.zip`` +``sudo unzip payara-5.201.zip`` -``sudo chown -R $USER /usr/local/glassfish4`` +``sudo chown -R $USER /usr/local/payara5`` -Test Glassfish Startup Time on Mac -++++++++++++++++++++++++++++++++++ +Test Payara Startup Time on Mac ++++++++++++++++++++++++++++++++ -``cd /usr/local/glassfish4/glassfish/bin`` +``cd /usr/local/payara5/glassfish/bin`` ``./asadmin start-domain`` -``grep "startup time" /usr/local/glassfish4/glassfish/domains/domain1/logs/server.log`` +``grep "startup time" /usr/local/payara5/glassfish/domains/domain1/logs/server.log`` If you are seeing startup times in the 30 second range (31,584ms for "Felix" for example) please be aware that startup time can be greatly reduced (to less than 1.5 seconds in our testing) if you make a small edit to your ``/etc/hosts`` file as described at https://stackoverflow.com/questions/39636792/jvm-takes-a-long-time-to-resolve-ip-address-for-localhost/39698914#39698914 and https://thoeni.io/post/macos-sierra-java/ @@ -172,17 +172,11 @@ To install Solr, execute the following commands: Run the Dataverse Installer Script ---------------------------------- -Navigate to the directory where you cloned the Dataverse git repo and run these commands: +Navigate to the directory where you cloned the Dataverse git repo change directories to the ``scripts/installer`` directory like this: ``cd scripts/installer`` -``./install`` - -It's fine to accept the default values. - -After a while you will see ``Enter admin user name [Enter to accept default]>`` and you can just hit Enter. - -**NEW in v.4.19:** We have added a new implementation of the installer script written in Python. It is intended to eventually replace the old installer above (written in Perl). For now it is being offered as an (experimental) alternative. See README_python.txt, in the scripts/installer directory, for more information on how to run it (please give it a try, and let us know if there are any problems and/or if you have any suggestions!) +Follow the instructions in :download:`README_python.txt <../../../../scripts/installer/README_python.txt>` which can be found in the directory above. Verify Dataverse is Running ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index 3ebfecd4a35..2ce9c52bd37 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -24,9 +24,9 @@ If you have any questions at all, please reach out to other developers via the c Core Technologies ----------------- -Dataverse is a `Java EE `_ application that is compiled into a war file and deployed to an application server (Glassfish) which is configured to work with a relational database (PostgreSQL) and a search engine (Solr). +Dataverse is a `Jakarta EE `_ application that is compiled into a WAR file and deployed to an application server (app server) which is configured to work with a relational database (PostgreSQL) and a search engine (Solr). -We make use of a variety of Java EE technologies such as JPA, JAX-RS, JMS, and JSF. The front end is built using PrimeFaces and Bootstrap. +We make use of a variety of Jakarta EE technologies such as JPA, JAX-RS, JMS, and JSF. The front end is built using PrimeFaces and Bootstrap. Roadmap ------- diff --git a/doc/sphinx-guides/source/developers/make-data-count.rst b/doc/sphinx-guides/source/developers/make-data-count.rst index 253304d78ea..d7d20a02e11 100644 --- a/doc/sphinx-guides/source/developers/make-data-count.rst +++ b/doc/sphinx-guides/source/developers/make-data-count.rst @@ -38,9 +38,9 @@ Next you need to have Dataverse add some entries to the log that Counter Process Next you should run Counter Processor to convert the log into a SUSHI report, which is in JSON format. Before running Counter Processor, you need to put a configuration file into place. As a starting point use :download:`counter-processor-config.yaml <../../../../scripts/vagrant/counter-processor-config.yaml>` and edit the file, paying particular attention to the following settings: -- ``log_name_pattern`` You might want something like ``/usr/local/glassfish4/glassfish/domains/domain1/logs/counter_(yyyy-mm-dd).log`` +- ``log_name_pattern`` You might want something like ``/usr/local/payara5/glassfish/domains/domain1/logs/counter_(yyyy-mm-dd).log`` - ``year_month`` You should probably set this to the current month. -- ``output_file`` This needs to be a directory that the "glassfish" Unix user can read but that the "counter" user can write to. In dev, you can probably get away with "/tmp" as the directory. +- ``output_file`` This needs to be a directory that the "dataverse" Unix user can read but that the "counter" user can write to. In dev, you can probably get away with "/tmp" as the directory. - ``platform`` Out of the box from Counter Processor this is set to ``Dash`` but we're not 100% sure if this should be "Dataverse" or a branch for a Dataverse installation like "LibreScholar". - ``upload_to_hub`` This should be "False" unless you are testing sending SUSHI reports to the DataCite hub. - ``simulate_date`` You should probably set this to tomorrow. diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index 07f65f6828a..6d49f927a9e 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -17,7 +17,7 @@ In the past (before adopting Flyway) we used to keep SQL upgrade scripts in ``sc How to Determine if You Need to Create a SQL Upgrade Script ----------------------------------------------------------- -If you are creating a new database table (which maps to an ``@Entity`` in JPA), you do not need to create or update a SQL upgrade script. The reason for this is that we use ``create-tables`` in ``src/main/resources/META-INF/persistence.xml`` so that new tables are automatically created by Glassfish when you deploy your war file. +If you are creating a new database table (which maps to an ``@Entity`` in JPA), you do not need to create or update a SQL upgrade script. The reason for this is that we use ``create-tables`` in ``src/main/resources/META-INF/persistence.xml`` so that new tables are automatically created by the app server when you deploy your war file. If you are doing anything other than creating a new database table such as adding a column to an existing table, you must create or update a SQL upgrade script. diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 51737d5cbbc..a240f4ea119 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -149,7 +149,7 @@ Edit ``main.yml`` to set the desired GitHub repo, branch, and to ensure that the - ``dataverse.api.test_suite: true`` - ``dataverse.sampledata.enabled: true`` -If you wish, you may pass the local path of a logging directory, which will tell ec2-create-instance.sh to `grab glassfish, maven and other logs`_ for your review. +If you wish, you may pass the local path of a logging directory, which will tell ec2-create-instance.sh to `grab various logs `_ for your review. Finally, run the script: @@ -157,7 +157,7 @@ Finally, run the script: $ ./ec2-create-instance.sh -g main.yml -l log_dir -Near the beginning and at the end of the ec2-create-instance.sh output you will see instructions for connecting to the instance via SSH. If you are actively working on a branch and want to refresh the warfile after each commit, you may wish to call a `redeploy.sh`_ script placed by the Ansible role, which will do a "git pull" against your branch, build the warfile, deploy the warfile, then restart glassfish. By default this script is written to /tmp/dataverse/redeploy.sh. You may invoke the script by appending it to the SSH command in ec2-create's output: +Near the beginning and at the end of the ec2-create-instance.sh output you will see instructions for connecting to the instance via SSH. If you are actively working on a branch and want to refresh the warfile after each commit, you may wish to call a `redeploy.sh `_ script placed by the Ansible role, which will do a "git pull" against your branch, build the warfile, deploy the warfile, then restart the app server. By default this script is written to /tmp/dataverse/redeploy.sh. You may invoke the script by appending it to the SSH command in ec2-create's output: .. code-block:: bash @@ -168,7 +168,7 @@ Running the full API test suite using Docker To run the full suite of integration tests on your laptop, we recommend using the "all in one" Docker configuration described in ``conf/docker-aio/readme.txt`` in the root of the repo. -Alternatively, you can run tests against Glassfish running on your laptop by following the "getting set up" steps below. +Alternatively, you can run tests against the app server running on your laptop by following the "getting set up" steps below. Getting Set Up to Run REST Assured Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -296,33 +296,35 @@ To run these tests, simply call out to Maven: Measuring Coverage of Integration Tests --------------------------------------- -Measuring the code coverage of integration tests with Jacoco requires several steps. In order to make these steps clear we'll use "/usr/local/glassfish4" as the Glassfish directory and "glassfish" as the Glassfish Unix user. +Measuring the code coverage of integration tests with Jacoco requires several steps. In order to make these steps clear we'll use "/usr/local/payara5" as the Payara directory and "dataverse" as the Payara Unix user. -Add jacocoagent.jar to Glassfish -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Please note that this was tested under Glassfish 4 but it is hoped that the same steps will work with Payara 5. + +Add jacocoagent.jar to Payara +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order to get code coverage reports out of Glassfish we'll be adding jacocoagent.jar to the Glassfish "lib" directory. +In order to get code coverage reports out of Payara we'll be adding jacocoagent.jar to the Payara "lib" directory. First, we need to download Jacoco. Look in pom.xml to determine which version of Jacoco we are using. As of this writing we are using 0.8.1 so in the example below we download the Jacoco zip from https://github.com/jacoco/jacoco/releases/tag/v0.8.1 -Note that we are running the following commands as the user "glassfish". In short, we stop Glassfish, add the Jacoco jar file, and start up Glassfish again. +Note that we are running the following commands as the user "dataverse". In short, we stop Payara, add the Jacoco jar file, and start up Payara again. .. code-block:: bash - su - glassfish - cd /home/glassfish + su - dataverse + cd /home/dataverse mkdir -p local/jacoco-0.8.1 cd local/jacoco-0.8.1 wget https://github.com/jacoco/jacoco/releases/download/v0.8.1/jacoco-0.8.1.zip unzip jacoco-0.8.1.zip - /usr/local/glassfish4/bin/asadmin stop-domain - cp /home/glassfish/local/jacoco-0.8.1/lib/jacocoagent.jar /usr/local/glassfish4/glassfish/lib - /usr/local/glassfish4/bin/asadmin start-domain + /usr/local/payara5/bin/asadmin stop-domain + cp /home/dataverse/local/jacoco-0.8.1/lib/jacocoagent.jar /usr/local/payara5/glassfish/lib + /usr/local/payara5/bin/asadmin start-domain Add jacococli.jar to the WAR File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -As the "glassfish" user download :download:`instrument_war_jacoco.bash <../_static/util/instrument_war_jacoco.bash>` (or skip ahead to the "git clone" step to get the script that way) and give it two arguments: +As the "dataverse" user download :download:`instrument_war_jacoco.bash <../_static/util/instrument_war_jacoco.bash>` (or skip ahead to the "git clone" step to get the script that way) and give it two arguments: - path to your pristine WAR file - path to the new WAR file the script will create with jacococli.jar in it @@ -336,25 +338,25 @@ Deploy the Instrumented WAR File Please note that you'll want to undeploy the old WAR file first, if necessary. -Run this as the "glassfish" user. +Run this as the "dataverse" user. .. code-block:: bash - /usr/local/glassfish4/bin/asadmin deploy dataverse-jacoco.war + /usr/local/payara5/bin/asadmin deploy dataverse-jacoco.war -Note that after deployment the file "/usr/local/glassfish4/glassfish/domains/domain1/config/jacoco.exec" exists and is empty. +Note that after deployment the file "/usr/local/payara5/glassfish/domains/domain1/config/jacoco.exec" exists and is empty. Run Integration Tests ~~~~~~~~~~~~~~~~~~~~~ -Note that even though you see "docker-aio" in the command below, we assume you are not necessarily running the test suite within Docker. (Some day we'll probably move this script to another directory.) For this reason, we pass the URL with the normal port (8080) that Glassfish runs on to the ``run-test-suite.sh`` script. +Note that even though you see "docker-aio" in the command below, we assume you are not necessarily running the test suite within Docker. (Some day we'll probably move this script to another directory.) For this reason, we pass the URL with the normal port (8080) that app servers run on to the ``run-test-suite.sh`` script. -Note that "/usr/local/glassfish4/glassfish/domains/domain1/config/jacoco.exec" will become non-empty after you stop and start Glassfish. You must stop and start Glassfish before every run of the integration test suite. +Note that "/usr/local/payara5/glassfish/domains/domain1/config/jacoco.exec" will become non-empty after you stop and start Payara. You must stop and start Payara before every run of the integration test suite. .. code-block:: bash - /usr/local/glassfish4/bin/asadmin stop-domain - /usr/local/glassfish4/bin/asadmin start-domain + /usr/local/payara5/bin/asadmin stop-domain + /usr/local/payara5/bin/asadmin start-domain git clone https://github.com/IQSS/dataverse.git cd dataverse conf/docker-aio/run-test-suite.sh http://localhost:8080 @@ -364,12 +366,12 @@ Note that "/usr/local/glassfish4/glassfish/domains/domain1/config/jacoco.exec" w Create Code Coverage Report ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Run these commands as the "glassfish" user. The ``cd dataverse`` means that you should change to the directory where you cloned the "dataverse" git repo. +Run these commands as the "dataverse" user. The ``cd dataverse`` means that you should change to the directory where you cloned the "dataverse" git repo. .. code-block:: bash cd dataverse - java -jar /home/glassfish/local/jacoco-0.8.1/lib/jacococli.jar report --classfiles target/classes --sourcefiles src/main/java --html target/coverage-it/ /usr/local/glassfish4/glassfish/domains/domain1/config/jacoco.exec + java -jar /home/dataverse/local/jacoco-0.8.1/lib/jacococli.jar report --classfiles target/classes --sourcefiles src/main/java --html target/coverage-it/ /usr/local/payara5/glassfish/domains/domain1/config/jacoco.exec Read Code Coverage Report ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -477,7 +479,7 @@ Future Work on Unit Tests - Review pull requests from @bencomp for ideas for approaches to testing: https://github.com/IQSS/dataverse/pulls?q=is%3Apr+author%3Abencomp - Come up with a way to test commands: http://irclog.iq.harvard.edu/dataverse/2015-11-04#i_26750 -- Test EJBs using Arquillian, embedded Glassfish, or similar. @bmckinney kicked the tires on Arquillian at https://github.com/bmckinney/bio-dataverse/commit/2f243b1db1ca704a42cd0a5de329083763b7c37a +- Test EJBs using Arquillian, embedded app servers, or similar. @bmckinney kicked the tires on Arquillian at https://github.com/bmckinney/bio-dataverse/commit/2f243b1db1ca704a42cd0a5de329083763b7c37a Future Work on Integration Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index 9084f3d1993..457016a8ac0 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -2,7 +2,7 @@ Tips ==== -If you just followed the steps in :doc:`dev-environment` for the first time, you will need to get set up to deploy code to Glassfish. Below you'll find other tips as well. +If you just followed the steps in :doc:`dev-environment` for the first time, you will need to get set up to deploy code to your app server. Below you'll find other tips as well. .. contents:: |toctitle| :local: @@ -10,40 +10,38 @@ If you just followed the steps in :doc:`dev-environment` for the first time, you Iterating on Code and Redeploying --------------------------------- -When you followed the steps in the :doc:`dev-environment` section, the war file was deployed to Glassfish by the ``install`` script. That's fine but once you're ready to make a change to the code you will need to get comfortable with undeploying and redeploying code (a war file) to Glassfish. +When you followed the steps in the :doc:`dev-environment` section, the war file was deployed to Payara by the Dataverse installation script. That's fine but once you're ready to make a change to the code you will need to get comfortable with undeploying and redeploying code (a war file) to Payara. -It's certainly possible to manage deployment and undeployment of the war file via the command line using the ``asadmin`` command that ships with Glassfish (that's what the ``install`` script uses and the steps are documented below), but we recommend getting set up with and IDE such as Netbeans to manage deployment for you. +It's certainly possible to manage deployment and undeployment of the war file via the command line using the ``asadmin`` command that ships with Payara (that's what the Dataverse installation script uses and the steps are documented below), but we recommend getting set up with and IDE such as Netbeans to manage deployment for you. -Undeploy the war File from the ``install`` Script -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Undeploy the war File from the Dataverse Installation Script +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Because the initial deployment of the war file was done outside of Netbeans by the ``install`` script, it's a good idea to undeploy that war file to give Netbeans a clean slate to work with. +Because the initial deployment of the war file was done outside of Netbeans by the Dataverse installation script, it's a good idea to undeploy that war file to give Netbeans a clean slate to work with. -Assuming you installed Glassfish in ``/usr/local/glassfish4``, run the following ``asadmin`` command to see the version of Dataverse that the ``install`` script deployed: +Assuming you installed Payara in ``/usr/local/payara5``, run the following ``asadmin`` command to see the version of Dataverse that the Dataverse installation script deployed: -``/usr/local/glassfish4/bin/asadmin list-applications`` +``/usr/local/payara5/bin/asadmin list-applications`` -You will probably see something like ``dataverse-4.8.5 `` as the output. To undeploy, use whichever version you see like this: +You will probably see something like ``dataverse-5.0 `` as the output. To undeploy, use whichever version you see like this: -``/usr/local/glassfish4/bin/asadmin undeploy dataverse-4.8.5`` +``/usr/local/payara5/bin/asadmin undeploy dataverse-5.0`` -Now that Glassfish doesn't have anything deployed, we can proceed with getting Netbeans set up to deploy the code. +Now that Payara doesn't have anything deployed, we can proceed with getting Netbeans set up to deploy the code. -Add Glassfish 4.1 as a Server in Netbeans -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Add Payara as a Server in Netbeans +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Dataverse only works with a specific version of Glassfish (see https://github.com/IQSS/dataverse/issues/2628 ) so you need to make sure Netbeans is deploying to that version rather than a newer version of Glassfish that may have come bundled with Netbeans. +Launch Netbeans and click "Tools" and then "Servers". Click "Add Server" and select "Payara Server" and set the installation location to ``/usr/local/payara5``. The defaults are fine so you can click "Next" and "Finish". -Launch Netbeans and click "Tools" and then "Servers". Click "Add Server" and select "Glassfish Server" and set the installation location to ``/usr/local/glassfish4``. The default are fine so you can click "Next" and "Finish". If you are running Netbeans 8.2 and there is already a bundled version of Glassfish, you should probably remove it because (again) you need a specific version of Glassfish (4.1 as of this writing). +Please note that if you are on a Mac, Netbeans may be unable to start Payara due to proxy settings in Netbeans. Go to the "General" tab in Netbeans preferences and click "Test connection" to see if you are affected. If you get a green checkmark, you're all set. If you get a red exclamation mark, change "Proxy Settings" to "No Proxy" and retest. A more complicated answer having to do with changing network settings is available at https://discussions.apple.com/thread/7680039?answerId=30715103022#30715103022 and the bug is also described at https://netbeans.org/bugzilla/show_bug.cgi?id=268076 -Please note that if you are on a Mac, Netbeans may be unable to start Glassfish due to proxy settings in Netbeans. Go to the "General" tab in Netbeans preferences and click "Test connection" to see if you are affected. If you get a green checkmark, you're all set. If you get a red exclamation mark, change "Proxy Settings" to "No Proxy" and retest. A more complicated answer having to do with changing network settings is available at https://discussions.apple.com/thread/7680039?answerId=30715103022#30715103022 and the bug is also described at https://netbeans.org/bugzilla/show_bug.cgi?id=268076 +At this point you can manage Payara using Netbeans. Click "Window" and then "Services". Expand "Servers" and right-click Payara to stop and then start it so that it appears in the Output window. Note that you can expand "Payara" and "Applications" to see if any applications are deployed. -At this point you can manage Glassfish using Netbeans. Click "Window" and then "Services". Expand "Servers" and right-click Glassfish to stop and then start it so that it appears in the Output window. Note that you can expand "Glassfish" and "Applications" to see if any applications are deployed. +Ensure that Dataverse Will Be Deployed to Payara +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Ensure that Dataverse Will Be Deployed to Glassfish 4.1 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Click "Window" and then "Projects". Click "File" and then "Project Properties (dataverse)". Click "Run" and change "Server" from "No Server Selected" to your installation of Glassfish 4.1. Click OK. +Click "Window" and then "Projects". Click "File" and then "Project Properties (dataverse)". Click "Run" and change "Server" from "No Server Selected" to your installation of Payara. Click OK. .. _custom_build_num_script: @@ -106,7 +104,7 @@ Deploying With ``asadmin`` Sometimes you want to deploy code without using Netbeans or from the command line on a server you have ssh'ed into. -For the ``asadmin`` commands below, we assume you have already changed directories to ``/usr/local/glassfish4/glassfish/bin`` or wherever you have installed Glassfish. +For the ``asadmin`` commands below, we assume you have already changed directories to ``/usr/local/payara5/glassfish/bin`` or wherever you have installed Payara. There are four steps to this process: @@ -116,15 +114,15 @@ There are four steps to this process: 4. Copy the war file to the server (if necessary) 5. Deploy the new code: ``./asadmin deploy /path/to/dataverse-VERSION.war`` -Running the Dataverse ``install`` Script in Non-Interactive Mode ----------------------------------------------------------------- +Running the Dataverse Installation Script in Non-Interactive Mode +----------------------------------------------------------------- Rather than running the installer in "interactive" mode, it's possible to put the values in a file. See "non-interactive mode" in the :doc:`/installation/installation-main` section of the Installation Guide. -Preventing Glassfish from Phoning Home --------------------------------------- +Preventing Payara from Phoning Home +----------------------------------- -By default, Glassfish reports analytics information. The administration guide suggests this can be disabled with ``./asadmin create-jvm-options -Dcom.sun.enterprise.tools.admingui.NO_NETWORK=true``, should this be found to be undesirable for development purposes. +By default, Glassfish reports analytics information. The administration guide suggests this can be disabled with ``./asadmin create-jvm-options -Dcom.sun.enterprise.tools.admingui.NO_NETWORK=true``, should this be found to be undesirable for development purposes. It is unknown if Payara phones home or not. Solr ---- @@ -172,6 +170,14 @@ You may want to populate your **non-production** installation(s) of Dataverse wi - Code in https://github.com/IQSS/dataverse-sample-data (recommended). This set of sample data includes several common data types, data subsetted from production datasets in dataverse.harvard.edu, datasets with file hierarchy, and more. - Scripts called from ``scripts/deploy/phoenix.dataverse.org/post``. +Switching from Glassfish to Payara +---------------------------------- + +If you already have a working dev environment with Glassfish and want to switch to Payara, you must do the following: + +- Copy the "domain1" directory from Glassfish to Payara. +- Copy the PostgreSQL driver into place like this: ``cp scripts/installer/pgdriver/postgresql-42.*.jar /usr/local/payara5/glassfish/lib`` + ---- Previous: :doc:`dev-environment` | Next: :doc:`troubleshooting` diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst index 236dee2e3c9..859c84e983d 100755 --- a/doc/sphinx-guides/source/developers/tools.rst +++ b/doc/sphinx-guides/source/developers/tools.rst @@ -30,13 +30,15 @@ With Maven installed you can run ``mvn package`` and ``mvn test`` from the comma Vagrant +++++++ +Please note that the Vagrant environment is currently broken. For details, see https://github.com/IQSS/dataverse/issues/6849 + Vagrant allows you to spin up a virtual machine running Dataverse on your development workstation. You'll need to install Vagrant from https://www.vagrantup.com and VirtualBox from https://www.virtualbox.org. We assume you have already cloned the repo from https://github.com/IQSS/dataverse as explained in the :doc:`/developers/dev-environment` section. From the root of the git repo (where the ``Vagrantfile`` is), run ``vagrant up`` and eventually you should be able to reach an installation of Dataverse at http://localhost:8888 (the ``forwarded_port`` indicated in the ``Vagrantfile``). -Please note that running ``vagrant up`` for the first time should run the ``downloads/download.sh`` script for you to download required software such as Glassfish and Solr and any patches. However, these dependencies change over time so it's a place to look if ``vagrant up`` was working but later fails. +Please note that running ``vagrant up`` for the first time should run the ``downloads/download.sh`` script for you to download required software such as an app server, Solr, etc. However, these dependencies change over time so it's a place to look if ``vagrant up`` was working but later fails. On Windows if you see an error like ``/usr/bin/perl^M: bad interpreter`` you might need to run ``dos2unix`` on the installation scripts. @@ -52,11 +54,11 @@ The Memory Analyzer Tool (MAT) from Eclipse can help you analyze heap dumps, sho It can be downloaded from http://www.eclipse.org/mat -If the heap dump provided to you was created with ``gcore`` (such as with ``gcore -o /tmp/gf.core $glassfish_pid``) rather than ``jmap``, you will need to convert the file before you can open it in MAT. Using ``gf.core.13849`` as example of the original 33 GB file, here is how you could convert it into a 26 GB ``gf.core.13849.hprof`` file. Please note that this operation took almost 90 minutes: +If the heap dump provided to you was created with ``gcore`` (such as with ``gcore -o /tmp/app.core $app_pid``) rather than ``jmap``, you will need to convert the file before you can open it in MAT. Using ``app.core.13849`` as example of the original 33 GB file, here is how you could convert it into a 26 GB ``app.core.13849.hprof`` file. Please note that this operation took almost 90 minutes: -``/usr/java7/bin/jmap -dump:format=b,file=gf.core.13849.hprof /usr/java7/bin/java gf.core.13849`` +``/usr/java7/bin/jmap -dump:format=b,file=app.core.13849.hprof /usr/java7/bin/java app.core.13849`` -A file of this size may not "just work" in MAT. When you attempt to open it you may see something like "An internal error occurred during: "Parsing heap dump from '/tmp/heapdumps/gf.core.13849.hprof'". Java heap space". If so, you will need to increase the memory allocated to MAT. On Mac OS X, this can be done by editing ``MemoryAnalyzer.app/Contents/MacOS/MemoryAnalyzer.ini`` and increasing the value "-Xmx1024m" until it's high enough to open the file. See also http://wiki.eclipse.org/index.php/MemoryAnalyzer/FAQ#Out_of_Memory_Error_while_Running_the_Memory_Analyzer +A file of this size may not "just work" in MAT. When you attempt to open it you may see something like "An internal error occurred during: "Parsing heap dump from '/tmp/heapdumps/app.core.13849.hprof'". Java heap space". If so, you will need to increase the memory allocated to MAT. On Mac OS X, this can be done by editing ``MemoryAnalyzer.app/Contents/MacOS/MemoryAnalyzer.ini`` and increasing the value "-Xmx1024m" until it's high enough to open the file. See also http://wiki.eclipse.org/index.php/MemoryAnalyzer/FAQ#Out_of_Memory_Error_while_Running_the_Memory_Analyzer PageKite ++++++++ @@ -71,7 +73,7 @@ Sign up at https://pagekite.net and follow the installation instructions or simp The first time you run ``./pagekite.py`` a file at ``~/.pagekite.rc`` will be created. You can edit this file to configure PageKite to serve up port 8080 -(the default GlassFish HTTP port) or the port of your choosing. +(the default app server HTTP port) or the port of your choosing. According to https://pagekite.net/support/free-for-foss/ PageKite (very generously!) offers free accounts to developers writing software the meets http://opensource.org/docs/definition.php such as Dataverse. @@ -131,7 +133,7 @@ Look for "RESOURCE_LEAK", for example. lsof ++++ -If file descriptors are not closed, eventually the open but unused resources can cause problems with system (glassfish in particular) stability. +If file descriptors are not closed, eventually the open but unused resources can cause problems with system (app servers in particular) stability. Static analysis and heap dumps are not always sufficient to identify the sources of these problems. For a quick sanity check, it can be helpful to check that the number of file descriptors does not increase after a request has finished processing. @@ -155,7 +157,7 @@ jmap allows you to look at the contents of the java heap. It can be used to crea .. code-block:: bash - $ jmap -histo + $ jmap -histo will output a list of all classes, sorted by the number of instances of each individual class, with the size in bytes. This can be very useful when looking for memory leaks in the application. Another useful tool is ``jstat``, that can be used in combination with ``jmap`` to monitor the effectiveness of garbage collection in reclaiming allocated memory. @@ -168,7 +170,7 @@ In the example script below we stress running Dataverse applicatione with GET re #!/bin/sh - # the script takes the numeric id of the glassfish process as the command line argument: + # the script takes the numeric id of the app server process as the command line argument: id=$1 while : diff --git a/doc/sphinx-guides/source/developers/troubleshooting.rst b/doc/sphinx-guides/source/developers/troubleshooting.rst index 2182c9768ad..63f00ce02f6 100755 --- a/doc/sphinx-guides/source/developers/troubleshooting.rst +++ b/doc/sphinx-guides/source/developers/troubleshooting.rst @@ -14,7 +14,7 @@ For unknown reasons, Netbeans will sometimes change the following line under ``s ``/`` -Sometimes Netbeans will change ``/`` to ``/dataverse``. Sometimes it will delete the line entirely. Either way, you will see very strange behavior when attempting to click around Dataverse in a browser. The homepage will load but icons will be missing. Any other page will fail to load entirely and you'll see a Glassfish error. +Sometimes Netbeans will change ``/`` to ``/dataverse``. Sometimes it will delete the line entirely. Either way, you will see very strange behavior when attempting to click around Dataverse in a browser. The homepage will load but icons will be missing. Any other page will fail to load entirely and you'll see an app server error. The solution is to put the file back to how it was before Netbeans touched it. If anyone knows of an open Netbeans bug about this, please let us know. @@ -35,9 +35,9 @@ You can check the current SMTP server with the ``asadmin`` command: ``./asadmin get server.resources.mail-resource.mail/notifyMailSession.host`` -This command helps verify what host your domain is using to send mail. Even if it's the correct hostname, you may still need to adjust settings. If all else fails, there are some free SMTP service options available such as Gmail and MailGun. This can be configured from the GlassFish console or the command line. +This command helps verify what host your domain is using to send mail. Even if it's the correct hostname, you may still need to adjust settings. If all else fails, there are some free SMTP service options available such as Gmail and MailGun. This can be configured from the Payara console or the command line. -1. First, navigate to your Glassfish admin console: http://localhost:4848 +1. First, navigate to your Payara admin console: http://localhost:4848 2. From the left-side panel, select **JavaMail Sessions** 3. You should see one session named **mail/notifyMailSession** -- click on that. @@ -67,7 +67,7 @@ mail.smtp.socketFactory.class javax.net.ssl.SSLSocketFactory **\*WARNING**: Entering a password here will *not* conceal it on-screen. It’s recommended to use an *app password* (for smtp.gmail.com users) or utilize a dedicated/non-personal user account with SMTP server auths so that you do not risk compromising your password. -Save these changes at the top of the page and restart your Glassfish server to try it out. +Save these changes at the top of the page and restart your app server to try it out. The mail session can also be set from command line. To use this method, you will need to delete your notifyMailSession and create a new one. See the below example: @@ -76,7 +76,7 @@ The mail session can also be set from command line. To use this method, you will These properties can be tailored to your own preferred mail service, but if all else fails these settings work fine with Dataverse development environments for your localhost. -+ If you're seeing a "Relay access denied" error in your Glassfish logs when your app attempts to send an email, double check your user/password credentials for the Mail Host you're using. ++ If you're seeing a "Relay access denied" error in your app server logs when Dataverse attempts to send an email, double check your user/password credentials for the Mail Host you're using. + If you're seeing a "Connection refused" / similar error upon email sending, try another port. As another example, here is how to create a Mail Host via command line for Amazon SES: diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 0386713a161..ec5d4cb971f 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -4,7 +4,7 @@ Windows Development Development on Windows is not well supported, unfortunately. You will have a much easier time if you develop on Mac or Linux as described under :doc:`dev-environment` section. -If you want to try using Windows for Dataverse development, your best best is to use Vagrant, as described below. Minishift is also an option. These instructions were tested on Windows 10. +Vagrant commands appear below and were tested on Windows 10 but the Vagrant environement is currently broken. Please see https://github.com/IQSS/dataverse/issues/6849 .. contents:: |toctitle| :local: @@ -71,64 +71,6 @@ The ``dataverse`` directory you changed is the one you just cloned. Vagrant will After a long while you hopefully will have Dataverse installed at http://localhost:8888 -Running Dataverse in Minishift ------------------------------- - -Minishift is a dev environment for OpenShift, which is Red Hat's distribution of Kubernetes. The :doc:`containers` section contains much more detail but the essential steps for using Minishift on Windows are described below. - -Install VirtualBox -~~~~~~~~~~~~~~~~~~ - -Download and install VirtualBox from https://www.virtualbox.org - -Install Git -~~~~~~~~~~~ - -Download and install Git from https://git-scm.com - -Install Minishift -~~~~~~~~~~~~~~~~~ - -Download Minishift from https://docs.openshift.org/latest/minishift/getting-started/installing.html . It should be a zip file. - -From Git Bash: - -``cd ~/Downloads`` - -``unzip minishift*.zip`` - -``mkdir ~/bin`` - -``cp minishift*/minishift.exe ~/bin`` - -Clone Git Repo -~~~~~~~~~~~~~~ - -From Git Bash, run the following commands: - -``git config --global core.autocrlf input`` - -``git clone https://github.com/IQSS/dataverse.git`` - -Start Minishift VM and Run Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``minishift start --vm-driver=virtualbox --memory=8GB`` - -``eval $(minishift oc-env)`` - -``oc new-project project1`` - -``cd ~/dataverse`` - -``oc new-app conf/openshift/openshift.json`` - -``minishift console`` - -This should open a web browser. In Microsoft Edge we saw ``INET_E_RESOURCE_NOT_FOUND`` so if you see that, try Chrome instead. A cert error is expected. Log in with the username "developer" and any password such as "asdf". - -Under "Overview" you should see a URL that has "dataverse-project1" in it. You should be able to click it and log into Dataverse with the username "dataverseAdmin" and the password "admin". - Improving Windows Support ------------------------- @@ -140,4 +82,4 @@ We have been unable to get Windows Subsystem for Linux (WSL) to work. We tried f Discussion and Feedback ~~~~~~~~~~~~~~~~~~~~~~~ -For more discussion of Windows support for Dataverse development see our community list thread `"Do you want to develop on Windows?" `_ We would be happy to inconrporate feedback from Windows developers into this page. The :doc:`documentation` section describes how. +For more discussion of Windows support for Dataverse development see our community list thread `"Do you want to develop on Windows?" `_ We would be happy to incorporate feedback from Windows developers into this page. The :doc:`documentation` section describes how. diff --git a/doc/sphinx-guides/source/installation/advanced.rst b/doc/sphinx-guides/source/installation/advanced.rst index a60d7dbc23f..9bcba1a9fac 100644 --- a/doc/sphinx-guides/source/installation/advanced.rst +++ b/doc/sphinx-guides/source/installation/advanced.rst @@ -7,22 +7,22 @@ Advanced installations are not officially supported but here we are at least doc .. contents:: |toctitle| :local: -Multiple Glassfish Servers --------------------------- +Multiple App Servers +-------------------- -You should be conscious of the following when running multiple Glassfish servers. +You should be conscious of the following when running multiple app servers. -- Only one Glassfish server can be the dedicated timer server, as explained in the :doc:`/admin/timers` section of the Admin Guide. -- When users upload a logo or footer for their dataverse using the "theme" feature described in the :doc:`/user/dataverse-management` section of the User Guide, these logos are stored only on the Glassfish server the user happend to be on when uploading the logo. By default these logos and footers are written to the directory ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/logos``. -- When a sitemp is created by a Glassfish server it is written to the filesystem of just that Glassfish server. By default the sitemap is written to the directory ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/sitemap``. -- If Make Data Count is used, its raw logs must be copied from each Glassfish server to single instance of Counter Processor. See also :ref:`:MDCLogPath` section in the Configuration section of this guide and the :doc:`/admin/make-data-count` section of the Admin Guide. -- Dataset draft version logging occurs separately on each Glassfish server. See :ref:`edit-draft-versions-logging` section in Monitoring of the Admin Guide for details. -- Password aliases (``db_password_alias``, etc.) are stored per Glassfish server. +- Only one app server can be the dedicated timer server, as explained in the :doc:`/admin/timers` section of the Admin Guide. +- When users upload a logo or footer for their dataverse using the "theme" feature described in the :doc:`/user/dataverse-management` section of the User Guide, these logos are stored only on the app server the user happend to be on when uploading the logo. By default these logos and footers are written to the directory ``/usr/local/payara5/glassfish/domains/domain1/docroot/logos``. +- When a sitemap is created by an app server it is written to the filesystem of just that app server. By default the sitemap is written to the directory ``/usr/local/payara5/glassfish/domains/domain1/docroot/sitemap``. +- If Make Data Count is used, its raw logs must be copied from each app server to single instance of Counter Processor. See also :ref:`:MDCLogPath` section in the Configuration section of this guide and the :doc:`/admin/make-data-count` section of the Admin Guide. +- Dataset draft version logging occurs separately on each app server. See :ref:`edit-draft-versions-logging` section in Monitoring of the Admin Guide for details. +- Password aliases (``db_password_alias``, etc.) are stored per app server. -Detecting Which Glassfish Server a User Is On -+++++++++++++++++++++++++++++++++++++++++++++ +Detecting Which App Server a User Is On ++++++++++++++++++++++++++++++++++++++++ -If you have successfully installed multiple Glassfish servers behind a load balancer you might like to know which server a user has landed on. A straightforward solution is to place a file called ``host.txt`` in a directory that is served up by Apache such as ``/var/www/html`` and then configure Apache not to proxy requests to ``/host.txt`` to Glassfish. Here are some example commands on RHEL/CentOS 7 that accomplish this:: +If you have successfully installed multiple app servers behind a load balancer you might like to know which server a user has landed on. A straightforward solution is to place a file called ``host.txt`` in a directory that is served up by Apache such as ``/var/www/html`` and then configure Apache not to proxy requests to ``/host.txt`` to the app server. Here are some example commands on RHEL/CentOS 7 that accomplish this:: [root@server1 ~]# vim /etc/httpd/conf.d/ssl.conf [root@server1 ~]# grep host.txt /etc/httpd/conf.d/ssl.conf @@ -32,6 +32,6 @@ If you have successfully installed multiple Glassfish servers behind a load bala [root@server1 ~]# curl https://dataverse.example.edu/host.txt server1.example.edu -You would repeat the steps above for all of your Glassfish servers. If users seem to be having a problem with a particular server, you can ask them to visit https://dataverse.example.edu/host.txt and let you know what they see there (e.g. "server1.example.edu") to help you know which server to troubleshoot. +You would repeat the steps above for all of your app servers. If users seem to be having a problem with a particular server, you can ask them to visit https://dataverse.example.edu/host.txt and let you know what they see there (e.g. "server1.example.edu") to help you know which server to troubleshoot. -Please note that :ref:`network-ports` under the Configuration section has more information on fronting Glassfish with Apache. The :doc:`shibboleth` section talks about the use of ``ProxyPassMatch``. +Please note that :ref:`network-ports` under the Configuration section has more information on fronting your app server with Apache. The :doc:`shibboleth` section talks about the use of ``ProxyPassMatch``. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 3fe67638fd0..fbd84a7453a 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -37,7 +37,7 @@ It's also possible to prevent file uploads via API by adjusting the :ref:`:Uploa Forcing HTTPS +++++++++++++ -To avoid having your users send credentials in the clear, it's strongly recommended to force all web traffic to go through HTTPS (port 443) rather than HTTP (port 80). The ease with which one can install a valid SSL cert into Apache compared with the same operation in Glassfish might be a compelling enough reason to front Glassfish with Apache. In addition, Apache can be configured to rewrite HTTP to HTTPS with rules such as those found at https://wiki.apache.org/httpd/RewriteHTTPToHTTPS or in the section on :doc:`shibboleth`. +To avoid having your users send credentials in the clear, it's strongly recommended to force all web traffic to go through HTTPS (port 443) rather than HTTP (port 80). The ease with which one can install a valid SSL cert into Apache compared with the same operation in Payara might be a compelling enough reason to front Payara with Apache. In addition, Apache can be configured to rewrite HTTP to HTTPS with rules such as those found at https://wiki.apache.org/httpd/RewriteHTTPToHTTPS or in the section on :doc:`shibboleth`. .. _PrivacyConsiderations: @@ -52,12 +52,12 @@ Out of the box, Dataverse will list email addresses of the contacts for datasets Additional Recommendations ++++++++++++++++++++++++++ -Run Glassfish as a User Other Than Root -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Run Payara as a User Other Than Root +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -See the :ref:`glassfish` section of :doc:`prerequisites` for details and init scripts for running Glassfish as non-root. +See the :ref:`payara` section of :doc:`prerequisites` for details and init scripts for running Payara as non-root. -Related to this is that you should remove ``/root/.glassfish/pass`` to ensure that Glassfish isn't ever accidentally started as root. Without the password, Glassfish won't be able to start as root, which is a good thing. +Related to this is that you should remove ``/root/.payara/pass`` to ensure that Payara isn't ever accidentally started as root. Without the password, Payara won't be able to start as root, which is a good thing. Enforce Strong Passwords for User Accounts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -84,27 +84,27 @@ Network Ports Remember how under "Decisions to Make" in the :doc:`prep` section we mentioned you'll need to make a decision about whether or not to introduce a proxy in front of Dataverse such as Apache or nginx? The time has come to make that decision. -The need to redirect port HTTP (port 80) to HTTPS (port 443) for security has already been mentioned above and the fact that Glassfish puts these services on 8080 and 8181, respectively, was touched on in the :doc:`installation-main` section. In production, you don't want to tell your users to use Dataverse on ports 8080 and 8181. You should have them use the standard HTTPS port, which is 443. +The need to redirect port HTTP (port 80) to HTTPS (port 443) for security has already been mentioned above and the fact that Payara puts these services on 8080 and 8181, respectively, was touched on in the :doc:`installation-main` section. In production, you don't want to tell your users to use Dataverse on ports 8080 and 8181. You should have them use the standard HTTPS port, which is 443. -Your decision to proxy or not should primarily be driven by which features of Dataverse you'd like to use. If you'd like to use Shibboleth, the decision is easy because proxying or "fronting" Glassfish with Apache is required. The details are covered in the :doc:`shibboleth` section. +Your decision to proxy or not should primarily be driven by which features of Dataverse you'd like to use. If you'd like to use Shibboleth, the decision is easy because proxying or "fronting" Payara with Apache is required. The details are covered in the :doc:`shibboleth` section. If you'd like to use TwoRavens, you should also consider fronting with Apache because you will be required to install an Apache anyway to make use of the rApache module. For details, see the :doc:`r-rapache-tworavens` section. -Even if you have no interest in Shibboleth nor TwoRavens, you may want to front Dataverse with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse team `_, but the process of adding a certificate to Glassfish is arduous and not for the faint of heart. The Dataverse team cannot provide much help with adding certificates to Glassfish beyond linking to `tips `_ on the web. +Even if you have no interest in Shibboleth nor TwoRavens, you may want to front Dataverse with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse team `_, but the process of adding a certificate to Payara is arduous and not for the faint of heart. The Dataverse team cannot provide much help with adding certificates to Payara beyond linking to `tips `_ on the web. -Still not convinced you should put Glassfish behind another web server? Even if you manage to get your SSL certificate into Glassfish, how are you going to run Glassfish on low ports such as 80 and 443? Are you going to run Glassfish as root? Bad idea. This is a security risk. Under "Additional Recommendations" under "Securing Your Installation" above you are advised to configure Glassfish to run as a user other than root. (The Dataverse team will close https://github.com/IQSS/dataverse/issues/1934 after updating the Glassfish init script provided in the :doc:`prerequisites` section to not require root.) +Still not convinced you should put Payara behind another web server? Even if you manage to get your SSL certificate into Payara, how are you going to run Payara on low ports such as 80 and 443? Are you going to run Payara as root? Bad idea. This is a security risk. Under "Additional Recommendations" under "Securing Your Installation" above you are advised to configure Payara to run as a user other than root. There's also the issue of serving a production-ready version of robots.txt. By using a proxy such as Apache, this is a one-time "set it and forget it" step as explained below in the "Going Live" section. -If you are convinced you'd like to try fronting Glassfish with Apache, the :doc:`shibboleth` section should be good resource for you. +If you are convinced you'd like to try fronting Payara with Apache, the :doc:`shibboleth` section should be good resource for you. -If you really don't want to front Glassfish with any proxy (not recommended), you can configure Glassfish to run HTTPS on port 443 like this: +If you really don't want to front Payara with any proxy (not recommended), you can configure Payara to run HTTPS on port 443 like this: ``./asadmin set server-config.network-config.network-listeners.network-listener.http-listener-2.port=443`` -What about port 80? Even if you don't front Dataverse with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Glassfish uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Glassfish can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: http://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to +What about port 80? Even if you don't front Dataverse with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Payara uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Payara can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: http://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to -If you are running an installation with Apache and Glassfish on the same server, and would like to restrict Glassfish from responding to any requests to port 8080 from external hosts (in other words, not through Apache), you can restrict the AJP listener to localhost only with: +If you are running an installation with Apache and Payara on the same server, and would like to restrict Payara from responding to any requests to port 8080 from external hosts (in other words, not through Apache), you can restrict the AJP listener to localhost only with: ``./asadmin set server-config.network-config.network-listeners.network-listener.http-listener-1.address=127.0.0.1`` @@ -129,7 +129,7 @@ By default, the installer configures a default DOI namespace (10.5072) with Data -Ddoi.username=... -Ddoi.password=... -and restart Glassfish. The prefix can be configured via the API (where it is referred to as "Authority"): +and restart Payara. The prefix can be configured via the API (where it is referred to as "Authority"): ``curl -X PUT -d 10.xxxx http://localhost:8080/api/admin/settings/:Authority`` @@ -219,7 +219,7 @@ As for the "Remote only" authentication mode, it means that: File Storage: Using a Local Filesystem and/or Swift and/or S3 object stores --------------------------------------------------------------------------- -By default, a Dataverse installation stores all data files (files uploaded by end users) on the filesystem at ``/usr/local/glassfish4/glassfish/domains/domain1/files``. This path can vary based on answers you gave to the installer (see the :ref:`dataverse-installer` section of the Installation Guide) or afterward by reconfiguring the ``dataverse.files.directory`` JVM option described below. +By default, a Dataverse installation stores all data files (files uploaded by end users) on the filesystem at ``/usr/local/payara5/glassfish/domains/domain1/files``. This path can vary based on answers you gave to the installer (see the :ref:`dataverse-installer` section of the Installation Guide) or afterward by reconfiguring the ``dataverse.files.directory`` JVM option described below. Dataverse can alternately store files in a Swift or S3-compatible object store, and can now be configured to support multiple stores at once. With a multi-store configuration, the location for new files can be controlled on a per-dataverse basis. @@ -356,7 +356,7 @@ First: Set Up Accounts and Access Credentials ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Dataverse and the AWS SDK make use of the "AWS credentials profile file" and "AWS config profile file" located in -``~/.aws/`` where ``~`` is the home directory of the user you run Glassfish as. This file can be generated via either +``~/.aws/`` where ``~`` is the home directory of the user you run Payara as. This file can be generated via either of two methods described below: 1. Manually through creation of the credentials and config files or @@ -442,7 +442,7 @@ Additional profiles can be added to these files by appending the relevant inform aws_access_key_id = aws_secret_access_key = -Place these two files in a folder named ``.aws`` under the home directory for the user running your Dataverse Glassfish +Place these two files in a folder named ``.aws`` under the home directory for the user running your Dataverse Payara instance. (From the `AWS Command Line Interface Documentation `_: "In order to separate credentials from less sensitive options, region and output format are stored in a separate file named config in the same folder") @@ -480,7 +480,7 @@ Then, we'll need to identify which S3 bucket we're using. Replace ``.bucket-name="`` -Optionally, you can have users download files from S3 directly rather than having files pass from S3 through Glassfish to your users. To accomplish this, set ``dataverse.files..download-redirect`` to ``true`` like this: +Optionally, you can have users download files from S3 directly rather than having files pass from S3 through Payara to your users. To accomplish this, set ``dataverse.files..download-redirect`` to ``true`` like this: ``./asadmin create-jvm-options "-Ddataverse.files..download-redirect=true"`` @@ -497,7 +497,7 @@ In case you would like to configure Dataverse to use a custom S3 service instead add the options for the custom URL and region as documented below. Please read above if your desired combination has been tested already and what other options have been set for a successful integration. -Lastly, go ahead and restart your glassfish server. With Dataverse deployed and the site online, you should be able to upload datasets and data files and see the corresponding files in your S3 bucket. Within a bucket, the folder structure emulates that found in local file storage. +Lastly, go ahead and restart your Payara server. With Dataverse deployed and the site online, you should be able to upload datasets and data files and see the corresponding files in your S3 bucket. Within a bucket, the folder structure emulates that found in local file storage. S3 Storage Options ################## @@ -582,7 +582,7 @@ Custom Navbar Logo Dataverse allows you to replace the default Dataverse icon and name branding in the navbar with your own custom logo. Note that this logo is separate from the *root dataverse theme* logo. -The custom logo image file is expected to be small enough to fit comfortably in the navbar, no more than 50 pixels in height and 160 pixels in width. Create a ``navbar`` directory in your Glassfish ``logos`` directory and place your custom logo there. By Glassfish default, your logo image file will be located at ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/logos/navbar/logo.png``. +The custom logo image file is expected to be small enough to fit comfortably in the navbar, no more than 50 pixels in height and 160 pixels in width. Create a ``navbar`` directory in your Payara ``logos`` directory and place your custom logo there. By default, your logo image file will be located at ``/usr/local/payara5/glassfish/domains/domain1/docroot/logos/navbar/logo.png``. Once you have the location of your custom logo image file, run this curl command to add it to your settings: @@ -641,13 +641,13 @@ The presence of the :ref:`:Languages` database setting adds a dropdown in the he Configuring the "lang" Directory ++++++++++++++++++++++++++++++++ -Translations for Dataverse are stored in "properties" files in a directory on disk (e.g. ``/home/glassfish/langBundles``) that you specify with the :ref:`dataverse.lang.directory` ``dataverse.lang.directory`` JVM option, like this: +Translations for Dataverse are stored in "properties" files in a directory on disk (e.g. ``/home/dataverse/langBundles``) that you specify with the :ref:`dataverse.lang.directory` ``dataverse.lang.directory`` JVM option, like this: -``./asadmin create-jvm-options '-Ddataverse.lang.directory=/home/glassfish/langBundles'`` +``./asadmin create-jvm-options '-Ddataverse.lang.directory=/home/dataverse/langBundles'`` Go ahead and create the directory you specified. -``mkdir /home/glassfish/langBundles`` +``mkdir /home/dataverse/langBundles`` Creating a languages.zip File +++++++++++++++++++++++++++++ @@ -762,7 +762,7 @@ Duracloud Configuration Also note that while the current Chronopolis implementation generates the bag and submits it to the archive's DuraCloud interface, the step to make a 'snapshot' of the space containing the Bag (and verify it's successful submission) are actions a curator must take in the DuraCloud interface. -The minimal configuration to support an archiver integration involves adding a minimum of two Dataverse Keys and any required Glassfish jvm options. The example instructions here are specific to the DuraCloud Archiver\: +The minimal configuration to support an archiver integration involves adding a minimum of two Dataverse Keys and any required Payara jvm options. The example instructions here are specific to the DuraCloud Archiver\: \:ArchiverClassName - the fully qualified class to be used for archiving. For example: @@ -780,7 +780,7 @@ The DPN archiver defines three custom settings, one of which is required (the ot :DuraCloudPort and :DuraCloudContext are also defined if you are not using the defaults ("443" and "duracloud" respectively). (Note\: these settings are only in effect if they are listed in the \:ArchiverSettings. Otherwise, they will not be passed to the DuraCloud Archiver class.) -Archivers may require glassfish settings as well. For the Chronopolis archiver, the username and password associated with your organization's Chronopolis/DuraCloud account should be configured in Glassfish: +Archivers may require JVM options as well. For the Chronopolis archiver, the username and password associated with your organization's Chronopolis/DuraCloud account should be configured in Payara: ``./asadmin create-jvm-options '-Dduracloud.username=YOUR_USERNAME_HERE'`` @@ -866,16 +866,16 @@ You can of course modify your own robots.txt to suit your specific needs as nece (See the sample robots.txt file linked above for some comments on how to set up different "Allow" and "Disallow" rules for different crawler bots) -You have a couple of options for putting an updated robots.txt file into production. If you are fronting Glassfish with Apache as recommended above, you can place robots.txt in the root of the directory specified in your ``VirtualHost`` and to your Apache config a ``ProxyPassMatch`` line like the one below to prevent Glassfish from serving the version of robots.txt that is embedded in the Dataverse war file: +You have a couple of options for putting an updated robots.txt file into production. If you are fronting Payara with Apache as recommended above, you can place robots.txt in the root of the directory specified in your ``VirtualHost`` and to your Apache config a ``ProxyPassMatch`` line like the one below to prevent Payara from serving the version of robots.txt that is embedded in the Dataverse war file: .. code-block:: text - # don't let Glassfish serve its version of robots.txt + # don't let Payara serve its version of robots.txt ProxyPassMatch ^/robots.txt$ ! For more of an explanation of ``ProxyPassMatch`` see the :doc:`shibboleth` section. -If you are not fronting Glassfish with Apache you'll need to prevent Glassfish from serving the robots.txt file embedded in the war file by overwriting robots.txt after the war file has been deployed. The downside of this technique is that you will have to remember to overwrite robots.txt in the "exploded" war file each time you deploy the war file, which probably means each time you upgrade to a new version of Dataverse. Furthermore, since the version of Dataverse is always incrementing and the version can be part of the file path, you will need to be conscious of where on disk you need to replace the file. For example, for Dataverse 4.6.1 the path to robots.txt may be ``/usr/local/glassfish4/glassfish/domains/domain1/applications/dataverse-4.6.1/robots.txt`` with the version number ``4.6.1`` as part of the path. +If you are not fronting Payara with Apache you'll need to prevent Payara from serving the robots.txt file embedded in the war file by overwriting robots.txt after the war file has been deployed. The downside of this technique is that you will have to remember to overwrite robots.txt in the "exploded" war file each time you deploy the war file, which probably means each time you upgrade to a new version of Dataverse. Furthermore, since the version of Dataverse is always incrementing and the version can be part of the file path, you will need to be conscious of where on disk you need to replace the file. For example, for Dataverse 4.6.1 the path to robots.txt may be ``/usr/local/payara5/glassfish/domains/domain1/applications/dataverse-4.6.1/robots.txt`` with the version number ``4.6.1`` as part of the path. Creating a Sitemap and Submitting it to Search Engines ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -886,11 +886,11 @@ Create or update your sitemap by adding the following curl command to cron to ru ``curl -X POST http://localhost:8080/api/admin/sitemap`` -This will create or update a file in the following location unless you have customized your installation directory for Glassfish: +This will create or update a file in the following location unless you have customized your installation directory for Payara: -``/usr/local/glassfish4/glassfish/domains/domain1/docroot/sitemap/sitemap.xml`` +``/usr/local/payara5/glassfish/domains/domain1/docroot/sitemap/sitemap.xml`` -On an installation of Dataverse with many datasets, the creation or updating of the sitemap can take a while. You can check Glassfish's server.log file for "BEGIN updateSiteMap" and "END updateSiteMap" lines to know when the process started and stopped and any errors in between. +On an installation of Dataverse with many datasets, the creation or updating of the sitemap can take a while. You can check Payara's server.log file for "BEGIN updateSiteMap" and "END updateSiteMap" lines to know when the process started and stopped and any errors in between. https://demo.dataverse.org/sitemap.xml is the sitemap URL for the Dataverse Demo site and yours should be similar. @@ -921,7 +921,7 @@ Before going live, you might want to consider setting up integrations to make it JVM Options ----------- -JVM stands for Java Virtual Machine and as a Java application, Glassfish can read JVM options when it is started. A number of JVM options are configured by the installer below is a complete list of the Dataverse-specific JVM options. You can inspect the configured options by running: +JVM stands for Java Virtual Machine and as a Java application, Payara can read JVM options when it is started. A number of JVM options are configured by the installer below is a complete list of the Dataverse-specific JVM options. You can inspect the configured options by running: ``./asadmin list-jvm-options | egrep 'dataverse|doi'`` @@ -931,7 +931,7 @@ When changing values these values with ``asadmin``, you'll need to delete the ol ``./asadmin create-jvm-options "-Ddataverse.fqdn=dataverse.example.com"`` -It's also possible to change these values by stopping Glassfish, editing ``glassfish4/glassfish/domains/domain1/config/domain.xml``, and restarting Glassfish. +It's also possible to change these values by stopping Payara, editing ``payara5/glassfish/domains/domain1/config/domain.xml``, and restarting Payara. dataverse.fqdn ++++++++++++++ @@ -1108,7 +1108,7 @@ If you want to use different index than the default 300 dataverse.timerServer +++++++++++++++++++++ -This JVM option is only relevant if you plan to run multiple Glassfish servers for redundancy. Only one Glassfish server can act as the dedicated timer server and for details on promoting or demoting a Glassfish server to handle this responsibility, see :doc:`/admin/timers`. +This JVM option is only relevant if you plan to run multiple Payara servers for redundancy. Only one Payara server can act as the dedicated timer server and for details on promoting or demoting a Payara server to handle this responsibility, see :doc:`/admin/timers`. .. _dataverse.lang.directory: @@ -1564,7 +1564,7 @@ Limit the number of files in a zip that Dataverse will accept. :SolrHostColonPort ++++++++++++++++++ -By default Dataverse will attempt to connect to Solr on port 8983 on localhost. Use this setting to change the hostname or port. You must restart Glassfish after making this change. +By default Dataverse will attempt to connect to Solr on port 8983 on localhost. Use this setting to change the hostname or port. You must restart Payara after making this change. ``curl -X PUT -d localhost:8983 http://localhost:8080/api/admin/settings/:SolrHostColonPort`` @@ -1977,7 +1977,7 @@ Sets how long a cached metrics result is used before re-running the query for a Sets the path where the raw Make Data Count logs are stored before being processed. If not set, no logs will be created for Make Data Count. See also the :doc:`/admin/make-data-count` section of the Admin Guide. -``curl -X PUT -d '/usr/local/glassfish4/glassfish/domains/domain1/logs' http://localhost:8080/api/admin/settings/:MDCLogPath`` +``curl -X PUT -d '/usr/local/payara5/glassfish/domains/domain1/logs' http://localhost:8080/api/admin/settings/:MDCLogPath`` .. _:DisplayMDCMetrics: diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index 3853369bbf3..36f42fbec05 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -12,11 +12,11 @@ Now that the :doc:`prerequisites` are in place, we are ready to execute the Data Running the Dataverse Installer ------------------------------- -A scripted, interactive installer is provided. This script will configure your Glassfish environment, create the database, set some required options and start the application. Some configuration tasks will still be required after you run the installer! So make sure to consult the next section. +A scripted, interactive installer is provided. This script will configure your app server environment, create the database, set some required options and start the application. Some configuration tasks will still be required after you run the installer! So make sure to consult the next section. As mentioned in the :doc:`prerequisites` section, RHEL/CentOS is the recommended Linux distribution. (The installer is also known to work on Mac OS X for setting up a development environment.) -Generally, the installer has a better chance of succeeding if you run it against a freshly installed Glassfish node that still has all the default configuration settings. In any event, please make sure that it is still configured to accept http connections on port 8080 - because that's where the installer expects to find the application once it's deployed. +Generally, the installer has a better chance of succeeding if you run it against a freshly installed Payara node that still has all the default configuration settings. In any event, please make sure that it is still configured to accept http connections on port 8080 - because that's where the installer expects to find the application once it's deployed. You should have already downloaded the installer from https://github.com/IQSS/dataverse/releases when setting up and starting Solr under the :doc:`prerequisites` section. Again, it's a zip file with "dvinstall" in the name. @@ -24,31 +24,31 @@ Unpack the zip file - this will create the directory ``dvinstall``. **Important:** The installer will need to use the PostgreSQL command line utility ``psql`` in order to configure the database. If the executable is not in your system PATH, the installer will try to locate it on your system. However, we strongly recommend that you check and make sure it is in the PATH. This is especially important if you have multiple versions of PostgreSQL installed on your system. Make sure the psql that came with the version that you want to use with your Dataverse is the first on your path. For example, if the PostgreSQL distribution you are running is installed in /Library/PostgreSQL/9.6, add /Library/PostgreSQL/9.6/bin to the beginning of your $PATH variable. If you are *running* multiple PostgreSQL servers, make sure you know the port number of the one you want to use, as the installer will need it in order to connect to the database (the first PostgreSQL distribution installed on your system is likely using the default port 5432; but the second will likely be on 5433, etc.) Does every word in this paragraph make sense? If it does, great - because you definitely need to be comfortable with basic system tasks in order to install Dataverse. If not - if you don't know how to check where your PostgreSQL is installed, or what port it is running on, or what a $PATH is... it's not too late to stop. Because it will most likely not work. And if you contact us for help, these will be the questions we'll be asking you - so, again, you need to be able to answer them comfortably for it to work. -Execute the installer script like this (but first read the note below about not running the installer as root):: - - $ cd dvinstall - $ ./install - - **It is no longer necessary to run the installer as root!** Just make sure the user running the installer has write permission to: -- /usr/local/glassfish4/glassfish/lib -- /usr/local/glassfish4/glassfish/domains/domain1 +- /usr/local/payara5/glassfish/lib +- /usr/local/payara5/glassfish/domains/domain1 - the current working directory of the installer (it currently writes its logfile there), and - your jvm-option specified files.dir -**NEW in v.4.19:** We have added a new implementation of the installer script written in Python. It is intended to eventually replace the old installer above (written in Perl). But for now it is being offered as an (experimental) alternative. See README_python.txt, included in the installer bundle, for more information on how to run it. +The only reason to run Payara as root would be to allow Payara itself to listen on the default HTTP(S) ports 80 and 443, or any other port below 1024. However, it is simpler and more secure to run Payara run on its default port of 8080 and hide it behind an Apache Proxy, via AJP, running on port 80 or 443. This configuration is required if you're going to use Shibboleth authentication. See more discussion on this here: :doc:`shibboleth`.) +Read the installer script directions like this:: -The only reason to run Glassfish as root would be to allow Glassfish itself to listen on the default HTTP(S) ports 80 and 443, or any other port below 1024. However, it is simpler and more secure to run Glassfish run on its default port of 8080 and hide it behind an Apache Proxy, via AJP, running on port 80 or 443. This configuration is required if you're going to use Shibboleth authentication. See more discussion on this here: :doc:`shibboleth`.) + $ cd dvinstall + $ less README_python.txt + +Alternatively you can download :download:`README_python.txt <../../../../scripts/installer/README_python.txt>` from this guides. + +Follow the instructions in the text file. The script will prompt you for some configuration values. If this is a test/evaluation installation, it may be possible to accept the default values provided for most of the settings: - Internet Address of your host: localhost -- Glassfish Directory: /usr/local/glassfish4 -- Glassfish User: current user running the installer script +- Payara Directory: /usr/local/payara5 +- Payara User: current user running the installer script - Administrator email address for this Dataverse: (none) - SMTP (mail) server to relay notification messages: localhost - Postgres Server Address: [127.0.0.1] @@ -70,15 +70,17 @@ If desired, these default values can be configured by creating a ``default.confi This allows the installer to be run in non-interactive mode (with ``./install -y -f > install.out 2> install.err``), which can allow for easier interaction with automated provisioning tools. -All the Glassfish configuration tasks performed by the installer are isolated in the shell script ``dvinstall/glassfish-setup.sh`` (as ``asadmin`` commands). +All the Payara configuration tasks performed by the installer are isolated in the shell script ``dvinstall/as-setup.sh`` (as ``asadmin`` commands). -**IMPORTANT:** As a security measure, the ``glassfish-setup.sh`` script stores passwords as "aliases" rather than plaintext. If you change your database password, for example, you will need to update the alias with ``asadmin update-password-alias db_password_alias``, for example. Here is a list of the password aliases that are set by the installation process and entered into Glassfish's ``domain.xml`` file: +The installer patches the app server, copying ``jakarta.faces_dv.jar`` into ``/usr/local/payara5/glassfish/modules``. For details on this patch, see https://github.com/eclipse-ee4j/mojarra/issues/4647 + +**IMPORTANT:** As a security measure, the ``as-setup.sh`` script stores passwords as "aliases" rather than plaintext. If you change your database password, for example, you will need to update the alias with ``asadmin update-password-alias db_password_alias``, for example. Here is a list of the password aliases that are set by the installation process and entered into Payara's ``domain.xml`` file: - ``db_password_alias`` - ``doi_password_alias`` - ``rserve_password_alias`` -Glassfish does not provide up to date documentation but Payara (a fork of Glassfish) does so for more information, please see https://docs.payara.fish/documentation/payara-server/password-aliases/password-alias-asadmin-commands.html +For more information, please see https://docs.payara.fish/documentation/payara-server/password-aliases/password-alias-asadmin-commands.html **IMPORTANT:** The installer will also ask for an external site URL for Dataverse. It is *imperative* that this value be supplied accurately, or a long list of functions will be inoperable, including: @@ -95,16 +97,14 @@ The supplied site URL will be saved under the JVM option :ref:`dataverse.siteUrl **IMPORTANT:** Please note, that "out of the box" the installer will configure the Dataverse to leave unrestricted access to the administration APIs from (and only from) localhost. Please consider the security implications of this arrangement (anyone with shell access to the server can potentially mess with your Dataverse). An alternative solution would be to block open access to these sensitive API endpoints completely; and to only allow requests supplying a pre-defined "unblock token" (password). If you prefer that as a solution, please consult the supplied script ``post-install-api-block.sh`` for examples on how to set it up. See also "Securing Your Installation" under the :doc:`config` section. -Dataverse uses JHOVE_ to help identify the file format (CSV, PNG, etc.) for files that users have uploaded. The installer places files called ``jhove.conf`` and ``jhoveConfig.xsd`` into the directory ``/usr/local/glassfish4/glassfish/domains/domain1/config`` by default and makes adjustments to the jhove.conf file based on the directory into which you chose to install Glassfish. +Dataverse uses JHOVE_ to help identify the file format (CSV, PNG, etc.) for files that users have uploaded. The installer places files called ``jhove.conf`` and ``jhoveConfig.xsd`` into the directory ``/usr/local/payara5/glassfish/domains/domain1/config`` by default and makes adjustments to the jhove.conf file based on the directory into which you chose to install Payara. .. _JHOVE: http://jhove.openpreservation.org -The script is to a large degree a derivative of the old installer from DVN 3.x. It is written in Perl. If someone in the community is eager to rewrite it, perhaps in a different language, please get in touch. :) - Logging In ---------- -Out of the box, Glassfish runs on port 8080 and 8181 rather than 80 and 443, respectively, so visiting http://localhost:8080 (substituting your hostname) should bring up a login page. See the :doc:`shibboleth` page for more on ports, but for now, let's confirm we can log in by using port 8080. Poke a temporary hole in your firewall, if needed. +Out of the box, Payara runs on port 8080 and 8181 rather than 80 and 443, respectively, so visiting http://localhost:8080 (substituting your hostname) should bring up a login page. See the :doc:`shibboleth` page for more on ports, but for now, let's confirm we can log in by using port 8080. Poke a temporary hole in your firewall, if needed. Superuser Account ^^^^^^^^^^^^^^^^^ @@ -128,7 +128,7 @@ Next you'll want to check out the :doc:`config` section, especially the section Troubleshooting --------------- -If the following doesn't apply, please get in touch as explained in the :doc:`intro`. You may be asked to provide ``glassfish4/glassfish/domains/domain1/logs/server.log`` for debugging. +If the following doesn't apply, please get in touch as explained in the :doc:`intro`. You may be asked to provide ``payara5/glassfish/domains/domain1/logs/server.log`` for debugging. Dataset Cannot Be Published ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -138,22 +138,22 @@ Check to make sure you used a fully qualified domain name when installing Datave Problems Sending Email ^^^^^^^^^^^^^^^^^^^^^^ -If your Dataverse installation is not sending system emails, you may need to provide authentication for your mail host. First, double check the SMTP server being used with this Glassfish asadmin command: +If your Dataverse installation is not sending system emails, you may need to provide authentication for your mail host. First, double check the SMTP server being used with this Payara asadmin command: ``./asadmin get server.resources.mail-resource.mail/notifyMailSession.host`` This should return the DNS of the mail host you configured during or after installation. mail/notifyMailSession is the JavaMail Session that's used to send emails to users. -If the command returns a host you don't want to use, you can modify your notifyMailSession with the Glassfish ``asadmin set`` command with necessary options (`click here for the manual page `_), or via the admin console at http://localhost:4848 with your domain running. +If the command returns a host you don't want to use, you can modify your notifyMailSession with the Payara ``asadmin set`` command with necessary options (`click here for the manual page `_), or via the admin console at http://localhost:4848 with your domain running. If your mail host requires a username/password for access, continue to the next section. Mail Host Configuration & Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you need to alter your mail host address, user, or provide a password to connect with, these settings are easily changed in the Glassfish admin console or via command line. +If you need to alter your mail host address, user, or provide a password to connect with, these settings are easily changed in the Payara admin console or via command line. -For the Glassfish console, load a browser with your domain online, navigate to http://localhost:4848 and on the side panel find JavaMail Sessions. By default, Dataverse uses a session named mail/notifyMailSession for routing outgoing emails. Click this mail session in the window to modify it. +For the Payara console, load a browser with your domain online, navigate to http://localhost:4848 and on the side panel find JavaMail Sessions. By default, Dataverse uses a session named mail/notifyMailSession for routing outgoing emails. Click this mail session in the window to modify it. When fine tuning your JavaMail Session, there are a number of fields you can edit. The most important are: @@ -193,7 +193,7 @@ The mail session can also be set from command line. To use this method, you will - Delete: ``./asadmin delete-javamail-resource mail/notifyMailSession`` - Create (remove brackets and replace the variables inside): ``./asadmin create-javamail-resource --mailhost [smtp.gmail.com] --mailuser [test\@test\.com] --fromaddress [test\@test\.com] --property mail.smtp.auth=[true]:mail.smtp.password=[password]:mail.smtp.port=[465]:mail.smtp.socketFactory.port=[465]:mail.smtp.socketFactory.fallback=[false]:mail.smtp.socketFactory.class=[javax.net.ssl.SSLSocketFactory] mail/notifyMailSession`` -Be sure you save the changes made here and then restart your Glassfish server to test it out. +Be sure you save the changes made here and then restart your Payara server to test it out. UnknownHostException While Deploying ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -205,17 +205,17 @@ If you are seeing "Caused by: java.net.UnknownHostException: myhost: Name or ser Fresh Reinstall --------------- -Early on when you're installing Dataverse, you may think, "I just want to blow away what I've installed and start over." That's fine. You don't have to uninstall the various components like Glassfish, PostgreSQL and Solr, but you should be conscious of how to clear out their data. For Glassfish, a common helpful process is to: +Early on when you're installing Dataverse, you may think, "I just want to blow away what I've installed and start over." That's fine. You don't have to uninstall the various components like Payara, PostgreSQL and Solr, but you should be conscious of how to clear out their data. For Payara, a common helpful process is to: -- Stop Glassfish; +- Stop Payara; - Remove the ``generated`` and ``osgi-cache`` directories; - Delete all the rows from the ``EJB__TIMER__TBL`` table in the database; -- Start Glassfish +- Start Payara Drop database ^^^^^^^^^^^^^ -In order to drop the database, you have to stop Glassfish, which will have open connections. Before you stop Glassfish, you may as well undeploy the war file. First, find the name like this: +In order to drop the database, you have to stop Payara, which will have open connections. Before you stop Payara, you may as well undeploy the war file. First, find the name like this: ``./asadmin list-applications`` @@ -223,11 +223,11 @@ Then undeploy it like this: ``./asadmin undeploy dataverse-VERSION`` -Stop Glassfish with the init script provided in the :doc:`prerequisites` section or just use: +Stop Payara with the init script provided in the :doc:`prerequisites` section or just use: ``./asadmin stop-domain`` -With Glassfish down, you should now be able to drop your database and recreate it: +With Payara down, you should now be able to drop your database and recreate it: ``psql -U dvnapp -c 'DROP DATABASE "dvndb"' template1`` @@ -244,7 +244,7 @@ Deleting Uploaded Files The path below will depend on the value for ``dataverse.files.directory`` as described in the :doc:`config` section: -``rm -rf /usr/local/glassfish4/glassfish/domains/domain1/files`` +``rm -rf /usr/local/payara5/glassfish/domains/domain1/files`` Rerun Installer ^^^^^^^^^^^^^^^ diff --git a/doc/sphinx-guides/source/installation/oauth2.rst b/doc/sphinx-guides/source/installation/oauth2.rst index 300df5e20e0..4484ca72168 100644 --- a/doc/sphinx-guides/source/installation/oauth2.rst +++ b/doc/sphinx-guides/source/installation/oauth2.rst @@ -65,7 +65,7 @@ Edit the JSON template and replace the two "FIXME" values with the Client ID and ``curl -X POST -H 'Content-type: application/json' --upload-file github.json http://localhost:8080/api/admin/authenticationProviders`` -After restarting Glassfish you should see the new provider under "Other options" on the Log In page, as described in the :doc:`/user/account` section of the User Guide. +After restarting your app server you should see the new provider under "Other options" on the Log In page, as described in the :doc:`/user/account` section of the User Guide. By default, the Log In page will show the "builtin" provider, but you can adjust this via the ``:DefaultAuthProvider`` configuration option. For details, see :doc:`config`. @@ -77,8 +77,6 @@ This template can be used for configuring this setting (**this is not something - :download:`orcid-sandbox.json <../_static/installation/files/root/auth-providers/orcid-sandbox.json>` -Please note that the :doc:`prerequisites` section contains an step regarding CA certs in Glassfish that must be followed to get ORCID login to work. - Converting Local Users to OAuth ------------------------------- diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst index 6d206d2576b..768a5b66cd7 100644 --- a/doc/sphinx-guides/source/installation/oidc.rst +++ b/doc/sphinx-guides/source/installation/oidc.rst @@ -78,7 +78,7 @@ Now load the configuration into Dataverse using the same API as with :doc:`oauth ``curl -X POST -H 'Content-type: application/json' --upload-file my-oidc-provider.json http://localhost:8080/api/admin/authenticationProviders`` -Dataverse will automatically try to load the provider and retrieve the metadata. Watch the Glassfish log for errors. +Dataverse will automatically try to load the provider and retrieve the metadata. Watch the app server log for errors. You should see the new provider under "Other options" on the Log In page, as described in the :doc:`/user/account` section of the User Guide. diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index cd43fb6e436..f64a27c4d54 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -14,15 +14,8 @@ We'll try to get you up and running as quickly as possible, but we thought you m Choose Your Own Installation Adventure -------------------------------------- -Vagrant (for Testing Only) -++++++++++++++++++++++++++ - -If you are looking to simply kick the tires on installing Dataverse and are familiar with Vagrant, you are welcome to read through the :ref:`vagrant` section of the :doc:`/developers/tools` section of the Developer Guide. Checking out a tagged release is recommended rather than running ``vagrant up`` on unreleased code. - -Pilot Installation -++++++++++++++++++ - -Vagrant is not a bad way for a sysadmin to get a quick sense of how an application like Dataverse is put together in a sandbox (a virtual machine running on a laptop for example), but to allow end users to start playing with Dataverse, you'll need to install Dataverse on a server. +Standard Installation ++++++++++++++++++++++ Installing Dataverse involves some system configuration followed by executing an installation script that will guide you through the installation process as described in :doc:`installation-main`, but reading about the :ref:`architecture` of Dataverse is recommended first. @@ -40,7 +33,7 @@ There are some community-lead projects to use configuration management tools suc The Dataverse development team is happy to "bless" additional community efforts along these lines (i.e. Docker, Chef, Salt, etc.) by creating a repo under https://github.com/IQSS and managing team access. -Dataverse permits a fair amount of flexibility in where you choose to install the various components. The diagram below shows a load balancer, multiple proxies and web servers, redundant database servers, and offloading of potentially resource intensive work to a separate server. +Dataverse permits a fair amount of flexibility in where you choose to install the various components. The diagram below shows a load balancer, multiple proxies and web servers, redundant database servers, and offloading of potentially resource intensive work to a separate server. (Glassfish is shown rather than Payara.) |3webservers| @@ -51,7 +44,7 @@ A setup such as this is advanced enough to be considered out of scope for this g Architecture and Components --------------------------- -Dataverse is a Java Enterprise Edition (EE) web application that is shipped as a war (web archive) file. +Dataverse is a Java Enterprise Edition (EE) web application that is shipped as a WAR (Web ARchive) file. This WAR file is deployed to an application server or app server. Required Components +++++++++++++++++++ @@ -59,7 +52,7 @@ Required Components When planning your installation you should be aware of the following components of the Dataverse architecture: - Linux: RHEL/CentOS is highly recommended since all development and QA happens on this distribution. -- Glassfish: a Java EE application server to which the Dataverse application (war file) is to be deployed. +- App server: Payara is the recommended Jakarta EE application server - PostgreSQL: a relational database. - Solr: a search engine. A Dataverse-specific schema is provided. - SMTP server: for sending mail for password resets and other notifications. @@ -73,7 +66,7 @@ There are a number of optional components you may choose to install or configure - External Tools: Third party tools for data exploration can be added to Dataverse by following the instructions in the :doc:`/admin/external-tools` section of the Admin Guide. - R, rApache, Zelig, and TwoRavens: :doc:`/user/data-exploration/tworavens` describes the feature and :doc:`r-rapache-tworavens` describes how to install these components. :doc:`/admin/external-tools` explains how third-party tools like TwoRavens can be added to Dataverse. - Dropbox integration :ref:`dataverse.dropbox.key`: for uploading files from the Dropbox API. -- Apache: a web server that can "reverse proxy" Glassfish applications and rewrite HTTP traffic. +- Apache: a web server that can "reverse proxy" Jakarta EE applications (like Dataverse) and rewrite HTTP traffic. - Shibboleth: an authentication system described in :doc:`shibboleth`. Its use with Dataverse requires Apache. - OAuth2: an authentication system described in :doc:`oauth2`. - Geoconnect: a system that allows users to create maps from geospatial files, described in :doc:`geoconnect`. @@ -88,11 +81,11 @@ Hardware Requirements A basic installation of Dataverse runs fine on modest hardware. For example, as of this writing the test installation at http://phoenix.dataverse.org is backed by a single virtual machine with two 2.8 GHz processors, 8 GB of RAM and 50 GB of disk. -In contrast, before we moved it to the Amazon Cloud, the production installation at https://dataverse.harvard.edu was backed by six servers with two Intel Xeon 2.53 Ghz CPUs and either 48 or 64 GB of RAM. The three servers with 48 GB of RAM run were web frontends running Glassfish and Apache and were load balanced by a hardware device. The remaining three servers with 64 GB of RAM were the primary and backup database servers and a server dedicated to running Rserve. Multiple TB of storage were mounted from a SAN via NFS. +In contrast, before we moved it to the Amazon Cloud, the production installation at https://dataverse.harvard.edu was backed by six servers with two Intel Xeon 2.53 Ghz CPUs and either 48 or 64 GB of RAM. The three servers with 48 GB of RAM run were web frontends running Glassfish 4 and Apache and were load balanced by a hardware device. The remaining three servers with 64 GB of RAM were the primary and backup database servers and a server dedicated to running Rserve. Multiple TB of storage were mounted from a SAN via NFS. Currently, Harvard Dataverse is served by four AWS server nodes: two "m4.4xlarge" instances (64GB/16 vCPU) as web frontends, one 32GB/8 vCPU ("m4.2xlarge") instance for the Solr search engine, and one 16GB/4 vCPU ("m4.xlarge") instance for R and TwoRavens. The PostgreSQL database is served by Amazon RDS, and physical files are stored on Amazon S3. -The Dataverse installation script will attempt to give Glassfish the right amount of RAM based on your system. +The Dataverse installation script will attempt to give your app server the right amount of RAM based on your system. Experimentation and testing with various hardware configurations is encouraged, or course, but do reach out as explained in the :doc:`intro` as needed for assistance. @@ -111,7 +104,7 @@ Here are some questions to keep in the back of your mind as you test and move in - How much storage do I need? - Which features do I want based on :ref:`architecture`? - How do I want my users to log in to Dataverse? With local accounts? With Shibboleth/SAML? With OAuth providers such as ORCID, GitHub, or Google? -- Do I want to to run Glassfish on the standard web ports (80 and 443) or do I want to "front" Glassfish with a proxy such as Apache or nginx? See "Network Ports" in the :doc:`config` section. +- Do I want to to run my app server on the standard web ports (80 and 443) or do I want to "front" my app server with a proxy such as Apache or nginx? See "Network Ports" in the :doc:`config` section. - How many points of failure am I willing to tolerate? How much complexity do I want? - How much does it cost to subscribe to a service to create persistent identifiers such as DOIs or handles? diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index bfa8e11e6fc..c01b9b2a9d3 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -4,12 +4,10 @@ Prerequisites ============= -Before running the Dataverse installation script, you must install and configure Linux, Java, Glassfish, PostgreSQL, Solr, and jq. The other software listed below is optional but can provide useful features. +Before running the Dataverse installation script, you must install and configure Linux, Java, Payara, PostgreSQL, Solr, and jq. The other software listed below is optional but can provide useful features. After following all the steps below, you can proceed to the :doc:`installation-main` section. -You **may** find it helpful to look at how the configuration is done automatically by various tools such as Vagrant, Puppet, or Ansible. See the :doc:`prep` section for pointers on diving into these scripts. - .. contents:: |toctitle| :local: @@ -44,79 +42,54 @@ If you don't want to be prompted, here is an example of the non-interactive invo # alternatives --set java /usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin/java -.. _glassfish: +.. _payara: -Glassfish ---------- +Payara +------ -Glassfish Version 4.1 is required. There are known issues with newer versions of the Glassfish 4.x series so it should be avoided. For details, see https://github.com/IQSS/dataverse/issues/2628 . The issue we are using the track support for Glassfish 5 is https://github.com/IQSS/dataverse/issues/4248 . +Payara 5.201 is recommended. Newer versions might work fine. -Installing Glassfish -==================== +Installing Payara +================= -**Note:** The Dataverse installer need not be run as root, and it is recommended that Glassfish not run as root either. We suggest the creation of a glassfish service account for this purpose. +**Note:** The Dataverse installer need not be run as root, and it is recommended that Payara not run as root either. We suggest the creation of a "dataverse" service account for this purpose. -- Download and install Glassfish (installed in ``/usr/local/glassfish4`` in the example commands below):: +- Download and install Payara (installed in ``/usr/local/payara5`` in the example commands below):: - # wget http://dlc-cdn.sun.com/glassfish/4.1/release/glassfish-4.1.zip - # unzip glassfish-4.1.zip - # mv glassfish4 /usr/local + # wget --content-disposition https://search.maven.org/remotecontent?filepath=fish/payara/distributions/payara/5.201/payara-5.201.zip + # unzip payara-5.201.zip + # mv payara5 /usr/local -If you intend to install and run Glassfish under a service account (and we hope you do), chown -R the Glassfish hierarchy to root to protect it but give the service account access to the below directories: +If you intend to install and run Payara under a service account (and we hope you do), chown -R the Payara hierarchy to root to protect it but give the service account access to the below directories: - Set service account permissions:: - # chown -R root:root /usr/local/glassfish4 - # chown glassfish /usr/local/glassfish4/glassfish/lib - # chown -R glassfish:glassfish /usr/local/glassfish4/glassfish/domains/domain1 + # chown -R root:root /usr/local/payara5 + # chown dataverse /usr/local/payara5/glassfish/lib + # chown -R dataverse:dataverse /usr/local/payara5/glassfish/domains/domain1 After installation, you may chown the lib/ directory back to root; the installer only needs write access to copy the JDBC driver into that directory. -Once Glassfish is installed, you'll need a newer version of the Weld library (v2.2.10.SP1) to fix a serious issue in the library supplied with Glassfish 4.1 (see https://github.com/IQSS/dataverse/issues/647 for details). If you plan to front Glassfish with Apache you must also patch Grizzly as explained in the :doc:`shibboleth` section. - -- Remove the stock Weld jar; download Weld v2.2.10.SP1 and install it in the modules folder:: - - # cd /usr/local/glassfish4/glassfish/modules - # rm weld-osgi-bundle.jar - # curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar - - Change from ``-client`` to ``-server`` under ``-client``:: - # vim /usr/local/glassfish4/glassfish/domains/domain1/config/domain.xml + # vim /usr/local/payara5/glassfish/domains/domain1/config/domain.xml This recommendation comes from http://www.c2b2.co.uk/middleware-blog/glassfish-4-performance-tuning-monitoring-and-troubleshooting.php among other places. -- Start Glassfish and verify the Weld version:: - - # /usr/local/glassfish4/bin/asadmin start-domain - # /usr/local/glassfish4/bin/asadmin osgi lb | grep 'Weld OSGi Bundle' - -The Certificate Authority (CA) certificate bundle file from Glassfish contains certs that expired in August 2018, causing problems with ORCID login. - -- The actual expiration date is August 22, 2018, which you can see with the following command:: - - # keytool -list -v -keystore /usr/local/glassfish4/glassfish/domains/domain1/config/cacerts.jks - -- Overwrite Glassfish's CA certs file with the file that ships with the operating system and restart Glassfish:: - - # cp /etc/pki/ca-trust/extracted/java/cacerts /usr/local/glassfish4/glassfish/domains/domain1/config/cacerts.jks - # /usr/local/glassfish4/bin/asadmin stop-domain - # /usr/local/glassfish4/bin/asadmin start-domain - -Launching Glassfish on system boot -================================== +Launching Payara on System Boot +=============================== -The Dataverse installation script will start Glassfish if necessary, but you may find the following scripts helpful to launch Glassfish start automatically on boot. +The Dataverse installation script will start Payara if necessary, but you may find the following scripts helpful to launch Payara start automatically on boot. They were originally written for Glassfish but have been adjusted for Payara. -- This :download:`Systemd file<../_static/installation/files/etc/systemd/glassfish.service>` may be serve as a reference for systems using Systemd (such as RHEL/CentOS 7 or Ubuntu 16+) -- This :download:`init script<../_static/installation/files/etc/init.d/glassfish.init.service>` may be useful for RHEL/CentOS 6 or Ubuntu >= 14 if you're using a Glassfish service account, or -- This :download:`Glassfish init script <../_static/installation/files/etc/init.d/glassfish.init.root>` may be helpful if you're just going to run Glassfish as root. +- This :download:`Systemd file<../_static/installation/files/etc/systemd/payara.service>` may be serve as a reference for systems using Systemd (such as RHEL/CentOS 7 or Ubuntu 16+) +- This :download:`init script<../_static/installation/files/etc/init.d/payara.init.service>` may be useful for RHEL/CentOS 6 or Ubuntu >= 14 if you're using a Payara service account, or +- This :download:`Payara init script <../_static/installation/files/etc/init.d/payara.init.root>` may be helpful if you're just going to run Payara as root (not recommended). -It is not necessary for Glassfish to be running before you execute the Dataverse installation script; it will start Glassfish for you. +It is not necessary for Payara to be running before you execute the Dataverse installation script; it will start Payara for you. -Please note that you must run Glassfish in an English locale. If you are using something like ``LANG=de_DE.UTF-8``, ingest of tabular data will fail with the message "RoundRoutines:decimal separator no in right place". +Please note that you must run Payara in an English locale. If you are using something like ``LANG=de_DE.UTF-8``, ingest of tabular data will fail with the message "RoundRoutines:decimal separator no in right place". -Also note that Glassfish may utilize more than the default number of file descriptors, especially when running batch jobs such as harvesting. We have increased ours by adding ulimit -n 32768 to our glassfish init script. On operating systems which use systemd such as RHEL or CentOS 7, file descriptor limits may be increased by adding a line like LimitNOFILE=32768 to the systemd unit file. You may adjust the file descriptor limits on running processes by using the prlimit utility:: +Also note that Payara may utilize more than the default number of file descriptors, especially when running batch jobs such as harvesting. We have increased ours by adding ulimit -n 32768 to our Payara init script. On operating systems which use systemd such as RHEL or CentOS 7, file descriptor limits may be increased by adding a line like LimitNOFILE=32768 to the systemd unit file. You may adjust the file descriptor limits on running processes by using the prlimit utility:: # sudo prlimit --pid pid --nofile=32768:32768 @@ -149,7 +122,7 @@ Configuring Database Access for the Dataverse Application (and the Dataverse Ins - The application and the installer script will be connecting to PostgreSQL over TCP/IP, using password authentication. In this section we explain how to configure PostgreSQL to accept these connections. -- If PostgreSQL is running on the same server as Glassfish, find the localhost (127.0.0.1) entry that's already in the ``pg_hba.conf`` and modify it to look like this:: +- If PostgreSQL is running on the same server as Payara, find the localhost (127.0.0.1) entry that's already in the ``pg_hba.conf`` and modify it to look like this:: host all all 127.0.0.1/32 md5 @@ -167,7 +140,7 @@ Configuring Database Access for the Dataverse Application (and the Dataverse Ins host all all [ADDRESS] 255.255.255.255 md5 - Where ``[ADDRESS]`` is the numeric IP address of the Glassfish server. Enter this address when the installer asks for the PostgreSQL server address. + Where ``[ADDRESS]`` is the numeric IP address of the Payara server. Enter this address when the installer asks for the PostgreSQL server address. - In some distributions, PostgreSQL is pre-configured so that it doesn't accept network connections at all. Check that the ``listen_address`` line in the configuration file ``postgresql.conf`` is not commented out and looks like this:: @@ -316,7 +289,7 @@ When installed using standard ``yum`` mechanism, above, the executable for the I On MacOS you can compile ImageMagick from sources, or use one of the popular installation frameworks, such as brew. -If the installed location of the convert executable is different from ``/usr/bin/convert``, you will also need to specify it in your Glassfish configuration using the JVM option, below. For example:: +If the installed location of the convert executable is different from ``/usr/bin/convert``, you will also need to specify it in your Payara configuration using the JVM option, below. For example:: -Ddataverse.path.imagemagick.convert=/opt/local/bin/convert diff --git a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst index 95368e31cd7..1ba0bd083ec 100644 --- a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst +++ b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst @@ -9,6 +9,8 @@ for the future of the Dataverse/TwoRavens collaboration are still being worked o such, **support for TwoRavens is somewhat limited at the moment (as of Spring of 2017).** +Please note that in the text below, Glassfish was changed to Payara but not tested. + Any questions regarding the features of TwoRavens, bug reports and such, should be addressed directly to the developers of the application. The `TwoRavens GitHub repository @@ -78,7 +80,7 @@ server to download the file **directly from the Dataverse application**. Access URLs need to be configured for this to work properly (this is done by the TwoRavens installer script in step ``3.``) -If you install all components on a single server and front Glassfish with Apache +If you install all components on a single server and front the app server with Apache (see :ref:`network-ports` under the :doc:`config` section), the component and data flow diagram might looks something like this: @@ -291,7 +293,7 @@ Dataverse URL ``http://{your hostname}:8080`` URL of the Dataverse t Please note the default values above. The installer assumes - that you are running both the Dataverse and TwoRavens/rApache on the same host; -- the default ports for Apache (80) and Glassfish that is serving your Dataverse (8080); +- the default ports for Apache (80) and the app server that is serving your Dataverse (8080); - ``http`` (not ``https``!) for both . This configuration is recommended if you are simply trying out/testing Dataverse @@ -330,7 +332,7 @@ Compare the two files. **It is important that the two copies are identical**. **If different**: -- the **TwoRavens version wins**. Meaning, you need to copy the version supplied with this TwoRavens distribution and overwrite the Glassfish version (above); then restart Glassfish. +- the **TwoRavens version wins**. Meaning, you need to copy the version supplied with this TwoRavens distribution and overwrite the app server version (above); then restart the app server. - unless this is a brand new Dataverse installation, it may have cached summary statistics fragments that were produced with the older version of this R code. You **must remove** all such cached files:: @@ -368,7 +370,7 @@ If the file does NOT appear as Tabular Data - if it is shown as Stata/dta, and no tabular attributes - the numbers of Variables and Observations and the UNF - are being displayed, try to refresh the page a couple of times. If that doesn't change the view to Tabular, it likely means that something went very wrong with the -tabular ingest. Consult the Glassfish server log for any error messages that may +tabular ingest. Consult the app server log for any error messages that may explain the failure. If the file is showing as Tabular Data, but the ``Explore`` button isn't present, @@ -418,9 +420,9 @@ You should get the output that looks like this:: ... If you are getting an error message instead, this is likely an Rserve connection problem. -Consult the Glassfish server log for any Rserve-related "connection refused" messages. +Consult the app server log for any Rserve-related "connection refused" messages. See if Rserve is running, and start it with ``service rserve start``, if necessary. -Check if the Rserve host name, username and password in the Glassfish configuration match +Check if the Rserve host name, username and password in the app server configuration match the actual Rserve configuration. (this is discussed in the section ``2.`` of the guide). Correct this, if necessary, then try again. @@ -444,7 +446,7 @@ Symptom: the variables view is initialized properly, but no model output appears I. Ports configuration discussion --------------------------------- -By default, Glassfish will install itself on ports 8080 and 8181 (for +By default, the app server will install itself on ports 8080 and 8181 (for ``HTTP`` and ``HTTPS``, respectively). Apache will install itself on port 80 (the default port for ``HTTP``). Under this configuration, your Dataverse will be accessible at ``http://{your host}:8080``, and rApache at @@ -458,16 +460,16 @@ defaults, and you should have a working installation in no time. However, if you are planning to use this installation to actually serve data to real users, you will most likely want to run your Dataverse on a standard port; and to use ``HTTPS``. It is definitely possible to configure -Glassfish to serve the application under ``HTTPS`` on port 443. However, we -**do not recommend** this setup! For at least 2 reasons: 1. Running Glassfish on +the app server to serve the application under ``HTTPS`` on port 443. However, we +**do not recommend** this setup! For at least 2 reasons: 1. Running the app server on port 443 will require you to **run it as root** user; which should be avoided, if possible, for reasons of security. Also, 2) installing ``SSL`` certificates under -Glassfish is unnecessarily complicated. The alternative configuration that -we recommend is to "hide" your Glassfish behind Apache. In this setup Apache +the app server is unnecessarily complicated. The alternative configuration that +we recommend is to "hide" your app server behind Apache. In this setup Apache serves as the ``HTTPS`` front running on port 443, proxying the traffic to -Glassfish using ``mod_proxy_ajp``; and Glassfish is running as +the app server using ``mod_proxy_ajp``; and the app server is running as an non-privileged user on a high port that's not accessible from the outside. -Unlike Glassfish, Apache has a mechanism for running on a privileged port (in +Unlike the app server, Apache has a mechanism for running on a privileged port (in this case, 443) as a non-privileged user. It is possible to use this configuration, and have this Apache instance serve TwoRavens and rApache too, all on the same server. Please see :ref:`network-ports` under the :doc:`config` @@ -534,7 +536,7 @@ replacing *every* instance of ``production<-FALSE`` line with ``production<-TRUE **In** ``dataexplore/rook/rooksource.R`` **the following line:** -``setwd("/usr/local/glassfish4/glassfish/domains/domain1/docroot/dataexplore/rook")`` +``setwd("/usr/local/payara5/glassfish/domains/domain1/docroot/dataexplore/rook")`` needs to be changed to: diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst index 9dd8fe8ce02..3d24c823e77 100644 --- a/doc/sphinx-guides/source/installation/shibboleth.rst +++ b/doc/sphinx-guides/source/installation/shibboleth.rst @@ -28,7 +28,7 @@ Only Red Hat Enterprise Linux (RHEL) and derivatives such as CentOS have been te Install Apache ~~~~~~~~~~~~~~ -We will be "fronting" Glassfish with Apache so that we can make use of the ``mod_shib`` Apache module. We will also make use of the ``mod_proxy_ajp`` module built in to Apache. +We will be "fronting" the app server with Apache so that we can make use of the ``mod_shib`` Apache module. We will also make use of the ``mod_proxy_ajp`` module built in to Apache. We include the ``mod_ssl`` package to enforce HTTPS per below. @@ -65,26 +65,17 @@ Please note that during the installation it's ok to import GPG keys from the Shi ``yum install shibboleth`` -Configure Glassfish -------------------- - -Apply GRIZZLY-1787 Patch -~~~~~~~~~~~~~~~~~~~~~~~~ - -In order for the Dataverse "download as zip" feature to work well with large files without causing ``OutOfMemoryError`` problems on Glassfish 4.1 when fronted with Apache, you should stop Glassfish, with ``./asadmin stop-domain domain1``, make a backup of ``glassfish4/glassfish/modules/glassfish-grizzly-extra-all.jar``, replace it with a patched version of ``glassfish-grizzly-extra-all.jar`` downloaded from :download:`here ` (the md5 is in the :download:`README <../_static/installation/files/issues/2180/grizzly-patch/readme.md>`), and start Glassfish again with ``./asadmin start-domain domain1``. - -For more background on the patch, please see https://java.net/jira/browse/GRIZZLY-1787 and https://github.com/IQSS/dataverse/issues/2180 and https://github.com/payara/Payara/issues/350 - -This problem has been reported to Glassfish at https://java.net/projects/glassfish/lists/users/archive/2015-07/message/1 and while Glassfish 4.1.1 includes a new enough version of Grizzly to fix the bug, other complicating factors prevent its adoption (look for "Glassfish 4.1.1" in the :doc:`prerequisites` section for details on why it is not recommended). +Configure Payara +---------------- -Glassfish HTTP and HTTPS ports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +App Server HTTP and HTTPS ports +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Apache will be listening on ports 80 and 443 so we need to make sure Glassfish isn't using them. If you've been changing the default ports used by Glassfish per the :doc:`config` section, revert the Glassfish HTTP service to listen on 8080, the default port: +Apache will be listening on ports 80 and 443 so we need to make sure the app server isn't using them. If you've been changing the default ports used by the app server per the :doc:`config` section, revert the HTTP service to listen on 8080, the default port: ``./asadmin set server-config.network-config.network-listeners.network-listener.http-listener-1.port=8080`` -Likewise, if necessary, revert the Glassfish HTTPS service to listen on port 8181: +Likewise, if necessary, revert the HTTPS service to listen on port 8181: ``./asadmin set server-config.network-config.network-listeners.network-listener.http-listener-2.port=8181`` @@ -102,7 +93,9 @@ This enables the `AJP protocol