diff --git a/Dockerfile b/Dockerfile index 5f492ea0594..b0864a0c55f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1 @@ -# See `conf/docker` for Docker images +# See http://guides.dataverse.org/en/latest/developers/containers.html diff --git a/conf/docker/build.sh b/conf/docker/build.sh deleted file mode 100755 index 27145d199e1..00000000000 --- a/conf/docker/build.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# Creates images and pushes them to Docker Hub. -# The "latest" tag under "iqss" should be relatively stable. Don't push breaking changes there. -# None of the tags are suitable for production use. See https://github.com/IQSS/dataverse/issues/4040 -# To iterate on images, push to custom tags or tags based on branch names or a non-iqss Docker Hub org/username. Consider trying "internal" to push to the internal Minishift registry. - -# Docker Hub organization or username -HUBORG=iqss -# The most stable tag we have. -STABLE=latest -#FIXME: Use a real flag/argument parser. download-files.sh uses "getopts" for example. -if [ -z "$1" ]; then - echo "No argument supplied. For experiments, specify \"branch\" or \"custom my-custom-tag\" or \"huborg \" or \"internal\". Specify \"stable\" to push to the \"$STABLE\" tag under \"$HUBORG\" if your change won't break anything." - exit 1 -fi - -if [ "$1" == 'branch' ]; then - echo "We'll push a tag to the branch you're on." - GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) - TAG=$GIT_BRANCH -elif [ "$1" == 'stable' ]; then - echo "We'll push a tag to the most stable tag (which isn't saying much!)." - TAG=$STABLE -elif [ "$1" == 'custom' ]; then - if [ -z "$2" ]; then - echo "You must provide a custom tag as the second argument. Something other than \"$STABLE\"." - exit 1 - else - echo "We'll push a custom tag." - TAG=$2 - fi -elif [ "$1" == 'huborg' ]; then - if [ -z "$2" ]; then - echo "You must provide your Docker Hub organization or username as the second argument. \"$USER\" or whatever." - exit 1 - else - HUBORG=$2 - TAG=$STABLE - echo "We'll push to the Docker Hub organization or username you specified: $HUBORG." - fi -elif [ "$1" == 'internal' ]; then - echo "Building for internal Minishift registry." - TAG=$STABLE -else - echo "Unexpected argument: $1. Exiting. Run with no arguments for help." - TAG=$STABLE - exit 1 -fi -echo Images are being built for registry org/username \"$HUBORG\" with the tag \"$TAG\". -# -# Build dataverse-solr -# -cp ../solr/7.3.0/schema.xml solr/ -# move solr*.tgz to the solr image -cp ../../downloads/solr-7.3.0.tgz solr/ -docker build -t $HUBORG/dataverse-solr:$TAG solr/ -if [ "$1" == 'internal' ]; then - echo "Skipping docker push because we're using the internal Minishift registry." -else - # FIXME: Check the output of "docker build" and only push on success. - docker push $HUBORG/dataverse-solr:$TAG -fi -# -# Build dataverse-glassfish -# -# TODO: Think about if we really need dataverse.war because it's in dvinstall.zip. -cd ../.. -mvn clean -scripts/installer/custom-build-number -mvn package -cd conf/docker -cp ../../target/dataverse*.war dataverse-glassfish/dataverse.war -if [[ "$?" -ne 0 ]]; then - echo "Unable to copy war file into place. Did 'mvn package' work?" - exit 1 -fi -cd ../../scripts/installer -make clean -make -cd ../../conf/docker -cp ../../scripts/installer/dvinstall.zip dataverse-glassfish -if [[ "$?" -ne 0 ]]; then - echo "Unable to copy dvinstall.zip file into place. Did 'make' work?" - exit 1 -fi -cp ../../downloads/glassfish-4.1.zip dataverse-glassfish -if [[ "$?" -ne 0 ]]; then - echo "Unable to copy Glassfish zip file into place. You must run the download script in that directory once. " - exit 1 -fi -# We'll assume at this point that the download script has been run. -cp ../../downloads/weld-osgi-bundle-2.2.10.Final-glassfish4.jar dataverse-glassfish -docker build -t $HUBORG/dataverse-glassfish:$TAG dataverse-glassfish -if [ "$1" == 'internal' ]; then - echo "Skipping docker push because we're using the internal Minishift registry." -else - # FIXME: Check the output of "docker build" and only push on success. - docker push $HUBORG/dataverse-glassfish:$TAG -fi -# -# Build init-container -# -cp ../../scripts/installer/install dataverse-glassfish/init-container -docker build -t $HUBORG/init-container:$TAG dataverse-glassfish/init-container -if [ "$1" == 'internal' ]; then - echo "Skipping docker push because we're using the internal Minishift registry." -else - # FIXME: Check the output of "docker build" and only push on success. - docker push $HUBORG/init-container:$TAG -fi diff --git a/conf/docker/dataverse-glassfish/.gitignore b/conf/docker/dataverse-glassfish/.gitignore deleted file mode 100644 index 2084aa8849e..00000000000 --- a/conf/docker/dataverse-glassfish/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -glassfish-4.1.zip -weld-osgi-bundle-2.2.10.Final-glassfish4.jar -dvinstall.zip -dataverse.war -init-container/postgres-setup diff --git a/conf/docker/dataverse-glassfish/Dockerfile b/conf/docker/dataverse-glassfish/Dockerfile deleted file mode 100644 index d92bfbc54f0..00000000000 --- a/conf/docker/dataverse-glassfish/Dockerfile +++ /dev/null @@ -1,101 +0,0 @@ -FROM centos:7 -MAINTAINER Dataverse (support@dataverse.org) - -COPY glassfish-4.1.zip /tmp -COPY weld-osgi-bundle-2.2.10.Final-glassfish4.jar /tmp -COPY default.config /tmp -# Install dependencies -#RUN yum install -y unzip -RUN yum install -y \ - cronie \ - git \ - java-1.8.0-openjdk-devel \ - nc \ - perl \ - postgresql \ - sha1sum \ - unzip \ - wget - -ENV GLASSFISH_DOWNLOAD_SHA1 d1a103d06682eb08722fbc9a93089211befaa080 -ENV GLASSFISH_DIRECTORY "/usr/local/glassfish4" -ENV HOST_DNS_ADDRESS "localhost" -ENV POSTGRES_DB "dvndb" -ENV POSTGRES_USER "dvnapp" -ENV RSERVE_USER "rserve" -ENV RSERVE_PASSWORD "rserve" - -#RUN exitEarlyBeforeJq -RUN yum -y install epel-release \ - jq - -COPY dvinstall.zip /tmp - -#RUN ls /tmp -# -RUN find /tmp -# -#RUN exitEarly - -# Install Glassfish 4.1 -RUN cd /tmp \ - && unzip glassfish-4.1.zip \ - && mv glassfish4 /usr/local \ - && cd /usr/local/glassfish4/glassfish/modules \ - && rm weld-osgi-bundle.jar \ - && cp /tmp/weld-osgi-bundle-2.2.10.Final-glassfish4.jar . \ - && cd /tmp && unzip /tmp/dvinstall.zip \ - && chmod 777 -R /tmp/dvinstall/ \ - #FIXME: Patch Grizzly too! - && echo "Done installing and patching Glassfish" - -RUN chmod g=u /etc/passwd - -RUN mkdir -p /home/glassfish -RUN chgrp -R 0 /home/glassfish && \ - chmod -R g=u /home/glassfish - -RUN mkdir -p /usr/local/glassfish4 -RUN chgrp -R 0 /usr/local/glassfish4 && \ - chmod -R g=u /usr/local/glassfish4 - -#JHOVE -RUN cp /tmp/dvinstall/jhove* /usr/local/glassfish4/glassfish/domains/domain1/config - - -#SETUP JVM OPTIONS -ARG DOCKER_BUILD="true" -RUN echo $DOCKER_BUILD -RUN /tmp/dvinstall/glassfish-setup.sh -###glassfish-setup will handle everything in Dockerbuild - -##install jdbc driver -RUN cp /tmp/dvinstall/pgdriver/postgresql-42.2.12.jar /usr/local/glassfish4/glassfish/domains/domain1/lib - -# Customized persistence xml to avoid database recreation -#RUN mkdir -p /tmp/WEB-INF/classes/META-INF/ -#COPY WEB-INF/classes/META-INF/persistence.xml /tmp/WEB-INF/classes/META-INF/ - -# Install iRods iCommands -#RUN cd /tmp \ -# && yum -y install epel-release \ -# && yum -y install ftp://ftp.renci.org/pub/irods/releases/4.1.6/centos7/irods-icommands-4.1.6-centos7-x86_64.rpm - -#COPY config-glassfish /root/dvinstall -#COPY restart-glassfish /root/dvinstall -#COPY config-dataverse /root/dvinstall - -#RUN cd /root/dvinstall && ./config-dataverse -COPY ./entrypoint.sh / -#COPY ./ddl /root/dvinstall -#COPY ./init-postgres /root/dvinstall -#COPY ./init-glassfish /root/dvinstall -#COPY ./init-dataverse /root/dvinstall -#COPY ./setup-all.sh /root/dvinstall -#COPY ./setup-irods.sh /root/dvinstall -COPY ./Dockerfile / - -EXPOSE 8080 - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["dataverse"] diff --git a/conf/docker/dataverse-glassfish/default.config b/conf/docker/dataverse-glassfish/default.config deleted file mode 100644 index 7af10336f30..00000000000 --- a/conf/docker/dataverse-glassfish/default.config +++ /dev/null @@ -1,16 +0,0 @@ -HOST_DNS_ADDRESS localhost -GLASSFISH_DIRECTORY /usr/local/glassfish4 -ADMIN_EMAIL -MAIL_SERVER mail.hmdc.harvard.edu -POSTGRES_ADMIN_PASSWORD secret -POSTGRES_SERVER dataverse-postgresql-0.dataverse-postgresql-service -POSTGRES_PORT 5432 -POSTGRES_DATABASE dvndb -POSTGRES_USER dvnapp -POSTGRES_PASSWORD secret -SOLR_LOCATION dataverse-solr-service:8983 -TWORAVENS_LOCATION NOT INSTALLED -RSERVE_HOST localhost -RSERVE_PORT 6311 -RSERVE_USER rserve -RSERVE_PASSWORD rserve diff --git a/conf/docker/dataverse-glassfish/entrypoint.sh b/conf/docker/dataverse-glassfish/entrypoint.sh deleted file mode 100755 index 55bbbdedbb7..00000000000 --- a/conf/docker/dataverse-glassfish/entrypoint.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash -x - -# Entrypoint script for Dataverse web application. This script waits -# for dependent services (Rserve, Postgres, Solr) to start before -# initializing Glassfish. - -echo "whoami before..." -whoami -if ! whoami &> /dev/null; then - if [ -w /etc/passwd ]; then - # Make `whoami` return the glassfish user. # See https://docs.openshift.org/3.6/creating_images/guidelines.html#openshift-origin-specific-guidelines - # Fancy bash magic from https://github.com/RHsyseng/container-rhel-examples/blob/1208dcd7d4f431fc6598184dba6341b9465f4197/starter-arbitrary-uid/bin/uid_entrypoint#L4 - echo "${USER_NAME:-glassfish}:x:$(id -u):0:${USER_NAME:-glassfish} user:/home/glassfish:/bin/bash" >> /etc/passwd - fi -fi -echo "whoami after" -whoami - -set -e - -if [ "$1" = 'dataverse' ]; then - - export GLASSFISH_DIRECTORY=/usr/local/glassfish4 - export HOST_DNS_ADDRESS=localhost - - TIMEOUT=30 - - if [ -n "$RSERVE_SERVICE_HOST" ]; then - RSERVE_HOST=$RSERVE_SERVICE_HOST - elif [ -n "$RSERVE_PORT_6311_TCP_ADDR" ]; then - RSERVE_HOST=$RSERVE_PORT_6311_TCP_ADDR - elif [ -z "$RSERVE_HOST" ]; then - RSERVE_HOST="localhost" - fi - export RSERVE_HOST - - if [ -n "$RSERVE_SERVICE_PORT" ]; then - RSERVE_PORT=$RSERVE_SERVICE_PORT - elif [ -n "$RSERVE_PORT_6311_TCP_PORT" ]; then - RSERVE_PORT=$RSERVE_PORT_6311_TCP_PORT - elif [ -z "$RSERVE_PORT" ]; then - RSERVE_PORT="6311" - fi - export RSERVE_PORT - - echo "Using Rserve at $RSERVE_HOST:$RSERVE_PORT" - - if ncat $RSERVE_HOST $RSERVE_PORT -w $TIMEOUT --send-only < /dev/null > /dev/null 2>&1 ; then - echo Rserve running; - else - echo Optional service Rserve not running. - fi - - - # postgres - if [ -n "$POSTGRES_SERVICE_HOST" ]; then - POSTGRES_HOST=$POSTGRES_SERVICE_HOST - elif [ -n "$POSTGRES_PORT_5432_TCP_ADDR" ]; then - POSTGRES_HOST=$POSTGRES_PORT_5432_TCP_ADDR - elif [ -z "$POSTGRES_HOST" ]; then - POSTGRES_HOST="localhost" - fi - export POSTGRES_HOST - - if [ -n "$POSTGRES_SERVICE_PORT" ]; then - POSTGRES_PORT=$POSTGRES_SERVICE_PORT - elif [ -n "$POSTGRES_PORT_5432_TCP_PORT" ]; then - POSTGRES_PORT=$POSTGRES_PORT_5432_TCP_PORT - else - POSTGRES_PORT=5432 - fi - export POSTGRES_PORT - - echo "Using Postgres at $POSTGRES_HOST:$POSTGRES_PORT" - - if ncat $POSTGRES_HOST $POSTGRES_PORT -w $TIMEOUT --send-only < /dev/null > /dev/null 2>&1 ; then - echo Postgres running; - else - echo Required service Postgres not running. Have you started the required services? - exit 1 - fi - - # solr - if [ -n "$SOLR_SERVICE_HOST" ]; then - SOLR_HOST=$SOLR_SERVICE_HOST - elif [ -n "$SOLR_PORT_8983_TCP_ADDR" ]; then - SOLR_HOST=$SOLR_PORT_8983_TCP_ADDR - elif [ -z "$SOLR_HOST" ]; then - SOLR_HOST="localhost" - fi - export SOLR_HOST - - if [ -n "$SOLR_SERVICE_PORT" ]; then - SOLR_PORT=$SOLR_SERVICE_PORT - elif [ -n "$SOLR_PORT_8983_TCP_PORT" ]; then - SOLR_PORT=$SOLR_PORT_8983_TCP_PORT - else - SOLR_PORT=8983 - fi - export SOLR_PORT - - echo "Using Solr at $SOLR_HOST:$SOLR_PORT" - - if ncat $SOLR_HOST $SOLR_PORT -w $TIMEOUT --send-only < /dev/null > /dev/null 2>&1 ; then - echo Solr running; - else - echo Required service Solr not running. Have you started the required services? - exit 1 - fi - - GLASSFISH_INSTALL_DIR="/usr/local/glassfish4" - cd /tmp/dvinstall - echo Copying the non-interactive file into place - cp /tmp/default.config . - echo Looking at first few lines of default.config - head default.config - # non-interactive install - echo Running non-interactive install - #./install -y -f > install.out 2> install.err - ./install -y -f - -# if [ -n "$DVICAT_PORT_1247_TCP_PORT" ]; then -# ./setup-irods.sh -# fi - - # We do change the Solr server in Minishift/OpenShift, which is - # the primary target for all of the work under conf/docker. - # echo -e "\n\nRestarting Dataverse in case Solr host was changed..." - # /usr/local/glassfish4/glassfish/bin/asadmin stop-domain - # sleep 3 - # /usr/local/glassfish4/glassfish/bin/asadmin start-domain - - echo -e "\n\nDataverse started" - - sleep infinity -else - exec "$@" -fi - diff --git a/conf/docker/dataverse-glassfish/init-container/Dockerfile b/conf/docker/dataverse-glassfish/init-container/Dockerfile deleted file mode 100644 index 829ab03bcb2..00000000000 --- a/conf/docker/dataverse-glassfish/init-container/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM centos:7 -MAINTAINER Dataverse (support@dataverse.org) - -### init-container is an Init Container for glassfish service in OpenShift or other Kubernetes environment -# This initContainer will take care of setting up glassfish - -# Install dependencies -RUN yum install -y \ - nc \ - perl \ - postgresql \ - sha1sum - -COPY install / - -ENTRYPOINT ["/install", "--pg_only", "--yes"] diff --git a/conf/docker/dataverse-glassfish/init-container/default.config b/conf/docker/dataverse-glassfish/init-container/default.config deleted file mode 100644 index 7af10336f30..00000000000 --- a/conf/docker/dataverse-glassfish/init-container/default.config +++ /dev/null @@ -1,16 +0,0 @@ -HOST_DNS_ADDRESS localhost -GLASSFISH_DIRECTORY /usr/local/glassfish4 -ADMIN_EMAIL -MAIL_SERVER mail.hmdc.harvard.edu -POSTGRES_ADMIN_PASSWORD secret -POSTGRES_SERVER dataverse-postgresql-0.dataverse-postgresql-service -POSTGRES_PORT 5432 -POSTGRES_DATABASE dvndb -POSTGRES_USER dvnapp -POSTGRES_PASSWORD secret -SOLR_LOCATION dataverse-solr-service:8983 -TWORAVENS_LOCATION NOT INSTALLED -RSERVE_HOST localhost -RSERVE_PORT 6311 -RSERVE_USER rserve -RSERVE_PASSWORD rserve diff --git a/conf/docker/postgresql/Dockerfile b/conf/docker/postgresql/Dockerfile deleted file mode 100644 index 81ecf0fdeb8..00000000000 --- a/conf/docker/postgresql/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -# PostgreSQL for Dataverse (but consider switching to the image from CentOS) -# -# See also conf/docker/dataverse-glassfish/Dockerfile diff --git a/conf/docker/solr/.gitignore b/conf/docker/solr/.gitignore deleted file mode 100644 index a6237a89914..00000000000 --- a/conf/docker/solr/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -solr-7.3.0.tgz -schema.xml diff --git a/conf/docker/solr/Dockerfile b/conf/docker/solr/Dockerfile deleted file mode 100644 index 993c2909f4d..00000000000 --- a/conf/docker/solr/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM centos:7 -MAINTAINER Dataverse (support@dataverse.org) - -RUN yum install -y unzip java-1.8.0-openjdk-devel lsof - -# Install Solr 7.3.0 -# The context of the build is the "conf" directory. -COPY solr-7.3.0.tgz /tmp -RUN cd /tmp \ - && tar xvfz solr-7.3.0.tgz \ - && rm solr-7.3.0.tgz \ - && mkdir /usr/local/solr \ - && mv solr-7.3.0 /usr/local/solr/ - -COPY schema.xml /tmp -COPY solrconfig_master.xml /tmp -COPY solrconfig_slave.xml /tmp - -RUN chmod g=u /etc/passwd - -RUN chgrp -R 0 /usr/local/solr && \ - chmod -R g=u /usr/local/solr - -EXPOSE 8983 - -COPY Dockerfile / -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] -USER 1001 -CMD ["solr"] diff --git a/conf/docker/solr/backup_cron.sh b/conf/docker/solr/backup_cron.sh deleted file mode 100644 index 95f31b4b53a..00000000000 --- a/conf/docker/solr/backup_cron.sh +++ /dev/null @@ -1 +0,0 @@ -0 */6 * * * curl 'http://localhost:8983/solr/collection1/replication?command=backup&location=/home/share' diff --git a/conf/docker/solr/entrypoint.sh b/conf/docker/solr/entrypoint.sh deleted file mode 100755 index 5d003f9c56b..00000000000 --- a/conf/docker/solr/entrypoint.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -if ! whoami &> /dev/null; then - if [ -w /etc/passwd ]; then - echo "${USER_NAME:-default}:x:$(id -u):0:${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd - fi -fi - -SOLR_DIR=/usr/local/solr/solr-7.3.0 - -if [ "$1" = 'solr' ]; then - - cp -r $SOLR_DIR/server/solr/configsets/_default $SOLR_DIR/server/solr/collection1 - cp /tmp/schema.xml $SOLR_DIR/server/solr/collection1/conf - - if [ $HOSTNAME = "dataverse-solr-0" ]; then - echo "I am the master" - mv /tmp/solrconfig_master.xml $SOLR_DIR/server/solr/collection1/conf/solrconfig.xml - cp /tmp/solrconfig_slave.xml $SOLR_DIR/server/solr/collection1/conf - - else - echo "I am the slave" - cp /tmp/solrconfig_slave.xml $SOLR_DIR/server/solr/collection1/conf - mv $SOLR_DIR/server/solr/collection1/conf/solrconfig_slave.xml $SOLR_DIR/server/solr/collection1/conf/solrconfig.xml - fi - cd $SOLR_DIR - bin/solr start - bin/solr create_core -c collection1 -d server/solr/collection1/conf - if [ $HOSTNAME = "dataverse-solr-0" ]; then - curl 'http://localhost:8983/solr/collection1/replication?command=restore&location=/home/share' - fi - - sleep infinity -elif [ "$1" = 'usage' ]; then - echo 'docker run -d iqss/dataverse-solr solr' -else - exec "$@" -fi diff --git a/conf/docker/solr/solrconfig_master.xml b/conf/docker/solr/solrconfig_master.xml deleted file mode 100644 index d409f70b5d5..00000000000 --- a/conf/docker/solr/solrconfig_master.xml +++ /dev/null @@ -1,1431 +0,0 @@ - - - - - - - - - 7.3.0 - - - - - - - - - - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - edismax - 0.075 - - dvName^400 - authorName^180 - dvSubject^190 - dvDescription^180 - dvAffiliation^170 - title^130 - subject^120 - keyword^110 - topicClassValue^100 - dsDescriptionValue^90 - authorAffiliation^80 - publicationCitation^60 - producerName^50 - fileName^30 - fileDescription^30 - variableLabel^20 - variableName^10 - _text_^1.0 - - - dvName^200 - authorName^100 - dvSubject^100 - dvDescription^100 - dvAffiliation^100 - title^75 - subject^75 - keyword^75 - topicClassValue^75 - dsDescriptionValue^75 - authorAffiliation^75 - publicationCitation^75 - producerName^75 - - - - isHarvested:false^25000 - - - - - - - - - - - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - _text_ - - - - - - - true - ignored_ - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - - - true - false - - - terms - - - - - - - - - string - - - - - - explicit - - - elevator - - - - - - - - startup - commit - - - - - schema.xml,stopwords.txt,elevate.xml - solrconfig_slave.xml:solrconfig.xml,x.xml,y.xml - - - - - 1 - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - - 5 - - - - - - - - - - - - - - diff --git a/conf/docker/solr/solrconfig_slave.xml b/conf/docker/solr/solrconfig_slave.xml deleted file mode 100644 index c31710ebace..00000000000 --- a/conf/docker/solr/solrconfig_slave.xml +++ /dev/null @@ -1,1442 +0,0 @@ - - - - - - - - - 7.3.0 - - - - - - - - - - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - edismax - 0.075 - - dvName^400 - authorName^180 - dvSubject^190 - dvDescription^180 - dvAffiliation^170 - title^130 - subject^120 - keyword^110 - topicClassValue^100 - dsDescriptionValue^90 - authorAffiliation^80 - publicationCitation^60 - producerName^50 - fileName^30 - fileDescription^30 - variableLabel^20 - variableName^10 - _text_^1.0 - - - dvName^200 - authorName^100 - dvSubject^100 - dvDescription^100 - dvAffiliation^100 - title^75 - subject^75 - keyword^75 - topicClassValue^75 - dsDescriptionValue^75 - authorAffiliation^75 - publicationCitation^75 - producerName^75 - - - - isHarvested:false^25000 - - - - - - - - - - - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - _text_ - - - - - - - true - ignored_ - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - - - true - false - - - terms - - - - - - - - - string - - - - - - explicit - - - elevator - - - - - - - - - http://dataverse-solr-0.dataverse-solr-service:8983/solr/collection1 - - - 00:00:20 - - - internal - - 5000 - 10000 - - - username - password - - - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - - 5 - - - - - - - - - - - - - - diff --git a/doc/sphinx-guides/source/developers/containers.rst b/doc/sphinx-guides/source/developers/containers.rst index 7d08ddb6e4f..ecf82533160 100755 --- a/doc/sphinx-guides/source/developers/containers.rst +++ b/doc/sphinx-guides/source/developers/containers.rst @@ -7,411 +7,11 @@ The Dataverse community is exploring the use of Docker, Kubernetes, and other co - https://github.com/IQSS/dataverse-kubernetes - https://github.com/IQSS/dataverse-docker -Much of the content below was written in 2017-2018 and references an old version of an application server (Glassfish 4) that is no longer used in development. Pull requests are welcome to completely rewrite this page. +The :doc:`testing` section mentions using Docker for integration tests. .. contents:: |toctitle| :local: -OpenShift ---------- - -From the Dataverse perspective, we are in the business of providing a "template" for OpenShift that describes how the various components we build our application on (Glassfish, PostgreSQL, Solr, the Dataverse war file itself, etc.) work together. We publish Docker images to DockerHub at https://hub.docker.com/u/iqss/ that are used in this OpenShift template. - -Dataverse's (light) use of Docker is documented below in a separate section. We actually started with Docker in the context of OpenShift, which is why OpenShift is listed first but we can imagine rearranging this in the future. - -The OpenShift template for Dataverse can be found at ``conf/openshift/openshift.json`` and if you need to hack on the template or related files under ``conf/docker`` it is recommended that you iterate on them using Minishift. - -The instructions below will walk you through spinning up Dataverse within Minishift. It is recommended that you do this on the "develop" branch to make sure everything is working before changing anything. - -Install Minishift -~~~~~~~~~~~~~~~~~ - -Minishift requires a hypervisor and since we already use VirtualBox for Vagrant, you should install VirtualBox from http://virtualbox.org . - -Download the Minishift tarball from https://docs.openshift.org/latest/minishift/getting-started/installing.html and put the ``minishift`` binary in ``/usr/local/bin`` or somewhere in your ``$PATH``. This assumes Mac or Linux. These instructions were last tested on version ``v1.14.0+1ec5877`` of Minishift. - -At this point, you might want to consider going through the Minishift quickstart to get oriented: https://docs.openshift.org/latest/minishift/getting-started/quickstart.html - -Start Minishift -~~~~~~~~~~~~~~~ - -``minishift start --vm-driver=virtualbox --memory=8GB`` - -Make the OpenShift Client Binary (oc) Executable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``eval $(minishift oc-env)`` - -Log in to Minishift from the Command Line -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Note that if you just installed and started Minishift, you are probably logged in already. This ``oc login`` step is included in case you aren't logged in anymore. - -``oc login --username developer --password=whatever`` - -Use "developer" as the username and a couple characters as the password. - -Create a Minishift Project -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Calling the project "project1" is fairly arbitrary. We'll probably want to revisit this name in the future. A project is necessary in order to create an OpenShift app. - -``oc new-project project1`` - -Note that ``oc projects`` will return a list of projects. - -Create a Dataverse App within the Minishift Project -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following command operates on the ``conf/openshift/openshift.json`` file that resides in the main Dataverse git repo. It will download images from Docker Hub and use them to spin up Dataverse within Minishift/OpenShift. Later we will cover how to make changes to the images on Docker Hub. - -``oc new-app conf/openshift/openshift.json`` - -Log into Minishift and Visit Dataverse in your Browser -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -After running the ``oc new-app`` command above, deployment of Dataverse within Minishift/OpenShift will begin. You should log into the OpenShift web interface to check on the status of the deployment. If you just created the Minishift VM with the ``minishift start`` command above, the ``oc new-app`` step is expected to take a while because the images need to be downloaded from Docker Hub. Also, the installation of Dataverse takes a while. - -Typing ``minishift console`` should open the OpenShift web interface in your browser. The IP address might not be "192.168.99.100" but it's used below as an example. - -- https://192.168.99.100:8443 (or URL from ``minishift console``) -- username: developer -- password: - -In the OpenShift web interface you should see a link that looks something like http://dataverse-project1.192.168.99.100.nip.io but the IP address will vary and will match the output of ``minishift ip``. Eventually, after deployment is complete, the Dataverse web interface will appear at this URL and you will be able to log in with the username "dataverseAdmin" and the password "admin". - -Another way to verify that Dataverse has been succesfully deployed is to make sure that the Dataverse "info" API endpoint returns a version (note that ``minishift ip`` is used because the IP address will vary): - -``curl http://dataverse-project1.`minishift ip`.nip.io/api/info/version`` - -From the perspective of OpenShift and the ``openshift.json`` config file, the HTTP link to Dataverse in called a route. See also documentation for ``oc expose``. - -Troubleshooting -~~~~~~~~~~~~~~~ - -Here are some tips on troubleshooting your deployment of Dataverse to Minishift. - -Check Status of Dataverse Deployment to Minishift -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``oc status`` - -Once images have been downloaded from Docker Hub, the output below will change from ``Pulling`` to ``Pulled``. - -``oc get events | grep Pull`` - -This is a deep dive: - -``oc get all`` - -Review Logs of Dataverse Deployment to Minishift -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Logs are provided in the web interface to each of the deployment configurations. The URLs should be something like this (but the IP address) will vary and you should click "View Log". The installation of Dataverse is done within the one Glassfish deployment configuration: - -- https://192.168.99.100:8443/console/project/project1/browse/dc/dataverse-glassfish -- https://192.168.99.100:8443/console/project/project1/browse/dc/dataverse-postgresql -- https://192.168.99.100:8443/console/project/project1/browse/dc/dataverse-solr - -You can also see logs from each of the components (Glassfish, PostgreSQL, and Solr) from the command line with ``oc logs`` like this (just change the ``grep`` at the end): - -``oc logs $(oc get po -o json | jq '.items[] | select(.kind=="Pod").metadata.name' -r | grep glassfish)`` - -Get a Shell (ssh/rsh) on Containers Deployed to Minishift -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can get a shell on any of the containers for each of the components (Glassfish, PostgreSQL, and Solr) with ``oc rc`` (just change the ``grep`` at the end): - -``oc rsh $(oc get po -o json | jq '.items[] | select(.kind=="Pod").metadata.name' -r | grep glassfish)`` - -From the ``rsh`` prompt of the Glassfish container you could run something like the following to make sure that Dataverse is running on port 8080: - -``curl http://localhost:8080/api/info/version`` - -Cleaning up -~~~~~~~~~~~ - -If you simply wanted to try out Dataverse on Minishift and want to clean up, you can run ``oc delete project project1`` to delete the project or ``minishift stop`` and ``minishift delete`` to delete the entire Minishift VM and all the Docker containers inside it. - -Making Changes -~~~~~~~~~~~~~~ - -Making Changes to Docker Images -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you're interested in using Minishift for development and want to change the Dataverse code, you will need to get set up to create Docker images based on your changes and make them available within Minishift. - -It is recommended to add experimental images to Minishift's internal registry. Note that despite what https://docs.openshift.org/latest/minishift/openshift/openshift-docker-registry.html says you will not use ``docker push`` because we have seen "unauthorized: authentication required” when trying to push to it as reported at https://github.com/minishift/minishift/issues/817 . Rather you will run ``docker build`` and run ``docker images`` to see that your newly build images are listed in Minishift's internal registry. - -First, set the Docker environment variables so that ``docker build`` and ``docker images`` refer to the internal Minishift registry rather than your normal Docker setup: - -``eval $(minishift docker-env)`` - -When you're ready to build, change to the right directory: - -``cd conf/docker`` - -And then run the build script in "internal" mode: - -``./build.sh internal`` - -Note that ``conf/openshift/openshift.json`` must not have ``imagePullPolicy`` set to ``Always`` or it will pull from "iqss" on Docker Hub. Changing it to ``IfNotPresent`` allow Minishift to use the images shown from ``docker images`` rather than the ones on Docker Hub. - -Using Minishift for day to day Dataverse development might be something we want to investigate in the future. These blog posts talk about developing Java applications using Minishift/OpenShift: - -- https://blog.openshift.com/fast-iterative-java-development-on-openshift-kubernetes-using-rsync/ -- https://blog.openshift.com/debugging-java-applications-on-openshift-kubernetes/ - -Making Changes to the OpenShift Config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you are interested in changing the OpenShift config file for Dataverse at ``conf/openshift/openshift.json`` note that in many cases once you have Dataverse running in Minishift you can use ``oc process`` and ``oc apply`` like this (but please note that some errors and warnings are expected): - -``oc process -f conf/openshift/openshift.json | oc apply -f -`` - -The slower way to iterate on the ``openshift.json`` file is to delete the project and re-create it. - -Making Changes to the PostgreSQL Database from the Glassfish Pod -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can access and modify the PostgreSQL database via an interactive terminal called psql. - -To log in to psql from the command line of the Glassfish pod, type the following command: - -``PGPASSWORD=$POSTGRES_PASSWORD; export PGPASSWORD; /usr/bin/psql -h $POSTGRES_SERVER.$POSTGRES_SERVICE_HOST -U $POSTGRES_USER -d $POSTGRES_DATABASE`` - -To log in as an admin, type this command instead: - -``PGPASSWORD=$POSTGRESQL_ADMIN_PASSWORD; export PGPASSWORD; /usr/bin/psql -h $POSTGRES_SERVER.$POSTGRES_SERVICE_HOST -U postgres -d $POSTGRES_DATABASE`` - -Scaling Dataverse by Increasing Replicas in a StatefulSet -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Glassfish, Solr and PostgreSQL Pods are in a "StatefulSet" which is a concept from OpenShift and Kubernetes that you can read about at https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ - -As of this writing, the ``openshift.json`` file has a single "replica" for each of these two stateful sets. It's possible to increase the number of replicas from 1 to 3, for example, with this command: - -``oc scale statefulset/dataverse-glassfish --replicas=3`` - -The command above should result in two additional Glassfish pods being spun up. The name of the pods is significant and there is special logic in the "zeroth" pod ("dataverse-glassfish-0" and "dataverse-postgresql-0"). For example, only "dataverse-glassfish-0" makes itself the dedicated timer server as explained in :doc:`/admin/timers` section of the Admin Guide. "dataverse-glassfish-1" and other higher number pods will not be configured as a timer server. - -Once you have multiple Glassfish servers you may notice bugs that will require additional configuration to fix. One such bug has to do with Dataverse logos which are stored at ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/logos`` on each of the Glassfish servers. This means that the logo will look fine when you just uploaded it because you're on the server with the logo on the local file system but when you visit that dataverse in the future and you're on a differernt Glassfish server, you will see a broken image. (You can find some discussion of this logo bug at https://github.com/IQSS/dataverse-aws/issues/10 and http://irclog.iq.harvard.edu/dataverse/2016-10-21 .) This is all "advanced" installation territory (see the :doc:`/installation/advanced` section of the Installation Guide) and OpenShift might be a good environment in which to work on some of these bugs. - -Multiple PostgreSQL servers are possible within the OpenShift environment as well and have been set up with some amount of replication. "dataverse-postgresql-0" is the master and non-zero pods are the slaves. We have just scratched the surface of this configuration but replication from master to slave seems to we working. Future work could include failover and making Dataverse smarter about utilizing multiple PostgreSQL servers for reads. Right now we assume Dataverse is only being used with a single PostgreSQL server and that it's the master. - -Solr supports index distribution and replication for scaling. For OpenShift use, we choose replication. It's possible to scale up Solr using the method method similar to Glassfish, as mentioned aboved -In OpenShift, the first Solr pod, dataverse-solr-0, will be the master node, and the rest will be slave nodes - - -Configuring Persistent Volumes and Solr master node recovery -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Solr requires backing up the search index to persistent storage. For our proof of concept, we configure a hostPath, which allows Solr containers to access the hosts' file system, for our Solr containers backups. To read more about OpenShift/Kubernetes' persistent volumes, please visit: https://kubernetes.io/docs/concepts/storage/persistent-volumes - -To allow containers to use a host's storage, we need to allow access to that directory first. In this example, we expose /tmp/share to the containers:: - -# mkdir /tmp/share -# chcon -R -t svirt_sandbox_file_t -# chgrp root -R /tmp/share -# oc login -u system:admin -# oc edit scc restricted # Update allowHostDirVolumePlugin to true and runAsUser type to RunAsAny - - -To add a persistent volume and persistent volume claim, in conf/docker/openshift/openshift.json, add the following to objects in openshift.json. -Here, we are using hostPath for development purposes. Since OpenShift supports many types of cluster storages, -if the administrator wishes to use any cluster storage like EBS, Google Cloud Storage, etc, they would have to use a different type of Persistent Storage:: - - { - "kind" : "PersistentVolume", - "apiVersion" : "v1", - "metadata":{ - "name" : "solr-index-backup", - "labels":{ - "name" : "solr-index-backup", - "type" : "local" - } - }, - "spec":{ - "capacity":{ - "storage" : "8Gi" - }, - "accessModes":[ - "ReadWriteMany", "ReadWriteOnce", "ReadOnlyMany" - ], - "hostPath": { - "path" : "/tmp/share" - } - } - }, - { - "kind" : "PersistentVolumeClaim", - "apiVersion": "v1", - "metadata": { - "name": "solr-claim" - }, - "spec": { - "accessModes": [ - "ReadWriteMany", "ReadWriteOnce", "ReadOnlyMany" - ], - "resources": { - "requests": { - "storage": "3Gi" - } - }, - "selector":{ - "matchLabels":{ - "name" : "solr-index-backup", - "type" : "local" - } - } - } - } - - -To make solr container mount the hostPath, add the following part under .spec.spec (for Solr StatefulSet):: - - { - "kind": "StatefulSet", - "apiVersion": "apps/v1beta1", - "metadata": { - "name": "dataverse-solr", - .... - - "spec": { - "serviceName" : "dataverse-solr-service", - ..... - - "spec": { - "volumes": [ - { - "name": "solr-index-backup", - "persistentVolumeClaim": { - "claimName": "solr-claim" - } - } - ], - - "containers": [ - .... - - "volumeMounts":[ - { - "mountPath" : "/var/share", - "name" : "solr-index-backup" - } - - - -Solr is now ready for backup and recovery. In order to backup:: - - oc rsh dataverse-solr-0 - curl 'http://localhost:8983/solr/collection1/replication?command=backup&location=/var/share' - - -In solr entrypoint.sh, it's configured so that if dataverse-solr-0 failed, it will get the latest version of the index in the backup and restore. All backups are stored in /tmp/share in the host, or /home/share in solr containers. - -Running Containers to Run as Root in Minishift -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is **not** recommended to run containers as root in Minishift because for security reasons OpenShift doesn't support running containers as root. However, it's good to know how to allow containers to run as root in case you need to work on a Docker image to make it run as non-root. - -For more information on improving Docker images to run as non-root, see "Support Arbitrary User IDs" at https://docs.openshift.org/latest/creating_images/guidelines.html#openshift-origin-specific-guidelines - -Let's say you have a container that you suspect works fine when it runs as root. You want to see it working as-is before you start hacking on the Dockerfile and entrypoint file. You can configure Minishift to allow containers to run as root with this command: - -``oc adm policy add-scc-to-user anyuid -z default --as system:admin`` - -Once you are done testing you can revert Minishift back to not allowing containers to run as root with this command: - -``oc adm policy remove-scc-from-user anyuid -z default --as system:admin`` - -Minishift Resources -~~~~~~~~~~~~~~~~~~~ - -The following resources might be helpful. - -- https://blog.openshift.com/part-1-from-app-to-openshift-runtimes-and-templates/ -- https://blog.openshift.com/part-2-creating-a-template-a-technical-walkthrough/ -- https://docs.openshift.com/enterprise/3.0/architecture/core_concepts/templates.html - -Docker ------- - -From the Dataverse perspective, Docker is important for a few reasons: - -- There is interest from the community in running Dataverse on OpenShift and some initial work has been done to get Dataverse running on Minishift in Docker containers. Minishift makes use of Docker images on Docker Hub. To build new Docker images and push them to Docker Hub, you'll need to install Docker. The main issue to follow is https://github.com/IQSS/dataverse/issues/4040 . -- Docker may aid in testing efforts if we can easily spin up Docker images based on code in pull requests and run the full integration suite against those images. See the :doc:`testing` section for more information on integration tests. - -Installing Docker -~~~~~~~~~~~~~~~~~ - -On Linux, you can probably get Docker from your package manager. - -On Mac, download the ``.dmg`` from https://www.docker.com and install it. As of this writing is it known as Docker Community Edition for Mac. - -On Windows, we have heard reports of success using Docker on a Linux VM running in VirtualBox or similar. There's something called "Docker Community Edition for Windows" but we haven't tried it. See also the :doc:`windows` section. - -As explained above, we use Docker images in two different contexts: - -- Testing using an "all in one" Docker image (ephemeral, unpublished) -- Future production use on Minishift/OpenShift/Kubernetes (published to Docker Hub) - -All In One Docker Images for Testing -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The "all in one" Docker files are in ``conf/docker-aio`` and you should follow the readme in that directory for more information on how to use them. - -Future production use on Minishift/OpenShift/Kubernetes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -FIXME: rewrite this section to talk about only pushing stable images to Docker Hub. - -When working with Docker in the context of Minishift, follow the instructions above and make sure you get the Dataverse Docker images running in Minishift before you start messing with them. - -As of this writing, the Dataverse Docker images we publish under https://hub.docker.com/u/iqss/ are highly experimental. They were originally tagged with branch names like ``kick-the-tires`` and as of this writing the ``latest`` tag should be considered highly experimental and not for production use. See https://github.com/IQSS/dataverse/issues/4040 for the latest status and please reach out if you'd like to help! - -Change to the docker directory: - -``cd conf/docker`` - -Edit one of the files: - -``vim dataverse-glassfish/Dockerfile`` - -At this point you want to build the image and run it. We are assuming you want to run it in your Minishift environment. We will be building your image and pushing it to Docker Hub. - -Log in to Docker Hub with an account that has access to push to the ``iqss`` organization: - -``docker login`` - -(If you don't have access to push to the ``iqss`` organization, you can push elsewhere and adjust your ``openshift.json`` file accordingly.) - -Build and push the images to Docker Hub: - -``./build.sh`` - -Note that you will see output such as ``digest: sha256:213b6380e6ee92607db5d02c9e88d7591d81f4b6d713224d47003d5807b93d4b`` that should later be reflected in Minishift to indicate that you are using the latest image you just pushed to Docker Hub. - -You can get a list of all repos under the ``iqss`` organization with this: - -``curl https://hub.docker.com/v2/repositories/iqss/`` - -To see a specific repo: - -``curl https://hub.docker.com/v2/repositories/iqss/dataverse-glassfish/`` - -Known Issues with Dataverse Images on Docker Hub -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Again, Dataverse Docker images on Docker Hub are highly experimental at this point. As of this writing, their purpose is primarily for kicking the tires on Dataverse. Here are some known issues: - -- The Dataverse installer is run in the entrypoint script every time you run the image. Ideally, Dataverse would be installed in the Dockerfile instead. Dataverse is being installed in the entrypoint script because it needs PosgreSQL to be up already so that database tables can be created when the war file is deployed. -- The storage should be abstracted. Storage of data files and PostgreSQL data. Probably Solr data. -- Better tuning of memory by examining ``/sys/fs/cgroup/memory/memory.limit_in_bytes`` and incorporating this into the Dataverse installation script. -- Only a single Glassfish server can be used. See "Dedicated timer server in a Dataverse server cluster" in the :doc:`/admin/timers` section of the Installation Guide. -- Only a single PostgreSQL server can be used. -- Only a single Solr server can be used. - ---- Previous: :doc:`deployment` | Next: :doc:`making-releases` diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst index 0386713a161..ec5d4cb971f 100755 --- a/doc/sphinx-guides/source/developers/windows.rst +++ b/doc/sphinx-guides/source/developers/windows.rst @@ -4,7 +4,7 @@ Windows Development Development on Windows is not well supported, unfortunately. You will have a much easier time if you develop on Mac or Linux as described under :doc:`dev-environment` section. -If you want to try using Windows for Dataverse development, your best best is to use Vagrant, as described below. Minishift is also an option. These instructions were tested on Windows 10. +Vagrant commands appear below and were tested on Windows 10 but the Vagrant environement is currently broken. Please see https://github.com/IQSS/dataverse/issues/6849 .. contents:: |toctitle| :local: @@ -71,64 +71,6 @@ The ``dataverse`` directory you changed is the one you just cloned. Vagrant will After a long while you hopefully will have Dataverse installed at http://localhost:8888 -Running Dataverse in Minishift ------------------------------- - -Minishift is a dev environment for OpenShift, which is Red Hat's distribution of Kubernetes. The :doc:`containers` section contains much more detail but the essential steps for using Minishift on Windows are described below. - -Install VirtualBox -~~~~~~~~~~~~~~~~~~ - -Download and install VirtualBox from https://www.virtualbox.org - -Install Git -~~~~~~~~~~~ - -Download and install Git from https://git-scm.com - -Install Minishift -~~~~~~~~~~~~~~~~~ - -Download Minishift from https://docs.openshift.org/latest/minishift/getting-started/installing.html . It should be a zip file. - -From Git Bash: - -``cd ~/Downloads`` - -``unzip minishift*.zip`` - -``mkdir ~/bin`` - -``cp minishift*/minishift.exe ~/bin`` - -Clone Git Repo -~~~~~~~~~~~~~~ - -From Git Bash, run the following commands: - -``git config --global core.autocrlf input`` - -``git clone https://github.com/IQSS/dataverse.git`` - -Start Minishift VM and Run Dataverse -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``minishift start --vm-driver=virtualbox --memory=8GB`` - -``eval $(minishift oc-env)`` - -``oc new-project project1`` - -``cd ~/dataverse`` - -``oc new-app conf/openshift/openshift.json`` - -``minishift console`` - -This should open a web browser. In Microsoft Edge we saw ``INET_E_RESOURCE_NOT_FOUND`` so if you see that, try Chrome instead. A cert error is expected. Log in with the username "developer" and any password such as "asdf". - -Under "Overview" you should see a URL that has "dataverse-project1" in it. You should be able to click it and log into Dataverse with the username "dataverseAdmin" and the password "admin". - Improving Windows Support ------------------------- @@ -140,4 +82,4 @@ We have been unable to get Windows Subsystem for Linux (WSL) to work. We tried f Discussion and Feedback ~~~~~~~~~~~~~~~~~~~~~~~ -For more discussion of Windows support for Dataverse development see our community list thread `"Do you want to develop on Windows?" `_ We would be happy to inconrporate feedback from Windows developers into this page. The :doc:`documentation` section describes how. +For more discussion of Windows support for Dataverse development see our community list thread `"Do you want to develop on Windows?" `_ We would be happy to incorporate feedback from Windows developers into this page. The :doc:`documentation` section describes how.