IQSS · kcondon · Aug 28, 2023 · Aug 14, 2023 · Aug 16, 2023 · Aug 16, 2023
diff --git a/.env b/.env
@@ -1,4 +1,4 @@
 APP_IMAGE=gdcc/dataverse:unstable
 POSTGRES_VERSION=13
 DATAVERSE_DB_USER=dataverse
-SOLR_VERSION=8.11.1
+SOLR_VERSION=9.3.0
diff --git a/conf/solr/8.11.1/readme.md b/conf/solr/8.11.1/readme.md
diff --git a/conf/solr/8.11.1/schema.xml → conf/solr/9.3.0/schema.xml b/conf/solr/8.11.1/schema.xml → conf/solr/9.3.0/schema.xml
diff --git a/conf/solr/8.11.1/solrconfig.xml → conf/solr/9.3.0/solrconfig.xml b/conf/solr/8.11.1/solrconfig.xml → conf/solr/9.3.0/solrconfig.xml
@@ -1353,7 +1353,7 @@
        in Solr's conf/xslt directory.  Changes to xslt files are checked for
        every xsltCacheLifetimeSeconds.
     -->
-  <queryResponseWriter name="xslt" class="solr.XSLTResponseWriter">
+  <queryResponseWriter name="xslt" class="solr.scripting.xslt.XSLTResponseWriter">
     <int name="xsltCacheLifetimeSeconds">5</int>
   </queryResponseWriter>
 

diff --git a/conf/solr/8.11.1/update-fields.sh → conf/solr/9.3.0/update-fields.sh b/conf/solr/8.11.1/update-fields.sh → conf/solr/9.3.0/update-fields.sh
diff --git a/doc/release-notes/9260-solr930.md b/doc/release-notes/9260-solr930.md
@@ -0,0 +1,3 @@
+Solr has been upgraded to Solr 9. You should install Solr fresh and reindex. 
+
+Note that init scripts have been updated and starting Solr with -Dsolr.modules=scripting is now required.
diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr
@@ -5,9 +5,9 @@
 # chkconfig: 35 92 08
 # description: Starts and stops Apache Solr
 
-SOLR_DIR="/usr/local/solr/solr-8.11.1"
+SOLR_DIR="/usr/local/solr/solr-9.3.0"
 SOLR_COMMAND="bin/solr"
-SOLR_ARGS="-m 1g -j jetty.host=127.0.0.1"
+SOLR_ARGS="-m 1g -j jetty.host=127.0.0.1 -Dsolr.modules=scripting"
 SOLR_USER=solr
 
 case $1 in

diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service
@@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target
 [Service]
 User = solr
 Type = forking
-WorkingDirectory = /usr/local/solr/solr-8.11.1
-ExecStart = /usr/local/solr/solr-8.11.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1"
-ExecStop = /usr/local/solr/solr-8.11.1/bin/solr stop
+WorkingDirectory = /usr/local/solr/solr-9.3.0
+ExecStart = /usr/local/solr/solr-9.3.0/bin/solr start -m 1g -j "jetty.host=127.0.0.1" -Dsolr.modules=scripting
+ExecStop = /usr/local/solr/solr-9.3.0/bin/solr stop
 LimitNOFILE=65000
 LimitNPROC=65000
 Restart=on-failure

diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -516,7 +516,7 @@ the Solr schema configuration, including any enabled metadata schemas:
 
 ``curl "http://localhost:8080/api/admin/index/solr/schema"``
 
-You can use :download:`update-fields.sh <../../../../conf/solr/8.11.1/update-fields.sh>` to easily add these to the
+You can use :download:`update-fields.sh <../../../../conf/solr/9.3.0/update-fields.sh>` to easily add these to the
 Solr schema you installed for your Dataverse installation.
 
 The script needs a target XML file containing your Solr schema. (See the :doc:`/installation/prerequisites/` section of
@@ -540,7 +540,7 @@ from some place else than your Dataverse installation).
 Please note that reconfigurations of your Solr index might require a re-index. Usually release notes indicate
 a necessary re-index, but for your custom metadata you will need to keep track on your own.
 
-Please note also that if you are going to make a pull request updating ``conf/solr/8.11.1/schema.xml`` with fields you have
+Please note also that if you are going to make a pull request updating ``conf/solr/9.3.0/schema.xml`` with fields you have
 added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you
 don't care about) to create a complete list of fields. (This might change in the future.)
 

diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst
@@ -134,7 +134,7 @@ On Linux, you should just install PostgreSQL using your favorite package manager
 Install Solr
 ^^^^^^^^^^^^
 
-`Solr <http://lucene.apache.org/solr/>`_ 8.11.1 is required.
+`Solr <http://lucene.apache.org/solr/>`_ 9.3.0 is required.
 
 To install Solr, execute the following commands:
 
@@ -144,29 +144,29 @@ To install Solr, execute the following commands:
 
 ``cd /usr/local/solr``
 
-``curl -O http://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz``
+``curl -O http://archive.apache.org/dist/lucene/solr/9.3.0/solr-9.3.0.tgz``
 
-``tar xvfz solr-8.11.1.tgz``
+``tar xvfz solr-9.3.0.tgz``
 
-``cd solr-8.11.1/server/solr``
+``cd solr-9.3.0/server/solr``
 
 ``cp -r configsets/_default collection1``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/schema.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/9.3.0/schema.xml``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/schema_dv_mdb_fields.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/9.3.0/schema_dv_mdb_fields.xml``
 
 ``mv schema*.xml collection1/conf``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/solrconfig.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/9.3.0/solrconfig.xml``
 
 ``mv solrconfig.xml collection1/conf/solrconfig.xml``
 
-``cd /usr/local/solr/solr-8.11.1``
+``cd /usr/local/solr/solr-9.3.0``
 
 (Please note that the extra jetty argument below is a security measure to limit connections to Solr to only your computer. For extra security, run a firewall.)
 
-``bin/solr start -j "-Djetty.host=127.0.0.1"``
+``bin/solr start -j "-Djetty.host=127.0.0.1 -Dsolr.modules=scripting"``
 
 ``bin/solr create_core -c collection1 -d server/solr/collection1/conf``
 

diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst
@@ -159,7 +159,7 @@ The Dataverse Software search index is powered by Solr.
 Supported Versions
 ==================
 
-The Dataverse Software has been tested with Solr version 8.11.1. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported.
+The Dataverse Software has been tested with Solr version 9.3.0. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported.
 
 Installing Solr
 ===============
@@ -174,19 +174,19 @@ Become the ``solr`` user and then download and configure Solr::
 
         su - solr
         cd /usr/local/solr
-        wget https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz
-        tar xvzf solr-8.11.1.tgz
-        cd solr-8.11.1
+        wget https://archive.apache.org/dist/lucene/solr/9.3.0/solr-9.3.0.tgz
+        tar xvzf solr-9.3.0.tgz
+        cd solr-9.3.0
         cp -r server/solr/configsets/_default server/solr/collection1
 
 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place::
 
-        cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf
-        cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf
+        cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf
+        cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf
 
 Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ``<str name="qf">`` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev .
 
-A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-8.11.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
+A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.3.0/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
 
 Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``::
 
@@ -205,7 +205,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo
 
 Finally, you need to tell Solr to create the core "collection1" on startup::
 
-        echo "name=collection1" > /usr/local/solr/solr-8.11.1/server/solr/collection1/core.properties
+        echo "name=collection1" > /usr/local/solr/solr-9.3.0/server/solr/collection1/core.properties
 
 Solr Init Script
 ================
@@ -226,6 +226,8 @@ For systems using init.d (like CentOS 6), download this :download:`Solr init scr
         service start solr
         chkconfig solr on
 
+Please note that as of Solr 9 the following argument `to enable a module <https://solr.apache.org/guide/solr/latest/configuration-guide/solr-modules.html>`_ has been added to the init scripts: ``-Dsolr.modules=scripting``
+
 Securing Solr
 =============
 

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
@@ -78,6 +78,8 @@ services:
     depends_on:
       - dev_solr_initializer
     restart: on-failure
+    environment:
+     - SOLR_MODULES=scripting
     ports:
       - "8983:8983"
     networks:

diff --git a/downloads/download.sh b/downloads/download.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
 curl -L -O https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.7/payara-6.2023.7.zip
-curl -L -O https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz
+curl -L -O https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz
 curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar
 curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar
diff --git a/modules/container-configbaker/assembly.xml b/modules/container-configbaker/assembly.xml
@@ -8,7 +8,7 @@
         </fileSet>
         <!-- Get our custom Solr files -->
         <fileSet>
-            <directory>conf/solr/8.11.1</directory>
+            <directory>conf/solr/9.3.0</directory>
             <outputDirectory>solr</outputDirectory>
         </fileSet>
         <!-- Get the setup scripts from the installer (selected choice only) -->
@@ -43,4 +43,4 @@
             </excludes>
         </fileSet>
     </fileSets>
-</assembly>
+</assembly>
diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
@@ -150,7 +150,7 @@
         <!-- Major system components and dependencies -->
         <payara.version>6.2023.7</payara.version>
         <postgresql.version>42.5.1</postgresql.version>
-        <solr.version>8.11.1</solr.version>
+        <solr.version>9.3.0</solr.version>
         <aws.version>1.12.290</aws.version>
         <google.cloud.version>0.177.0</google.cloud.version>
 

diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile
@@ -55,13 +55,13 @@ ${JHOVE_SCHEMA}:	../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}
 	@echo copying jhove schema file
 	/bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}
 
-${SOLR_SCHEMA}:		../../conf/solr/8.11.1/schema.xml ../../conf/solr/8.11.1/update-fields.sh ${INSTALLER_ZIP_DIR}
+${SOLR_SCHEMA}:		../../conf/solr/9.3.0/schema.xml ../../conf/solr/9.3.0/update-fields.sh ${INSTALLER_ZIP_DIR}
 	@echo copying Solr schema file
-	/bin/cp ../../conf/solr/8.11.1/schema.xml ../../conf/solr/8.11.1/update-fields.sh ${INSTALLER_ZIP_DIR}
+	/bin/cp ../../conf/solr/9.3.0/schema.xml ../../conf/solr/9.3.0/update-fields.sh ${INSTALLER_ZIP_DIR}
 
-${SOLR_CONFIG}:		../../conf/solr/8.11.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
+${SOLR_CONFIG}:		../../conf/solr/9.3.0/solrconfig.xml ${INSTALLER_ZIP_DIR}
 	@echo copying Solr config file
-	/bin/cp ../../conf/solr/8.11.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
+	/bin/cp ../../conf/solr/9.3.0/solrconfig.xml ${INSTALLER_ZIP_DIR}
 
 ${PYTHON_FILES}: README_python.txt install.py installConfig.py installAppServer.py installUtils.py requirements.txt default.config interactive.config ${INSTALLER_ZIP_DIR}
 	@echo copying Python installer files

diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh
@@ -5,12 +5,12 @@ SOLR_USER=solr
 SOLR_HOME=/usr/local/solr
 mkdir $SOLR_HOME
 chown $SOLR_USER:$SOLR_USER $SOLR_HOME
-su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.11.1.tgz $SOLR_HOME"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-8.11.1.tgz"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.11.1/server/solr && cp -r configsets/_default . && mv _default collection1"
-su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.11.1/schema*.xml $SOLR_HOME/solr-8.11.1/server/solr/collection1/conf/"
-su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.11.1/solrconfig.xml $SOLR_HOME/solr-8.11.1/server/solr/collection1/conf/solrconfig.xml"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.11.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
+su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-9.3.0.tgz $SOLR_HOME"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-9.3.0.tgz"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-9.3.0/server/solr && cp -r configsets/_default . && mv _default collection1"
+su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/9.3.0/schema*.xml $SOLR_HOME/solr-9.3.0/server/solr/collection1/conf/"
+su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/9.3.0/solrconfig.xml $SOLR_HOME/solr-9.3.0/server/solr/collection1/conf/solrconfig.xml"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-9.3.0 && bin/solr start -Dsolr.modules=scripting && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
 cp /dataverse/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr /etc/init.d/solr
 chmod 755 /etc/init.d/solr
 /etc/init.d/solr stop

diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh
@@ -29,7 +29,7 @@
 mkdir /opt/maven
 mv apache-maven-3.8.2/* /opt/maven/
 echo "export JAVA_HOME=/usr/lib/jvm/jre-openjdk" > /etc/profile.d/maven.sh
 echo "export M2_HOME=/opt/maven" >> /etc/profile.d/maven.sh
 echo "export MAVEN_HOME=/opt/maven" >> /etc/profile.d/maven.sh
 echo "export PATH=/opt/maven/bin:${PATH}" >> /etc/profile.d/maven.sh
 chmod 0755 /etc/profile.d/maven.sh
@@ -52,7 +52,7 @@
 useradd $SOLR_USER || :
 DOWNLOAD_DIR='/dataverse/downloads'
 PAYARA_ZIP="$DOWNLOAD_DIR/payara-6.2023.7.zip"
-SOLR_TGZ="$DOWNLOAD_DIR/solr-8.11.1.tgz"
+SOLR_TGZ="$DOWNLOAD_DIR/solr-9.3.0.tgz"
 if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then
     echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...."
     cd $DOWNLOAD_DIR && ./download.sh && cd

diff --git a/tests/shell/spec/update_fields_spec.sh b/tests/shell/spec/update_fields_spec.sh
@@ -1,16 +1,16 @@
 #shellcheck shell=sh
 
 update_fields() {
-  ../../conf/solr/8.11.1/update-fields.sh "$@"
+  ../../conf/solr/9.3.0/update-fields.sh "$@"
 }
 
 Describe "Update fields command"
 
   Describe "can operate on upstream data"
-    copyUpstreamSchema() { cp ../../conf/solr/8.11.1/schema.xml data/solr/upstream-schema.xml; }
+    copyUpstreamSchema() { cp ../../conf/solr/9.3.0/schema.xml data/solr/upstream-schema.xml; }
     AfterAll 'copyUpstreamSchema'
 
-    Path schema-xml="../../conf/solr/8.11.1/schema.xml"
+    Path schema-xml="../../conf/solr/9.3.0/schema.xml"
     It "needs upstream schema.xml"
       The path schema-xml should be exist
     End
@@ -115,8 +115,8 @@
    End

    Describe "succeeds because"
      setup() { cp data/solr/minimal-schema.xml data/solr/minimal-schema-work.xml; }
      cleanup() { rm data/solr/minimal-schema-work.xml; }
      BeforeEach 'setup'
      AfterEach 'cleanup'

@@ -124,7 +124,7 @@
      AfterAll 'deleteUpstreamSchema'

      match_content() {
        grep -q "$@" "${match_content}"
      }

      It "prints nothing when editing minimal schema"