diff --git a/.env b/.env index 4acb2891..a8ac2d49 100644 --- a/.env +++ b/.env @@ -27,6 +27,10 @@ TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_te CKAN_SOLR_URL=http://solr:8983/solr/ckan CKAN_REDIS_URL=redis://redis:6379/1 +CKAN_SOLR_BASE_URL=http://solr:8983 +CKAN_SOLR_USER=admin +CKAN_SOLR_PASSWORD=pass + TEST_CKAN_SOLR_URL=http://solr:8983/solr/ckan TEST_CKAN_REDIS_URL=redis://redis:6379/1 @@ -142,4 +146,4 @@ CKANEXT__SAML2AUTH__REQUESTED_AUTHN_CONTEXT=http://idmanagement.gov/ns/assurance CKANEXT__SAML2AUTH__REQUESTED_AUTHN_CONTEXT_COMPARISON=exact # Avoid double package_show call to add tracking info -CKANEXT__DATAGOVCATALOG__ADD_PACKAGES_TRACKING_INFO=false \ No newline at end of file +CKANEXT__DATAGOVCATALOG__ADD_PACKAGES_TRACKING_INFO=false diff --git a/.gitignore b/.gitignore index 5bd23765..47a82750 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ e2e/cypress/results/output.xml node_modules package-lock.json -solr/*.zip \ No newline at end of file +ckan/setup/solr/*.zip +ckan/setup/solr/managed-schema diff --git a/.profile b/.profile index 8bcb77fd..466239a7 100755 --- a/.profile +++ b/.profile @@ -82,7 +82,7 @@ export CKANEXT__SAML2AUTH__SYSADMINS_LIST=$(echo $VCAP_SERVICES | jq --raw-outpu # Set up the collection in Solr echo Setting up Solr collection export SOLR_COLLECTION=ckan -./solr/migrate-solrcloud-schema.sh $SOLR_COLLECTION +./ckan/setup/migrate-solrcloud-schema.sh $SOLR_COLLECTION export CKAN_SOLR_URL=$CKAN_SOLR_BASE_URL/solr/$SOLR_COLLECTION # Write out any files and directories diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 927bb974..4d030862 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -5,7 +5,7 @@ FROM openknowledge/ckan-dev:2.9 ENV GIT_BRANCH=2.9 # add dependencies for cryptography and vim -RUN apk add libressl-dev musl-dev libffi-dev xmlsec vim xmlsec-dev openjdk11 +RUN apk add libressl-dev musl-dev libffi-dev xmlsec vim xmlsec-dev openjdk11 zip # Download Saxon jar for FGDC2ISO transform (geodatagov) ARG saxon_ver=9.9.1-7 ADD \ @@ -26,6 +26,9 @@ COPY docker-entrypoint.d/* /docker-entrypoint.d/ COPY setup/gunicorn.conf.py ${APP_DIR}/ COPY setup/server_start.sh ${APP_DIR}/ +# Custom prerun script for Solr 8 +COPY setup/GSA_prerun.py ${APP_DIR}/ + COPY saml2 ${APP_DIR}/saml2 # COPY the ini test file to the container @@ -39,4 +42,4 @@ RUN ln -s /usr/bin/python3 /usr/bin/python # harvests, we need to setup a cron for the run command COPY setup/harvest-check-cron /etc/crontabs/root -# RUN sudo -u ckan -EH pip3 install git+https://github.com/nickumia-reisys/werkzeug@e1f6527604ab30e4b46b5430a5fb97e7a7055cd7#egg=werkzeug \ No newline at end of file +# RUN sudo -u ckan -EH pip3 install git+https://github.com/nickumia-reisys/werkzeug@e1f6527604ab30e4b46b5430a5fb97e7a7055cd7#egg=werkzeug diff --git a/ckan/requirements.in b/ckan/requirements.in index dc857fd9..fd453af3 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -1,6 +1,6 @@ # CKAN requirements and extensions -ckan @ git+https://github.com/ckan/ckan.git@ckan-2.9.4 +ckan @ git+https://github.com/ckan/ckan.git@dev-v2.9 -e git+https://github.com/GSA/ckanext-datagovcatalog.git@main#egg=ckanext-datagovcatalog -e git+https://github.com/GSA/ckanext-datagovtheme.git@main#egg=ckanext-datagovtheme -e git+https://github.com/GSA/ckanext-datajson.git@main#egg=ckanext-datajson diff --git a/ckan/requirements.txt b/ckan/requirements.txt index b188663a..8ec16fb9 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -6,20 +6,20 @@ boto==2.49.0 certifi==2021.10.8 cffi==1.15.0 chardet==3.0.4 -ckan @ git+https://github.com/ckan/ckan.git@6731c5a821a6a5f4bdaa20f4e793e0b6ba44f823 +ckan @ git+https://github.com/ckan/ckan.git@ef1432fa4177edbd0a2f457bad363a33b6b61344 -e git+https://github.com/GSA/ckanext-datagovcatalog.git@64e65702ae1eb5e46d9f37139dc4044c0f253526#egg=ckanext_datagovcatalog --e git+https://github.com/GSA/ckanext-datagovtheme.git@fa53b7c836d4cc3f34b0c5f156b04f9bacb8a67e#egg=ckanext_datagovtheme +-e git+https://github.com/GSA/ckanext-datagovtheme.git@d20b0f696cc16251dc94ee811f3f36cbde6673dc#egg=ckanext_datagovtheme -e git+https://github.com/GSA/ckanext-datajson.git@a5d8c9458a7efe955c31b90eeac1e7797881d014#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@2d2c8a894bea8c97b0c8544465094f9979ac516b ckanext-envvars @ git+https://github.com/GSA/ckanext-envvars.git@33f7e190ab332244cb961a425e09af592d9b647b -e git+https://github.com/GSA/ckanext-geodatagov.git@af6378074fcbc2705e7e33960d5ddd2c8e46ed4c#egg=ckanext_geodatagov ckanext-googleanalyticsbasic @ git+https://github.com/GSA/ckanext-googleanalyticsbasic.git@c6a425d5e14d658c0fa3661fdc4423162161c3f4 --e git+https://github.com/ckan/ckanext-harvest.git@9d5679f0461f5aac05b7f800e11b6a62afb7feeb#egg=ckanext_harvest +-e git+https://github.com/ckan/ckanext-harvest.git@d84d847b09f28ab97bf1ca0baa651fdc05693d03#egg=ckanext_harvest ckanext-saml2auth @ git+https://github.com/keitaroinc/ckanext-saml2auth.git@7412ff7aba3d215f95a08f99216410e72e60c5bc -e git+https://github.com/gsa/ckanext-spatial.git@3828c6e7efe7c4b5cef02f4e7163339c7b5c5710#egg=ckanext_spatial ckantoolkit==0.0.3 click==7.1.2 -cryptography==35.0.0 +cryptography==36.0.1 defusedxml==0.7.1 Deprecated==1.2.13 distro==1.6.0 @@ -33,28 +33,28 @@ flask-multistatic==1.0 future==0.18.2 GeoAlchemy2==0.5.0 geomet==0.3.0 -gevent==21.8.0 +gevent==21.12.0 google-compute-engine==2.8.13 greenlet==1.1.2 gunicorn==20.1.0 html5lib==1.1 idna==2.10 importlib-resources==5.4.0 -isodate==0.6.0 +isodate==0.6.1 itsdangerous==2.0.1 Jinja2==2.11.3 json-table-schema==0.2.1 jsonschema==2.4.0 LEPL==5.1.3 -lxml==4.6.4 -Mako==1.1.5 +lxml==4.7.1 +Mako==1.1.6 Markdown==3.1.1 MarkupSafe==2.0.1 messytables==0.15.2 newrelic==7.2.4.171 nose==1.3.7 OWSLib==0.18.0 -packaging==21.2 +packaging==21.3 passlib==1.7.3 PasteDeploy==2.0.1 pathtools==0.1.2 @@ -65,7 +65,7 @@ psycopg2==2.8.6 pycparser==2.21 PyJWT==1.7.1 pyOpenSSL==20.0.1 -pyparsing==2.4.7 +pyparsing==3.0.6 pyproj==2.6.1 pysaml2==7.0.1 pysolr==3.6.0 @@ -78,7 +78,7 @@ PyUtilib==5.7.1 PyYAML==5.4 PyZ3950 @ git+https://github.com/danizen/PyZ3950@6d44a4ab85c8bda3a7542c2c9efdfad46c830219 rdflib==4.2.2 -redis==4.0.0 +redis==4.0.2 repoze.lru==0.7 repoze.who==2.3 requests==2.25.0 @@ -100,7 +100,7 @@ WebOb==1.8.7 Werkzeug==1.0.0 wrapt==1.13.3 xlrd==2.0.1 -xmlschema==1.8.2 +xmlschema==1.9.1 zipp==3.6.0 zope.event==4.5.0 zope.interface==5.4.0 diff --git a/ckan/setup/GSA_prerun.py b/ckan/setup/GSA_prerun.py new file mode 100644 index 00000000..0d0c985c --- /dev/null +++ b/ckan/setup/GSA_prerun.py @@ -0,0 +1,54 @@ +import os +import sys +import time +try: + from urllib.request import urlopen + from urllib.error import URLError +except ImportError: + from urllib2 import urlopen + from urllib2 import URLError + +import prerun as pr + +RETRY = 5 + + +def check_solr_connection(retry=None): + if retry is None: + retry = RETRY + elif retry == 0: + print("[prerun] Giving up after 5 tries...") + sys.exit(1) + + url = os.environ.get("CKAN_SOLR_URL", "") + search_url = "{url}/select/?q=*&wt=json".format(url=url) + + try: + connection = urlopen(search_url) + except URLError as e: + print(str(e)) + print("[prerun] Unable to connect to solr, waiting...") + time.sleep(10) + check_solr_connection(retry=retry - 1) + else: + try: + pythonified = str(connection.read()).replace('true', 'True') + eval(pythonified) + except TypeError: + pass + + +if __name__ == "__main__": + + maintenance = os.environ.get("MAINTENANCE_MODE", "").lower() == "true" + + if maintenance: + print("[prerun] Maintenance mode, skipping setup...") + else: + pr.check_main_db_connection() + pr.init_db() + pr.update_plugins() + pr.check_datastore_db_connection() + pr.init_datastore_db() + check_solr_connection() + pr.create_sysadmin() diff --git a/ckan/setup/ckan_setup.sh b/ckan/setup/ckan_setup.sh index 5f287c96..ef2d008e 100755 --- a/ckan/setup/ckan_setup.sh +++ b/ckan/setup/ckan_setup.sh @@ -50,8 +50,23 @@ ckan config-tool $SRC_DIR/ckan/test-core.ini \ "solr_url = $TEST_CKAN_SOLR_URL" \ "ckan.redis.url = $TEST_CKAN_REDIS_URL" +# SOLR takes a while to boot up in zookeeper mode, make sure it's up before +echo "Validating SOLR is up..." +NEXT_WAIT_TIME=0 +until [ $NEXT_WAIT_TIME -eq 10 ] || curl --get --fail --quiet --location-trusted --user $CKAN_SOLR_USER:$CKAN_SOLR_PASSWORD \ + $CKAN_SOLR_BASE_URL/solr/admin/collections \ + --data-urlencode action=list \ + --data-urlencode wt=json; do + sleep $(( NEXT_WAIT_TIME++ )) + echo "SOLR still not up, trying for the $NEXT_WAIT_TIME time" +done +[ $NEXT_WAIT_TIME -lt 10 ] + +# Add ckan core to solr +/app/ckan/setup/migrate-solrcloud-schema.sh + # Run the prerun script to init CKAN and create the default admin user -python3 prerun.py +python GSA_prerun.py # Run any startup scripts provided by images extending this one if [[ -d "/docker-entrypoint.d" ]] @@ -66,4 +81,4 @@ then done fi -exec /app/ckan/setup/server_start.sh \ No newline at end of file +exec /app/ckan/setup/server_start.sh diff --git a/solr/migrate-solrcloud-schema.sh b/ckan/setup/migrate-solrcloud-schema.sh similarity index 71% rename from solr/migrate-solrcloud-schema.sh rename to ckan/setup/migrate-solrcloud-schema.sh index 91bec5e2..ff2e38f4 100755 --- a/solr/migrate-solrcloud-schema.sh +++ b/ckan/setup/migrate-solrcloud-schema.sh @@ -23,9 +23,18 @@ if ! (curl --get --fail --location-trusted --user $CKAN_SOLR_USER:$CKAN_SOLR_PA --data-urlencode action=list \ --data-urlencode wt=json | grep -q $COLLECTION_NAME); then + cd $(dirname $0)/solr + + CKAN_BRANCH="dev-v2.9" + curl https://raw.githubusercontent.com/ckan/ckan/$CKAN_BRANCH/ckan/config/solr/schema.xml -o managed-schema + + # Fix from https://github.com/ckan/ckan/issues/5585#issuecomment-953586246 + sed -i "s/text<\/defaultSearchField>/text<\/df>/" managed-schema + sed -i "s///" managed-schema + # Zip solr configSet - cd solr && zip ckan_2.9_solr_config.zip \ - currency.xml elevate.xml protwords.txt schema.xml solrconfig.xml stopwords.txt synonyms.txt + zip ckan_2.9_solr_config.zip \ + managed-schema solrconfig.xml protwords.txt stopwords.txt synonyms.txt echo "Uploading config set..." curl --fail --location-trusted --user $CKAN_SOLR_USER:$CKAN_SOLR_PASSWORD \ @@ -36,5 +45,6 @@ if ! (curl --get --fail --location-trusted --user $CKAN_SOLR_USER:$CKAN_SOLR_PA curl --fail --location-trusted --user $CKAN_SOLR_USER:$CKAN_SOLR_PASSWORD \ "$CKAN_SOLR_BASE_URL/solr/admin/collections?action=create&name=$COLLECTION_NAME&collection.configName=$COLLECTION_NAME&numShards=1" \ -X POST + cd - fi diff --git a/solr/protwords.txt b/ckan/setup/solr/protwords.txt similarity index 97% rename from solr/protwords.txt rename to ckan/setup/solr/protwords.txt index 1dfc0abe..5a32e503 100644 --- a/solr/protwords.txt +++ b/ckan/setup/solr/protwords.txt @@ -1,21 +1,21 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -# Use a protected word file to protect against the stemmer reducing two -# unrelated words to the same base word. - -# Some non-words that normally won't be encountered, -# just to test that they won't be stemmed. -dontstems -zwhacky - +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/ckan/setup/solr/solrconfig.xml b/ckan/setup/solr/solrconfig.xml new file mode 100644 index 00000000..f76fca4a --- /dev/null +++ b/ckan/setup/solr/solrconfig.xml @@ -0,0 +1,1295 @@ + + + + + + + + + 8.11.1 + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + ${solr.max.booleanClauses:1024} + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + + + + + + + + + + + + + + + + explicit + json + true + + + + + + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + false + + + terms + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z + yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z + yyyy-MM-dd HH:mm[:ss[.SSS]][z + yyyy-MM-dd HH:mm[:ss[,SSS]][z + [EEE, ]dd MMM yyyy HH:mm[:ss] z + EEEE, dd-MMM-yy HH:mm:ss z + EEE MMM ppd HH:mm:ss [z ]yyyy + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + + + + + + diff --git a/solr/stopwords.txt b/ckan/setup/solr/stopwords.txt similarity index 98% rename from solr/stopwords.txt rename to ckan/setup/solr/stopwords.txt index ae1e83ee..25b47f6a 100644 --- a/solr/stopwords.txt +++ b/ckan/setup/solr/stopwords.txt @@ -1,14 +1,14 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/solr/synonyms.txt b/ckan/setup/solr/synonyms.txt similarity index 97% rename from solr/synonyms.txt rename to ckan/setup/solr/synonyms.txt index eab4ee87..d1ed0dc7 100644 --- a/solr/synonyms.txt +++ b/ckan/setup/solr/synonyms.txt @@ -1,29 +1,29 @@ -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#----------------------------------------------------------------------- -#some test synonym mappings unlikely to appear in real input text -aaafoo => aaabar -bbbfoo => bbbfoo bbbbar -cccfoo => cccbar cccbaz -fooaaa,baraaa,bazaaa - -# Some synonym groups specific to this example -GB,gib,gigabyte,gigabytes -MB,mib,megabyte,megabytes -Television, Televisions, TV, TVs -#notice we use "gib" instead of "GiB" so any WordDelimiterGraphFilter coming -#after us won't split it into two words. - -# Synonym mappings can be used for spelling correction too -pixima => pixma - +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterGraphFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/docker-compose.yml b/docker-compose.yml index 7c3fbd79..7cc835be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,11 +24,56 @@ services: - pg_data:/var/lib/postgresql/data solr: - image: ghcr.io/gsa/catalog.data.gov.solr + image: ghcr.io/gsa/catalog.data.gov.solr:8-curl + environment: + - ZK_HOST=zookeeper1:2181,zookeeper2:2182,zookeeper3:2183 ports: - "8983:8983" + depends_on: + - zookeeper1 + - zookeeper2 + - zookeeper3 + volumes: + - solr_data:/var/solr + + zookeeper1: + image: zookeeper:3.7 + restart: always + ports: + - 2181:2181 + hostname: zookeeper1 + environment: + ZOO_MY_ID: 1 + ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 server.2=zookeeper2:2888:3888;2181 server.3=zookeeper3:2888:3888;2181 + volumes: + - zookeeperdata:/data + - zookeeperdatalog:/datalog + + zookeeper2: + image: zookeeper:3.7 + restart: always + ports: + - 2182:2181 + hostname: zookeeper2 + environment: + ZOO_MY_ID: 2 + ZOO_SERVERS: server.1=zookeeper1:2888:3888;2181 server.2=0.0.0.0:2888:3888;2181 server.3=zookeeper3:2888:3888;2181 + volumes: + - zookeeperdata2:/data + - zookeeperdatalog2:/datalog + + zookeeper3: + image: zookeeper:3.7 + restart: always + ports: + - 2183:2181 + hostname: zookeeper3 + environment: + ZOO_MY_ID: 3 + ZOO_SERVERS: server.1=zookeeper1:2888:3888;2181 server.2=zookeeper2:2888:3888;2181 server.3=0.0.0.0:2888:3888;2181 volumes: - - solr_data:/opt/solr/server/solr/ckan/data/index + - zookeeperdata3:/data + - zookeeperdatalog3:/datalog redis: image: redis:alpine @@ -53,3 +98,9 @@ volumes: ckan_storage: pg_data: solr_data: + zookeeperdata: + zookeeperdatalog: + zookeeperdata2: + zookeeperdatalog2: + zookeeperdata3: + zookeeperdatalog3: diff --git a/e2e/cypress/integration/ckan_extensions.spec.js b/e2e/cypress/integration/ckan_extensions.spec.js index 41baca29..27cf31ea 100644 --- a/e2e/cypress/integration/ckan_extensions.spec.js +++ b/e2e/cypress/integration/ckan_extensions.spec.js @@ -3,7 +3,7 @@ describe('CKAN Extensions', () => { it('Uses CKAN 2.9', () => { cy.request('/api/action/status_show').should((response) => { expect(response.body).to.have.property('success', true); - expect(response.body.result).to.have.property('ckan_version', '2.9.4'); + expect(response.body.result).to.have.property('ckan_version', '2.9.5b'); }); }) @@ -43,4 +43,4 @@ describe('CKAN Extensions', () => { // expect(installed_extensions).to.include('qa'); }); }) -}) \ No newline at end of file +}) diff --git a/solr/Dockerfile b/solr/Dockerfile index 584263d8..5adf0440 100644 --- a/solr/Dockerfile +++ b/solr/Dockerfile @@ -1,27 +1,14 @@ -FROM solr:6.6.6-slim +FROM solr:8.11-slim MAINTAINER Open Knowledge -# Enviroment -ENV SOLR_CORE ckan -ENV CKAN_VERSION dev-v2.9 - -# Set user to root for initial configuration USER root -# Create Directories -RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf -RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data - -# Adding Files -ADD solrconfig.xml \ -schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.6/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ -/opt/solr/server/solr/$SOLR_CORE/conf/ +####################### +## GSA Specific Changes (also need root user) +####################### +# Add curl to access API routes +RUN apt-get update && apt-get install -y curl # Fix Issue https://github.com/GSA/datagov-deploy/issues/3285 # Update the angularjs library files @@ -74,12 +61,11 @@ RUN perl -0777 -i -pe 's/ \ /opt/solr/server/etc/webdefault.xml -# Create Core.properties -RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties - # Giving ownership to user 'solr' -RUN mkdir /opt/solr/server/solr/$SOLR_CORE/data/index +RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data/index RUN chown -R $SOLR_USER:$SOLR_USER /opt/solr/server/solr/ - +# # Set user to 'solr' for to comply with solr security USER solr + +# CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"] diff --git a/solr/currency.xml b/solr/currency.xml deleted file mode 100644 index 3a9c58af..00000000 --- a/solr/currency.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/solr/elevate.xml b/solr/elevate.xml deleted file mode 100644 index 2c09ebed..00000000 --- a/solr/elevate.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - diff --git a/solr/schema.xml b/solr/schema.xml deleted file mode 100644 index 1929d6dc..00000000 --- a/solr/schema.xml +++ /dev/null @@ -1,208 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -index_id -text - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/solr/service-config.json b/solr/service-config.json index 1c595ada..9e0c61fd 100644 --- a/solr/service-config.json +++ b/solr/service-config.json @@ -1,4 +1,4 @@ { "solrImageRepo": "ghcr.io/gsa/catalog.data.gov.solr", - "solrImageTag": "latest" -} \ No newline at end of file + "solrImageTag": "8-curl" +} diff --git a/solr/solrconfig.xml b/solr/solrconfig.xml deleted file mode 100644 index 9ac620c2..00000000 --- a/solr/solrconfig.xml +++ /dev/null @@ -1,343 +0,0 @@ - - - - - - 6.0.0 - - - - - - - - - - - ${solr.data.dir:} - - - - - - - ${solr.lock.type:native} - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - ${solr.autoCommit.maxTime:15000} - false - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - 1024 - - - - - true - 20 - 200 - - - - - - - - - - false - 2 - - - - - - - - - - - - - - explicit - 10 - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - - - _text_ - - - - - - - - add-unknown-fields-to-the-schema - - - - - - - - true - ignored_ - _text_ - - - - - - - - - - - explicit - true - - - - - - - text_general - - - default - _text_ - solr.DirectSolrSpellChecker - internal - 0.5 - 2 - 1 - 5 - 4 - 0.01 - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - true - false - - - - terms - - - - - - - string - elevate.xml - - - - - - - explicit - - - elevator - - - - - - - - - - - 100 - - - - - - 70 - 0.5 - [-\w ,/\n\"']{20,200} - - - - - - ]]> - ]]> - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - - 10 - .,!? - - - - - - WORD - en - US - - - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - text/plain; charset=UTF-8 - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - 5 - - -