From 190520678edfd06d95447852b4ace1288c0b36e7 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 18 Sep 2024 17:16:06 -0400 Subject: [PATCH] Update to newer smui + new SMUI features (#168) * it's 2024, it's time to ditch docker-compose * now showing rules usage Signed-off-by: Eric Pugh --- README.md | 10 +++++----- TECHNICAL_DETAILS.md | 2 +- docker-compose.yml | 10 ++++++---- embeddings/app/main.py | 6 +++--- embeddings/requirements.txt | 1 + helpers.sh | 6 +++--- katas/000_setting_up_chorus.md | 6 +++--- katas/012_vector_search.md | 2 +- keycloak/Dockerfile | 2 +- quickstart.sh | 14 +++++++------- smui/Dockerfile | 2 +- smui/data/rules-usage-stats.csv | 4 ++++ 12 files changed, 36 insertions(+), 29 deletions(-) create mode 100644 smui/data/rules-usage-stats.csv diff --git a/README.md b/README.md index 3a8bb55..b7128c9 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ There is also a video series that is very closely related called [Meet Pete](htt To start your environment, but still run each command to setup the integrations manually, run: ``` -docker-compose up --build -d +docker compose up --build -d ``` The quickstart command will launch a Solr cluster, load the configsets and product data for the _ecommerce_ index, and launch the SMUI user interface: @@ -115,18 +115,18 @@ To include the observability features, run: To see what is happening in the Chorus stack you can tail the logs for all the components via: ``` -docker-compose logs -tf +docker compose logs -tf ``` If you want to narrow down to just one component of the Chorus stack do: ``` -docker-compose ps # list out the names of the components -docker-compose logs -tf solr1 solr2 # tail solr1 and solr2 only +docker compose ps # list out the names of the components +docker compose logs -tf solr1 solr2 # tail solr1 and solr2 only ``` To destroy your environment (including any volumes created like the mysql db), just run: ``` -docker-compose down -v +docker compose down -v ``` or diff --git a/TECHNICAL_DETAILS.md b/TECHNICAL_DETAILS.md index 864faa3..58e3b32 100644 --- a/TECHNICAL_DETAILS.md +++ b/TECHNICAL_DETAILS.md @@ -43,7 +43,7 @@ curl http://localhost:8983/solr/ecommerce/config -H 'Content-type:application/js Create a user with the email `demo@example.com` and the password `password`: ```sh -docker-compose run quepid bundle exec thor user:create -a demo@example.com "Demo User" password +docker compose run quepid bundle exec thor user:create -a demo@example.com "Demo User" password ``` For Quepid case, pick `name` for title, and `id` for identifier. Add `thumb:imageUrl` to the list of fields. diff --git a/docker-compose.yml b/docker-compose.yml index e4a1c90..2463619 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: blacklight: @@ -58,6 +57,9 @@ services: - SMUI_TOGGLE_EVENTHISTORY=true - SMUI_RULE_TAGGING_ACTIVE=true - SMUI_PREDEFINED_TAGS_FILE=/smui/conf/predefined_tags.json + - SMUI_RULE_USAGE_STATISTICS_LOCATION=file:///smui/data/rules-usage-stats.csv + volumes: + - ./smui/data/:/smui/data depends_on: - mysql @@ -125,7 +127,7 @@ services: - keycloak zoo1: - image: zookeeper:3.7.0 + image: zookeeper:3.9.2 container_name: zoo1 hostname: zoo1 ports: @@ -139,7 +141,7 @@ services: zoo2: - image: zookeeper:3.7.0 + image: zookeeper:3.9.2 container_name: zoo2 hostname: zoo2 ports: @@ -153,7 +155,7 @@ services: zoo3: - image: zookeeper:3.7.0 + image: zookeeper:3.9.2 container_name: zoo3 hostname: zoo3 ports: diff --git a/embeddings/app/main.py b/embeddings/app/main.py index 27ea6ff..f0da4eb 100644 --- a/embeddings/app/main.py +++ b/embeddings/app/main.py @@ -1,11 +1,11 @@ from fastapi import FastAPI from clipL14 import router as router_clip from minilm import router as router_minilm -from finetuned import router as router_finetuned +#from finetuned import router as router_finetuned app = FastAPI() app.include_router(router_clip.router, prefix="/clip") app.include_router(router_minilm.router, prefix="/minilm") -app.include_router(router_finetuned.sbert_router, prefix="/ftbert") -app.include_router(router_finetuned.clip_router, prefix="/ftclip") +#app.include_router(router_finetuned.sbert_router, prefix="/ftbert") +#app.include_router(router_finetuned.clip_router, prefix="/ftclip") diff --git a/embeddings/requirements.txt b/embeddings/requirements.txt index 7241179..56b84c3 100644 --- a/embeddings/requirements.txt +++ b/embeddings/requirements.txt @@ -1,3 +1,4 @@ +docarray==0.21.0 # See https://github.com/querqy/chorus/issues/169 sentence-transformers fastapi uvicorn[standard] diff --git a/helpers.sh b/helpers.sh index 941d5a0..8986b85 100644 --- a/helpers.sh +++ b/helpers.sh @@ -58,7 +58,7 @@ function log_prerequisite_missing() { function check_prerequisites() { set +e - PREREQUISITES=("curl" "jq" "docker-compose" "zip") + PREREQUISITES=("curl" "jq" "docker" "zip") log_major "Checking availability of prerequisites..." COUNT_MISSING_PREQUISITES=0 for PREREQUISITE in "${PREREQUISITES[@]}" @@ -81,10 +81,10 @@ function check_prerequisites() { } function display_help_message() { - echo -e "Use the option --with-offline-lab | -lab to include Quepid and RRE services in Chorus." + echo -e "Use the option --with-offline-lab | -lab to include Quepid in Chorus." echo -e "Use the option --with-observability | -obs to include Grafana, Prometheus, and Solr Exporter services in Chorus." + echo -e "Use the option --with-active-search-management | -active to include Active Search Management via SMUI in Chorus." echo -e "Use the option --with-vector-search | -vector to include Vector Search services in Chorus." - echo -e "Use the option --with-active-search-management | -active to include Active Search Management in Chorus." echo -e "Use the option --shutdown | -s to shutdown and remove the Docker containers and data." echo -e "Use the option --online-deployment | -online to update configuration to run on chorus.dev.o19s.com environment." } diff --git a/katas/000_setting_up_chorus.md b/katas/000_setting_up_chorus.md index c5e83ca..c2b3cf1 100644 --- a/katas/000_setting_up_chorus.md +++ b/katas/000_setting_up_chorus.md @@ -7,7 +7,7 @@ We use a Docker Compose based environment to manage firing up all the components Open up a terminal window and run: ```sh -docker-compose up --build +docker compose up --build ``` Wait a while, because you'll be downloading and building quite a few images! You may think it's frozen at various points, but go for a walk and come back and it'll be up and running. @@ -139,13 +139,13 @@ We'll start with Quepid and then move on to RRE. First we need to create the database for Quepid: ```sh -docker-compose run --rm quepid bin/rake db:setup +docker compose run --rm quepid bin/rake db:setup ``` We also need to create you an account with Administrator permissions: ```sh -docker-compose run quepid bundle exec thor user:create -a admin@choruselectronics.com "Chorus Admin" password +docker compose run quepid bundle exec thor user:create -a admin@choruselectronics.com "Chorus Admin" password ``` Visit Quepid at http://localhost:3000 and log in with the email and password you just set up. diff --git a/katas/012_vector_search.md b/katas/012_vector_search.md index 85ff579..fc79309 100644 --- a/katas/012_vector_search.md +++ b/katas/012_vector_search.md @@ -55,7 +55,7 @@ Also, notice that this also triggers an additional 'embedding service' based on ```bash if $vector_search; then log_major "Setting up Embeddings service" - docker-compose up -d --build embeddings + docker compose up -d --build embeddings ./embeddings/wait-for-api.sh fi ``` diff --git a/keycloak/Dockerfile b/keycloak/Dockerfile index a81bfc8..4f4f857 100644 --- a/keycloak/Dockerfile +++ b/keycloak/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/keycloak/keycloak:20.0.5 as builder +FROM quay.io/keycloak/keycloak:20.0.5 AS builder # Enable health and metrics support ENV KC_HEALTH_ENABLED=true diff --git a/quickstart.sh b/quickstart.sh index 2d27544..22f2a40 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -51,7 +51,7 @@ do shift done -# Function check_prerequisites makes sure that you have curl, jq, docker-compose, and zip installed. See helpers.sh for details. +# Function check_prerequisites makes sure that you have curl, jq, docker, and zip installed. See helpers.sh for details. check_prerequisites services="blacklight solr1 solr2 solr3 keycloak" @@ -91,7 +91,7 @@ if ! $local_deploy; then sed -i.bu 's/keycloak:9080/chorus.dev.o19s.com:9080/g' ./docker-compose.yml fi -docker-compose down -t 30 -v +docker compose down -t 30 -v if $shutdown; then exit fi @@ -105,7 +105,7 @@ if $active_search_management; then fi -docker-compose up -d --build ${services} +docker compose up -d --build ${services} echo -e "${MAJOR}Waiting for Solr cluster to start up and all three nodes to be online.${RESET}" ./solr/wait-for-solr-cluster.sh # Wait for all three Solr nodes to be online @@ -400,15 +400,15 @@ if $offline_lab; then log_major "Setting up Quepid" ./mysql/wait-for-mysql.sh - docker-compose run --rm quepid bundle exec bin/rake db:setup - docker-compose run quepid bundle exec thor user:create -a admin@choruselectronics.com "Chorus Admin" password + docker compose run --rm quepid bundle exec bin/rake db:setup + docker compose run quepid bundle exec thor user:create -a admin@choruselectronics.com "Chorus Admin" password log_minor "Setting up Chorus Baseline Relevance case" if $local_deploy; then solr_collection_url=http://localhost:8983/solr/ecommerce/select else solr_collection_url=http://chorus.dev.o19s.com:8983/solr/ecommerce/select fi - docker-compose run quepid bundle exec thor case:create "Chorus Baseline Relevance" solr ${solr_collection_url} JSONP "id:id, title:title, thumb:img_500x500, name, brand, product_type" "q=#\$query##&useParams=visible_products,querqy_algo" nDCG@10 admin@choruselectronics.com + docker compose run quepid bundle exec thor case:create "Chorus Baseline Relevance" solr ${solr_collection_url} JSONP "id:id, title:title, thumb:img_500x500, name, brand, product_type" "q=#\$query##&useParams=visible_products,querqy_algo" nDCG@10 admin@choruselectronics.com docker cp ./katas/Broad_Query_Set_rated.csv quepid:/srv/app/Broad_Query_Set_rated.csv docker exec quepid bundle exec thor ratings:import 1 /srv/app/Broad_Query_Set_rated.csv >> /dev/null @@ -424,7 +424,7 @@ fi if $vector_search; then log_major "Setting up Embeddings service" - docker-compose up -d --build embeddings + docker compose up -d --build embeddings ./embeddings/wait-for-api.sh fi diff --git a/smui/Dockerfile b/smui/Dockerfile index 2eb14d3..f1fa28c 100644 --- a/smui/Dockerfile +++ b/smui/Dockerfile @@ -1,4 +1,4 @@ -FROM querqy/smui:4.0.11 +FROM querqy/smui:4.3.0 COPY --chown=smui:smui conf/* conf/ diff --git a/smui/data/rules-usage-stats.csv b/smui/data/rules-usage-stats.csv new file mode 100644 index 0000000..21655b2 --- /dev/null +++ b/smui/data/rules-usage-stats.csv @@ -0,0 +1,4 @@ +SMUI_GUID,USER_QUERY,FREQUENCY +c4b10dea-1cd5-4361-83c6-d5c72691e343,laptop,6 +c4b10dea-1cd5-4361-83c6-d5c72691e343,notebook,2 +7bc66070-7d88-43d4-8d09-0dd0210c4ba3,laptop,20