From b4e26c67cf99356bc1ab62879356e0ad0ba63527 Mon Sep 17 00:00:00 2001 From: MajoBerger Date: Fri, 23 Jun 2023 15:50:22 +0200 Subject: [PATCH 01/10] separate ports for multiple instance run --- docker/cli.yml | 10 ++++++---- docker/docker-compose-rest.yml | 29 ++++++++++++++--------------- docker/docker-compose.yml | 14 +++++++------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/docker/cli.yml b/docker/cli.yml index 56baf577efd..d568b50b65a 100644 --- a/docker/cli.yml +++ b/docker/cli.yml @@ -16,7 +16,7 @@ version: "3.7" services: dspace-cli: - image: "${DOCKER_OWNER:-dspace}/dspace-cli:${DSPACE_VER:-dspace-7_x}" + image: "${DOCKER_OWNER:-dataquest}/dspace-cli:${DSPACE_VER:-dspace-7_x}" container_name: dspace-cli environment: # Below syntax may look odd, but it is how to override dspace.cfg settings via env variables. @@ -26,10 +26,12 @@ services: # dspace.dir dspace__P__dir: /dspace # db.url: Ensure we are using the 'dspacedb' image for our database - db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace' + db__P__url: 'jdbc:postgresql://dspacedb:543${INSTANCE}/dspace' # solr.server: Ensure we are using the 'dspacesolr' image for Solr - solr__P__server: http://dspacesolr:8983/solr + solr__P__server: http://dspacesolr:898${INSTANCE}/solr + + # S3 assetstore__P__index__P__primary: ${S3_STORAGE:-0} assetstore__P__s3__P__enabled: ${S3_ENABLED:-false} assetstore__P__s3__P__useRelativePath: ${S3_RELATIVE_PATH:-false} @@ -37,7 +39,7 @@ services: assetstore__P__s3__P__subfolder: ${S3_SUBFOLDER:-} assetstore__P__s3__P__awsAccessKey: ${S3_ACCESS:-} assetstore__P__s3__P__awsSecretKey: ${S3_SECRET:-} - assetstore__P__s3__P__awsRegionName: ${S3_REGION_NAME:-} + assetstore__P__s3__P__awsRegionName: ${S3_REGION_NAME:-} volumes: - "assetstore:/dspace/assetstore" diff --git a/docker/docker-compose-rest.yml b/docker/docker-compose-rest.yml index e0998f15c3b..3c7d586139a 100644 --- a/docker/docker-compose-rest.yml +++ b/docker/docker-compose-rest.yml @@ -17,7 +17,7 @@ services: # DSpace (backend) webapp container dspace: restart: unless-stopped - container_name: dspace + container_name: dspace${INSTANCE} environment: # Below syntax may look odd, but it is how to override dspace.cfg settings via env variables. # See https://github.com/DSpace/DSpace/blob/main/dspace/config/config-definition.xml @@ -29,10 +29,9 @@ services: dspace__P__ui__P__url: ${UI_URL:-http://127.0.0.1:4000} dspace__P__name: 'DSpace Started with Docker Compose' # db.url: Ensure we are using the 'dspacedb' image for our database - db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace' + db__P__url: 'jdbc:postgresql://dspacedb:543${INSTANCE}/dspace' # solr.server: Ensure we are using the 'dspacesolr' image for Solr - solr__P__server: http://dspacesolr:8983/solr - + solr__P__server: http://dspacesolr:898${INSTANCE}/solr #S3 config assetstore__P__index__P__primary: ${S3_STORAGE:-0} assetstore__P__s3__P__enabled: ${S3_ENABLED:-false} @@ -42,16 +41,15 @@ services: assetstore__P__s3__P__awsAccessKey: ${S3_ACCESS:-} assetstore__P__s3__P__awsSecretKey: ${S3_SECRET:-} assetstore__P__s3__P__awsRegionName: ${S3_REGION_NAME:-} - image: ${DSPACE_REST_IMAGE:-dataquest/dspace:dtq-dev-7.5} depends_on: - dspacedb networks: dspacenet: ports: - - published: 8080 + - published: 808${INSTANCE} target: 8080 - - published: 8000 + - published: 800${INSTANCE} target: 8000 stdin_open: true tty: true @@ -67,29 +65,30 @@ services: - /bin/bash - '-c' - | - while (! /dev/null 2>&1; do sleep 1; done; + while (! /dev/null 2>&1; do sleep 1; done; /dspace/bin/dspace database migrate force custom_run.sh # DSpace database container dspacedb: restart: unless-stopped - container_name: dspacedb + container_name: dspacedb${INSTANCE} environment: PGDATA: /pgdata image: dspace/dspace-postgres-pgcrypto networks: dspacenet: ports: - - published: 5432 - target: 5432 + - published: 543${INSTANCE} + target: 543${INSTANCE} stdin_open: true tty: true volumes: - pgdata:/pgdata + command: -p 543${INSTANCE} # DSpace Solr container dspacesolr: restart: unless-stopped - container_name: dspacesolr + container_name: dspacesolr${INSTANCE} # Uses official Solr image at https://hub.docker.com/_/solr/ image: solr:8.11-slim # Needs main 'dspace' container to start first to guarantee access to solr_configs @@ -98,8 +97,8 @@ services: networks: dspacenet: ports: - - published: 8983 - target: 8983 + - published: 898${INSTANCE} + target: 898${INSTANCE} stdin_open: true tty: true working_dir: /var/solr/data @@ -126,7 +125,7 @@ services: cp -r -u /opt/solr/server/solr/configsets/dspace/search/* search precreate-core statistics /opt/solr/server/solr/configsets/dspace/statistics cp -r -u /opt/solr/server/solr/configsets/dspace/statistics/* statistics - exec solr -f -m 4g + exec solr -p 898${INSTANCE} -f -m 4g volumes: assetstore: pgdata: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b223d9276b0..f81f0ad4be7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -15,15 +15,15 @@ networks: services: dspace-angular: restart: unless-stopped - container_name: dspace-angular + container_name: dspace-angular${INSTANCE} environment: DSPACE_UI_SSL: 'false' DSPACE_UI_HOST: dspace-angular - DSPACE_UI_PORT: '4000' + DSPACE_UI_PORT: 400${INSTANCE} DSPACE_UI_NAMESPACE: / DSPACE_REST_SSL: 'false' DSPACE_REST_HOST: ${DSPACE_REST_HOST:-localhost} - DSPACE_REST_PORT: 8080 + DSPACE_REST_PORT: 808${INSTANCE} DSPACE_REST_NAMESPACE: /server image: ${DSPACE_UI_IMAGE:-dataquest/dspace-angular:dspace-7_x} build: @@ -32,9 +32,9 @@ services: networks: dspacenet: ports: - - published: 4000 - target: 4000 - - published: 9876 - target: 9876 + - published: 400${INSTANCE} + target: 400${INSTANCE} + - published: 987${INSTANCE} + target: 987${INSTANCE} stdin_open: true tty: true From 426b4386f73c9f22476f9db388d1b9a6ce584c75 Mon Sep 17 00:00:00 2001 From: MajoBerger Date: Mon, 26 Jun 2023 11:41:00 +0200 Subject: [PATCH 02/10] modified deploy --- .github/workflows/deploy.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7c1eae926c5..69d904d3400 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -25,12 +25,22 @@ jobs: touch .env.dev-5 || true echo DSPACE_REST_HOST=dev-5.pc > .env.dev-5 - echo REST_URL=http://dev-5.pc:8080/server >> .env.dev-5 - echo UI_URL=http://dev-5.pc >> .env.dev-5 - echo DSPACE_REST_IMAGE=dataquest/dspace:dspace-7_x >> .env.dev-5 + echo REST_URL=http://dev-5.pc:8085/server >> .env.dev-5 + echo UI_URL=http://dev-5.pc:4005 >> .env.dev-5 + echo DSPACE_REST_IMAGE=dataquest/dspace:dtq-dev-7.5 >> .env.dev-5 echo DOCKER_OWNER=dataquest >> .env.dev-5 ## echo NODE_ENV="development" >> .env.dev-5 - echo DSPACE_UI_IMAGE="dataquest/dspace-angular:dspace-7_x" >> .env.dev-5 + echo DSPACE_UI_IMAGE="dataquest/dspace-angular:dtq-dev-7.5" >> .env.dev-5 + echo INSTANCE=5 >> .env.dev-5 + + echo S3_STORAGE=1 >> .env.dev-5 + echo S3_ENABLED=true >> .env.dev-5 + echo S3_RELATIVE_PATH=true >> .env.dev-5 + echo S3_BUCKET=dev-5 >> .env.dev-5 + echo S3_SUBFOLDER=seven-five >> .env.dev-5 + echo S3_ACCESS=${{ secrets.AMAZON_ACCESS }} >> .env.dev-5 + echo S3_SECRET=${{ secrets.AMAZON_SECRET }} >> .env.dev-5 + echo S3_REGION_NAME=us-east-1 >> .env.dev-5 export ENVFILE=$(pwd)/.env.dev-5 ./start.sh From 23846330e6e2c12877befa50eac39045a52ed885 Mon Sep 17 00:00:00 2001 From: MajoBerger Date: Mon, 26 Jun 2023 13:19:46 +0200 Subject: [PATCH 03/10] parametrized dspace project name in docker compose --- .github/workflows/deploy.yml | 2 +- build-scripts/run/start.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 69d904d3400..82fe00aa36f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,4 +43,4 @@ jobs: echo S3_REGION_NAME=us-east-1 >> .env.dev-5 export ENVFILE=$(pwd)/.env.dev-5 - ./start.sh + ./start.sh dspace-5 diff --git a/build-scripts/run/start.sh b/build-scripts/run/start.sh index 0aa268964e4..792fe94ab8a 100755 --- a/build-scripts/run/start.sh +++ b/build-scripts/run/start.sh @@ -11,15 +11,15 @@ docker pull $DSPACE_UI_IMAGE pushd ../.. docker-compose --env-file $ENVFILE -f docker/docker-compose.yml -f docker/docker-compose-rest.yml pull -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build +docker-compose --env-file $ENVFILE ${1:-unnamed_dspace} -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build popd # Create admin user # set DOCKER_OWNER to match our image (see cli.yml) pushd ../.. -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/matomo-w-db.yml pull -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/matomo-w-db.yml up -d --no-build -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en -o dataquest -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/cli.yml run --rm dspace-cli user --add -m user@test.edu -g meno -s priezvisko -l en -p user -o dataquest -docker-compose --env-file $ENVFILE -p dq-d7 -f docker/cli.yml run --rm dspace-cli version +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/matomo-w-db.yml pull +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/matomo-w-db.yml up -d --no-build +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en -o dataquest +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli user --add -m user@test.edu -g meno -s priezvisko -l en -p user -o dataquest +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/cli.yml run --rm dspace-cli version popd From eab71120d13ac86d417693706aadc03bd9a634d8 Mon Sep 17 00:00:00 2001 From: MajoBerger Date: Mon, 26 Jun 2023 14:24:06 +0200 Subject: [PATCH 04/10] fixed .env for deployment on dev-5 --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 82fe00aa36f..741e60c8d7f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,7 +26,7 @@ jobs: touch .env.dev-5 || true echo DSPACE_REST_HOST=dev-5.pc > .env.dev-5 echo REST_URL=http://dev-5.pc:8085/server >> .env.dev-5 - echo UI_URL=http://dev-5.pc:4005 >> .env.dev-5 + echo UI_URL=http://dev-5.pc:85 >> .env.dev-5 echo DSPACE_REST_IMAGE=dataquest/dspace:dtq-dev-7.5 >> .env.dev-5 echo DOCKER_OWNER=dataquest >> .env.dev-5 ## echo NODE_ENV="development" >> .env.dev-5 From b70d48a3ac58523a9b3661a164a12a6d59520acf Mon Sep 17 00:00:00 2001 From: MajoBerger Date: Mon, 26 Jun 2023 14:28:15 +0200 Subject: [PATCH 05/10] fix forgotten -p parameter on deploy --- build-scripts/run/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-scripts/run/start.sh b/build-scripts/run/start.sh index 792fe94ab8a..1e23583e0bb 100755 --- a/build-scripts/run/start.sh +++ b/build-scripts/run/start.sh @@ -11,7 +11,7 @@ docker pull $DSPACE_UI_IMAGE pushd ../.. docker-compose --env-file $ENVFILE -f docker/docker-compose.yml -f docker/docker-compose-rest.yml pull -docker-compose --env-file $ENVFILE ${1:-unnamed_dspace} -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build +docker-compose --env-file $ENVFILE -p ${1:-unnamed_dspace} -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --no-build popd # Create admin user From ab6d92fca7acc6174c6bd73d5e6b017809a31b9d Mon Sep 17 00:00:00 2001 From: MajoBerger Date: Mon, 26 Jun 2023 14:46:30 +0200 Subject: [PATCH 06/10] fixed region --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 741e60c8d7f..b243ec141ce 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -40,7 +40,7 @@ jobs: echo S3_SUBFOLDER=seven-five >> .env.dev-5 echo S3_ACCESS=${{ secrets.AMAZON_ACCESS }} >> .env.dev-5 echo S3_SECRET=${{ secrets.AMAZON_SECRET }} >> .env.dev-5 - echo S3_REGION_NAME=us-east-1 >> .env.dev-5 + echo S3_REGION_NAME=eu-central-1 >> .env.dev-5 export ENVFILE=$(pwd)/.env.dev-5 ./start.sh dspace-5 From eac108fa34a1ae68736d3180a6e3b29079776708 Mon Sep 17 00:00:00 2001 From: MajoBerger Date: Mon, 26 Jun 2023 14:52:19 +0200 Subject: [PATCH 07/10] fixed bucket name --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b243ec141ce..1367ce6c7cb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -36,7 +36,7 @@ jobs: echo S3_STORAGE=1 >> .env.dev-5 echo S3_ENABLED=true >> .env.dev-5 echo S3_RELATIVE_PATH=true >> .env.dev-5 - echo S3_BUCKET=dev-5 >> .env.dev-5 + echo S3_BUCKET=dev-5-dspace >> .env.dev-5 echo S3_SUBFOLDER=seven-five >> .env.dev-5 echo S3_ACCESS=${{ secrets.AMAZON_ACCESS }} >> .env.dev-5 echo S3_SECRET=${{ secrets.AMAZON_SECRET }} >> .env.dev-5 From c393891389531fd8b46dbffafb8ecf138e99929e Mon Sep 17 00:00:00 2001 From: MajoBerger <88670521+MajoBerger@users.noreply.github.com> Date: Tue, 27 Jun 2023 10:39:08 +0200 Subject: [PATCH 08/10] add DSPACE_VER variable to deploy --- .github/workflows/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1367ce6c7cb..f5964210790 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -31,6 +31,7 @@ jobs: echo DOCKER_OWNER=dataquest >> .env.dev-5 ## echo NODE_ENV="development" >> .env.dev-5 echo DSPACE_UI_IMAGE="dataquest/dspace-angular:dtq-dev-7.5" >> .env.dev-5 + echo DSPACE_VER="dtq-dev-7.5" >> .env.dev-5 echo INSTANCE=5 >> .env.dev-5 echo S3_STORAGE=1 >> .env.dev-5 From d06988f2e2e89b2aaa5180fa7f7dcff15ea5a6d5 Mon Sep 17 00:00:00 2001 From: MajoBerger <88670521+MajoBerger@users.noreply.github.com> Date: Tue, 27 Jun 2023 12:50:57 +0200 Subject: [PATCH 09/10] remove automatic deploy --- .github/workflows/deploy.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f5964210790..23f58d746e3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,13 +2,6 @@ name: Deploy DSpace on: - workflow_run: - workflows: ["Docker images", "Erase database"] - types: - - completed - branches: - - dtq-dev - - dtq-dev-present workflow_dispatch: jobs: From 2c241a5fc744086df3920244a14468e4bd1ea6d8 Mon Sep 17 00:00:00 2001 From: MajoBerger <88670521+MajoBerger@users.noreply.github.com> Date: Tue, 27 Jun 2023 14:06:13 +0200 Subject: [PATCH 10/10] unify erasing db --- .github/workflows/erase_db.yml | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/erase_db.yml b/.github/workflows/erase_db.yml index 2fcf17f473e..30b5dfb06fb 100644 --- a/.github/workflows/erase_db.yml +++ b/.github/workflows/erase_db.yml @@ -4,26 +4,24 @@ on: workflow_dispatch: jobs: - deploy: + restart_db: + env: + INSTANCE: 5 runs-on: dspace-dep-1 steps: - uses: actions/checkout@v3 with: submodules: false - - name: deploy + - name: stop and remove containers run: | - cd $GITHUB_WORKSPACE/build-scripts/run/ - pwd - touch .env.dev-5 || true - echo DSPACE_REST_HOST=dev-5.pc > .env.dev-5 - echo REST_URL=http://dev-5.pc:8080/server >> .env.dev-5 - echo UI_URL=http://dev-5.pc >> .env.dev-5 - echo DSPACE_REST_IMAGE=dataquest/dspace:dspace-7_x >> .env.dev-5 - echo DOCKER_OWNER=dataquest >> .env.dev-5 - ## echo NODE_ENV="development" >> .env.dev-5 - echo DSPACE_UI_IMAGE="dataquest/dspace-angular:dspace-7_x" >> .env.dev-5 - - export ENVFILE=$(pwd)/.env.dev-5 - docker stop dspacedb dspace dspace-angular - docker rm dspacedb dspace dspace-angular - docker volume rm dq-d7_pgdata + docker stop dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE + docker rm dspacedb$INSTANCE dspace$INSTANCE dspace-angular$INSTANCE + - name: remove volumes + if: '!cancelled()' + run: | + docker volume rm dspace-${{env.INSTANCE}}_pgdata dspace-${{env.INSTANCE}}_assetstore + - name: redeploy + if: '!cancelled()' + run: | + curl -H "Accept: application/vnd.github.everest-preview+json" -H "Authorization: token ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }}" --request POST https://api.github.com/repos/dataquest-dev/dspace-angular/actions/workflows/deploy.yml/dispatches --data "{\"ref\":\"refs/heads/internal/allow-several-concurrent-instances-in-docker\"}" +