[tests-only] CERNBox setup for ScienceMesh tests (#4391)

* Improved sciencemesh config * Reworked temporary deployment of cernbox web * Changelog * Fixes: the CERNBox web UI now almost works, not yet the invitations * Fixed meshdir for CERNBox to support invitations and removed duplicates * ocm provider authorizer: configuration to disable check takes precedence This is preliminary to be able to support OCM without ScienceMesh * Fixed public links provider and some other entries * Patched web link to meshdir * Revert for now the revad build without gaia * Further minor fixes
cs3org · Dec 20, 2023 · 8200a2a · 8200a2a
1 parent 26ebe9b
commit 8200a2a
Show file tree

Hide file tree

Showing 18 changed files with 106 additions and 91 deletions.
diff --git a/changelog/unreleased/sm-cernbox.md b/changelog/unreleased/sm-cernbox.md
@@ -0,0 +1,6 @@
+Enhancement: CERNBox setup for ScienceMesh tests
+
+This PR includes a bundled CERNBox-like web UI and backend
+to test the ScienceMesh workflows with OC10 and NC
+
+https://github.com/cs3org/reva/pull/4391
diff --git a/examples/cernbox/cernbox-extensions-bundle.tgz b/examples/cernbox/cernbox-extensions-bundle.tgz
diff --git a/examples/cernbox/cernbox.toml b/examples/cernbox/cernbox.toml
@@ -20,6 +20,7 @@ level = "debug"
 
 [shared]
 gatewaysvc = "{{ vars.internal_gateway }}:19000"
+jwt_secret = "reva-secret"
 
 [grpc.services.gateway]
 address = ":19000"
@@ -29,6 +30,7 @@ storageregistrysvc = "{{ grpc.services.storageregistry.address }}"
 preferencessvc = "{{ grpc.services.userprovider.address }}"
 userprovidersvc = "{{ grpc.services.userprovider.address }}"
 usershareprovidersvc = "{{ grpc.services.usershareprovider.address }}"
+publicshareprovidersvc = "{{ grpc.services.publicshareprovider.address }}"
 ocmcoresvc = "{{ grpc.services.ocmcore.address }}"
 ocmshareprovidersvc = "{{ grpc.services.ocmshareprovider.address }}"
 ocminvitemanagersvc = "{{ grpc.services.ocminvitemanager.address }}"
@@ -84,13 +86,14 @@ app_int_url = "http://collabora.docker:9980"
 # app_int_url = "https://codimd.docker"
 
 
-### AUTH PROVIDERS ###
+### AUTH ###
 
 [grpc.services.authregistry]
 driver = "static"
 
 [grpc.services.authregistry.drivers.static.rules]
 basic = "{{ grpc.services.authprovider[0].address }}"
+bearer = "{{ grpc.services.authprovider[0].address }}"
 machine = "{{ grpc.services.authprovider[1].address }}"
 ocmshares = "{{ grpc.services.authprovider[2].address }}"
 
@@ -117,16 +120,16 @@ gateway_addr = "{{ vars.internal_gateway }}:19000"
 auth_manager = "ocmshares"
 
 
-### STORAGE PROVIDERS ###
+### STORAGE ###
 
 [grpc.services.storageregistry]
 driver = "static"
 
 [grpc.services.storageregistry.drivers.static]
-home_provider = "/home"
+home_provider = "/"
 
 [grpc.services.storageregistry.drivers.static.rules]
-"/home" = {"address" = "{{ grpc.services.storageprovider[0].address }}"}
+"/" = {"address" = "{{ grpc.services.storageprovider[0].address }}"}
 "localhome" = {"address" = "{{ grpc.services.storageprovider[0].address }}"}
 "/ocm" = {"address" = "{{ grpc.services.storageprovider[1].address }}"}
 "ocm" = {"address" = "{{ grpc.services.storageprovider[1].address }}"}
@@ -135,11 +138,11 @@ home_provider = "/home"
 
 [[grpc.services.storageprovider]]
 driver = "localhome"
-mount_path = "/home"
+mount_path = "/"
 mount_id = "localhome"
 expose_data_server = true
 data_server_url = "https://localhost:{{ http.services.dataprovider[0].address.port }}/data"
-enable_home_creation = false
+enable_home_creation = true
 
 [grpc.services.storageprovider.drivers.localhome]
 user_layout = "{{.Username}}"
@@ -172,6 +175,8 @@ driver = "memory"
 [grpc.services.publicshareprovider]
 driver = "memory"
 
+[grpc.services.preferences]
+
 [grpc.services.ocmcore]
 driver = "json"
 
@@ -233,9 +238,15 @@ file = ""
 
 ### HTTP ENDPOINTS ###
 
+[http.middlewares.auth]
+credential_chain = ["publicshares", "basic", "bearer"]
+token_strategy_chain = ["bearer", "header"]
+
+[http.middlewares.auth.credentials_by_user_agent]
+"mirall" = "basic"
+
 [http.services.appprovider]
 address = ":443"
-insecure = true
 
 [http.services.datagateway]
 address = ":443"
@@ -259,7 +270,7 @@ driver = "ocmreceived"
 [http.services.sciencemesh]
 address = ":443"
 provider_domain = "{{ vars.provider_domain }}"
-mesh_directory_url = "https://sciencemesh.cesnet.cz/iop/meshdir"
+mesh_directory_url = "https:/meshdir.docker/meshdir"
 ocm_mount_point = "/sciencemesh"
 
 [http.services.sciencemesh.smtp_credentials]
@@ -376,6 +387,7 @@ string = "10.0.11"
 
 [http.services.ocdav]
 address = ":443"
+insecure = true
 
 [http.services.prometheus]
 address = ":443"
@@ -386,4 +398,10 @@ address = ":443"
 #address = ":443"
 
 [http.middlewares.cors]
+allowed_origins = ["*"]
+allowed_methods = ["OPTIONS", "LOCK", "GET", "HEAD", "POST", "DELETE", "PROPPATCH", "COPY", "MOVE", "UNLOCK", "PROPFIND", "MKCOL", "REPORT", "SEARCH", "PUT"]
+allowed_headers = ["Accept", "Accept-Language", "Authorization", "Content-Language", "Content-Type", "Depth", "OCS-APIREQUEST", "Referer", "sec-ch-ua", "sec-ch-ua-mobile", "sec-ch-ua-platform", "User-Agent", "X-Requested-With"]
+debug = true
+exposed_headers = []
+
 [http.middlewares.log]
diff --git a/examples/cernbox/custom-mime-types-demo.json b/examples/cernbox/custom-mime-types-demo.json
diff --git a/examples/cernbox/keycloak/cernbox.json b/examples/cernbox/keycloak/cernbox.json
@@ -636,7 +636,9 @@
       "redirectUris": [
         "/realms/cernbox/account/*"
       ],
-      "webOrigins": [],
+      "webOrigins": [
+        "*"
+      ],
       "notBefore": 0,
       "bearerOnly": false,
       "consentRequired": false,
@@ -680,7 +682,9 @@
       "redirectUris": [
         "/realms/cernbox/account/*"
       ],
-      "webOrigins": [],
+      "webOrigins": [
+        "*"
+      ],
       "notBefore": 0,
       "bearerOnly": false,
       "consentRequired": false,
@@ -815,8 +819,7 @@
         "https://cernbox2.docker/*"
       ],
       "webOrigins": [
-        "https://cernbox1.docker/*",
-        "https://cernbox2.docker/*"
+        "*"
       ],
       "notBefore": 0,
       "bearerOnly": false,
@@ -903,7 +906,7 @@
         "/admin/cernbox/console/*"
       ],
       "webOrigins": [
-        "+"
+        "*"
       ],
       "notBefore": 0,
       "bearerOnly": false,

diff --git a/examples/cernbox/nginx/nginx.conf b/examples/cernbox/nginx/nginx.conf
@@ -106,6 +106,10 @@ http {
       proxy_set_header Host $host;
     }
 
+    location ^~ /otg {
+      return 204;
+    }
+
     location ^~ /app/ {
       proxy_pass       https://revad;
       proxy_set_header Host $host;
@@ -139,7 +143,9 @@ http {
     }
 
     location ^~ /cernbox {
-      root /var/www/cernbox;
+      root /var/www;
+      add_header Cache-Control "no-cache";
+      add_header Access-Control-Allow-Origin "https://idp.docker:8443" always;
       etag off;
       gzip_static on;
     }
@@ -148,7 +154,7 @@ http {
       root /var/www/web;
       add_header Cache-Control "no-cache";
       add_header Access-Control-Allow-Origin "https://idp.docker:8443" always;
-      etag on;
+      etag off;
       gzip_static on;
       try_files $uri /index.html;
     }

diff --git a/examples/cernbox/providers.testnet.json b/examples/cernbox/providers.testnet.json
diff --git a/examples/cernbox/web-bundle.tgz b/examples/cernbox/web-bundle.tgz
diff --git a/examples/cernbox/web.json b/examples/cernbox/web.json
@@ -12,11 +12,10 @@
   "options": {
     "contextHelpers": true,
     "enableAdvancedTable": true,
-    "runningOnEos": true,
     "cernFeatures": true,
     "hoverableQuickActions": true,
     "disableFeedbackLink": true,
-    "homeFolder": "/home/{{.Id}}",
+    "homeFolder": "/{{.Id}}",
     "previewFileMimeTypes" : [
       "image/gif",
       "image/png",

diff --git a/examples/sciencemesh/providers.testnet.json b/examples/sciencemesh/providers.testnet.json
@@ -1,26 +1,34 @@
 [
-  {	"domain": "revad1.docker",	"services": [
+  {	"domain": "revad1.docker", "services": [
     { "endpoint": { "type": { "name": "OCM" }, "path": "https://revad1.docker/ocm/" }, "host": "revad1.docker" },
     { "endpoint": { "type": { "name": "Webdav" }, "path": "https://revad1.docker/remote.php/webdav/" }, "host": "revad1.docker" }
   ] },
   { "domain": "revad2.docker", "services": [
     { "endpoint": { "type": { "name": "OCM" }, "path": "https://revad2.docker/ocm/" }, "host": "revad2.docker" },
     { "endpoint": { "type": { "name": "Webdav" }, "path": "https://revad2.docker/remote.php/webdav/" }, "host": "revad2.docker" }
   ] },
-  {	"domain": "revanextcloud1.docker",	"services": [
+  {	"domain": "revanextcloud1.docker", "services": [
     { "endpoint": { "type": { "name": "OCM" }, "path": "https://revanextcloud1.docker/ocm/" }, "host": "revanextcloud1.docker" },
-    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://nc1.docker/remote.php/webdav/" }, "host": "nc1.docker" }
+    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://nc1.docker/remote.php/webdav/" }, "host": "nextcloud1.docker" }
   ] },
   { "domain": "revanextcloud2.docker", "services": [
     { "endpoint": { "type": { "name": "OCM" }, "path": "https://revanextcloud2.docker/ocm/" }, "host": "revanextcloud2.docker" },
-    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://nc2.docker/remote.php/webdav/" }, "host": "nc2.docker" }
+    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://nc2.docker/remote.php/webdav/" }, "host": "nextcloud2.docker" }
   ] },
-  {	"domain": "revaowncloud1.docker",	"services": [
+  {	"domain": "revaowncloud1.docker", "services": [
     { "endpoint": { "type": { "name": "OCM" }, "path": "https://revaowncloud1.docker/ocm/" }, "host": "revaowncloud1.docker" },
-    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://oc1.docker/remote.php/webdav/" }, "host": "oc1.docker" }
+    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://owncloud1.docker/remote.php/webdav/" }, "host": "owncloud1.docker" }
   ] },
   { "domain": "revaowncloud2.docker", "services": [
     { "endpoint": { "type": { "name": "OCM" }, "path": "https://revaowncloud2.docker/ocm/" }, "host": "revaowncloud2.docker" },
-    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://oc2.docker/remote.php/webdav/" }, "host": "oc2.docker" }
+    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://owncloud2.docker/remote.php/dav/" }, "host": "owncloud2.docker" }
+  ] },
+  {	"domain": "revacernbox1.docker", "services": [
+    { "endpoint": { "type": { "name": "OCM" }, "path": "https://revacernbox1.docker/ocm/" }, "host": "revacernbox1.docker" },
+    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://cernbox1.docker/remote.php/dav/" }, "host": "cernbox1.docker" }
+  ] },
+  { "domain": "revacernbox2.docker", "services": [
+    { "endpoint": { "type": { "name": "OCM" }, "path": "https://revacernbox2.docker/ocm/" }, "host": "revacernbox2.docker" },
+    { "endpoint": { "type": { "name": "Webdav" }, "path": "https://cernbox2.docker/remote.php/dav/" }, "host": "cernbox2.docker" }
   ] }
 ]
diff --git a/examples/sciencemesh/sciencemesh.toml b/examples/sciencemesh/sciencemesh.toml
@@ -161,8 +161,7 @@ driver = "nextcloud"
 provider_domain = "{{ vars.provider_domain }}"
 webdav_endpoint = "{{ vars.external_reva_endpoint }}"
 webdav_prefix = "{{ vars.external_reva_endpoint }}/remote.php/dav/files"
-# TODO the following should become {{ vars.external_reva_endpoint }}/external/{{.Token}}/...
-webapp_template = "https://your.revad.org/external/sciencemesh/{{.Token}}/{relative-path-to-shared-resource}"
+webapp_template = "{{ vars.external_reva_endpoint }}/external/sciencemesh/{{.Token}}/{relative-path-to-shared-resource}"
 
 [grpc.services.ocmshareprovider.drivers.nextcloud]
 webdav_host = "{{ vars.external_reva_endpoint }}"

diff --git a/pkg/ocm/provider/authorizer/json/json.go b/pkg/ocm/provider/authorizer/json/json.go
@@ -130,10 +130,10 @@ func (a *authorizer) IsProviderAllowed(ctx context.Context, pi *ocmprovider.Prov
 	}
 
 	switch {
-	case !providerAuthorized:
-		return errtypes.NotFound(pi.GetDomain())
 	case !a.conf.VerifyRequestHostname:
 		return nil
+	case !providerAuthorized:
+		return errtypes.NotFound(pi.GetDomain())
 	case len(pi.Services) == 0:
 		return errtypes.NotSupported("No IP provided")
 	}

diff --git a/pkg/ocm/provider/authorizer/mentix/mentix.go b/pkg/ocm/provider/authorizer/mentix/mentix.go
@@ -185,10 +185,10 @@ func (a *authorizer) IsProviderAllowed(ctx context.Context, pi *ocmprovider.Prov
 	}
 
 	switch {
-	case !providerAuthorized:
-		return errtypes.NotFound(pi.GetDomain())
 	case !a.conf.VerifyRequestHostname:
 		return nil
+	case !providerAuthorized:
+		return errtypes.NotFound(pi.GetDomain())
 	case len(pi.Services) == 0:
 		return errtypes.NotSupported(
 			fmt.Sprintf("mentix: provider %s has no supported services", pi.GetDomain()))

diff --git a/tests/sciencemesh/init.sh b/tests/sciencemesh/init.sh
@@ -9,9 +9,6 @@ BRANCH_NEXTCLOUD_APP=nextcloud
 REPO_OWNCLOUD_APP=https://github.com/sciencemesh/nc-sciencemesh
 BRANCH_OWNCLOUD_APP=owncloud
 
-# TODO will be dropped in favour of Reva directly serving the UI
-CBOX_WEB=https://github.com/cernbox/web-release/releases/latest/download
-
 REPO_WOPISERVER=https://github.com/cs3org/wopiserver
 TAG_WOPISERVER=master
 
@@ -45,27 +42,33 @@ TAG_WOPISERVER=master
     pondersource/dev-stock-owncloud-sciencemesh                                 \
     composer install
 
-# CERNBox web and extensions sources: uid=101 is nginx in the nginx container.
-# TODO the extensions are temporarily extracted from a tgz
-[ ! -d "cernbox-web-sciencemesh" ] &&                                           \
-    mkdir -p temp/cernbox-1-conf temp/cernbox-2-conf &&                         \
-    cp cernbox/nginx/* temp/cernbox-1-conf &&                                   \
-    cp cernbox/nginx/* temp/cernbox-2-conf &&                                   \
+# CERNBox web bundle (temporary, to be served by Reva in the future):
+# uid=101 is 'nginx' in the nginx container.
+[ ! -d "cernbox-web-sciencemesh" ] &&
     mkdir cernbox-web-sciencemesh &&                                            \
-    cd cernbox-web-sciencemesh &&
-    mkdir -p ./web && mkdir -p ./cernbox &&                                     \
-    wget ${CBOX_WEB}/web.tar.gz &&                                              \
-    tar xf web.tar.gz -C ./web --strip-components=1 &&                          \
-    rm -rf web.tar.gz &&                                                        \
-    tar xf ../cernbox/cernbox-extensions-bundle.tgz &&                          \
+    cd cernbox-web-sciencemesh &&                                               \
+    tar xf ../cernbox/web-bundle.tgz &&                                         \
+    cd web/js && sed -i "s|sciencemesh\.cesnet\.cz\/iop|meshdir\.docker|"       \
+           web-app-science*mjs &&                                               \
+    rm web-app-science*mjs.gz && gzip web-app-science*mjs &&                    \
+    cd ../.. &&                                                                 \
     chmod -R 755 ./* && chown -R 101:101 ./* &&                                 \
-    cd -
+    cd ..
 
 # wopiserver source code for the config.
-[ ! -d "wopi-sciencemesh" ] &&                                                       \
-    git clone --branch ${TAG_WOPISERVER} ${REPO_WOPISERVER} wopi-sciencemesh &&      \
-    mkdir -p temp/wopi-1-conf temp/wopi-2-conf &&                                    \
-    cp wopi-sciencemesh/wopiserver.conf temp/wopi-1-conf/wopiserver.defaults.conf && \
-    echo "shared-secret-2" > temp/wopi-1-conf/iopsecret &&                           \
-    echo "wopisecret" > temp/wopi-1-conf/wopisecret &&                               \
-    cp temp/wopi-1-conf/* temp/wopi-2-conf/
+[ ! -d "wopi-sciencemesh" ] &&                                                  \
+    git clone --branch ${TAG_WOPISERVER} ${REPO_WOPISERVER} wopi-sciencemesh    \
+
+# Runtime configurations for WOPI and CERNBox.
+[ ! -d "temp" ] &&                                                              \
+    mkdir -p temp/cernbox-1-conf temp/cernbox-2-conf &&                         \
+    cp cernbox/nginx/* temp/cernbox-1-conf &&                                   \
+    cp cernbox/nginx/* temp/cernbox-2-conf &&                                   \
+    mkdir -p temp/wopi-1-conf temp/wopi-2-conf &&                               \
+    cp wopi-sciencemesh/wopiserver.conf                                         \
+       temp/wopi-1-conf/wopiserver.defaults.conf &&                             \
+    echo "shared-secret-2" > temp/wopi-1-conf/iopsecret &&                      \
+    echo "wopisecret" > temp/wopi-1-conf/wopisecret &&                          \
+    cp temp/wopi-1-conf/* temp/wopi-2-conf/ &&                                  \
+    echo "temp folder for runtime configurations created"
+
diff --git a/tests/sciencemesh/scripts/build-reva.sh b/tests/sciencemesh/scripts/build-reva.sh
@@ -5,5 +5,7 @@ set -e
 git config --global --add safe.directory /reva
 # go mod tidy
 go mod vendor
+#make gaia
+#gaia build --with github.com/cernbox/reva-ocweb-plugin --with github.com/cs3org/reva=$(shell pwd) -o ./cmd/revad/revad
 make revad
 make reva