Merge pull request #5089 from wkloucek/wopi-validator

[full-ci] run the wopivalidator in CI

.drone.star

@@ -96,6 +96,10 @@ config = {
         "skip": False,
         "earlyFail": True,
     },
+    "wopiValidatorTests": {
+        "skip": False,
+        "earlyFail": True,
+    },
     "localApiTests": {
         "skip": False,
         "earlyFail": True,
@@ -343,6 +347,8 @@ def testPipelines(ctx):
 
     if "skip" not in config["cs3ApiTests"] or not config["cs3ApiTests"]["skip"]:
         pipelines.append(cs3ApiTests(ctx, "ocis", "default"))
+    if "skip" not in config["wopiValidatorTests"] or not config["wopiValidatorTests"]["skip"]:
+        pipelines.append(wopiValidatorTests(ctx, "ocis", "default"))
     if "skip" not in config["localApiTests"] or not config["localApiTests"]["skip"]:
         pipelines += [
             localApiTests(ctx, "ocis", "apiAccountsHashDifficulty"),
@@ -750,6 +756,100 @@ def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4):
         },
     }
 
+def wopiValidatorTests(ctx, storage, accounts_hash_difficulty = 4):
+    early_fail = config["wopiValidatorTests"]["earlyFail"] if "earlyFail" in config["wopiValidatorTests"] else False
+
+    testgroups = [
+        "BaseWopiViewing",
+        "CheckFileInfoSchema",
+        "EditFlows",
+        "Locks",
+        "AccessTokens",
+        "GetLock",
+        "ExtendedLockLength",
+        "FileVersion",
+        "Features",
+    ]
+
+    validatorTests = []
+
+    for testgroup in testgroups:
+        validatorTests.append({
+            "name": "wopiValidatorTests-%s-%s" % (storage, testgroup),
+            "image": "owncloudci/wopi-validator",
+            "commands": [
+                "export WOPI_TOKEN=$(cat accesstoken)",
+                "echo $WOPI_TOKEN",
+                "export WOPI_TTL=$(cat accesstokenttl)",
+                "echo $WOPI_TTL",
+                "export WOPI_SRC=$(cat wopisrc)",
+                "echo $WOPI_SRC",
+                "cd /app",
+                "/app/Microsoft.Office.WopiValidator -s -t $WOPI_TOKEN -w $WOPI_SRC -l $WOPI_TTL --testgroup %s" % testgroup,
+            ],
+        })
+
+    return {
+        "kind": "pipeline",
+        "type": "docker",
+        "name": "wopiValidatorTests-%s" % (storage),
+        "platform": {
+            "os": "linux",
+            "arch": "amd64",
+        },
+        "steps": skipIfUnchanged(ctx, "acceptance-tests") +
+                 restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
+                 [
+                     {
+                         "name": "fakeoffice",
+                         "image": OC_CI_ALPINE,
+                         "detach": True,
+                         "environment": {},
+                         "commands": [
+                             "sh %s/tests/config/drone/serve-hosting-discovery.sh" % (dirs["base"]),
+                         ],
+                     },
+                     {
+                         "name": "wopiserver",
+                         "image": "cs3org/wopiserver:v9.2.5",
+                         "detach": True,
+                         "commands": [
+                             "cp %s/tests/config/drone/wopiserver.conf /etc/wopi/wopiserver.conf" % (dirs["base"]),
+                             "echo 123 > /etc/wopi/wopisecret",
+                             "/app/wopiserver.py",
+                         ],
+                     },
+                 ] +
+                 ocisServer(storage, accounts_hash_difficulty, [], [], "wopi_validator") +
+                 [
+                     {
+                         "name": "prepare-test-file-%s" % (storage),
+                         "image": OC_CI_ALPINE,
+                         "environment": {},
+                         "commands": [
+                             "curl -k 'https://ocis-server:9200/remote.php/webdav/test.wopitest' --fail --retry-connrefused --retry 7 --retry-all-errors -X PUT -u admin:admin -D headers.txt",
+                             "export FILE_ID=$(cat headers.txt | sed -n -e 's/^.*Oc-Fileid: //p')",
+                             "export URL=\"https://ocis-server:9200/app/open?app_name=FakeOffice&file_id=$FILE_ID\"",
+                             "export URL=$(echo $URL | tr -d '[:cntrl:]')",
+                             "curl -k -X POST \"$URL\" -u admin:admin -v > open.json",
+                             "cat open.json | jq .form_parameters.access_token | tr -d '\"' > accesstoken",
+                             "cat open.json | jq .form_parameters.access_token_ttl | tr -d '\"' > accesstokenttl",
+                             "echo -n 'http://wopiserver:8880/wopi/files/' > wopisrc",
+                             "cat open.json | jq .app_url | sed -n -e 's/^.*files%2F//p' | tr -d '\"' >> wopisrc",
+                         ],
+                     },
+                 ] +
+                 validatorTests +
+                 failEarly(ctx, early_fail),
+        "depends_on": getPipelineNames([buildOcisBinaryForTesting(ctx)]),
+        "trigger": {
+            "ref": [
+                "refs/heads/master",
+                "refs/pull/**",
+            ],
+        },
+    }
+
 def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", accounts_hash_difficulty = 4):
     early_fail = config["apiTests"]["earlyFail"] if "earlyFail" in config["apiTests"] else False
     filterTags = "~@skipOnGraph&&~@skipOnOcis&&~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@local_storage&&~@skipOnOcis-%s-Storage&&~@caldav&&~@carddav" % ("OC" if storage == "owncloud" else "OCIS")
@@ -1874,8 +1974,8 @@ def notify():
         },
     }
 
-def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = [], testing_parallel_deploy = False):
-    if not testing_parallel_deploy:
+def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = [], deploy_type = ""):
+    if deploy_type == "":
         user = "0:0"
         environment = {
             "OCIS_URL": OCIS_URL,
@@ -1902,7 +2002,43 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             ],
             "depends_on": depends_on,
         }
-    else:
+
+    if deploy_type == "wopi_validator":
+        user = "0:0"
+        environment = {
+            "OCIS_URL": OCIS_URL,
+            "OCIS_CONFIG_DIR": "/root/.ocis/config",
+            "STORAGE_USERS_DRIVER": "%s" % (storage),
+            "STORAGE_USERS_DRIVER_LOCAL_ROOT": "%s/local/root" % dirs["ocis"],
+            "STORAGE_USERS_DRIVER_OCIS_ROOT": "%s/storage/users" % dirs["ocis"],
+            "STORAGE_SYSTEM_DRIVER_OCIS_ROOT": "%s/storage/metadata" % dirs["ocis"],
+            "SHARING_USER_JSON_FILE": "%s/shares.json" % dirs["ocis"],
+            "PROXY_ENABLE_BASIC_AUTH": True,
+            "WEB_UI_CONFIG": "%s/%s" % (dirs["base"], dirs["ocisConfig"]),
+            "OCIS_LOG_LEVEL": "error",
+            "SETTINGS_DATA_PATH": "%s/settings" % dirs["ocis"],
+            "IDM_CREATE_DEMO_USERS": True,
+            "IDM_ADMIN_PASSWORD": "admin",  # override the random admin password from `ocis init`
+            "FRONTEND_SEARCH_MIN_LENGTH": "2",
+            "GATEWAY_GRPC_ADDR": "0.0.0.0:9142",  # make gateway available to wopi server
+            "APP_PROVIDER_EXTERNAL_ADDR": "127.0.0.1:9164",
+            "APP_PROVIDER_DRIVER": "wopi",
+            "APP_PROVIDER_WOPI_APP_NAME": "FakeOffice",
+            "APP_PROVIDER_WOPI_APP_URL": "http://fakeoffice:8080",
+            "APP_PROVIDER_WOPI_INSECURE": "true",
+            "APP_PROVIDER_WOPI_WOPI_SERVER_EXTERNAL_URL": "http://wopiserver:8880",
+            "APP_PROVIDER_WOPI_FOLDER_URL_BASE_URL": "https://ocis-server:9200",
+        }
+        wait_for_ocis = {
+            "name": "wait-for-ocis-server",
+            "image": OC_CI_WAIT_FOR,
+            "commands": [
+                "wait-for -it ocis-server:9200 -t 300",
+            ],
+            "depends_on": depends_on,
+        }
+
+    if deploy_type == "parallel":
         user = "33:33"
         environment = {
             # Keycloak IDP specific configuration
@@ -2527,7 +2663,7 @@ def parallelDeployAcceptancePipeline(ctx):
                              4,
                              [stepVolumeOC10OCISData, stepVolumeOCISConfig],
                              ["fix-shared-data-permissions"],
-                             True,
+                             "parallel",
                          ) +
                          parallelAcceptance(environment) +
                          failEarly(ctx, early_fail),

tests/config/drone/hosting-discovery.xml

@@ -0,0 +1,8 @@
+<wopi-discovery>
+<net-zone name="external-http">
+<app favIconUrl="https://fakeoffice.owncloud.test/favicon.ico" name="wopitest">
+<action default="true" ext="wopitest" name="view" urlsrc="https://fakeoffice.owncloud.test/not/relevant?"/>
+<action default="true" ext="wopitest" name="edit" urlsrc="https://fakeoffice.owncloud.test/not/relevant?"/>
+</app>
+</net-zone>
+</wopi-discovery>

tests/config/drone/serve-hosting-discovery.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+while true; do
+  echo -e "HTTP/1.1 200 OK\n\n$(cat /drone/src/tests/config/drone/hosting-discovery.xml)" | nc -l -k -p 8080
+done

tests/config/drone/wopiserver.conf

@@ -0,0 +1,131 @@
+#
+# This config is based on https://github.com/cs3org/wopiserver/blob/master/wopiserver.conf
+#
+# wopiserver.conf
+#
+# Default configuration file for the WOPI server for oCIS
+#
+##############################################################
+
+[general]
+# Storage access layer to be loaded in order to operate this WOPI server
+# only "cs3" is supported with oCIS
+storagetype = cs3
+
+# Port where to listen for WOPI requests
+port = 8880
+
+# Logging level. Debug enables the Flask debug mode as well.
+# Valid values are: Debug, Info, Warning, Error.
+loglevel = Debug
+loghandler = stream
+logdest = stdout
+
+# URL of your WOPI server or your HA proxy in front of it
+wopiurl = http://wopiserver
+
+# URL for direct download of files. The complete URL that is sent
+# to clients will include the access_token argument
+downloadurl = http://wopiserver/wopi/cbox/download
+
+# The internal server engine to use (defaults to flask).
+# Set to waitress for production installations.
+internalserver = waitress
+
+# List of file extensions deemed incompatible with LibreOffice:
+# interoperable locking will be disabled for such files
+nonofficetypes = .md .zmd .txt .epd
+
+# List of file extensions to be supported by Collabora (deprecated)
+codeofficetypes = .odt .ott .ods .ots .odp .otp .odg .otg .doc .dot .xls .xlt .xlm .ppt .pot .pps .vsd .dxf .wmf .cdr .pages .number .key
+
+brandingname=CS3org WOPI server
+brandingurl=https://github.com/cs3org/wopiserver
+
+# WOPI access token expiration time [seconds]
+tokenvalidity = 86400
+
+# WOPI lock expiration time [seconds]
+wopilockexpiration = 3600
+
+# WOPI lock strict check: if True, WOPI locks will be compared according to specs,
+# that is their representation must match. False (default) allows for a more relaxed
+# comparison, which compensates incorrect lock requests from Microsoft Office Online
+# on-premise setups.
+wopilockstrictcheck = False
+
+# Enable support of rename operations from WOPI apps. This is currently
+# disabled by default as it has been observed that both MS Office and Collabora
+# Online do not play well with this feature.
+# Not supported with oCIS, must always be set to "False"
+enablerename = False
+
+# Detection of external Microsoft Office or LibreOffice locks. By default, lock files
+# compatible with Office for Desktop applications are detected, assuming that the
+# underlying storage can be mounted as a remote filesystem: in this case, WOPI GetLock
+# and SetLock operations return such locks and prevent online apps from entering edit mode.
+# This feature can be disabled in order to operate a pure WOPI server for online apps.
+# Not supported with oCIS, must always be set to "False"
+detectexternallocks = False
+
+# Location of the webconflict files. By default, such files are stored in the same path
+# as the original file. If that fails (e.g. because of missing permissions),
+# an attempt is made to store such files in this path if specified, otherwise
+# the system falls back to the recovery space (cf. io|recoverypath).
+# The keywords <user_initial> and <username> are replaced with the actual username's
+# initial letter and the actual username, respectively, so you can use e.g.
+# /your_storage/home/user_initial/username
+#conflictpath = /
+
+# ownCloud's WOPI proxy configuration. Disabled by default.
+#wopiproxy = https://external-wopi-proxy.com
+#wopiproxysecretfile = /path/to/your/shared-key-file
+#proxiedappname = Name of your proxied app
+
+[security]
+# Location of the secret files. Requires a restart of the
+# WOPI server when either the files or their content change.
+wopisecretfile = /etc/wopi/wopisecret
+# iop secret is not used for cs3 storage type
+#iopsecretfile = /etc/wopi/iopsecret
+
+# Use https as opposed to http (requires certificate)
+usehttps = no
+
+# Certificate and key for https. Requires a restart
+# to apply a change.
+wopicert = /etc/grid-security/host.crt
+wopikey = /etc/grid-security/host.key
+
+[bridge]
+# SSL certificate check for the connected apps
+sslverify = True
+
+# Minimal time interval between two consecutive save operations [seconds]
+#saveinterval = 200
+
+# Minimal time interval before a closed file is WOPI-unlocked [seconds]
+#unlockinterval = 90
+
+# CodiMD: disable creating zipped bundles when files contain pictures
+#disablezip = False
+
+[io]
+# Size used for buffered reads [bytes]
+chunksize = 4194304
+
+# Path to a recovery space in case of I/O errors when reaching to the remote storage.
+# This is expected to be a local path, and it is provided in order to ease user support.
+# Defaults to the indicated spool folder.
+recoverypath = /var/spool/wopirecovery
+
+[cs3]
+# Host and port of the Reva(-like) CS3-compliant GRPC gateway endpoint
+revagateway = ocis-server:9142
+
+# Reva/gRPC authentication token expiration time [seconds]
+# The default value matches Reva's default
+authtokenvalidity = 3600
+
+# SSL certificate check for Reva
+sslverify = False