diff --git a/README.md b/README.md index e8f89647..a2f2cf1c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,14 @@ This service is part of the ScienceMesh Interoperability Platform (IOP) and impl It enables ScienceMesh EFSS storages to integrate Office Online platforms including Microsoft Office Online and Collabora Online. In addition it implements a [bridge](src/bridge/readme.md) module with dedicated extensions to support apps like CodiMD and Etherpad. Author: Giuseppe Lo Presti (@glpatcern)
-Contributions: Michael DSilva (@madsi1m), Lovisa Lugnegaard (@LovisaLugnegard), Samuel Alfageme (@SamuAlfageme), Ishank Arora (@ishank011), Willy Kloucek (@wkloucek) +Contributors: +- Michael DSilva (@madsi1m) +- Lovisa Lugnegaard (@LovisaLugnegard) +- Samuel Alfageme (@SamuAlfageme) +- Ishank Arora (@ishank011) +- Willy Kloucek (@wkloucek) +- Gianmaria Del Monte (@gmgigi96) +- Klaas Freitag (@dragotin) Initial revision: December 2016
First production version for CERNBox: September 2017 (presented at [oCCon17](https://occon17.owncloud.org) - [slides](https://www.slideshare.net/giuseppelopresti/collaborative-editing-and-more-in-cernbox))
diff --git a/requirements.txt b/requirements.txt index 8d20811c..fdad10b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,5 @@ requests more_itertools tuspy prometheus-flask-exporter -cs3apis>=0.1.dev66 +cs3apis>=0.1.dev87 waitress diff --git a/src/bridge/__init__.py b/src/bridge/__init__.py index a3a886b1..4c6a4733 100644 --- a/src/bridge/__init__.py +++ b/src/bridge/__init__.py @@ -1,7 +1,7 @@ ''' -The WOPI Bridge for IOP. This connector service supports CodiMD and Etherpad. +The WOPI bridge extension for IOP. This connector service supports CodiMD and Etherpad. -Author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' import os @@ -20,6 +20,7 @@ from base64 import urlsafe_b64encode import flask import bridge.wopiclient as wopic +import core.wopiutils as utils # The supported plugins integrated with this WOPI Bridge @@ -171,6 +172,7 @@ def appopen(wopisrc, acctok): 'lastsave': int(time.time()) - WB.saveinterval, 'toclose': {acctok[-20:]: False}, 'docid': wopilock['docid'], + 'app': os.path.splitext(filemd['BaseFileName'])[1][1:], } # also clear any potential stale response for this document try: @@ -282,15 +284,15 @@ def run(self): self.cleanup(openfile, wopisrc, wopilock) except Exception as e: # pylint: disable=broad-except ex_type, ex_value, ex_traceback = sys.exc_info() - WB.log.error('msg="SaveThread: unexpected exception caught" ex="%s" type="%s" traceback="%s"' % - (e, ex_type, traceback.format_exception(ex_type, ex_value, ex_traceback))) - WB.log.info('msg="SaveThread terminated, shutting down"') + WB.log.critical('msg="SaveThread: unexpected exception caught" ex="%s" type="%s" traceback="%s"' % + (e, ex_type, traceback.format_exception(ex_type, ex_value, ex_traceback))) def savedirty(self, openfile, wopisrc): '''save documents that are dirty for more than `saveinterval` or that are being closed''' wopilock = None if openfile['tosave'] and (_intersection(openfile['toclose']) or (openfile['lastsave'] < time.time() - WB.saveinterval)): + app = BRIDGE_EXT_PLUGINS.get(openfile['app']) if 'app' in openfile else None try: wopilock = wopic.getlock(wopisrc, openfile['acctok']) except wopic.InvalidLock: @@ -301,14 +303,23 @@ def savedirty(self, openfile, wopisrc): wopisrc, openfile['acctok'], openfile['docid'], _intersection(openfile['toclose'])) except wopic.InvalidLock as ile: # even this attempt failed, give up - # TODO here we should save the file on a local storage to help later recovery WB.saveresponses[wopisrc] = wopic.jsonify(str(ile)), http.client.INTERNAL_SERVER_ERROR + # attempt to save to local storage to help for later recovery: this is a feature of the core wopiserver + content = rc = None + if app: + content, rc = WB.plugins[app].savetostorage(wopisrc, openfile['acctok'], False, {'docid': openfile['docid']}, onlyfetch=True) + if rc == http.client.OK: + utils.storeForRecovery(content, wopisrc[wopisrc.rfind('/')+1:], openfile['acctok'][-20:], ile) + if rc != http.client.OK: + WB.log.error('msg="SaveThread: failed to fetch file for recovery to local storage" token="%s" docid="%s" app="%s" response="%s"' % + (openfile['acctok'][-20:], openfile['docid'], app, content)) # set some 'fake' metadata, will be automatically cleaned up later openfile['lastsave'] = int(time.time()) openfile['tosave'] = False openfile['toclose'] = {'invalid-lock': True} return None - app = BRIDGE_EXT_PLUGINS.get(wopilock['app']) + if not app: + app = BRIDGE_EXT_PLUGINS.get(wopilock['app']) if not app: WB.log.error('msg="SaveThread: malformed app attribute in WOPI lock" lock="%s"' % wopilock) WB.saveresponses[wopisrc] = wopic.jsonify('Unrecognized app for this file'), http.client.BAD_REQUEST @@ -369,14 +380,17 @@ def cleanup(self, openfile, wopisrc, wopilock): del WB.openfiles[wopisrc] elif openfile['toclose'] != wopilock['toclose']: # some user still on it, refresh lock if the toclose part has changed - wopic.refreshlock(wopisrc, openfile['acctok'], wopilock, toclose=openfile['toclose']) + try: + wopic.refreshlock(wopisrc, openfile['acctok'], wopilock, toclose=openfile['toclose']) + except wopic.InvalidLock: + WB.log.warning('msg="SaveThread: failed to refresh lock, will try again later" url="%s"' % wopisrc) @atexit.register def stopsavethread(): '''Exit handler to cleanly stop the storage sync thread''' if WB.savethread: - WB.log.info('msg="Waiting for SaveThread to complete"') + WB.log.info('msg="Waiting for SaveThread to complete"') # TODO when this handler is called, the logger is not accessible any longer with WB.savecv: WB.active = False WB.savecv.notify() diff --git a/src/bridge/codimd.py b/src/bridge/codimd.py index 4930ad6b..006e91b7 100644 --- a/src/bridge/codimd.py +++ b/src/bridge/codimd.py @@ -3,7 +3,7 @@ The CodiMD-specific code used by the WOPI bridge. -Author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' import os @@ -233,13 +233,16 @@ def _getattachments(mddoc, docfilename, forcezip=False): return zip_buffer.getvalue(), response -def savetostorage(wopisrc, acctok, isclose, wopilock): +def savetostorage(wopisrc, acctok, isclose, wopilock, onlyfetch=False): '''Copy document from CodiMD back to storage''' # get document from CodiMD try: log.info('msg="Fetching file from CodiMD" isclose="%s" appurl="%s" token="%s"' % (isclose, appurl + wopilock['docid'], acctok[-20:])) mddoc = _fetchfromcodimd(wopilock, acctok) + if onlyfetch: + # this is used only in case of recovery to local storage + return mddoc, http.client.OK except AppFailure: return wopic.jsonify('Could not save file, failed to fetch document from CodiMD'), http.client.INTERNAL_SERVER_ERROR @@ -269,11 +272,15 @@ def savetostorage(wopisrc, acctok, isclose, wopilock): if isclose and wopilock['digest'] == 'dirty': h = hashlib.sha1() h.update(mddoc) - wopilock = wopic.refreshlock(wopisrc, acctok, wopilock, digest=(h.hexdigest() if h else 'dirty')) - log.info('msg="Save completed" filename="%s" isclose="%s" token="%s"' % - (wopilock['filename'], isclose, acctok[-20:])) - # combine the responses - return attresponse if attresponse else (wopic.jsonify('File saved successfully'), http.client.OK) + try: + wopilock = wopic.refreshlock(wopisrc, acctok, wopilock, digest=(h.hexdigest() if h else 'dirty')) + log.info('msg="Save completed" filename="%s" isclose="%s" token="%s"' % + (wopilock['filename'], isclose, acctok[-20:])) + # combine the responses + return attresponse if attresponse else (wopic.jsonify('File saved successfully'), http.client.OK) + except wopic.InvalidLock: + return wopic.jsonify('File saved, but failed to refresh lock'), http.client.INTERNAL_SERVER_ERROR + # on close, use saveas for either the new bundle, if this is the first time we have attachments, # or the single md file, if there are no more attachments. diff --git a/src/bridge/etherpad.py b/src/bridge/etherpad.py index 59cd38e5..88bd2cf9 100644 --- a/src/bridge/etherpad.py +++ b/src/bridge/etherpad.py @@ -3,7 +3,7 @@ The Etherpad-specific code used by the WOPI bridge. -Author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' from random import choice @@ -134,13 +134,16 @@ def _fetchfrometherpad(wopilock, acctok): raise AppFailure -def savetostorage(wopisrc, acctok, isclose, wopilock): +def savetostorage(wopisrc, acctok, isclose, wopilock, onlyfetch=False): '''Copy document from Etherpad back to storage''' # get document from Etherpad try: log.info('msg="Fetching file from Etherpad" isclose="%s" appurl="%s" token="%s"' % (isclose, appurl + '/p' + wopilock['docid'], acctok[-20:])) epfile = _fetchfrometherpad(wopilock, acctok) + if onlyfetch: + # this is used only in case of recovery to local storage + return epfile, http.client.OK except AppFailure: return wopic.jsonify('Could not save file, failed to fetch document from Etherpad'), http.client.INTERNAL_SERVER_ERROR @@ -158,7 +161,10 @@ def savetostorage(wopisrc, acctok, isclose, wopilock): reply = wopic.handleputfile('PutFile', wopisrc, res) if reply: return reply - wopilock = wopic.refreshlock(wopisrc, acctok, wopilock, digest='dirty') - log.info('msg="Save completed" filename="%s" isclose="%s" token="%s"' % - (wopilock['filename'], isclose, acctok[-20:])) - return wopic.jsonify('File saved successfully'), http.client.OK + try: + wopilock = wopic.refreshlock(wopisrc, acctok, wopilock, digest='dirty') + log.info('msg="Save completed" filename="%s" isclose="%s" token="%s"' % + (wopilock['filename'], isclose, acctok[-20:])) + return wopic.jsonify('File saved successfully'), http.client.OK + except wopic.InvalidLock: + return wopic.jsonify('File saved, but failed to refresh lock'), http.client.INTERNAL_SERVER_ERROR diff --git a/src/bridge/wopiclient.py b/src/bridge/wopiclient.py index 507396b1..49db4d16 100644 --- a/src/bridge/wopiclient.py +++ b/src/bridge/wopiclient.py @@ -1,9 +1,9 @@ ''' wopiclient.py -A set of WOPI functions for the WOPI bridge service. +A set of WOPI client functions for the WOPI bridge service. -Author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' import os @@ -113,7 +113,7 @@ def refreshlock(wopisrc, acctok, wopilock, digest=None, toclose=None): # else fail log.error('msg="Calling WOPI RefreshLock failed" url="%s" response="%d" reason="%s"' % (wopisrc, res.status_code, res.headers.get('X-WOPI-LockFailureReason'))) - return None + raise InvalidLock('Failed to refresh the lock') def relock(wopisrc, acctok, docid, isclose): @@ -154,8 +154,8 @@ def handleputfile(wopicall, wopisrc, res): return jsonify('Error saving the file. %s' % res.headers.get('X-WOPI-LockFailureReason')), \ http.client.INTERNAL_SERVER_ERROR if res.status_code != http.client.OK: + # hopefully the server has kept a local copy for later recovery log.error('msg="Calling WOPI %s failed" url="%s" response="%s"' % (wopicall, wopisrc, res.status_code)) - # TODO need to save the file on a local storage for later recovery return jsonify('Error saving the file, please contact support'), http.client.INTERNAL_SERVER_ERROR return None diff --git a/src/core/commoniface.py b/src/core/commoniface.py index d203b330..6985677c 100644 --- a/src/core/commoniface.py +++ b/src/core/commoniface.py @@ -1,13 +1,17 @@ ''' commoniface.py -common entities used by all storage interfaces for the IOP WOPI server +Common entities used by all storage interfaces for the IOP WOPI server. +Includes functions to store and retrieve Reva-compatible locks. -Author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' import time import json +from base64 import urlsafe_b64encode, urlsafe_b64decode +from binascii import Error as B64Error + # standard file missing message ENOENT_MSG = 'No such file or directory' @@ -19,8 +23,56 @@ ACCESS_ERROR = 'Operation not permitted' # name of the xattr storing the Reva lock -LOCKKEY = 'user.iop.lock' +LOCKKEY = 'iop.lock' + +# the prefix used for the lock-id payload to be WebDAV compatible: +# see https://github.com/cs3org/wopiserver/pull/51#issuecomment-1038798545 for more details; +# the UUID is fully random and hard coded to identify WOPI locks, no need to have it dynamic +WEBDAV_LOCK_PREFIX = 'opaquelocktoken:797356a8-0500-4ceb-a8a0-c94c8cde7eba' + +# reference to global config +config = None + + +# Manipulate Reva-compliant locks, i.e. JSON structs with the following format: +#{ +# "lock_id": "id1234", +# "type": 2, +# "user": { +# "idp": "https://your-idprovider.org", +# "opaque_id": "username", +# "type": 1 +# }, +# "app_name": "your_app", +# "expiration": { +# "seconds": 1665446400 +# } +#} def genrevalock(appname, value): - '''Return a JSON-formatted lock compatible with the Reva implementation of the CS3 Lock API''' - return json.dumps({'h': appname if appname else 'wopi', 't': int(time.time()), 'md': value}) + '''Return a base64-encoded lock compatible with the Reva implementation of the CS3 Lock API + cf. https://github.com/cs3org/cs3apis/blob/main/cs3/storage/provider/v1beta1/resources.proto''' + return urlsafe_b64encode(json.dumps( + {'lock_id': value, + 'type': 2, # LOCK_TYPE_WRITE + 'app_name': appname if appname else 'wopi', + 'user': {}, + 'expiration': { + 'seconds': int(time.time()) + config.getint('general', 'wopilockexpiration') + }, + }).encode()).decode() + + +def retrieverevalock(rawlock): + '''Restores the JSON payload from a base64-encoded Reva lock''' + try: + l = json.loads(urlsafe_b64decode(rawlock).decode()) + if 'h' in l: + # temporary code to support the data structure from WOPI 8.0 + l['app_name'] = l['h'] + l['lock_id'] = WEBDAV_LOCK_PREFIX + ' ' + l['md'] + l['expiration'] = {} + l['expiration']['seconds'] = l['exp'] + return l + except (B64Error, json.JSONDecodeError) as e: + raise IOError(e) diff --git a/src/core/cs3iface.py b/src/core/cs3iface.py index 84da2f36..a82ada6f 100644 --- a/src/core/cs3iface.py +++ b/src/core/cs3iface.py @@ -3,9 +3,7 @@ CS3 API based interface for the IOP WOPI server -Authors: -Giuseppe.LoPresti@cern.ch, CERN/IT-ST -Lovisa.Lugnegaard@cern.ch, CERN/IT-ST +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' import time @@ -25,23 +23,24 @@ # module-wide state ctx = {} # "map" to store some module context: cf. init() -tokens = {} # map userid [string] to {authentication token, token expiration time} +log = None - -def init(config, log): +def init(inconfig, inlog): '''Init module-level variables''' - ctx['log'] = log - ctx['chunksize'] = config.getint('io', 'chunksize') - ctx['ssl_verify'] = config.getboolean('cs3', 'sslverify', fallback=True) - ctx['authtokenvalidity'] = config.getint('cs3', 'authtokenvalidity') - if config.has_option('cs3', 'revagateway'): - revagateway = config.get('cs3', 'revagateway') + global log # pylint: disable=global-statement + log = inlog + ctx['chunksize'] = inconfig.getint('io', 'chunksize') + ctx['ssl_verify'] = inconfig.getboolean('cs3', 'sslverify', fallback=True) + ctx['authtokenvalidity'] = inconfig.getint('cs3', 'authtokenvalidity') + ctx['lockexpiration'] = inconfig.getint('general', 'wopilockexpiration') + if inconfig.has_option('cs3', 'revagateway'): + revagateway = inconfig.get('cs3', 'revagateway') else: # legacy entry, to be dropped at next major release - revagateway = config.get('cs3', 'revahost') + revagateway = inconfig.get('cs3', 'revahost') # prepare the gRPC connection ch = grpc.insecure_channel(revagateway) - ctx['cs3stub'] = cs3gw_grpc.GatewayAPIStub(ch) + ctx['cs3gw'] = cs3gw_grpc.GatewayAPIStub(ch) def getuseridfromcreds(token, _wopiuser): @@ -53,8 +52,8 @@ def getuseridfromcreds(token, _wopiuser): def authenticate_for_test(userid, userpwd): '''Use basic authentication against Reva for testing purposes''' authReq = cs3gw.AuthenticateRequest(type='basic', client_id=userid, client_secret=userpwd) - authRes = ctx['cs3stub'].Authenticate(authReq) - ctx['log'].debug('msg="Authenticated user" res="%s"' % authRes) + authRes = ctx['cs3gw'].Authenticate(authReq) + log.debug('msg="Authenticated user" res="%s"' % authRes) if authRes.status.code != cs3code.CODE_OK: raise IOError('Failed to authenticate as user ' + userid + ': ' + authRes.status.message) return authRes.token @@ -73,16 +72,15 @@ def stat(endpoint, fileid, userid, versioninv=1): else: # assume we have an opaque fileid ref = cs3spr.Reference(resource_id=cs3spr.ResourceId(storage_id=endpoint, opaque_id=fileid)) - statInfo = ctx['cs3stub'].Stat(request=cs3sp.StatRequest(ref=ref), - metadata=[('x-access-token', userid)]) + statInfo = ctx['cs3gw'].Stat(request=cs3sp.StatRequest(ref=ref), metadata=[('x-access-token', userid)]) tend = time.time() - ctx['log'].info('msg="Invoked stat" inode="%s" elapsedTimems="%.1f"' % (fileid, (tend-tstart)*1000)) + log.info('msg="Invoked stat" inode="%s" elapsedTimems="%.1f"' % (fileid, (tend-tstart)*1000)) if statInfo.status.code == cs3code.CODE_OK: - ctx['log'].debug('msg="Stat result" data="%s"' % statInfo) + log.debug('msg="Stat result" data="%s"' % statInfo) if statInfo.info.type == cs3spr.RESOURCE_TYPE_CONTAINER: raise IOError('Is a directory') if statInfo.info.type not in (cs3spr.RESOURCE_TYPE_FILE, cs3spr.RESOURCE_TYPE_SYMLINK): - ctx['log'].warning('msg="Stat: unexpected type" type="%d"' % statInfo.info.type) + log.warning('msg="Stat: unexpected type" type="%d"' % statInfo.info.type) raise IOError('Unexpected type %d' % statInfo.info.type) # we base64-encode the inode so it can be used in a WOPISrc inode = urlsafe_b64encode(statInfo.info.id.opaque_id.encode()).decode() @@ -93,7 +91,7 @@ def stat(endpoint, fileid, userid, versioninv=1): 'size': statInfo.info.size, 'mtime': statInfo.info.mtime.seconds } - ctx['log'].info('msg="Failed stat" inode="%s" reason="%s"' % (fileid, statInfo.status.message.replace('"', "'"))) + log.info('msg="Failed stat" inode="%s" reason="%s"' % (fileid, statInfo.status.message.replace('"', "'"))) raise IOError(common.ENOENT_MSG if statInfo.status.code == cs3code.CODE_NOT_FOUND else statInfo.status.message) @@ -102,129 +100,187 @@ def statx(endpoint, fileid, userid, versioninv=0): return stat(endpoint, fileid, userid, versioninv) -def setxattr(_endpoint, filepath, userid, key, value): +def setxattr(_endpoint, filepath, userid, key, value, lockid): '''Set the extended attribute to using the given userid as access token''' reference = cs3spr.Reference(path=filepath) - arbitrary_metadata = cs3spr.ArbitraryMetadata() - arbitrary_metadata.metadata.update({key: str(value)}) # pylint: disable=no-member - req = cs3sp.SetArbitraryMetadataRequest(ref=reference, arbitrary_metadata=arbitrary_metadata) - res = ctx['cs3stub'].SetArbitraryMetadata(request=req, - metadata=[('x-access-token', userid)]) + md = cs3spr.ArbitraryMetadata() + md.metadata.update({key: str(value)}) # pylint: disable=no-member + req = cs3sp.SetArbitraryMetadataRequest(ref=reference, arbitrary_metadata=md, lock_id=lockid) + res = ctx['cs3gw'].SetArbitraryMetadata(request=req, metadata=[('x-access-token', userid)]) if res.status.code != cs3code.CODE_OK: - ctx['log'].error('msg="Failed to setxattr" filepath="%s" key="%s" reason="%s"' % (filepath, key, res.status.message.replace('"', "'"))) + log.error('msg="Failed to setxattr" filepath="%s" key="%s" code="%s" reason="%s"' % + (filepath, key, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - ctx['log'].debug('msg="Invoked setxattr" result="%s"' % res) + log.debug('msg="Invoked setxattr" result="%s"' % res) def getxattr(_endpoint, filepath, userid, key): '''Get the extended attribute using the given userid as access token''' tstart = time.time() reference = cs3spr.Reference(path=filepath) - statInfo = ctx['cs3stub'].Stat(request=cs3sp.StatRequest(ref=reference), - metadata=[('x-access-token', userid)]) + statInfo = ctx['cs3gw'].Stat(request=cs3sp.StatRequest(ref=reference), metadata=[('x-access-token', userid)]) tend = time.time() + if statInfo.status.code == cs3code.CODE_NOT_FOUND: + log.debug('msg="Invoked stat for getxattr on missing file" filepath="%s"' % filepath) + return None if statInfo.status.code != cs3code.CODE_OK: - ctx['log'].error('msg="Failed to stat" filepath="%s" key="%s" reason="%s"' % (filepath, key, statInfo.status.message.replace('"', "'"))) + log.error('msg="Failed to stat" filepath="%s" key="%s" reason="%s"' % + (filepath, key, statInfo.status.message.replace('"', "'"))) raise IOError(statInfo.status.message) try: xattrvalue = statInfo.info.arbitrary_metadata.metadata[key] if xattrvalue == '': raise KeyError - ctx['log'].debug('msg="Invoked stat for getxattr" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend-tstart)*1000)) + log.debug('msg="Invoked stat for getxattr" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend-tstart)*1000)) return xattrvalue except KeyError: - ctx['log'].warning('msg="Empty value or key not found in getxattr" filepath="%s" key="%s" metadata="%s"' % (filepath, key, statInfo.info.arbitrary_metadata.metadata)) + log.warning('msg="Empty value or key not found in getxattr" filepath="%s" key="%s" metadata="%s"' % + (filepath, key, statInfo.info.arbitrary_metadata.metadata)) return None -def rmxattr(_endpoint, filepath, userid, key): +def rmxattr(_endpoint, filepath, userid, key, lockid): '''Remove the extended attribute using the given userid as access token''' reference = cs3spr.Reference(path=filepath) - req = cs3sp.UnsetArbitraryMetadataRequest(ref=reference, arbitrary_metadata_keys=[key]) - res = ctx['cs3stub'].UnsetArbitraryMetadata(request=req, metadata=[('x-access-token', userid)]) + req = cs3sp.UnsetArbitraryMetadataRequest(ref=reference, arbitrary_metadata_keys=[key], lock_id=lockid) + res = ctx['cs3gw'].UnsetArbitraryMetadata(request=req, metadata=[('x-access-token', userid)]) if res.status.code != cs3code.CODE_OK: - ctx['log'].error('msg="Failed to rmxattr" filepath="%s" key="%s" reason="%s"' % (filepath, key, res.status.message.replace('"', "'"))) + log.error('msg="Failed to rmxattr" filepath="%s" key="%s" reason="%s"' % (filepath, key, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - ctx['log'].debug('msg="Invoked rmxattr" result="%s"' % res) + log.debug('msg="Invoked rmxattr" result="%s"' % res.status) -def setlock(endpoint, filepath, userid, appname, value): +def setlock(_endpoint, filepath, userid, appname, value): '''Set a lock to filepath with the given value metadata and appname as holder''' - raise NotImplementedError + reference = cs3spr.Reference(path=filepath) + lock = cs3spr.Lock(type=cs3spr.LOCK_TYPE_WRITE, app_name=appname, lock_id=value, \ + expiration={'seconds': int(time.time() + ctx['lockexpiration'])}) + req = cs3sp.SetLockRequest(ref=reference, lock=lock) + res = ctx['cs3gw'].SetLock(request=req, metadata=[('x-access-token', userid)]) + if res.status.code == cs3code.CODE_FAILED_PRECONDITION: + log.info('msg="Invoked setlock on an already locked entity" filepath="%s" appname="%s" reason="%s"' % + (filepath, appname, res.status.message.replace('"', "'"))) + raise IOError(common.EXCL_ERROR) + if res.status.code != cs3code.CODE_OK: + log.error('msg="Failed to setlock" filepath="%s" appname="%s" value="%s" code="%s" reason="%s"' % + (filepath, appname, value, res.status.code, res.status.message.replace('"', "'"))) + raise IOError(res.status.message) + log.debug('msg="Invoked setlock" filepath="%s" value="%s" result="%s"' % (filepath, value, res.status)) -def getlock(endpoint, filepath, userid, appname): +def getlock(_endpoint, filepath, userid): '''Get the lock metadata for the given filepath''' - raise NotImplementedError + reference = cs3spr.Reference(path=filepath) + req = cs3sp.GetLockRequest(ref=reference) + res = ctx['cs3gw'].GetLock(request=req, metadata=[('x-access-token', userid)]) + if res.status.code == cs3code.CODE_NOT_FOUND: + log.debug('msg="Invoked getlock on unlocked or missing file" filepath="%s"' % filepath) + return None + if res.status.code != cs3code.CODE_OK: + log.error('msg="Failed to getlock" filepath="%s" code="%s" reason="%s"' % + (filepath, res.status.code, res.status.message.replace('"', "'"))) + raise IOError(res.status.message) + log.debug('msg="Invoked getlock" filepath="%s" result="%s"' % (filepath, res.lock)) + # return a dict that mimics the internal JSON structure used by Reva, cf. commoniface.py + return { + 'lock_id': res.lock.lock_id, + 'type': res.lock.type, + 'app_name': res.lock.app_name, + 'user': {'opaque_id' : res.lock.user.opaque_id, + 'idp': res.lock.user.idp, + 'type': 1 + } if res.lock.user.opaque_id else {}, + 'expiration': { + 'seconds': res.lock.expiration.seconds + } + } -def refreshlock(endpoint, filepath, userid, appname, value): +def refreshlock(_endpoint, filepath, userid, appname, value): '''Refresh the lock metadata for the given filepath''' - raise NotImplementedError + reference = cs3spr.Reference(path=filepath) + lock = cs3spr.Lock(type=cs3spr.LOCK_TYPE_WRITE, app_name=appname, lock_id=value, \ + expiration={'seconds': int(time.time() + ctx['lockexpiration'])}) + req = cs3sp.RefreshLockRequest(ref=reference, lock=lock) + res = ctx['cs3gw'].RefreshLock(request=req, metadata=[('x-access-token', userid)]) + if res.status.code != cs3code.CODE_OK: + log.warning('msg="Failed to refreshlock" filepath="%s" appname="%s" value="%s" code="%s" reason="%s"' % + (filepath, appname, value, res.status.code, res.status.message.replace('"', "'"))) + raise IOError(res.status.message) + log.debug('msg="Invoked refreshlock" filepath="%s" value="%s" result="%s"' % (filepath, value, res.status)) -def unlock(endpoint, filepath, userid, appname): +def unlock(_endpoint, filepath, userid, appname, value): '''Remove the lock for the given filepath''' - raise NotImplementedError + reference = cs3spr.Reference(path=filepath) + lock = cs3spr.Lock(type=cs3spr.LOCK_TYPE_WRITE, app_name=appname, lock_id=value) + req = cs3sp.UnlockRequest(ref=reference, lock=lock) + res = ctx['cs3gw'].Unlock(request=req, metadata=[('x-access-token', userid)]) + if res.status.code != cs3code.CODE_OK: + log.error('msg="Failed to unlock" filepath="%s" code="%s" reason="%s"' % + (filepath, res.status.code, res.status.message.replace('"', "'"))) + raise IOError(res.status.message) + log.debug('msg="Invoked unlock" filepath="%s" value="%s" result="%s"' % (filepath, value, res.status)) -def readfile(_endpoint, filepath, userid): +def readfile(_endpoint, filepath, userid, lockid): '''Read a file using the given userid as access token. Note that the function is a generator, managed by Flask.''' tstart = time.time() # prepare endpoint - req = cs3sp.InitiateFileDownloadRequest(ref=cs3spr.Reference(path=filepath)) - initfiledownloadres = ctx['cs3stub'].InitiateFileDownload(request=req, metadata=[('x-access-token', userid)]) + req = cs3sp.InitiateFileDownloadRequest(ref=cs3spr.Reference(path=filepath), lock_id=lockid) + initfiledownloadres = ctx['cs3gw'].InitiateFileDownload(request=req, metadata=[('x-access-token', userid)]) if initfiledownloadres.status.code == cs3code.CODE_NOT_FOUND: - ctx['log'].info('msg="File not found on read" filepath="%s"' % filepath) + log.info('msg="File not found on read" filepath="%s"' % filepath) yield IOError(common.ENOENT_MSG) elif initfiledownloadres.status.code != cs3code.CODE_OK: - ctx['log'].error('msg="Failed to initiateFileDownload on read" filepath="%s" reason="%s"' % - (filepath, initfiledownloadres.status.message.replace('"', "'"))) + log.error('msg="Failed to initiateFileDownload on read" filepath="%s" code="%s" reason="%s"' % + (filepath, initfiledownloadres.status.code, initfiledownloadres.status.message.replace('"', "'"))) yield IOError(initfiledownloadres.status.message) - ctx['log'].debug('msg="readfile: InitiateFileDownloadRes returned" protocols="%s"' % initfiledownloadres.protocols) + log.debug('msg="readfile: InitiateFileDownloadRes returned" protocols="%s"' % initfiledownloadres.protocols) # Download try: protocol = [p for p in initfiledownloadres.protocols if p.protocol == "simple"][0] headers = { 'x-access-token': userid, - 'X-Reva-Transfer': protocol.token # needed if the downloads pass through the data gateway in reva + 'x-reva-transfer': protocol.token # needed if the downloads pass through the data gateway in reva } fileget = requests.get(url=protocol.download_endpoint, headers=headers, verify=ctx['ssl_verify']) except requests.exceptions.RequestException as e: - ctx['log'].error('msg="Exception when downloading file from Reva" reason="%s"' % e) + log.error('msg="Exception when downloading file from Reva" reason="%s"' % e) yield IOError(e) tend = time.time() data = fileget.content if fileget.status_code != http.client.OK: - ctx['log'].error('msg="Error downloading file from Reva" code="%d" reason="%s"' % (fileget.status_code, fileget.reason.replace('"', "'"))) + log.error('msg="Error downloading file from Reva" code="%d" reason="%s"' % + (fileget.status_code, fileget.reason.replace('"', "'"))) yield IOError(fileget.reason) else: - ctx['log'].info('msg="File open for read" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend-tstart)*1000)) + log.info('msg="File open for read" filepath="%s" elapsedTimems="%.1f"' % (filepath, (tend-tstart)*1000)) for i in range(0, len(data), ctx['chunksize']): yield data[i:i+ctx['chunksize']] -def writefile(_endpoint, filepath, userid, content, islock=False): +def writefile(_endpoint, filepath, userid, content, lockid, islock=False): '''Write a file using the given userid as access token. The entire content is written and any pre-existing file is deleted (or moved to the previous version if supported). - The islock flag is currently not supported. TODO the backend should at least support + The islock flag is currently not supported. The backend should at least support writing the file with O_CREAT|O_EXCL flags to prevent races.''' if islock: - ctx['log'].warning('msg="Lock (no-overwrite) flag not yet supported, going for standard upload"') + log.warning('msg="Lock (no-overwrite) flag not supported, going for standard upload"') tstart = time.time() # prepare endpoint if isinstance(content, str): content = bytes(content, 'UTF-8') size = str(len(content)) metadata = types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode(size))}) - req = cs3sp.InitiateFileUploadRequest(ref=cs3spr.Reference(path=filepath), opaque=metadata) - initfileuploadres = ctx['cs3stub'].InitiateFileUpload(request=req, metadata=[('x-access-token', userid)]) + req = cs3sp.InitiateFileUploadRequest(ref=cs3spr.Reference(path=filepath), lock_id=lockid, opaque=metadata) + initfileuploadres = ctx['cs3gw'].InitiateFileUpload(request=req, metadata=[('x-access-token', userid)]) if initfileuploadres.status.code != cs3code.CODE_OK: - ctx['log'].error('msg="Failed to initiateFileUpload on write" filepath="%s" reason="%s"' % \ - (filepath, initfileuploadres.status.message.replace('"', "'"))) + log.error('msg="Failed to initiateFileUpload on write" filepath="%s" code="%s" reason="%s"' % \ + (filepath, initfileuploadres.status.code, initfileuploadres.status.message.replace('"', "'"))) raise IOError(initfileuploadres.status.message) - ctx['log'].debug('msg="writefile: InitiateFileUploadRes returned" protocols="%s"' % initfileuploadres.protocols) + log.debug('msg="writefile: InitiateFileUploadRes returned" protocols="%s"' % initfileuploadres.protocols) # Upload try: @@ -233,42 +289,45 @@ def writefile(_endpoint, filepath, userid, content, islock=False): headers = { 'x-access-token': userid, 'Upload-Length': size, - 'X-Reva-Transfer': protocol.token # needed if the uploads pass through the data gateway in reva + 'x-reva-transfer': protocol.token # needed if the uploads pass through the data gateway in reva } putres = requests.put(url=protocol.upload_endpoint, data=content, headers=headers, verify=ctx['ssl_verify']) except requests.exceptions.RequestException as e: - ctx['log'].error('msg="Exception when uploading file to Reva" reason="%s"' % e) + log.error('msg="Exception when uploading file to Reva" reason="%s"' % e) raise IOError(e) tend = time.time() + if putres.status_code == http.client.UNAUTHORIZED: + log.warning('msg="Access denied uploading file to Reva" reason="%s"' % putres.reason) + raise IOError(common.ACCESS_ERROR) if putres.status_code != http.client.OK: - ctx['log'].error('msg="Error uploading file to Reva" code="%d" reason="%s"' % (putres.status_code, putres.reason)) + log.error('msg="Error uploading file to Reva" code="%d" reason="%s"' % (putres.status_code, putres.reason)) raise IOError(putres.reason) - ctx['log'].info('msg="File written successfully" filepath="%s" elapsedTimems="%.1f" islock="%s"' % \ + log.info('msg="File written successfully" filepath="%s" elapsedTimems="%.1f" islock="%s"' % \ (filepath, (tend-tstart)*1000, islock)) -def renamefile(_endpoint, filepath, newfilepath, userid): +def renamefile(_endpoint, filepath, newfilepath, userid, lockid): '''Rename a file from origfilepath to newfilepath using the given userid as access token.''' - source = cs3spr.Reference(path=filepath) - destination = cs3spr.Reference(path=newfilepath) - req = cs3sp.MoveRequest(source=source, destination=destination) - res = ctx['cs3stub'].Move(request=req, metadata=[('x-access-token', userid)]) + req = cs3sp.MoveRequest(source=cs3spr.Reference(path=filepath), \ + destination=cs3spr.Reference(path=newfilepath), lock_id=lockid) + res = ctx['cs3gw'].Move(request=req, metadata=[('x-access-token', userid)]) if res.status.code != cs3code.CODE_OK: - ctx['log'].error('msg="Failed to rename file" filepath="%s" reason="%s"' % (filepath, res.status.message.replace('"', "'"))) + log.error('msg="Failed to rename file" filepath="%s" code="%s" reason="%s"' % + (filepath, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - ctx['log'].debug('msg="Invoked renamefile" result="%s"' % res) + log.debug('msg="Invoked renamefile" result="%s"' % res) -def removefile(_endpoint, filepath, userid, force=False): +def removefile(_endpoint, filepath, userid, _force=False): '''Remove a file using the given userid as access token. The force argument is ignored for now for CS3 storage.''' - reference = cs3spr.Reference(path=filepath) - req = cs3sp.DeleteRequest(ref=reference) - res = ctx['cs3stub'].Delete(request=req, metadata=[('x-access-token', userid)]) + req = cs3sp.DeleteRequest(ref=cs3spr.Reference(path=filepath)) + res = ctx['cs3gw'].Delete(request=req, metadata=[('x-access-token', userid)]) if res.status.code != cs3code.CODE_OK: if str(res) == common.ENOENT_MSG: - ctx['log'].info('msg="Invoked removefile on non-existing file" filepath="%s"' % filepath) + log.info('msg="Invoked removefile on non-existing file" filepath="%s"' % filepath) else: - ctx['log'].error('msg="Failed to remove file" filepath="%s" reason="%s"' % (filepath, res.status.message.replace('"', "'"))) + log.error('msg="Failed to remove file" filepath="%s" code="%s" reason="%s"' % + (filepath, res.status.code, res.status.message.replace('"', "'"))) raise IOError(res.status.message) - ctx['log'].debug('msg="Invoked removefile" result="%s"' % res) + log.debug('msg="Invoked removefile" result="%s"' % res) diff --git a/src/core/discovery.py b/src/core/discovery.py index 8b2ca8ed..704788b0 100644 --- a/src/core/discovery.py +++ b/src/core/discovery.py @@ -1,9 +1,12 @@ ''' discovery.py -Helper code for the WOPI discovery phase, as well as -for integrating the apps supported by the bridge functionality. -This code is going to be deprecated once the new Reva AppProvider is fully functional. +Helper code for the WOPI discovery phase, as well as for integrating the apps +supported by the bridge functionality. +This code is deprecated and is only used in conjunction with the xroot storage interface: +when the WOPI server is interfaced to Reva via the cs3 storage interface this code is disabled. + +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' from xml.etree import ElementTree as ET diff --git a/src/core/ioplocks.py b/src/core/ioplocks.py index 05cd3110..e3d80ab8 100644 --- a/src/core/ioplocks.py +++ b/src/core/ioplocks.py @@ -2,6 +2,8 @@ ioplocks.py Implementation of the interoperable (iop) locking used for OnlyOffice and Office Desktop applications. + +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' import time @@ -129,7 +131,7 @@ def createLock(filestat, filename, userid, endpoint): (srv.wopiurl, time.strftime('%d.%m.%Y %H:%M', time.localtime(time.time())), lockid) # try to write in exclusive mode (and if a valid WOPI lock exists, assume the corresponding LibreOffice lock # is still there so the write will fail) - st.writefile(endpoint, utils.getLibreOfficeLockName(filename), userid, lolockcontent, islock=True) + st.writefile(endpoint, utils.getLibreOfficeLockName(filename), userid, lolockcontent, None, islock=True) log.info('msg="cboxLock: created LibreOffice-compatible lock file" filename="%s" fileid="%s" lockid="%ld"' % (filename, filestat['inode'], lockid)) return str(lockid), http.client.OK @@ -179,7 +181,7 @@ def createLock(filestat, filename, userid, endpoint): lockid = int(time.time()) lolockcontent = ',OnlyOffice Online Editor,%s,%s,ExtWebApp;\n%d;' % \ (srv.wopiurl, time.strftime('%d.%m.%Y %H:%M', time.localtime(time.time())), lockid) - st.writefile(endpoint, utils.getLibreOfficeLockName(filename), userid, lolockcontent, islock=False) + st.writefile(endpoint, utils.getLibreOfficeLockName(filename), userid, lolockcontent, None, islock=False) log.info('msg="cboxLock: refreshed LibreOffice-compatible lock file" filename="%s" fileid="%s" mtime="%ld" lockid="%ld"' % (filename, filestat['inode'], filestat['mtime'], lockid)) return str(lockid), http.client.OK @@ -206,7 +208,7 @@ def iopunlock(filename, userid, endpoint): lock = lock.decode('UTF-8') if 'OnlyOffice Online Editor' in lock: # remove the LibreOffice-compatible lock file - st.removefile(endpoint, utils.getLibreOfficeLockName(filename), userid, 1) + st.removefile(endpoint, utils.getLibreOfficeLockName(filename), userid, True) # and log this along with the previous lockid for reference lockid = int(lock.split(';\n')[1].strip(';')) log.info('msg="cboxUnlock: successfully removed LibreOffice-compatible lock file" filename="%s" lockid="%ld"' % diff --git a/src/core/localiface.py b/src/core/localiface.py index 1399ba90..3fbed110 100644 --- a/src/core/localiface.py +++ b/src/core/localiface.py @@ -3,14 +3,13 @@ Local storage interface for the IOP WOPI server -Author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' import time import os import warnings from stat import S_ISDIR -import json import core.commoniface as common # module-wide state @@ -29,7 +28,7 @@ def init(inconfig, inlog): global config # pylint: disable=global-statement global log # pylint: disable=global-statement global homepath # pylint: disable=global-statement - config = inconfig + common.config = config = inconfig log = inlog homepath = config.get('local', 'storagehomepath') try: @@ -75,11 +74,11 @@ def statx(endpoint, filepath, userid, versioninv=1): return stat(endpoint, filepath, userid) -def setxattr(_endpoint, filepath, _userid, key, value): +def setxattr(_endpoint, filepath, _userid, key, value, _lockid): '''Set the extended attribute to on behalf of the given userid''' try: os.setxattr(_getfilepath(filepath), 'user.' + key, str(value).encode()) - except (FileNotFoundError, PermissionError, OSError) as e: + except OSError as e: log.error('msg="Failed to setxattr" filepath="%s" key="%s" exception="%s"' % (filepath, key, e)) raise IOError(e) @@ -89,25 +88,26 @@ def getxattr(_endpoint, filepath, _userid, key): try: filepath = _getfilepath(filepath) return os.getxattr(filepath, 'user.' + key).decode('UTF-8') - except (FileNotFoundError, PermissionError, OSError) as e: + except OSError as e: log.error('msg="Failed to getxattr" filepath="%s" key="%s" exception="%s"' % (filepath, key, e)) return None -def rmxattr(_endpoint, filepath, _userid, key): +def rmxattr(_endpoint, filepath, _userid, key, _lockid): '''Remove the extended attribute on behalf of the given userid''' try: os.removexattr(_getfilepath(filepath), 'user.' + key) - except (FileNotFoundError, PermissionError, OSError) as e: + except OSError as e: log.error('msg="Failed to rmxattr" filepath="%s" key="%s" exception="%s"' % (filepath, key, e)) raise IOError(e) def setlock(endpoint, filepath, _userid, appname, value): '''Set the lock as an xattr on behalf of the given userid''' + log.debug('msg="Invoked setlock" filepath="%s" value="%s"' % (filepath, value)) if not getxattr(endpoint, filepath, '0:0', common.LOCKKEY): # we do not protect from race conditions here - setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value)) + setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), None) else: raise IOError(common.EXCL_ERROR) @@ -116,26 +116,33 @@ def getlock(endpoint, filepath, _userid): '''Get the lock metadata as an xattr on behalf of the given userid''' l = getxattr(endpoint, filepath, '0:0', common.LOCKKEY) if l: - return json.loads(l) + return common.retrieverevalock(l) return None def refreshlock(endpoint, filepath, _userid, appname, value): '''Refresh the lock value as an xattr on behalf of the given userid''' + log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) l = getlock(endpoint, filepath, _userid) if not l: + log.warning('msg="Failed to refreshlock" filepath="%s" appname="%s" reason="%s"' % + (filepath, appname, 'File is not locked')) raise IOError('File was not locked') - if l['h'] != appname and l['h'] != 'wopi': - raise IOError('File is locked by %s' % l['h']) + if l['app_name'] != appname and l['app_name'] != 'wopi': + log.warning('msg="Failed to refreshlock" filepath="%s" appname="%s" reason="%s"' % + (filepath, appname, 'File is locked by %s' % l['app_name'])) + raise IOError('File is locked by %s' % l['app_name']) + log.debug('msg="Invoked refreshlock" filepath="%s" value="%s"' % (filepath, value)) # this is non-atomic, but the lock was already held - setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value)) + setxattr(endpoint, filepath, '0:0', common.LOCKKEY, common.genrevalock(appname, value), None) -def unlock(endpoint, filepath, _userid, _appname): +def unlock(endpoint, filepath, _userid, _appname, value): '''Remove the lock as an xattr on behalf of the given userid''' - rmxattr(endpoint, filepath, '0:0', common.LOCKKEY) + log.debug('msg="Invoked unlock" filepath="%s" value="%s' % (filepath, value)) + rmxattr(endpoint, filepath, '0:0', common.LOCKKEY, None) -def readfile(_endpoint, filepath, _userid): +def readfile(_endpoint, filepath, _userid, _lockid): '''Read a file on behalf of the given userid. Note that the function is a generator, managed by Flask.''' log.debug('msg="Invoking readFile" filepath="%s"' % filepath) try: @@ -159,7 +166,7 @@ def readfile(_endpoint, filepath, _userid): yield IOError(e) -def writefile(_endpoint, filepath, _userid, content, islock=False): +def writefile(_endpoint, filepath, _userid, content, _lockid, islock=False): '''Write a file via xroot on behalf of the given userid. The entire content is written and any pre-existing file is deleted (or moved to the previous version if supported). With islock=True, the file is opened with O_CREAT|O_EXCL.''' @@ -200,7 +207,7 @@ def writefile(_endpoint, filepath, _userid, content, islock=False): (filepath, (tend-tstart)*1000, islock)) -def renamefile(_endpoint, origfilepath, newfilepath, _userid): +def renamefile(_endpoint, origfilepath, newfilepath, _userid, _lockid): '''Rename a file from origfilepath to newfilepath on behalf of the given userid.''' try: os.rename(_getfilepath(origfilepath), _getfilepath(newfilepath)) @@ -210,7 +217,7 @@ def renamefile(_endpoint, origfilepath, newfilepath, _userid): def removefile(_endpoint, filepath, _userid, force=False): '''Remove a file on behalf of the given userid. - The force argument is irrelevant and ignored for local storage.''' + The force argument is irrelevant and ignored for local storage.''' try: os.remove(_getfilepath(filepath)) except (FileNotFoundError, PermissionError, IsADirectoryError, OSError) as e: diff --git a/src/core/wopi.py b/src/core/wopi.py index 6fb65fa6..608f5c30 100644 --- a/src/core/wopi.py +++ b/src/core/wopi.py @@ -3,6 +3,8 @@ srv.py Implementation of the core WOPI API + +Main author: Giuseppe.LoPresti@cern.ch, CERN/IT-ST ''' import time @@ -10,14 +12,14 @@ import configparser import json import http.client -from urllib.parse import quote_plus as url_quote_plus +from datetime import datetime from more_itertools import peekable import jwt import flask import core.wopiutils as utils import core.commoniface as common -IO_ERROR = 'I/O Error' +IO_ERROR = 'I/O Error, please contact support' # convenience references to global entities st = None @@ -31,11 +33,21 @@ def checkFileInfo(fileid): srv.refreshconfig() try: acctok = jwt.decode(flask.request.args['access_token'], srv.wopisecret, algorithms=['HS256']) - acctok['viewmode'] = utils.ViewMode(acctok['viewmode']) if acctok['exp'] < time.time(): raise jwt.exceptions.ExpiredSignatureError - log.info('msg="CheckFileInfo" user="%s" filename="%s" fileid="%s" token="%s"' % - (acctok['userid'][-20:], acctok['filename'], fileid, flask.request.args['access_token'][-20:])) + wopits = 'NA' + if 'X-WOPI-TimeStamp' in flask.request.headers: + # typically not present, but if it's there it must be checked for expiration (comes from WOPI validator tests) + try: + wopits = int(flask.request.headers['X-WOPI-Timestamp'])/10000000 # convert .NET Ticks to seconds since AD 1 + if wopits < (datetime.utcnow() - datetime(1, 1, 1)).total_seconds() - 20*60: + # timestamps older than 20 minutes must be considered expired + raise ValueError + except ValueError: + raise KeyError('Invalid or expired X-WOPI-Timestamp header') + log.info('msg="CheckFileInfo" user="%s" filename="%s" fileid="%s" token="%s" wopits="%s"' % + (acctok['userid'][-20:], acctok['filename'], fileid, flask.request.args['access_token'][-20:], wopits)) + acctok['viewmode'] = utils.ViewMode(acctok['viewmode']) statInfo = st.statx(acctok['endpoint'], acctok['filename'], acctok['userid'], versioninv=1) # compute some entities for the response wopiSrc = 'WOPISrc=%s&access_token=%s' % (utils.generateWopiSrc(fileid), flask.request.args['access_token']) @@ -43,9 +55,9 @@ def checkFileInfo(fileid): fmd = {} fmd['BaseFileName'] = fmd['BreadcrumbDocName'] = os.path.basename(acctok['filename']) furl = acctok['folderurl'] - # encode the path part as it is going to be an URL GET argument - fmd['BreadcrumbFolderUrl'] = furl[:furl.find('=')+1] + url_quote_plus(furl[furl.find('=')+1:]) if furl != '/' else '' + fmd['BreadcrumbFolderUrl'] = furl if furl != '/' else '' if acctok['username'] == '': + fmd['IsAnonymousUser'] = True fmd['UserFriendlyName'] = 'Guest ' + utils.randomString(3) if '?path' in furl and furl[-1] != '/' and furl[-1] != '=': # this is a subfolder of a public share, show it @@ -65,14 +77,15 @@ def checkFileInfo(fileid): fmd['UserId'] = acctok['wopiuser'] # typically same as OwnerId; different when accessing shared documents fmd['Size'] = statInfo['size'] # TODO the version is generated like this in ownCloud: 'V' . $file->getEtag() . \md5($file->getChecksum()); - fmd['Version'] = statInfo['mtime'] # mtime is used as version here + fmd['Version'] = 'v%d' % statInfo['mtime'] # mtime is used as version here fmd['SupportsExtendedLockLength'] = fmd['SupportsGetLock'] = True fmd['SupportsUpdate'] = fmd['UserCanWrite'] = fmd['SupportsLocks'] = \ fmd['SupportsDeleteFile'] = acctok['viewmode'] == utils.ViewMode.READ_WRITE fmd['UserCanNotWriteRelative'] = acctok['viewmode'] != utils.ViewMode.READ_WRITE + fmd['SupportsContainers'] = False # TODO this is all to be implemented fmd['HostViewUrl'] = '%s%s%s' % (acctok['appviewurl'], '&' if '?' in acctok['appviewurl'] else '?', wopiSrc) fmd['HostEditUrl'] = '%s%s%s' % (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', wopiSrc) - fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and utils.ViewMode.READ_WRITE + fmd['SupportsRename'] = fmd['UserCanRename'] = enablerename and (acctok['viewmode'] == utils.ViewMode.READ_WRITE) # populate app-specific metadata if acctok['appname'].find('Microsoft') > 0: # the following is to enable the 'Edit in Word/Excel/PowerPoint' (desktop) action (probably broken) @@ -88,8 +101,9 @@ def checkFileInfo(fileid): res = flask.Response(json.dumps(fmd), mimetype='application/json') # amend sensitive metadata for the logs - fmd['HostViewUrl'] = fmd['HostEditUrl'] = fmd['DownloadUrl'] = '_amended_' - log.info('msg="File metadata response" token="%s" metadata="%s"' % (flask.request.args['access_token'][-20:], fmd)) + fmd['HostViewUrl'] = fmd['HostEditUrl'] = fmd['DownloadUrl'] = '_redacted_' + log.info('msg="File metadata response" token="%s" session="%s" metadata="%s"' % + (flask.request.args['access_token'][-20:], flask.request.headers.get('X-WOPI-SessionId'), fmd)) return res except IOError as e: log.info('msg="Requested file not found" filename="%s" token="%s" error="%s"' % @@ -114,13 +128,18 @@ def getFile(fileid): log.info('msg="GetFile" user="%s" filename="%s" fileid="%s" token="%s"' % (acctok['userid'][-20:], acctok['filename'], fileid, flask.request.args['access_token'][-20:])) # get the file reader generator - f = peekable(st.readfile(acctok['endpoint'], acctok['filename'], acctok['userid'])) + # TODO for the time being we do not look if the file is locked. Once exclusive locks are implemented in Reva, + # the lock must be fetched prior to the following call in order to access the file. + f = peekable(st.readfile(acctok['endpoint'], acctok['filename'], acctok['userid'], None)) firstchunk = f.peek() if isinstance(firstchunk, IOError): return ('Failed to fetch file from storage: %s' % firstchunk), http.client.INTERNAL_SERVER_ERROR + # stat the file to get the version, TODO this should be cached inside the access token + statInfo = st.stat(acctok['endpoint'], acctok['filename'], acctok['userid']) # stream file from storage to client resp = flask.Response(f, mimetype='application/octet-stream') resp.status_code = http.client.OK + resp.headers['X-WOPI-ItemVersion'] = 'v%d' % statInfo['mtime'] return resp except StopIteration as e: # File is empty, still return OK (strictly speaking, we should return 204 NO_CONTENT) @@ -153,14 +172,13 @@ def setLock(fileid, reqheaders, acctok): else: savetime = None if not savetime or not savetime.isdigit(): - return utils.makeConflictResponse(op, '', lock, oldLock, acctok['filename'], + return utils.makeConflictResponse(op, None, lock, oldLock, acctok['filename'], 'The file was not locked' + ' and got modified' if validateTarget else '') # LOCK or REFRESH_LOCK: atomically set the lock to the given one, including the expiration time, # and return conflict response if the file was already locked try: - return utils.storeWopiLock(fileid, op, lock, oldLock, acctok, \ - os.path.splitext(acctok['filename'])[1] not in srv.nonofficetypes) + return utils.storeWopiLock(fileid, op, lock, oldLock, acctok) except IOError as e: # expected failures are handled in storeWopiLock log.error('msg="%s: unable to store WOPI lock" filename="%s" token="%s" lock="%s" reason="%s"' % @@ -173,9 +191,9 @@ def getLock(fileid, _reqheaders_unused, acctok): resp = flask.Response() lock, _ = utils.retrieveWopiLock(fileid, 'GETLOCK', '', acctok) resp.status_code = http.client.OK if lock else http.client.NOT_FOUND - if lock: - resp.headers['X-WOPI-Lock'] = lock - # for statistical purposes, check whether a lock exists and update internal bookkeeping + resp.headers['X-WOPI-Lock'] = lock if lock else '' + # for statistical purposes, check whether a lock exists and update internal bookkeeping + if lock and lock != 'External': try: # the file was already opened for write, check whether this is a new user if not acctok['username'] in srv.openfiles[acctok['filename']][1]: @@ -184,22 +202,13 @@ def getLock(fileid, _reqheaders_unused, acctok): if len(srv.openfiles[acctok['filename']][1]) > 1: # for later monitoring, explicitly log that this file is being edited by at least two users log.info('msg="Collaborative editing detected" filename="%s" token="%s" users="%s"' % - (acctok['filename'], flask.request.args['access_token'][-20:], - list(srv.openfiles[acctok['filename']][1]))) + (acctok['filename'], flask.request.args['access_token'][-20:], + list(srv.openfiles[acctok['filename']][1]))) except KeyError: # existing lock but missing srv.openfiles[acctok['filename']] ? log.warning('msg="Repopulating missing metadata" filename="%s" token="%s" friendlyname="%s"' % (acctok['filename'], flask.request.args['access_token'][-20:], acctok['username'])) srv.openfiles[acctok['filename']] = (time.asctime(), set([acctok['username']])) - # we might want to check if a non-WOPI lock exists for this file: - #if os.path.splitext(acctok['filename'])[1] not in srv.nonofficetypes: - # try: - # lockstat = st.stat(acctok['endpoint'], utils.getLibreOfficeLockName(acctok['filename']), acctok['userid']) - # return utils.makeConflictResponse('GetLock', 'External App', '', '', acctok['filename'], \ - # 'The file was locked by LibreOffice for Desktop') - # except IOError: - # pass - # however implications have to be properly understood as we've seen cases of locks left behind return resp @@ -208,19 +217,22 @@ def unlock(fileid, reqheaders, acctok): lock = reqheaders['X-WOPI-Lock'] retrievedLock, _ = utils.retrieveWopiLock(fileid, 'UNLOCK', lock, acctok) if not utils.compareWopiLocks(retrievedLock, lock): - return utils.makeConflictResponse('UNLOCK', retrievedLock, lock, '', acctok['filename']) - # OK, the lock matches. Remove any extended attribute related to locks and conflicts handling + return utils.makeConflictResponse('UNLOCK', retrievedLock, lock, 'NA', acctok['filename'], 'Lock mismatch') + # OK, the lock matches. Remove the lock try: - st.unlock(acctok['endpoint'], acctok['filename'], acctok['userid'], acctok['appname']) - except IOError: - # ignore, it's not worth to report anything here - pass + # validate that the underlying file is still there + statInfo = st.stat(acctok['endpoint'], acctok['filename'], acctok['userid']) + st.unlock(acctok['endpoint'], acctok['filename'], acctok['userid'], acctok['appname'], utils.encodeLock(lock)) + except IOError as e: + if common.ENOENT_MSG in e: + return 'File not found', http.client.NOT_FOUND + return IO_ERROR, http.client.INTERNAL_SERVER_ERROR try: # also remove the LibreOffice-compatible lock file when relevant if os.path.splitext(acctok['filename'])[1] not in srv.nonofficetypes: - st.removefile(acctok['endpoint'], utils.getLibreOfficeLockName(acctok['filename']), acctok['userid'], force=True) + st.removefile(acctok['endpoint'], utils.getLibreOfficeLockName(acctok['filename']), acctok['userid'], True) except IOError: - # same as above + # ignore, it's not worth to report anything here pass # and update our internal list of opened files try: @@ -228,7 +240,10 @@ def unlock(fileid, reqheaders, acctok): except KeyError: # already removed? pass - return 'OK', http.client.OK + resp = flask.Response() + resp.status_code = http.client.OK + resp.headers['X-WOPI-ItemVersion'] = 'v%d' % statInfo['mtime'] + return resp def putRelative(fileid, reqheaders, acctok): @@ -243,9 +258,10 @@ def putRelative(fileid, reqheaders, acctok): suggTarget, relTarget, overwriteTarget, flask.request.args['access_token'][-20:])) # either one xor the other must be present; note we can't use `^` as we have a mix of str and NoneType if (suggTarget and relTarget) or (not suggTarget and not relTarget): - return 'Not supported', http.client.NOT_IMPLEMENTED + return '', http.client.NOT_IMPLEMENTED if suggTarget: - # the suggested target is a filename that can be changed to avoid collisions + # the suggested target is a UTF7-encoded (!) filename that can be changed to avoid collisions + suggTarget = suggTarget.encode().decode('utf-7') if suggTarget[0] == '.': # we just have the extension here targetName = os.path.splitext(acctok['filename'])[0] + suggTarget else: @@ -262,13 +278,13 @@ def putRelative(fileid, reqheaders, acctok): # OK, the targetName is good to go break # we got another error with this file, fail - log.info('msg="PutRelative" user="%s" filename="%s" token="%s" suggTarget="%s" error="%s"' % - (acctok['userid'][-20:], targetName, flask.request.args['access_token'][-20:], \ - suggTarget, str(e))) - return 'Illegal filename %s: %s' % (targetName, e), http.client.BAD_REQUEST + log.warning('msg="PutRelative" user="%s" filename="%s" token="%s" suggTarget="%s" error="%s"' % + (acctok['userid'][-20:], targetName, flask.request.args['access_token'][-20:], \ + suggTarget, str(e))) + return '', http.client.BAD_REQUEST else: - # the relative target is a filename to be respected, and that may overwrite an existing file - relTarget = os.path.dirname(acctok['filename']) + os.path.sep + relTarget # make full path + # the relative target is a UTF7-encoded filename to be respected, and that may overwrite an existing file + relTarget = os.path.dirname(acctok['filename']) + os.path.sep + relTarget.encode().decode('utf-7') # make full path try: # check for file existence + lock fileExists = st.stat(acctok['endpoint'], relTarget, acctok['userid']) @@ -276,17 +292,20 @@ def putRelative(fileid, reqheaders, acctok): except IOError: fileExists = False if fileExists and (not overwriteTarget or retrievedTargetLock): - return utils.makeConflictResponse('PUT_RELATIVE', retrievedTargetLock, '', '', relTarget, - 'Target file already exists') + return utils.makeConflictResponse('PUT_RELATIVE', retrievedTargetLock, 'NA', 'NA', relTarget, + {'message': 'Target file already exists', + # specs (the WOPI validator) require these to be populated with valid values + 'Name': os.path.basename(relTarget), + 'Url': utils.generateWopiSrc('0'), + }) # else we can use the relative target targetName = relTarget # either way, we now have a targetName to save the file: attempt to do so try: utils.storeWopiFile(flask.request, None, acctok, utils.LASTSAVETIMEKEY, targetName) except IOError as e: - log.info('msg="Error writing file" filename="%s" token="%s" error="%s"' % - (targetName, flask.request.args['access_token'][-20:], e)) - return IO_ERROR, http.client.INTERNAL_SERVER_ERROR + utils.storeForRecovery(flask.request.get_data(), targetName, flask.request.args['access_token'][-20:], e) # lgtm [py/path-injection] + return '', http.client.INTERNAL_SERVER_ERROR # generate an access token for the new file log.info('msg="PutRelative: generating new access token" user="%s" filename="%s" ' \ 'mode="ViewMode.READ_WRITE" friendlyname="%s"' % @@ -302,8 +321,10 @@ def putRelative(fileid, reqheaders, acctok): putrelmd['HostEditUrl'] = '%s%sWOPISrc=%s&access_token=%s' % \ (acctok['appediturl'], '&' if '?' in acctok['appediturl'] else '?', utils.generateWopiSrc(inode), newacctok) + resp = flask.Response(json.dumps(putrelmd), mimetype='application/json') + putrelmd['Url'] = putrelmd['HostEditUrl'] = '_redacted_' log.debug('msg="PutRelative response" token="%s" metadata="%s"' % (newacctok[-20:], putrelmd)) - return flask.Response(json.dumps(putrelmd), mimetype='application/json') + return resp def deleteFile(fileid, _reqheaders_unused, acctok): @@ -311,7 +332,8 @@ def deleteFile(fileid, _reqheaders_unused, acctok): retrievedLock, _ = utils.retrieveWopiLock(fileid, 'DELETE', '', acctok) if retrievedLock is not None: # file is locked and cannot be deleted - return utils.makeConflictResponse('DELETE', retrievedLock, '', '', acctok['filename']) + return utils.makeConflictResponse('DELETE', retrievedLock, 'NA', 'NA', acctok['filename'], \ + 'Cannot delete a locked file') try: st.removefile(acctok['endpoint'], acctok['filename'], acctok['userid']) return 'OK', http.client.OK @@ -326,17 +348,17 @@ def renameFile(fileid, reqheaders, acctok): lock = reqheaders['X-WOPI-Lock'] if 'X-WOPI-Lock' in reqheaders else None retrievedLock, _ = utils.retrieveWopiLock(fileid, 'RENAMEFILE', lock, acctok) if retrievedLock is not None and not utils.compareWopiLocks(retrievedLock, lock): - return utils.makeConflictResponse('RENAMEFILE', retrievedLock, lock, '', acctok['filename']) + return utils.makeConflictResponse('RENAMEFILE', retrievedLock, lock, 'NA', acctok['filename']) try: # the destination name comes without base path and without extension targetName = os.path.dirname(acctok['filename']) + '/' + targetName + os.path.splitext(acctok['filename'])[1] log.info('msg="RenameFile" user="%s" filename="%s" token="%s" targetname="%s"' % (acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:], targetName)) - st.renamefile(acctok['endpoint'], acctok['filename'], targetName, acctok['userid']) + st.renamefile(acctok['endpoint'], acctok['filename'], targetName, acctok['userid'], utils.encodeLock(retrievedLock)) # also rename the locks if os.path.splitext(acctok['filename'])[1] not in srv.nonofficetypes: st.renamefile(acctok['endpoint'], utils.getLibreOfficeLockName(acctok['filename']), \ - utils.getLibreOfficeLockName(targetName), acctok['userid']) + utils.getLibreOfficeLockName(targetName), acctok['userid'], None) # prepare and send the response as JSON renamemd = {} renamemd['Name'] = reqheaders['X-WOPI-RequestedName'] @@ -364,13 +386,19 @@ def _createNewFile(fileid, acctok): return 'File exists', http.client.CONFLICT except IOError: # indeed the file did not exist, so we write it for the first time - utils.storeWopiFile(flask.request, None, acctok, utils.LASTSAVETIMEKEY) - log.info('msg="File stored successfully" action="editnew" user="%s" filename="%s" token="%s"' % - (acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:])) - # and we keep track of it as an open file with timestamp = Epoch, despite not having any lock yet. - # XXX this is to work around an issue with concurrent editing of newly created files (cf. iopOpen) - srv.openfiles[acctok['filename']] = ('0', set([acctok['username']])) - return 'OK', http.client.OK + try: + utils.storeWopiFile(flask.request, None, acctok, utils.LASTSAVETIMEKEY) + log.info('msg="File stored successfully" action="editnew" user="%s" filename="%s" token="%s"' % + (acctok['userid'][-20:], acctok['filename'], flask.request.args['access_token'][-20:])) + # and we keep track of it as an open file with timestamp = Epoch, despite not having any lock yet. + # XXX this is to work around an issue with concurrent editing of newly created files (cf. iopOpen) + srv.openfiles[acctok['filename']] = ('0', set([acctok['username']])) + return 'OK', http.client.OK + except IOError as e: + utils.storeForRecovery(flask.request.get_data(), acctok['filename'], \ + flask.request.args['access_token'][-20:], e) + return IO_ERROR, http.client.INTERNAL_SERVER_ERROR + def putFile(fileid): @@ -380,73 +408,82 @@ def putFile(fileid): acctok = jwt.decode(flask.request.args['access_token'], srv.wopisecret, algorithms=['HS256']) if acctok['exp'] < time.time(): raise jwt.exceptions.ExpiredSignatureError - if 'X-WOPI-Lock' not in flask.request.headers: - # no lock given: assume we are in creation mode (cf. editnew WOPI action) - return _createNewFile(fileid, acctok) - # otherwise, check that the caller holds the current lock on the file - lock = flask.request.headers['X-WOPI-Lock'] - retrievedLock, lockHolder = utils.retrieveWopiLock(fileid, 'PUTFILE', lock, acctok) - if retrievedLock is None: - return utils.makeConflictResponse('PUTFILE', retrievedLock, lock, '', acctok['filename'], \ - 'Cannot overwrite unlocked file') - if not utils.compareWopiLocks(retrievedLock, lock): - return utils.makeConflictResponse('PUTFILE', retrievedLock, lock, '', acctok['filename'], \ - 'Cannot overwrite file locked by %s' % \ - (lockHolder if lockHolder != 'wopi' else 'another application')) - # OK, we can save the file now - log.info('msg="PutFile" user="%s" filename="%s" fileid="%s" action="edit" token="%s"' % - (acctok['userid'][-20:], acctok['filename'], fileid, flask.request.args['access_token'][-20:])) - try: - # check now the destination file against conflicts - savetime = st.getxattr(acctok['endpoint'], acctok['filename'], acctok['userid'], utils.LASTSAVETIMEKEY) - mtime = None - mtime = st.stat(acctok['endpoint'], acctok['filename'], acctok['userid'])['mtime'] - if savetime is None or not savetime.isdigit() or int(mtime) > int(savetime): - # no xattr was there or we got our xattr but mtime is more recent: someone may have updated the file - # from a different source (e.g. FUSE or SMB mount), therefore force conflict. - # Note we can't get a time resolution better than one second! - log.info('msg="Forcing conflict based on lastWopiSaveTime" user="%s" filename="%s" ' \ - 'savetime="%s" lastmtime="%s" token="%s"' % - (acctok['userid'][-20:], acctok['filename'], savetime, mtime, flask.request.args['access_token'][-20:])) - raise IOError - log.debug('msg="Got lastWopiSaveTime" user="%s" filename="%s" savetime="%s" lastmtime="%s" token="%s"' % - (acctok['userid'][-20:], acctok['filename'], savetime, mtime, flask.request.args['access_token'][-20:])) - - except IOError: - # either the file was deleted or it was updated/overwritten by others: force conflict - newname, ext = os.path.splitext(acctok['filename']) - # !!! typical EFSS formats are like '_conflict--