diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d393bb6f..e2b5f8422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Change Log +## 1.21.1 + +### Improvements +- Parse qptiff and imagej vendor information ([#1168](../../pull/1168)) + +## 1.21.0 + +### Improvements +- getPixel method and endpoint now returns rawer values ([#1155](../../pull/1155)) +- Allow more sophisticated filtering on item lists from the Girder client ([#1157](../../pull/1157), [#1159](../../pull/1159), [#1161](../../pull/1161)) + +### Changes +- Rename tiff exceptions to be better follow python standards ([#1162](../../pull/1162)) +- Require a newer version of girder ([#1163](../../pull/1163)) +- Add manual mongo index names where index names might get too long ([#1166](../../pull/1166)) +- Avoid using $unionWith for annotation searches as it isn't supported everywhere ([#1167](../../pull/1167)) + +### Bug Fixes +- The deepzoom tile source misreported the format of its tile output ([#1158](../../pull/1158)) +- Guard against errors in a log message ([#1164](../../pull/1164)) +- Fix thumbnail query typo ([#1165](../../pull/1165)) + ## 1.20.6 ### Improvements diff --git a/girder/girder_large_image/models/image_item.py b/girder/girder_large_image/models/image_item.py index 051cceac0..4431f96dc 100644 --- a/girder/girder_large_image/models/image_item.py +++ b/girder/girder_large_image/models/image_item.py @@ -499,7 +499,7 @@ def removeThumbnailFiles(self, item, keep=0, sort=None, imageKey=None, } if imageKey and key == 'isLargeImageThumbnail': if imageKey == 'none': - query['thumbnailKey'] = {'not': {'$regex': '"imageKey":'}} + query['thumbnailKey'] = {'$not': {'$regex': '"imageKey":'}} else: query['thumbnailKey'] = {'$regex': '"imageKey":"%s"' % imageKey} query.update(kwargs) diff --git a/girder/girder_large_image/rest/__init__.py b/girder/girder_large_image/rest/__init__.py index 701d3397e..21b8090cc 100644 --- a/girder/girder_large_image/rest/__init__.py +++ b/girder/girder_large_image/rest/__init__.py @@ -35,6 +35,11 @@ def altItemFind(self, folderId, text, name, limit, offset, sort, filters=None): text = None except Exception as exc: logger.warning('Failed to parse _filter_ from text field: %r', exc) + if filters: + try: + logger.debug('Item find filters: %s', json.dumps(filters)) + except Exception: + pass if recurse: return _itemFindRecursive( self, origItemFind, folderId, text, name, limit, offset, sort, filters) diff --git a/girder/girder_large_image/web_client/package.json b/girder/girder_large_image/web_client/package.json index 04de14806..f22866988 100644 --- a/girder/girder_large_image/web_client/package.json +++ b/girder/girder_large_image/web_client/package.json @@ -27,7 +27,7 @@ "vue-color": "^2.8.1", "vue-loader": "~15.9.8", "vue-template-compiler": "~2.6.14", - "webpack": "^2.7.0", + "webpack": "^3", "yaml": "^2.1.1" }, "main": "./index.js", diff --git a/girder/girder_large_image/web_client/views/itemList.js b/girder/girder_large_image/web_client/views/itemList.js index 7b397bfa8..7ec71b164 100644 --- a/girder/girder_large_image/web_client/views/itemList.js +++ b/girder/girder_large_image/web_client/views/itemList.js @@ -219,50 +219,97 @@ wrap(ItemListWidget, 'render', function (render) { addToRoute({filter: this._generalFilter}); }; + this._unescapePhrase = (val) => { + if (val !== undefined) { + val = val.replace('\\\'', '\'').replace('\\"', '"').replace('\\\\', '\\'); + } + return val; + }; + this._setFilter = () => { const val = this._generalFilter; let filter; const usedPhrases = {}; const columns = (this._confList() || {}).columns || []; if (val !== undefined && val !== '' && columns.length) { + // a value can be surrounded by single or double quotes, which will + // be removed. + const quotedValue = /((?:"((?:[^\\"]|\\\\|\\")*)(?:"|$)|'((?:[^\\']|\\\\|\\')*)(?:'|$)|([^:,\s]+)))/g; + const phraseRE = new RegExp( + new RegExp('((?:' + quotedValue.source + ':|))').source + + /(-?)/.source + + quotedValue.source + + new RegExp('((?:,' + quotedValue.source + ')*)').source, 'g'); filter = []; - val.match(/"[^"]*"|'[^']*'|\S+/g).forEach((phrase) => { - if (!phrase.length || usedPhrases[phrase]) { - return; + [...val.matchAll(phraseRE)].forEach((match) => { + const coltag = this._unescapePhrase(match[5] || match[4] || match[3]); + const phrase = this._unescapePhrase(match[10] || match[9] || match[8]); + const negation = match[6] === '-'; + var phrases = [{phrase: phrase, exact: match[8] !== undefined}]; + if (match[11]) { + [...match[11].matchAll(quotedValue)].forEach((submatch) => { + const subphrase = this._unescapePhrase(submatch[4] || submatch[3] || submatch[2]); + // remove dupes? + if (subphrase && subphrase.length) { + phrases.push({phrase: subphrase, exact: submatch[2] !== undefined}); + } + }); } - usedPhrases[phrase] = true; - if (phrase[0] === phrase.substr(phrase.length - 1) && ['"', "'"].includes(phrase[0])) { - phrase = phrase.substr(1, phrase.length - 2); + const key = `${coltag}:` + phrases.join('|||'); + if (!phrases.length || usedPhrases[key]) { + return; } - const numval = +phrase; - /* If numval is a non-zero number not in exponential notation. - * delta is the value of one for the least significant digit. - * This will be NaN if phrase is not a number. */ - const delta = Math.abs(+numval.toString().replace(/\d(?=.*[1-9](0*\.|)0*$)/g, '0').replace(/[1-9]/, '1')); - // escape for regex - phrase = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + usedPhrases[key] = true; const clause = []; - columns.forEach((col) => { - let key; + phrases.forEach(({phrase, exact}) => { + const numval = +phrase; + /* If numval is a non-zero number not in exponential + * notation, delta is the value of one for the least + * significant digit. This will be NaN if phrase is not a + * number. */ + const delta = Math.abs(+numval.toString().replace(/\d(?=.*[1-9](0*\.|)0*$)/g, '0').replace(/[1-9]/, '1')); + // escape for regex + phrase = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - if (col.type === 'record' && col.value !== 'controls') { - key = col.value; - } else if (col.type === 'metadata') { - key = 'meta.' + col.value; - } - if (key) { - clause.push({[key]: {$regex: phrase, $options: 'i'}}); - if (!_.isNaN(numval)) { - clause.push({[key]: {$eq: numval}}); - if (numval > 0 && delta) { - clause.push({[key]: {$gte: numval, $lt: numval + delta}}); - } else if (numval < 0 && delta) { - clause.push({[key]: {$lte: numval, $gt: numval + delta}}); + columns.forEach((col) => { + let key; + if (coltag && + coltag.localeCompare(col.title || col.value, undefined, {sensitivity: 'accent'}) && + coltag.localeCompare(col.value, undefined, {sensitivity: 'accent'}) + ) { + return; + } + if (col.type === 'record' && col.value !== 'controls') { + key = col.value; + } else if (col.type === 'metadata') { + key = 'meta.' + col.value; + } + if (!coltag && !exact) { + const r = new RegExp('^' + (phrase.substr(phrase.length - 1) === ':' ? phrase.substr(0, phrase.length - 1) : phrase), 'i'); + if (r.exec(col.value) || r.exec(col.title || col.value)) { + clause.push({[key]: {$exists: true}}); } } - } + if (key && exact) { + clause.push({[key]: {$regex: '^' + phrase + '$', $options: 'i'}}); + } else if (key) { + clause.push({[key]: {$regex: phrase, $options: 'i'}}); + if (!_.isNaN(numval)) { + clause.push({[key]: {$eq: numval}}); + if (numval > 0 && delta) { + clause.push({[key]: {$gte: numval, $lt: numval + delta}}); + } else if (numval < 0 && delta) { + clause.push({[key]: {$lte: numval, $gt: numval + delta}}); + } + } + } + }); }); - filter.push({$or: clause}); + if (clause.length > 0) { + filter.push(!negation ? {$or: clause} : {$nor: clause}); + } else if (!negation) { + filter.push({$or: [{_no_such_value_: '_no_such_value_'}]}); + } }); if (filter.length === 0) { filter = undefined; @@ -296,7 +343,16 @@ wrap(ItemListWidget, 'render', function (render) { func = 'before'; } if (base.length) { - base[func]('Filter: '); + base[func]('Filter: '); if (this._generalFilter) { root.find('.li-item-list-filter-input').val(this._generalFilter); } diff --git a/girder/setup.py b/girder/setup.py index ff5c0b4cd..a939af74f 100644 --- a/girder/setup.py +++ b/girder/setup.py @@ -53,7 +53,7 @@ def prerelease_local_scheme(version): 'Programming Language :: Python :: 3.11', ], install_requires=[ - 'girder>=3.0.4', + 'girder>=3.1.18', 'girder-jobs>=3.0.3', f'large_image{limit_version}', 'importlib-metadata<5 ; python_version < "3.8"', diff --git a/girder_annotation/girder_large_image_annotation/handlers.py b/girder_annotation/girder_large_image_annotation/handlers.py index 4cdaa602c..c7a5177f5 100644 --- a/girder_annotation/girder_large_image_annotation/handlers.py +++ b/girder_annotation/girder_large_image_annotation/handlers.py @@ -111,13 +111,10 @@ def process_annotations(event): # noqa: C901 item = results['item'] user = results['user'] - startTime = time.time() file = File().load( event.info.get('file', {}).get('_id'), level=AccessType.READ, user=user ) - if time.time() - startTime > 10: - logger.info('Loaded annotation file in %5.3fs', time.time() - startTime) startTime = time.time() if not file: diff --git a/girder_annotation/girder_large_image_annotation/models/annotationelement.py b/girder_annotation/girder_large_image_annotation/models/annotationelement.py index 46d22c6aa..dcd0d809f 100644 --- a/girder_annotation/girder_large_image_annotation/models/annotationelement.py +++ b/girder_annotation/girder_large_image_annotation/models/annotationelement.py @@ -63,16 +63,22 @@ def initialize(self): ('bbox.lowx', SortDir.DESCENDING), ('bbox.highx', SortDir.ASCENDING), ('bbox.size', SortDir.DESCENDING), - ], {}), + ], { + 'name': 'annotationBboxIdx' + }), ([ ('annotationId', SortDir.ASCENDING), ('bbox.size', SortDir.DESCENDING), - ], {}), + ], { + 'name': 'annotationBboxSizeIdx' + }), ([ ('annotationId', SortDir.ASCENDING), ('_version', SortDir.DESCENDING), ('element.group', SortDir.ASCENDING), - ], {}), + ], { + 'name': 'annotationGroupIdx' + }), ([ ('created', SortDir.ASCENDING), ('_version', SortDir.ASCENDING), diff --git a/girder_annotation/girder_large_image_annotation/rest/annotation.py b/girder_annotation/girder_large_image_annotation/rest/annotation.py index 98d25fe17..6483370e0 100644 --- a/girder_annotation/girder_large_image_annotation/rest/annotation.py +++ b/girder_annotation/girder_large_image_annotation/rest/annotation.py @@ -582,20 +582,7 @@ def deleteItemAnnotations(self, item): def getFolderAnnotations(self, id, recurse, user, limit=False, offset=False, sort=False, sortDir=False, count=False): - recursivePipeline = [ - {'$graphLookup': { - 'from': 'folder', - 'startWith': '$_id', - 'connectFromField': '_id', - 'connectToField': 'parentId', - 'as': '__children' - }}, - {'$unwind': {'path': '$__children'}}, - {'$replaceRoot': {'newRoot': '$__children'}}, - {'$unionWith': { - 'coll': 'folder', - 'pipeline': [{'$match': {'_id': ObjectId(id)}}] - }}] if recurse else [] + accessPipeline = [ {'$match': { '$or': [ @@ -612,36 +599,59 @@ def getFolderAnnotations(self, id, recurse, user, limit=False, offset=False, sor ] }} ] if not user['admin'] else [] - pipeline = [ - {'$match': {'_id': 'none'}}, - {'$unionWith': { - 'coll': 'folder', - 'pipeline': [{'$match': {'_id': ObjectId(id)}}] + - recursivePipeline + - [{'$lookup': { - 'from': 'item', - 'localField': '_id', - 'foreignField': 'folderId', - 'as': '__items' - }}, {'$lookup': { - 'from': 'annotation', - 'localField': '__items._id', - 'foreignField': 'itemId', - 'as': '__annotations' - }}, {'$unwind': '$__annotations'}, - {'$replaceRoot': {'newRoot': '$__annotations'}}, - {'$match': {'_active': {'$ne': False}}} - ] + accessPipeline + recursivePipeline = [ + {'$match': {'_id': ObjectId(id)}}, + {'$facet': { + 'documents1': [{'$match': {'_id': ObjectId(id)}}], + 'documents2': [ + {'$graphLookup': { + 'from': 'folder', + 'startWith': '$_id', + 'connectFromField': '_id', + 'connectToField': 'parentId', + 'as': '__children' + }}, + {'$unwind': {'path': '$__children'}}, + {'$replaceRoot': {'newRoot': '$__children'}} + ] }}, - ] + {'$project': {'__children': {'$concatArrays': [ + '$documents1', '$documents2' + ]}}}, + {'$unwind': {'path': '$__children'}}, + {'$replaceRoot': {'newRoot': '$__children'}} + ] if recurse else [{'$match': {'_id': ObjectId(id)}}] + + # We are only finding anntoations that we can change the permissions + # on. If we wanted to expose annotations based on a permissions level, + # we need to add a folder access pipeline immediately after the + # recursivePipleine that for write and above would include the + # ANNOTATION_ACCSESS_FLAG + pipeline = recursivePipeline + [ + {'$lookup': { + 'from': 'item', + 'localField': '_id', + 'foreignField': 'folderId', + 'as': '__items' + }}, + {'$lookup': { + 'from': 'annotation', + 'localField': '__items._id', + 'foreignField': 'itemId', + 'as': '__annotations' + }}, + {'$unwind': '$__annotations'}, + {'$replaceRoot': {'newRoot': '$__annotations'}}, + {'$match': {'_active': {'$ne': False}}} + ] + accessPipeline + if count: pipeline += [{'$count': 'count'}] else: pipeline = pipeline + [{'$sort': {sort: sortDir}}] if sort else pipeline pipeline = pipeline + [{'$skip': offset}] if offset else pipeline pipeline = pipeline + [{'$limit': limit}] if limit else pipeline - - return Annotation().collection.aggregate(pipeline) + return Folder().collection.aggregate(pipeline) @autoDescribeRoute( Description('Check if the user owns any annotations for the items in a folder') diff --git a/girder_annotation/test_annotation/test_annotations_rest.py b/girder_annotation/test_annotation/test_annotations_rest.py index 8adfbb262..ca6b26a78 100644 --- a/girder_annotation/test_annotation/test_annotations_rest.py +++ b/girder_annotation/test_annotation/test_annotations_rest.py @@ -16,6 +16,8 @@ from girder_large_image_annotation.models.annotation import Annotation from girder.constants import AccessType + from girder.models.collection import Collection + from girder.models.folder import Folder from girder.models.item import Item from girder.models.setting import Setting except ImportError: @@ -808,3 +810,66 @@ def testMetadataSearch(server, admin, fsAssetstore): params={'q': 'key:key2 value', 'mode': 'li_annotation_metadata', 'types': '["item"]'}) assert utilities.respStatus(resp) == 200 assert len(resp.json['item']) == 0 + + +@pytest.mark.usefixtures('unbindLargeImage', 'unbindAnnotation') +@pytest.mark.plugin('large_image_annotation') +def testFolderEndpoints(server, admin, user): + collection = Collection().createCollection( + 'collection A', user) + colFolderA = Folder().createFolder( + collection, 'folder A', parentType='collection', + creator=user) + colFolderB = Folder().createFolder( + colFolderA, 'folder B', creator=user) + colFolderC = Folder().createFolder( + colFolderA, 'folder C', creator=admin, public=False) + colFolderC = Folder().setAccessList(colFolderC, access={'users': [], 'groups': []}, save=True) + itemA1 = Item().createItem('sample A1', user, colFolderA) + itemA2 = Item().createItem('sample A1', user, colFolderA) + itemB1 = Item().createItem('sample B1', user, colFolderB) + itemB2 = Item().createItem('sample B1', user, colFolderB) + itemC1 = Item().createItem('sample C1', admin, colFolderC) + itemC2 = Item().createItem('sample C1', admin, colFolderC) + Annotation().createAnnotation(itemA1, user, sampleAnnotation) + ann = Annotation().createAnnotation(itemA1, admin, sampleAnnotation, public=False) + Annotation().setAccessList(ann, access={'users': [], 'groups': []}, save=True) + Annotation().createAnnotation(itemA2, user, sampleAnnotation) + Annotation().createAnnotation(itemB1, user, sampleAnnotation) + ann = Annotation().createAnnotation(itemB1, admin, sampleAnnotation, public=False) + Annotation().setAccessList(ann, access={'users': [], 'groups': []}, save=True) + Annotation().createAnnotation(itemB2, user, sampleAnnotation) + Annotation().createAnnotation(itemC1, user, sampleAnnotation) + ann = Annotation().createAnnotation(itemC1, admin, sampleAnnotation, public=False) + Annotation().setAccessList(ann, access={'users': [], 'groups': []}, save=True) + Annotation().createAnnotation(itemC2, user, sampleAnnotation) + + resp = server.request( + path='/annotation/folder/' + str(colFolderA['_id']), user=admin, + params={'recurse': False}) + assert utilities.respStatus(resp) == 200 + assert len(resp.json) == 3 + + resp = server.request( + path='/annotation/folder/' + str(colFolderA['_id']), user=admin, + params={'recurse': True}) + assert utilities.respStatus(resp) == 200 + assert len(resp.json) == 9 + + resp = server.request( + path='/annotation/folder/' + str(colFolderA['_id']), user=user, + params={'recurse': False}) + assert utilities.respStatus(resp) == 200 + assert len(resp.json) == 2 + + resp = server.request( + path='/annotation/folder/' + str(colFolderA['_id']), user=user, + params={'recurse': True}) + assert utilities.respStatus(resp) == 200 + assert len(resp.json) == 6 + + resp = server.request( + path='/annotation/folder/' + str(colFolderC['_id']), user=user, + params={'recurse': True}) + assert utilities.respStatus(resp) == 200 + assert len(resp.json) == 2 diff --git a/large_image/tilesource/base.py b/large_image/tilesource/base.py index f88b91e3f..dec026ce8 100644 --- a/large_image/tilesource/base.py +++ b/large_image/tilesource/base.py @@ -2716,12 +2716,13 @@ def getPixel(self, includeTileRecord=False, **kwargs): # img, format = self.getRegion(format=TILE_FORMAT_PIL, **regionArgs) # where img is the PIL image (rather than tile['tile'], but using # _tileIteratorInfo and the _tileIterator is slightly more efficient. - iterInfo = self._tileIteratorInfo(format=TILE_FORMAT_PIL, **regionArgs) + iterInfo = self._tileIteratorInfo(format=TILE_FORMAT_NUMPY, **regionArgs) if iterInfo is not None: tile = next(self._tileIterator(iterInfo), None) if includeTileRecord: pixel['tile'] = tile - img = tile['tile'] + pixel['value'] = [v.item() for v in tile['tile'][0][0]] + img = _imageToPIL(tile['tile']) if img.size[0] >= 1 and img.size[1] >= 1: if len(img.mode) > 1: pixel.update(dict(zip(img.mode.lower(), img.load()[0, 0]))) diff --git a/requirements-dev.txt b/requirements-dev.txt index d6500ef5a..44c071231 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,5 @@ # Top level dependencies -girder>=3.0.4 ; python_version < '3.8' -girder>=3.0.14 ; python_version >= '3.8' +girder>=3.1.18 girder-jobs>=3.0.3 -e sources/bioformats -e sources/deepzoom diff --git a/requirements-test.txt b/requirements-test.txt index 5807d555e..89b473670 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,5 @@ # Top level dependencies -girder>=3.0.4 ; python_version < '3.8' -girder>=3.0.14 ; python_version >= '3.8' +girder>=3.1.18 girder-jobs>=3.0.3 sources/bioformats sources/deepzoom diff --git a/sources/deepzoom/large_image_source_deepzoom/__init__.py b/sources/deepzoom/large_image_source_deepzoom/__init__.py index 5f9e6268f..762a1c4aa 100644 --- a/sources/deepzoom/large_image_source_deepzoom/__init__.py +++ b/sources/deepzoom/large_image_source_deepzoom/__init__.py @@ -8,7 +8,7 @@ import PIL.Image from large_image.cache_util import LruCacheMetaclass, methodcache -from large_image.constants import TILE_FORMAT_NUMPY, SourcePriority +from large_image.constants import TILE_FORMAT_PIL, SourcePriority from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError from large_image.tilesource import FileTileSource, etreeToDict @@ -113,7 +113,7 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): overlap if x else 0, overlap if y else 0, self.tileWidth + (overlap if x else 0), self.tileHeight + (overlap if y else 0))) - return self._outputTile(tile, TILE_FORMAT_NUMPY, x, y, z, + return self._outputTile(tile, TILE_FORMAT_PIL, x, y, z, pilImageAllowed, numpyAllowed, **kwargs) diff --git a/sources/multi/docs/specification.rst b/sources/multi/docs/specification.rst index 9f6dc0c03..27a873ed6 100644 --- a/sources/multi/docs/specification.rst +++ b/sources/multi/docs/specification.rst @@ -126,7 +126,7 @@ Transforms can be applied to scale the individual sources: y: 360 - path: ./test_orient4.tif position: - scale: 360 + scale: 2 x: 180 y: 180 diff --git a/sources/ometiff/large_image_source_ometiff/__init__.py b/sources/ometiff/large_image_source_ometiff/__init__.py index 28f075822..d2c56ec5f 100644 --- a/sources/ometiff/large_image_source_ometiff/__init__.py +++ b/sources/ometiff/large_image_source_ometiff/__init__.py @@ -22,10 +22,8 @@ import numpy import PIL.Image from large_image_source_tiff import TiffFileTileSource -from large_image_source_tiff.tiff_reader import (InvalidOperationTiffException, - IOTiffException, - TiffException, - TiledTiffDirectory) +from large_image_source_tiff.exceptions import InvalidOperationTiffError, IOTiffError, TiffError +from large_image_source_tiff.tiff_reader import TiledTiffDirectory from large_image.cache_util import LruCacheMetaclass, methodcache from large_image.constants import TILE_FORMAT_NUMPY, TILE_FORMAT_PIL, SourcePriority @@ -102,7 +100,7 @@ def __init__(self, path, **kwargs): try: base = TiledTiffDirectory(self._largeImagePath, 0, mustBeTiled=None) - except TiffException: + except TiffError: if not os.path.isfile(self._largeImagePath): raise TileSourceFileNotFoundError(self._largeImagePath) from None raise TileSourceError('Not a recognized OME Tiff') @@ -355,9 +353,9 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, format = TILE_FORMAT_NUMPY return self._outputTile(tile, format, x, y, z, pilImageAllowed, numpyAllowed, **kwargs) - except InvalidOperationTiffException as e: + except InvalidOperationTiffError as e: raise TileSourceError(e.args[0]) - except IOTiffException as e: + except IOTiffError as e: return self.getTileIOTiffError( x, y, z, pilImageAllowed=pilImageAllowed, numpyAllowed=numpyAllowed, sparseFallback=sparseFallback, diff --git a/sources/tiff/large_image_source_tiff/__init__.py b/sources/tiff/large_image_source_tiff/__init__.py index c9c7de9b5..7c7dbbe87 100644 --- a/sources/tiff/large_image_source_tiff/__init__.py +++ b/sources/tiff/large_image_source_tiff/__init__.py @@ -32,9 +32,9 @@ from large_image.exceptions import TileSourceError, TileSourceFileNotFoundError from large_image.tilesource import FileTileSource, nearPowerOfTwo -from .tiff_reader import (InvalidOperationTiffException, IOTiffException, - IOTiffOpenException, TiffException, - TiledTiffDirectory, ValidationTiffException) +from .exceptions import (InvalidOperationTiffError, IOOpenTiffError, + IOTiffError, TiffError, ValidationTiffError) +from .tiff_reader import TiledTiffDirectory try: from importlib.metadata import PackageNotFoundError @@ -105,9 +105,9 @@ def __init__(self, path, **kwargs): # noqa try: alldir = self._scanDirectories() - except IOTiffOpenException: + except IOOpenTiffError: raise TileSourceError('File cannot be opened via tiff source.') - except (ValidationTiffException, TiffException) as exc: + except (ValidationTiffError, TiffError) as exc: alldir = [] lastException = exc @@ -169,6 +169,7 @@ def __init__(self, path, **kwargs): # noqa self.sizeX = highest.imageWidth self.sizeY = highest.imageHeight self._checkForInefficientDirectories() + self._checkForVendorSpecificTags() def _scanDirectories(self): lastException = None @@ -193,11 +194,11 @@ def _scanDirectories(self): dir._setDirectory(directoryNum) dir._loadMetadata() dir._validate() - except ValidationTiffException as exc: + except ValidationTiffError as exc: lastException = exc associatedDirs.append(directoryNum) continue - except TiffException as exc: + except TiffError as exc: if not lastException: lastException = exc break @@ -347,6 +348,7 @@ def _initWithTiffTools(self): # noqa for idx in range(self.levels - 1)] self._tiffDirectories.append(dir0) self._checkForInefficientDirectories() + self._checkForVendorSpecificTags() return True def _checkForInefficientDirectories(self, warn=True): @@ -396,6 +398,27 @@ def _reorient_numpy_image(self, image, orientation): image = image[::, ::-1, ::] return image + def _checkForVendorSpecificTags(self): + if not hasattr(self, '_frames') or len(self._frames) <= 1: + return + if self._frames[0].get('frame', {}).get('IndexC'): + return + dir = self._tiffDirectories[-1] + if not hasattr(dir, '_description_record'): + return + if dir._description_record.get('PerkinElmer-QPI-ImageDescription', {}).get('Biomarker'): + channels = [] + for frame in range(len(self._frames)): + dir = self._getDirFromCache(*self._frames[frame]['dirs'][-1]) + channels.append(dir._description_record.get( + 'PerkinElmer-QPI-ImageDescription', {}).get('Biomarker')) + if channels[-1] is None: + return + self._frames[0]['channels'] = channels + for idx, frame in enumerate(self._frames): + frame.setdefault('frame', {}) + frame['frame']['IndexC'] = idx + def _addAssociatedImage(self, largeImagePath, directoryNum, mustBeTiled=False, topImage=None): """ Check if the specified TIFF directory contains an image with a sensible @@ -438,7 +461,7 @@ def _addAssociatedImage(self, largeImagePath, directoryNum, mustBeTiled=False, t return image = self._reorient_numpy_image(image, associated._tiffInfo.get('orientation')) self._associatedImages[id] = image - except (TiffException, AttributeError): + except (TiffError, AttributeError): # If we can't validate or read an associated image or it has no # useful imagedescription, fail quietly without adding an # associated image. @@ -582,10 +605,10 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, if not kwargs.get('inSparseFallback'): tile = self.getTileFromEmptyDirectory(x, y, z, **kwargs) else: - raise IOTiffException('Missing z level %d' % z) + raise IOTiffError('Missing z level %d' % z) except Exception: if sparseFallback: - raise IOTiffException('Missing z level %d' % z) + raise IOTiffError('Missing z level %d' % z) else: raise allowStyle = False @@ -599,9 +622,9 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, format = TILE_FORMAT_NUMPY return self._outputTile(tile, format, x, y, z, pilImageAllowed, numpyAllowed, applyStyle=allowStyle, **kwargs) - except InvalidOperationTiffException as e: + except InvalidOperationTiffError as e: raise TileSourceError(e.args[0]) - except IOTiffException as e: + except IOTiffError as e: return self.getTileIOTiffError( x, y, z, pilImageAllowed=pilImageAllowed, numpyAllowed=numpyAllowed, sparseFallback=sparseFallback, @@ -619,7 +642,7 @@ def _getDirFromCache(self, dirnum, subdir=None): try: result = TiledTiffDirectory( self._largeImagePath, dirnum, mustBeTiled=None, subDirectoryNum=subdir) - except IOTiffException: + except IOTiffError: result = None self._directoryCache[key] = result return result diff --git a/sources/tiff/large_image_source_tiff/exceptions.py b/sources/tiff/large_image_source_tiff/exceptions.py new file mode 100644 index 000000000..6d67b1b68 --- /dev/null +++ b/sources/tiff/large_image_source_tiff/exceptions.py @@ -0,0 +1,37 @@ +class TiffError(Exception): + pass + + +class InvalidOperationTiffError(TiffError): + """ + An exception caused by the user making an invalid request of a TIFF file. + """ + + pass + + +class IOTiffError(TiffError): + """ + An exception caused by an internal failure, due to an invalid file or other + error. + """ + + pass + + +class IOOpenTiffError(IOTiffError): + """ + An exception caused by an internal failure where the file cannot be opened + by the main library. + """ + + pass + + +class ValidationTiffError(TiffError): + """ + An exception caused by the TIFF reader not being able to support a given + file. + """ + + pass diff --git a/sources/tiff/large_image_source_tiff/tiff_reader.py b/sources/tiff/large_image_source_tiff/tiff_reader.py index 2bec9b926..5616cbdc1 100644 --- a/sources/tiff/large_image_source_tiff/tiff_reader.py +++ b/sources/tiff/large_image_source_tiff/tiff_reader.py @@ -31,6 +31,8 @@ from large_image.cache_util import methodcache, strhash from large_image.tilesource import etreeToDict +from .exceptions import InvalidOperationTiffError, IOOpenTiffError, IOTiffError, ValidationTiffError + try: from libtiff import libtiff_ctypes except ValueError as exc: @@ -68,45 +70,6 @@ def patchLibtiff(): patchLibtiff() -class TiffException(Exception): - pass - - -class InvalidOperationTiffException(TiffException): - """ - An exception caused by the user making an invalid request of a TIFF file. - """ - - pass - - -class IOTiffException(TiffException): - """ - An exception caused by an internal failure, due to an invalid file or other - error. - """ - - pass - - -class IOTiffOpenException(IOTiffException): - """ - An exception caused by an internal failure where the file cannot be opened - by the main library. - """ - - pass - - -class ValidationTiffException(TiffException): - """ - An exception caused by the TIFF reader not being able to support a given - file. - """ - - pass - - class TiledTiffDirectory: CoreFunctions = [ @@ -131,8 +94,8 @@ def __init__(self, filePath, directoryNum, mustBeTiled=True, subDirectoryNum=0, :type subDirectoryNum: int :param validate: if False, don't validate that images can be read. :type mustBeTiled: bool - :raises: InvalidOperationTiffException or IOTiffException or - ValidationTiffException + :raises: InvalidOperationTiffError or IOTiffError or + ValidationTiffError """ self.logger = config.getConfig('logger') # create local cache to store Jpeg tables and getTileByteCountsType @@ -150,7 +113,7 @@ def __init__(self, filePath, directoryNum, mustBeTiled=True, subDirectoryNum=0, try: if validate: self._validate() - except ValidationTiffException: + except ValidationTiffError: self._close() raise @@ -167,11 +130,11 @@ def _open(self, filePath, directoryNum, subDirectoryNum=0): :type directoryNum: int :param subDirectoryNum: The number of the TIFF sub-IFD to be used. :type subDirectoryNum: int - :raises: InvalidOperationTiffException or IOTiffException + :raises: InvalidOperationTiffError or IOTiffError """ self._close() if not os.path.isfile(filePath): - raise InvalidOperationTiffException( + raise InvalidOperationTiffError( 'TIFF file does not exist: %s' % filePath) try: bytePath = filePath @@ -179,7 +142,7 @@ def _open(self, filePath, directoryNum, subDirectoryNum=0): bytePath = filePath.encode() self._tiffFile = libtiff_ctypes.TIFF.open(bytePath) except TypeError: - raise IOTiffOpenException( + raise IOOpenTiffError( 'Could not open TIFF file: %s' % filePath) # pylibtiff changed the case of some functions between version 0.4 and # the version that supports libtiff 4.0.6. To support both, ensure @@ -195,19 +158,19 @@ def _setDirectory(self, directoryNum, subDirectoryNum=0): self._directoryNum = directoryNum if self._tiffFile.SetDirectory(self._directoryNum) != 1: self._tiffFile.close() - raise IOTiffException( + raise IOTiffError( 'Could not set TIFF directory to %d' % directoryNum) self._subDirectoryNum = subDirectoryNum if self._subDirectoryNum: subifds = self._tiffFile.GetField('subifd') if (subifds is None or self._subDirectoryNum < 1 or self._subDirectoryNum > len(subifds)): - raise IOTiffException( + raise IOTiffError( 'Could not set TIFF subdirectory to %d' % subDirectoryNum) subifd = subifds[self._subDirectoryNum - 1] if self._tiffFile.SetSubDirectory(subifd) != 1: self._tiffFile.close() - raise IOTiffException( + raise IOTiffError( 'Could not set TIFF subdirectory to %d' % subDirectoryNum) def _close(self): @@ -219,22 +182,22 @@ def _validate(self): # noqa """ Validate that this TIFF file and directory are suitable for reading. - :raises: ValidationTiffException + :raises: ValidationTiffError """ if not self._mustBeTiled: if self._mustBeTiled is not None and self._tiffInfo.get('istiled'): - raise ValidationTiffException('Expected a non-tiled TIFF file') + raise ValidationTiffError('Expected a non-tiled TIFF file') # For any non-supported file, we probably can add a conversion task in # the create_image.py script, such as flatten or colourspace. These # should only be done if necessary, which would require the conversion # job to check output and perform subsequent processing as needed. if (not self._tiffInfo.get('samplesperpixel') or self._tiffInfo.get('samplesperpixel') < 1): - raise ValidationTiffException( + raise ValidationTiffError( 'Only RGB and greyscale TIFF files are supported') if self._tiffInfo.get('bitspersample') not in (8, 16, 32, 64): - raise ValidationTiffException( + raise ValidationTiffError( 'Only 8 and 16 bits-per-sample TIFF files are supported') if self._tiffInfo.get('sampleformat') not in { @@ -242,20 +205,20 @@ def _validate(self): # noqa libtiff_ctypes.SAMPLEFORMAT_UINT, libtiff_ctypes.SAMPLEFORMAT_INT, libtiff_ctypes.SAMPLEFORMAT_IEEEFP}: - raise ValidationTiffException( + raise ValidationTiffError( 'Only unsigned int sampled TIFF files are supported') if (self._tiffInfo.get('planarconfig') != libtiff_ctypes.PLANARCONFIG_CONTIG and self._tiffInfo.get('photometric') not in { libtiff_ctypes.PHOTOMETRIC_MINISBLACK}): - raise ValidationTiffException( + raise ValidationTiffError( 'Only contiguous planar configuration TIFF files are supported') if self._tiffInfo.get('photometric') not in { libtiff_ctypes.PHOTOMETRIC_MINISBLACK, libtiff_ctypes.PHOTOMETRIC_RGB, libtiff_ctypes.PHOTOMETRIC_YCBCR}: - raise ValidationTiffException( + raise ValidationTiffError( 'Only greyscale (black is 0), RGB, and YCbCr photometric ' 'interpretation TIFF files are supported') @@ -269,32 +232,32 @@ def _validate(self): # noqa libtiff_ctypes.ORIENTATION_RIGHTBOT, libtiff_ctypes.ORIENTATION_LEFTBOT, None}: - raise ValidationTiffException( + raise ValidationTiffError( 'Unsupported TIFF orientation') if self._mustBeTiled and ( not self._tiffInfo.get('istiled') or not self._tiffInfo.get('tilewidth') or not self._tiffInfo.get('tilelength')): - raise ValidationTiffException('A tiled TIFF is required.') + raise ValidationTiffError('A tiled TIFF is required.') if self._mustBeTiled is False and ( self._tiffInfo.get('istiled') or not self._tiffInfo.get('rowsperstrip')): - raise ValidationTiffException('A non-tiled TIFF with strips is required.') + raise ValidationTiffError('A non-tiled TIFF with strips is required.') if (self._tiffInfo.get('compression') == libtiff_ctypes.COMPRESSION_JPEG and self._tiffInfo.get('jpegtablesmode') != libtiff_ctypes.JPEGTABLESMODE_QUANT | libtiff_ctypes.JPEGTABLESMODE_HUFF): - raise ValidationTiffException( + raise ValidationTiffError( 'Only TIFF files with separate Huffman and quantization ' 'tables are supported') if self._tiffInfo.get('compression') == libtiff_ctypes.COMPRESSION_JPEG: try: self._getJpegTables() - except IOTiffException: + except IOTiffError: self._completeJpeg = True def _loadMetadata(self): @@ -385,18 +348,18 @@ def _getJpegTables(self): libtiff_ctypes.TIFFTAG_JPEGTABLES, ctypes.byref(tableSize), ctypes.byref(tableBuffer)) != 1: - raise IOTiffException('Could not get JPEG Huffman / quantization tables') + raise IOTiffError('Could not get JPEG Huffman / quantization tables') tableSize = tableSize.value tableBuffer = ctypes.cast(tableBuffer, ctypes.POINTER(ctypes.c_char)) if tableBuffer[:2] != b'\xff\xd8': - raise IOTiffException( + raise IOTiffError( 'Missing JPEG Start Of Image marker in tables') if tableBuffer[tableSize - 2:tableSize] != b'\xff\xd9': - raise IOTiffException('Missing JPEG End Of Image marker in tables') + raise IOTiffError('Missing JPEG End Of Image marker in tables') if tableBuffer[2:4] not in (b'\xff\xc4', b'\xff\xdb'): - raise IOTiffException( + raise IOTiffError( 'Missing JPEG Huffman or Quantization Table marker') # Strip the Start / End Of Image markers @@ -415,26 +378,26 @@ def _toTileNum(self, x, y, transpose=False): :type tranpose: boolean :return: The internal tile number of the desired tile. :rtype int - :raises: InvalidOperationTiffException + :raises: InvalidOperationTiffError """ # TIFFCheckTile and TIFFComputeTile require pixel coordinates if not transpose: pixelX = int(x * self._tileWidth) pixelY = int(y * self._tileHeight) if x < 0 or y < 0 or pixelX >= self._imageWidth or pixelY >= self._imageHeight: - raise InvalidOperationTiffException( + raise InvalidOperationTiffError( 'Tile x=%d, y=%d does not exist' % (x, y)) else: pixelX = int(x * self._tileHeight) pixelY = int(y * self._tileWidth) if x < 0 or y < 0 or pixelX >= self._imageHeight or pixelY >= self._imageWidth: - raise InvalidOperationTiffException( + raise InvalidOperationTiffError( 'Tile x=%d, y=%d does not exist' % (x, y)) # We had been using TIFFCheckTile, but with z=0 and sample=0, this is # just a check that x, y is within the image # if libtiff_ctypes.libtiff.TIFFCheckTile( # self._tiffFile, pixelX, pixelY, 0, 0) == 0: - # raise InvalidOperationTiffException( + # raise InvalidOperationTiffError( # 'Tile x=%d, y=%d does not exist' % (x, y)) if self._tiffInfo.get('istiled'): tileNum = libtiff_ctypes.libtiff.TIFFComputeTile( @@ -452,7 +415,7 @@ def _getTileByteCountsType(self): :return: The element type in TIFFTAG_TILEBYTECOUNTS. :rtype: ctypes.c_uint64 or ctypes.c_uint16 - :raises: IOTiffException + :raises: IOTiffError """ tileByteCountsFieldInfo = libtiff_ctypes.libtiff.TIFFFieldWithTag( self._tiffFile, libtiff_ctypes.TIFFTAG_TILEBYTECOUNTS).contents @@ -464,7 +427,7 @@ def _getTileByteCountsType(self): libtiff_ctypes.TIFFDataType.TIFF_SHORT: return ctypes.c_uint16 else: - raise IOTiffException( + raise IOTiffError( 'Invalid type for TIFFTAG_TILEBYTECOUNTS: %s' % tileByteCountsLibtiffType) def _getJpegFrameSize(self, tileNum): @@ -475,7 +438,7 @@ def _getJpegFrameSize(self, tileNum): :type tileNum: int :return: The size in bytes of the raw tile data for the desired tile. :rtype: int - :raises: InvalidOperationTiffException or IOTiffException + :raises: InvalidOperationTiffError or IOTiffError """ # TODO: is it worth it to memoize this? @@ -483,7 +446,7 @@ def _getJpegFrameSize(self, tileNum): totalTileCount = libtiff_ctypes.libtiff.TIFFNumberOfTiles( self._tiffFile).value if tileNum >= totalTileCount: - raise InvalidOperationTiffException('Tile number out of range') + raise InvalidOperationTiffError('Tile number out of range') # pylibtiff treats the output of TIFFTAG_TILEBYTECOUNTS as a scalar # uint32; libtiff's documentation specifies that the output will be an @@ -508,7 +471,7 @@ def _getJpegFrameSize(self, tileNum): self._tiffFile, libtiff_ctypes.TIFFTAG_TILEBYTECOUNTS, ctypes.byref(rawTileSizes)) != 1: - raise IOTiffException('Could not get raw tile size') + raise IOTiffError('Could not get raw tile size') # In practice, this will never overflow, and it's simpler to convert the # long to an int @@ -524,12 +487,12 @@ def _getJpegFrame(self, tileNum, entire=False): # noqa container information. :return: The JPEG image frame, including a JPEG Start Of Frame marker. :rtype: bytes - :raises: InvalidOperationTiffException or IOTiffException + :raises: InvalidOperationTiffError or IOTiffError """ - # This raises an InvalidOperationTiffException if the tile doesn't exist + # This raises an InvalidOperationTiffError if the tile doesn't exist rawTileSize = self._getJpegFrameSize(tileNum) if rawTileSize <= 0: - raise IOTiffException('No raw tile data') + raise IOTiffError('No raw tile data') frameBuffer = ctypes.create_string_buffer(rawTileSize) @@ -537,20 +500,20 @@ def _getJpegFrame(self, tileNum, entire=False): # noqa self._tiffFile, tileNum, frameBuffer, rawTileSize).value if bytesRead == -1: - raise IOTiffException('Failed to read raw tile') + raise IOTiffError('Failed to read raw tile') elif bytesRead < rawTileSize: - raise IOTiffException('Buffer underflow when reading tile') + raise IOTiffError('Buffer underflow when reading tile') elif bytesRead > rawTileSize: # It's unlikely that this will ever occur, but incomplete reads will # be checked for by looking for the JPEG end marker - raise IOTiffException('Buffer overflow when reading tile') + raise IOTiffError('Buffer overflow when reading tile') if entire: return frameBuffer.raw[:] if frameBuffer.raw[:2] != b'\xff\xd8': - raise IOTiffException('Missing JPEG Start Of Image marker in frame') + raise IOTiffError('Missing JPEG Start Of Image marker in frame') if frameBuffer.raw[-2:] != b'\xff\xd9': - raise IOTiffException('Missing JPEG End Of Image marker in frame') + raise IOTiffError('Missing JPEG End Of Image marker in frame') if frameBuffer.raw[2:4] in (b'\xff\xc0', b'\xff\xc2'): frameStartPos = 2 else: @@ -562,7 +525,7 @@ def _getJpegFrame(self, tileNum, entire=False): # noqa if frameStartPos == -1: frameStartPos = frameBuffer.raw.find(b'\xff\xc2', 2, -2) if frameStartPos == -1: - raise IOTiffException('Missing JPEG Start Of Frame marker') + raise IOTiffError('Missing JPEG Start Of Frame marker') # If the photometric value is RGB and the JPEG component ids are just # 0, 1, 2, change the component ids to R, G, B to ensure color space # information is preserved. @@ -591,7 +554,7 @@ def _getUncompressedTile(self, tileNum): :type tileNum: int :return: the tile as a PIL 8-bit-per-channel images. :rtype: PIL.Image - :raises: IOTiffException + :raises: IOTiffError """ with self._tileLock: if self._tiffInfo.get('istiled'): @@ -615,14 +578,14 @@ def _getUncompressedTile(self, tileNum): ctypes.byref(imageBuffer, stripSize * stripNum), stripSize).value if chunkSize <= 0: - raise IOTiffException( + raise IOTiffError( 'Read an unexpected number of bytes from an encoded strip') readSize += chunkSize if readSize < tileSize: ctypes.memset(ctypes.byref(imageBuffer, readSize), 0, tileSize - readSize) readSize = tileSize if readSize < tileSize: - raise IOTiffException( + raise IOTiffError( 'Read an unexpected number of bytes from an encoded tile' if readSize >= 0 else 'Failed to read from an encoded tile') tw, th = self._tileWidth, self._tileHeight @@ -724,7 +687,7 @@ def _getTileRotated(self, x, y): :min(subtile.shape[1], tw - stx)] tile[sty:sty + subtile.shape[0], stx:stx + subtile.shape[1]] = subtile if tile is None: - raise InvalidOperationTiffException( + raise InvalidOperationTiffError( 'Tile x=%d, y=%d does not exist' % (x, y)) if self._tiffInfo.get('orientation') in { libtiff_ctypes.ORIENTATION_BOTRIGHT, @@ -788,13 +751,13 @@ def getTile(self, x, y): :type y: int :return: either a buffer with a JPEG or a PIL image. :rtype: bytes - :raises: InvalidOperationTiffException or IOTiffException + :raises: InvalidOperationTiffError or IOTiffError """ if self._tiffInfo.get('orientation') not in { libtiff_ctypes.ORIENTATION_TOPLEFT, None}: return self._getTileRotated(x, y) - # This raises an InvalidOperationTiffException if the tile doesn't exist + # This raises an InvalidOperationTiffError if the tile doesn't exist tileNum = self._toTileNum(x, y) if (not self._tiffInfo.get('istiled') or diff --git a/sources/tifffile/large_image_source_tifffile/__init__.py b/sources/tifffile/large_image_source_tifffile/__init__.py index c4cc769c1..9548abef7 100644 --- a/sources/tifffile/large_image_source_tifffile/__init__.py +++ b/sources/tifffile/large_image_source_tifffile/__init__.py @@ -1,3 +1,4 @@ +import json import logging import math import os @@ -76,6 +77,7 @@ class TifffileFileTileSource(FileTileSource, metaclass=LruCacheMetaclass): _tileSize = 512 _minImageSize = 128 _minTileSize = 128 + _singleTileSize = 1024 _maxTileSize = 2048 _minAssociatedImageSize = 64 _maxAssociatedImageSize = 8192 @@ -102,6 +104,8 @@ def __init__(self, path, **kwargs): self.tileWidth = self.tileHeight = self._tileSize s = self._tf.series[maxseries] self._baseSeries = s + if len(s.levels) == 1: + self.tileWidth = self.tileHeight = self._singleTileSize page = s.pages[0] if ('TileWidth' in page.tags and self._minTileSize <= page.tags['TileWidth'].value <= self._maxTileSize): @@ -243,6 +247,15 @@ def _findAssociatedImages(self): max(entry['width'], entry['height']) >= self._minAssociatedImageSize): self._associatedImages[id] = entry + def _handle_imagej(self): + try: + ijm = self._tf.pages[0].tags['IJMetadata'].value + if (ijm['Labels'] and len(ijm['Labels']) == self._framecount and + not getattr(self, '_channels', None)): + self._channels = ijm['Labels'] + except Exception: + pass + def _handle_scn(self): # noqa """ For SCN files, parse the xml and possibly adjust how associated images @@ -359,7 +372,8 @@ def getInternalMetadata(self, **kwargs): pages.extend([page for page in self._tf.pages if page not in pagesInSeries]) for page in pages: for tag in getattr(page, 'tags', []): - if tag.dtype_name == 'ASCII' and tag.value: + if (tag.dtype_name == 'ASCII' or ( + tag.dtype_name == 'BYTE' and isinstance(tag.value, dict))) and tag.value: key = basekey = tag.name suffix = 0 while key in result: @@ -368,6 +382,13 @@ def getInternalMetadata(self, **kwargs): suffix += 1 key = '%s_%d' % (basekey, suffix) result[key] = tag.value + if isinstance(result[key], dict): + result[key] = result[key].copy() + for subkey in list(result[key]): + try: + json.dumps(result[key][subkey]) + except Exception: + del result[key][subkey] if hasattr(self, '_xml') and 'xml' not in result: result.pop('ImageDescription', None) result['xml'] = self._xml @@ -420,11 +441,13 @@ def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): if sidx not in self._zarrcache: if len(self._zarrcache) > 10: self._zarrcache = {} - self._zarrcache[sidx] = zarr.open(series.aszarr(), mode='r') - za = self._zarrcache[sidx] + za = zarr.open(series.aszarr(), mode='r') + hasgbs = hasattr(za[0], 'get_basic_selection') + self._zarrcache[sidx] = (za, hasgbs) + za, hasgbs = self._zarrcache[sidx] xidx = series.axes.index('X') yidx = series.axes.index('Y') - if hasattr(za[0], 'get_basic_selection'): + if hasgbs: bza = za[0] # we could cache this for ll in range(len(series.levels) - 1, 0, -1): diff --git a/test.Dockerfile b/test.Dockerfile index 87d1bf407..b5c2fd4f0 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -116,11 +116,8 @@ RUN . ~/.bashrc && \ nvm install 12 && \ nvm alias default 12 && \ nvm use default && \ - rm /usr/local/bin/node || true && \ - rm /usr/local/bin/npm || true && \ - rm /usr/local/bin/npx || true && \ - ln -s `which node` /usr/local/bin/. && \ - ln -s `which npm` /usr/local/bin/. && \ - ln -s `which npx` /usr/local/bin/. + ln -s $(dirname `which npm`) /usr/local/node + +ENV PATH="/usr/local/node:$PATH" WORKDIR /app diff --git a/test/test_source_openslide.py b/test/test_source_openslide.py index 1acd46832..2f29f63ca 100644 --- a/test/test_source_openslide.py +++ b/test/test_source_openslide.py @@ -321,16 +321,16 @@ def testGetPixel(): source = large_image_source_openslide.open(imagePath, style={'icc': False}) pixel = source.getPixel(region={'left': 12125, 'top': 10640}) - assert pixel == {'r': 156, 'g': 98, 'b': 138, 'a': 255} + assert pixel == {'r': 156, 'g': 98, 'b': 138, 'a': 255, 'value': [156, 98, 138, 255]} pixel = source.getPixel(region={'left': 3.0555, 'top': 2.68128, 'units': 'mm'}) - assert pixel == {'r': 156, 'g': 98, 'b': 138, 'a': 255} + assert pixel == {'r': 156, 'g': 98, 'b': 138, 'a': 255, 'value': [156, 98, 138, 255]} pixel = source.getPixel(region={'top': 10640, 'right': 12126, 'bottom': 12000}) - assert pixel == {'r': 156, 'g': 98, 'b': 138, 'a': 255} + assert pixel == {'r': 156, 'g': 98, 'b': 138, 'a': 255, 'value': [156, 98, 138, 255]} pixel = source.getPixel(region={'left': 12125, 'top': 10640, 'right': 13000}) - assert pixel == {'r': 156, 'g': 98, 'b': 138, 'a': 255} + assert pixel == {'r': 156, 'g': 98, 'b': 138, 'a': 255, 'value': [156, 98, 138, 255]} pixel = source.getPixel(region={'left': 12125, 'top': 10640}, includeTileRecord=True) assert 'tile' in pixel @@ -342,7 +342,7 @@ def testGetPixelWithICCCorrection(): '4a70-9ae3-50e3ab45e242.svs') source = large_image_source_openslide.open(imagePath) pixel = source.getPixel(region={'left': 12125, 'top': 10640}) - assert pixel == {'r': 169, 'g': 99, 'b': 151, 'a': 255} + assert pixel == {'r': 169, 'g': 99, 'b': 151, 'a': 255, 'value': [169, 99, 151, 255]} source = large_image_source_openslide.open(imagePath, style={'icc': True}) pixel2 = source.getPixel(region={'left': 12125, 'top': 10640}) assert pixel == pixel2 diff --git a/tox.ini b/tox.ini index 9ed683499..d64940f81 100644 --- a/tox.ini +++ b/tox.ini @@ -129,9 +129,11 @@ skip_install = true deps = autopep8 isort + unify commands = isort {posargs:.} autopep8 -ria large_image sources utilities girder girder_annotation examples docs test + unify --in-place --recursive large_image sources utilities girder girder_annotation examples docs test [testenv:lintclient] description = Lint the girder large_image plugin client