From 788b1651175759f023bb6d174f71bbb1960850c4 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Fri, 28 Jun 2024 15:18:37 -0400 Subject: [PATCH 01/11] Create large images for all items in a folder --- girder/girder_large_image/rest/__init__.py | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/girder/girder_large_image/rest/__init__.py b/girder/girder_large_image/rest/__init__.py index 07c3c5654..965cf6550 100644 --- a/girder/girder_large_image/rest/__init__.py +++ b/girder/girder_large_image/rest/__init__.py @@ -7,6 +7,8 @@ from girder.constants import AccessType, TokenScope from girder.models.folder import Folder from girder.models.item import Item +from girder.models.file import File +from girder_large_image.models.image_item import ImageItem def addSystemEndpoints(apiRoot): @@ -17,6 +19,7 @@ def addSystemEndpoints(apiRoot): """ apiRoot.folder.route('GET', (':id', 'yaml_config', ':name'), getYAMLConfigFile) apiRoot.folder.route('PUT', (':id', 'yaml_config', ':name'), putYAMLConfigFile) + apiRoot.folder.route('PUT', (':id', 'items'), createLargeImages) origItemFind = apiRoot.item._find origFolderFind = apiRoot.folder._find @@ -101,6 +104,36 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset, return Item().findWithPermissions(filters, offset, limit, sort=sort, user=user) return origItemFind(folderId, text, name, limit, offset, sort, filters) +@access.user(scope=TokenScope.DATA_WRITE) +@autoDescribeRoute( + Description('Creates large images for all items within a folder, given they each only have one file.') + .responseClass('Folder') + .modelParam('id', model=Folder, level=AccessType.WRITE, required=True) + .errorResponse('ID was invalid.') + .errorResponse('Write access was denied for the folder.', 403) +) +@boundHandler +def createLargeImages(self, folder, params): + user=self.getCurrentUser() + for item in Folder().childItems(folder=folder): + if item.get('largeImage'): + if item['largeImage'].get('expected'): + pass + else: + try: + Item().getMetadata(item) + continue + except Exception: + previousFileId = item['largeImage'].get('originalId', item['largeImage']['fileId']) + ImageItem().delete(item) + ImageItem().createImageItem(item, File().load(user=user, id=previousFileId)) + else: + largeImageFileId = params.get('fileId') + if largeImageFileId is None : + files = list(Item().childFiles(item=item, limit=0)) + if len(files) == 1: + largeImageFileId = str(files[0]['_id']) + ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId)) @access.public(scope=TokenScope.DATA_READ) @autoDescribeRoute( From 69d38d4412ff5f02cb014dd3a671c4b22d2282f7 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Mon, 1 Jul 2024 16:42:12 -0400 Subject: [PATCH 02/11] Add option to create local job with each large image --- girder/girder_large_image/rest/__init__.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/girder/girder_large_image/rest/__init__.py b/girder/girder_large_image/rest/__init__.py index 965cf6550..f48474e1d 100644 --- a/girder/girder_large_image/rest/__init__.py +++ b/girder/girder_large_image/rest/__init__.py @@ -106,15 +106,19 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset, @access.user(scope=TokenScope.DATA_WRITE) @autoDescribeRoute( - Description('Creates large images for all items within a folder, given they each only have one file.') + Description('Create large images for all items within a folder.') + .notes('Does not work for items with multiple files.') .responseClass('Folder') .modelParam('id', model=Folder, level=AccessType.WRITE, required=True) + .param('localJobs', 'Whether a local job should be created for each large image', required=False, + default=False, dataType='boolean') .errorResponse('ID was invalid.') .errorResponse('Write access was denied for the folder.', 403) ) @boundHandler def createLargeImages(self, folder, params): user=self.getCurrentUser() + createJobs = 'always' or params.get('localJobs') for item in Folder().childItems(folder=folder): if item.get('largeImage'): if item['largeImage'].get('expected'): @@ -126,14 +130,15 @@ def createLargeImages(self, folder, params): except Exception: previousFileId = item['largeImage'].get('originalId', item['largeImage']['fileId']) ImageItem().delete(item) - ImageItem().createImageItem(item, File().load(user=user, id=previousFileId)) + ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), createJob=createJobs, + localJob=params.get('localJobs')) else: - largeImageFileId = params.get('fileId') - if largeImageFileId is None : - files = list(Item().childFiles(item=item, limit=0)) - if len(files) == 1: - largeImageFileId = str(files[0]['_id']) - ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId)) + largeImageFileId = None + files = list(Item().childFiles(item=item, limit=0)) + if len(files) == 1: + largeImageFileId = str(files[0]['_id']) + ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId), createJob=createJobs, + localJob=params.get('localJobs')) @access.public(scope=TokenScope.DATA_READ) @autoDescribeRoute( From 9b7bf9a042588c4ce509f86cee63ea38d53da524 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Tue, 2 Jul 2024 13:32:20 -0400 Subject: [PATCH 03/11] Add option to recurse child folders and allow creation of non-local jobs --- girder/girder_large_image/rest/__init__.py | 26 +++++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/girder/girder_large_image/rest/__init__.py b/girder/girder_large_image/rest/__init__.py index f48474e1d..06a33f5cb 100644 --- a/girder/girder_large_image/rest/__init__.py +++ b/girder/girder_large_image/rest/__init__.py @@ -110,15 +110,29 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset, .notes('Does not work for items with multiple files.') .responseClass('Folder') .modelParam('id', model=Folder, level=AccessType.WRITE, required=True) - .param('localJobs', 'Whether a local job should be created for each large image', required=False, + .param('createJobs','Whether a job should be created for each large image.', required=False, default=False, dataType='boolean') + .param('localJobs', 'Whether a the jobs created should be local.', required=False, + default=False, dataType='boolean') + .param('recurse', 'Whether child folders should be recursed', required=False, default=False, + dataType='boolean') .errorResponse('ID was invalid.') .errorResponse('Write access was denied for the folder.', 403) ) @boundHandler def createLargeImages(self, folder, params): user=self.getCurrentUser() - createJobs = 'always' or params.get('localJobs') + createJobs = params.get('createJobs') + if createJobs: + createJobs = 'always' or True + createImagesRecurseOption(self, folder=folder, createJobs=createJobs, user=user, + recurse=params.get('recurse'), localJobs=params.get('localJobs')) + +def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs): + if recurse: + for childFolder in Folder().childFolders(parent=folder, parentType='folder'): + createImagesRecurseOption(self, folder=childFolder, createJobs=createJobs, user=user, + recurse=recurse, localJobs=localJobs) for item in Folder().childItems(folder=folder): if item.get('largeImage'): if item['largeImage'].get('expected'): @@ -130,15 +144,15 @@ def createLargeImages(self, folder, params): except Exception: previousFileId = item['largeImage'].get('originalId', item['largeImage']['fileId']) ImageItem().delete(item) - ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), createJob=createJobs, - localJob=params.get('localJobs')) + ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), + createJob=createJobs, localJob=localJobs) else: largeImageFileId = None files = list(Item().childFiles(item=item, limit=0)) if len(files) == 1: largeImageFileId = str(files[0]['_id']) - ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId), createJob=createJobs, - localJob=params.get('localJobs')) + ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId), + createJob=createJobs, localJob=localJobs) @access.public(scope=TokenScope.DATA_READ) @autoDescribeRoute( From 41bbec51024cdbced7cf42dbb68b862c9be8ed89 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Wed, 10 Jul 2024 10:29:44 -0400 Subject: [PATCH 04/11] Move function to rest/large_image_resource.py, rename route, and squash commits from master # dependency-type: direct:production #update-type: version-update:semver-major --- .../rest/large_image_resource.py | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index ab544980f..81303e191 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -35,10 +35,11 @@ from girder.api import access from girder.api.describe import Description, autoDescribeRoute, describeRoute from girder.api.rest import Resource -from girder.constants import SortDir, TokenScope +from girder.constants import SortDir, TokenScope, AccessType from girder.exceptions import RestException from girder.models.file import File from girder.models.item import Item +from girder.models.folder import Folder from girder.models.setting import Setting from large_image import cache_util from large_image.exceptions import TileGeneralError @@ -255,6 +256,7 @@ def __init__(self): self.route('GET', ('histograms',), self.countHistograms) self.route('DELETE', ('histograms',), self.deleteHistograms) self.route('DELETE', ('tiles', 'incomplete'), self.deleteIncompleteTiles) + self.route('PUT', ('folder', ':id', 'tiles'), self.createLargeImages) @describeRoute( Description('Clear tile source caches to release resources and file handles.'), @@ -445,6 +447,55 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): removed += 1 return removed + @access.user(scope=TokenScope.DATA_WRITE) + @autoDescribeRoute( + Description('Create large images for all items within a folder.') + .notes('Does not work for items with multiple files.') + .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) + .param('createJobs','Whether a job should be created for each large image.', required=False, + default=False, dataType='boolean') + .param('localJobs', 'Whether a the jobs created should be local.', required=False, + default=False, dataType='boolean') + .param('recurse', 'Whether child folders should be recursed', required=False, default=False, + dataType='boolean') + .errorResponse('ID was invalid.') + .errorResponse('Write access was denied for the folder.', 403) + ) + def createLargeImages(self, folder, params): + user=self.getCurrentUser() + createJobs = params.get('createJobs') + if createJobs: + createJobs = 'always' or True + self.createImagesRecurseOption(folder=folder, createJobs=createJobs, user=user, + recurse=params.get('recurse'), localJobs=params.get('localJobs')) + + def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs): + count = {'large images created': 0} + if recurse: + for childFolder in Folder().childFolders(parent=folder, parentType='folder'): + self.createImagesRecurseOption(folder=childFolder, createJobs=createJobs, user=user, + recurse=recurse, localJobs=localJobs) + for item in Folder().childItems(folder=folder): + if item.get('largeImage'): + if item['largeImage'].get('expected'): + pass + else: + try: + Item().getMetadata(item) + continue + except Exception: + previousFileId = item['largeImage'].get('originalId', item['largeImage']['fileId']) + ImageItem().delete(item) + ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), + createJob=createJobs, localJob=localJobs) + else: + largeImageFileId = None + files = list(Item().childFiles(item=item, limit=0)) + if len(files) == 1: + largeImageFileId = str(files[0]['_id']) + ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId), + createJob=createJobs, localJob=localJobs) + @describeRoute( Description('Remove large images from items where the large image job ' 'incomplete.') From 1259c9c63b3b6bb5a9941073fafd6706f884697b Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Wed, 10 Jul 2024 11:44:39 -0400 Subject: [PATCH 05/11] Restoring girder/girder_large_image/rest/__init__.py --- girder/girder_large_image/rest/__init__.py | 52 ---------------------- 1 file changed, 52 deletions(-) diff --git a/girder/girder_large_image/rest/__init__.py b/girder/girder_large_image/rest/__init__.py index 06a33f5cb..07c3c5654 100644 --- a/girder/girder_large_image/rest/__init__.py +++ b/girder/girder_large_image/rest/__init__.py @@ -7,8 +7,6 @@ from girder.constants import AccessType, TokenScope from girder.models.folder import Folder from girder.models.item import Item -from girder.models.file import File -from girder_large_image.models.image_item import ImageItem def addSystemEndpoints(apiRoot): @@ -19,7 +17,6 @@ def addSystemEndpoints(apiRoot): """ apiRoot.folder.route('GET', (':id', 'yaml_config', ':name'), getYAMLConfigFile) apiRoot.folder.route('PUT', (':id', 'yaml_config', ':name'), putYAMLConfigFile) - apiRoot.folder.route('PUT', (':id', 'items'), createLargeImages) origItemFind = apiRoot.item._find origFolderFind = apiRoot.folder._find @@ -104,55 +101,6 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset, return Item().findWithPermissions(filters, offset, limit, sort=sort, user=user) return origItemFind(folderId, text, name, limit, offset, sort, filters) -@access.user(scope=TokenScope.DATA_WRITE) -@autoDescribeRoute( - Description('Create large images for all items within a folder.') - .notes('Does not work for items with multiple files.') - .responseClass('Folder') - .modelParam('id', model=Folder, level=AccessType.WRITE, required=True) - .param('createJobs','Whether a job should be created for each large image.', required=False, - default=False, dataType='boolean') - .param('localJobs', 'Whether a the jobs created should be local.', required=False, - default=False, dataType='boolean') - .param('recurse', 'Whether child folders should be recursed', required=False, default=False, - dataType='boolean') - .errorResponse('ID was invalid.') - .errorResponse('Write access was denied for the folder.', 403) -) -@boundHandler -def createLargeImages(self, folder, params): - user=self.getCurrentUser() - createJobs = params.get('createJobs') - if createJobs: - createJobs = 'always' or True - createImagesRecurseOption(self, folder=folder, createJobs=createJobs, user=user, - recurse=params.get('recurse'), localJobs=params.get('localJobs')) - -def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs): - if recurse: - for childFolder in Folder().childFolders(parent=folder, parentType='folder'): - createImagesRecurseOption(self, folder=childFolder, createJobs=createJobs, user=user, - recurse=recurse, localJobs=localJobs) - for item in Folder().childItems(folder=folder): - if item.get('largeImage'): - if item['largeImage'].get('expected'): - pass - else: - try: - Item().getMetadata(item) - continue - except Exception: - previousFileId = item['largeImage'].get('originalId', item['largeImage']['fileId']) - ImageItem().delete(item) - ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), - createJob=createJobs, localJob=localJobs) - else: - largeImageFileId = None - files = list(Item().childFiles(item=item, limit=0)) - if len(files) == 1: - largeImageFileId = str(files[0]['_id']) - ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId), - createJob=createJobs, localJob=localJobs) @access.public(scope=TokenScope.DATA_READ) @autoDescribeRoute( From ef19b736766a8f3410552291b78805fc62bc4563 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Thu, 11 Jul 2024 14:47:18 -0400 Subject: [PATCH 06/11] Return summary message & fix descriptions --- .../rest/large_image_resource.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index 81303e191..c43fb7a9c 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -449,12 +449,12 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): @access.user(scope=TokenScope.DATA_WRITE) @autoDescribeRoute( - Description('Create large images for all items within a folder.') - .notes('Does not work for items with multiple files.') + Description('Create new large images for all items within a folder.') + .notes('Does not work for items with multiple files and removes existing large images from items.') .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) - .param('createJobs','Whether a job should be created for each large image.', required=False, + .param('createJobs','Whether job(s) should be created for each large image.', required=False, default=False, dataType='boolean') - .param('localJobs', 'Whether a the jobs created should be local.', required=False, + .param('localJobs', 'Whether the job(s) created should be local.', required=False, default=False, dataType='boolean') .param('recurse', 'Whether child folders should be recursed', required=False, default=False, dataType='boolean') @@ -466,16 +466,22 @@ def createLargeImages(self, folder, params): createJobs = params.get('createJobs') if createJobs: createJobs = 'always' or True - self.createImagesRecurseOption(folder=folder, createJobs=createJobs, user=user, + return self.createImagesRecurseOption(folder=folder, createJobs=createJobs, user=user, recurse=params.get('recurse'), localJobs=params.get('localJobs')) def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs): - count = {'large images created': 0} + result = {'childFoldersRecursed': 0, + 'largeImagesCreated': 0, + 'largeImagesRemovedAndRecreated': 0, + 'totalItems': 0} if recurse: for childFolder in Folder().childFolders(parent=folder, parentType='folder'): - self.createImagesRecurseOption(folder=childFolder, createJobs=createJobs, user=user, + result['childFoldersRecursed']+=1 + childResult = self.createImagesRecurseOption(folder=childFolder, createJobs=createJobs, user=user, recurse=recurse, localJobs=localJobs) + for key in childResult: result[key]+= childResult[key] for item in Folder().childItems(folder=folder): + result['totalItems'] +=1 if item.get('largeImage'): if item['largeImage'].get('expected'): pass @@ -488,13 +494,16 @@ def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs ImageItem().delete(item) ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), createJob=createJobs, localJob=localJobs) + result['largeImagesRemovedAndRecreated']+=1 else: largeImageFileId = None - files = list(Item().childFiles(item=item, limit=0)) + files = list(Item().childFiles(item=item)) if len(files) == 1: largeImageFileId = str(files[0]['_id']) ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId), createJob=createJobs, localJob=localJobs) + result['largeImagesCreated']+=1 + return result @describeRoute( Description('Remove large images from items where the large image job ' From 5fc12af4ddc3b392258c986f2cf91b48898398e1 Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Fri, 12 Jul 2024 09:15:39 -0400 Subject: [PATCH 07/11] Lint fixes --- .../rest/large_image_resource.py | 58 ++++++++++--------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index c43fb7a9c..dc6b96149 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -35,11 +35,11 @@ from girder.api import access from girder.api.describe import Description, autoDescribeRoute, describeRoute from girder.api.rest import Resource -from girder.constants import SortDir, TokenScope, AccessType +from girder.constants import AccessType, SortDir, TokenScope from girder.exceptions import RestException from girder.models.file import File -from girder.models.item import Item from girder.models.folder import Folder +from girder.models.item import Item from girder.models.setting import Setting from large_image import cache_util from large_image.exceptions import TileGeneralError @@ -450,25 +450,28 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): @access.user(scope=TokenScope.DATA_WRITE) @autoDescribeRoute( Description('Create new large images for all items within a folder.') - .notes('Does not work for items with multiple files and removes existing large images from items.') - .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) - .param('createJobs','Whether job(s) should be created for each large image.', required=False, - default=False, dataType='boolean') - .param('localJobs', 'Whether the job(s) created should be local.', required=False, - default=False, dataType='boolean') - .param('recurse', 'Whether child folders should be recursed', required=False, default=False, - dataType='boolean') + .notes('Does not work for items with multiple files and removes existing ' + 'large images from items.') + .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, + required=True) + .param('createJobs', 'Whether job(s) should be created for each large image.', + required=False, default=False, dataType='boolean') + .param('localJobs', 'Whether the job(s) created should be local.', required=False, + default=False, dataType='boolean') + .param('recurse', 'Whether child folders should be recursed.', required=False, + default=False, dataType='boolean') .errorResponse('ID was invalid.') - .errorResponse('Write access was denied for the folder.', 403) + .errorResponse('Write access was denied for the folder.', 403), ) def createLargeImages(self, folder, params): - user=self.getCurrentUser() + user = self.getCurrentUser() createJobs = params.get('createJobs') if createJobs: createJobs = 'always' or True - return self.createImagesRecurseOption(folder=folder, createJobs=createJobs, user=user, - recurse=params.get('recurse'), localJobs=params.get('localJobs')) - + return self.createImagesRecurseOption(folder=folder, createJobs=createJobs, user=user, + recurse=params.get('recurse'), + localJobs=params.get('localJobs')) + def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs): result = {'childFoldersRecursed': 0, 'largeImagesCreated': 0, @@ -476,33 +479,36 @@ def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs 'totalItems': 0} if recurse: for childFolder in Folder().childFolders(parent=folder, parentType='folder'): - result['childFoldersRecursed']+=1 - childResult = self.createImagesRecurseOption(folder=childFolder, createJobs=createJobs, user=user, - recurse=recurse, localJobs=localJobs) - for key in childResult: result[key]+= childResult[key] + result['childFoldersRecursed'] += 1 + childResult = self.createImagesRecurseOption(folder=childFolder, + createJobs=createJobs, user=user, + recurse=recurse, localJobs=localJobs) + for key in childResult: + result[key] += childResult[key] for item in Folder().childItems(folder=folder): - result['totalItems'] +=1 + result['totalItems'] += 1 if item.get('largeImage'): if item['largeImage'].get('expected'): pass else: try: Item().getMetadata(item) - continue + continue except Exception: - previousFileId = item['largeImage'].get('originalId', item['largeImage']['fileId']) + previousFileId = item['largeImage'].get( + 'originalId', item['largeImage']['fileId']) ImageItem().delete(item) - ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), + ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), createJob=createJobs, localJob=localJobs) - result['largeImagesRemovedAndRecreated']+=1 + result['largeImagesRemovedAndRecreated'] += 1 else: largeImageFileId = None files = list(Item().childFiles(item=item)) if len(files) == 1: largeImageFileId = str(files[0]['_id']) - ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId), + ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId), createJob=createJobs, localJob=localJobs) - result['largeImagesCreated']+=1 + result['largeImagesCreated'] += 1 return result @describeRoute( From 71107ba9aa59ce3d162431c3bee4b1144444c44e Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Mon, 15 Jul 2024 14:11:35 -0400 Subject: [PATCH 08/11] Get necessary data & simplify passing file --- girder/girder_large_image/rest/large_image_resource.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index dc6b96149..c2839e4ca 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -502,12 +502,10 @@ def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs createJob=createJobs, localJob=localJobs) result['largeImagesRemovedAndRecreated'] += 1 else: - largeImageFileId = None - files = list(Item().childFiles(item=item)) + files = list(Item().childFiles(item=item, limit=2)) if len(files) == 1: - largeImageFileId = str(files[0]['_id']) - ImageItem().createImageItem(item, File().load(user=user, id=largeImageFileId), - createJob=createJobs, localJob=localJobs) + ImageItem().createImageItem(item, files[0], createJob=createJobs, + localJob=localJobs) result['largeImagesCreated'] += 1 return result From 90cf2ff15182a0078c9574150644781ec50b1c7c Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Mon, 15 Jul 2024 15:57:35 -0400 Subject: [PATCH 09/11] Skip over existing and unfinished large images --- .../rest/large_image_resource.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index c2839e4ca..8b8ab0c7d 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -42,7 +42,7 @@ from girder.models.item import Item from girder.models.setting import Setting from large_image import cache_util -from large_image.exceptions import TileGeneralError +from large_image.exceptions import TileGeneralError, TileSourceError from .. import constants, girder_tilesource from ..models.image_item import ImageItem @@ -450,8 +450,8 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): @access.user(scope=TokenScope.DATA_WRITE) @autoDescribeRoute( Description('Create new large images for all items within a folder.') - .notes('Does not work for items with multiple files and removes existing ' - 'large images from items.') + .notes('Does not work for items with multiple files and skips over items with ' + 'existing or unfinished large images.') .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) .param('createJobs', 'Whether job(s) should be created for each large image.', @@ -474,6 +474,7 @@ def createLargeImages(self, folder, params): def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs): result = {'childFoldersRecursed': 0, + 'itemsSkipped': 0, 'largeImagesCreated': 0, 'largeImagesRemovedAndRecreated': 0, 'totalItems': 0} @@ -489,14 +490,15 @@ def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs result['totalItems'] += 1 if item.get('largeImage'): if item['largeImage'].get('expected'): - pass + result['itemsSkipped'] += 1 else: try: - Item().getMetadata(item) + ImageItem().getMetadata(item) + result['itemsSkipped'] += 1 continue - except Exception: - previousFileId = item['largeImage'].get( - 'originalId', item['largeImage']['fileId']) + except TileSourceError: + previousFileId = item['largeImage'].get('originalId', + item['largeImage']['fileId']) ImageItem().delete(item) ImageItem().createImageItem(item, File().load(user=user, id=previousFileId), createJob=createJobs, localJob=localJobs) @@ -507,6 +509,8 @@ def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs ImageItem().createImageItem(item, files[0], createJob=createJobs, localJob=localJobs) result['largeImagesCreated'] += 1 + else: + result['itemsSkipped'] += 1 return result @describeRoute( From 35c894a532b8c8e438d606889937bc9745d3e41d Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Mon, 15 Jul 2024 16:35:09 -0400 Subject: [PATCH 10/11] Add test --- girder/test_girder/test_large_image.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/girder/test_girder/test_large_image.py b/girder/test_girder/test_large_image.py index fb9ab6a39..ddf13f876 100644 --- a/girder/test_girder/test_large_image.py +++ b/girder/test_girder/test_large_image.py @@ -237,6 +237,25 @@ def testThumbnailFileJob(server, admin, user, fsAssetstore): Setting().set(constants.PluginSettings.LARGE_IMAGE_MAX_THUMBNAIL_FILES, 0) +@pytest.mark.usefixtures('unbindLargeImage') +@pytest.mark.plugin('large_image') +def testFolderCreateImages(server, admin, user, fsAssetstore): + file = utilities.uploadExternalFile('sample_image.ptif', admin, fsAssetstore) + itemId = file['itemId'] + item = Item().load(itemId, user=admin) + folderId = str(item['folderId']) + # Remove the large image from this item + ImageItem().delete(item) + # Ask to make all items in this folder large images + resp = server.request( + method='PUT', path=f'/large_image/folder/{folderId}/tiles', user=admin) + assert utilities.respStatus(resp) == 200 + assert resp.json['largeImagesCreated'] == 1 + item = Item().load(itemId, user=admin) + # Check that this item became a large image again + assert 'largeImage' in item + + @pytest.mark.singular() @pytest.mark.usefixtures('unbindLargeImage') @pytest.mark.plugin('large_image') From 5ae4647dec4600d5aced0a51286e02ec1958e64d Mon Sep 17 00:00:00 2001 From: Gabrielle Duplan Date: Tue, 16 Jul 2024 11:11:11 -0400 Subject: [PATCH 11/11] Catch KeyError, extend test and change flag name --- .../girder_large_image/rest/large_image_resource.py | 8 +++----- girder/test_girder/test_large_image.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/girder/girder_large_image/rest/large_image_resource.py b/girder/girder_large_image/rest/large_image_resource.py index 8b8ab0c7d..cb39c054e 100644 --- a/girder/girder_large_image/rest/large_image_resource.py +++ b/girder/girder_large_image/rest/large_image_resource.py @@ -454,7 +454,7 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): 'existing or unfinished large images.') .modelParam('id', 'The ID of the folder.', model=Folder, level=AccessType.WRITE, required=True) - .param('createJobs', 'Whether job(s) should be created for each large image.', + .param('force', 'Whether creation job(s) should be forced for each large image.', required=False, default=False, dataType='boolean') .param('localJobs', 'Whether the job(s) created should be local.', required=False, default=False, dataType='boolean') @@ -465,9 +465,7 @@ def _deleteCachedImages(self, spec, associatedImages=False, imageKey=None): ) def createLargeImages(self, folder, params): user = self.getCurrentUser() - createJobs = params.get('createJobs') - if createJobs: - createJobs = 'always' or True + createJobs = 'always' if self.boolParam('force', params, default=False) else True return self.createImagesRecurseOption(folder=folder, createJobs=createJobs, user=user, recurse=params.get('recurse'), localJobs=params.get('localJobs')) @@ -496,7 +494,7 @@ def createImagesRecurseOption(self, folder, createJobs, user, recurse, localJobs ImageItem().getMetadata(item) result['itemsSkipped'] += 1 continue - except TileSourceError: + except (TileSourceError, KeyError): previousFileId = item['largeImage'].get('originalId', item['largeImage']['fileId']) ImageItem().delete(item) diff --git a/girder/test_girder/test_large_image.py b/girder/test_girder/test_large_image.py index ddf13f876..c6072ace1 100644 --- a/girder/test_girder/test_large_image.py +++ b/girder/test_girder/test_large_image.py @@ -254,6 +254,18 @@ def testFolderCreateImages(server, admin, user, fsAssetstore): item = Item().load(itemId, user=admin) # Check that this item became a large image again assert 'largeImage' in item + # Hitting the endpoint again should skip the item + resp = server.request( + method='PUT', path=f'/large_image/folder/{folderId}/tiles', user=admin) + assert utilities.respStatus(resp) == 200 + assert resp.json['itemsSkipped'] == 1 + # If the item's source isn't working, it should be recreated. + item['largeImage']['sourceName'] = 'unknown' + Item().updateItem(item) + resp = server.request( + method='PUT', path=f'/large_image/folder/{folderId}/tiles', user=admin) + assert utilities.respStatus(resp) == 200 + assert resp.json['largeImagesRemovedAndRecreated'] == 1 @pytest.mark.singular()