diff --git a/src/doppkit/__init__.py b/src/doppkit/__init__.py index 1af0edc..d324f53 100644 --- a/src/doppkit/__init__.py +++ b/src/doppkit/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.3" +__version__ = "0.3.4" from . import cache from . import grid diff --git a/src/doppkit/cache.py b/src/doppkit/cache.py index e259112..c745e3a 100644 --- a/src/doppkit/cache.py +++ b/src/doppkit/cache.py @@ -5,6 +5,7 @@ import pathlib import logging import asyncio +import shutil import httpx from io import BytesIO from .util import parse_options_header @@ -37,6 +38,8 @@ def complete_task(self, name: str, url: str) -> None: ... + + class Content: def __init__( self, @@ -89,6 +92,9 @@ def get_data(self) -> bytes: data = property(get_data) +completed_downloads: dict[str, Content] = dict() + + async def cache( app: 'Application', urls: Iterable[DownloadUrl], @@ -179,15 +185,27 @@ async def cache_url( # create parent directory/directories if needed if c.target.parent is not None: c.target.parent.mkdir(parents=True, exist_ok=True) - # we are writing to disk asynchronously - async with aiofiles.open(c.target, "wb+") as f: - async for chunk in response.aiter_bytes(): - await f.write(chunk) - chunk_count += 1 - if args.progress and progress is not None: - progress.update( - name, url.url, completed=response.num_bytes_downloaded - ) + + # check if we already downloaded this asset + if url.url in completed_downloads: + # get the path stored previously + previous_content = completed_downloads[url.url] + logger.info(f"Download cache hit on {c.target.name}, copying from {previous_content.target}") + shutil.copy(previous_content.target, c.target) + else: + # we are writing to disk asynchronously + async with aiofiles.open(c.target, "wb+") as f: + async for chunk in response.aiter_bytes(): + await f.write(chunk) + chunk_count += 1 + if args.progress and progress is not None: + progress.update( + name, + url.url, + completed=response.num_bytes_downloaded + ) + completed_downloads[url.url] = c + if args.progress and progress is not None: # we can hide the task now that it's finished progress.complete_task(name, url.url) diff --git a/src/doppkit/gui/window.py b/src/doppkit/gui/window.py index 24fb1ce..180e24a 100644 --- a/src/doppkit/gui/window.py +++ b/src/doppkit/gui/window.py @@ -44,7 +44,7 @@ def __init__(self): # key is AOI PK self.aois: dict[int, list[ProgressTracking]] = defaultdict(list) - self.urls_to_export_id: dict[str, int] = {} + self.urls_to_export_id: dict[str, list[int]] = defaultdict(list) self.export_files: dict[int, dict[str, ExportFileProgress]] = defaultdict(dict) @@ -66,40 +66,40 @@ def create_task(self, name: str, url: str, total: int): None """ - export_id = self.urls_to_export_id[url] - self.export_files[export_id][url] = ExportFileProgress( - url, - export_id=export_id, - total=total - ) + export_ids = self.urls_to_export_id[url] + for export_id in export_ids: + self.export_files[export_id][url] = ExportFileProgress( + url, + export_id=export_id, + total=total + ) def update(self, name: str, url: str, completed: int) -> None: - export_id = self.urls_to_export_id[url] - - old_progress = self.export_files[export_id][url] - self.export_files[export_id][url] = ExportFileProgress( - url, - export_id=export_id, - current=completed, - total=old_progress.total, - is_complete=completed >= old_progress.total - ) - - export_progress = self.export_progress[export_id] - export_downloaded = sum(file_progress.current for file_progress in self.export_files[export_id].values()) - export_progress.update(export_downloaded) - self.taskUpdated.emit(export_progress) + export_ids = self.urls_to_export_id[url] + for export_id in export_ids: + old_progress = self.export_files[export_id][url] + self.export_files[export_id][url] = ExportFileProgress( + url, + export_id=export_id, + current=completed, + total=old_progress.total, + is_complete=completed >= old_progress.total + ) + export_progress = self.export_progress[export_id] + export_downloaded = sum(file_progress.current for file_progress in self.export_files[export_id].values()) + export_progress.update(export_downloaded) + self.taskUpdated.emit(export_progress) def complete_task(self, name: str, url: str) -> None: - export_id = self.urls_to_export_id[url] - old_progress = self.export_files[export_id][url] - - self.update(name, url, old_progress.total) + export_ids = self.urls_to_export_id[url] - export_progress = self.export_progress[export_id] - if all(export_file.is_complete for export_file in self.export_files[export_id].values()): - export_progress.is_complete = True - self.taskCompleted.emit(export_progress) + for export_id in export_ids: + old_progress = self.export_files[export_id][url] + self.update(name, url, old_progress.total) + export_progress = self.export_progress[export_id] + if all(export_file.is_complete for export_file in self.export_files[export_id].values()): + export_progress.is_complete = True + self.taskCompleted.emit(export_progress) def update_export_progress(self): pass @@ -369,7 +369,11 @@ async def listExports(self): # there is no AOI entered... self.buttonList.setEnabled(True) return None - self.AOIs = await api.get_aois(self.AOI_ids[0]) + try: + self.AOIs = await api.get_aois(self.AOI_ids[0]) + finally: + # no need to elave the button list grayed out if there is an exception... + self.buttonList.setEnabled(True) model = ExportModel() model.clear() @@ -397,9 +401,7 @@ async def listExports(self): treeViewSize = self.exportView.size() treeViewSize.setWidth(400) self.exportView.resize(treeViewSize) - self.exportView.show() - self.buttonList.setEnabled(True) @qasync.asyncSlot() @@ -435,9 +437,8 @@ async def downloadExports(self): urls.append( download_file ) - print(f"{download_file.save_path} = {download_file.total} bytes") download_size += download_file.total - self.progressInterconnect.urls_to_export_id[download_file.url] = export["id"] + self.progressInterconnect.urls_to_export_id[download_file.url].append(export["id"]) progress_tracker = ProgressTracking( export["id"], export_name=export["name"], @@ -449,8 +450,11 @@ async def downloadExports(self): self.progressInterconnect.export_progress[export["id"]] = progress_tracker self.progressInterconnect.aois[aoi["id"]].append(progress_tracker) - _ = await cache(self.doppkit, urls, {}, progress=self.progressInterconnect) - self.buttonDownload.setEnabled(True) + try: + _ = await cache(self.doppkit, urls, {}, progress=self.progressInterconnect) + logger.info("Download AOI Exports Complete") + finally: + self.buttonDownload.setEnabled(True) @dataclass class ProgressTracking: