Skip to content

Commit

Permalink
Address couple of issues with the same asset being in multiple exports (
Browse files Browse the repository at this point in the history
#67)

* Fix problems with downloading same file in multiple exports

* Download progress reporting is now correct for all exports where the
  same file was referenced in multiple places
* If a file was already downloaded, instead of downloading a second time
  copy the already completed version of the file to the new destination
* Re-enable the list and download exports buttons after unhandled
  exceptions
  • Loading branch information
j9ac9k authored Feb 23, 2024
1 parent a84465d commit 31d3b7f
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 47 deletions.
2 changes: 1 addition & 1 deletion src/doppkit/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
__version__ = "0.3.3"
__version__ = "0.3.4"

from . import cache
from . import grid
Expand Down
36 changes: 27 additions & 9 deletions src/doppkit/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import pathlib
import logging
import asyncio
import shutil
import httpx
from io import BytesIO
from .util import parse_options_header
Expand Down Expand Up @@ -37,6 +38,8 @@ def complete_task(self, name: str, url: str) -> None:
...




class Content:
def __init__(
self,
Expand Down Expand Up @@ -89,6 +92,9 @@ def get_data(self) -> bytes:
data = property(get_data)


completed_downloads: dict[str, Content] = dict()


async def cache(
app: 'Application',
urls: Iterable[DownloadUrl],
Expand Down Expand Up @@ -179,15 +185,27 @@ async def cache_url(
# create parent directory/directories if needed
if c.target.parent is not None:
c.target.parent.mkdir(parents=True, exist_ok=True)
# we are writing to disk asynchronously
async with aiofiles.open(c.target, "wb+") as f:
async for chunk in response.aiter_bytes():
await f.write(chunk)
chunk_count += 1
if args.progress and progress is not None:
progress.update(
name, url.url, completed=response.num_bytes_downloaded
)

# check if we already downloaded this asset
if url.url in completed_downloads:
# get the path stored previously
previous_content = completed_downloads[url.url]
logger.info(f"Download cache hit on {c.target.name}, copying from {previous_content.target}")
shutil.copy(previous_content.target, c.target)
else:
# we are writing to disk asynchronously
async with aiofiles.open(c.target, "wb+") as f:
async for chunk in response.aiter_bytes():
await f.write(chunk)
chunk_count += 1
if args.progress and progress is not None:
progress.update(
name,
url.url,
completed=response.num_bytes_downloaded
)
completed_downloads[url.url] = c

if args.progress and progress is not None:
# we can hide the task now that it's finished
progress.complete_task(name, url.url)
Expand Down
78 changes: 41 additions & 37 deletions src/doppkit/gui/window.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def __init__(self):
# key is AOI PK
self.aois: dict[int, list[ProgressTracking]] = defaultdict(list)

self.urls_to_export_id: dict[str, int] = {}
self.urls_to_export_id: dict[str, list[int]] = defaultdict(list)

self.export_files: dict[int, dict[str, ExportFileProgress]] = defaultdict(dict)

Expand All @@ -66,40 +66,40 @@ def create_task(self, name: str, url: str, total: int):
None
"""

export_id = self.urls_to_export_id[url]
self.export_files[export_id][url] = ExportFileProgress(
url,
export_id=export_id,
total=total
)
export_ids = self.urls_to_export_id[url]
for export_id in export_ids:
self.export_files[export_id][url] = ExportFileProgress(
url,
export_id=export_id,
total=total
)

def update(self, name: str, url: str, completed: int) -> None:
export_id = self.urls_to_export_id[url]

old_progress = self.export_files[export_id][url]
self.export_files[export_id][url] = ExportFileProgress(
url,
export_id=export_id,
current=completed,
total=old_progress.total,
is_complete=completed >= old_progress.total
)

export_progress = self.export_progress[export_id]
export_downloaded = sum(file_progress.current for file_progress in self.export_files[export_id].values())
export_progress.update(export_downloaded)
self.taskUpdated.emit(export_progress)
export_ids = self.urls_to_export_id[url]
for export_id in export_ids:
old_progress = self.export_files[export_id][url]
self.export_files[export_id][url] = ExportFileProgress(
url,
export_id=export_id,
current=completed,
total=old_progress.total,
is_complete=completed >= old_progress.total
)
export_progress = self.export_progress[export_id]
export_downloaded = sum(file_progress.current for file_progress in self.export_files[export_id].values())
export_progress.update(export_downloaded)
self.taskUpdated.emit(export_progress)

def complete_task(self, name: str, url: str) -> None:
export_id = self.urls_to_export_id[url]
old_progress = self.export_files[export_id][url]

self.update(name, url, old_progress.total)
export_ids = self.urls_to_export_id[url]

export_progress = self.export_progress[export_id]
if all(export_file.is_complete for export_file in self.export_files[export_id].values()):
export_progress.is_complete = True
self.taskCompleted.emit(export_progress)
for export_id in export_ids:
old_progress = self.export_files[export_id][url]
self.update(name, url, old_progress.total)
export_progress = self.export_progress[export_id]
if all(export_file.is_complete for export_file in self.export_files[export_id].values()):
export_progress.is_complete = True
self.taskCompleted.emit(export_progress)

def update_export_progress(self):
pass
Expand Down Expand Up @@ -369,7 +369,11 @@ async def listExports(self):
# there is no AOI entered...
self.buttonList.setEnabled(True)
return None
self.AOIs = await api.get_aois(self.AOI_ids[0])
try:
self.AOIs = await api.get_aois(self.AOI_ids[0])
finally:
# no need to elave the button list grayed out if there is an exception...
self.buttonList.setEnabled(True)
model = ExportModel()
model.clear()

Expand Down Expand Up @@ -397,9 +401,7 @@ async def listExports(self):
treeViewSize = self.exportView.size()
treeViewSize.setWidth(400)
self.exportView.resize(treeViewSize)

self.exportView.show()
self.buttonList.setEnabled(True)


@qasync.asyncSlot()
Expand Down Expand Up @@ -435,9 +437,8 @@ async def downloadExports(self):
urls.append(
download_file
)
print(f"{download_file.save_path} = {download_file.total} bytes")
download_size += download_file.total
self.progressInterconnect.urls_to_export_id[download_file.url] = export["id"]
self.progressInterconnect.urls_to_export_id[download_file.url].append(export["id"])
progress_tracker = ProgressTracking(
export["id"],
export_name=export["name"],
Expand All @@ -449,8 +450,11 @@ async def downloadExports(self):
self.progressInterconnect.export_progress[export["id"]] = progress_tracker
self.progressInterconnect.aois[aoi["id"]].append(progress_tracker)

_ = await cache(self.doppkit, urls, {}, progress=self.progressInterconnect)
self.buttonDownload.setEnabled(True)
try:
_ = await cache(self.doppkit, urls, {}, progress=self.progressInterconnect)
logger.info("Download AOI Exports Complete")
finally:
self.buttonDownload.setEnabled(True)

@dataclass
class ProgressTracking:
Expand Down

0 comments on commit 31d3b7f

Please sign in to comment.