From b5c6ddc4db804c5def9942f47a7091b2261b5cc2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 25 Jan 2021 23:50:25 +0100 Subject: [PATCH] Download: Use '.partial' extensions whilst downloading. To prevent accidentally using partial broken downloads, use an additional .partial file extension whilst download is running and rename the file to remove this when complete. --- nf_core/download.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 75d7ba033..d353ccd8f 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -478,7 +478,13 @@ def singularity_download_image(self, container, out_path, cache_path, progress): nice_name = container.split("/")[-1][:50] task = progress.add_task(nice_name, start=False, total=False, progress_type="download") try: - with open(output_path, "wb") as fh: + # Set a temporary filename to download to + output_path_tmp = f"{output_path}.partial" + if os.path.exists(output_path_tmp): + os.remove(output_path_tmp) + + # Open file handle and download + with open(output_path_tmp, "wb") as fh: # Disable caching as this breaks streamed downloads with requests_cache.disabled(): r = requests.get(container, allow_redirects=True, stream=True, timeout=60 * 5) @@ -492,13 +498,16 @@ def singularity_download_image(self, container, out_path, cache_path, progress): progress.update(task, advance=len(data)) fh.write(data) - # Copy cached download if we are using the cache - if cache_path: - log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) - progress.update(task, description="Copying from cache to target directory") - shutil.copyfile(cache_path, out_path) + # Rename partial filename to final filename + os.rename(output_path_tmp, output_path) + + # Copy cached download if we are using the cache + if cache_path: + log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) + progress.update(task, description="Copying from cache to target directory") + shutil.copyfile(cache_path, out_path) - progress.remove_task(task) + progress.remove_task(task) except: # Kill the progress bars