add option to skip download existing files

binh-vu · Jan 8, 2024 · 7db7965 · 7db7965
1 parent 3e4b19f
commit 7db7965
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/kgdata/misc/download.py b/kgdata/misc/download.py
@@ -128,7 +128,7 @@ def list_files(
         return files
 
     def create_download_jobs(
-        self, files: list[DumpFile], rootdir: str | Path
+        self, files: list[DumpFile], rootdir: str | Path, skip_if_exists: bool = True
     ) -> list[tuple[str, Path]]:
         rootdir = Path(rootdir)
         main_category_urls = {
@@ -143,6 +143,8 @@ def create_download_jobs(
         jobs = []
         for file in files:
             outfile = dump_dir / get_url_filename(file.url)
+            if outfile.exists() and skip_if_exists:
+                continue
             jobs.append((file.url, outfile))
         return jobs
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "kgdata"
-version = "6.2.0"
+version = "6.2.1"
 description = "Library to process dumps of knowledge graphs (Wikipedia, DBpedia, Wikidata)"
 readme = "README.md"
 authors = [{ name = "Binh Vu", email = "binh@toan2.com" }]