Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow users to specify whether to download in parallel #363

Merged
merged 1 commit into from
Mar 1, 2024
Merged

Allow users to specify whether to download in parallel #363

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 73 additions & 10 deletions mapreader/download/sheet_downloader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,7 @@ def _check_map_sheet_exists(self, feature: dict) -> bool:
return False
return False

def _download_map(self, feature: dict) -> bool:
def _download_map(self, feature: dict, download_in_parallel: bool = True) -> bool:
"""
Downloads a single map sheet and saves as png file.

Expand All @@ -573,7 +573,9 @@ def _download_map(self, feature: dict) -> bool:
True if map was downloaded successfully, False if not.
"""
map_name = str("map_" + feature["properties"]["IMAGE"])
self.downloader.download_tiles(feature["grid_bb"])
self.downloader.download_tiles(
feature["grid_bb"], download_in_parallel=download_in_parallel
)
success = self.merger.merge(feature["grid_bb"], map_name)
if success:
print(f'[INFO] Downloaded "{map_name}.png"')
Expand Down Expand Up @@ -681,6 +683,7 @@ def _download_map_sheets(
path_save: str | None = "maps",
metadata_fname: str | None = "metadata.csv",
overwrite: bool | None = False,
download_in_parallel: bool = True,
**kwargs: dict | None,
):
"""Download map sheets from a list of features.
Expand All @@ -695,6 +698,8 @@ def _download_map_sheets(
Name to use for metadata file, by default "metadata.csv"
overwrite : bool, optional
Whether to overwrite existing maps, by default ``False``.
download_in_parallel : bool, optional
Whether to download tiles in parallel, by default ``True``.
**kwargs : dict, optional
Keyword arguments to pass to the ``_save_metadata()`` method.
"""
Expand All @@ -703,7 +708,9 @@ def _download_map_sheets(
if not overwrite:
if self._check_map_sheet_exists(feature):
continue
success = self._download_map(feature)
success = self._download_map(
feature, download_in_parallel=download_in_parallel
)
if success:
metadata_path = f"{path_save}/{metadata_fname}"
self._save_metadata(
Expand All @@ -715,6 +722,7 @@ def download_all_map_sheets(
path_save: str | None = "maps",
metadata_fname: str | None = "metadata.csv",
overwrite: bool | None = False,
download_in_parallel: bool = True,
**kwargs: dict | None,
) -> None:
"""
Expand All @@ -728,6 +736,8 @@ def download_all_map_sheets(
Name to use for metadata file, by default "metadata.csv"
overwrite : bool, optional
Whether to overwrite existing maps, by default ``False``.
download_in_parallel : bool, optional
Whether to download tiles in parallel, by default ``True``.
**kwargs : dict, optional
Keyword arguments to pass to the ``_download_map_sheets()`` method.
"""
Expand All @@ -739,7 +749,12 @@ def download_all_map_sheets(

features = self.features
self._download_map_sheets(
features, path_save, metadata_fname, overwrite, **kwargs
features,
path_save,
metadata_fname,
overwrite,
download_in_parallel=download_in_parallel,
**kwargs,
)

def download_map_sheets_by_wfs_ids(
Expand All @@ -748,6 +763,7 @@ def download_map_sheets_by_wfs_ids(
path_save: str | None = "maps",
metadata_fname: str | None = "metadata.csv",
overwrite: bool | None = False,
download_in_parallel: bool = True,
**kwargs: dict | None,
) -> None:
"""
Expand All @@ -763,6 +779,8 @@ def download_map_sheets_by_wfs_ids(
Name to use for metadata file, by default "metadata.csv"
overwrite : bool, optional
Whether to overwrite existing maps, by default ``False``.
download_in_parallel : bool, optional
Whether to download tiles in parallel, by default ``True``.
**kwargs : dict, optional
Keyword arguments to pass to the ``_download_map_sheets()`` method.
"""
Expand Down Expand Up @@ -794,7 +812,12 @@ def download_map_sheets_by_wfs_ids(
features.append(feature)

self._download_map_sheets(
features, path_save, metadata_fname, overwrite, **kwargs
features,
path_save,
metadata_fname,
overwrite,
download_in_parallel=download_in_parallel,
**kwargs,
)

def download_map_sheets_by_polygon(
Expand All @@ -804,6 +827,7 @@ def download_map_sheets_by_polygon(
metadata_fname: str | None = "metadata.csv",
mode: str | None = "within",
overwrite: bool | None = False,
download_in_parallel: bool = True,
**kwargs: dict | None,
) -> None:
"""
Expand All @@ -824,6 +848,8 @@ def download_map_sheets_by_polygon(
By default "within".
overwrite : bool, optional
Whether to overwrite existing maps, by default ``False``.
download_in_parallel : bool, optional
Whether to download tiles in parallel, by default ``True``.
**kwargs : dict, optional
Keyword arguments to pass to the ``_download_map_sheets()`` method.

Expand Down Expand Up @@ -869,7 +895,12 @@ def download_map_sheets_by_polygon(
features.append(feature)

self._download_map_sheets(
features, path_save, metadata_fname, overwrite, **kwargs
features,
path_save,
metadata_fname,
overwrite,
download_in_parallel=download_in_parallel,
**kwargs,
)

def download_map_sheets_by_coordinates(
Expand All @@ -878,6 +909,7 @@ def download_map_sheets_by_coordinates(
path_save: str | None = "maps",
metadata_fname: str | None = "metadata.csv",
overwrite: bool | None = False,
download_in_parallel: bool = True,
**kwargs: dict | None,
) -> None:
"""
Expand All @@ -894,6 +926,8 @@ def download_map_sheets_by_coordinates(
Name to use for metadata file, by default "metadata.csv"
overwrite : bool, optional
Whether to overwrite existing maps, by default ``False``.
download_in_parallel : bool, optional
Whether to download tiles in parallel, by default ``True``.
**kwargs : dict, optional
Keyword arguments to pass to the ``_download_map_sheets()`` method.
"""
Expand Down Expand Up @@ -922,7 +956,12 @@ def download_map_sheets_by_coordinates(
features.append(feature)

self._download_map_sheets(
features, path_save, metadata_fname, overwrite, **kwargs
features,
path_save,
metadata_fname,
overwrite,
download_in_parallel=download_in_parallel,
**kwargs,
)

def download_map_sheets_by_line(
Expand All @@ -931,6 +970,7 @@ def download_map_sheets_by_line(
path_save: str | None = "maps",
metadata_fname: str | None = "metadata.csv",
overwrite: bool | None = False,
download_in_parallel: bool = True,
**kwargs: dict | None,
) -> None:
"""
Expand All @@ -946,6 +986,8 @@ def download_map_sheets_by_line(
Name to use for metadata file, by default "metadata.csv"
overwrite : bool, optional
Whether to overwrite existing maps, by default ``False``
download_in_parallel : bool, optional
Whether to download tiles in parallel, by default ``True``.
**kwargs : dict, optional
Keyword arguments to pass to the ``_download_map_sheets()`` method.

Expand Down Expand Up @@ -980,7 +1022,12 @@ def download_map_sheets_by_line(
features.append(feature)

self._download_map_sheets(
features, path_save, metadata_fname, overwrite, **kwargs
features,
path_save,
metadata_fname,
overwrite,
download_in_parallel=download_in_parallel,
**kwargs,
)

def download_map_sheets_by_string(
Expand All @@ -990,6 +1037,7 @@ def download_map_sheets_by_string(
path_save: str | None = "maps",
metadata_fname: str | None = "metadata.csv",
overwrite: bool | None = False,
download_in_parallel: bool = True,
**kwargs: dict | None,
) -> None:
"""
Expand All @@ -1013,6 +1061,8 @@ def download_map_sheets_by_string(
Name to use for metadata file, by default "metadata.csv"
overwrite : bool, optional
Whether to overwrite existing maps, by default ``False``.
download_in_parallel : bool, optional
Whether to download tiles in parallel, by default ``True``.
**kwargs : dict, optional
Keyword arguments to pass to the ``_download_map_sheets()`` method.

Expand Down Expand Up @@ -1053,14 +1103,20 @@ def download_map_sheets_by_string(
features.append(feature)

self._download_map_sheets(
features, path_save, metadata_fname, overwrite, **kwargs
features,
path_save,
metadata_fname,
overwrite,
download_in_parallel=download_in_parallel,
**kwargs,
)

def download_map_sheets_by_queries(
self,
path_save: str | None = "maps",
metadata_fname: str | None = "metadata.csv",
overwrite: bool | None = False,
download_in_parallel: bool = True,
**kwargs: dict | None,
) -> None:
"""
Expand All @@ -1074,6 +1130,8 @@ def download_map_sheets_by_queries(
Name to use for metadata file, by default "metadata.csv"
overwrite : bool, optional
Whether to overwrite existing maps, by default ``False``.
download_in_parallel : bool, optional
Whether to download tiles in parallel, by default ``True``.
**kwargs : dict, optional
Keyword arguments to pass to the ``_download_map_sheets()`` method.
"""
Expand All @@ -1088,7 +1146,12 @@ def download_map_sheets_by_queries(

features = self.found_queries
self._download_map_sheets(
features, path_save, metadata_fname, overwrite, **kwargs
features,
path_save,
metadata_fname,
overwrite,
download_in_parallel=download_in_parallel,
**kwargs,
)

def hist_published_dates(self, **kwargs) -> None:
Expand Down
Loading