From 51d90e82a46c83583e19c79121f40128dab8777d Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Fri, 22 Mar 2024 12:40:00 +0100 Subject: [PATCH 1/2] fix fsspec tqdm callback --- src/datasets/utils/file_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/datasets/utils/file_utils.py b/src/datasets/utils/file_utils.py index 7c823831676..e31d80a7134 100644 --- a/src/datasets/utils/file_utils.py +++ b/src/datasets/utils/file_utils.py @@ -327,8 +327,12 @@ def stack_multiprocessing_download_progress_bars(): class TqdmCallback(fsspec.callbacks.TqdmCallback): def __init__(self, tqdm_kwargs=None, *args, **kwargs): - super().__init__(tqdm_kwargs, *args, **kwargs) - self._tqdm = _tqdm # replace tqdm.tqdm by datasets.tqdm.tqdm + if config.FSSPEC_VERSION < version.parse("2024.2.0"): + super().__init__(tqdm_kwargs, *args, **kwargs) + self._tqdm = _tqdm # replace tqdm.tqdm by datasets.tqdm.tqdm + else: + kwargs["tqdm_cls"] = _tqdm.tqdm + super().__init__(tqdm_kwargs, *args, **kwargs) def fsspec_get(url, temp_file, storage_options=None, desc=None): From aa00e55a352f810cf99b2e9206a3be2e3246f9f2 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Fri, 22 Mar 2024 12:44:03 +0100 Subject: [PATCH 2/2] minor --- src/datasets/utils/file_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/utils/file_utils.py b/src/datasets/utils/file_utils.py index e31d80a7134..6384c521124 100644 --- a/src/datasets/utils/file_utils.py +++ b/src/datasets/utils/file_utils.py @@ -329,7 +329,7 @@ class TqdmCallback(fsspec.callbacks.TqdmCallback): def __init__(self, tqdm_kwargs=None, *args, **kwargs): if config.FSSPEC_VERSION < version.parse("2024.2.0"): super().__init__(tqdm_kwargs, *args, **kwargs) - self._tqdm = _tqdm # replace tqdm.tqdm by datasets.tqdm.tqdm + self._tqdm = _tqdm # replace tqdm module by datasets.utils.tqdm module else: kwargs["tqdm_cls"] = _tqdm.tqdm super().__init__(tqdm_kwargs, *args, **kwargs)