Merge branch 'develop'

MobileTeleSystems · Aug 17, 2023 · 5f0f86a · 5f0f86a
2 parents b448bbd + 801fff2
commit 5f0f86a
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 4 deletions.
diff --git a/docs/changelog/0.9.1.rst b/docs/changelog/0.9.1.rst
@@ -0,0 +1,9 @@
+0.9.1 (2023-08-17)
+==================
+
+Bug Fixes
+---------
+
+- Fixed bug then number of threads created by ``FileDownloader`` / ``FileUploader`` / ``FileMover`` was
+  not ``min(workers, len(files))``, but ``max(workers, len(files))``. leading to create too much workers
+  on large files list.
diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst
@@ -4,6 +4,7 @@
 
     DRAFT
     NEXT_RELEASE
+    0.9.1
     0.9.0
     0.8.1
     0.8.0

diff --git a/onetl/VERSION b/onetl/VERSION
@@ -1 +1 @@
-0.9.0
+0.9.1
diff --git a/onetl/file/file_downloader/file_downloader.py b/onetl/file/file_downloader/file_downloader.py
@@ -719,7 +719,7 @@ def _bulk_download(
 
         if workers > 1:
             with ThreadPoolExecutor(
-                max_workers=max(workers, len(to_download)),
+                max_workers=min(workers, len(to_download)),
                 thread_name_prefix=self.__class__.__name__,
             ) as executor:
                 futures = [

diff --git a/onetl/file/file_mover/file_mover.py b/onetl/file/file_mover/file_mover.py
@@ -516,7 +516,7 @@ def _bulk_move(
 
         if workers > 1:
             with ThreadPoolExecutor(
-                max_workers=max(workers, len(to_move)),
+                max_workers=min(workers, len(to_move)),
                 thread_name_prefix=self.__class__.__name__,
             ) as executor:
                 futures = [

diff --git a/onetl/file/file_uploader/file_uploader.py b/onetl/file/file_uploader/file_uploader.py
@@ -544,7 +544,7 @@ def _bulk_upload(
 
         if workers > 1:
             with ThreadPoolExecutor(
-                max_workers=max(workers, len(to_upload)),
+                max_workers=min(workers, len(to_upload)),
                 thread_name_prefix=self.__class__.__name__,
             ) as executor:
                 futures = [