From a56e014cbb6e343f050fbd9aa774d507de7f73fc Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Mon, 6 Feb 2023 18:09:43 +0100 Subject: [PATCH 1/4] Fix WorkerLoop.loop worker_factory --- workers/datasets_based/src/datasets_based/worker_loop.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workers/datasets_based/src/datasets_based/worker_loop.py b/workers/datasets_based/src/datasets_based/worker_loop.py index 3d03d2078f..4046e3f8e5 100644 --- a/workers/datasets_based/src/datasets_based/worker_loop.py +++ b/workers/datasets_based/src/datasets_based/worker_loop.py @@ -10,7 +10,7 @@ from libcommon.queue import EmptyQueueError, Queue from psutil import cpu_count, disk_usage, getloadavg, swap_memory, virtual_memory -from datasets_based.worker import WorkerFactory +from datasets_based.worker_factory import DatasetBasedWorkerFactory @dataclass @@ -31,7 +31,7 @@ class WorkerLoop: """ queue: Queue - worker_factory: WorkerFactory + worker_factory: DatasetBasedWorkerFactory worker_loop_config: WorkerLoopConfig def log(self, level: int, msg: str) -> None: From b59ef2428308ecfab7a9b005ac631394f53a8b07 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Mon, 6 Feb 2023 18:34:43 +0100 Subject: [PATCH 2/4] Fix test --- workers/datasets_based/tests/test_worker_loop.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/workers/datasets_based/tests/test_worker_loop.py b/workers/datasets_based/tests/test_worker_loop.py index f233bac14b..89a54a83fc 100644 --- a/workers/datasets_based/tests/test_worker_loop.py +++ b/workers/datasets_based/tests/test_worker_loop.py @@ -6,7 +6,8 @@ from libcommon.queue import Queue from datasets_based.config import AppConfig -from datasets_based.worker import JobInfo, Worker, WorkerFactory +from datasets_based.worker import JobInfo, Worker +from datasets_based.worker_factory import DatasetBasedWorkerFactory from datasets_based.worker_loop import WorkerLoop @@ -33,8 +34,9 @@ def compute(self) -> Mapping[str, Any]: return {"key": "value"} -class DummyWorkerFactory(WorkerFactory): - def __init__(self, processing_step: ProcessingStep) -> None: +class DummyWorkerFactory(DatasetBasedWorkerFactory): + def __init__(self, app_config: AppConfig, processing_step: ProcessingStep) -> None: + self.app_config = app_config self.common_config = CommonConfig() self.processing_step = processing_step @@ -43,10 +45,11 @@ def _create_worker(self, job_info: JobInfo) -> Worker: def test_process_next_job( + app_config: AppConfig, test_processing_step: ProcessingStep, queue_config: QueueConfig, ) -> None: - worker_factory = DummyWorkerFactory(processing_step=test_processing_step) + worker_factory = DummyWorkerFactory(app_config=app_config, processing_step=test_processing_step) queue = Queue(type=test_processing_step.endpoint, max_jobs_per_namespace=queue_config.max_jobs_per_namespace) worker_loop = WorkerLoop( worker_factory=worker_factory, From 245a99cd0dfd492d702813b2a3b893f8191c6eb9 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Tue, 7 Feb 2023 10:41:09 +0100 Subject: [PATCH 3/4] Revert changes Revert "Fix test" This reverts commit b59ef2428308ecfab7a9b005ac631394f53a8b07. Revert "Fix WorkerLoop.loop worker_factory" This reverts commit a56e014cbb6e343f050fbd9aa774d507de7f73fc. --- .../datasets_based/src/datasets_based/worker_loop.py | 4 ++-- workers/datasets_based/tests/test_worker_loop.py | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/workers/datasets_based/src/datasets_based/worker_loop.py b/workers/datasets_based/src/datasets_based/worker_loop.py index 4046e3f8e5..3d03d2078f 100644 --- a/workers/datasets_based/src/datasets_based/worker_loop.py +++ b/workers/datasets_based/src/datasets_based/worker_loop.py @@ -10,7 +10,7 @@ from libcommon.queue import EmptyQueueError, Queue from psutil import cpu_count, disk_usage, getloadavg, swap_memory, virtual_memory -from datasets_based.worker_factory import DatasetBasedWorkerFactory +from datasets_based.worker import WorkerFactory @dataclass @@ -31,7 +31,7 @@ class WorkerLoop: """ queue: Queue - worker_factory: DatasetBasedWorkerFactory + worker_factory: WorkerFactory worker_loop_config: WorkerLoopConfig def log(self, level: int, msg: str) -> None: diff --git a/workers/datasets_based/tests/test_worker_loop.py b/workers/datasets_based/tests/test_worker_loop.py index 89a54a83fc..f233bac14b 100644 --- a/workers/datasets_based/tests/test_worker_loop.py +++ b/workers/datasets_based/tests/test_worker_loop.py @@ -6,8 +6,7 @@ from libcommon.queue import Queue from datasets_based.config import AppConfig -from datasets_based.worker import JobInfo, Worker -from datasets_based.worker_factory import DatasetBasedWorkerFactory +from datasets_based.worker import JobInfo, Worker, WorkerFactory from datasets_based.worker_loop import WorkerLoop @@ -34,9 +33,8 @@ def compute(self) -> Mapping[str, Any]: return {"key": "value"} -class DummyWorkerFactory(DatasetBasedWorkerFactory): - def __init__(self, app_config: AppConfig, processing_step: ProcessingStep) -> None: - self.app_config = app_config +class DummyWorkerFactory(WorkerFactory): + def __init__(self, processing_step: ProcessingStep) -> None: self.common_config = CommonConfig() self.processing_step = processing_step @@ -45,11 +43,10 @@ def _create_worker(self, job_info: JobInfo) -> Worker: def test_process_next_job( - app_config: AppConfig, test_processing_step: ProcessingStep, queue_config: QueueConfig, ) -> None: - worker_factory = DummyWorkerFactory(app_config=app_config, processing_step=test_processing_step) + worker_factory = DummyWorkerFactory(processing_step=test_processing_step) queue = Queue(type=test_processing_step.endpoint, max_jobs_per_namespace=queue_config.max_jobs_per_namespace) worker_loop = WorkerLoop( worker_factory=worker_factory, From 0cc2bae9f303735c8c8444cc52fdf96034f60b40 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Tue, 7 Feb 2023 10:44:40 +0100 Subject: [PATCH 4/4] Remove log from WorkerLoop.loop --- workers/datasets_based/src/datasets_based/worker_loop.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workers/datasets_based/src/datasets_based/worker_loop.py b/workers/datasets_based/src/datasets_based/worker_loop.py index 3d03d2078f..0c27d5c38c 100644 --- a/workers/datasets_based/src/datasets_based/worker_loop.py +++ b/workers/datasets_based/src/datasets_based/worker_loop.py @@ -97,7 +97,6 @@ def sleep(self) -> None: time.sleep(duration) def loop(self) -> None: - self.info(f"Using endpoint {self.worker_factory.app_config.common.hf_endpoint}") self.info("Worker started") try: while True: