From 0411ec47862413dab9ceda724ce8a97208233ad5 Mon Sep 17 00:00:00 2001 From: Matthew The Date: Tue, 2 Jan 2024 17:23:46 +0100 Subject: [PATCH] Limit number of BLAS threads during parallelization. Fixes #1 --- job_pool/job_pool.py | 2 ++ poetry.lock | 13 ++++++++++++- pyproject.toml | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/job_pool/job_pool.py b/job_pool/job_pool.py index 8933460..b3530c0 100644 --- a/job_pool/job_pool.py +++ b/job_pool/job_pool.py @@ -1,3 +1,4 @@ +from threadpoolctl import threadpool_limits import time import signal from typing import Optional @@ -58,6 +59,7 @@ class AbnormalWorkerTerminationError(Exception): class JobPool: + @threadpool_limits.wrap(limits=1, user_api='blas') def __init__( self, processes: int = 1, diff --git a/poetry.lock b/poetry.lock index 5e0851e..5f38a0f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -177,6 +177,17 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] +[[package]] +name = "threadpoolctl" +version = "3.2.0" +description = "threadpoolctl" +optional = false +python-versions = ">=3.8" +files = [ + {file = "threadpoolctl-3.2.0-py3-none-any.whl", hash = "sha256:2b7818516e423bdaebb97c723f86a7c6b0a83d3f3b0970328d66f4d9104dc032"}, + {file = "threadpoolctl-3.2.0.tar.gz", hash = "sha256:c96a0ba3bdddeaca37dc4cc7344aafad41cdb8c313f74fdfe387a867bba93355"}, +] + [[package]] name = "tomli" version = "2.0.1" @@ -211,4 +222,4 @@ telegram = ["requests"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "5232cf250867c5edbb47cbac9b1e6aea142698d7867c5d80554ebe3519fdf8fd" +content-hash = "0c4259c19f8ac5f066855aa08204b3b17e232b86d448ad7ae475f30131031750" diff --git a/pyproject.toml b/pyproject.toml index 1476d27..5b313a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ packages = [{include = "job_pool"}] [tool.poetry.dependencies] python = "^3.8" tqdm = "^4.66.1" +threadpoolctl = "^3.2.0" [tool.poetry.group.dev.dependencies]