diff --git a/torchbenchmark/canary_models/DALLE2_pytorch/install.py b/torchbenchmark/canary_models/DALLE2_pytorch/install.py index 22f7c52a3d..9ec98b38c0 100644 --- a/torchbenchmark/canary_models/DALLE2_pytorch/install.py +++ b/torchbenchmark/canary_models/DALLE2_pytorch/install.py @@ -2,6 +2,7 @@ import patch import subprocess import sys +from utils.python_utils import pip_install_requirements def patch_dalle2(): import dalle2_pytorch @@ -12,8 +13,8 @@ def patch_dalle2(): print("Failed to patch dalle2_pytorch/dalle2_pytorch.py. Exit.") exit(1) -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +def pip_install_requirements_dalle2(): + pip_install_requirements() # DALLE2_pytorch requires embedding-reader # https://github.com/lucidrains/DALLE2-pytorch/blob/00e07b7d61e21447d55e6d06d5c928cf8b67601d/setup.py#L34 # embedding-reader requires an old version of pandas and pyarrow @@ -22,5 +23,5 @@ def pip_install_requirements(): subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-U', 'pandas', 'pyarrow']) if __name__ == '__main__': - pip_install_requirements() + pip_install_requirements_dalle2() patch_dalle2() \ No newline at end of file diff --git a/torchbenchmark/canary_models/codellama/install.py b/torchbenchmark/canary_models/codellama/install.py index cc27b6e7cf..9ebc7b8179 100644 --- a/torchbenchmark/canary_models/codellama/install.py +++ b/torchbenchmark/canary_models/codellama/install.py @@ -1,6 +1,4 @@ -import subprocess -import sys import os from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model diff --git a/torchbenchmark/canary_models/fambench_dlrm/install.py b/torchbenchmark/canary_models/fambench_dlrm/install.py index df5435fecc..2bb9cae7c4 100644 --- a/torchbenchmark/canary_models/fambench_dlrm/install.py +++ b/torchbenchmark/canary_models/fambench_dlrm/install.py @@ -2,6 +2,7 @@ import sys import subprocess from torchbenchmark import REPO_PATH +from utils.python_utils import pip_install_requirements def update_fambench_submodule(): @@ -17,12 +18,6 @@ def update_fambench_submodule(): subprocess.check_call(update_command, cwd=REPO_PATH) -def pip_install_requirements(): - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "-q", "-r", "requirements.txt"] - ) - - if __name__ == "__main__": update_fambench_submodule() pip_install_requirements() diff --git a/torchbenchmark/canary_models/fambench_xlmr/install.py b/torchbenchmark/canary_models/fambench_xlmr/install.py index c36963ae7e..fad0952bdb 100644 --- a/torchbenchmark/canary_models/fambench_xlmr/install.py +++ b/torchbenchmark/canary_models/fambench_xlmr/install.py @@ -2,7 +2,7 @@ import sys import subprocess from torchbenchmark import REPO_PATH - +from utils.python_utils import pip_install_requirements def update_fambench_submodule(): "Update FAMBench submodule of the benchmark repo" @@ -19,9 +19,7 @@ def update_fambench_submodule(): def pip_install_requirements(): try: - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "-q", "-r", "requirements.txt"] - ) + pip_install_requirements() # pin fairseq version # ignore deps specified in requirements.txt subprocess.check_call( diff --git a/torchbenchmark/canary_models/gat/install.py b/torchbenchmark/canary_models/gat/install.py index cdcba10a41..e67c0c5317 100644 --- a/torchbenchmark/canary_models/gat/install.py +++ b/torchbenchmark/canary_models/gat/install.py @@ -1,13 +1,6 @@ - -import subprocess -import sys from utils import s3_utils - - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt', '-f', 'https://data.pyg.org/whl/torch-2.1.0+cpu.html']) - +from utils.python_utils import pip_install_requirements if __name__ == '__main__': s3_utils.checkout_s3_data("INPUT_TARBALLS", "Reddit_minimal.tar.gz", decompress=True) - pip_install_requirements() + pip_install_requirements(extra_args=["-f", "https://data.pyg.org/whl/torch-2.1.0+cpu.html"]) diff --git a/torchbenchmark/canary_models/hf_MPT_7b_instruct/install.py b/torchbenchmark/canary_models/hf_MPT_7b_instruct/install.py index 1a49905932..123372a8de 100644 --- a/torchbenchmark/canary_models/hf_MPT_7b_instruct/install.py +++ b/torchbenchmark/canary_models/hf_MPT_7b_instruct/install.py @@ -1,10 +1,6 @@ -import subprocess -import sys import os from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/canary_models/hf_Yi/install.py b/torchbenchmark/canary_models/hf_Yi/install.py index 1a49905932..aab10dfd24 100644 --- a/torchbenchmark/canary_models/hf_Yi/install.py +++ b/torchbenchmark/canary_models/hf_Yi/install.py @@ -2,9 +2,7 @@ import sys import os from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/canary_models/hf_mixtral/install.py b/torchbenchmark/canary_models/hf_mixtral/install.py index 1a49905932..b3013e1fa7 100644 --- a/torchbenchmark/canary_models/hf_mixtral/install.py +++ b/torchbenchmark/canary_models/hf_mixtral/install.py @@ -1,10 +1,7 @@ -import subprocess -import sys import os from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model +from utils.python_utils import pip_install_requirements -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/canary_models/phi_1_5/install.py b/torchbenchmark/canary_models/phi_1_5/install.py index 1a49905932..aab10dfd24 100644 --- a/torchbenchmark/canary_models/phi_1_5/install.py +++ b/torchbenchmark/canary_models/phi_1_5/install.py @@ -2,9 +2,7 @@ import sys import os from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/canary_models/phi_2/install.py b/torchbenchmark/canary_models/phi_2/install.py index 1a49905932..123372a8de 100644 --- a/torchbenchmark/canary_models/phi_2/install.py +++ b/torchbenchmark/canary_models/phi_2/install.py @@ -1,10 +1,6 @@ -import subprocess -import sys import os from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/canary_models/sage/install.py b/torchbenchmark/canary_models/sage/install.py index cdcba10a41..4a57983e56 100644 --- a/torchbenchmark/canary_models/sage/install.py +++ b/torchbenchmark/canary_models/sage/install.py @@ -1,12 +1,5 @@ - -import subprocess -import sys from utils import s3_utils - - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt', '-f', 'https://data.pyg.org/whl/torch-2.1.0+cpu.html']) - +from utils.python_utils import pip_install_requirements if __name__ == '__main__': s3_utils.checkout_s3_data("INPUT_TARBALLS", "Reddit_minimal.tar.gz", decompress=True) diff --git a/torchbenchmark/canary_models/torchrec_dlrm/install.py b/torchbenchmark/canary_models/torchrec_dlrm/install.py index e92d8783ca..df4fdfae39 100644 --- a/torchbenchmark/canary_models/torchrec_dlrm/install.py +++ b/torchbenchmark/canary_models/torchrec_dlrm/install.py @@ -1,12 +1,4 @@ -import subprocess -import sys - - -def pip_install_requirements(): - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "-q", "-r", "requirements.txt"] - ) - +from utils.python_utils import pip_install_requirements if __name__ == "__main__": pip_install_requirements() diff --git a/torchbenchmark/models/maml_omniglot/install.py b/torchbenchmark/models/maml_omniglot/install.py index d3e20cdb9b..117c8cf1dc 100644 --- a/torchbenchmark/models/maml_omniglot/install.py +++ b/torchbenchmark/models/maml_omniglot/install.py @@ -1,10 +1,5 @@ -import subprocess -import sys from utils import s3_utils - - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/moondream/install.py b/torchbenchmark/models/moondream/install.py index 1a49905932..b3013e1fa7 100644 --- a/torchbenchmark/models/moondream/install.py +++ b/torchbenchmark/models/moondream/install.py @@ -1,10 +1,7 @@ -import subprocess -import sys import os from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model +from utils.python_utils import pip_install_requirements -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/nvidia_deeprecommender/install.py b/torchbenchmark/models/nvidia_deeprecommender/install.py index 76fc6193d5..6b1178892f 100644 --- a/torchbenchmark/models/nvidia_deeprecommender/install.py +++ b/torchbenchmark/models/nvidia_deeprecommender/install.py @@ -1,9 +1,6 @@ import subprocess import sys - - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/opacus_cifar10/install.py b/torchbenchmark/models/opacus_cifar10/install.py index e89de1f48e..44bd90cc2c 100644 --- a/torchbenchmark/models/opacus_cifar10/install.py +++ b/torchbenchmark/models/opacus_cifar10/install.py @@ -1,8 +1,4 @@ -import subprocess -import sys - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/install.py b/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/install.py index 1ff77b2a20..0c2da1f20f 100644 --- a/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/install.py +++ b/torchbenchmark/models/pytorch_CycleGAN_and_pix2pix/install.py @@ -1,10 +1,5 @@ -import subprocess -import sys from utils import s3_utils - - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': s3_utils.checkout_s3_data("INPUT_TARBALLS", "pytorch_CycleGAN_and_pix2pix_inputs.tar.gz", decompress=True) diff --git a/torchbenchmark/models/pytorch_stargan/install.py b/torchbenchmark/models/pytorch_stargan/install.py index 1db8bdc758..9b7af53603 100644 --- a/torchbenchmark/models/pytorch_stargan/install.py +++ b/torchbenchmark/models/pytorch_stargan/install.py @@ -1,9 +1,5 @@ -import subprocess -import sys from utils import s3_utils - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': s3_utils.checkout_s3_data("INPUT_TARBALLS", "pytorch_stargan_inputs.tar.gz", decompress=True) diff --git a/torchbenchmark/models/pytorch_unet/install.py b/torchbenchmark/models/pytorch_unet/install.py index 65ac1173a6..44bd90cc2c 100644 --- a/torchbenchmark/models/pytorch_unet/install.py +++ b/torchbenchmark/models/pytorch_unet/install.py @@ -1,9 +1,4 @@ -import subprocess -import sys - - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'pytorch_unet/requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': pip_install_requirements() diff --git a/torchbenchmark/models/sam/install.py b/torchbenchmark/models/sam/install.py index 7c94fa7bae..96f6d890f4 100644 --- a/torchbenchmark/models/sam/install.py +++ b/torchbenchmark/models/sam/install.py @@ -2,6 +2,7 @@ import subprocess import sys import requests +from utils.python_utils import pip_install_requirements def download(uri): directory = '.data' @@ -16,10 +17,6 @@ def download(uri): else: print(f'Failed to download file with status code {response.status_code}') - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) - def download_checkpoint(): download('https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth') diff --git a/torchbenchmark/models/sam_fast/install.py b/torchbenchmark/models/sam_fast/install.py index 7c94fa7bae..c12254ce53 100644 --- a/torchbenchmark/models/sam_fast/install.py +++ b/torchbenchmark/models/sam_fast/install.py @@ -1,7 +1,6 @@ import os -import subprocess -import sys import requests +from utils.python_utils import pip_install_requirements def download(uri): directory = '.data' @@ -16,10 +15,6 @@ def download(uri): else: print(f'Failed to download file with status code {response.status_code}') - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) - def download_checkpoint(): download('https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth') diff --git a/torchbenchmark/models/soft_actor_critic/install.py b/torchbenchmark/models/soft_actor_critic/install.py index e92d8783ca..df4fdfae39 100644 --- a/torchbenchmark/models/soft_actor_critic/install.py +++ b/torchbenchmark/models/soft_actor_critic/install.py @@ -1,12 +1,4 @@ -import subprocess -import sys - - -def pip_install_requirements(): - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "-q", "-r", "requirements.txt"] - ) - +from utils.python_utils import pip_install_requirements if __name__ == "__main__": pip_install_requirements() diff --git a/torchbenchmark/models/speech_transformer/install.py b/torchbenchmark/models/speech_transformer/install.py index 2627e23a15..c8352f35d6 100644 --- a/torchbenchmark/models/speech_transformer/install.py +++ b/torchbenchmark/models/speech_transformer/install.py @@ -1,9 +1,5 @@ -import sys -import subprocess from utils import s3_utils - -def pip_install_requirements(): - subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt']) +from utils.python_utils import pip_install_requirements if __name__ == '__main__': s3_utils.checkout_s3_data("INPUT_TARBALLS", "speech_transformer_inputs.tar.gz", decompress=True) diff --git a/torchbenchmark/util/framework/lit_llama.py b/torchbenchmark/util/framework/lit_llama.py index d3f9e9b82a..d8c2e2dd92 100644 --- a/torchbenchmark/util/framework/lit_llama.py +++ b/torchbenchmark/util/framework/lit_llama.py @@ -5,6 +5,7 @@ from pathlib import Path from torchbenchmark import REPO_PATH +from utils.python_utils import pip_install_requirements LIT_LLAMA_PATH = os.path.join(REPO_PATH, "submodules", "lit-llama") @@ -21,20 +22,6 @@ def update_lit_llama_submodule(): subprocess.check_call(update_command, cwd=REPO_PATH) -def pip_install_requirements(): - subprocess.check_call( - [ - sys.executable, - "-m", - "pip", - "install", - "-q", - "-r", - os.path.join(LIT_LLAMA_PATH, "requirements.txt"), - ] - ) - - def openllama_download(): if os.path.exists( os.path.join(LIT_LLAMA_PATH, "checkpoints/lit-llama/7B/lit-llama.pth") diff --git a/utils/python_utils.py b/utils/python_utils.py index dcb33b4aed..470b41a511 100644 --- a/utils/python_utils.py +++ b/utils/python_utils.py @@ -2,6 +2,8 @@ from pathlib import Path import subprocess +from typing import Optional, List + DEFAULT_PYTHON_VERSION = "3.11" PYTHON_VERSION_MAP = { @@ -23,7 +25,10 @@ def create_conda_env(pyver: str, name: str): subprocess.check_call(command) -def pip_install_requirements(requirements_txt="requirements.txt", continue_on_fail=False, no_build_isolation=False): +def pip_install_requirements(requirements_txt="requirements.txt", + continue_on_fail=False, + no_build_isolation=False, + extra_args: Optional[List[str]]=None): import sys constraints_file = REPO_DIR.joinpath("build", "constraints.txt") if not constraints_file.exists(): @@ -33,6 +38,8 @@ def pip_install_requirements(requirements_txt="requirements.txt", continue_on_fa constraints_parameters = ["-c", str(constraints_file.resolve())] if no_build_isolation: constraints_parameters.append("--no-build-isolation") + if extra_args and isinstance(extra_args, list): + constraints_parameters.extend(extra_args) if not continue_on_fail: subprocess.check_call( [sys.executable, "-m", "pip", "install", "-r", requirements_txt] + constraints_parameters,