From c09f36cba4645e24f8dd4f979c3d9239fda7773d Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Thu, 15 Aug 2024 22:43:14 -0400 Subject: [PATCH 1/3] [Build] Implement install_requirements.sh with Python Move the details of install_requirements.sh into install_requirements.py. Then, call install_requirements.py from install_requirements.sh. This will avoid duplication when add Windows version of installing_requirements. This is for issue #4661. --- install_requirements.py | 153 +++++++++++++++++++++++++++++++++++++++ install_requirements.sh | 154 +--------------------------------------- 2 files changed, 154 insertions(+), 153 deletions(-) create mode 100644 install_requirements.py diff --git a/install_requirements.py b/install_requirements.py new file mode 100644 index 0000000000..d11b3dcf87 --- /dev/null +++ b/install_requirements.py @@ -0,0 +1,153 @@ +import os +import platform +import re +import sys +import subprocess + +# Before doing anything, cd to the directory containing this script. +os.chdir(os.path.dirname(os.path.abspath(__file__))) + +# Find the names of the python tools to use. +PYTHON_EXECUTABLE = os.getenv('PYTHON_EXECUTABLE') + +if not PYTHON_EXECUTABLE: + CONDA_DEFAULT_ENV = os.getenv('CONDA_DEFAULT_ENV') + if not CONDA_DEFAULT_ENV or CONDA_DEFAULT_ENV == "base" or not subprocess.call(["which", "python"]): + PYTHON_EXECUTABLE = "python3" + else: + PYTHON_EXECUTABLE = "python" + +if PYTHON_EXECUTABLE == "python": + PIP_EXECUTABLE = "pip" +else: + PIP_EXECUTABLE = "pip3" + +print(f"Using Python executable: {PYTHON_EXECUTABLE}") +print(f"Using Pip executable: {PIP_EXECUTABLE}") + +def python_is_compatible(): + # Scrape the version range from pyproject.toml, which should be in the current directory. + version_specifier = None + with open('pyproject.toml', 'r') as file: + for line in file: + if line.startswith('requires-python'): + match = re.search(r'"([^"]*)"', line) + if match: + version_specifier = match.group(1) + break + + if not version_specifier: + print("WARNING: Skipping python version check: version range not found", file=sys.stderr) + return False + + # Install the packaging module if necessary. + try: + import packaging + except ImportError: + subprocess.check_call([PIP_EXECUTABLE, 'install', 'packaging']) + # Compare the current python version to the range in version_specifier. Exits + # with status 1 if the version is not compatible, or with status 0 if the + # version is compatible or the logic itself fails. + try: + import packaging.version + import packaging.specifiers + + python_version = packaging.version.parse(platform.python_version()) + version_range = packaging.specifiers.SpecifierSet(version_specifier) + if python_version not in version_range: + print( + f"ERROR: ExecuTorch does not support python version {python_version}: must satisfy \"{version_specifier}\"", + file=sys.stderr, + ) + sys.exit(1) + except Exception as e: + print(f"WARNING: Skipping python version check: {e}", file=sys.stderr) + sys.exit(0) + return False + return True + +if not python_is_compatible(): + sys.exit(1) + +# Parse options. +EXECUTORCH_BUILD_PYBIND = "OFF" +CMAKE_ARGS = os.getenv("CMAKE_ARGS", "") +CMAKE_BUILD_ARGS = os.getenv("CMAKE_BUILD_ARGS", "") + +for arg in sys.argv[1:]: + if arg == "--pybind": + EXECUTORCH_BUILD_PYBIND = "ON" + elif arg in ["coreml", "mps", "xnnpack"]: + if EXECUTORCH_BUILD_PYBIND == "ON": + arg_upper = arg.upper() + CMAKE_ARGS += f" -DEXECUTORCH_BUILD_{arg_upper}=ON" + else: + print(f"Error: {arg} must follow --pybind") + sys.exit(1) + else: + print(f"Error: Unknown option {arg}") + sys.exit(1) + +print(f"EXECUTORCH_BUILD_PYBIND: {EXECUTORCH_BUILD_PYBIND}") +print(f"CMAKE_ARGS: {CMAKE_ARGS}") +print(f"CMAKE_BUILD_ARGS: {CMAKE_BUILD_ARGS}") + +NIGHTLY_VERSION = "dev20240716" + +# The pip repository that hosts nightly torch packages. +TORCH_NIGHTLY_URL = "https://download.pytorch.org/whl/nightly/cpu" + +# pip packages needed by exir. +EXIR_REQUIREMENTS = [ + f"torch==2.5.0.{NIGHTLY_VERSION}", + f"torchvision==0.20.0.{NIGHTLY_VERSION}" # For testing. +] + +# pip packages needed for development. +DEVEL_REQUIREMENTS = [ + "cmake", # For building binary targets. + "pip>=23", # For building the pip package. + "pyyaml", # Imported by the kernel codegen tools. + "setuptools>=63", # For building the pip package. + "tomli", # Imported by extract_sources.py when using python < 3.11. + "wheel", # For building the pip package archive. + "zstd" # Imported by resolve_buck.py. +] + +# pip packages needed to run examples. +# TODO: Make each example publish its own requirements.txt +EXAMPLES_REQUIREMENTS = [ + "timm==1.0.7", + f"torchaudio==2.4.0.{NIGHTLY_VERSION}", + "torchsr==1.0.4", + "transformers==4.42.4" +] + +# Assemble the list of requirements to actually install. +# TODO: Add options for reducing the number of requirements. +REQUIREMENTS_TO_INSTALL = EXIR_REQUIREMENTS + DEVEL_REQUIREMENTS + EXAMPLES_REQUIREMENTS + +print("Requirements to install:") +for requirement in REQUIREMENTS_TO_INSTALL: + print(requirement) + +# Install the requirements. `--extra-index-url` tells pip to look for package +# versions on the provided URL if they aren't available on the default URL. +subprocess.check_call([PIP_EXECUTABLE, "install", *REQUIREMENTS_TO_INSTALL, "--extra-index-url", TORCH_NIGHTLY_URL]) + +# +# Install executorch pip package. This also makes `flatc` available on the path. +# The --extra-index-url may be necessary if pyproject.toml has a dependency on a +# pre-release or nightly version of a torch package. +# + +# Set environment variables +os.environ["EXECUTORCH_BUILD_PYBIND"] = EXECUTORCH_BUILD_PYBIND +os.environ["CMAKE_ARGS"] = CMAKE_ARGS +os.environ["CMAKE_BUILD_ARGS"] = CMAKE_BUILD_ARGS + +# Run the pip install command +subprocess.check_call([ + PIP_EXECUTABLE, "install", ".", "--no-build-isolation", "-v", + "--extra-index-url", TORCH_NIGHTLY_URL +]) diff --git a/install_requirements.sh b/install_requirements.sh index 97f85a3ee5..81f7c08044 100755 --- a/install_requirements.sh +++ b/install_requirements.sh @@ -19,156 +19,4 @@ then fi fi -if [[ "$PYTHON_EXECUTABLE" == "python" ]]; -then - PIP_EXECUTABLE=pip -else - PIP_EXECUTABLE=pip3 -fi - -# Returns 0 if the current python version is compatible with the version range -# in pyprojects.toml, or returns 1 if it is not compatible. If the check logic -# itself fails, prints a warning and returns 0. -python_is_compatible() { - # Scrape the version range from pyproject.toml, which should be - # in the current directory. - local version_specifier - version_specifier="$( - grep "^requires-python" pyproject.toml \ - | head -1 \ - | sed -e 's/[^"]*"//;s/".*//' - )" - if [[ -z ${version_specifier} ]]; then - echo "WARNING: Skipping python version check: version range not found" >& 2 - return 0 - fi - - # Install the packaging module if necessary. - if ! python -c 'import packaging' 2> /dev/null ; then - ${PIP_EXECUTABLE} install packaging - fi - - # Compare the current python version to the range in version_specifier. Exits - # with status 1 if the version is not compatible, or with status 0 if the - # version is compatible or the logic itself fails. -${PYTHON_EXECUTABLE} <=23" # For building the pip package. - pyyaml # Imported by the kernel codegen tools. - "setuptools>=63" # For building the pip package. - tomli # Imported by extract_sources.py when using python < 3.11. - wheel # For building the pip package archive. - zstd # Imported by resolve_buck.py. -) - -# pip packages needed to run examples. -# TODO(dbort): Make each example publish its own requirements.txt -EXAMPLES_REQUIREMENTS=( - timm==1.0.7 - torchaudio=="2.4.0.${NIGHTLY_VERSION}" - torchsr==1.0.4 - transformers==4.42.4 -) - -# Assemble the list of requirements to actually install. -# TODO(dbort): Add options for reducing the number of requirements. -REQUIREMENTS_TO_INSTALL=( - "${EXIR_REQUIREMENTS[@]}" - "${DEVEL_REQUIREMENTS[@]}" - "${EXAMPLES_REQUIREMENTS[@]}" -) - -# Install the requirements. `--extra-index-url` tells pip to look for package -# versions on the provided URL if they aren't available on the default URL. -$PIP_EXECUTABLE install --extra-index-url "${TORCH_NIGHTLY_URL}" \ - "${REQUIREMENTS_TO_INSTALL[@]}" - -# -# Install executorch pip package. This also makes `flatc` available on the path. -# The --extra-index-url may be necessary if pyproject.toml has a dependency on a -# pre-release or nightly version of a torch package. -# - -EXECUTORCH_BUILD_PYBIND="${EXECUTORCH_BUILD_PYBIND}" \ - CMAKE_ARGS="${CMAKE_ARGS}" \ - CMAKE_BUILD_ARGS="${CMAKE_BUILD_ARGS}" \ - $PIP_EXECUTABLE install . --no-build-isolation -v \ - --extra-index-url "${TORCH_URL}" +$PYTHON_EXECUTABLE ./install_requirements.py "$@" From 008b808d0b60783d760dc0e211c15eaf7873c419 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Fri, 16 Aug 2024 20:34:51 -0400 Subject: [PATCH 2/3] Update per comment. --- install_requirements.py | 100 +++++++++++++++++++++++----------------- install_requirements.sh | 3 ++ 2 files changed, 60 insertions(+), 43 deletions(-) diff --git a/install_requirements.py b/install_requirements.py index d11b3dcf87..c3d4ff3220 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -1,71 +1,66 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + import os import platform import re -import sys import subprocess +import sys # Before doing anything, cd to the directory containing this script. os.chdir(os.path.dirname(os.path.abspath(__file__))) -# Find the names of the python tools to use. -PYTHON_EXECUTABLE = os.getenv('PYTHON_EXECUTABLE') - -if not PYTHON_EXECUTABLE: - CONDA_DEFAULT_ENV = os.getenv('CONDA_DEFAULT_ENV') - if not CONDA_DEFAULT_ENV or CONDA_DEFAULT_ENV == "base" or not subprocess.call(["which", "python"]): - PYTHON_EXECUTABLE = "python3" - else: - PYTHON_EXECUTABLE = "python" - -if PYTHON_EXECUTABLE == "python": - PIP_EXECUTABLE = "pip" -else: - PIP_EXECUTABLE = "pip3" - -print(f"Using Python executable: {PYTHON_EXECUTABLE}") -print(f"Using Pip executable: {PIP_EXECUTABLE}") def python_is_compatible(): # Scrape the version range from pyproject.toml, which should be in the current directory. version_specifier = None - with open('pyproject.toml', 'r') as file: + with open("pyproject.toml", "r") as file: for line in file: - if line.startswith('requires-python'): + if line.startswith("requires-python"): match = re.search(r'"([^"]*)"', line) if match: version_specifier = match.group(1) break if not version_specifier: - print("WARNING: Skipping python version check: version range not found", file=sys.stderr) + print( + "WARNING: Skipping python version check: version range not found", + file=sys.stderr, + ) return False # Install the packaging module if necessary. try: import packaging except ImportError: - subprocess.check_call([PIP_EXECUTABLE, 'install', 'packaging']) + subprocess.run( + [sys.executable, "-m", "pip", "install", "packaging"], check=True + ) # Compare the current python version to the range in version_specifier. Exits # with status 1 if the version is not compatible, or with status 0 if the # version is compatible or the logic itself fails. try: - import packaging.version import packaging.specifiers + import packaging.version python_version = packaging.version.parse(platform.python_version()) version_range = packaging.specifiers.SpecifierSet(version_specifier) if python_version not in version_range: print( - f"ERROR: ExecuTorch does not support python version {python_version}: must satisfy \"{version_specifier}\"", + f'ERROR: ExecuTorch does not support python version {python_version}: must satisfy "{version_specifier}"', file=sys.stderr, ) - sys.exit(1) + return False except Exception as e: print(f"WARNING: Skipping python version check: {e}", file=sys.stderr) - sys.exit(0) - return False return True + if not python_is_compatible(): sys.exit(1) @@ -88,10 +83,12 @@ def python_is_compatible(): print(f"Error: Unknown option {arg}") sys.exit(1) -print(f"EXECUTORCH_BUILD_PYBIND: {EXECUTORCH_BUILD_PYBIND}") -print(f"CMAKE_ARGS: {CMAKE_ARGS}") -print(f"CMAKE_BUILD_ARGS: {CMAKE_BUILD_ARGS}") - +# Since ExecuTorch often uses main-branch features of pytorch, only the nightly +# pip versions will have the required features. +# +# NOTE: If a newly-fetched version of the executorch repo changes the value of +# NIGHTLY_VERSION, you should re-run this script to install the necessary +# package versions. NIGHTLY_VERSION = "dev20240716" # The pip repository that hosts nightly torch packages. @@ -100,7 +97,7 @@ def python_is_compatible(): # pip packages needed by exir. EXIR_REQUIREMENTS = [ f"torch==2.5.0.{NIGHTLY_VERSION}", - f"torchvision==0.20.0.{NIGHTLY_VERSION}" # For testing. + f"torchvision==0.20.0.{NIGHTLY_VERSION}", # For testing. ] # pip packages needed for development. @@ -111,7 +108,7 @@ def python_is_compatible(): "setuptools>=63", # For building the pip package. "tomli", # Imported by extract_sources.py when using python < 3.11. "wheel", # For building the pip package archive. - "zstd" # Imported by resolve_buck.py. + "zstd", # Imported by resolve_buck.py. ] # pip packages needed to run examples. @@ -120,20 +117,27 @@ def python_is_compatible(): "timm==1.0.7", f"torchaudio==2.4.0.{NIGHTLY_VERSION}", "torchsr==1.0.4", - "transformers==4.42.4" + "transformers==4.42.4", ] # Assemble the list of requirements to actually install. # TODO: Add options for reducing the number of requirements. REQUIREMENTS_TO_INSTALL = EXIR_REQUIREMENTS + DEVEL_REQUIREMENTS + EXAMPLES_REQUIREMENTS -print("Requirements to install:") -for requirement in REQUIREMENTS_TO_INSTALL: - print(requirement) - # Install the requirements. `--extra-index-url` tells pip to look for package # versions on the provided URL if they aren't available on the default URL. -subprocess.check_call([PIP_EXECUTABLE, "install", *REQUIREMENTS_TO_INSTALL, "--extra-index-url", TORCH_NIGHTLY_URL]) +subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + *REQUIREMENTS_TO_INSTALL, + "--extra-index-url", + TORCH_NIGHTLY_URL, + ], + check=True, +) # # Install executorch pip package. This also makes `flatc` available on the path. @@ -147,7 +151,17 @@ def python_is_compatible(): os.environ["CMAKE_BUILD_ARGS"] = CMAKE_BUILD_ARGS # Run the pip install command -subprocess.check_call([ - PIP_EXECUTABLE, "install", ".", "--no-build-isolation", "-v", - "--extra-index-url", TORCH_NIGHTLY_URL -]) +subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + ".", + "--no-build-isolation", + "-v", + "--extra-index-url", + TORCH_NIGHTLY_URL, + ], + check=True, +) diff --git a/install_requirements.sh b/install_requirements.sh index 81f7c08044..6caaa880e6 100755 --- a/install_requirements.sh +++ b/install_requirements.sh @@ -20,3 +20,6 @@ then fi $PYTHON_EXECUTABLE ./install_requirements.py "$@" + +# Exit with the same status as the python script. +exit $? From dd5b83e7040731381a17e895f8abbb0cb3e93f39 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Mon, 19 Aug 2024 20:03:21 -0400 Subject: [PATCH 3/3] Keep sync after rebase. --- install_requirements.py | 1 + 1 file changed, 1 insertion(+) diff --git a/install_requirements.py b/install_requirements.py index c3d4ff3220..5c9ecc182f 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -98,6 +98,7 @@ def python_is_compatible(): EXIR_REQUIREMENTS = [ f"torch==2.5.0.{NIGHTLY_VERSION}", f"torchvision==0.20.0.{NIGHTLY_VERSION}", # For testing. + "typing-extensions", ] # pip packages needed for development.