Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Build] Implement install_requirements.sh with Python #4748

Merged
merged 3 commits into from
Aug 20, 2024
Merged

[Build] Implement install_requirements.sh with Python #4748

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c09f36c
[Build] Implement install_requirements.sh with Python
python3kgae Aug 16, 2024
008b808
Update per comment.
python3kgae Aug 17, 2024
dd5b83e
Keep sync after rebase.
python3kgae Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 168 additions & 0 deletions install_requirements.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

import os
python3kgae marked this conversation as resolved.
Show resolved Hide resolved
import platform
import re
import subprocess
import sys

# Before doing anything, cd to the directory containing this script.
os.chdir(os.path.dirname(os.path.abspath(__file__)))


def python_is_compatible():
# Scrape the version range from pyproject.toml, which should be in the current directory.
version_specifier = None
with open("pyproject.toml", "r") as file:
for line in file:
if line.startswith("requires-python"):
match = re.search(r'"([^"]*)"', line)
if match:
version_specifier = match.group(1)
break

if not version_specifier:
print(
"WARNING: Skipping python version check: version range not found",
file=sys.stderr,
)
return False

# Install the packaging module if necessary.
try:
import packaging
except ImportError:
subprocess.run(
[sys.executable, "-m", "pip", "install", "packaging"], check=True
)
# Compare the current python version to the range in version_specifier. Exits
# with status 1 if the version is not compatible, or with status 0 if the
# version is compatible or the logic itself fails.
try:
import packaging.specifiers
import packaging.version

python_version = packaging.version.parse(platform.python_version())
version_range = packaging.specifiers.SpecifierSet(version_specifier)
if python_version not in version_range:
print(
f'ERROR: ExecuTorch does not support python version {python_version}: must satisfy "{version_specifier}"',
file=sys.stderr,
)
return False
except Exception as e:
print(f"WARNING: Skipping python version check: {e}", file=sys.stderr)
return True


if not python_is_compatible():
sys.exit(1)

# Parse options.
EXECUTORCH_BUILD_PYBIND = "OFF"
CMAKE_ARGS = os.getenv("CMAKE_ARGS", "")
CMAKE_BUILD_ARGS = os.getenv("CMAKE_BUILD_ARGS", "")

for arg in sys.argv[1:]:
if arg == "--pybind":
EXECUTORCH_BUILD_PYBIND = "ON"
elif arg in ["coreml", "mps", "xnnpack"]:
if EXECUTORCH_BUILD_PYBIND == "ON":
arg_upper = arg.upper()
CMAKE_ARGS += f" -DEXECUTORCH_BUILD_{arg_upper}=ON"
else:
print(f"Error: {arg} must follow --pybind")
sys.exit(1)
else:
print(f"Error: Unknown option {arg}")
sys.exit(1)

# Since ExecuTorch often uses main-branch features of pytorch, only the nightly
# pip versions will have the required features.
#
# NOTE: If a newly-fetched version of the executorch repo changes the value of
# NIGHTLY_VERSION, you should re-run this script to install the necessary
# package versions.
NIGHTLY_VERSION = "dev20240716"
dbort marked this conversation as resolved.
Show resolved Hide resolved
python3kgae marked this conversation as resolved.
Show resolved Hide resolved
dbort marked this conversation as resolved.
Show resolved Hide resolved

# The pip repository that hosts nightly torch packages.
TORCH_NIGHTLY_URL = "https://download.pytorch.org/whl/nightly/cpu"

# pip packages needed by exir.
EXIR_REQUIREMENTS = [
f"torch==2.5.0.{NIGHTLY_VERSION}",
f"torchvision==0.20.0.{NIGHTLY_VERSION}", # For testing.
"typing-extensions",
]

# pip packages needed for development.
DEVEL_REQUIREMENTS = [
"cmake", # For building binary targets.
"pip>=23", # For building the pip package.
"pyyaml", # Imported by the kernel codegen tools.
"setuptools>=63", # For building the pip package.
"tomli", # Imported by extract_sources.py when using python < 3.11.
"wheel", # For building the pip package archive.
"zstd", # Imported by resolve_buck.py.
]

# pip packages needed to run examples.
# TODO: Make each example publish its own requirements.txt
EXAMPLES_REQUIREMENTS = [
"timm==1.0.7",
f"torchaudio==2.4.0.{NIGHTLY_VERSION}",
"torchsr==1.0.4",
"transformers==4.42.4",
]

# Assemble the list of requirements to actually install.
# TODO: Add options for reducing the number of requirements.
REQUIREMENTS_TO_INSTALL = EXIR_REQUIREMENTS + DEVEL_REQUIREMENTS + EXAMPLES_REQUIREMENTS

# Install the requirements. `--extra-index-url` tells pip to look for package
# versions on the provided URL if they aren't available on the default URL.
subprocess.run(
[
sys.executable,
"-m",
"pip",
"install",
*REQUIREMENTS_TO_INSTALL,
"--extra-index-url",
TORCH_NIGHTLY_URL,
],
check=True,
)

#
# Install executorch pip package. This also makes `flatc` available on the path.
# The --extra-index-url may be necessary if pyproject.toml has a dependency on a
# pre-release or nightly version of a torch package.
#

# Set environment variables
os.environ["EXECUTORCH_BUILD_PYBIND"] = EXECUTORCH_BUILD_PYBIND
os.environ["CMAKE_ARGS"] = CMAKE_ARGS
os.environ["CMAKE_BUILD_ARGS"] = CMAKE_BUILD_ARGS

# Run the pip install command
subprocess.run(
[
sys.executable,
"-m",
"pip",
"install",
".",
"--no-build-isolation",
"-v",
"--extra-index-url",
TORCH_NIGHTLY_URL,
],
check=True,
)
155 changes: 3 additions & 152 deletions install_requirements.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,156 +19,7 @@ then
fi
fi

if [[ "$PYTHON_EXECUTABLE" == "python" ]];
then
PIP_EXECUTABLE=pip
else
PIP_EXECUTABLE=pip3
fi

# Returns 0 if the current python version is compatible with the version range
# in pyprojects.toml, or returns 1 if it is not compatible. If the check logic
# itself fails, prints a warning and returns 0.
python_is_compatible() {
# Scrape the version range from pyproject.toml, which should be
# in the current directory.
local version_specifier
version_specifier="$(
grep "^requires-python" pyproject.toml \
| head -1 \
| sed -e 's/[^"]*"//;s/".*//'
)"
if [[ -z ${version_specifier} ]]; then
echo "WARNING: Skipping python version check: version range not found" >& 2
return 0
fi

# Install the packaging module if necessary.
if ! python -c 'import packaging' 2> /dev/null ; then
${PIP_EXECUTABLE} install packaging
fi

# Compare the current python version to the range in version_specifier. Exits
# with status 1 if the version is not compatible, or with status 0 if the
# version is compatible or the logic itself fails.
${PYTHON_EXECUTABLE} <<EOF
import sys
try:
import packaging.version
import packaging.specifiers
import platform

python_version = packaging.version.parse(platform.python_version())
version_range = packaging.specifiers.SpecifierSet("${version_specifier}")
if python_version not in version_range:
print(
"ERROR: ExecuTorch does not support python version "
+ f"{python_version}: must satisfy \"${version_specifier}\"",
file=sys.stderr,
)
sys.exit(1)
except Exception as e:
print(f"WARNING: Skipping python version check: {e}", file=sys.stderr)
sys.exit(0)
EOF

return $?
}

# Fail fast if the wheel build will fail because the current python version
# isn't supported. But don't fail if the check logic itself has problems: the
# wheel build will do a final check before proceeding.
if ! python_is_compatible; then
exit 1
fi

# Parse options.
EXECUTORCH_BUILD_PYBIND=OFF

for arg in "$@"; do
case $arg in
--pybind)
EXECUTORCH_BUILD_PYBIND=ON
;;
coreml|mps|xnnpack)
if [[ "$EXECUTORCH_BUILD_PYBIND" == "ON" ]]; then
arg_upper="$(echo "${arg}" | tr '[:lower:]' '[:upper:]')"
CMAKE_ARGS="$CMAKE_ARGS -DEXECUTORCH_BUILD_${arg_upper}=ON"
else
echo "Error: $arg must follow --pybind"
exit 1
fi
;;
*)
echo "Error: Unknown option $arg"
exit 1
;;
esac
done

#
# Install pip packages used by code in the ExecuTorch repo.
#

# Since ExecuTorch often uses main-branch features of pytorch, only the nightly
# pip versions will have the required features.
#
# NOTE: If a newly-fetched version of the executorch repo changes the value of
# NIGHTLY_VERSION, you should re-run this script to install the necessary
# package versions.
NIGHTLY_VERSION=dev20240716

# The pip repository that hosts nightly torch packages.
TORCH_NIGHTLY_URL="https://download.pytorch.org/whl/nightly/cpu"

# pip packages needed by exir.
EXIR_REQUIREMENTS=(
torch=="2.5.0.${NIGHTLY_VERSION}"
torchvision=="0.20.0.${NIGHTLY_VERSION}" # For testing.
typing-extensions
)

# pip packages needed for development.
DEVEL_REQUIREMENTS=(
cmake # For building binary targets.
"pip>=23" # For building the pip package.
pyyaml # Imported by the kernel codegen tools.
"setuptools>=63" # For building the pip package.
tomli # Imported by extract_sources.py when using python < 3.11.
wheel # For building the pip package archive.
zstd # Imported by resolve_buck.py.
)

# pip packages needed to run examples.
# TODO(dbort): Make each example publish its own requirements.txt
EXAMPLES_REQUIREMENTS=(
timm==1.0.7
torchaudio=="2.4.0.${NIGHTLY_VERSION}"
torchsr==1.0.4
transformers==4.42.4
)

# Assemble the list of requirements to actually install.
# TODO(dbort): Add options for reducing the number of requirements.
REQUIREMENTS_TO_INSTALL=(
"${EXIR_REQUIREMENTS[@]}"
"${DEVEL_REQUIREMENTS[@]}"
"${EXAMPLES_REQUIREMENTS[@]}"
)

# Install the requirements. `--extra-index-url` tells pip to look for package
# versions on the provided URL if they aren't available on the default URL.
$PIP_EXECUTABLE install --extra-index-url "${TORCH_NIGHTLY_URL}" \
"${REQUIREMENTS_TO_INSTALL[@]}"

#
# Install executorch pip package. This also makes `flatc` available on the path.
# The --extra-index-url may be necessary if pyproject.toml has a dependency on a
# pre-release or nightly version of a torch package.
#
$PYTHON_EXECUTABLE ./install_requirements.py "$@"
python3kgae marked this conversation as resolved.
Show resolved Hide resolved

EXECUTORCH_BUILD_PYBIND="${EXECUTORCH_BUILD_PYBIND}" \
CMAKE_ARGS="${CMAKE_ARGS}" \
CMAKE_BUILD_ARGS="${CMAKE_BUILD_ARGS}" \
$PIP_EXECUTABLE install . --no-build-isolation -v \
--extra-index-url "${TORCH_URL}"
# Exit with the same status as the python script.
exit $?
Loading