forked from pytorch/executorch
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add CI scripts to run supported executorch networks through MPS
- Loading branch information
1 parent
9682172
commit 96adccc
Showing
3 changed files
with
208 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
name: pull_mps | ||
|
||
on: | ||
pull_request: | ||
push: | ||
branches: | ||
- main | ||
workflow_dispatch: | ||
|
||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} | ||
cancel-in-progress: true | ||
|
||
jobs: | ||
gather-models-mps: | ||
runs-on: macos-executorch | ||
outputs: | ||
models: ${{ steps.gather-models-mps.outputs.models }} | ||
steps: | ||
- uses: actions/checkout@v3 | ||
with: | ||
submodules: 'true' | ||
- uses: actions/setup-python@v4 | ||
with: | ||
python-version: '3.10' | ||
cache: pip | ||
- name: Extract the list of models to test | ||
id: gather-models-mps | ||
run: | | ||
set -eux | ||
source .ci/scripts/utils.sh | ||
# This is a simple Python script but as it tries to import executorch.examples.models, | ||
# it requires a whole bunch of ExecuTorch dependencies on the Docker image | ||
install_pip_dependencies | ||
install_executorch | ||
PYTHONPATH="${PWD}" python -m backends.apple.mps.ci.scripts.gather_test_models_mps | ||
test-mps-delegate-macos: | ||
name: test-mps-delegate-macos | ||
runs-on: macos-executorch | ||
needs: gather-models-mps | ||
strategy: | ||
matrix: ${{ fromJSON(needs.gather-models-mps.outputs.models) }} | ||
fail-fast: false | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- name: Run test ${{ matrix.test }} | ||
if: always() | ||
run: | | ||
cd .. | ||
rm -r -f executorch || true | ||
cp -a executorch_apple_internal executorch && cd executorch | ||
WORKSPACE=$(pwd) | ||
pushd "${WORKSPACE}" | ||
MODEL_NAME=${{ matrix.model }} | ||
BUILD_TOOL=${{ matrix.build-tool }} | ||
# Setup MacOS dependencies as there is no Docker support on MacOS atm | ||
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" | ||
PYTHON_EXECUTABLE=python bash backends/apple/mps/install_requirements.sh | ||
# Build and test ExecuTorch | ||
PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" | ||
popd | ||
- name: Cleanup test ${{ matrix.test }} | ||
if: always() | ||
run: | | ||
cd .. && mv executorch executorch_apple_internal && cd executorch_apple_internal |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
# | ||
# Copyright (c) 2023 Apple Inc. All rights reserved. | ||
# Provided subject to the LICENSE file in the top level directory. | ||
# | ||
|
||
import json | ||
import os | ||
from typing import Any | ||
|
||
from examples.models import MODEL_NAME_TO_MODEL | ||
from examples.xnnpack import MODEL_NAME_TO_OPTIONS | ||
|
||
BUILD_TOOLS = { | ||
"cmake": {"macos-14"}, | ||
} | ||
DEFAULT_RUNNERS = { | ||
"macos-14": "macos-executorch", | ||
} | ||
|
||
def parse_args() -> Any: | ||
from argparse import ArgumentParser | ||
|
||
parser = ArgumentParser("Gather all models to test on CI for macOS MPS delegate") | ||
parser.add_argument( | ||
"--target-os", | ||
type=str, | ||
choices=["macos-14"], | ||
default="macos-14", | ||
help="the target OS", | ||
) | ||
return parser.parse_args() | ||
|
||
|
||
def set_output(name: str, val: Any) -> None: | ||
""" | ||
Set the GitHb output so that it can be accessed by other jobs | ||
""" | ||
print(f"Setting {val} to GitHub output") | ||
|
||
if os.getenv("GITHUB_OUTPUT"): | ||
with open(str(os.getenv("GITHUB_OUTPUT")), "a") as env: | ||
print(f"{name}={val}", file=env) | ||
else: | ||
print(f"::set-output name={name}::{val}") | ||
|
||
|
||
def export_models_for_ci() -> None: | ||
""" | ||
This gathers all the example models that we want to test on GitHub OSS CI | ||
""" | ||
args = parse_args() | ||
target_os = args.target_os | ||
|
||
# This is the JSON syntax for configuration matrix used by GitHub | ||
# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs | ||
models = {"include": []} | ||
for name in MODEL_NAME_TO_MODEL.keys(): | ||
delegation_configs = { | ||
name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].delegation, | ||
} | ||
for build_tool in BUILD_TOOLS.keys(): | ||
if target_os not in BUILD_TOOLS[build_tool]: | ||
continue | ||
|
||
record = { | ||
"build-tool": build_tool, | ||
"model": name, | ||
"runner": DEFAULT_RUNNERS.get(target_os), | ||
} | ||
|
||
models["include"].append(record) | ||
|
||
set_output("models", json.dumps(models)) | ||
|
||
|
||
if __name__ == "__main__": | ||
export_models_for_ci() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
#!/bin/bash | ||
# | ||
# Copyright (c) 2023 Apple Inc. All rights reserved. | ||
# Provided subject to the LICENSE file in the top level directory. | ||
# | ||
|
||
set -exu | ||
|
||
# shellcheck source=/dev/null | ||
# source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" | ||
|
||
MODEL_NAME=$1 | ||
if [[ -z "${MODEL_NAME:-}" ]]; then | ||
echo "Missing model name, exiting..." | ||
exit 1 | ||
fi | ||
|
||
BUILD_TOOL=$2 | ||
if [[ -z "${BUILD_TOOL:-}" ]]; then | ||
echo "Missing build tool (require cmake), exiting..." | ||
exit 1 | ||
fi | ||
|
||
which "${PYTHON_EXECUTABLE}" | ||
CMAKE_OUTPUT_DIR=cmake-out | ||
|
||
build_cmake_mps_executor_runner() { | ||
echo "Building mps_executor_runner" | ||
SITE_PACKAGES="$(${PYTHON_EXECUTABLE} -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')" | ||
CMAKE_PREFIX_PATH="${SITE_PACKAGES}/torch" | ||
|
||
(rm -rf ${CMAKE_OUTPUT_DIR} \ | ||
&& mkdir ${CMAKE_OUTPUT_DIR} \ | ||
&& cd ${CMAKE_OUTPUT_DIR} \ | ||
&& cmake -DBUCK2=buck2 \ | ||
-DEXECUTORCH_BUILD_MPS=ON \ | ||
-DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" \ | ||
-DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" ..) | ||
|
||
cmake --build ${CMAKE_OUTPUT_DIR} -j4 | ||
} | ||
|
||
test_model_with_mps() { | ||
if [[ "${MODEL_NAME}" == "llama2" ]]; then | ||
cd examples/third-party/llama | ||
pip install -e . | ||
cd ../../.. | ||
fi | ||
|
||
"${PYTHON_EXECUTABLE}" -m examples.apple.mps.scripts.mps_example --model_name="${MODEL_NAME}" --bundled | ||
|
||
OUTPUT_MODEL_PATH="${MODEL_NAME}_mps_bundled.pte" | ||
|
||
if [[ "${BUILD_TOOL}" == "cmake" ]]; then | ||
if [[ ! -f ${CMAKE_OUTPUT_DIR}/examples/apple/mps/mps_executor_runner ]]; then | ||
build_cmake_mps_executor_runner | ||
fi | ||
./${CMAKE_OUTPUT_DIR}/examples/apple/mps/mps_executor_runner --model_path "${OUTPUT_MODEL_PATH}" --bundled_program | ||
else | ||
echo "Invalid build tool ${BUILD_TOOL}. Only cmake is supported atm" | ||
exit 1 | ||
fi | ||
} | ||
|
||
echo "Testing ${MODEL_NAME} with MPS..." | ||
test_model_with_mps |