Add CI scripts to run supported executorch networks through MPS (#1)

* Add CI scripts to run supported executorch networks through MPS

* Fix CI

* Fix CI #2

.github/workflows/pull_mps.yml

@@ -0,0 +1,60 @@
+name: pull_mps
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
+  cancel-in-progress: true
+
+jobs:
+  gather-models-mps:
+    runs-on: macos-executorch
+    outputs:
+      models: ${{ steps.gather-models-mps.outputs.models }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: pip
+      - name: Extract the list of models to test
+        id: gather-models-mps
+        run: |
+          set -eux
+          source .ci/scripts/utils.sh
+          # This is a simple Python script but as it tries to import executorch.examples.models,
+          # it requires a whole bunch of ExecuTorch dependencies on the Docker image
+          install_pip_dependencies
+          install_executorch
+          PYTHONPATH="${PWD}" python -m backends.apple.mps.ci.scripts.gather_test_models_mps
+  test-mps-delegate-macos:
+    name: test-mps-delegate-macos
+    runs-on: macos-executorch
+    needs: gather-models-mps
+    strategy:
+      matrix: ${{ fromJSON(needs.gather-models-mps.outputs.models) }}
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+      - name: Run test ${{ matrix.test }}
+        if: always()
+        run: |
+          WORKSPACE=$(pwd)
+          pushd "${WORKSPACE}"
+          MODEL_NAME=${{ matrix.model }}
+          BUILD_TOOL=${{ matrix.build-tool }}
+          # Setup MacOS dependencies as there is no Docker support on MacOS atm
+          PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
+          PYTHON_EXECUTABLE=python bash backends/apple/mps/install_requirements.sh
+          # Build and test ExecuTorch
+          PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}"
+          popd

backends/apple/mps/ci/scripts/gather_test_models_mps.py

@@ -0,0 +1,77 @@
+#
+#  Copyright (c) 2023 Apple Inc. All rights reserved.
+#  Provided subject to the LICENSE file in the top level directory.
+#
+
+import json
+import os
+from typing import Any
+
+from examples.models import MODEL_NAME_TO_MODEL
+from examples.xnnpack import MODEL_NAME_TO_OPTIONS
+
+BUILD_TOOLS = {
+    "cmake": {"macos-14"},
+}
+DEFAULT_RUNNERS = {
+    "macos-14": "macos-executorch",
+}
+
+def parse_args() -> Any:
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser("Gather all models to test on CI for macOS MPS delegate")
+    parser.add_argument(
+        "--target-os",
+        type=str,
+        choices=["macos-14"],
+        default="macos-14",
+        help="the target OS",
+    )
+    return parser.parse_args()
+
+
+def set_output(name: str, val: Any) -> None:
+    """
+    Set the GitHb output so that it can be accessed by other jobs
+    """
+    print(f"Setting {val} to GitHub output")
+
+    if os.getenv("GITHUB_OUTPUT"):
+        with open(str(os.getenv("GITHUB_OUTPUT")), "a") as env:
+            print(f"{name}={val}", file=env)
+    else:
+        print(f"::set-output name={name}::{val}")
+
+
+def export_models_for_ci() -> None:
+    """
+    This gathers all the example models that we want to test on GitHub OSS CI
+    """
+    args = parse_args()
+    target_os = args.target_os
+
+    # This is the JSON syntax for configuration matrix used by GitHub
+    # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
+    models = {"include": []}
+    for name in MODEL_NAME_TO_MODEL.keys():
+        delegation_configs = {
+            name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].delegation,
+        }
+        for build_tool in BUILD_TOOLS.keys():
+            if target_os not in BUILD_TOOLS[build_tool]:
+                continue
+
+            record = {
+                "build-tool": build_tool,
+                "model": name,
+                "runner": DEFAULT_RUNNERS.get(target_os),
+            }
+
+            models["include"].append(record)
+
+    set_output("models", json.dumps(models))
+
+
+if __name__ == "__main__":
+    export_models_for_ci()

backends/apple/mps/ci/scripts/test-mps.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+#  Copyright (c) 2023 Apple Inc. All rights reserved.
+#  Provided subject to the LICENSE file in the top level directory.
+#
+
+set -exu
+
+# shellcheck source=/dev/null
+# source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+MODEL_NAME=$1
+if [[ -z "${MODEL_NAME:-}" ]]; then
+  echo "Missing model name, exiting..."
+  exit 1
+fi
+
+BUILD_TOOL=$2
+if [[ -z "${BUILD_TOOL:-}" ]]; then
+  echo "Missing build tool (require cmake), exiting..."
+  exit 1
+fi
+
+which "${PYTHON_EXECUTABLE}"
+CMAKE_OUTPUT_DIR=cmake-out
+
+build_cmake_mps_executor_runner() {
+  echo "Building mps_executor_runner"
+  SITE_PACKAGES="$(${PYTHON_EXECUTABLE} -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
+  CMAKE_PREFIX_PATH="${SITE_PACKAGES}/torch"
+
+  (rm -rf ${CMAKE_OUTPUT_DIR} \
+    && mkdir ${CMAKE_OUTPUT_DIR} \
+    && cd ${CMAKE_OUTPUT_DIR} \
+    && cmake -DBUCK2=buck2 \
+      -DEXECUTORCH_BUILD_MPS=ON \
+      -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" \
+      -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" ..)
+
+  cmake --build ${CMAKE_OUTPUT_DIR} -j4
+}
+
+test_model_with_mps() {
+  if [[ "${MODEL_NAME}" == "llama2" ]]; then
+    cd examples/third-party/llama
+    pip install -e .
+    cd ../../..
+  fi
+
+  "${PYTHON_EXECUTABLE}" -m examples.apple.mps.scripts.mps_example --model_name="${MODEL_NAME}" --bundled
+
+  OUTPUT_MODEL_PATH="${MODEL_NAME}_mps_bundled.pte"
+
+  if [[ "${BUILD_TOOL}" == "cmake" ]]; then
+    if [[ ! -f ${CMAKE_OUTPUT_DIR}/examples/apple/mps/mps_executor_runner ]]; then
+      build_cmake_mps_executor_runner
+    fi
+    ./${CMAKE_OUTPUT_DIR}/examples/apple/mps/mps_executor_runner --model_path "${OUTPUT_MODEL_PATH}" --bundled_program
+  else
+    echo "Invalid build tool ${BUILD_TOOL}. Only cmake is supported atm"
+    exit 1
+  fi
+}
+
+echo "Testing ${MODEL_NAME} with MPS..."
+test_model_with_mps

examples/apple/mps/CMakeLists.txt

@@ -46,7 +46,6 @@ foreach(fbs_file ${_bundled_program_schema__srcs})
   string(REGEX REPLACE "[.]fbs$" "_generated.h" generated "${fbs_file}")
   list(APPEND _bundled_program_schema__outputs
         "${generated}")
-  message("OUTPUTS: " ${_bundled_program_schema__outputs})
 endforeach()
 
 add_custom_command(