Move benchmark apps to extension/benchmark dir

.github/workflows/android-perf.yml

@@ -292,7 +292,7 @@ jobs:
               --output-dir benchmark-results \
               --repo ${{ github.repository }} \
               --head-branch ${{ github.head_ref || github.ref_name }} \
-              --workflow-name ${{ github.workflow }} \
+              --workflow-name "${{ github.workflow }}" \
               --workflow-run-id ${{ github.run_id }} \
               --workflow-run-attempt ${{ github.run_attempt }}
           done

.github/workflows/android.yml

@@ -15,6 +15,7 @@ on:
       - install_requirements.sh
       - examples/demo-apps/android/**
       - extension/android/**
+      - extension/benchmark/android/**
       - extension/module/**
   workflow_dispatch:
 

.github/workflows/apple-perf.yml

@@ -235,17 +235,17 @@ jobs:
         PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
           build/build_apple_frameworks.sh --coreml --custom --mps --optimized --portable --quantized --xnnpack
 
-        mkdir -p extension/apple/Benchmark/Frameworks
+        mkdir -p extension/benchmark/apple/Benchmark/Frameworks
         for FRAMEWORK in "${FRAMEWORKS[@]}"; do (
-          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/apple/Benchmark/Frameworks/
+          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/benchmark/apple/Benchmark/Frameworks/
         ) done
         echo "::endgroup::"
 
         # NB: Although exported models can be copied to this directory and bundled together with the
         # app, we don't use this in CI and rely on AWS extra data parameter to make the model and the
         # tokenizer available to the benchmark. This decouples the app and the model. We just need to
         # create the directory here to pass the build
-        mkdir -p extension/apple/Benchmark/Models
+        mkdir -p extension/benchmark/apple/Benchmark/Models
         ${CONDA_RUN} --no-capture-output \
           build/build_apple_llm_demo.sh ${ARTIFACTS_DIR_NAME}
 

.github/workflows/apple.yml

@@ -18,6 +18,7 @@ on:
       - build/test_ios_ci.sh
       - examples/demo-apps/apple_ios/**
       - extension/apple/**
+      - extension/benchmark/apple/**
       - extension/module/**
   workflow_dispatch:
 
@@ -272,14 +273,14 @@ jobs:
         PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
           build/build_apple_frameworks.sh --coreml --custom --mps --optimized --portable --quantized --xnnpack
 
-        mkdir -p extension/apple/Benchmark/Frameworks
+        mkdir -p extension/benchmark/apple/Benchmark/Frameworks
         for FRAMEWORK in "${FRAMEWORKS[@]}"; do (
-          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/apple/Benchmark/Frameworks/
+          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/benchmark/apple/Benchmark/Frameworks/
         ) done
         echo "::endgroup::"
 
         echo "::group::Build ExecuTorch benchmark app"
-        mkdir -p extension/apple/Benchmark/Models
+        mkdir -p extension/benchmark/apple/Benchmark/Models
         ${CONDA_RUN} --no-capture-output \
           build/build_apple_llm_demo.sh ${ARTIFACTS_DIR_NAME}
         echo "::endgroup::"

.github/workflows/lint.yml

@@ -66,7 +66,7 @@ jobs:
         FILES_NEEDS_FORMAT=$(/opt/google-java-format -n extension/android/src/main/java/org/pytorch/executorch/*.java \
           examples/demo-apps/android/ExecuTorchDemo/app/src/main/java/com/example/executorchdemo/*.java \
           examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/*.java \
-          extension/android/benchmark/app/src/main/java/org/pytorch/minibench/*.java)
+          extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/*.java)
         if [ -n "$FILES_NEEDS_FORMAT" ]; then
           echo "Warning: The following files need formatting. Please use google-java-format."
           echo "Use a binary from https://github.com/google/google-java-format/releases/"

.github/workflows/upload-android-test-specs.yml

@@ -4,13 +4,13 @@ on:
   pull_request:
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - extension/android/benchmark/android-llm-device-farm-test-spec.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
   push:
     branches:
       - main
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - extension/android/benchmark/android-llm-device-farm-test-spec.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
 
 concurrency:
   # NB: This concurency group needs to be different than the one used in android-perf, otherwise
@@ -32,7 +32,7 @@ jobs:
             ${{ github.repository }}/${{ github.run_id }}/artifacts
           retention-days: 1
           if-no-files-found: error
-          path: extension/android/benchmark/android-llm-device-farm-test-spec.yml
+          path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
 
   validate-android-test-spec:
     needs: upload-android-test-spec-for-validation
@@ -77,7 +77,7 @@ jobs:
 
       - name: Upload the spec to S3 ossci-android bucket
         shell: bash
-        working-directory: extension/android/benchmark/
+        working-directory: extension/benchmark/android/benchmark/
         env:
           SPEC_FILE: android-llm-device-farm-test-spec.yml
         run: |

.github/workflows/upload-apple-test-specs.yml

@@ -4,13 +4,13 @@ on:
   pull_request:
     paths:
       - .github/workflows/upload-apple-test-specs.yml
-      - examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml
+      - extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
   push:
     branches:
       - main
     paths:
       - .github/workflows/upload-apple-test-specs.yml
-      - examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml
+      - extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
 
 concurrency:
   # NB: This concurency group needs to be different than the one used in apple-perf, otherwise
@@ -32,7 +32,7 @@ jobs:
             ${{ github.repository }}/${{ github.run_id }}/artifacts
           retention-days: 1
           if-no-files-found: error
-          path: examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml
+          path: extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
 
   validate-apple-test-spec:
     needs: upload-apple-test-spec-for-validation
@@ -78,7 +78,7 @@ jobs:
 
       - name: Upload the spec to S3 ossci-ios bucket
         shell: bash
-        working-directory: examples/demo-apps/apple_ios
+        working-directory: extension/benchmark/apple/Benchmark/
         env:
           SPEC_FILE: default-ios-device-farm-appium-test-spec.yml
         run: |

build/build_android_llm_demo.sh

@@ -108,9 +108,9 @@ build_android_demo_apps() {
   ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest
   popd
 
-  mkdir -p extension/android/benchmark/app/libs
-  cp ${BUILD_AAR_DIR}/executorch.aar extension/android/benchmark/app/libs
-  pushd extension/android/benchmark
+  mkdir -p extension/benchmark/android/benchmark/app/libs
+  cp ${BUILD_AAR_DIR}/executorch.aar extension/benchmark/android/benchmark/app/libs
+  pushd extension/benchmark/android/benchmark
   ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest
   popd
 }
@@ -135,8 +135,8 @@ collect_artifacts_to_be_uploaded() {
   # Collect MiniBench APK
   MINIBENCH_APP_DIR="${ARTIFACTS_DIR_NAME}/minibench"
   mkdir -p "${MINIBENCH_APP_DIR}"
-  cp extension/android/benchmark/app/build/outputs/apk/debug/*.apk "${MINIBENCH_APP_DIR}"
-  cp extension/android/benchmark/app/build/outputs/apk/androidTest/debug/*.apk "${MINIBENCH_APP_DIR}"
+  cp extension/benchmark/android/benchmark/app/build/outputs/apk/debug/*.apk "${MINIBENCH_APP_DIR}"
+  cp extension/benchmark/android/benchmark/app/build/outputs/apk/androidTest/debug/*.apk "${MINIBENCH_APP_DIR}"
 }
 
 BUILD_AAR_DIR="$(mktemp -d)"

build/build_apple_llm_demo.sh

@@ -8,7 +8,7 @@
 set -euo pipefail
 
 ARTIFACTS_DIR_NAME="$1"
-APP_PATH="extension/apple/Benchmark/Benchmark"
+APP_PATH="extension/benchmark/apple/Benchmark/Benchmark"
 
 xcodebuild build-for-testing \
   -project "${APP_PATH}.xcodeproj" \

extension/benchmark/apple/.clang-format

@@ -0,0 +1,45 @@
+---
+AccessModifierOffset: -1
+AlignEscapedNewlinesLeft: true
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BreakBeforeBinaryOperators: false
+BreakBeforeBraces: Attach
+BreakConstructorInitializersBeforeComma: false
+ColumnLimit:     80
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+ExperimentalAutoDetectBinPacking: true
+IndentCaseLabels: false
+IndentFunctionDeclarationAfterType: false
+IndentWidth:     2
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 10
+PenaltyBreakComment: 60
+PenaltyBreakFirstLessLess: 20
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+SpaceAfterControlStatementKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
+...

extension/apple/Benchmark/README.md → extension/benchmark/apple/Benchmark/README.md

@@ -42,7 +42,7 @@ You have two options:
 Run the provided script to download the prebuilt frameworks:
 
 ```bash
-./extension/apple/Benchmark/Frameworks/download_frameworks.sh
+./extension/benchmark/apple/Benchmark/Frameworks/download_frameworks.sh
 ```
 </details>
 
@@ -56,7 +56,7 @@ Alternatively, you can build the frameworks yourself by following the [guide](ht
 Once the frameworks are downloaded or built, verify that the `Frameworks` directory contains the necessary `.xcframework` files:
 
 ```bash
-ls extension/apple/Benchmark/Frameworks
+ls extension/benchmark/apple/Benchmark/Frameworks
 ```
 
 You should see:
@@ -74,16 +74,16 @@ kernels_quantized.xcframework
 
 ## Adding Models and Resources
 
-Place your exported model files (`.pte`) and any other resources (e.g., `tokenizer.bin`) into the `extension/apple/Benchmark/Resources` directory:
+Place your exported model files (`.pte`) and any other resources (e.g., `tokenizer.bin`) into the `extension/benchmark/apple/Benchmark/Resources` directory:
 
 ```bash
-cp <path/to/my_model.pte> <path/to/llama3.pte> <path/to/tokenizer.bin> extension/apple/Benchmark/Resources
+cp <path/to/my_model.pte> <path/to/llama3.pte> <path/to/tokenizer.bin> extension/benchmark/apple/Benchmark/Resources
 ```
 
 Optionally, check that the files are there:
 
 ```bash
-ls extension/apple/Benchmark/Resources
+ls extension/benchmark/apple/Benchmark/Resources
 ```
 
 For this example you should see:
@@ -103,7 +103,7 @@ The app automatically bundles these resources and makes them available to the te
 Open the Benchmark Xcode project:
 
 ```bash
-open extension/apple/Benchmark/Benchmark.xcodeproj
+open extension/benchmark/apple/Benchmark/Benchmark.xcodeproj
 ```
 
 Select the destination device or simulator and press `Command+U`, or click `Product` > `Test` in the menu to run the test suite.
@@ -333,13 +333,13 @@ You can also run the tests using `xcodebuild`:
 
 ```bash
 # Run on an iOS Simulator
-xcodebuild test -project extension/apple/Benchmark/Benchmark.xcodeproj \
+xcodebuild test -project extension/benchmark/apple/Benchmark/Benchmark.xcodeproj \
 -scheme Benchmark \
 -destination 'platform=iOS Simulator,name=<SimulatorName>' \
 -testPlan Tests
 
 # Run on a physical iOS device
-xcodebuild test -project extension/apple/Benchmark/Benchmark.xcodeproj \
+xcodebuild test -project extension/benchmark/apple/Benchmark/Benchmark.xcodeproj \
 -scheme Benchmark \
 -destination 'platform=iOS,name=<DeviceName>' \
 -testPlan Tests \

extension/apple/Benchmark/Tests/Tests.xcconfig → extension/benchmark/apple/Benchmark/Tests/Tests.xcconfig

@@ -1,5 +1,5 @@
 HEADER_SEARCH_PATHS[sdk=iphonesimulator*] = $(inherited) \
-$(SRCROOT)/../../../.. \
+$(SRCROOT)/../../../../.. \
 $(TEMP_DIR)/cmake/include
 
 LIBRARY_SEARCH_PATHS[sdk=iphonesimulator*] = $(inherited) \
@@ -16,7 +16,7 @@ OTHER_LDFLAGS[sdk=iphonesimulator*] = $(inherited) \
 -force_load $(BUILT_PRODUCTS_DIR)/libkernels_quantized-simulator-release.a
 
 HEADER_SEARCH_PATHS[sdk=iphoneos*] = $(inherited) \
-$(SRCROOT)/../../../.. \
+$(SRCROOT)/../../../../.. \
 $(TEMP_DIR)/cmake/include
 
 LIBRARY_SEARCH_PATHS[sdk=iphoneos*] = $(inherited) \
@@ -33,7 +33,7 @@ OTHER_LDFLAGS[sdk=iphoneos*] = $(inherited) \
 -force_load $(BUILT_PRODUCTS_DIR)/libkernels_quantized-ios-release.a
 
 HEADER_SEARCH_PATHS[sdk=macos*] = $(inherited) \
-$(SRCROOT)/../../../.. \
+$(SRCROOT)/../../../../.. \
 $(TEMP_DIR)/cmake/include
 
 LIBRARY_SEARCH_PATHS[sdk=macos*] = $(inherited) \