Move benchmark apps to extension/benchmark dir (#5951)

Summary: Pull Request resolved: #5951 Reviewed By: huydhn Differential Revision: D64012575 Pulled By: guangy10 fbshipit-source-id: 27bb25586af2aafb8df6c0623d502482b4fa238b
pytorch · Oct 8, 2024 · ac2ae07 · ac2ae07
1 parent 986d001
commit ac2ae07
Show file tree

Hide file tree

Showing 50 changed files with 88 additions and 41 deletions.
diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
@@ -292,7 +292,7 @@ jobs:
               --output-dir benchmark-results \
               --repo ${{ github.repository }} \
               --head-branch ${{ github.head_ref || github.ref_name }} \
-              --workflow-name ${{ github.workflow }} \
+              --workflow-name "${{ github.workflow }}" \
               --workflow-run-id ${{ github.run_id }} \
               --workflow-run-attempt ${{ github.run_attempt }}
           done

diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
@@ -15,6 +15,7 @@ on:
       - install_requirements.sh
       - examples/demo-apps/android/**
       - extension/android/**
+      - extension/benchmark/android/**
       - extension/module/**
   workflow_dispatch:
 

diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml
@@ -235,17 +235,17 @@ jobs:
         PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
           build/build_apple_frameworks.sh --coreml --custom --mps --optimized --portable --quantized --xnnpack
 
-        mkdir -p extension/apple/Benchmark/Frameworks
+        mkdir -p extension/benchmark/apple/Benchmark/Frameworks
         for FRAMEWORK in "${FRAMEWORKS[@]}"; do (
-          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/apple/Benchmark/Frameworks/
+          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/benchmark/apple/Benchmark/Frameworks/
         ) done
         echo "::endgroup::"
 
         # NB: Although exported models can be copied to this directory and bundled together with the
         # app, we don't use this in CI and rely on AWS extra data parameter to make the model and the
         # tokenizer available to the benchmark. This decouples the app and the model. We just need to
         # create the directory here to pass the build
-        mkdir -p extension/apple/Benchmark/Models
+        mkdir -p extension/benchmark/apple/Benchmark/Models
         ${CONDA_RUN} --no-capture-output \
           build/build_apple_llm_demo.sh ${ARTIFACTS_DIR_NAME}
 

diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml
@@ -18,6 +18,7 @@ on:
       - build/test_ios_ci.sh
       - examples/demo-apps/apple_ios/**
       - extension/apple/**
+      - extension/benchmark/apple/**
       - extension/module/**
   workflow_dispatch:
 
@@ -272,14 +273,14 @@ jobs:
         PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \
           build/build_apple_frameworks.sh --coreml --custom --mps --optimized --portable --quantized --xnnpack
 
-        mkdir -p extension/apple/Benchmark/Frameworks
+        mkdir -p extension/benchmark/apple/Benchmark/Frameworks
         for FRAMEWORK in "${FRAMEWORKS[@]}"; do (
-          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/apple/Benchmark/Frameworks/
+          cp -r "cmake-out/${FRAMEWORK}.xcframework" extension/benchmark/apple/Benchmark/Frameworks/
         ) done
         echo "::endgroup::"
 
         echo "::group::Build ExecuTorch benchmark app"
-        mkdir -p extension/apple/Benchmark/Models
+        mkdir -p extension/benchmark/apple/Benchmark/Models
         ${CONDA_RUN} --no-capture-output \
           build/build_apple_llm_demo.sh ${ARTIFACTS_DIR_NAME}
         echo "::endgroup::"
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -66,7 +66,7 @@ jobs:
         FILES_NEEDS_FORMAT=$(/opt/google-java-format -n extension/android/src/main/java/org/pytorch/executorch/*.java \
           examples/demo-apps/android/ExecuTorchDemo/app/src/main/java/com/example/executorchdemo/*.java \
           examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/*.java \
-          extension/android/benchmark/app/src/main/java/org/pytorch/minibench/*.java)
+          extension/benchmark/android/benchmark/app/src/main/java/org/pytorch/minibench/*.java)
         if [ -n "$FILES_NEEDS_FORMAT" ]; then
           echo "Warning: The following files need formatting. Please use google-java-format."
           echo "Use a binary from https://github.com/google/google-java-format/releases/"

diff --git a/.github/workflows/upload-android-test-specs.yml b/.github/workflows/upload-android-test-specs.yml
@@ -4,13 +4,13 @@ on:
   pull_request:
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - extension/android/benchmark/android-llm-device-farm-test-spec.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
   push:
     branches:
       - main
     paths:
       - .github/workflows/upload-android-test-specs.yml
-      - extension/android/benchmark/android-llm-device-farm-test-spec.yml
+      - extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
 
 concurrency:
   # NB: This concurency group needs to be different than the one used in android-perf, otherwise
@@ -32,7 +32,7 @@ jobs:
             ${{ github.repository }}/${{ github.run_id }}/artifacts
           retention-days: 1
           if-no-files-found: error
-          path: extension/android/benchmark/android-llm-device-farm-test-spec.yml
+          path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
 
   validate-android-test-spec:
     needs: upload-android-test-spec-for-validation
@@ -77,7 +77,7 @@ jobs:
 
       - name: Upload the spec to S3 ossci-android bucket
         shell: bash
-        working-directory: extension/android/benchmark/
+        working-directory: extension/benchmark/android/benchmark/
         env:
           SPEC_FILE: android-llm-device-farm-test-spec.yml
         run: |

diff --git a/.github/workflows/upload-apple-test-specs.yml b/.github/workflows/upload-apple-test-specs.yml
@@ -4,13 +4,13 @@ on:
   pull_request:
     paths:
       - .github/workflows/upload-apple-test-specs.yml
-      - examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml
+      - extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
   push:
     branches:
       - main
     paths:
       - .github/workflows/upload-apple-test-specs.yml
-      - examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml
+      - extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
 
 concurrency:
   # NB: This concurency group needs to be different than the one used in apple-perf, otherwise
@@ -32,7 +32,7 @@ jobs:
             ${{ github.repository }}/${{ github.run_id }}/artifacts
           retention-days: 1
           if-no-files-found: error
-          path: examples/demo-apps/apple_ios/default-ios-device-farm-appium-test-spec.yml
+          path: extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
 
   validate-apple-test-spec:
     needs: upload-apple-test-spec-for-validation
@@ -78,7 +78,7 @@ jobs:
 
       - name: Upload the spec to S3 ossci-ios bucket
         shell: bash
-        working-directory: examples/demo-apps/apple_ios
+        working-directory: extension/benchmark/apple/Benchmark/
         env:
           SPEC_FILE: default-ios-device-farm-appium-test-spec.yml
         run: |

diff --git a/build/build_android_llm_demo.sh b/build/build_android_llm_demo.sh
@@ -108,9 +108,9 @@ build_android_demo_apps() {
   ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest
   popd
 
-  mkdir -p extension/android/benchmark/app/libs
-  cp ${BUILD_AAR_DIR}/executorch.aar extension/android/benchmark/app/libs
-  pushd extension/android/benchmark
+  mkdir -p extension/benchmark/android/benchmark/app/libs
+  cp ${BUILD_AAR_DIR}/executorch.aar extension/benchmark/android/benchmark/app/libs
+  pushd extension/benchmark/android/benchmark
   ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build assembleAndroidTest
   popd
 }
@@ -135,8 +135,8 @@ collect_artifacts_to_be_uploaded() {
   # Collect MiniBench APK
   MINIBENCH_APP_DIR="${ARTIFACTS_DIR_NAME}/minibench"
   mkdir -p "${MINIBENCH_APP_DIR}"
-  cp extension/android/benchmark/app/build/outputs/apk/debug/*.apk "${MINIBENCH_APP_DIR}"
-  cp extension/android/benchmark/app/build/outputs/apk/androidTest/debug/*.apk "${MINIBENCH_APP_DIR}"
+  cp extension/benchmark/android/benchmark/app/build/outputs/apk/debug/*.apk "${MINIBENCH_APP_DIR}"
+  cp extension/benchmark/android/benchmark/app/build/outputs/apk/androidTest/debug/*.apk "${MINIBENCH_APP_DIR}"
 }
 
 BUILD_AAR_DIR="$(mktemp -d)"

diff --git a/build/build_apple_llm_demo.sh b/build/build_apple_llm_demo.sh
@@ -8,7 +8,7 @@
 set -euo pipefail
 
 ARTIFACTS_DIR_NAME="$1"
-APP_PATH="extension/apple/Benchmark/Benchmark"
+APP_PATH="extension/benchmark/apple/Benchmark/Benchmark"
 
 xcodebuild build-for-testing \
   -project "${APP_PATH}.xcodeproj" \

diff --git a/extension/android/benchmark/.gitignore → ...on/benchmark/android/benchmark/.gitignore b/extension/android/benchmark/.gitignore → ...on/benchmark/android/benchmark/.gitignore
diff --git a/extension/android/benchmark/README.md → ...ion/benchmark/android/benchmark/README.md b/extension/android/benchmark/README.md → ...ion/benchmark/android/benchmark/README.md
diff --git a/...ark/android-llm-device-farm-test-spec.yml → ...ark/android-llm-device-farm-test-spec.yml b/...ark/android-llm-device-farm-test-spec.yml → ...ark/android-llm-device-farm-test-spec.yml
diff --git a/extension/android/benchmark/app/.gitignore → ...enchmark/android/benchmark/app/.gitignore b/extension/android/benchmark/app/.gitignore → ...enchmark/android/benchmark/app/.gitignore
diff --git a/...on/android/benchmark/app/build.gradle.kts → ...rk/android/benchmark/app/build.gradle.kts b/...on/android/benchmark/app/build.gradle.kts → ...rk/android/benchmark/app/build.gradle.kts
diff --git a/.../android/benchmark/app/proguard-rules.pro → .../android/benchmark/app/proguard-rules.pro b/.../android/benchmark/app/proguard-rules.pro → .../android/benchmark/app/proguard-rules.pro
diff --git a/...ch/minibench/ExampleInstrumentedTest.java → ...ch/minibench/ExampleInstrumentedTest.java b/...ch/minibench/ExampleInstrumentedTest.java → ...ch/minibench/ExampleInstrumentedTest.java
diff --git a/...enchmark/app/src/main/AndroidManifest.xml → ...enchmark/app/src/main/AndroidManifest.xml b/...enchmark/app/src/main/AndroidManifest.xml → ...enchmark/app/src/main/AndroidManifest.xml
diff --git a/.../pytorch/minibench/BenchmarkActivity.java → .../pytorch/minibench/BenchmarkActivity.java b/.../pytorch/minibench/BenchmarkActivity.java → .../pytorch/minibench/BenchmarkActivity.java
diff --git a/...rg/pytorch/minibench/BenchmarkMetric.java → ...rg/pytorch/minibench/BenchmarkMetric.java b/...rg/pytorch/minibench/BenchmarkMetric.java → ...rg/pytorch/minibench/BenchmarkMetric.java
diff --git a/...torch/minibench/LlmBenchmarkActivity.java → ...torch/minibench/LlmBenchmarkActivity.java b/...torch/minibench/LlmBenchmarkActivity.java → ...torch/minibench/LlmBenchmarkActivity.java
diff --git a/...va/org/pytorch/minibench/ModelRunner.java → ...va/org/pytorch/minibench/ModelRunner.java b/...va/org/pytorch/minibench/ModelRunner.java → ...va/org/pytorch/minibench/ModelRunner.java
diff --git a/...ytorch/minibench/ModelRunnerCallback.java → ...ytorch/minibench/ModelRunnerCallback.java b/...ytorch/minibench/ModelRunnerCallback.java → ...ytorch/minibench/ModelRunnerCallback.java
diff --git a/...chmark/app/src/main/res/values/colors.xml → ...chmark/app/src/main/res/values/colors.xml b/...chmark/app/src/main/res/values/colors.xml → ...chmark/app/src/main/res/values/colors.xml
diff --git a/...hmark/app/src/main/res/values/strings.xml → ...hmark/app/src/main/res/values/strings.xml b/...hmark/app/src/main/res/values/strings.xml → ...hmark/app/src/main/res/values/strings.xml
diff --git a/...chmark/app/src/main/res/values/themes.xml → ...chmark/app/src/main/res/values/themes.xml b/...chmark/app/src/main/res/values/themes.xml → ...chmark/app/src/main/res/values/themes.xml
diff --git a/...rg/pytorch/minibench/ExampleUnitTest.java → ...rg/pytorch/minibench/ExampleUnitTest.java b/...rg/pytorch/minibench/ExampleUnitTest.java → ...rg/pytorch/minibench/ExampleUnitTest.java
diff --git a/extension/android/benchmark/build.gradle.kts → ...chmark/android/benchmark/build.gradle.kts b/extension/android/benchmark/build.gradle.kts → ...chmark/android/benchmark/build.gradle.kts
diff --git a/...nsion/android/benchmark/gradle.properties → ...hmark/android/benchmark/gradle.properties b/...nsion/android/benchmark/gradle.properties → ...hmark/android/benchmark/gradle.properties
diff --git a/...nchmark/gradle/wrapper/gradle-wrapper.jar → ...nchmark/gradle/wrapper/gradle-wrapper.jar b/...nchmark/gradle/wrapper/gradle-wrapper.jar → ...nchmark/gradle/wrapper/gradle-wrapper.jar
diff --git a/.../gradle/wrapper/gradle-wrapper.properties → .../gradle/wrapper/gradle-wrapper.properties b/.../gradle/wrapper/gradle-wrapper.properties → .../gradle/wrapper/gradle-wrapper.properties
diff --git a/extension/android/benchmark/gradlew → ...nsion/benchmark/android/benchmark/gradlew b/extension/android/benchmark/gradlew → ...nsion/benchmark/android/benchmark/gradlew
diff --git a/extension/android/benchmark/gradlew.bat → ...n/benchmark/android/benchmark/gradlew.bat b/extension/android/benchmark/gradlew.bat → ...n/benchmark/android/benchmark/gradlew.bat
diff --git a/...ion/android/benchmark/settings.gradle.kts → ...ark/android/benchmark/settings.gradle.kts b/...ion/android/benchmark/settings.gradle.kts → ...ark/android/benchmark/settings.gradle.kts
diff --git a/extension/benchmark/apple/.clang-format b/extension/benchmark/apple/.clang-format
@@ -0,0 +1,45 @@
+---
+AccessModifierOffset: -1
+AlignEscapedNewlinesLeft: true
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BreakBeforeBinaryOperators: false
+BreakBeforeBraces: Attach
+BreakConstructorInitializersBeforeComma: false
+ColumnLimit:     80
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+ExperimentalAutoDetectBinPacking: true
+IndentCaseLabels: false
+IndentFunctionDeclarationAfterType: false
+IndentWidth:     2
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 10
+PenaltyBreakComment: 60
+PenaltyBreakFirstLessLess: 20
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+SpaceAfterControlStatementKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
+...
diff --git a/...sion/apple/Benchmark/App/App.entitlements → ...mark/apple/Benchmark/App/App.entitlements b/...sion/apple/Benchmark/App/App.entitlements → ...mark/apple/Benchmark/App/App.entitlements
diff --git a/extension/apple/Benchmark/App/App.swift → ...n/benchmark/apple/Benchmark/App/App.swift b/extension/apple/Benchmark/App/App.swift → ...n/benchmark/apple/Benchmark/App/App.swift
diff --git a/...hmark/Benchmark.xcodeproj/project.pbxproj → ...hmark/Benchmark.xcodeproj/project.pbxproj b/...hmark/Benchmark.xcodeproj/project.pbxproj → ...hmark/Benchmark.xcodeproj/project.pbxproj
@@ -66,10 +66,10 @@
 		032A73FD2CAFBB7800932D36 /* tiktoken.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = tiktoken.h; sourceTree = "<group>"; };
 		032A73FE2CAFBB7800932D36 /* tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = tiktoken.cpp; sourceTree = "<group>"; };
 		032A73FF2CAFBB7800932D36 /* tokenizer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = tokenizer.h; sourceTree = "<group>"; };
-		032A74212CAFC1B300932D36 /* runner.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = runner.h; path = ../../../examples/models/llama2/runner/runner.h; sourceTree = SOURCE_ROOT; };
-		032A74222CAFC1B300932D36 /* runner.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = runner.cpp; path = ../../../examples/models/llama2/runner/runner.cpp; sourceTree = SOURCE_ROOT; };
-		032A74242CAFC34800932D36 /* llama_tiktoken.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = llama_tiktoken.h; path = ../../../examples/models/llama2/tokenizer/llama_tiktoken.h; sourceTree = SOURCE_ROOT; };
-		032A74252CAFC34800932D36 /* llama_tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = llama_tiktoken.cpp; path = ../../../examples/models/llama2/tokenizer/llama_tiktoken.cpp; sourceTree = SOURCE_ROOT; };
+		032A74212CAFC1B300932D36 /* runner.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = runner.h; path = ../../../../examples/models/llama2/runner/runner.h; sourceTree = SOURCE_ROOT; };
+		032A74222CAFC1B300932D36 /* runner.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = runner.cpp; path = ../../../../examples/models/llama2/runner/runner.cpp; sourceTree = SOURCE_ROOT; };
+		032A74242CAFC34800932D36 /* llama_tiktoken.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = llama_tiktoken.h; path = ../../../../examples/models/llama2/tokenizer/llama_tiktoken.h; sourceTree = SOURCE_ROOT; };
+		032A74252CAFC34800932D36 /* llama_tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = llama_tiktoken.cpp; path = ../../../../examples/models/llama2/tokenizer/llama_tiktoken.cpp; sourceTree = SOURCE_ROOT; };
 		037C96A02C8A570B00B3DF38 /* Tests.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = Tests.xctestplan; sourceTree = "<group>"; };
 		03B0118B2CAC567900054791 /* DynamicTestCase.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DynamicTestCase.h; sourceTree = "<group>"; };
 		03B0118C2CAC567900054791 /* DynamicTestCase.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = DynamicTestCase.m; sourceTree = "<group>"; };
@@ -158,7 +158,7 @@
 				032A73DF2CAFBB7800932D36 /* util.h */,
 			);
 			name = runner;
-			path = ../../llm/runner;
+			path = ../../../llm/runner;
 			sourceTree = SOURCE_ROOT;
 		};
 		032A73E92CAFBB7800932D36 /* sampler */ = {
@@ -168,7 +168,7 @@
 				032A73E62CAFBB7800932D36 /* sampler.cpp */,
 			);
 			name = sampler;
-			path = ../../llm/sampler;
+			path = ../../../llm/sampler;
 			sourceTree = SOURCE_ROOT;
 		};
 		032A74022CAFBB7800932D36 /* tokenizer */ = {
@@ -184,7 +184,7 @@
 				032A73FF2CAFBB7800932D36 /* tokenizer.h */,
 			);
 			name = tokenizer;
-			path = ../../llm/tokenizer;
+			path = ../../../llm/tokenizer;
 			sourceTree = SOURCE_ROOT;
 		};
 		03B0118D2CAC567900054791 /* TestUtils */ = {
@@ -364,7 +364,7 @@
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;
-			shellScript = "set -e\n\nif ! command -v cmake &> /dev/null\nthen\n    echo \"Cmake not found, please install Cmake. \\n1. Download Cmake.app from https://cmake.org/download with version > 3.19. \\n2. Install it to Applications/ folder and run sudo /Applications/CMake.app/Contents/bin/cmake-gui --install to install CMake commandline tools.\"\n    exit 1\nfi\n\nCMAKE_DIR=\"$TEMP_DIR/cmake\"\nrm -rf \"$CMAKE_DIR\"\n\nPLATFORM=\"SIMULATORARM64\"\nDEPLOYMENT_TARGET=\"17.0\"\n\nif [[ \"$PLATFORM_NAME\" == *\"iphoneos\"* ]]; then\n  PLATFORM=\"OS64\"\nelif [[ \"$PLATFORM_NAME\" == *\"macos\"* ]]; then\n  PLATFORM=\"MAC_ARM64\"\n  DEPLOYMENT_TARGET=\"10.15\"\nfi\n\ncmake_build() {\n    local src_dir=$1\n    local target=$2\n    shift 2\n    local extra_args=(\"$@\")\n    local build_dir=\"$CMAKE_DIR/build/$(basename \"$src_dir\")\"\n\n    mkdir -p \"$build_dir\" && cd \"$build_dir\"\n\n    if [[ \"$PLATFORM\" == \"MAC_ARM64\" ]]; then\n        extra_args+=(-DCMAKE_INSTALL_BUNDLEDIR=\"${CMAKE_DIR}/bin\")\n        extra_args+=(-DCMAKE_MACOSX_BUNDLE=OFF)\n    fi\n    cmake -G Xcode \\\n          -DCMAKE_BUILD_TYPE=\"Release\" \\\n          -DCMAKE_CXX_STANDARD=17 \\\n          -DCMAKE_TOOLCHAIN_FILE=\"$SRCROOT/../../../third-party/ios-cmake/ios.toolchain.cmake\" \\\n          -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD=\"c++17\" \\\n          -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY=\"libc++\" \\\n          -DPLATFORM=\"$PLATFORM\" \\\n          -DDEPLOYMENT_TARGET=\"$DEPLOYMENT_TARGET\" \\\n          -DCMAKE_INSTALL_PREFIX=\"$CMAKE_DIR\" \\\n          \"${extra_args[@]}\" \\\n          \"$src_dir\"\n    cmake --build . --config \"Release\" --target \"$target\"\n    if [[ \"$target\" == \"install\" ]]; then\n        cmake --install . --prefix \"$CMAKE_DIR\"\n    fi\n}\n\ncmake_build \"$SRCROOT/../../llm/third-party/abseil-cpp\" \"install\" \\\n    -DABSL_PROPAGATE_CXX_STD=ON\n\ncmake_build \"$SRCROOT/../../llm/third-party/re2\" \"install\"\n\ncmake_build \"$SRCROOT/../../llm/third-party/sentencepiece\" \"sentencepiece-static\" \\\n    -DSPM_ENABLE_SHARED=OFF\n\necho \"$(find $CMAKE_DIR/lib -name \"*.a\" | sed -E 's|^.*/lib([^/]+)\\.a|-l\\1|g' | tr '\\n' ' ')\" > \"$CMAKE_DIR/linker_flags\"\n";
+			shellScript = "set -e\n\nif ! command -v cmake &> /dev/null\nthen\n    echo \"Cmake not found, please install Cmake. \\n1. Download Cmake.app from https://cmake.org/download with version > 3.19. \\n2. Install it to Applications/ folder and run sudo /Applications/CMake.app/Contents/bin/cmake-gui --install to install CMake commandline tools.\"\n    exit 1\nfi\n\nCMAKE_DIR=\"$TEMP_DIR/cmake\"\nrm -rf \"$CMAKE_DIR\"\n\nPLATFORM=\"SIMULATORARM64\"\nDEPLOYMENT_TARGET=\"17.0\"\n\nif [[ \"$PLATFORM_NAME\" == *\"iphoneos\"* ]]; then\n  PLATFORM=\"OS64\"\nelif [[ \"$PLATFORM_NAME\" == *\"macos\"* ]]; then\n  PLATFORM=\"MAC_ARM64\"\n  DEPLOYMENT_TARGET=\"10.15\"\nfi\n\ncmake_build() {\n    local src_dir=$1\n    local target=$2\n    shift 2\n    local extra_args=(\"$@\")\n    local build_dir=\"$CMAKE_DIR/build/$(basename \"$src_dir\")\"\n\n    mkdir -p \"$build_dir\" && cd \"$build_dir\"\n\n    if [[ \"$PLATFORM\" == \"MAC_ARM64\" ]]; then\n        extra_args+=(-DCMAKE_INSTALL_BUNDLEDIR=\"${CMAKE_DIR}/bin\")\n        extra_args+=(-DCMAKE_MACOSX_BUNDLE=OFF)\n    fi\n    cmake -G Xcode \\\n          -DCMAKE_BUILD_TYPE=\"Release\" \\\n          -DCMAKE_CXX_STANDARD=17 \\\n          -DCMAKE_TOOLCHAIN_FILE=\"$SRCROOT/../../../../third-party/ios-cmake/ios.toolchain.cmake\" \\\n          -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD=\"c++17\" \\\n          -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY=\"libc++\" \\\n          -DPLATFORM=\"$PLATFORM\" \\\n          -DDEPLOYMENT_TARGET=\"$DEPLOYMENT_TARGET\" \\\n          -DCMAKE_INSTALL_PREFIX=\"$CMAKE_DIR\" \\\n          \"${extra_args[@]}\" \\\n          \"$src_dir\"\n    cmake --build . --config \"Release\" --target \"$target\"\n    if [[ \"$target\" == \"install\" ]]; then\n        cmake --install . --prefix \"$CMAKE_DIR\"\n    fi\n}\n\ncmake_build \"$SRCROOT/../../../llm/third-party/abseil-cpp\" \"install\" \\\n    -DABSL_PROPAGATE_CXX_STD=ON\n\ncmake_build \"$SRCROOT/../../../llm/third-party/re2\" \"install\"\n\ncmake_build \"$SRCROOT/../../../llm/third-party/sentencepiece\" \"sentencepiece-static\" \\\n    -DSPM_ENABLE_SHARED=OFF\n\necho \"$(find $CMAKE_DIR/lib -name \"*.a\" | sed -E 's|^.*/lib([^/]+)\\.a|-l\\1|g' | tr '\\n' ' ')\" > \"$CMAKE_DIR/linker_flags\"\n";
 		};
 /* End PBXShellScriptBuildPhase section */
 

diff --git a/...xcshareddata/xcschemes/Benchmark.xcscheme → ...xcshareddata/xcschemes/Benchmark.xcscheme b/...xcshareddata/xcschemes/Benchmark.xcscheme → ...xcshareddata/xcschemes/Benchmark.xcscheme
diff --git a/...nchmark/Frameworks/download_frameworks.sh → ...nchmark/Frameworks/download_frameworks.sh b/...nchmark/Frameworks/download_frameworks.sh → ...nchmark/Frameworks/download_frameworks.sh
diff --git a/extension/apple/Benchmark/README.md → ...nsion/benchmark/apple/Benchmark/README.md b/extension/apple/Benchmark/README.md → ...nsion/benchmark/apple/Benchmark/README.md
@@ -42,7 +42,7 @@ You have two options:
 Run the provided script to download the prebuilt frameworks:
 
 ```bash
-./extension/apple/Benchmark/Frameworks/download_frameworks.sh
+./extension/benchmark/apple/Benchmark/Frameworks/download_frameworks.sh
 ```
 </details>
 
@@ -56,7 +56,7 @@ Alternatively, you can build the frameworks yourself by following the [guide](ht
 Once the frameworks are downloaded or built, verify that the `Frameworks` directory contains the necessary `.xcframework` files:
 
 ```bash
-ls extension/apple/Benchmark/Frameworks
+ls extension/benchmark/apple/Benchmark/Frameworks
 ```
 
 You should see:
@@ -74,16 +74,16 @@ kernels_quantized.xcframework
 
 ## Adding Models and Resources
 
-Place your exported model files (`.pte`) and any other resources (e.g., `tokenizer.bin`) into the `extension/apple/Benchmark/Resources` directory:
+Place your exported model files (`.pte`) and any other resources (e.g., `tokenizer.bin`) into the `extension/benchmark/apple/Benchmark/Resources` directory:
 
 ```bash
-cp <path/to/my_model.pte> <path/to/llama3.pte> <path/to/tokenizer.bin> extension/apple/Benchmark/Resources
+cp <path/to/my_model.pte> <path/to/llama3.pte> <path/to/tokenizer.bin> extension/benchmark/apple/Benchmark/Resources
 ```
 
 Optionally, check that the files are there:
 
 ```bash
-ls extension/apple/Benchmark/Resources
+ls extension/benchmark/apple/Benchmark/Resources
 ```
 
 For this example you should see:
@@ -103,7 +103,7 @@ The app automatically bundles these resources and makes them available to the te
 Open the Benchmark Xcode project:
 
 ```bash
-open extension/apple/Benchmark/Benchmark.xcodeproj
+open extension/benchmark/apple/Benchmark/Benchmark.xcodeproj
 ```
 
 Select the destination device or simulator and press `Command+U`, or click `Product` > `Test` in the menu to run the test suite.
@@ -333,13 +333,13 @@ You can also run the tests using `xcodebuild`:
 
 ```bash
 # Run on an iOS Simulator
-xcodebuild test -project extension/apple/Benchmark/Benchmark.xcodeproj \
+xcodebuild test -project extension/benchmark/apple/Benchmark/Benchmark.xcodeproj \
 -scheme Benchmark \
 -destination 'platform=iOS Simulator,name=<SimulatorName>' \
 -testPlan Tests
 
 # Run on a physical iOS device
-xcodebuild test -project extension/apple/Benchmark/Benchmark.xcodeproj \
+xcodebuild test -project extension/benchmark/apple/Benchmark/Benchmark.xcodeproj \
 -scheme Benchmark \
 -destination 'platform=iOS,name=<DeviceName>' \
 -testPlan Tests \

diff --git a/...nsion/apple/Benchmark/Resources/README.md → ...hmark/apple/Benchmark/Resources/README.md b/...nsion/apple/Benchmark/Resources/README.md → ...hmark/apple/Benchmark/Resources/README.md
diff --git a/...ple/Benchmark/TestUtils/DynamicTestCase.h → ...ple/Benchmark/TestUtils/DynamicTestCase.h b/...ple/Benchmark/TestUtils/DynamicTestCase.h → ...ple/Benchmark/TestUtils/DynamicTestCase.h
diff --git a/...ple/Benchmark/TestUtils/DynamicTestCase.m → ...ple/Benchmark/TestUtils/DynamicTestCase.m b/...ple/Benchmark/TestUtils/DynamicTestCase.m → ...ple/Benchmark/TestUtils/DynamicTestCase.m
diff --git a/...le/Benchmark/TestUtils/ResourceTestCase.h → ...le/Benchmark/TestUtils/ResourceTestCase.h b/...le/Benchmark/TestUtils/ResourceTestCase.h → ...le/Benchmark/TestUtils/ResourceTestCase.h
diff --git a/...le/Benchmark/TestUtils/ResourceTestCase.m → ...le/Benchmark/TestUtils/ResourceTestCase.m b/...le/Benchmark/TestUtils/ResourceTestCase.m → ...le/Benchmark/TestUtils/ResourceTestCase.m
diff --git a/...ion/apple/Benchmark/Tests/GenericTests.mm → ...ark/apple/Benchmark/Tests/GenericTests.mm b/...ion/apple/Benchmark/Tests/GenericTests.mm → ...ark/apple/Benchmark/Tests/GenericTests.mm
diff --git a/...apple/Benchmark/Tests/LLaMA/LLaMATests.mm → ...apple/Benchmark/Tests/LLaMA/LLaMATests.mm b/...apple/Benchmark/Tests/LLaMA/LLaMATests.mm → ...apple/Benchmark/Tests/LLaMA/LLaMATests.mm