-
Notifications
You must be signed in to change notification settings - Fork 8
190 lines (167 loc) · 7.19 KB
/
benchmark_test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
name: Benchmark Tests
on:
workflow_dispatch:
jobs:
Run_Benchmark:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
benchmark: [aww, vww, resnet, toycar]
# benchmark: [aww]
target: [spike_rv32, spike_rv32_min, spike_rv64]
# target: [spike_rv32, spike_rv32_min]
backend: [tvmaot, tflmi]
# backend: [tvmaot]
toolchain: [gcc, llvm]
optimize: [s, 3]
# optimize: [s]
unroll: [0]
steps:
- name: Maximize build space
uses: easimon/maximize-build-space@master
with:
root-reserve-mb: 32768
temp-reserve-mb: 1024
remove-dotnet: 'true'
remove-android: 'true'
remove-haskell: 'true'
remove-codeql: 'true'
- uses: actions/checkout@v4
- name: Install TFLM dependencies
run: |
sudo apt-get install python3-setuptools
pip3 install wheel
pip3 install Pillow
pip3 install Wave
- name: Get dates
id: dates
run: |
echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
echo "yesterday=$(date +'%Y-%m-%d' -d yesterday)" >> $GITHUB_OUTPUT
echo "date=$(date +'%Y-%m-%d' -d 'last saturday')" >> $GITHUB_OUTPUT
- name: Download Cached GCC Binaries
id: cache-gcc
uses: actions/cache/restore@v4
env:
cache-name: cache-gcc-binaries
with:
path: |
./Toolchain
!./Toolchain/*.sh
fail-on-cache-miss: true
key: gcc-binaries-${{ steps.dates.outputs.today }}
restore-keys: |
gcc-binaries-${{ steps.dates.outputs.yesterday }}
gcc-binaries-${{ steps.dates.outputs.date }}
gcc-binaries-
- name: TEST
run: |
ls Toolchain/
ls Toolchain/rv32gc
ls Toolchain/rv64gc
ls Toolchain/rv32gcp
ls Toolchain/rv32gcv
ls Toolchain/rv64gcv
ls Toolchain/rv32im
ls Toolchain/rv32im_zve32x
ls Toolchain/llvm
./Toolchain/rv32gc/bin/riscv32-unknown-elf-gcc --version
./Toolchain/rv64gc/bin/riscv64-unknown-elf-gcc --version
./Toolchain/rv32gcp/bin/riscv32-unknown-elf-gcc --version
./Toolchain/rv32gcv/bin/riscv32-unknown-elf-gcc --version
./Toolchain/rv64gcv/bin/riscv64-unknown-elf-gcc --version
./Toolchain/rv32im/bin/riscv32-unknown-elf-gcc --version
./Toolchain/rv32im_zve32x/bin/riscv32-unknown-elf-gcc --version
./Toolchain/llvm/bin/clang --version
- name: Install Spike Dependencies
run: |
sudo apt-get install libboost-all-dev
sudo apt-get install device-tree-compiler
ls Toolchain/
- name: Setup MLonMCU
working-directory: ./Integration/MLonMCU
run: |
export PREBUILD_TVM=true
python3 -m pip install --user virtualenv
# ./setup_mlonmcu.sh --enable-tflm --enable-tvm --enable-spike
./setup_mlonmcu.sh --enable-tflm --enable-tvm --enable-spike --enable-vext --enable-gcc --enable-llvm --enable-rv64
- name: Run ${{matrix.benchmark}} Benchmark
working-directory: ./Integration/MLonMCU
run: |
set -e
source common.sh
source $VENV_DIR/bin/activate
export MLONMCU_HOME=$WORKSPACE_DIR
cat $MLONMCU_HOME/environment.yml
# python gen_muriscnn_benchmarks.py --verbose -b ${{matrix.backend}} -t spike --vlen 128 --vlen 256 --vlen 512 --vlen 1024 --vlen 2048 --vlen 4096 --post -p --parallel 4 -f muriscvnn -f vext -f pext --autotuned --toolchain ${{matrix.toolchain}} -f auto_vectorize --opt ${{matrix.optimize}} --unroll ${{matrix.unroll}} --baseline $(case "${{matrix.backend}}-${{matrix.toolchain}}" in tvm*-gcc) echo -n "32" ;; tvm*-llvm) echo -n "28" ;; tflm*-gcc) echo -n "8" ;; tflm*-llvm) echo -n "7" ;; *) echo -n "?" ;; esac) ${{matrix.benchmark}} --fail
python gen_muriscnn_benchmarks.py --verbose -b ${{matrix.backend}} -t ${{matrix.target}} --vlen 128 --vlen 256 --vlen 512 --vlen 1024 --vlen 2048 --vlen 4096 --post -p --parallel 4 -f muriscvnn -f vext --toolchain ${{matrix.toolchain}} -f auto_vectorize --opt ${{matrix.optimize}} --unroll ${{matrix.unroll}} --baseline $(case "${{matrix.backend}}-${{matrix.toolchain}}" in tvm*-gcc) echo -n "14" ;; tvm*-llvm) echo -n "14" ;; tflm*-gcc) echo -n "7" ;; tflm*-llvm) echo -n "7" ;; *) echo -n "?" ;; esac) ${{matrix.benchmark}} --portable
# TODO: add --fail
- name: Archive Benchmark Results
uses: actions/upload-artifact@v4
with:
name: ${{matrix.benchmark}}-benchmark-${{matrix.backend}}-${{matrix.toolchain}}-${{matrix.optimize}}-${{matrix.unroll}}-${{matrix.target}}-${{ steps.dates.outputs.today }}
path: ./Integration/MLonMCU/out.csv
Update_Wiki:
runs-on: ubuntu-22.04
needs: Run_Benchmark
steps:
- uses: actions/checkout@v4
- name: Download Cached GCC Binaries
id: cache-gcc
uses: actions/cache/restore@v4
env:
cache-name: cache-gcc-binaries
with:
path: |
./Toolchain
!./Toolchain/*.sh
fail-on-cache-miss: true
key: gcc-binaries-${{ steps.dates.outputs.today }}
restore-keys: |
gcc-binaries-${{ steps.dates.outputs.yesterday }}
gcc-binaries-${{ steps.dates.outputs.date }}
gcc-binaries-
- name: Download Artifacts
uses: actions/download-artifact@v4
with:
path: ./Wiki
- name: Download dependencies
run: |
pip install -r Wiki/requirements.txt
- name: Get current date
id: date-today
run: echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
- name: Create New Wiki Page
working-directory: ./Wiki
run: |
source ../Integration/MLonMCU/common.sh
export RV32GC_VERSION=$(../Toolchain/rv32gc/bin/riscv32-unknown-elf-gcc --version | head -1)
export RV32GCV_VERSION=$(../Toolchain/rv32gcv/bin/riscv32-unknown-elf-gcc --version | head -1)
export RV32GCP_VERSION=$(../Toolchain/rv32gcp/bin/riscv32-unknown-elf-gcc --version | head -1)
export LLVM_VERSION=$(../Toolchain/llvm/bin/clang --version | head -1)
python3 create_wiki.py */*.csv --split
cat Benchmarks-*.md >> $GITHUB_STEP_SUMMARY
- name: Clone Wiki Repo
uses: actions/checkout@v4
with:
repository: ${{github.repository}}.wiki
path: Wiki-repo
- name: Push New Page to Wiki
working-directory: ./Wiki-repo
run: |
# ls
cp ../Wiki/*.md .
cp ../Wiki/*.csv .
# ls
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
find . -name "Benchmarks-${{ steps.date-today.outputs.today }}-*.csv" -exec ../Wiki/compare_with_prev.sh {} 2 \;
python3 ../Wiki/gen_sidebar.py Benchmarks-*.md --open first > _Sidebar.md
python3 ../Wiki/gen_sidebar.py Benchmarks-*.md --open none --hide-models > _Footer.md
rm README.md
git add *.md
git add *.csv
git status
git commit -m "Added Benchmarks on ${{ steps.date-today.outputs.today }}"
git push origin master