[microTVM][Zephyr] Update RVM to Zephyr 2.7 (#10138)

* Update to zephyr2.7 and Refactor

* Temporary for testing

* Update cmake version

* fix import path and format

* Fix test script

* address comments

* fix path

* fix image name

apps/microtvm/reference-vm/base-box-tool.py

@@ -51,7 +51,10 @@
 # in [platform]/base-box/base_box_provision.sh
 EXTRA_SCRIPTS = {
     "arduino": (),
-    "zephyr": ("docker/install/ubuntu_init_zephyr_project.sh",),
+    "zephyr": (
+        "docker/install/ubuntu_init_zephyr_project.sh",
+        "docker/install/ubuntu_install_zephyr_sdk.sh",
+    ),
 }
 
 PACKER_FILE_NAME = "packer.json"
@@ -230,6 +233,12 @@ def generate_packer_config(platform, file_path, providers):
         filename = os.path.basename(script_path)
         provisioners.append({"type": "file", "source": script_path, "destination": f"~/{filename}"})
 
+    provisioners.append(
+        {
+            "type": "shell",
+            "script": "base_box_setup.sh",
+        }
+    )
     provisioners.append(
         {
             "type": "shell",

apps/microtvm/reference-vm/zephyr/Vagrantfile

@@ -16,7 +16,7 @@
 # under the License.
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "tlcpack/microtvm-zephyr-2.5"
+  config.vm.box = "tlcpack/microtvm-zephyr-2.7"
 
   if ENV.has_key?("TVM_RVM_NUM_CORES")
     num_cores = ENV["TVM_RVM_NUM_CORES"]

apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh

@@ -27,9 +27,7 @@ source ~/.profile
 
 # Init Zephyr
 cd ~
-# Using most recent commit that passes all the tests.
-ZEPHYR_VERSION="v2.5-branch"
-~/ubuntu_init_zephyr_project.sh ~/zephyr ${ZEPHYR_VERSION} --commit dabf23758417fd041fec2a2a821d8f526afac29d
+~/ubuntu_init_zephyr_project.sh ~/zephyr
 
 # Cleanup
-rm -f *.sh
+rm -f ubuntu_init_zephyr_project.sh

apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh

@@ -38,7 +38,7 @@ sudo apt update
 # NOTE: latest cmake cannot be installed due to
 # https://github.com/zephyrproject-rtos/zephyr/issues/30232
 sudo apt install -y --no-install-recommends git \
-     cmake=3.18.4-0kitware1 cmake-data=3.18.4-0kitware1 \
+     cmake=3.22.2-0kitware1ubuntu18.04.1 cmake-data=3.22.2-0kitware1ubuntu18.04.1 \
      ninja-build gperf ccache dfu-util device-tree-compiler wget \
      python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \
      make gcc gcc-multilib g++-multilib libsdl2-dev
@@ -85,13 +85,8 @@ echo PATH=$PATH
 
 cd ~
 echo "Downloading zephyr SDK..."
-ZEPHYR_SDK_VERSION=0.12.3
-ZEPHYR_SDK_FILE=zephyr-sdk-linux-setup.run
-wget --no-verbose -O $ZEPHYR_SDK_FILE \
-    https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}-x86_64-linux-setup.run
-chmod +x $ZEPHYR_SDK_FILE
-"./$ZEPHYR_SDK_FILE" -- -d ~/zephyr-sdk -y
-rm -rf "${ZEPHYR_SDK_FILE}"
+~/ubuntu_install_zephyr_sdk.sh ~/zephyr-sdk
+rm -f ubuntu_install_zephyr_sdk.sh
 
 # GDB for Zephyr SDK depends on python3.8
 sudo add-apt-repository ppa:deadsnakes/ppa

apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh

@@ -31,9 +31,4 @@ fi
 board=$1
 
 pytest tests/micro/zephyr/test_zephyr.py --zephyr-board=${board}
-
-if [ $board == "stm32f746g_disco" ] || [ $board == "nucleo_f746zg" ]; then
-    echo "NOTE: skipped test_zephyr_aot.py on $board -- known failure"
-else
-    pytest tests/micro/zephyr/test_zephyr_aot.py --zephyr-board=${board}
-fi
+pytest tests/micro/zephyr/test_zephyr_aot.py --zephyr-board=${board}

apps/microtvm/zephyr/template_project/microtvm_api_server.py

@@ -590,7 +590,14 @@ def _set_nonblock(fd):
 class ZephyrSerialTransport:
     @classmethod
     def _lookup_baud_rate(cls, options):
-        sys.path.insert(0, os.path.join(get_zephyr_base(options), "scripts", "dts"))
+        # TODO(mehrdadh): remove this hack once dtlib.py is a standalone project
+        # https://github.com/zephyrproject-rtos/zephyr/blob/v2.7-branch/scripts/dts/README.txt
+        sys.path.insert(
+            0,
+            os.path.join(
+                get_zephyr_base(options), "scripts", "dts", "python-devicetree", "src", "devicetree"
+            ),
+        )
         try:
             import dtlib  # pylint: disable=import-outside-toplevel
         finally:

docker/Dockerfile.ci_qemu

@@ -67,6 +67,7 @@ RUN bash /install/ubuntu_install_sccache.sh
 # Zephyr SDK deps
 COPY install/ubuntu_install_zephyr.sh /install/ubuntu_install_zephyr.sh
 COPY install/ubuntu_init_zephyr_project.sh /install/ubuntu_init_zephyr_project.sh
+COPY install/ubuntu_install_zephyr_sdk.sh /install/ubuntu_install_zephyr_sdk.sh
 RUN bash /install/ubuntu_install_zephyr.sh
 ENV ZEPHYR_BASE=/opt/zephyrproject/zephyr
 

docker/install/ubuntu_init_zephyr_project.sh

@@ -19,32 +19,55 @@
 #
 # Initialize Zephyr Project.
 #
-# Usage: ubuntu_init_zephyr_project.sh path branch [--commit hash]
-#   path is the installation path for the repository.
-#   branch is the zephyr branch.
-#   --commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit.
+# Usage: docker/install/ubuntu_init_zephyr_project.sh <INSTALLATION_PATH> [--branch BRANCH]
+#         [--commit HASH]
+# INSTALLATION_PATH is the installation path for the repository.
+# --branch is the zephyr branch. If not specified, it uses the default.
+# --commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit.
 #
 
 set -x
 
-DOWNLOAD_DIR=$1
-shift
-ZEPHYR_BRANCH=$1
+function show_usage() {
+    cat <<EOF
+Usage: docker/install/ubuntu_init_zephyr_project.sh <INSTALLATION_PATH> [--branch BRANCH]
+        [--commit COMMIT]
+INSTALLATION_PATH is the installation path for the repository.
+--branch is the zephyr branch. If not specified, it uses the default.
+--commit is the commit hash number of zephyrproject repository. If not specified, it uses the latest commit.
+EOF
+}
+
+if [ "$#" -lt 1 -o "$1" == "--help" -o "$1" == "-h" ]; then
+    show_usage
+    exit -1
+fi
+
+INSTALLATION_PATH=$1
 shift
 
-commit_hash=
+if [ "$1" == "--branch" ]; then
+    shift
+    BRANCH=$1
+    shift
+else
+    BRANCH="v2.7-branch"
+fi
+
+COMMIT=
 if [ "$1" == "--commit" ]; then
     shift
-    commit_hash=$1
+    COMMIT=$1
+    shift
 fi
 
-west init --mr ${ZEPHYR_BRANCH} ${DOWNLOAD_DIR}
+west init --mr ${BRANCH} ${INSTALLATION_PATH}
 
-if [ -n "$commit_hash" ]; then
-    cd ${DOWNLOAD_DIR}/zephyr
-    git checkout ${commit_hash}
+if [ -n "$COMMIT" ]; then
+    cd ${INSTALLATION_PATH}/zephyr
+    git checkout ${COMMIT}
 fi
 
-cd ${DOWNLOAD_DIR}
+cd ${INSTALLATION_PATH}
 west update
 west zephyr-export

docker/install/ubuntu_install_zephyr.sh

@@ -44,13 +44,8 @@ sudo apt-get install -y cmake
 pip3 install west
 
 # Init ZephyrProject
-# To keep in sync with the version 
-# defined in apps/microtvm/zephyr/template_project/microtvm_api_server.py
-# We use `-branch` tag since it tracks the same version with extra patches for bugs.
-ZEPHYR_VERSION="v2.7-branch"
 ZEPHYR_PROJECT_PATH=/opt/zephyrproject
-ZEPHYR_INIT_SCRIPT=$(find -name "ubuntu_init_zephyr_project.sh")
-bash ${ZEPHYR_INIT_SCRIPT} ${ZEPHYR_PROJECT_PATH} ${ZEPHYR_VERSION}
+bash /install/ubuntu_init_zephyr_project.sh ${ZEPHYR_PROJECT_PATH}
 cd ${ZEPHYR_PROJECT_PATH}
 
 # As part of the build process, Zephyr needs to touch some symlinks in zephyr/misc/generated/syscalls_links (this path is relative to the
@@ -69,10 +64,4 @@ chmod o+rwx zephyr/.cache
 #/opt/west/bin/pip3 install -r /opt/zephyrproject/zephyr/scripts/requirements.txt
 pip3 install -r /opt/zephyrproject/zephyr/scripts/requirements.txt
 
-ZEPHYR_SDK_VERSION=0.13.2
-ZEPHYR_SDK_FILE=zephyr-sdk-linux-setup.run
-wget --no-verbose -O $ZEPHYR_SDK_FILE \
-    https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}-linux-x86_64-setup.run
-chmod +x $ZEPHYR_SDK_FILE
-"./$ZEPHYR_SDK_FILE" -- -d /opt/zephyr-sdk
-rm "$ZEPHYR_SDK_FILE"
+bash /install/ubuntu_install_zephyr_sdk.sh /opt/zephyr-sdk

docker/install/ubuntu_install_zephyr_sdk.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Install Zephyr SDK
+#
+# Usage: docker/install/ubuntu_install_zephyr_sdk.sh <INSTALLATION_PATH>
+# INSTALLATION_PATH is the installation path for the SDK.
+#
+
+set -e
+set -x
+
+function show_usage() {
+    cat <<EOF
+Usage: docker/install/ubuntu_install_zephyr_sdk.sh <INSTALLATION_PATH>
+INSTALLATION_PATH is the installation path for the SDK.
+EOF
+}
+
+if [ "$#" -lt 1 -o "$1" == "--help" -o "$1" == "-h" ]; then
+    show_usage
+    exit -1
+fi
+
+INSTALLATION_PATH=$1
+shift
+
+ZEPHYR_SDK_VERSION=0.13.2
+ZEPHYR_SDK_FILE=zephyr-sdk-linux-setup.run
+wget --no-verbose -O $ZEPHYR_SDK_FILE \
+    https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}-linux-x86_64-setup.run
+chmod +x $ZEPHYR_SDK_FILE
+"./$ZEPHYR_SDK_FILE" -- -d ${INSTALLATION_PATH}
+rm "$ZEPHYR_SDK_FILE"