2024-03-13 nightly release (3507412)

.buckconfig

@@ -0,0 +1,25 @@
+[executorch]
+  is_oss = 1
+
+[buildfile]
+  name = TARGETS
+
+[repositories]
+  root = .
+  prelude = third-party/prelude
+  shim = shim
+
+[repository_aliases]
+  config = prelude
+  ovr_config = prelude
+  toolchains = shim
+  fbcode = shim
+  fbcode_macros = shim
+  fbsource = shim
+  buck = shim
+
+[cxx]
+  cxxflags = -g -std=c++17
+
+[parser]
+  target_platform_detector_spec = target:root//...->prelude//platforms:default target:shim//...->prelude//platforms:default

.ci/docker/README.md

@@ -0,0 +1,25 @@
+# Docker images for ExecuTorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in ExecuTorch CI. The content of this directory are copied
+from PyTorch CI https://github.com/pytorch/pytorch/tree/main/.ci/docker.
+It also uses the same directory structure as PyTorch.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS=ubuntu
+OS_VERSION=22.04
+CLANG_VERSION=""
+GCC_VERSION=""
+PYTHON_VERSION=3.10
+MINICONDA_VERSION=23.10.0-1
+BUCK2_VERSION=$(cat ci_commit_pins/buck2.txt)
+
+case "${IMAGE_NAME}" in
+  executorch-ubuntu-22.04-gcc9)
+    LINTRUNNER=""
+    GCC_VERSION=9
+    ;;
+  executorch-ubuntu-22.04-clang12)
+    LINTRUNNER=""
+    CLANG_VERSION=12
+    ;;
+  executorch-ubuntu-22.04-linter)
+    LINTRUNNER=yes
+    CLANG_VERSION=12
+    ;;
+  executorch-ubuntu-22.04-arm-sdk)
+    ARM_SDK=yes
+    CLANG_VERSION=12
+    ;;
+  *)
+    echo "Invalid image name ${IMAGE_NAME}"
+    exit 1
+esac
+
+TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt)
+BUILD_DOCS=1
+
+# Copy requirements-lintrunner.txt from root to here
+cp ../../requirements-lintrunner.txt ./
+
+# Copy arm setup script from root to here
+# TODO(huydhn): Figure out a way to rebuild the Docker image automatically
+# with a new image hash when the content here is updated
+cp -r ../../examples/arm/ ./arm
+
+docker build \
+  --no-cache \
+  --progress=plain \
+  --build-arg "OS_VERSION=${OS_VERSION}" \
+  --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+  --build-arg "GCC_VERSION=${GCC_VERSION}" \
+  --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \
+  --build-arg "MINICONDA_VERSION=${MINICONDA_VERSION}" \
+  --build-arg "TORCH_VERSION=${TORCH_VERSION}" \
+  --build-arg "BUCK2_VERSION=${BUCK2_VERSION}" \
+  --build-arg "LINTRUNNER=${LINTRUNNER:-}" \
+  --build-arg "BUILD_DOCS=${BUILD_DOCS}" \
+  --build-arg "ARM_SDK=${ARM_SDK:-}" \
+  -f "${OS}"/Dockerfile \
+  "$@" \
+  .

.ci/docker/ci_commit_pins/buck2.txt

@@ -0,0 +1 @@
+2024-02-15

.ci/docker/ci_commit_pins/pytorch.txt

@@ -0,0 +1 @@
+6ca9ae4f8693639c395544327f7e362441a58c79

.ci/docker/common/install_base.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+install_ubuntu() {
+  apt-get update
+
+  apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    git \
+    wget \
+    sudo \
+    vim \
+    jq \
+    vim \
+    unzip \
+    gdb \
+    rsync \
+    libssl-dev
+
+  # Cleanup package manager
+  apt-get autoclean && apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+install_ubuntu() {
+  apt-get update
+  apt-get install -y zstd
+
+  BUCK2=buck2-x86_64-unknown-linux-gnu.zst
+  wget -q "https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/${BUCK2}"
+  zstd -d "${BUCK2}" -o buck2
+
+  chmod +x buck2
+  mv buck2 /usr/bin/
+
+  rm "${BUCK2}"
+  # Cleanup package manager
+  apt-get autoclean && apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac

.ci/docker/common/install_cache.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# NB: This script is adopted from PyTorch core repo at
+# https://github.com/pytorch/pytorch/blob/main/.ci/docker/common/install_cache.sh
+set -ex
+
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+install_binary() {
+  echo "Downloading sccache binary from S3 repo"
+  curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
+  chmod +x /opt/cache/bin/sccache
+}
+
+mkdir -p /opt/cache/bin
+sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
+export PATH="/opt/cache/bin:$PATH"
+
+# NB: Install the pre-built binary from S3 as building from source
+# https://github.com/pytorch/sccache has started failing mysteriously
+# in which sccache server couldn't start with the following error:
+#   sccache: error: Invalid argument (os error 22)
+install_binary
+
+function write_sccache_stub() {
+  BINARY=$1
+  printf "#!/bin/sh\nif [ \$(env -u LD_PRELOAD ps -p \$PPID -o comm=) != sccache ]; then\n  exec sccache %s \"\$@\"\nelse\n  exec %s \"\$@\"\nfi" "$(which "${BINARY}")" "$(which "${BINARY}")" > "/opt/cache/bin/${BINARY}"
+  chmod a+x "/opt/cache/bin/${BINARY}"
+}
+
+init_sccache() {
+  # This is the remote cache bucket
+  export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
+  export SCCACHE_S3_KEY_PREFIX=executorch
+  export SCCACHE_IDLE_TIMEOUT=0
+  export SCCACHE_ERROR_LOG=/tmp/sccache_error.log
+  export RUST_LOG=sccache::server=error
+
+  # NB: This function is adopted from PyTorch core at
+  # https://github.com/pytorch/pytorch/blob/main/.ci/pytorch/common-build.sh
+  as_ci_user sccache --stop-server > /dev/null 2>&1 || true
+  rm -f "${SCCACHE_ERROR_LOG}" || true
+
+  # Clear sccache stats before using it
+  as_ci_user sccache --zero-stats || true
+}
+
+write_sccache_stub cc
+write_sccache_stub c++
+write_sccache_stub gcc
+write_sccache_stub g++
+write_sccache_stub clang
+write_sccache_stub clang++
+init_sccache

.ci/docker/common/install_clang.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+install_ubuntu() {
+  apt-get update
+
+  apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+  apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+  # Also require LLD linker from llvm and libomp to build PyTorch from source
+  apt-get install -y lld "libomp-${CLANG_VERSION}-dev"
+
+  # Use update-alternatives to make this version the default
+  update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+  update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+  # Override cc/c++ to clang as well
+  update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+  update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+  # Cleanup package manager
+  apt-get autoclean && apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+  # Install base packages depending on the base OS
+  ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+  case "$ID" in
+    ubuntu)
+      install_ubuntu
+      ;;
+    *)
+      echo "Unable to determine OS..."
+      exit 1
+      ;;
+  esac
+fi

.ci/docker/common/install_conda.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+install_miniconda() {
+  BASE_URL="https://repo.anaconda.com/miniconda"
+  CONDA_FILE="Miniconda3-py${PYTHON_VERSION//./}_${MINICONDA_VERSION}-Linux-x86_64.sh"
+
+  mkdir -p /opt/conda
+  chown ci-user:ci-user /opt/conda
+
+  pushd /tmp
+  wget -q "${BASE_URL}/${CONDA_FILE}"
+  # Install miniconda
+  as_ci_user bash "${CONDA_FILE}" -b -f -p "/opt/conda"
+  # Clean up the download file
+  rm "${CONDA_FILE}"
+  popd
+
+  sed -e 's|PATH="\(.*\)"|PATH="/opt/conda/bin:\1"|g' -i /etc/environment
+  export PATH="/opt/conda/bin:$PATH"
+}
+
+install_python() {
+  pushd /opt/conda
+  # Install the correct Python version
+  as_ci_user conda create -n "py_${PYTHON_VERSION}" -y --file /opt/conda/conda-env-ci.txt python="${PYTHON_VERSION}"
+  popd
+}
+
+install_pip_dependencies() {
+  pushd /opt/conda
+  # Install all Python dependencies, including PyTorch
+  pip_install -r /opt/conda/requirements-ci.txt
+  popd
+}
+
+fix_conda_ubuntu_libstdcxx() {
+  cat /etc/issue
+  # WARNING: This is a HACK from PyTorch core to be able to build PyTorch on 22.04.
+  # Specifically, ubuntu-20+ all comes lib libstdc++ newer than 3.30+, but anaconda
+  # is stuck with 3.29. So, remove libstdc++6.so.3.29 as installed by
+  # https://anaconda.org/anaconda/libstdcxx-ng/files?version=11.2.0
+  #
+  # PyTorch sev: https://github.com/pytorch/pytorch/issues/105248
+  # Ref: https://github.com/pytorch/pytorch/blob/main/.ci/docker/common/install_conda.sh
+  if grep -e "2[02].04." /etc/issue >/dev/null; then
+    rm "/opt/conda/envs/py_${PYTHON_VERSION}/lib/libstdc++.so.6"
+  fi
+}
+
+install_miniconda
+install_python
+install_pip_dependencies
+fix_conda_ubuntu_libstdcxx