Skip to content

Commit

Permalink
Merge pull request #2115 from flatcar/chewi/catalyst-4
Browse files Browse the repository at this point in the history
Upgrade to Catalyst 4 and don't use repo snapshots for stage1
  • Loading branch information
chewi authored Jul 15, 2024
2 parents 354a0f2 + 186f311 commit 7c23476
Show file tree
Hide file tree
Showing 43 changed files with 580 additions and 423 deletions.
133 changes: 13 additions & 120 deletions bootstrap_sdk
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,19 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# This uses Gentoo's catalyst for very thoroughly building images from
# scratch. Using images based on this will eliminate some of the hackery
# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
#
# This uses Gentoo's catalyst for very thoroughly building images from scratch.
# For reference the procedure it performs is this:
#
# 1. snapshot: Grab a snapshot of the portage-stable repo from
# the current SDK's /var/lib/gentoo/repos/gentoo.
# Alternatively, check out a git ref specified via --portage-ref.
#
# 2. stage1: Using a "seed" tarball as a build environment, build a
# minimal root file system into a clean directory using ROOT=...
# and USE=-* The restricted USE flags are key be small and avoid
# circular dependencies.
# 1. seed: Take a recent SDK, dev container, or custom tarball as a seed to
# build stage 1 with. Before proceeding, update relevant packages that have
# changed sub-slot to avoid missing library issues later in the build.
#
# 2. stage1: Using the above seed tarball as a build environment, build a
# minimal root file system into a clean directory using ROOT=... and USE=-*
# The restricted USE flags are key be small and avoid circular dependencies.
# NOTE that stage1 LACKS PROPER STAGE ISOLATION. Binaries produced in stage1
# will be linked against the SEED SDK libraries, NOT against libraries
# built in stage 1. See "stage_repo()" documentation further below for more.
# This stage uses:
# - portage-stable from the SDK's /var/lib/gentoo/repos/gentoo
# or a custom path via --stage1_portage_path command line option
# - coreos-overlay from the SDK's /var/lib/gentoo/repos/coreos-overlay
# or a custom path via --stage1_overlay_path command line option
# Command line option refs need caution though, since
# stage1 must not contain updated ebuilds (see build_stage1 below).
# will be linked against the SEED SDK libraries, NOT against libraries built
# in stage 1.
#
# 3. stage2: Run portage-stable/scripts/bootstrap.sh
# This rebuilds the toolchain using Gentoo bootstrapping, ensuring it's not linked
Expand Down Expand Up @@ -59,24 +48,17 @@ TYPE="flatcar-sdk"
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1


DEFINE_string stage1_portage_path "" \
"Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
DEFINE_string stage1_overlay_path "" \
"Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"


## Define the stage4 config template
catalyst_stage4() {
cat <<EOF
target: stage4
pkgcache_path: $BINPKGS
stage4/packages: coreos-devel/sdk-depends
stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_sdk.sh
stage4/root_overlay: ${ROOT_OVERLAY}
stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb
stage4/empty: /root /var/cache/edb
stage4/rm: /etc/machine-id /etc/resolv.conf
EOF
catalyst_stage_default
catalyst_stage_default 4
}

# Switch to HTTP because early boostrap stages do not have SSL support.
Expand Down Expand Up @@ -118,7 +100,7 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
#
# No package updates must happen in stage 1, so we use the portage-stable and
# coreos-overlay paths included with the current SDK (from the SDK chroot's
# /var/lib/gentoo/repos/). "Current SDK" refers to the SDK we entered with
# /var/gentoo/repos/). "Current SDK" refers to the SDK we entered with
# 'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
#
# Using ebuilds from the above mentioned sources will ensure that stage 1 builds
Expand All @@ -143,95 +125,6 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
# by stage one. Therefore, these binaries will cease to work in stage 2 when linked against
# outdated "seed tarball" libraries which have been updated to newer versions in stage 1.

stage_repo() {
local repo=${1}
local path=${2}
local dest=${3}
local update_seed_file=${4}
local gitname="$repo"

if [ "$gitname" = "gentoo" ] ; then
gitname="portage-stable"
fi

if [ -z "$path" ]; then
cp -R "/var/gentoo/repos/${repo}" "$dest"
info "Using local SDK's ebuild repo '$repo' ('$gitname') in stage 1."
else
mkdir "$dest/$repo"
cp -R "${path}/"* "$dest/${repo}/"
info "Using custom path '$path' for ebuild repo '$repo' ('$gitname') in stage 1."
info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it."
fi
(
set -euo pipefail
local repo_var hook name

# FLAGS_coreos_overlay for gitname coreos-overlay
repo_var="FLAGS_${gitname//-/_}"
shopt -s nullglob
for hook in "${FLAGS_coreos_overlay}/coreos/stage1_hooks/"*"-${gitname}.sh"; do
name=${hook##*/}
name=${name%"-${gitname}.sh"}
info "Invoking stage1 ${gitname} hook ${name} on ${dest}/${repo}"
"${hook}" "${dest}/${repo}" "${!repo_var}" "${update_seed_file}"
done
)
}

build_stage1() {
# First, write out the default 4-stage catalyst configuration files
write_configs

# Prepare local copies of both the "known-good" portage-stable and the
# "known-good" coreos-overlay ebuild repos
local stage1_repos="$TEMPDIR/stage1-ebuild-repos"
info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
rm -rf "$stage1_repos"
mkdir "$stage1_repos"

# If the file exists and is not empty, seed will be updated.
# Stage1 hooks may decide that the seed SDK needs updating.
local update_seed_file="${TEMPDIR}/update_seed"

# prepare ebuild repos for stage 1, either from the local SDK (default)
# or from custom paths specified via command line flags
stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos" "${update_seed_file}"
stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos" "${update_seed_file}"

# Create a snapshot of "known-good" portage-stable repo copy for use in stage 1
# This requires us to create a custom catalyst config to point it to the
# repo copy we just created, for snapshotting.
catalyst_conf > "$TEMPDIR/catalyst-stage1.conf"
sed -i "s:^portdir.*:portdir=\"$stage1_repos/gentoo\":" \
"$TEMPDIR/catalyst-stage1.conf"
# take the "portage directory" (portage-stable copy) snapshot
build_snapshot "${TEMPDIR}/catalyst-stage1.conf" "${FLAGS_version}-stage1"

# Update the stage 1 spec to use the "known-good" portage-stable snapshot
# and coreos-overlay copy repository versions from above.
sed -i -e "s/^snapshot:.*/snapshot: $FLAGS_version-stage1/" \
-e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \
"$TEMPDIR/stage1.spec"

# If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
if [[ -n ${FLAGS_stage1_portage_path} ]] || [[ -n ${FLAGS_stage1_overlay_path} ]] || [[ -s ${update_seed_file} ]]; then
sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
>>"$TEMPDIR/stage1.spec"
fi
rm -f "${update_seed_file}"

# Finally, build stage 1
build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf"
}

if [[ "$STAGES" =~ stage1 ]]; then
build_stage1
STAGES="${STAGES/stage1/}"
SEED="${TYPE}/stage1-${ARCH}-latest"
fi

catalyst_build

if [[ "$STAGES" =~ stage4 ]]; then
Expand Down
5 changes: 2 additions & 3 deletions build_library/build_image_util.sh
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ get_metadata() {
local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
if [ -n "${mirror}" ]; then
# Take only first mirror, those not working should be removed
local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/third_party/portage-stable/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
fi
new_val+="${v} "
Expand Down Expand Up @@ -490,8 +490,7 @@ EOF
license_list="$(jq -r '.[] | "\(.licenses | .[])"' "${json_input}" | sort | uniq)"
local license_dirs=(
"/mnt/host/source/src/third_party/coreos-overlay/licenses/"
"/mnt/host/source/src/third_party/portage-stable/"
"/var/gentoo/repos/gentoo/licenses/"
"/mnt/host/source/src/third_party/portage-stable/licenses/"
"none"
)
for license_file in ${license_list}; do
Expand Down
Loading

0 comments on commit 7c23476

Please sign in to comment.