diff --git a/build_dev_binpkgs b/build_dev_binpkgs new file mode 100755 index 00000000000..37e0cc50bd2 --- /dev/null +++ b/build_dev_binpkgs @@ -0,0 +1,88 @@ +#!/bin/bash +# Copyright (c) 2023 by the Flatcar Maintainers. +# Use of this source code is governed by the Apache 2.0 license. + +. "$(dirname "$0")/common.sh" || exit 1 + +# Script must run inside the chroot +assert_inside_chroot +assert_not_root_user + +# Dependencies and packages to include by default. +packages_default=( "coreos-devel/board-packages" ) + +# Packages that are rdeps of the above but should not be included. +# (mostly large packages, e.g. programming languages etc.) +skip_packages_default="dev-lang/rust,virtual/rust,dev-lang/go,dev-lang/go-bootstrap,dev-go/go-md2man" + + +# Developer-visible flags. +DEFINE_string board "${DEFAULT_BOARD}" \ + "The board to build packages for." +DEFINE_string skip_packages "${skip_packages_default[@]}" \ + "Comma-separated list of packages in the dependency tree to skip." +DEFINE_boolean pretend "${FLAGS_FALSE}" \ + "List packages that would be built but do not actually build." + +FLAGS_HELP="usage: $(basename $0) [flags] [packages] + +build_dev_binpkgs builds binary packages for all dependencies of [packages] +that are not present in '/build//var/lib/portage/pkgs/'. +Useful for publishing a complete set of packages to a binhost. + +[packages] defaults to '${packages_default}' if not specified. +" + +# Parse command line +FLAGS "$@" || exit 1 +eval set -- "${FLAGS_ARGV}" + +# Die on any errors. +switch_to_strict_mode + +if [[ $# -eq 0 ]]; then + set -- "${packages_default[@]}" +fi +# -- + +function my_board_emerge() { + PORTAGE_CONFIGROOT="/build/${FLAGS_board}" SYSROOT="${SYSROOT:-/build/${FLAGS_board}}" ROOT="/build/${FLAGS_board}" sudo -E emerge --root-deps=rdeps "${@}" +} +# -- + +pkg_build_list="$(mktemp)" +pkg_skipped_list="${pkg_build_list}-skip" +trap 'rm -f "${pkg_build_list}" "${pkg_skipped_list}"' EXIT + +info "Collecting list of binpkgs to build" + +my_board_emerge --pretend --root-deps=rdeps --emptytree ${@} \ + | grep '\[ebuild' \ + | sed 's/^\[[^]]\+\] \([^ :]\+\)*:.*/\1/' \ + | while read pkg; do + if [ -f "/build/${FLAGS_board}/var/lib/portage/pkgs/${pkg}.tbz2" ] ; then + continue + fi + skip="" + for s in ${FLAGS_skip_packages//,/ }; do + if [[ ${pkg} = ${s}-* ]] ; then + echo -n "${pkg} " >> "${pkg_skipped_list}" + skip="true" + break + fi + done + [[ -z ${skip} ]] || continue + echo "=${pkg}" | tee -a "${pkg_build_list}" | sed 's/^/ /' +done +# -- + +if [ -f "${pkg_skipped_list}" ] ; then + info "Skipping binpkgs '$(cat "${pkg_skipped_list}")' because these are in the skip list." +fi + +pretend="" +if [[ "${FLAGS_pretend}" -eq "${FLAGS_TRUE}" ]]; then + pretend="--pretend" +fi + +my_board_emerge --buildpkg ${pretend} $(cat "${pkg_build_list}") diff --git a/build_packages b/build_packages index 1ad8800f48c..edc7b8967cb 100755 --- a/build_packages +++ b/build_packages @@ -42,6 +42,8 @@ DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \ "Don't build a new torcx store from the updated sysroot." DEFINE_string torcx_extra_pkg_url "" \ "URL to directory where the torcx packages will be available for downloading" +DEFINE_boolean only_resolve_circular_deps "${FLAGS_FALSE}" \ + "Don't build all packages; only resolve circular dependencies, then stop." # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -278,6 +280,11 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then sys-apps/systemd cryptsetup fi +if [[ "${FLAGS_only_resolve_circular_deps}" -eq "${FLAGS_TRUE}" ]]; then + info "Circular dependencies resolved. Stopping as requested." + exit +fi + export KBUILD_BUILD_USER="${BUILD_USER:-build}" export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}" diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh index 86439c599b7..46b177f437c 100644 --- a/ci-automation/push_pkgs.sh +++ b/ci-automation/push_pkgs.sh @@ -12,6 +12,10 @@ # This script will publish the packages from a pre-built packages container to # the buildcache server, effectively turning the build cache into a # binary packages server for the SDK. +# Before pushing packages the script will run ./build_dev_binpkgs to ensure all +# binary packages for the development container are actually built. +# Note that this may build packages not previously built by the "build_packages" +# step. # # PREREQUISITES: # @@ -88,11 +92,14 @@ function _push_packages_impl() { docker_image_from_buildcache "${packages}" "${docker_vernum}" + local my_name="flatcar-packages-publisher-${arch}-${docker_vernum}" + ./run_sdk_container -x ./ci-cleanup.sh -n "${my_name}" -C "${packages_image}" \ + ./build_dev_binpkgs --board="${arch}-usr" + local cmd="source ci-automation/push_pkgs.sh" cmd="$cmd; image_build__copy_to_bincache '$arch' '$vernum'" - local my_name="flatcar-packages-publisher-${arch}-${docker_vernum}" - ./run_sdk_container -x ./ci-cleanup.sh -n "${my_name}" -C "${packages_image}" \ + ./run_sdk_container -x ./ci-cleanup.sh -n "${my_name}" \ bash -c "$cmd" } # --