diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 214c0f46fc7..25ac63be6c2 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -19,6 +19,8 @@ fi BUILD_DIR="${FLAGS_output_root}/${BOARD}/${IMAGE_SUBDIR}" OUTSIDE_OUTPUT_DIR="../build/images/${BOARD}/${IMAGE_SUBDIR}" +source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 + set_build_symlinks() { local build=$(basename ${BUILD_DIR}) local link @@ -239,65 +241,6 @@ systemd_enable() { sudo ln -sf "../${unit_file}" "${wants_dir}/${unit_alias}" } -# Generate a ls-like listing of a directory tree. -# The ugly printf is used to predictable time format and size in bytes. -write_contents() { - info "Writing ${2##*/}" - pushd "$1" >/dev/null - # %M - file permissions - # %n - number of hard links to file - # %u - file's user name - # %g - file's group name - # %s - size in bytes - # %Tx - modification time (Y - year, m - month, d - day, H - hours, M - minutes) - # %P - file's path - # %l - symlink target (empty if not a symlink) - sudo TZ=UTC find -printf \ - '%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \ - | sed -e 's/ -> $//' > "$2" - popd >/dev/null -} - -# Generate a listing that can be used by other tools to analyze -# image/file size changes. -write_contents_with_technical_details() { - info "Writing ${2##*/}" - pushd "$1" >/dev/null - # %M - file permissions - # %D - ID of a device where file resides - # %i - inode number - # %n - number of hard links to file - # %s - size in bytes - # %P - file's path - sudo find -printf \ - '%M %D %i %n %s ./%P\n' > "$2" - popd >/dev/null -} - -# Generate a report like the following: -# -# File Size Used Avail Use% Type -# /boot 127M 62M 65M 50% vfat -# /usr 983M 721M 212M 78% ext2 -# / 6,0G 13M 5,6G 1% ext4 -# SUM 7,0G 796M 5,9G 12% - -write_disk_space_usage() { - info "Writing ${2##*/}" - pushd "${1}" >/dev/null - # The sed's first command turns './' into '/ ', second - # command replaces '- ' with 'SUM' for the total row. All this to - # keep the numbers neatly aligned in columns. - sudo df \ - --human-readable \ - --total \ - --output='file,size,used,avail,pcent,fstype' \ - ./boot ./usr ./ | \ - sed \ - -e 's#^\.\(/[^ ]*\)#\1 #' \ - -e 's/^- /SUM/' >"${2}" - popd >/dev/null -} - # "equery list" a potentially uninstalled board package query_available_package() { local pkg="$1" diff --git a/build_library/disk_util b/build_library/disk_util index f5fdd6d6fdc..e47f0932a23 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -743,18 +743,29 @@ def Tune(options): config, partitions = LoadPartitionConfig(options) GetPartitionTableFromImage(options, config, partitions) part = GetPartition(partitions, options.partition) + action_done = False if not part['image_compat']: raise InvalidLayout("Disk layout is incompatible with existing image") if options.disable2fs_rw is not None: + action_done = True if part.get('fs_type', None) in ('ext2', 'ext4'): Tune2fsReadWrite(options, part, options.disable2fs_rw) elif part.get('fs_type', None) == 'btrfs': ReadWriteSubvol(options, part, options.disable2fs_rw) else: raise Exception("Partition %s is not a ext2 or ext4 or btrfs" % options.partition) - else: + + if options.randomize_uuid is not None: + action_done = True + if part.get('fs_type', None) == 'btrfs': + with PartitionLoop(options, part) as loop_dev: + Sudo(['btrfstune', '-m', loop_dev]) + else: + raise Exception("Partition %s is not btrfs" % options.partition) + + if not action_done: raise Exception("No options specified!") @@ -1059,6 +1070,8 @@ def main(argv): help='disable mounting ext2 filesystems read-write') a.add_argument('--enable2fs_rw', action='store_false', dest='disable2fs_rw', help='re-enable mounting ext2 filesystems read-write') + a.add_argument('--randomize_uuid', action='store_true', default=None, + help='randomize btrfs UUIDs in the partition') a.add_argument('disk_image', help='path to disk image file') a.add_argument('partition', help='number or label of partition to edit') a.set_defaults(func=Tune) diff --git a/build_library/oem_sysext_util.sh b/build_library/oem_sysext_util.sh new file mode 100755 index 00000000000..21e014db481 --- /dev/null +++ b/build_library/oem_sysext_util.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# +# Copyright (c) 2023 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 + +_generate_listing() { + local rootfs="${1%/}"; shift + local listing="${1}"; shift + + local slashes="${rootfs//[^\/]}" + local slash_count="${#slashes}" + + # Invoking find with sudo as it's used for traversing root-owned + # rootfs, which means that some places may be unreachable by the + # sdk user. + sudo find "${rootfs}//" | cut -d/ -f$((slash_count + 2))- | sort >"${listing}" +} + +_prepend_action () { + local -n prepend_array="${1}"; shift + + prepend_array=( "${#}" "${@}" "${prepend_array[@]}" ) +} + +_invoke_actions () { + local arg_count + local command + while [[ "${#}" -gt 0 ]]; do + arg_count="${1}" + shift + command=( "${@:1:${arg_count}}" ) + shift "${arg_count}" + "${command[@]}" || : + done +} + +# Architecture values are taken from systemd.unit(5). +declare -A SYSEXT_ARCHES +SYSEXT_ARCHES['amd64-usr']='x86-64' +SYSEXT_ARCHES['arm64-usr']='arm64' + +declare -r SYSEXT_ARCHES + +# Usage: _get_sysext_arch board [board...] +_get_sysext_arch() { + local board + for board in "$@"; do + if [[ ${#SYSEXT_ARCHES["${board}"]} -ne 0 ]]; then + echo "${SYSEXT_ARCHES["${board}"]}" + else + die "Unknown board '${board}'" + fi + done +} + +oem_sysext_create() { + local oem="${1}"; shift + local board="${1}"; shift + local version_id="${1}"; shift + local prod_image="${1}"; shift + local prod_pkgdb="${1}"; shift + local work_dir="${1}"; shift + + local base_pkg="coreos-base/${oem}" + local sysext_work_dir="${work_dir}/sysext-${oem}" + local prod_rw_image="${sysext_work_dir}/prod_for_sysext.bin" + local prod_rw_rootfs="${sysext_work_dir}/prod_rw_rootfs" + + local cleanup_actions=() + trap '_invoke_actions "${cleanup_actions[@]}"' EXIT + + _prepend_action cleanup_actions rmdir "${sysext_work_dir}" + mkdir -p "${sysext_work_dir}" + + info 'Creating a production image copy for work rootfs' + _prepend_action cleanup_actions rm -f "${prod_rw_image}" + cp --sparse=always "${prod_image}" "${prod_rw_image}" + + info 'Preparing work image for mounting' + "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ + tune --randomize_uuid "${prod_rw_image}" OEM + "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ + tune --enable2fs_rw "${prod_rw_image}" USR-A + + info "Mounting work image to ${prod_rw_rootfs}" + _prepend_action cleanup_actions rmdir "${prod_rw_rootfs}" + _prepend_action cleanup_actions "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ + umount "${prod_rw_rootfs}" + "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ + mount --writable_verity "${prod_rw_image}" "${prod_rw_rootfs}" + + local initial_files="${sysext_work_dir}/initial_files" + info "Generating list of initial files in work image" + _prepend_action cleanup_actions rm -f "${initial_files}" + _generate_listing "${prod_rw_rootfs}" "${initial_files}" + + info "Stuffing package database into into ${prod_rw_rootfs}" + sudo tar -xf "${prod_pkgdb}" -C "${prod_rw_rootfs}" + + # Split into two steps because we want to always install + # $${base_pkg} from the ebuild (build_packages doesn't handle it) + # *but* we never want to build anything else from source + # here. emerge doesn't have a way to enforce this in a single + # command. + info "Building ${base_pkg}" + "emerge-${board}" --nodeps --buildpkgonly --usepkg n --verbose "${base_pkg}" + + info "Installing ${base_pkg} to ${prod_rw_rootfs}" + sudo emerge \ + --config-root="/build/${board}" \ + --root="${prod_rw_rootfs}" \ + --sysroot="${prod_rw_rootfs}" \ + --root-deps=rdeps \ + --usepkgonly \ + --verbose \ + "${base_pkg}" + + info "Removing portage db from ${prod_rw_rootfs}" + sudo rm -rf \ + "${prod_rw_rootfs}/var/cache/edb" \ + "${prod_rw_rootfs}/var/db/pkg" + + local all_files="${sysext_work_dir}/all_files" + local sysext_files="${sysext_work_dir}/sysext_files" + + info "Generating list of files in work image after installing OEM package" + _prepend_action cleanup_actions rm -f "${all_files}" + _generate_listing "${prod_rw_rootfs}" "${all_files}" + + info "Generating list of files for sysext image" + _prepend_action cleanup_actions rm -f "${sysext_files}" + comm -1 -3 "${initial_files}" "${all_files}" >"${sysext_files}" + + info "Copying files for sysext image" + local sysext_rootfs="${sysext_work_dir}/sysext_rootfs" + _prepend_action cleanup_actions rm -rf "${sysext_rootfs}" + rsync --links --files-from="${sysext_files}" "${prod_rw_rootfs}" "${sysext_rootfs}" + + info "Mangling files for sysext image" + local overlay_path mangle_fs + overlay_path=$(portageq get_repo_path / coreos) + mangle_fs="${overlay_path}/${base_pkg}/files/manglefs.sh" + if [[ -x "${mangle_fs}" ]]; then + "${mangle_fs}" "${sysext_rootfs}" + fi + + local entry + info "Removing non-/usr directories from sysext image" + for entry in "${sysext_rootfs}"/*; do + if [[ "${entry}" = */usr ]]; then + continue + fi + info " Removing ${entry##*/}" + rm -rf "${entry}" + done + + local metadata metadata_file metadata_version_entry + info "Adding sysext metadata" + mkdir -p "${sysext_rootfs}/usr/lib/extension-release.d" + if [[ "${version_id}" = 'initial' ]]; then + metadata_version_entry="SYSEXT_LEVEL=1.0" + else + metadata_version_entry="VERSION_ID=${version_id}" + fi + metadata=( + 'ID=flatcar' + "${metadata_version_entry}" + "ARCHITECTURE=$(_get_sysext_arch "${board}")" + ) + metadata_file="${sysext_rootfs}/usr/lib/extension-release.d/extension-release.${oem}" + printf '%s\n' "${metadata[@]}" >"${metadata_file}" + + info "Generating a squashfs image" + local sysext_raw_image_filename="${oem}.raw" + local output_raw_image="${sysext_work_dir}/${sysext_raw_image_filename}" + _prepend_action cleanup_actions rm -f "${output_raw_image}" + mksquashfs "${sysext_rootfs}" "${output_raw_image}" -all-root + + info "Generating image reports" + local sysext_mounted="${sysext_work_dir}/squashfs_mounted" + _prepend_action cleanup_actions rmdir "${sysext_mounted}" + mkdir "${sysext_mounted}" + _prepend_action cleanup_actions sudo umount "${sysext_mounted}" + sudo mount -t squashfs -o loop "${output_raw_image}" "${sysext_mounted}" + local contents="${sysext_raw_image_filename%.raw}_contents.txt" + local contents_wtd="${sysext_raw_image_filename%.raw}_contents_wtd.txt" + local disk_usage="${sysext_raw_image_filename%.raw}_disk_usage.txt" + _prepend_action cleanup_actions rm -f "${sysext_work_dir}/${contents}" + write_contents "${sysext_mounted}" "${sysext_work_dir}/${contents}" + _prepend_action cleanup_actions rm -f "${sysext_work_dir}/${contents_wtd}" + write_contents_with_technical_details "${sysext_mounted}" "${sysext_work_dir}/${contents_wtd}" + _prepend_action cleanup_actions rm -f "${sysext_work_dir}/${disk_usage}" + write_disk_space_usage_in_paths "${sysext_mounted}" "${sysext_work_dir}/${disk_usage}" + + local to_move + for to_move in "${sysext_raw_image_filename}" "${contents}" "${contents_wtd}" "${disk_usage}"; do + mv "${sysext_work_dir}/${to_move}" "${work_dir}/${to_move}" + done + + info "Alles jut, cleaning up" + trap - EXIT + _invoke_actions "${cleanup_actions[@]}" +} diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index bc2a39e182c..81457127055 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -77,6 +77,7 @@ create_prod_image() { local image_initrd_contents="${image_name%.bin}_initrd_contents.txt" local image_initrd_contents_wtd="${image_name%.bin}_initrd_contents_wtd.txt" local image_disk_usage="${image_name%.bin}_disk_usage.txt" + local image_pkgdb="${image_name%.bin}_pkgdb.tar.xz" start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" @@ -100,6 +101,8 @@ create_prod_image() { || die_notrace "coreos-au-key is missing the 'official' use flag" fi + tar -cf "${BUILD_DIR}/${image_pkgdb}" -C "${root_fs_dir}" var/cache/edb var/db/pkg + # clean-ups of things we do not need sudo rm ${root_fs_dir}/etc/csh.env sudo rm -rf ${root_fs_dir}/etc/env.d diff --git a/build_library/reports_util.sh b/build_library/reports_util.sh new file mode 100644 index 00000000000..0873d9e960b --- /dev/null +++ b/build_library/reports_util.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# +# Copyright (c) 2023 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +if [[ -n "${FLATCAR_REPORTS_UTIL_SH_INCLUDED:-}" ]]; then + return 0 +fi + +FLATCAR_REPORTS_UTIL_SH_INCLUDED=1 + +# Generate a ls-like listing of a directory tree. +# The ugly printf is used to predictable time format and size in bytes. +# +# Usage: +# write_contents "${rootfs}" ${contents_file}" +write_contents() { + local rootfs="${1}"; shift + local output="${1}"; shift + info "Writing ${output##*/}" + # Ensure output is an absolute path before we change the working + # directory. + output=$(realpath "${output}") + pushd "${rootfs}" >/dev/null + # %M - file permissions + # %n - number of hard links to file + # %u - file's user name + # %g - file's group name + # %s - size in bytes + # %Tx - modification time (Y - year, m - month, d - day, H - hours, M - minutes) + # %P - file's path + # %l - symlink target (empty if not a symlink) + sudo TZ=UTC find -printf \ + '%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \ + | sed -e 's/ -> $//' >"${output}" + popd >/dev/null +} + +# Generate a listing that can be used by other tools to analyze +# image/file size changes. +# +# Usage: +# write_contents_with_technical_details "${rootfs}" ${output_file}" +write_contents_with_technical_details() { + local rootfs="${1}"; shift + local output="${1}"; shift + info "Writing ${output##*/}" + # Ensure output is an absolute path before we change the working + # directory. + output=$(realpath "${output}") + pushd "${rootfs}" >/dev/null + # %M - file permissions + # %D - ID of a device where file resides + # %i - inode number + # %n - number of hard links to file + # %s - size in bytes + # %P - file's path + sudo find -printf \ + '%M %D %i %n %s ./%P\n' >"${output}" + popd >/dev/null +} + +# Generate a report like the following if more than one relative path +# in rootfs was passed: +# +# File Size Used Avail Use% Type +# /boot 127M 62M 65M 50% vfat +# /usr 983M 721M 212M 78% ext2 +# / 6,0G 13M 5,6G 1% ext4 +# SUM 7,0G 796M 5,9G 12% - +# +# or, in case of 0 or 1 relative path: +# +# File Size Used Avail Use% Type +# / 27M 27M 0 100% squashfs +# +# Usage: +# write_disk_space_usage_in_paths "${rootfs}" "${output_file}" ./boot ./usr ./ +write_disk_space_usage_in_paths() { + local rootfs="${1}"; shift + local output="${1}"; shift + info "Writing ${output##*/}" + # Ensure output is an absolute path before we change the working + # directory. + output=$(realpath "${output}") + pushd "${rootfs}" >/dev/null + local extra_flags + extra_flags=() + if [[ ${#} -eq 0 ]]; then + set -- ./ + fi + if [[ ${#} -gt 1 ]]; then + extra_flags+=('--total') + fi + # The sed's first command turns './' into '/ ', second + # command replaces '- ' with 'SUM' for the total row. All this to + # keep the numbers neatly aligned in columns. + sudo df \ + --human-readable \ + "${extra_flags[@]}" \ + --output='file,size,used,avail,pcent,fstype' \ + "${@}" | \ + sed \ + -e 's#^\.\(/[^ ]*\)#\1 #' \ + -e 's/^- /SUM/' >"${output}" + popd >/dev/null +} + +# Generate a report like the following: +# +# File Size Used Avail Use% Type +# /boot 127M 62M 65M 50% vfat +# /usr 983M 721M 212M 78% ext2 +# / 6,0G 13M 5,6G 1% ext4 +# SUM 7,0G 796M 5,9G 12% - +write_disk_space_usage() { + write_disk_space_usage_in_paths "${1}" "${2}" ./boot ./usr ./ +} diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 9738565aa54..802f0aa1378 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -71,6 +71,7 @@ VM_IMG_TYPE=DEFAULT # Set at runtime to the source and destination image paths VM_SRC_IMG= +VM_SRC_PKGDB= VM_TMP_IMG= VM_TMP_DIR= VM_TMP_ROOT= @@ -98,6 +99,12 @@ IMG_DEFAULT_OEM_USE= # Forced USE flags for the OEM package IMG_FORCE_OEM_USE= +# If set install the given package name to the OEM sysext image +IMG_DEFAULT_OEM_SYSEXT= + +# Forced OEM package name overriding what may be in the format +IMG_FORCE_OEM_SYSEXT= + # Hook to do any final tweaks or grab data while fs is mounted. IMG_DEFAULT_FS_HOOK= @@ -124,17 +131,23 @@ IMG_DEFAULT_CPUS=2 IMG_qemu_DISK_FORMAT=qcow2 IMG_qemu_DISK_LAYOUT=vm IMG_qemu_CONF_FORMAT=qemu -IMG_qemu_OEM_PACKAGE=oem-qemu +IMG_qemu_OEM_USE=qemu +IMG_qemu_OEM_PACKAGE=common-oem-files +IMG_qemu_OEM_SYSEXT=oem-qemu IMG_qemu_uefi_DISK_FORMAT=qcow2 IMG_qemu_uefi_DISK_LAYOUT=vm IMG_qemu_uefi_CONF_FORMAT=qemu_uefi -IMG_qemu_uefi_OEM_PACKAGE=oem-qemu +IMG_qemu_uefi_OEM_USE=qemu +IMG_qemu_uefi_OEM_PACKAGE=common-oem-files +IMG_qemu_uefi_OEM_SYSEXT=oem-qemu IMG_qemu_uefi_secure_DISK_FORMAT=qcow2 IMG_qemu_uefi_secure_DISK_LAYOUT=vm IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure -IMG_qemu_uefi_secure_OEM_PACKAGE=oem-qemu +IMG_qemu_uefi_secure_OEM_USE=qemu +IMG_qemu_uefi_secure_OEM_PACKAGE=common-oem-files +IMG_qemu_uefi_secure_OEM_SYSEXT=oem-qemu ## xen IMG_xen_CONF_FORMAT=xl @@ -272,7 +285,9 @@ IMG_exoscale_OEM_PACKAGE=oem-exoscale ## azure IMG_azure_DISK_FORMAT=vhd_fixed IMG_azure_DISK_LAYOUT=azure -IMG_azure_OEM_PACKAGE=oem-azure +IMG_azure_OEM_USE=azure +IMG_azure_OEM_PACKAGE=common-oem-files +IMG_azure_OEM_SYSEXT=oem-azure ## hyper-v IMG_hyperv_DISK_FORMAT=vhd @@ -345,13 +360,18 @@ set_vm_oem_pkg() { # Validate and set source vm image path set_vm_paths() { - local src_dir="$1" - local dst_dir="$2" - local src_name="$3" + local src_dir="${1}"; shift + local dst_dir="${1}"; shift + local src_name="${1}"; shift + local pkgdb_name="${1}"; shift VM_SRC_IMG="${src_dir}/${src_name}" if [[ ! -f "${VM_SRC_IMG}" ]]; then - die "Source image does not exist: $VM_SRC_IMG" + die "Source image does not exist: ${VM_SRC_IMG}" + fi + VM_SRC_PKGDB="${src_dir}/${pkgdb_name}" + if [[ ! -f "${VM_SRC_PKGDB}" ]]; then + die "Source package database does not exist: ${VM_SRC_PKGDB}" fi local dst_name="$(_src_to_dst_name "${src_name}" "_image.$(_disk_ext)")" @@ -517,6 +537,53 @@ install_oem_aci() { rm -rf "${aci_dir}" } +# Write the OEM sysext file into the OEM partition. +install_oem_sysext() { + local oem_sysext=$(_get_vm_opt OEM_SYSEXT) + + if [[ -z "${oem_sysext}" ]]; then + return 0 + fi + + local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext" + local built_sysext_filename="${oem_sysext}.raw" + local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}" + local build_oem_sysext_flags=( + --board="${BOARD}" + --build_dir="${built_sysext_dir}" + --prod_image_path="${VM_SRC_IMG}" + --prod_pkgdb_path="${VM_SRC_PKGDB}" + # TODO: Drop this when we implement updating OEM sysexts. + --version_id=initial + ) + + "${SCRIPT_ROOT}/build_oem_sysext" "${build_oem_sysext_flags[@]}" "${oem_sysext}" + + local installed_sysext_oem_dir='/oem/sysext' + local installed_sysext_file_prefix="${oem_sysext}-${FLATCAR_VERSION}" + local installed_sysext_filename="${installed_sysext_file_prefix}.raw" + local installed_sysext_abspath="${installed_sysext_oem_dir}/${installed_sysext_filename}" + info "Installing ${oem_sysext} sysext" + sudo install -Dpm 0644 \ + "${built_sysext_path}" \ + "${VM_TMP_ROOT}${installed_sysext_abspath}" || + die "Could not install ${oem_sysext} sysext" + # Move sysext image and reports to a destination directory to + # upload them, thus making them available as separate artifacts to + # download. + local upload_dir to_move + upload_dir="$(_dst_dir)" + for to_move in "${built_sysext_dir}/${oem_sysext}"*; do + mv "${to_move}" "${upload_dir}/${to_move##*/}" + done + # Remove sysext_dir if building sysext and installing it + # succeeded. + rm -rf "${built_sysext_dir}" + + # Mark the installed sysext as active. + sudo touch "${VM_TMP_ROOT}${installed_sysext_oem_dir}/active-${oem_sysext}" +} + # Any other tweaks required? run_fs_hook() { local fs_hook=$(_get_vm_opt FS_HOOK) diff --git a/build_oem_sysext b/build_oem_sysext new file mode 100755 index 00000000000..ece37ebd8f8 --- /dev/null +++ b/build_oem_sysext @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Copyright (c) 2023 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +SCRIPT_ROOT=$(dirname $(readlink -f "$0")) +. "${SCRIPT_ROOT}/common.sh" || exit 1 + +# Script must run inside the chroot +assert_inside_chroot + +assert_not_root_user + +# Developer-visible flags. +DEFINE_string board "${DEFAULT_BOARD}" \ + "The board to build an image for." +DEFINE_string build_dir "" \ + "Directory in which to place image result directories (named by version)" +DEFINE_string prod_image_path "" \ + "Path to the generic production image" +DEFINE_string prod_pkgdb_path "" \ + "Path to the tarball with portage package database from generic image production image" +DEFINE_string version_id "${FLATCAR_VERSION_ID}" \ + "Version ID stored inside the sysext extension" + +FLAGS_HELP="USAGE: build_oem_sysext [flags] [oem name]. +This script is used to build a Flatcar OEM sysext images. +The built image is in /oem-.raw. + +Examples: + +build_oem_sysext \ + --board=amd64-usr \ + --build_dir= \ + --prod_image_path= \ + --prod_pkgdb_path= \ + --version_id=\"\${FLATCAR_VERSION_ID}\" \ + oem-azure +... +" +show_help_if_requested "$@" + +# Parse command line. +FLAGS "$@" || exit 1 +if [[ -z "${FLAGS_ARGV}" ]]; then + echo 'No OEM given' + exit 0 +fi + +eval set -- "${FLAGS_ARGV}" + +# Only now can we die on error. shflags functions leak non-zero error codes, +# so will die prematurely if 'switch_to_strict_mode' is specified before now. +switch_to_strict_mode + +# N.B. Ordering matters for some of the libraries below, because +# some of the files contain initialization used by later files. +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/oem_sysext_util.sh" || exit 1 + +BUILD_DIR=${FLAGS_build_dir:-"${BUILD_DIR}"} + +if [[ -z "${FLAGS_prod_image_path}" ]]; then + error "--prod_image_path is required." + exit 1 +fi + +if [[ -z "${FLAGS_prod_pkgdb_path}" ]]; then + error "--prod_pkgdb_path is required." + exit 1 +fi + +for oem; do + oem_sysext_create "${oem}" "${BOARD}" "${FLAGS_version_id}" "${FLAGS_prod_image_path}" "${FLAGS_prod_pkgdb_path}" "${BUILD_DIR}" +done diff --git a/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md b/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md new file mode 100644 index 00000000000..5be23d3a692 --- /dev/null +++ b/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md @@ -0,0 +1 @@ +- Azure and QEMU OEM images now use systemd-sysext images for layering additional platform-specific software on top of `/usr`. For Azure images this also means that the image has a normal Python installation available through the sysext image. The OEM software is still not updated but this will be added soon. diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index 8dcd831e9aa..a27cba0175d 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -103,9 +103,11 @@ function _vm_build_impl() { formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g') local images_in="images-in/" + local file rm -rf "${images_in}" - copy_from_buildcache "images/${arch}/${vernum}/flatcar_production_image.bin.bz2" "${images_in}" - copy_from_buildcache "images/${arch}/${vernum}/version.txt" "${images_in}" + for file in flatcar_production_image.bin.bz2 flatcar_production_image_pkgdb.tar.xz version.txt; do + copy_from_buildcache "images/${arch}/${vernum}/${file}" "${images_in}" + done lbunzip2 "${images_in}/flatcar_production_image.bin.bz2" ./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \ -v "${vernum}" \ diff --git a/common.sh b/common.sh index c479295794e..eafbc928c17 100644 --- a/common.sh +++ b/common.sh @@ -425,6 +425,7 @@ BUILD_DIR= # Standard filenames FLATCAR_DEVELOPER_CONTAINER_NAME="flatcar_developer_container.bin" FLATCAR_PRODUCTION_IMAGE_NAME="flatcar_production_image.bin" +FLATCAR_PRODUCTION_IMAGE_PKGDB_NAME="flatcar_production_image_pkgdb.tar.xz" # ----------------------------------------------------------------------------- # Functions diff --git a/image_to_vm.sh b/image_to_vm.sh index c32d9d1d438..525490e313d 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -105,7 +105,7 @@ if [ -f "${FLAGS_from}/version.txt" ]; then FLATCAR_VERSION_STRING="${FLATCAR_VERSION}" fi -set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${FLATCAR_PRODUCTION_IMAGE_NAME}" +set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${FLATCAR_PRODUCTION_IMAGE_NAME}" "${FLATCAR_PRODUCTION_IMAGE_PKGDB_NAME}" # Make sure things are cleaned up on failure trap vm_cleanup EXIT @@ -118,6 +118,7 @@ setup_disk_image "${FLAGS_disk_layout}" # Optionally install any OEM packages install_oem_package install_oem_aci +install_oem_sysext run_fs_hook # Changes done, glue it together diff --git a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/0001-flatcar-changes.patch b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/0001-flatcar-changes.patch new file mode 100644 index 00000000000..6953cdea859 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/0001-flatcar-changes.patch @@ -0,0 +1,418 @@ +From 90b28746c0d8698a080eb7082e0e14054aee0a02 Mon Sep 17 00:00:00 2001 +From: Krzesimir Nowak +Date: Mon, 27 Feb 2023 15:59:21 +0100 +Subject: [PATCH] flatcar changes + +--- + azurelinuxagent/common/osutil/coreos.py | 39 +----- + azurelinuxagent/common/osutil/coreoscommon.py | 57 ++++++++ + azurelinuxagent/common/osutil/factory.py | 3 + + azurelinuxagent/common/osutil/flatcar.py | 41 ++++++ + config/flatcar/waagent.conf | 122 ++++++++++++++++++ + init/flatcar/10-waagent-sysext.conf | 2 + + init/flatcar/waagent.service | 30 +++++ + setup.py | 20 ++- + 8 files changed, 272 insertions(+), 42 deletions(-) + create mode 100644 azurelinuxagent/common/osutil/coreoscommon.py + create mode 100644 azurelinuxagent/common/osutil/flatcar.py + create mode 100644 config/flatcar/waagent.conf + create mode 100644 init/flatcar/10-waagent-sysext.conf + create mode 100644 init/flatcar/waagent.service + +diff --git a/azurelinuxagent/common/osutil/coreos.py b/azurelinuxagent/common/osutil/coreos.py +index fc0a6604..314008f0 100644 +--- a/azurelinuxagent/common/osutil/coreos.py ++++ b/azurelinuxagent/common/osutil/coreos.py +@@ -17,11 +17,10 @@ + # + + import os +-import azurelinuxagent.common.utils.shellutil as shellutil +-from azurelinuxagent.common.osutil.default import DefaultOSUtil ++from azurelinuxagent.common.osutil.coreoscommon import CoreosCommonUtil + + +-class CoreOSUtil(DefaultOSUtil): ++class CoreOSUtil(CoreosCommonUtil): + + def __init__(self): + super(CoreOSUtil, self).__init__() +@@ -46,40 +45,6 @@ class CoreOSUtil(DefaultOSUtil): + def get_agent_bin_path(): + return "/usr/share/oem/bin" + +- def is_sys_user(self, username): +- # User 'core' is not a sysuser. +- if username == 'core': +- return False +- return super(CoreOSUtil, self).is_sys_user(username) +- +- def is_dhcp_enabled(self): +- return True +- +- def start_network(self): +- return shellutil.run("systemctl start systemd-networkd", chk_err=False) +- +- def restart_if(self, ifname=None, retries=None, wait=None): +- shellutil.run("systemctl restart systemd-networkd") +- +- def restart_ssh_service(self): +- # SSH is socket activated on CoreOS. No need to restart it. +- pass +- +- def stop_dhcp_service(self): +- return shellutil.run("systemctl stop systemd-networkd", chk_err=False) +- +- def start_dhcp_service(self): +- return shellutil.run("systemctl start systemd-networkd", chk_err=False) +- +- def start_agent_service(self): +- return shellutil.run("systemctl start {0}".format(self.service_name), chk_err=False) +- +- def stop_agent_service(self): +- return shellutil.run("systemctl stop {0}".format(self.service_name), chk_err=False) +- +- def get_dhcp_pid(self): +- return self._get_dhcp_pid(["systemctl", "show", "-p", "MainPID", "systemd-networkd"]) +- + def conf_sshd(self, disable_password): + # In CoreOS, /etc/sshd_config is mount readonly. Skip the setting. + pass +diff --git a/azurelinuxagent/common/osutil/coreoscommon.py b/azurelinuxagent/common/osutil/coreoscommon.py +new file mode 100644 +index 00000000..fde9a456 +--- /dev/null ++++ b/azurelinuxagent/common/osutil/coreoscommon.py +@@ -0,0 +1,57 @@ ++# ++# Copyright 2023 Microsoft Corporation ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++# ++# Requires Python 2.6+ and Openssl 1.0+ ++# ++ ++import azurelinuxagent.common.utils.shellutil as shellutil ++from azurelinuxagent.common.osutil.default import DefaultOSUtil ++ ++ ++class CoreosCommonUtil(DefaultOSUtil): ++ ++ def is_sys_user(self, username): ++ # User 'core' is not a sysuser. ++ if username == 'core': ++ return False ++ return super(CoreOSUtil, self).is_sys_user(username) ++ ++ def is_dhcp_enabled(self): ++ return True ++ ++ def start_network(self): ++ return shellutil.run("systemctl start systemd-networkd", chk_err=False) ++ ++ def restart_if(self, ifname=None, retries=None, wait=None): ++ shellutil.run("systemctl restart systemd-networkd") ++ ++ def restart_ssh_service(self): ++ # SSH is socket activated on CoreOS. No need to restart it. ++ pass ++ ++ def stop_dhcp_service(self): ++ return shellutil.run("systemctl stop systemd-networkd", chk_err=False) ++ ++ def start_dhcp_service(self): ++ return shellutil.run("systemctl start systemd-networkd", chk_err=False) ++ ++ def start_agent_service(self): ++ return shellutil.run("systemctl start {0}".format(self.service_name), chk_err=False) ++ ++ def stop_agent_service(self): ++ return shellutil.run("systemctl stop {0}".format(self.service_name), chk_err=False) ++ ++ def get_dhcp_pid(self): ++ return self._get_dhcp_pid(["systemctl", "show", "-p", "MainPID", "systemd-networkd"]) +diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py +index b5ee0b09..9280c645 100644 +--- a/azurelinuxagent/common/osutil/factory.py ++++ b/azurelinuxagent/common/osutil/factory.py +@@ -27,6 +27,7 @@ from .clearlinux import ClearLinuxUtil + from .coreos import CoreOSUtil + from .debian import DebianOSBaseUtil, DebianOSModernUtil + from .default import DefaultOSUtil ++from .flatcar import FlatcarUtil + from .freebsd import FreeBSDOSUtil + from .gaia import GaiaOSUtil + from .iosxe import IosxeOSUtil +@@ -82,6 +83,8 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) + return DebianOSBaseUtil() + + if distro_name in ("flatcar", "coreos") or distro_code_name in ("flatcar", "coreos"): ++ if Version(distro_version) >= Version("3550"): ++ return FlatcarUtil() + return CoreOSUtil() + + if distro_name in ("suse", "sle_hpc", "sles", "opensuse"): +diff --git a/azurelinuxagent/common/osutil/flatcar.py b/azurelinuxagent/common/osutil/flatcar.py +new file mode 100644 +index 00000000..3d1bf535 +--- /dev/null ++++ b/azurelinuxagent/common/osutil/flatcar.py +@@ -0,0 +1,41 @@ ++# ++# Copyright 2023 Microsoft Corporation ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++# ++# Requires Python 2.6+ and Openssl 1.0+ ++# ++ ++import os ++import shutil ++ ++import azurelinuxagent.common.conf as conf ++ ++from azurelinuxagent.common.osutil.coreoscommon import CoreosCommonUtil ++ ++ ++class FlatcarUtil(CoreosCommonUtil): ++ ++ @staticmethod ++ def get_systemd_unit_file_install_path(): ++ return "/usr/lib/systemd/system" ++ ++ def conf_sshd(self, disable_password): ++ # make sure that the config file stops being a symlink ++ conf_file_path = conf.get_sshd_conf_file_path() ++ conf_file_path2 = f"{conf_file_path}.wal.tmp" ++ shutil.copy(conf_file_path, conf_file_path2) ++ os.remove(conf_file_path) ++ os.rename(conf_file_path2, conf_file_path) ++ super(CoreosCommonUtil, self).conf_sshd(disable_password) ++ pass +diff --git a/config/flatcar/waagent.conf b/config/flatcar/waagent.conf +new file mode 100644 +index 00000000..b453c634 +--- /dev/null ++++ b/config/flatcar/waagent.conf +@@ -0,0 +1,122 @@ ++# ++# Microsoft Azure Linux Agent Configuration ++# ++ ++# Enable extension handling. Do not disable this unless you do not need password reset, ++# backup, monitoring, or any extension handling whatsoever. ++Extensions.Enabled=y ++ ++# Which provisioning agent to use. Supported values are "auto" (default), "waagent", ++# "cloud-init", or "disabled". ++Provisioning.Agent=waagent ++ ++# Password authentication for root account will be unavailable. ++Provisioning.DeleteRootPassword=n ++ ++# Generate fresh host key pair. ++Provisioning.RegenerateSshHostKeyPair=n ++ ++# Supported values are "rsa", "dsa", "ecdsa", "ed25519", and "auto". ++# The "auto" option is supported on OpenSSH 5.9 (2011) and later. ++Provisioning.SshHostKeyPairType=auto ++ ++# Monitor host name changes and publish changes via DHCP requests. ++Provisioning.MonitorHostName=y ++ ++# Decode CustomData from Base64. ++Provisioning.DecodeCustomData=y ++ ++# Execute CustomData after provisioning. ++Provisioning.ExecuteCustomData=n ++ ++# Algorithm used by crypt when generating password hash. ++#Provisioning.PasswordCryptId=6 ++ ++# Length of random salt used when generating password hash. ++#Provisioning.PasswordCryptSaltLength=10 ++ ++# Allow reset password of sys user ++Provisioning.AllowResetSysUser=n ++ ++# Format if unformatted. If 'n', resource disk will not be mounted. ++ResourceDisk.Format=y ++ ++# File system on the resource disk ++# Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. ++ResourceDisk.Filesystem=ext4 ++ ++# Mount point for the resource disk ++ResourceDisk.MountPoint=/mnt/resource ++ ++# Create and use swapfile on resource disk. ++ResourceDisk.EnableSwap=n ++ ++# Size of the swapfile. ++ResourceDisk.SwapSizeMB=0 ++ ++# Comma-seperated list of mount options. See mount(8) for valid options. ++ResourceDisk.MountOptions=None ++ ++# Respond to load balancer probes if requested by Windows Azure. ++LBProbeResponder=y ++ ++# Enable verbose logging (y|n) ++Logs.Verbose=n ++ ++# Enable Console logging, default is y ++# Logs.Console=y ++ ++# Is FIPS enabled ++OS.EnableFIPS=n ++ ++# Set the path to SSH keys and configuration files ++OS.SshDir=/etc/ssh ++ ++# Root device timeout in seconds. ++OS.RootDeviceScsiTimeout=300 ++ ++# If "None", the system default version is used. ++OS.OpensslPath=None ++ ++# If set, agent will use proxy server to access internet ++#HttpProxy.Host=None ++#HttpProxy.Port=None ++ ++# Detect Scvmm environment, default is n ++# DetectScvmmEnv=n ++ ++# ++# Lib.Dir=/var/lib/waagent ++ ++# ++# DVD.MountPoint=/mnt/cdrom/secure ++ ++# ++# Pid.File=/var/run/waagent.pid ++ ++# ++# Extension.LogDir=/var/log/azure ++ ++# ++# Home.Dir=/home ++ ++# Enable RDMA management and set up, should only be used in HPC images ++# OS.EnableRDMA=y ++ ++# Enable or disable goal state processing auto-update, default is enabled ++AutoUpdate.Enabled=n ++ ++# Determine the update family, this should not be changed ++# AutoUpdate.GAFamily=Prod ++ ++# Determine if the overprovisioning feature is enabled. If yes, hold extension ++# handling until inVMArtifactsProfile.OnHold is false. ++# Default is enabled ++# EnableOverProvisioning=y ++ ++# Allow fallback to HTTP if HTTPS is unavailable ++# Note: Allowing HTTP (vs. HTTPS) may cause security risks ++# OS.AllowHTTP=n ++ ++# Add firewall rules to protect access to Azure host node services ++OS.EnableFirewall=y +diff --git a/init/flatcar/10-waagent-sysext.conf b/init/flatcar/10-waagent-sysext.conf +new file mode 100644 +index 00000000..f756dbc9 +--- /dev/null ++++ b/init/flatcar/10-waagent-sysext.conf +@@ -0,0 +1,2 @@ ++[Unit] ++Upholds=waagent.service +diff --git a/init/flatcar/waagent.service b/init/flatcar/waagent.service +new file mode 100644 +index 00000000..d0d6f7c8 +--- /dev/null ++++ b/init/flatcar/waagent.service +@@ -0,0 +1,30 @@ ++[Unit] ++Description=Microsoft Azure Linux Agent ++Wants=network-online.target sshd.service sshd-keygen.service ++After=network-online.target sshd-keygen.service ++ ++[Service] ++Type=simple ++ ++# Symlink the config if it's missing in /etc. This is a workaround for ++# the fact that this software comes to Flatcar as a sysext and as such ++# can't use the /etc overlay solution by putting the config into ++# /usr/share/flatcar/etc. ++# ++ExecStartPre=/bin/bash -c 'if [[ ! -e /etc/waagent.conf ]]; then ln -sf ../usr/share/waagent/waagent.conf /etc/waagent.conf; fi' ++ ++# This could be done also with: ++# ++# ExecStart=/usr/bin/python -u /usr/sbin/waagent -daemon ++# ++# But this would mean that logs from waagent in journal will be ++# denoted as coming from python instead. ++# ++Environment=PYTHONUNBUFFERED=x ++ExecStart=/usr/sbin/waagent -daemon ++ ++Restart=always ++RestartSec=5s ++ ++[Install] ++WantedBy=multi-user.target +diff --git a/setup.py b/setup.py +index d38d74d6..57b0edb9 100755 +--- a/setup.py ++++ b/setup.py +@@ -125,12 +125,22 @@ def get_data_files(name, version, fullname): # pylint: disable=R0912 + src=["init/arch/waagent.service"]) + elif name in ('coreos', 'flatcar'): + set_bin_files(data_files, dest=agent_bin_path) +- set_conf_files(data_files, dest="/usr/share/oem", +- src=["config/coreos/waagent.conf"]) + set_logrotate_files(data_files) +- set_udev_files(data_files) +- set_files(data_files, dest="/usr/share/oem", +- src=["init/coreos/cloud-config.yml"]) ++ if int(version.split('.')[0]) >= 3550: ++ # Not installing udev rules, Flatcar already has those ++ set_conf_files(data_files, dest="/usr/share/waagent", ++ src=["config/flatcar/waagent.conf"]) ++ set_systemd_files(data_files, dest=systemd_dir_path, ++ src=["init/flatcar/waagent.service"]) ++ multi_user_target_drop_in_dir=f"{systemd_dir_path}/multi-user.target.d" ++ set_systemd_files(data_files, dest=multi_user_target_drop_in_dir, ++ src=["init/flatcar/10-waagent-sysext.conf"]) ++ else: ++ set_udev_files(data_files) ++ set_conf_files(data_files, dest="/usr/share/oem", ++ src=["config/coreos/waagent.conf"]) ++ set_files(data_files, dest="/usr/share/oem", ++ src=["init/coreos/cloud-config.yml"]) + elif "Clear Linux" in fullname: + set_bin_files(data_files, dest=agent_bin_path) + set_conf_files(data_files, dest="/usr/share/defaults/waagent", +-- +2.25.1 + diff --git a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/waagent.conf b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/waagent.conf deleted file mode 100644 index 3d65d06b50d..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/waagent.conf +++ /dev/null @@ -1,127 +0,0 @@ -# -# Microsoft Azure Linux Agent Configuration -# - -# Enable instance creation -Provisioning.Enabled=y - -# Enable extension handling. Do not disable this unless you do not need password reset, -# backup, monitoring, or any extension handling whatsoever. -Extensions.Enabled=y - -# Rely on cloud-init to provision -Provisioning.UseCloudInit=n - -# Password authentication for root account will be unavailable. -Provisioning.DeleteRootPassword=n - -# Generate fresh host key pair. -Provisioning.RegenerateSshHostKeyPair=n - -# Supported values are "rsa", "dsa", "ecdsa", "ed25519", and "auto". -# The "auto" option is supported on OpenSSH 5.9 (2011) and later. -Provisioning.SshHostKeyPairType=auto - -# Monitor host name changes and publish changes via DHCP requests. -Provisioning.MonitorHostName=y - -# Decode CustomData from Base64. -Provisioning.DecodeCustomData=y - -# Execute CustomData after provisioning. -Provisioning.ExecuteCustomData=n - -# Algorithm used by crypt when generating password hash. -#Provisioning.PasswordCryptId=6 - -# Length of random salt used when generating password hash. -#Provisioning.PasswordCryptSaltLength=10 - -# Allow reset password of sys user -Provisioning.AllowResetSysUser=n - -# Format if unformatted. If 'n', resource disk will not be mounted. -ResourceDisk.Format=y - -# File system on the resource disk -# Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. -ResourceDisk.Filesystem=ext4 - -# Mount point for the resource disk -ResourceDisk.MountPoint=/mnt/resource - -# Create and use swapfile on resource disk. -ResourceDisk.EnableSwap=n - -# Size of the swapfile. -ResourceDisk.SwapSizeMB=0 - -# Comma-seperated list of mount options. See man(8) for valid options. -ResourceDisk.MountOptions=None - -# Enable verbose logging (y|n) -Logs.Verbose=n - -# Is FIPS enabled -OS.EnableFIPS=n - -# Root device timeout in seconds. -OS.RootDeviceScsiTimeout=300 - -# If "None", the system default version is used. -OS.OpensslPath=None - -# Set the SSH ClientAliveInterval -# OS.SshClientAliveInterval=180 - -# Set the path to SSH keys and configuration files -OS.SshDir=/etc/ssh - -# If set, agent will use proxy server to access internet -#HttpProxy.Host=None -#HttpProxy.Port=None - -# Detect Scvmm environment, default is n -# DetectScvmmEnv=n - -# -# Lib.Dir=/var/lib/waagent - -# -# DVD.MountPoint=/mnt/cdrom/secure - -# -# Pid.File=/var/run/waagent.pid - -# -# Extension.LogDir=/var/log/azure - -# -# Home.Dir=/home - -# Enable RDMA management and set up, should only be used in HPC images -# OS.EnableRDMA=y - -# Enable or disable goal state processing auto-update, default is enabled -# AutoUpdate.Enabled=y - -# Determine the update family, this should not be changed -# AutoUpdate.GAFamily=Prod - -# Determine if the overprovisioning feature is enabled. If yes, hold extension -# handling until inVMArtifactsProfile.OnHold is false. -# Default is enabled -# EnableOverProvisioning=y - -# Allow fallback to HTTP if HTTPS is unavailable -# Note: Allowing HTTP (vs. HTTPS) may cause security risks -# OS.AllowHTTP=n - -# Add firewall rules to protect access to Azure host node services -OS.EnableFirewall=y - -# Enforce control groups limits on the agent and extensions -CGroups.EnforceLimits=n - -# CGroups which are excluded from limits, comma separated -CGroups.Excluded=customscript,runcommand diff --git a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r2.ebuild b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r2.ebuild deleted file mode 100644 index 685a5a66e5b..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r2.ebuild +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2014 CoreOS, Inc.. All rights reserved. -# Distributed under the terms of the GNU General Public License v2 - -EAPI=7 - -DESCRIPTION="Windows Azure Linux Agent" -HOMEPAGE="https://github.com/Azure/WALinuxAgent" -KEYWORDS="amd64 arm64" -SRC_URI="${HOMEPAGE}/archive/v${PV}.tar.gz -> ${P}.tar.gz" - -LICENSE="Apache-2.0" -SLOT="0" -IUSE="" - -# Depending on specific version of python-oem allows us to notice when -# we update the major version of python and then to make sure that we -# install the package in correctly versioned site-packages directory. -DEP_PYVER="3.10" - -RDEPEND=" -dev-lang/python-oem:${DEP_PYVER} -dev-python/distro-oem -" - -S="${WORKDIR}/WALinuxAgent-${PV}" - -src_install() { - into "/oem" - dobin "${S}/bin/waagent" - - insinto "/oem/python/$(get_libdir)/python${DEP_PYVER}/site-packages" - doins -r "${S}/azurelinuxagent/" - - insinto "/oem" - doins "${FILESDIR}/waagent.conf" -} diff --git a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r3.ebuild b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r3.ebuild new file mode 100644 index 00000000000..ee70a047637 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r3.ebuild @@ -0,0 +1,33 @@ +# Copyright (c) 2014 CoreOS, Inc.. All rights reserved. +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +# Don't use DISTUTILS_USE_PEP517=setuptools because this installs +# everything inside /usr/lib/pythonX_Y/site-packages, even files that +# ought to be put into /etc or /sbin. +PYTHON_COMPAT=( python3_{9..11} ) + +inherit distutils-r1 + +DESCRIPTION="Windows Azure Linux Agent" +HOMEPAGE="https://github.com/Azure/WALinuxAgent" +SRC_URI="${HOMEPAGE}/archive/v${PV}.tar.gz -> ${P}.tar.gz" + +LICENSE="Apache-2.0" +KEYWORDS="amd64 arm64" +SLOT="0" +IUSE="" +RESTRICT="" + +BDEPEND=" + dev-python/distro +" +RDEPEND="${BDEPEND} +" + +S="${WORKDIR}/WALinuxAgent-${PV}" + +PATCHES=( + "${FILESDIR}/0001-flatcar-changes.patch" +) diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild new file mode 100644 index 00000000000..e3e58924f90 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild @@ -0,0 +1,88 @@ +# Copyright (c) 2023 The Flatcar Maintainers. +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +OEMIDS=( + qemu + azure +) + +DESCRIPTION='Common OEM files' +HOMEPAGE='https://www.flatcar.org/' + +LICENSE='Apache-2.0' +SLOT='0' +KEYWORDS='amd64 arm64' +IUSE="${OEMIDS[*]}" +REQUIRED_USE="^^ ( ${OEMIDS[*]} )" + +# No source directory. +S="${WORKDIR}" + +DEPEND="" +RDEPEND="${DEPEND}" +BDEPEND=" + app-portage/gentoolkit +" + +src_compile() { + local oemid package ebuild version name homepage lines + + for oemid in "${OEMIDS[@]}"; do + if use "${oemid}"; then break; fi + done + + package="coreos-base/oem-${oemid}" + ebuild=$(equery which "${package}") + version=${ebuild##*"oem-${oemid}-"} + version=${version%%'.ebuild'} + if [[ -z "${version}" ]]; then + die "Could not deduce a version from ebuild ${ebuild##*/} (${ebuild})" + fi + name=$(source <(grep -F 'OEM_NAME=' "${ebuild}"); echo "${OEM_NAME}") + if [[ -z "${name}" ]]; then + die "Missing OEM_NAME variable in ${ebuild##*/}" + fi + # We need to prefix the HOMEPAGE variable with SYSEXT_, because + # portage marks HOMEPAGE as readonly and this gets propagated to + # subshells, so sourcing a snippet with HOMEPAGE=foo won't + # overwrite the readonly variable. + homepage=$(source <(grep -F 'HOMEPAGE=' "${ebuild}" | sed -e 's/^/SYSEXT_/'); echo "${SYSEXT_HOMEPAGE}") + lines=( + "ID=${oemid}" + "VERSION_ID=${version}" + "NAME=\"${name}\"" + ) + if [[ -n "${homepage}" ]]; then + lines+=( "HOME_URL=\"${homepage}\"" ) + fi + lines+=( + 'BUG_REPORT_URL="https://issues.flatcar.org"' + ) + + { + printf '%s\n' "${lines[@]}" + if [[ -e "${FILESDIR}/${oemid}/oem-release.frag" ]]; then + cat "${FILESDIR}/${oemid}/oem-release.frag" + fi + } >"${T}/oem-release" + + lines=( + '# Flatcar GRUB settings' + '' + "set oem_id=\"${oemid}\"" + ) + { + printf '%s\n' "${lines[@]}" + if [[ -e "${FILESDIR}/${oemid}/grub.cfg.frag" ]]; then + cat "${FILESDIR}/${oemid}/grub.cfg.frag" + fi + } >"${T}/grub.cfg" +} + +src_install() { + insinto "/oem" + doins "${T}/grub.cfg" + doins "${T}/oem-release" +} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/grub.cfg b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/azure/grub.cfg.frag similarity index 89% rename from sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/grub.cfg rename to sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/azure/grub.cfg.frag index 15e7e50555b..48d22ee8865 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/grub.cfg +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/azure/grub.cfg.frag @@ -1,7 +1,3 @@ -# Flatcar GRUB settings - -set oem_id="azure" - set linux_append="flatcar.autologin" # Azure only has a serial console. diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/qemu/grub.cfg.frag b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/qemu/grub.cfg.frag new file mode 100644 index 00000000000..4f9e06c2c8c --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/qemu/grub.cfg.frag @@ -0,0 +1 @@ +set linux_append="flatcar.autologin" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/metadata.xml b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/metadata.xml new file mode 100644 index 00000000000..7c900b19e8e --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/metadata.xml @@ -0,0 +1,4 @@ + + + + diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-0.0.1-r180.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-0.0.1-r181.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-0.0.1-r180.ebuild rename to sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-0.0.1-r181.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild index 8fe3b1899d1..41c47517faf 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild @@ -10,7 +10,7 @@ CROS_WORKON_REPO="https://github.com" if [[ "${PV}" == 9999 ]]; then KEYWORDS="~amd64 ~arm ~arm64 ~x86" else - CROS_WORKON_COMMIT="93b80ace22806bae4ab521f16fa9f4d1d3172e77" # flatcar-master + CROS_WORKON_COMMIT="658eb0ea0fb8e89f8aa56ccf57867eb88b53fc27" # flatcar-master KEYWORDS="amd64 arm arm64 x86" fi diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/README b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/README deleted file mode 100644 index d128309fef3..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/README +++ /dev/null @@ -1,4 +0,0 @@ -These Ignition configs are part of the OEM configuration. Do not modify -them. If you want to write an Ignition config directly to disk, put it in -../config.ign and it will be applied at first boot instead of a config -in userdata. diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/base.ign b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/base.ign deleted file mode 100644 index 5c359a94103..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/base.ign +++ /dev/null @@ -1,37 +0,0 @@ -{ - "ignition": { - "version": "2.1.0" - }, - "storage": { - "files": [ - { - "filesystem": "root", - "path": "/etc/systemd/system/waagent.service", - "contents": { - "source": "oem:///units/waagent.service" - }, - "mode": 292 - }, - { - "filesystem": "root", - "path": "/etc/systemd/system/nvidia.service", - "contents": { - "source": "oem:///units/nvidia.service" - }, - "mode": 292 - } - ] - }, - "systemd": { - "units": [ - { - "name": "waagent.service", - "enabled": true - }, - { - "name": "nvidia.service", - "enabled": true - } - ] - } -} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh new file mode 100755 index 00000000000..28637a0eeb3 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +rootfs="${1}" + +to_delete=( + /usr/include + /usr/lib/debug + /usr/share/gdb + /usr/lib64/pkgconfig +) + +rm -rf "${to_delete[@]/#/${rootfs}}" + +ln -sf /usr/bin/true "${rootfs}/usr/bin/eject" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/oem-release b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/oem-release deleted file mode 100644 index fa11b4c3e02..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/oem-release +++ /dev/null @@ -1,5 +0,0 @@ -ID=azure -VERSION_ID=@@OEM_VERSION_ID@@ -NAME="Microsoft Azure" -HOME_URL="https://azure.microsoft.com/" -BUG_REPORT_URL="https://issues.flatcar.org" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/units/waagent.service b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/units/waagent.service deleted file mode 100644 index d8c6e71ad24..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/units/waagent.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Microsoft Azure Agent -Wants=network-online.target sshd-keygen.service -After=network-online.target sshd-keygen.service - -[Service] -Type=simple -Restart=always -RestartSec=5s -Environment=PATH=/oem/python/bin:/oem/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin -Environment=PYTHONUNBUFFERED=x -ExecStart=/oem/bin/waagent -daemon - -[Install] -WantedBy=multi-user.target diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r2.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r2.ebuild deleted file mode 100644 index 91a72143d40..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r2.ebuild +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2013 CoreOS, Inc.. All rights reserved. -# Distributed under the terms of the GNU General Public License v2 - -EAPI=7 - -DESCRIPTION="OEM suite for Azure" -HOMEPAGE="" -SRC_URI="" - -LICENSE="Apache-2.0" -SLOT="0" -KEYWORDS="amd64 arm64" -IUSE="" - -# no source directory -S="${WORKDIR}" - -RDEPEND=" - ~app-emulation/wa-linux-agent-${PV} - x11-drivers/nvidia-drivers -" - -src_prepare() { - default - sed -e "s\\@@OEM_VERSION_ID@@\\${PVR}\\g" \ - "${FILESDIR}/oem-release" > "${T}/oem-release" || die -} - -src_install() { - insinto "/oem" - doins "${FILESDIR}/grub.cfg" - doins "${T}/oem-release" - doins -r "${FILESDIR}/base" - doins -r "${FILESDIR}/units" - dosym "/usr/bin/true" "/oem/bin/eject" -} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r3.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r3.ebuild new file mode 100644 index 00000000000..24fa7fbe41c --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r3.ebuild @@ -0,0 +1,21 @@ +# Copyright (c) 2013 CoreOS, Inc.. All rights reserved. +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +DESCRIPTION="OEM suite for Azure" +HOMEPAGE="https://azure.microsoft.com/" +SRC_URI="" + +LICENSE="Apache-2.0" +SLOT="0" +KEYWORDS="amd64 arm64" +IUSE="" + +RDEPEND=" + ~app-emulation/wa-linux-agent-${PV} + x11-drivers/nvidia-drivers +" + +# for coreos-base/common-oem-files +OEM_NAME="Microsoft Azure" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/grub.cfg b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/grub.cfg deleted file mode 100644 index 2cd3a0a3105..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/grub.cfg +++ /dev/null @@ -1,4 +0,0 @@ -# Flatcar GRUB settings - -set oem_id="qemu" -set linux_append="flatcar.autologin" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/oem-release b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/oem-release deleted file mode 100644 index 280e43175de..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/oem-release +++ /dev/null @@ -1,5 +0,0 @@ -ID=qemu -VERSION_ID=@@OEM_VERSION_ID@@ -NAME="QEMU" -HOME_URL="https://www.qemu.org/" -BUG_REPORT_URL="https://issues.flatcar.org" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.1-r1.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.1-r1.ebuild deleted file mode 100644 index be8761ea9ee..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.1-r1.ebuild +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2020 Kinvolk GmbH. All rights reserved. -# Distributed under the terms of the GNU General Public License v2 - -EAPI=7 - -DESCRIPTION="OEM suite for QEMU" -HOMEPAGE="" -SRC_URI="" - -LICENSE="GPL-2" -SLOT="0" -KEYWORDS="amd64 arm64" -IUSE="" - -# no source directory -S="${WORKDIR}" - -src_prepare() { - default - sed -e "s\\@@OEM_VERSION_ID@@\\${PVR}\\g" \ - "${FILESDIR}/oem-release" > "${T}/oem-release" || die -} - -src_install() { - insinto "/oem" - doins "${FILESDIR}/grub.cfg" - doins "${T}/oem-release" -} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.2.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.2.ebuild new file mode 100644 index 00000000000..fb19d212a01 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.2.ebuild @@ -0,0 +1,15 @@ +# Copyright (c) 2020 Kinvolk GmbH. All rights reserved. +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +DESCRIPTION="OEM suite for QEMU" +HOMEPAGE="https://www.qemu.org/" +SRC_URI="" + +LICENSE="GPL-2" +SLOT="0" +KEYWORDS="amd64 arm64" +IUSE="" + +OEM_NAME="QEMU" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r10.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r11.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r10.ebuild rename to sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r11.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild index 0be0428a6b9..e95bd1644dc 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild @@ -34,6 +34,5 @@ RDEPEND=" coreos-base/coreos coreos-base/coreos-dev coreos-base/flatcar-eks - dev-lang/python-oem x11-drivers/nvidia-drivers " diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/Manifest b/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/Manifest deleted file mode 100644 index a60f8dcf2a3..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/Manifest +++ /dev/null @@ -1,3 +0,0 @@ -DIST Python-3.10.10.tar.xz 19627028 BLAKE2B 57fc6869fa05586158a170c1892d93a3036823bfafb9484b9d70bca6cdc3e76f75357622eace4bde9a4c0ca62a1bb79665e5751b41655f9f4d7e345547013ad8 SHA512 f0aee65970a68287b34c4eafcf35c6fa09c81ba234ac356db16fbbc6c36417e4ac67071e616d118f5e192d541d7f177dcab5585b9780e842f656c09e01c37ced -DIST Python-3.10.10.tar.xz.asc 833 BLAKE2B fd60e6268f7dd6676ea58bd7e80c513506ac9810c1a62ff060134207b0fd8e7b096d5f11f3cc536a1578144ff54c00bcb076d3c3f5889a69a898660dd280312b SHA512 591746d74c6123bf36c763b6e8e1de1554f02eeff30c855623ef0f12d3864d5573eb5efe96d6e142f24627c77b90738ada3456df4ad59bddcb008658f2ca8af9 -DIST python-gentoo-patches-3.10.10_p2.tar.xz 13992 BLAKE2B e18e708888dd28c8f238d4897aff79483a679396a168d8b5ff4f5e8c7f09cec5f1b13aeb327d3dc3e2149c2117c25da050987f1f1c3322b56c87245ba2d0b54d SHA512 14bc218a2f3c64ef9f42682fd1364208bcaa74f787dee39bd9566e40764c260a65fd42961be47a6e6c6227091cb2fef83e1d689302448647560689e20e07efe0 diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/README.md b/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/README.md deleted file mode 100644 index 9ebd462b493..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/README.md +++ /dev/null @@ -1,45 +0,0 @@ -Modifications made: - -- Keep using internal expat and libffi, thus dropping dev-libs/libffi - and dev-libs/expat from the dependencies. - -- Drop dev-python/gentoo-common dependency, it provides the - EXTERNALLY-MANAGED file, but we will provide our own. - -- Since this package is installed only for OEM partition as a binary - package, and the installation there happens after the packages - database is removed, we unset the RDEPEND variable. The RDEPEND - variable needs to be empty as it's also used during the binary - package installation. The contents of RDEPEND are already inside the - DEPEND variable, so we are safe. - -- We modify the configure flags: - - - Add `--prefix=/oem/python` as `/oem` is where the OEM partition is - mounted. - - - Add `--with-platlibdir="$(get_libdir)"`, this is to make sure that - consistent library directory gets picked. In our case for both - amd64 and arm64, it's lib64. - - - Change `--enable-shared` to `--disable-shared`. This will skip - building dynamic libraries, as we don't need them. - - - Add `--includedir=/discard/include` and change `--mandir` and - `--infodir` to also use `/discard` to install files there. Makes - it easy to remove the unnecessary files. - - - We disable loadable sqlite extensions. - - - As we want to use the internal versions of expat and libffi, we - change `--with-system-{expat,ffi}` to - `--without-system-{expat,ffi}`. - - - Comment out the `--with-wheel-pkg-dir` as it's some ensurepip - stuff we are disabling anyway. - -- Essentially drop `src_install` and write our own variant, where we - run `make altinstall`, remove unnecessary files (the original - `src_install` could be read to find out which files to remove), - creates a versionless python symlink, adds an EXTERNALLY-MANAGED - file, and removes the `/discard` directory. diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/metadata.xml b/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/metadata.xml deleted file mode 100644 index 66d5aec84c7..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/metadata.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - python@gentoo.org - Python - - - - Build Bluetooth protocol support in socket module - - - Install the ensurepip module that uses bundled wheels - to bootstrap pip and setuptools (if disabled, it will - be only possible to use venv `--without-pip`) - - - Link readline extension against dev-libs/libedit - instead of sys-libs/readline - - - Optimize the build using Profile Guided Optimization (PGO) - by running Python's test suite and collecting statistics - based on its performance. This will take longer to build. - - - Optimize the build using Link Time Optimization (LTO) - - - Disable pymalloc when running under - dev-util/valgrind is detected (may incur minor - performance penalty even when valgrind is not used) - - - Install Windows executables required to create an executable - installer for MS Windows - - - - cpe:/a:python:python - python/cpython - - diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/python-oem-3.10.10_p2-r1.ebuild b/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/python-oem-3.10.10_p2-r1.ebuild deleted file mode 100644 index ef732a7132f..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/python-oem-3.10.10_p2-r1.ebuild +++ /dev/null @@ -1,461 +0,0 @@ -# Copyright 1999-2023 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -EAPI="7" -WANT_LIBTOOL="none" - -inherit autotools check-reqs flag-o-matic multiprocessing pax-utils -inherit prefix python-utils-r1 toolchain-funcs verify-sig - -MY_PV=${PV/_rc/rc} -MY_P="Python-${MY_PV%_p*}" -PYVER=$(ver_cut 1-2) -PATCHSET="python-gentoo-patches-${MY_PV}" - -DESCRIPTION="An interpreted, interactive, object-oriented programming language" -HOMEPAGE=" - https://www.python.org/ - https://github.com/python/cpython/ -" -SRC_URI=" - https://www.python.org/ftp/python/${PV%%_*}/${MY_P}.tar.xz - https://dev.gentoo.org/~mgorny/dist/python/${PATCHSET}.tar.xz - verify-sig? ( - https://www.python.org/ftp/python/${PV%%_*}/${MY_P}.tar.xz.asc - ) -" -S="${WORKDIR}/${MY_P}" - -LICENSE="PSF-2" -SLOT="${PYVER}" -KEYWORDS="~alpha amd64 arm arm64 hppa ~ia64 ~loong ~m68k ~mips ppc ppc64 ~riscv ~s390 sparc x86" -IUSE=" - bluetooth build +ensurepip examples gdbm hardened libedit lto - +ncurses pgo +readline +sqlite +ssl test tk valgrind +xml -" -RESTRICT="!test? ( test )" - -# Do not add a dependency on dev-lang/python to this ebuild. -# If you need to apply a patch which requires python for bootstrapping, please -# run the bootstrap code on your dev box and include the results in the -# patchset. See bug 447752. - -# Flatcar: Drop a dependency on dev-libs/expat, we will use the internal one. -# Flatcar: Drop a dependency on dev-libs/libffi, we will use the internal one. -# Flatcar: Drop a dependency on dev-python/gentoo-common, we will install our own EXTERNALLY-MANAGED file -RDEPEND=" - app-arch/bzip2:= - app-arch/xz-utils:= - dev-lang/python-exec[python_targets_python3_10(-)] - dev-python/gentoo-common - sys-apps/util-linux:= - >=sys-libs/zlib-1.1.3:= - virtual/libcrypt:= - virtual/libintl - ensurepip? ( dev-python/ensurepip-wheels ) - gdbm? ( sys-libs/gdbm:=[berkdb] ) - ncurses? ( >=sys-libs/ncurses-5.2:= ) - readline? ( - !libedit? ( >=sys-libs/readline-4.1:= ) - libedit? ( dev-libs/libedit:= ) - ) - sqlite? ( >=dev-db/sqlite-3.3.8:3= ) - ssl? ( >=dev-libs/openssl-1.1.1:= ) - tk? ( - >=dev-lang/tcl-8.0:= - >=dev-lang/tk-8.0:= - dev-tcltk/blt:= - dev-tcltk/tix - ) - !! /dev/null || die - # We disable _ctypes and _crypt for CBUILD because Python's setup.py can't handle locating - # libdir correctly for cross. - PYTHON_DISABLE_MODULES="${PYTHON_DISABLE_MODULES} _ctypes _crypt" \ - ECONF_SOURCE="${S}" econf_build "${myeconfargs_cbuild[@]}" - - # Avoid as many dependencies as possible for the cross build. - cat >> Makefile <<-EOF || die - MODULE_NIS=disabled - MODULE__DBM=disabled - MODULE__GDBM=disabled - MODULE__DBM=disabled - MODULE__SQLITE3=disabled - MODULE__HASHLIB=disabled - MODULE__SSL=disabled - MODULE__CURSES=disabled - MODULE__CURSES_PANEL=disabled - MODULE_READLINE=disabled - MODULE__TKINTER=disabled - MODULE_PYEXPAT=disabled - MODULE_ZLIB=disabled - EOF - - # Unfortunately, we do have to build this immediately, and - # not in src_compile, because CHOST configure for Python - # will check the existence of the Python it was pointed to - # immediately. - PYTHON_DISABLE_MODULES="${PYTHON_DISABLE_MODULES} _ctypes _crypt" emake - popd &> /dev/null || die - fi - - # pass system CFLAGS & LDFLAGS as _NODIST, otherwise they'll get - # propagated to sysconfig for built extensions - local -x CFLAGS_NODIST=${CFLAGS} - local -x LDFLAGS_NODIST=${LDFLAGS} - local -x CFLAGS= LDFLAGS= - - # Fix implicit declarations on cross and prefix builds. Bug #674070. - if use ncurses; then - append-cppflags -I"${ESYSROOT}"/usr/include/ncursesw - fi - - hprefixify setup.py - econf "${myeconfargs[@]}" - - if grep -q "#define POSIX_SEMAPHORES_NOT_ENABLED 1" pyconfig.h; then - eerror "configure has detected that the sem_open function is broken." - eerror "Please ensure that /dev/shm is mounted as a tmpfs with mode 1777." - die "Broken sem_open function (bug 496328)" - fi - - # install epython.py as part of stdlib - echo "EPYTHON='python${PYVER}'" > Lib/epython.py || die -} - -src_compile() { - # Ensure sed works as expected - # https://bugs.gentoo.org/594768 - local -x LC_ALL=C - # Prevent using distutils bundled by setuptools. - # https://bugs.gentoo.org/823728 - export SETUPTOOLS_USE_DISTUTILS=stdlib - - # Save PYTHONDONTWRITEBYTECODE so that 'has_version' doesn't - # end up writing bytecode & violating sandbox. - # bug #831897 - local -x _PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE} - - if use pgo ; then - # bug 660358 - local -x COLUMNS=80 - local -x PYTHONDONTWRITEBYTECODE= - - addpredict "/usr/lib/python${PYVER}/site-packages" - fi - - # also need to clear the flags explicitly here or they end up - # in _sysconfigdata* - emake CPPFLAGS= CFLAGS= LDFLAGS= - - # Restore saved value from above. - local -x PYTHONDONTWRITEBYTECODE=${_PYTHONDONTWRITEBYTECODE} - - # Work around bug 329499. See also bug 413751 and 457194. - if has_version dev-libs/libffi[pax-kernel]; then - pax-mark E python - else - pax-mark m python - fi -} - -src_test() { - # Tests will not work when cross compiling. - if tc-is-cross-compiler; then - elog "Disabling tests due to crosscompiling." - return - fi - - local test_opts=( - -u-network - -j "$(makeopts_jobs)" - - # fails - -x test_gdb - ) - - if use sparc ; then - # bug #788022 - test_opts+=( - -x test_multiprocessing_fork - -x test_multiprocessing_forkserver - ) - fi - - # workaround docutils breaking tests - cat > Lib/docutils.py <<-EOF || die - raise ImportError("Thou shalt not import!") - EOF - - # bug 660358 - local -x COLUMNS=80 - local -x PYTHONDONTWRITEBYTECODE= - # workaround https://bugs.gentoo.org/775416 - addwrite "/usr/lib/python${PYVER}/site-packages" - - nonfatal emake test EXTRATESTOPTS="${test_opts[*]}" \ - CPPFLAGS= CFLAGS= LDFLAGS= < /dev/tty - local ret=${?} - - rm Lib/docutils.py || die - - [[ ${ret} -eq 0 ]] || die "emake test failed" -} - -# Flatcar: Rewrite src_install to just run make altinstall, remove -# some installed files (refer to the original src_install to see which -# files to drop), adding symlinks and the EXTERNALLY-MANAGED file, and -# removing the /discard directory. -src_install() { - local prefix=/oem/python - local eprefix="${ED}${prefix}" - local libdir="${prefix}/$(get_libdir)" - local elibdir="${eprefix}/$(get_libdir)" - local pythonplatlibdir="${libdir}/python${PYVER}" - local epythonplatlibdir="${elibdir}/python${PYVER}" - local bindir="${prefix}/bin" - local ebindir="${eprefix}/bin" - - emake DESTDIR="${D}" altinstall - - rm -r "${epythonplatlibdir}"/ensurepip || die - rm -r "${epythonplatlibdir}/"{sqlite3,test/test_sqlite*} || die - rm -r "${ebindir}/idle${PYVER}" || die - rm -r "${epythonplatlibdir}/"{idlelib,tkinter,test/test_tk*} || die - - # create a simple versionless 'python' symlink - dosym "python${PYVER}" "${bindir}/python" - dosym "python${PYVER}" "${bindir}/python3" - - insinto "${pythonplatlibdir}" - # https://peps.python.org/pep-0668/ - newins - EXTERNALLY-MANAGED <<-EOF - [externally-managed] - Error= - Please contact Flatcar maintainers if some python package - is necessary for this OEM image. - EOF - - rm -r "${ED}/discard" || die -} diff --git a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/Manifest b/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/Manifest deleted file mode 100644 index fbb1ae14b67..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/Manifest +++ /dev/null @@ -1 +0,0 @@ -DIST distro-1.7.0.tar.gz 58164 BLAKE2B 22bbd2daf9cac589530eac9a58767db6b9e389b77719516f7386a9377b49ba4c9b696165701acc42366b760b9a632c70a2243a58c12a367fef2a0a770a4aea44 SHA512 14516ecab33ee8c57c35a8279eb515fd699031fabac7d8886092ea98696797d55503179870aeb513a85e1a66c7e69f2f60bb6ea9fc935be975cb5135e1917ecc diff --git a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/README.md b/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/README.md deleted file mode 100644 index 7fcb570b329..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/README.md +++ /dev/null @@ -1,4 +0,0 @@ -This package is a hacked-up way to install a distro module for oem -packages to use. It's meant to be used by dev-lang/python-oem, thus -not using any python-specific eclasses and whatnot, to avoid pulling -python dependency into the production image. diff --git a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/distro-oem-1.7.0-r2.ebuild b/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/distro-oem-1.7.0-r2.ebuild deleted file mode 100644 index 330acec8824..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/distro-oem-1.7.0-r2.ebuild +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2021-2022 Microsoft Corporation -# Distributed under the terms of GNU General Public License v2 - -EAPI=8 - -MY_PN='distro' -MY_P="${MY_PN}-${PV}" - -DESCRIPTION="Reliable machine-readable Linux distribution information for Python" -HOMEPAGE=" - https://distro.readthedocs.io/en/latest/ - https://pypi.org/project/distro/ - https://github.com/python-distro/distro/" -SRC_URI="mirror://pypi/${MY_PN:0:1}/${MY_PN}/${MY_P}.tar.gz" - -LICENSE="Apache-2.0" -KEYWORDS="amd64 arm64" - -# Depending on specific version of python-oem allows us to notice when -# we update the major version of python and then to make sure that we -# install the package in correctly versioned site-packages directory. -DEP_PYVER="3.10" - -SLOT="0" -RDEPEND="dev-lang/python-oem:${DEP_PYVER}" - -S="${WORKDIR}/${MY_P}" - -src_compile() { - # nothing to do - : -} - -src_install() { - insinto "/oem/python/$(get_libdir)/python${DEP_PYVER}/site-packages" - local ssd="${S}/src/distro" - doins "${ssd}/distro.py" - doins "${ssd}/__init__.py" - doins "${ssd}/__main__.py" - doins "${ssd}/py.typed" -} diff --git a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/metadata.xml b/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/metadata.xml deleted file mode 100644 index 097975e3adc..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/metadata.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask index c02b3aba0da..ab7f1c9a9e5 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask @@ -26,7 +26,6 @@ sys-libs/glibc -crypt # We don't use pip. dev-lang/python ensurepip -dev-lang/python-oem ensurepip # Pulls dev-python/sphinx, which in turn pulls a lot of other python stuff. sys-fs/btrfs-progs man diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/package.use index 62ab6830760..269d4bdc211 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/package.use +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/package.use @@ -7,10 +7,6 @@ app-editors/vim minimal -crypt # minimal: Don't pull app-vim/gentoo-syntax app-editors/vim-core minimal dev-lang/python gdbm - -# Disable everything for python-oem except of build and xml -dev-lang/python-oem -bluetooth build -ensurepip -examples -gdbm -hardened -libedit -lto -ncurses -pgo -readline -sqlite -ssl -test -tk -valgrind xml - dev-libs/dbus-glib tools dev-libs/elfutils -utils dev-libs/openssl pkcs11 diff --git a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r14.ebuild b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r15.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r14.ebuild rename to sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r15.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild index 4a747a5175d..a8a4a65545c 100644 --- a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild @@ -10,7 +10,7 @@ CROS_WORKON_REPO="https://github.com" if [[ "${PV}" == 9999 ]]; then KEYWORDS="~amd64 ~arm ~arm64 ~x86" else - CROS_WORKON_COMMIT="130003986dfdab46a21c7f34054239e59583e0f6" # flatcar-master + CROS_WORKON_COMMIT="2c85973e01da92c60ad3c8cdcab702b4b508d10f" # flatcar-master KEYWORDS="amd64 arm arm64 x86" fi