From 32ba3d2a9cee1a87adb99dfcb6a12cf5137df3a5 Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Mon, 4 Sep 2023 21:58:50 +0200 Subject: [PATCH] kola: Add new OEM sysext update and migration test The A/B updated OEM systemd-sysext image gets activated when both /usr partitions have updated to a version that requires a sysext image. The old OEM contents get cleaned in this migration that happens on the boot after the final update. Test this migration and update logic by starting from an old image that isn't migrated, and emulate an OEM setup that requires an OEM sysext image now. The update first happens with the inbuilt Omaha kolet server which uses the fallback logic but then flatcar-update is used for the second update to supply the OEM payload directly. --- kola/tests/update/update.go | 82 +++++++++++++++++++++++++++++++++++++ kola/tests/util/update.go | 4 +- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/kola/tests/update/update.go b/kola/tests/update/update.go index b24b6ee9d..4dd5a1b7b 100644 --- a/kola/tests/update/update.go +++ b/kola/tests/update/update.go @@ -54,6 +54,48 @@ func init() { return kola.UpdatePayloadFile == "" }, }) + register.Register(®ister.Test{ + Name: "cl.update.oem", + Run: oemPayload, + ClusterSize: 1, + NativeFuncs: map[string]func() error{ + "Omaha": Serve, + }, + Distros: []string{"cl"}, + // This test uses its own OEM files and shouldn't run on other platforms + Platforms: []string{"qemu", "qemu-unpriv"}, + SkipFunc: func(version semver.Version, channel, arch, platform string) bool { + // This test can only run if the update payload to test is given. + // The image passed must also be an old release that does not have + // the OEM sysext setup because we want to test the migration path + // (see scripts/ci-automation/vendor-testing/qemu_update.sh) + return kola.UpdatePayloadFile == "" + }, + // This test is expected to run on a very old version as start image + UserData: conf.ContainerLinuxConfig(`storage: + filesystems: + - name: oem + mount: + device: "/dev/disk/by-label/OEM" + format: "ext4" + files: + - path: /oem-release + filesystem: oem + contents: + inline: | + ID=azure + - path: /python/shouldbedeleted + filesystem: oem + contents: + inline: | + should be deleted because its part of the Azure OEM cleanup paths + - path: /etc/systemd/system/waagent.service + contents: + inline: | + [Service] + ExecStart=/bin/echo "should be deleted because its part of the Azure OEM cleanup paths" +`), + }) register.Register(®ister.Test{ Name: "cl.sysext.boot.old", Run: sysextBootLogicOld, @@ -246,6 +288,46 @@ func splitNewlineEnv(envs string) map[string]string { return m } +func oemPayload(c cluster.TestCluster) { + // This test first consumes a payload locally given to kola + // and then also uses this payload again with flatcar-update, + // where the OEM extension will be fallback downloaded from + // bincache during the postinst hook, and then after the reboot + // there is a final update to itself with flatcar-update where + // the OEM extension is passed expliclitly and the machine + // reboots once more to migrate to the sysext OEM setup. + m := c.Machines()[0] + // The instance should now host its own update with a kolet + addr := configureOmahaServer(c, m) + configureMachineForUpdate(c, m, addr) + tutil.AssertBootedUsr(c, m, "USR-A") + // Updating will fail if there is no payload on bincache, + // so don't expect this to work for local or GitHub Action builds + updateMachine(c, m) + tutil.AssertBootedUsr(c, m, "USR-B") + tutil.InvalidateUsrPartition(c, m, "USR-A") + // Check that the instance is not yet migrated + _ = c.MustSSH(m, `test -e /oem/python/shouldbedeleted && test -e /etc/systemd/system/waagent.service`) + _ = c.MustSSH(m, `test ! -e /oem/sysext/active-oem-azure`) + version := string(c.MustSSH(m, `set -euo pipefail; grep -m 1 "^VERSION=" /usr/lib/os-release | cut -d = -f 2`)) + if version == "" { + c.Fatalf("Assertion for version string failed") + } + _ = c.MustSSH(m, `test -e /oem/sysext/oem-azure-`+version+`.raw`) + arch := strings.SplitN(kola.QEMUOptions.Board, "-", 2)[0] + _ = c.MustSSH(m, `curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://bincache.flatcar-linux.net/images/`+arch+`/`+version+`/flatcar_test_update-oem-azure.gz`) + _ = c.MustSSH(m, `sudo flatcar-update --to-version `+version+` --to-payload /updates/update.gz --extension ./flatcar_test_update-oem-azure.gz --disable-afterwards --force-dev-key`) + c.Logf("Rebooting test machine after flatcar-update run (2nd reboot)") + if err := m.Reboot(); err != nil { + c.Fatalf("reboot failed: %v", err) + } + tutil.AssertBootedUsr(c, m, "USR-A") + // Check that the instance has migrated + _ = c.MustSSH(m, `test ! -e /oem/python/shouldbedeleted && test ! -e /etc/systemd/system/waagent.service`) + _ = c.MustSSH(m, `test -e /oem/sysext/active-oem-azure`) + _ = c.MustSSH(m, `systemd-sysext status --json=pretty | jq --raw-output '.[] | select(.hierarchy == "/usr") | .extensions[]' | grep -q oem-azure`) +} + func sysextBootLogicOld(c cluster.TestCluster) { sysextBootLogic(c, "/usr/share/oem") } diff --git a/kola/tests/util/update.go b/kola/tests/util/update.go index 3ee4e6f3f..2547cf41f 100644 --- a/kola/tests/util/update.go +++ b/kola/tests/util/update.go @@ -31,8 +31,8 @@ func AssertBootedUsr(c cluster.TestCluster, m platform.Machine, usr string) { } func GetUsrDeviceNode(c cluster.TestCluster, m platform.Machine) string { - // find /usr dev - usrdev := c.MustSSH(m, "findmnt -no SOURCE /usr") + // find /usr dev (-f to see the first mount, not the sysext overlay mount) + usrdev := c.MustSSH(m, "findmnt -fno SOURCE /usr") // XXX: if the /usr dev is /dev/mapper/usr, we're on a verity enabled // image, so use dmsetup to find the real device.