kola: Add new OEM sysext update and migration test

The A/B updated OEM systemd-sysext image gets activated when both /usr
partitions have updated to a version that requires a sysext image. The
old OEM contents get cleaned in this migration that happens on the boot
after the final update.
Test this migration and update logic by starting from an old image that
isn't migrated, and emulate an OEM setup that requires an OEM sysext
image now. The update first happens with the inbuilt Omaha kolet server
which uses the fallback logic but then flatcar-update is used for the
second update to supply the OEM payload directly.

kola/tests/update/update.go

@@ -54,6 +54,48 @@ func init() {
 			return kola.UpdatePayloadFile == ""
 		},
 	})
+	register.Register(&register.Test{
+		Name:        "cl.update.oem",
+		Run:         oemPayload,
+		ClusterSize: 1,
+		NativeFuncs: map[string]func() error{
+			"Omaha": Serve,
+		},
+		Distros: []string{"cl"},
+		// This test uses its own OEM files and shouldn't run on other platforms
+		Platforms: []string{"qemu", "qemu-unpriv"},
+		SkipFunc: func(version semver.Version, channel, arch, platform string) bool {
+			// This test can only run if the update payload to test is given.
+			// The image passed must also be an old release that does not have
+			// the OEM sysext setup because we want to test the migration path
+			// (see scripts/ci-automation/vendor-testing/qemu_update.sh)
+			return kola.UpdatePayloadFile == ""
+		},
+		// This test is expected to run on a very old version as start image
+		UserData: conf.ContainerLinuxConfig(`storage:
+  filesystems:
+    - name: oem
+      mount:
+        device: "/dev/disk/by-label/OEM"
+        format: "ext4"
+  files:
+    - path: /oem-release
+      filesystem: oem
+      contents:
+        inline: |
+          ID=azure
+    - path: /python/shouldbedeleted
+      filesystem: oem
+      contents:
+        inline: |
+          should be deleted because its part of the Azure OEM cleanup paths
+    - path: /etc/systemd/system/waagent.service
+      contents:
+        inline: |
+          [Service]
+          ExecStart=/bin/echo "should be deleted because its part of the Azure OEM cleanup paths"
+`),
+	})
 	register.Register(&register.Test{
 		Name:        "cl.sysext.boot.old",
 		Run:         sysextBootLogicOld,
@@ -246,6 +288,46 @@ func splitNewlineEnv(envs string) map[string]string {
 	return m
 }
 
+func oemPayload(c cluster.TestCluster) {
+	// This test first consumes a payload locally given to kola
+	// and then also uses this payload again with flatcar-update,
+	// where the OEM extension will be fallback downloaded from
+	// bincache during the postinst hook, and then after the reboot
+	// there is a final update to itself with flatcar-update where
+	// the OEM extension is passed expliclitly and the machine
+	// reboots once more to migrate to the sysext OEM setup.
+	m := c.Machines()[0]
+	// The instance should now host its own update with a kolet
+	addr := configureOmahaServer(c, m)
+	configureMachineForUpdate(c, m, addr)
+	tutil.AssertBootedUsr(c, m, "USR-A")
+	// Updating will fail if there is no payload on bincache,
+	// so don't expect this to work for local or GitHub Action builds
+	updateMachine(c, m)
+	tutil.AssertBootedUsr(c, m, "USR-B")
+	tutil.InvalidateUsrPartition(c, m, "USR-A")
+	// Check that the instance is not yet migrated
+	_ = c.MustSSH(m, `test -e /oem/python/shouldbedeleted && test -e /etc/systemd/system/waagent.service`)
+	_ = c.MustSSH(m, `test ! -e /oem/sysext/active-oem-azure`)
+	version := string(c.MustSSH(m, `set -euo pipefail; grep -m 1 "^VERSION=" /usr/lib/os-release | cut -d = -f 2`))
+	if version == "" {
+		c.Fatalf("Assertion for version string failed")
+	}
+	_ = c.MustSSH(m, `test -e /oem/sysext/oem-azure-`+version+`.raw`)
+	arch := strings.SplitN(kola.QEMUOptions.Board, "-", 2)[0]
+	_ = c.MustSSH(m, `curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://bincache.flatcar-linux.net/images/`+arch+`/`+version+`/flatcar_test_update-oem-azure.gz`)
+	_ = c.MustSSH(m, `sudo flatcar-update --to-version `+version+` --to-payload /updates/update.gz --extension ./flatcar_test_update-oem-azure.gz --disable-afterwards --force-dev-key`)
+	c.Logf("Rebooting test machine after flatcar-update run (2nd reboot)")
+	if err := m.Reboot(); err != nil {
+		c.Fatalf("reboot failed: %v", err)
+	}
+	tutil.AssertBootedUsr(c, m, "USR-A")
+	// Check that the instance has migrated
+	_ = c.MustSSH(m, `test ! -e /oem/python/shouldbedeleted && test ! -e /etc/systemd/system/waagent.service`)
+	_ = c.MustSSH(m, `test -e /oem/sysext/active-oem-azure`)
+	_ = c.MustSSH(m, `systemd-sysext status --json=pretty | jq --raw-output '.[] | select(.hierarchy == "/usr") | .extensions[]' | grep -q oem-azure`)
+}
+
 func sysextBootLogicOld(c cluster.TestCluster) {
 	sysextBootLogic(c, "/usr/share/oem")
 }

kola/tests/util/update.go

@@ -31,8 +31,8 @@ func AssertBootedUsr(c cluster.TestCluster, m platform.Machine, usr string) {
 }
 
 func GetUsrDeviceNode(c cluster.TestCluster, m platform.Machine) string {
-	// find /usr dev
-	usrdev := c.MustSSH(m, "findmnt -no SOURCE /usr")
+	// find /usr dev (-f to see the first mount, not the sysext overlay mount)
+	usrdev := c.MustSSH(m, "findmnt -fno SOURCE /usr")
 
 	// XXX: if the /usr dev is /dev/mapper/usr, we're on a verity enabled
 	// image, so use dmsetup to find the real device.