Skip to content

Commit 517c099

Browse files
zehnmzehnm
committed
feat: U-Boot boot limit with fallback to old system
Use U-Boot boot limit functionality to detect a system update and failed startups. This uses the following U-Boot env variables: - `bootlimit`: defines the maximum number of reboot cycles allowed. Default value: 3 - `bootcount`: will be incremented at each reboot IF `upgrade_available` is set to 1 - `upgrade_available`: indicates that a system update has been applied. - SWUpdate sets this value to 1 after a new system update has been written. - Verified after boot with a systemd timer: If after a system update the remote app is still running after 3 minutes after boot up, the update is considered successful and `upgrade_available` and `bootcount` U-Boot env vars are cleared. - `rootfspart`: holds the current active root filesystem partition number to boot from. - `altbootcmd`: alternative U-Boot boot command if bootlimit is exceeded. - Switches the `rootfspart` to the old partition number to restore the old system.
1 parent 326a47e commit 517c099

File tree

16 files changed

+198
-78
lines changed

16 files changed

+198
-78
lines changed

‎buildroot-external/board/raspberrypi/rootfs-overlay/etc/swupdate/reset-bootcount.sh

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/sh
2+
3+
systemctl is-active --quiet app || {
4+
echo "Remote app not running!"
5+
exit 1
6+
}
7+
8+
ISUPGRADING=$(fw_printenv upgrade_available | awk -F'=' '{print $2}')
9+
echo "upgrade_available=$ISUPGRADING"
10+
if [ -z "$ISUPGRADING" ]
11+
then
12+
echo "No system update pending"
13+
else
14+
echo "System update pending, verifying system..."
15+
# TODO refactor once there are any common system checks:
16+
# - call script from systemd right after app start (instead of delayed timer)
17+
# - get pid of app
18+
# - perform system checks
19+
# - wait ~ 3 min
20+
# - check if app is still running and has same pid (i.e. didn't get auto-restarted)
21+
22+
# Perform extra checks here.
23+
# If anything went wrong, reboot again until the bootlimit is reached
24+
# which triggers a rollback of the RootFs
25+
26+
# It's all good! Clear upgrade status and mark partition ok.
27+
fw_setenv upgrade_available
28+
fw_setenv bootcount 0
29+
echo "System update successful. Marked current partition OK."
30+
fi

‎buildroot-external/board/raspberrypi/sw-description

+10-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,12 @@ software = {
1616
);
1717
bootenv: (
1818
{
19-
name = "yiospart";
19+
name = "rootfspart";
2020
value = "2";
21+
},
22+
{
23+
name = "upgrade_available";
24+
value = "1";
2125
}
2226
);
2327
}
@@ -32,8 +36,12 @@ software = {
3236
);
3337
bootenv: (
3438
{
35-
name = "yiospart";
39+
name = "rootfspart";
3640
value = "3";
41+
},
42+
{
43+
name = "upgrade_available";
44+
value = "1";
3745
}
3846
);
3947
}

‎buildroot-external/board/raspberrypi/uboot-boot.ush

+40-32
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,38 @@
11
# U-Boot boot script for YIOS developer image
2-
# Based on: https://github.com/Opentrons/buildroot/blob/opentrons-develop/board/opentrons/ot2/boot.scr
3-
# The job of this script is
4-
# - Figure out which partition is active
5-
# - Point U-Boot at that partition
6-
# - Boot it
2+
# fdtargs args patching based on: https://github.com/Opentrons/buildroot/blob/opentrons-develop/board/opentrons/ot2/boot.scr
3+
# Sets the correct root partition in the boot arguments inside the flattened device tree.
74

8-
# See if we have an env var for which partition to boot from (this might be
9-
# edited from the booted system, but also might not be present)
10-
if test -z "$yiospart"
5+
# Create the required env vars for boot count & limit support.
6+
# Note: bootcount and bootlimit are already defined in U-Boot.
7+
# - rootfspart holds the currently active root partition to boot
8+
if test -z "${rootfspart}"
119
then
12-
echo "No saved yiospart, defaulting to 2"
13-
setenv yiospart "2"
10+
echo "No saved rootfspart, defaulting to 2"
11+
setenv rootfspart "2"
1412
saveenv
1513
fi
1614

17-
if test "$yiospart" = "2"
15+
# - altbootcmd to execute when boot limit is reached
16+
if test -z "${altbootcmd}"
1817
then
19-
yiosold="3"
20-
elif test "$yiospart" = "3"
18+
echo "No saved altbootcmd, creating it"
19+
setenv altbootcmd "
20+
oldrootfspart='\${rootfspart}';
21+
if test \"'\${rootfspart}'\" = \"2\";
2122
then
22-
yiosold="2"
23+
setenv rootfspart 3;
2324
else
24-
echo "Bad yiospart $yiospart, defaulting to 2"
25-
setenv yiospart "2"
25+
setenv rootfspart 2;
26+
fi;
27+
setenv bootcount 0;
28+
saveenv;
29+
echo Rollback from rootfs '\${oldrootfspart}' to old rootfs '\${rootfspart}';
30+
boot"
2631
saveenv
27-
yiosold="3"
2832
fi
2933

30-
echo "Try booting from partition $yiospart with fallback to partition $yiosold"
31-
34+
# Parse and replace the boot arguments in the flattened device tree.
35+
# This allows us to keep the RPi cmdline.txt without hardcoded bootargs.
3236
# The raspi second stage bootloader (start.elf) puts the boot args in the
3337
# flattened device tree. The boot args contain the partition the kernel
3438
# will boot off of, so we have to pull the boot args, parse them, and then
@@ -40,24 +44,28 @@ newargs=""
4044
for arg in $fdtargs; do
4145
if test "root=/dev/mmcblk0p2" = $arg
4246
then
43-
echo "Found bootpart"
4447
bootpart=$arg
4548
else
4649
newargs="$newargs $arg"
4750
fi
4851
done
4952

50-
while true; do
51-
echo "Try booting from partition $yiospart"
53+
if test "$upgrade_available" = "1"
54+
then
55+
echo "Try booting system upgrade from partition ${rootfspart}. Boot count: $bootcount. Limit for rollback: $bootlimit"
56+
else
57+
echo "Booting from active partition ${rootfspart}"
58+
fi
5259

53-
to_boot="$newargs root=/dev/mmcblk0p$yiospart"
54-
fdt set /chosen bootargs "$to_boot"
55-
ext4load mmc 0:$yiospart $kernel_addr_r /zImage
56-
bootz $kernel_addr_r - $fdt_addr
60+
# Note: setting an invalid root param will hang the boot process due to required `rootwait` argument, which waits indefinitely!
61+
# A hackish workaround for a kernel panic to trigger a reboot would be using rootdelay=xx instead, but that delays the boot for xx seconds.
62+
# However: since we are loading the kernel from the same partition, the script will fail before booting the kernel. So it should be safe to use.
63+
to_boot="$newargs root=/dev/mmcblk0p${rootfspart}"
64+
fdt set /chosen bootargs "$to_boot"
65+
ext4load mmc 0:${rootfspart} $kernel_addr_r /zImage
66+
bootz $kernel_addr_r - $fdt_addr
5767

58-
echo "Boot failed! Switching"
59-
tmp=$otpart
60-
setenv yiospart $yiosold
61-
saveenv
62-
yiosold=$tmp
63-
done
68+
echo "Boot failed!"
69+
# Reset loop might not be the best choice, but there's not much we can do here.
70+
# Better solution would require to initialize the display in U-Boot and display an error screen. That's another project...
71+
reset

‎buildroot-external/board/raspberrypi/uboot.config

+4
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,7 @@ CONFIG_CMD_SETEXPR=y
1515
# set to -1 to disable autoboot.
1616
# set to -2 to autoboot with no delay and not check for abort
1717
CONFIG_BOOTDELAY=2
18+
CONFIG_BOOTCOUNT_ENV=y
19+
CONFIG_BOOTCOUNT_LIMIT=y
20+
CONFIG_BOOTCOUNT_BOOTLIMIT=3
21+
CONFIG_CMD_BOOTCOUNT=y

‎buildroot-external/board/remote/rootfs-overlay/etc/swupdate.cfg

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
globals :
44
{
5+
verbose = true;
56
postupdatecmd = "/etc/swupdate/postupdate.sh";
67
};
78

‎buildroot-external/board/remote/rootfs-overlay/etc/swupdate/reset-bootcount.sh

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/sh
2+
3+
systemctl is-active --quiet app || {
4+
echo "Remote app not running!"
5+
exit 1
6+
}
7+
8+
ISUPGRADING=$(fw_printenv upgrade_available | awk -F'=' '{print $2}')
9+
echo "upgrade_available=$ISUPGRADING"
10+
if [ -z "$ISUPGRADING" ]
11+
then
12+
echo "No system update pending"
13+
else
14+
echo "System update pending, verifying system..."
15+
# TODO refactor once there are any common system checks:
16+
# - call script from systemd right after app start (instead of delayed timer)
17+
# - get pid of app
18+
# - perform system checks
19+
# - wait ~ 3 min
20+
# - check if app is still running and has same pid (i.e. didn't get auto-restarted)
21+
22+
# Perform extra checks here.
23+
# If anything went wrong, reboot again until the bootlimit is reached
24+
# which triggers a rollback of the RootFs
25+
26+
# It's all good! Clear upgrade status and mark partition ok.
27+
fw_setenv upgrade_available
28+
fw_setenv bootcount 0
29+
echo "System update successful. Marked current partition OK."
30+
fi

‎buildroot-external/board/remote/sw-description

+10-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,12 @@ software = {
1616
);
1717
bootenv: (
1818
{
19-
name = "yiospart";
19+
name = "rootfspart";
2020
value = "2";
21+
},
22+
{
23+
name = "upgrade_available";
24+
value = "1";
2125
}
2226
);
2327
}
@@ -32,8 +36,12 @@ software = {
3236
);
3337
bootenv: (
3438
{
35-
name = "yiospart";
39+
name = "rootfspart";
3640
value = "3";
41+
},
42+
{
43+
name = "upgrade_available";
44+
value = "1";
3745
}
3846
);
3947
}

‎buildroot-external/board/remote/uboot-boot.ush

+40-32
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,38 @@
11
# U-Boot boot script for YIOS developer image
2-
# Based on: https://github.com/Opentrons/buildroot/blob/opentrons-develop/board/opentrons/ot2/boot.scr
3-
# The job of this script is
4-
# - Figure out which partition is active
5-
# - Point U-Boot at that partition
6-
# - Boot it
2+
# fdtargs args patching based on: https://github.com/Opentrons/buildroot/blob/opentrons-develop/board/opentrons/ot2/boot.scr
3+
# Sets the correct root partition in the boot arguments inside the flattened device tree.
74

8-
# See if we have an env var for which partition to boot from (this might be
9-
# edited from the booted system, but also might not be present)
10-
if test -z "$yiospart"
5+
# Create the required env vars for boot count & limit support.
6+
# Note: bootcount and bootlimit are already defined in U-Boot.
7+
# - rootfspart holds the currently active root partition to boot
8+
if test -z "${rootfspart}"
119
then
12-
echo "No saved yiospart, defaulting to 2"
13-
setenv yiospart "2"
10+
echo "No saved rootfspart, defaulting to 2"
11+
setenv rootfspart "2"
1412
saveenv
1513
fi
1614

17-
if test "$yiospart" = "2"
15+
# - altbootcmd to execute when boot limit is reached
16+
if test -z "${altbootcmd}"
1817
then
19-
yiosold="3"
20-
elif test "$yiospart" = "3"
18+
echo "No saved altbootcmd, creating it"
19+
setenv altbootcmd "
20+
oldrootfspart='\${rootfspart}';
21+
if test \"'\${rootfspart}'\" = \"2\";
2122
then
22-
yiosold="2"
23+
setenv rootfspart 3;
2324
else
24-
echo "Bad yiospart $yiospart, defaulting to 2"
25-
setenv yiospart "2"
25+
setenv rootfspart 2;
26+
fi;
27+
setenv bootcount 0;
28+
saveenv;
29+
echo Rollback from rootfs '\${oldrootfspart}' to old rootfs '\${rootfspart}';
30+
boot"
2631
saveenv
27-
yiosold="3"
2832
fi
2933

30-
echo "Try booting from partition $yiospart with fallback to partition $yiosold"
31-
34+
# Parse and replace the boot arguments in the flattened device tree.
35+
# This allows us to keep the RPi cmdline.txt without hardcoded bootargs.
3236
# The raspi second stage bootloader (start.elf) puts the boot args in the
3337
# flattened device tree. The boot args contain the partition the kernel
3438
# will boot off of, so we have to pull the boot args, parse them, and then
@@ -40,24 +44,28 @@ newargs=""
4044
for arg in $fdtargs; do
4145
if test "root=/dev/mmcblk0p2" = $arg
4246
then
43-
echo "Found bootpart"
4447
bootpart=$arg
4548
else
4649
newargs="$newargs $arg"
4750
fi
4851
done
4952

50-
while true; do
51-
echo "Try booting from partition $yiospart"
53+
if test "$upgrade_available" = "1"
54+
then
55+
echo "Try booting system upgrade from partition ${rootfspart}. Boot count: $bootcount. Limit for rollback: $bootlimit"
56+
else
57+
echo "Booting from active partition ${rootfspart}"
58+
fi
5259

53-
to_boot="$newargs root=/dev/mmcblk0p$yiospart"
54-
fdt set /chosen bootargs "$to_boot"
55-
ext4load mmc 0:$yiospart $kernel_addr_r /zImage
56-
bootz $kernel_addr_r - $fdt_addr
60+
# Note: setting an invalid root param will hang the boot process due to required `rootwait` argument, which waits indefinitely!
61+
# A hackish workaround for a kernel panic to trigger a reboot would be using rootdelay=xx instead, but that delays the boot for xx seconds.
62+
# However: since we are loading the kernel from the same partition, the script will fail before booting the kernel. So it should be safe to use.
63+
to_boot="$newargs root=/dev/mmcblk0p${rootfspart}"
64+
fdt set /chosen bootargs "$to_boot"
65+
ext4load mmc 0:${rootfspart} $kernel_addr_r /zImage
66+
bootz $kernel_addr_r - $fdt_addr
5767

58-
echo "Boot failed! Switching"
59-
tmp=$otpart
60-
setenv yiospart $yiosold
61-
saveenv
62-
yiosold=$tmp
63-
done
68+
echo "Boot failed!"
69+
# Reset loop might not be the best choice, but there's not much we can do here.
70+
# Better solution would require to initialize the display in U-Boot and display an error screen. That's another project...
71+
reset

‎buildroot-external/board/remote/uboot.config

+3
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,6 @@ CONFIG_SILENT_CONSOLE_UPDATE_ON_RELOC=y
3838
# CONFIG_CMD_ELF is not set
3939
# CONFIG_CMD_IMI is not set
4040
# CONFIG_CMD_XIMG is not set
41+
CONFIG_BOOTCOUNT_ENV=y
42+
CONFIG_BOOTCOUNT_LIMIT=y
43+
CONFIG_BOOTCOUNT_BOOTLIMIT=3

‎buildroot-external/rootfs-overlay/etc/systemd/system/app-update-bootcount.service

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[Unit]
2+
Description=Run bootcount reset check after app update script
3+
RefuseManualStart=no
4+
RefuseManualStop=no
5+
6+
[Service]
7+
Type=oneshot
8+
ExecStart=/etc/swupdate/reset-bootcount.sh

‎buildroot-external/rootfs-overlay/etc/systemd/system/app-update-bootcount.timer

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[Unit]
2+
Description=Bootcount reset check after app update timer
3+
4+
[Timer]
5+
OnBootSec=180
6+
7+
[Install]
8+
WantedBy=multi-user.target

‎buildroot-external/rootfs-overlay/etc/systemd/system/app.service

+7-2
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,14 @@ Description=YIO remote app
33
After=display-init.service
44

55
[Service]
6-
Type=simple
6+
Type=exec
77
ExecStart=/opt/yio/app-launch.sh
8-
RemainAfterExit=yes
8+
# Force reboot if app failed to startup 2 times within 60 seconds
9+
StartLimitIntervalSec=60
10+
StartLimitBurst=2
11+
Restart=always
12+
RestartSec=1
13+
StartLimitAction=reboot-force
914

1015
[Install]
1116
WantedBy=multi-user.target

‎buildroot-external/rootfs-overlay/etc/systemd/system/multi-user.target.wants/app-update-bootcount.timer

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../app-update-bootcount.timer

‎buildroot-external/rootfs-overlay/opt/yio/app-launch.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,5 @@ if [[ ! -d $YIO_APP_DIR ]] && [[ -d ${YIO_HOME}/app-previous ]]; then
1414
mv ${YIO_HOME}/app-previous $YIO_APP_DIR
1515
fi
1616

17-
${YIO_APP_DIR}/remote &
17+
# Do not fork! Otherwise systemd unit service doesn't work anymore
18+
${YIO_APP_DIR}/remote

0 commit comments

Comments
 (0)