[WIP] Make X230 and T430 use a universal flash-stage init

.circleci/config.yml

@@ -50,7 +50,7 @@ jobs:
 #          command: |
 #            ./build/make-4.2.1/make \
 #                CROSS=/cross/bin/x86_64-linux-musl- \
-#                --load 2 \
+#                CPUS=4 \
 #                V=1 \
 #                BOARD=qemu-linuxboot \
 #
@@ -59,13 +59,31 @@ jobs:
 #      - store-artifacts:
 #          path: build/qemu-linuxboot/hashes.txt
 
+      - run:
+          name: librem_mini
+          command: |
+            rm -rf build/librem_mini/* build/log/* && make CPUS=4 \
+                V=1 \
+                BOARD=librem_mini || (find ./build/ -name "*.log" -type f -mmin -1|while read log; do echo ""; echo '==>' "$log" '<=='; echo ""; cat $log;done; exit 1) \
+          no_output_timeout: 3h
+      - run:
+          name: Ouput librem_mini hashes
+          command: |
+            cat build/librem_mini/hashes.txt \
+      - run:
+          name: Archiving build logs for librem_mini
+          command: |
+             tar zcvf build/librem_mini/logs.tar.gz build/log/*
+      - store-artifacts:
+          path: build/librem_mini
+
       - run:
           name: x230-flash
           #We delete build/make-4.2.1/ directory until issue #799 is fixed.
           command: |
-            rm -rf build/x230-flash/* build/log/* && make --load 2 \
+            rm -rf build/x230-flash/* build/log/* && make CPUS=4 \
                 V=1 \
-                BOARD=x230-flash || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+                BOARD=x230-flash || (find ./build/ -name "*.log" -type f -mmin -1|while read log; do echo ""; echo '==>' "$log" '<=='; echo ""; cat $log;done; exit 1) \
           no_output_timeout: 3h
       - run:
           name: Ouput x230-flash hashes
@@ -81,9 +99,9 @@ jobs:
       - run:
           name: t430-flash
           command: |
-            rm -rf build/t430-flash/* build/log/* && make --load 2 \
+            rm -rf build/t430-flash/* build/log/* && make CPUS=4 \
                 V=1 \
-                BOARD=t430-flash || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+                BOARD=t430-flash || (find ./build/ -name "*.log" -type f -mmin -1|while read log; do echo ""; echo '==>' "$log" '<=='; echo ""; cat $log;done; exit 1) \
           no_output_timeout: 3h
       - run:
           name: Ouput t430-flash hashes
@@ -99,9 +117,9 @@ jobs:
       - run:
           name: t430
           command: |
-            rm -rf build/t430/* build/log/* && make --load 2 \
+            rm -rf build/t430/* build/log/* && make CPUS=4 \
                 V=1 \
-                BOARD=t430  || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+                BOARD=t430  || (find ./build/ -name "*.log" -type f -mmin -1|while read log; do echo ""; echo '==>' "$log" '<=='; echo ""; cat $log;done; exit 1) \
           no_output_timeout: 3h
       - run:
           name: Ouput t430 hashes
@@ -117,9 +135,9 @@ jobs:
       - run:
           name: x230
           command: |
-            rm -rf build/x230/* build/log/* && make --load 2 \
+            rm -rf build/x230/* build/log/* && make CPUS=4 \
                 V=1 \
-                BOARD=x230 || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+                BOARD=x230 || (find ./build/ -name "*.log" -type f -mmin -1|while read log; do echo ""; echo '==>' "$log" '<=='; echo ""; cat $log;done; exit 1) \
           no_output_timeout: 3h
       - run:
           name: Ouput x230 hashes
@@ -135,9 +153,9 @@ jobs:
       - run:
           name: x230-hotp-verification
           command: |
-            rm -rf build/x230-hotp-verification/* build/log/* && make --load 2 \
+            rm -rf build/x230-hotp-verification/* build/log/* && make CPUS=4 \
                 V=1 \
-                BOARD=x230-hotp-verification || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+                BOARD=x230-hotp-verification || (find ./build/ -name "*.log" -type f -mmin -1|while read log; do echo ""; echo '==>' "$log" '<=='; echo ""; cat $log;done; exit 1) \
           no_output_timeout: 3h
       - run:
           name: Ouput x230-hotp-verification hashes
@@ -153,9 +171,9 @@ jobs:
       - run:
           name: qemu-coreboot
           command: |
-            rm -rf build/qemu-coreboot/* build/log/* && make --load 2 \
+            rm -rf build/qemu-coreboot/* build/log/* && make CPUS=4 \
                 V=1 \
-                BOARD=qemu-coreboot || (find ./build/log/ -cmin 1|xargs tail; exit 1) \
+                BOARD=qemu-coreboot || (find ./build/ -name "*.log" -type f -mmin -1|while read log; do echo ""; echo '==>' "$log" '<=='; echo ""; cat $log;done; exit 1) \
           no_output_timeout: 3h
       - run:
           name: Output qemu-coreboot hashes

Makefile

@@ -16,7 +16,7 @@ INSTALL		:= $(pwd)/install
 log_dir		:= $(build)/log
 
 # Controls how many parallel jobs are invoked in subshells
-CPUS		:= $(shell nproc)
+CPUS		?= $(shell nproc)
 #MAKE_JOBS	?= -j$(CPUS) --max-load 16
 
 # Create the log directory if it doesn't already exist
@@ -36,6 +36,58 @@ DATE=`date --rfc-3339=seconds`
 
 # This is the correct version of Make
 
+# Check we have a suitable version of gawk
+# that's at least the same major version
+LOCAL_GAWK_VERSION := $(shell gawk --version 2>/dev/null | head -1 | cut -d' ' -f3 | cut -d, -f1)
+LOCAL_GAWK_MAJOR_VERSION := $(patsubst .%,.,$(LOCAL_GAWK_VERSION))
+include modules/gawk
+
+# Wrong version
+ifeq "" "$(filter $(LOCAL_GAWK_MAJOR_VERSION).%,$(gawk_version))"
+# Wrong gawk version detected -- build our local version
+# and re-invoke the Makefile with it instead.
+$(eval $(shell echo >&2 "$(DATE) Wrong gawk detected: $(LOCAL_GAWK_VERSION)"))
+HEADS_GAWK := $(build)/$(gawk_dir)/gawk
+
+# Once we have a suitable version of gawk, we can rerun make
+all linux cpio run: $(HEADS_GAWK)
+	LANG=C HEADS_GAWK=$(HEADS_GAWK) $(MAKE) $(MAKE_JOBS) $@
+%.clean %.vol %.menuconfig: $(HEADS_GAWK)
+	LANG=C HEADS_GAWK=$(HEADS_GAWK) $(MAKE) $@
+
+bootstrap: $(HEADS_GAWK)
+
+# How to download and build the correct version of gawk
+$(packages)/$(gawk_tar):
+	$(WGET) -O "$@.tmp" "$(gawk_url)"
+	if ! echo "$(gawk_hash)  $@.tmp" | sha256sum --check -; then \
+		exit 1 ; \
+	fi
+	mv "$@.tmp" "$@"
+
+$(build)/$(gawk_dir)/.extract: $(packages)/$(gawk_tar)
+	tar xf "$<" -C "$(build)"
+	touch "$@"
+
+$(build)/$(gawk_dir)/.patch: $(build)/$(gawk_dir)/.extract
+#	( cd "$(dir $@)" ; patch -p1 ) < "patches/gawk-$(gawk_version).patch"
+	touch "$@"
+
+$(build)/$(gawk_dir)/.configured: $(build)/$(gawk_dir)/.patch
+	cd "$(dir $@)" ; \
+	./configure 2>&1 \
+	| tee "$(log_dir)/gawk.configure.log" \
+	$(VERBOSE_REDIRECT)
+	touch "$@"
+
+$(HEADS_GAWK): $(build)/$(gawk_dir)/.configured
+	$(MAKE) -C "$(dir $@)" $(MAKE_JOBS) \
+		2>&1 \
+		| tee "$(log_dir)/gawk.log" \
+		$(VERBOSE_REDIRECT)
+endif
+
+
 BOARD		?= qemu-coreboot
 CONFIG		:= $(pwd)/boards/$(BOARD)/$(BOARD).config
 
@@ -124,6 +176,10 @@ CROSS_TOOLS_NOCC := \
 	PKG_CONFIG_PATH="$(INSTALL)/lib/pkgconfig" \
 	PKG_CONFIG_SYSROOT_DIR="$(INSTALL)" \
 
+ifneq "$(HEADS_GAWK)" ""
+CROSS_TOOLS_NOCC += AWK=$(HEADS_GAWK)
+endif
+
 CROSS_TOOLS := \
 	CC="$(heads_cc)" \
 	$(CROSS_TOOLS_NOCC) \

boards/librem_mini/librem_mini.config

@@ -26,7 +26,7 @@ CONFIG_HOTPKEY=y
 
 CONFIG_LINUX_USB=y
 
-export CONFIG_TPM=y
+export CONFIG_TPM=n
 export CONFIG_TPM_NO_LUKS_DISK_UNLOCK=y
 
 export CONFIG_BOOTSCRIPT=/bin/gui-init

boards/t430-flash/t430-flash.config

@@ -13,7 +13,7 @@ CONFIG_LINUX_CONFIG=config/linux-x230-flash.config
 CONFIG_LINUX_USB=y
 CONFIG_LINUX_E1000E=y
 
-export CONFIG_BOOTSCRIPT=/bin/t430-flash.init
+export CONFIG_BOOTSCRIPT=/bin/flash.init
 export CONFIG_FLASHROM_OPTIONS="--force --noverify-all -p internal --ifd --image bios"
 
 # This board is "special" in that we only want the top 4 MB of the ROM

boards/x230-flash/x230-flash.config

@@ -18,7 +18,7 @@ CONFIG_LINUX_CONFIG=config/linux-x230-flash.config
 CONFIG_LINUX_USB=y
 CONFIG_LINUX_E1000E=y
 
-export CONFIG_BOOTSCRIPT=/bin/x230-flash.init
+export CONFIG_BOOTSCRIPT=/bin/flash.init
 export CONFIG_FLASHROM_OPTIONS="--force --noverify-all -p internal --ifd --image bios"
 
 # This board is "special" in that we only want the top 4 MB of the ROM

config/coreboot-librem_mini.config

@@ -1,4 +1,5 @@
-CONFIG_LOCALVERSION="PureBoot-beta"
+CONFIG_LOCALVERSION="heads"
+CONFIG_ANY_TOOLCHAIN=y
 CONFIG_VENDOR_PURISM=y
 CONFIG_INTEL_GMA_VBT_FILE="3rdparty/purism-blobs/mainboard/purism/librem_whl/vbt.bin"
 CONFIG_IFD_BIN_PATH="3rdparty/purism-blobs/mainboard/purism/librem_whl/flashdescriptor.bin"

initrd/bin/t430-flash.init → initrd/bin/flash.init

@@ -20,7 +20,7 @@ echo ''
 echo 'To install from flash drive:'
 echo ''
 echo '  mount -o ro /dev/sdb1 /media'
-echo '  flash.sh /media/t430.rom'
+echo '  flash.sh /media/$CONFIG_BOARD.rom'
 echo ''
 
 exec /bin/ash

initrd/bin/kexec-save-default

@@ -120,7 +120,7 @@ if [ ! -d $paramsdir ]; then
 	mkdir -p $paramsdir \
 	|| die "Failed to create params directory"
 fi
-rm "$paramsdir/kexec_default.*.txt" 2>/dev/null || true
+rm $paramsdir/kexec_default.*.txt 2>/dev/null || true
 echo "$entry" > $ENTRY_FILE
 cd $bootdir && kexec-boot -b "$bootdir" -e "$entry" -f | \
 	xargs sha256sum > $HASH_FILE \

initrd/etc/functions

@@ -25,14 +25,20 @@ recovery() {
 	if [ "$CONFIG_TPM" = y ]; then
 		tpm extend -ix 4 -ic recovery
 	fi
-	echo >&2 "!!!!! Starting recovery shell"
-	sleep 1
 
-	if [ -x /bin/setsid ]; then
-		exec /bin/setsid -c /bin/ash
-	else
-		exec /bin/ash
-	fi
+	while [ true ]
+	do
+		echo >&2 "!!!!! Starting recovery shell"
+		sleep 1
+
+		if [ -x /bin/setsid ]; then
+			/bin/setsid -c /bin/ash
+		else
+			/bin/ash
+		fi
+		# clear screen
+		printf "\033c"
+	done
 }
 
 pause_recovery() {

modules/coreboot

@@ -61,18 +61,9 @@ coreboot_target := \
 coreboot_output := coreboot.rom
 coreboot_depend += linux initrd $(musl_dep)
 
-ifeq "$(CONFIG_COREBOOT_VERSION)" "4.8.1"
-COREBOOT_TOOLCHAIN=""
-$(COREBOOT_TOOLCHAIN):
-else
-COREBOOT_TOOLCHAIN="$(build)/$(coreboot_base_dir)/.xcompile"
-$(COREBOOT_TOOLCHAIN): $(build)/$(coreboot_base_dir)/.canary
-	$(MAKE) -C "$(build)/$(coreboot_base_dir)" CPUS=`nproc` crossgcc-i386
-endif
-
-$(build)/$(coreboot_dir)/.configured: $(COREBOOT_IASL) $(COREBOOT_TOOLCHAIN)
+$(build)/$(coreboot_dir)/.configured: $(COREBOOT_IASL)
 $(COREBOOT_IASL): $(build)/$(coreboot_base_dir)/.canary
-	$(MAKE) -C "$(build)/$(coreboot_base_dir)" CPUS=`nproc` iasl
+	$(MAKE) -C "$(build)/$(coreboot_base_dir)" CPUS=$$CPUS iasl
 
 # Force a rebuild if the inputs have changed
 $(build)/$(coreboot_dir)/.build: \

modules/flashtools

@@ -2,14 +2,11 @@ modules-$(CONFIG_FLASHTOOLS) += flashtools
 
 flashtools_depends := $(musl_dep)
 
-flashtools_version := git
-flashtools_repo := https://github.com/osresearch/flashtools
-
-flashtools_version := 0.0.1
+flashtools_version := 40d5170e84a3822552df7a500cd00aa870fdfe76
 flashtools_dir := flashtools-$(flashtools_version)
 flashtools_tar := flashtools-$(flashtools_version).tar.gz
-flashtools_url := https://github.com/osresearch/flashtools/archive/v$(flashtools_version).tar.gz
-flashtools_hash := e8205aa3d19e536080f5974ed06ab9a88c4c3f37870c2f6a3a08a2f39302c22c
+flashtools_url := https://github.com/osresearch/flashtools/archive/$(flashtools_version).tar.gz
+flashtools_hash := dca7f4fd129509bdcbf5e4646905d6dd82e91061d7faf62bbe7193c31bb7cd4c
 
 flashtools_target := \
 	$(CROSS_TOOLS) \

modules/gawk

@@ -0,0 +1,14 @@
+# This is not added to the module list since it is a special case
+# of things that need to be built for the host system, instead of the
+# target platform.
+#modules += gawk
+
+gawk_version := 4.2.1
+gawk_dir := gawk-$(gawk_version)
+gawk_tar := gawk-$(gawk_version).tar.xz
+
+gawk_url := http://gnu.mirror.constant.com/gawk/$(gawk_tar)
+gawk_hash := d1119785e746d46a8209d28b2de404a57f983aa48670f4e225531d3bdc175551
+
+# This is built for the local machine, not the target, so it doesn't have any
+# of the build instructions.

modules/linuxboot

@@ -21,7 +21,7 @@ linuxboot_configure := \
 	if [ "$(linuxboot_board)" = "qemu" ]; then \
 		echo >&2 "Pre-building edk2 OVMF" ; \
 		( cd $(build)/$(linuxboot_base_dir)/edk2/OvmfPkg ; \
-		./build.sh -n `nproc` \
+		./build.sh -n $$CPUS \
 		) || exit 1 ; \
 	fi ; \
 	touch .config ; \

patches/coreboot-4.12/0010-cross-compiler-support.patch

@@ -0,0 +1,28 @@
+diff --git a/Makefile b/Makefile
+index 027aae2faa..8251a9a478 100644
+--- a/Makefile
++++ b/Makefile
+@@ -176,6 +176,23 @@ $(if $(wildcard .xcompile)$(NOCOMPILE),,$(eval $(shell util/xcompile/xcompile $(
+
+ -include .xcompile
+
++ifneq "$(CROSS)" ""
++  $(info coreboot: Using $(CROSS)gcc)
++  CROSS_COMPILE_x86_32	:= $(CROSS)
++  CC_x86_32		:= $(CROSS_COMPILE_x86_32)gcc
++  CPP_x86_32		:= $(CROSS_COMPILE_x86_32)cpp
++  AS_x86_32		:= $(CROSS_COMPILE_x86_32)as --32
++  LD_x86_32		:= $(CROSS_COMPILE_x86_32)ld.bfd -b elf32-i386 -melf_i386
++  NM_x86_32		:= $(CROSS_COMPILE_x86_32)nm
++  OBJCOPY_x86_32	:= $(CROSS_COMPILE_x86_32)objcopy
++  OBJDUMP_x86_32	:= $(CROSS_COMPILE_x86_32)objdump
++  READELF_x86_32	:= $(CROSS_COMPILE_x86_32)readelf
++  STRIP_x86_32		:= $(CROSS_COMPILE_x86_32)strip
++  AR_x86_32		:= $(CROSS_COMPILE_x86_32)ar
++  GNATBIND_x86_32	:= $(CROSS_COMPILE_x86_32)gnatbind
++  COMPILER_RT_x86_32	:= $(shell $(CC_x86_32) --print-libgcc-file-name)
++endif
++
+ ifneq ($(XCOMPILE_COMPLETE),1)
+ $(shell rm -f .xcompile)
+ $(error .xcompile deleted because it's invalid. \