Switch from cpu based to u-root only model

This allows us to still mount from dev host without incurring overhead of running sshd under simulation.

Signed-off-by: Eric Van Hensbergen <ericvh@kernel.org>

.devcontainer/Dockerfile

@@ -38,7 +38,7 @@ RUN if [ `uname -m` = "aarch64" ]; then \
     else \
         export TARGETGOARCH="amd64"; \
     fi; \
-    wget https://go.dev/dl/go1.19.linux-$TARGETGOARCH.tar.gz; \
+    wget https://go.dev/dl/go1.20.3.linux-$TARGETGOARCH.tar.gz; \
     tar xf go*.tar.gz;rm go*.tar.gz;mv go /usr/local
 ENV GOROOT /usr/local/go
 ENV PATH /usr/local/go/bin:$PATH

.gitignore

@@ -0,0 +1 @@
+workspaces

Makefile

@@ -12,7 +12,7 @@ standalone: fvp-docker qemu downloadlatest
 
 lkvm: $(ARTIFACTS)/lkvm
 firmware: $(ARTIFACTS)/rmm.img $(ARTIFACTS)/bl1-linux.bin $(ARTIFACTS)/fip-linux.bin
-initramfs: $(ARTIFACTS)/initramfs.cpio $(ARTIFACTS)/cpu
+initramfs: $(ARTIFACTS)/initramfs.cpio $(ARTIFACTS)/cpu $(ARTIFACTS)/cpud
 linux: $(ARTIFACTS)/Image $(ARTIFACTS)/Image.guest
 qemu: $(ARTIFACTS)/qemu-system-aarch64
 
@@ -31,8 +31,12 @@ downloadlatest: $(ARTIFACTS)
 	wget -O $(ARTIFACTS)/lkvm https://github.com/ericvh/cca-cpu/releases/latest/download/lkvm > /dev/null 2>&1
 
 .PHONY: fvp-docker
-fvp-docker:
+fvp-docker: $(ARTIFACTS)/cpud
+	cp $(ARTIFACTS)/cpud fvp
+	cp $(ARTIFACTS)/identity.pub fvp
 	docker build -t cca-cpu/fvp fvp
+	rm -f fvp/cpud
+	rm -f fvp/identity.pub
 
 $(SRC_DIR)/kvmtool-cca/.git:
 	git clone --depth 1 -b cca/rfc-v1 https://git.gitlab.arm.com/linux-arm/kvmtool-cca.git $(SRC_DIR)/kvmtool-cca
@@ -54,6 +58,9 @@ $(ARTIFACTS)/initramfs.cpio:
 $(ARTIFACTS)/cpu:
 	make -C u-root-initramfs cpu
 
+$(ARTIFACTS)/cpud:
+	make -C u-root-initramfs cpud
+
 $(ARTIFACTS)/qemu-system-aarch64:
 	docker build --platform arm64 -t $(CONTAINER_NAME)-qemu-builder qemu
 	docker run --platform arm64 --rm -i -t -v /var/run/docker.sock:/var/run/docker.sock -v $(VOLUME_NAME):/workspaces --workdir=/workspaces/cca-cpu $(CONTAINER_NAME)-qemu-builder:latest make qemu-build

README.md

@@ -1,9 +1,127 @@
-# Cpu for CCA Demo
+# CPU for CCA Demo
+
+## Goal
+
+Build an environment which runs without buildroots to allow for easy development and test of CCA environments by leveraging u-root,
+cpu, and virtio 9p backmounts.  For more information on any of these environments see the links session at the end of this document.
 
 ## Components
 
+- _.devcontainer_ : devcontainer setup including primary Dockerfile 
+- _bin_ : scripts for executing the demo
 - _u-root-initramfs_ : u-root initramfs to avoid buildroots
 - _fvp_ : Container formula for FVP
 - _linux-cca_: kernels for guest and host
+- _tf-a_ : CCA enabled TF-A firmware
+- _tf-rmm_ : Realm Management Monitor for managing CCA realms
+- _qemu_ : CCA enabled qemu build environment.
+
+## Build
+
+The repository is setup as a vscode devcontainer, if you clone the repository into a vscode dev-container volume many things will be setup for you.
+The system is setup to build components using Docker volumes.  .devcontainer contains a Dockerfile for the primary development environment which will
+support building golang components, the linux kernel, TF-A, kvmtool, and qemu.
+
+In the devcontainer, this repo ends up in a /worksapces/cca-cpu directory - instead of cloning components into its tree, it clones them as peer directories
+and uses a peer build directory for intermediate files and binaries.  Final binaries are put in the /workspaces/artifacts directory.  If you aren't in a 
+devcontainer, please ensure you are okay with peer directories to the cloned directory being created or place everything in its own hierarchy.
+
+### Dev Container
+
+If you are already running in an arm64 dev container, you should be able to run:
+```
+make
+```
+and it will take care of building everything.  My native environment for development/test is an Arm laptop running OSX with docker desktop installed.
+It should be possible to run in linux/arm64 or even WSL/arm64 enviornments, but I have not directly tested.  It should also be possible to use binfmt
+and qemu-user-static to run cross-arch docker container environments (although build will be significantly slower).
+
+### Standalone and using pre-built artifacts
+
+There is a github CI action which builds many of the binaries, but not the FVP docker image -- so you'll need to build that yourself:
+```
+docker build -t cca-cpu/fvp:latest fvp
+```
+Then you can grab the latest artifacts and unpack them to artifacts.  _NOTE_: if you can't create /workspaces in your environment, then you need to modify the paths in all the scripts.
+```
+make downloadlatest
+sudo mkdir -p /workspaces
+sudo mv ../artifacts /workspaces/artifacts
+```
+
+### Standalone and building components
+
+This is not currently well tested and probably has bugs.
+
+```
+docker run -i -t -v `pwd`:/workspaces/cca-cpu -w /workspaces/cca-cpu ghcr.io/ericvh/cca-cpu /bin/bash
+make
+``` 
+
+## Use
+
+Once everything is built, you should be able to run the demo:
+
+```
+vscode ➜ /workspaces/cca-cpu (fvp-cpu) $ bin/start-fvp.bash 
+Starting FVP...hit ^a-d to exit
+...
+[    3.723671] Freeing unused kernel memory: 7936K
+[    3.724486] Run /init as init process
+2023/04/21 18:40:47 Welcome to u-root!
+                              _
+   _   _      _ __ ___   ___ | |_
+  | | | |____| '__/ _ \ / _ \| __|
+  | |_| |____| | | (_) | (_) | |_
+   \__,_|    |_|  \___/ \___/ \__|
+
+[    3.837078] cgroup: Unknown subsys name 'net_cls'
+root@172:/# /workspaces/cca-cpu/bin/test-lkvm.bash
+  # lkvm run -k /mnt/workspaces/artifacts/Image.guest -m 256 -c 1 --name guest-150
+[    1.345860] cacheinfo: Unable to detect cache hierarchy for CPU 0
+[    1.453459] loop: module loaded
+[    1.457073] tun: Universal TUN/TAP device driver, 1.6
+[    1.497107] net eth0: Fail to set guest offload.
+[    1.498512] virtio_net virtio2 eth0: set_features() failed (-22); wanted 0x0000000000134829, left 0x0080000000134829
+[    1.523209] VFIO - User Level meta-driver version: 0.3
+[    1.531093] SMCCC: SOC_ID: ARCH_SOC_ID not implemented, skipping ....
+[    1.537293] NET: Registered PF_PACKET protocol family
+[    1.538770] 9pnet: Installing 9P2000 support
+[    1.558259] NET: Registered PF_VSOCK protocol family
+[    1.616972] registered taskstats version 1
+[    1.730594] Freeing unused kernel memory: 1088K
+[    1.733815] Run /init as init process
+1970/01/01 00:00:01 Welcome to u-root!
+                              _
+   _   _      _ __ ___   ___ | |_
+  | | | |____| '__/ _ \ / _ \| __|
+  | |_| |____| | | (_) | (_) | |_
+   \__,_|    |_|  \___/ \___/ \__|
+
+[    2.612950] cgroup: Unknown subsys name 'net_cls'
+root@(none):/# 
+```
+
+The second invocation (test-lkvm) takes a couple minutes due to dialated simulation time.
+Both the FVP linux host (root@172 prompt) and the CCA realm (root@unknown) have the namespace
+from the originating vscode dev container mounted (this includes /workspaces, /home, as well
+as /usr, /bin, and /lib) -- so you can run all executables just like you would in your
+devcontainer.
+
+This should allow you to develop your own code in whatever language you like and run it inside
+the enclave.
+
+Everything in the scripts is run from inside screen, so use screen key combos for scrollback, etc.
+There are other screens inside the FVP container which have the RMM console, debug console, and FVP console itself.
+
+## Future
+
+- We aren't doing anything to attest the realm or applications within it yet.
+- Provide a vscode feature template which allows you to include appropraite artifacts and start scripts without operating in cca-cpu so that you can incorporate it into any (arm64) development environment
+- Enable networking and use cpu to provide multi-session access to realm.
 
+## Links
 
+- https://github.com/u-root/u-root
+- https://github.com/u-root/cpu
+- https://www.arm.com/architecture/security-features/arm-confidential-compute-architecture

bin/namespace.elv

@@ -0,0 +1,16 @@
+#!/bbin/elvish
+
+var ns = [ "/workspaces" "/home" "/lib" "/usr" "/bin" "/etc" ]
+
+mount -t 9p -o trans=virtio,protocol=9p2000.L FM /mnt
+
+for p $ns {
+    mkdir -p $p
+    mount -t none -o bind /mnt$p $p
+}
+
+if ( > (count $args) 0 ) {  
+    elvish -c $@args
+} else {
+    /bin/bash
+}

bin/start-fvp-stage2.bash

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+export ARTIFACTS="/workspaces/artifacts"
+export LINUX_BASE_ADDR="0x84000000"
+export INITRD_BASE_ADDR="0x83000000"
+
+screen -dmS FVP -t 'FVP' /opt/fvp/Base_RevC_AEMvA_pkg/models/Linux64*/FVP_Base_RevC-2xAEMvA \
+		-C bp.refcounter.non_arch_start_at_default=1 \
+		-C bp.refcounter.use_real_time=0 \
+		-C bp.ve_sysregs.exit_on_shutdown=1 \
+		-C cache_state_modelled=0 \
+		-C bp.dram_size=2 \
+		-C bp.dram_metadata.is_enabled=1 \
+		-C bp.secure_memory=1 \
+		-C cluster0.NUM_CORES=4 \
+		-C cluster0.PA_SIZE=48 \
+		-C cluster0.ecv_support_level=2 \
+		-C cluster0.gicv3.cpuintf-mmap-access-level=2 \
+		-C cluster0.gicv3.without-DS-support=1 \
+		-C cluster0.gicv4.mask-virtual-interrupt=1 \
+		-C cluster0.has_arm_v8-6=1 \
+		-C cluster0.has_amu=1 \
+		-C cluster0.has_branch_target_exception=1 \
+		-C cluster0.rme_support_level=2 \
+		-C cluster0.has_rndr=1 \
+		-C cluster0.has_v8_7_pmu_extension=2 \
+		-C cluster0.max_32bit_el=-1 \
+		-C cluster0.restriction_on_speculative_execution=2 \
+		-C cluster0.restriction_on_speculative_execution_aarch32=2 \
+		-C cluster0.stage12_tlb_size=1024 \
+		-C cluster0.check_memory_attributes=0 \
+		-C pci.pci_smmuv3.mmu.SMMU_AIDR=2 \
+		-C pci.pci_smmuv3.mmu.SMMU_IDR0=0x0046123B \
+		-C pci.pci_smmuv3.mmu.SMMU_IDR1=0x00600002 \
+		-C pci.pci_smmuv3.mmu.SMMU_IDR3=0x1714 \
+		-C pci.pci_smmuv3.mmu.SMMU_IDR5=0xFFFF0475 \
+		-C pci.pci_smmuv3.mmu.SMMU_S_IDR1=0xA0000002 \
+		-C pci.pci_smmuv3.mmu.SMMU_S_IDR2=0 \
+		-C pci.pci_smmuv3.mmu.SMMU_S_IDR3=0 \
+		-C pci.pci_smmuv3.mmu.SMMU_ROOT_IDR0=3 \
+		-C pci.pci_smmuv3.mmu.SMMU_ROOT_IIDR=0x43B \
+		-C pci.pci_smmuv3.mmu.root_register_page_offset=0x20000 \
+		-C pctl.startup=0.0.0.0 \
+		-C bp.smsc_91c111.enabled=1 \
+		-C bp.hostbridge.userNetworking=1 \
+		-C bp.hostbridge.userNetPorts=17010=17010 \
+		-C bp.pl011_uart0.uart_enable=1 \
+		-C bp.pl011_uart1.uart_enable=1 \
+		-C bp.pl011_uart2.uart_enable=1 \
+		-C bp.pl011_uart3.uart_enable=1 \
+		-C bp.terminal_0.terminal_command="screen -dmS Linux  -t 'Linux' telnet localhost %port" \
+		-C bp.terminal_1.terminal_command="screen -dmS RMM -t 'RMM' telnet localhost %port" \
+		-C bp.terminal_2.terminal_command="screen -dmS Debug -t 'Debug' telnet localhost %port" \
+		-C bp.terminal_3.terminal_command="screen -dmS Unknown -t 'Unknown' telnet localhost %port" \
+		-C bp.secureflashloader.fname=$ARTIFACTS/bl1-linux.bin \
+		-C bp.flashloader0.fname=$ARTIFACTS/fip-linux.bin \
+		--data cluster0.cpu0=$ARTIFACTS/Image@$LINUX_BASE_ADDR \
+        --data cluster0.cpu0=$ARTIFACTS/initramfs.cpio@$INITRD_BASE_ADDR \
+		-C bp.virtiop9device.root_path=/
+sleep 5
+screen -r Linux

bin/start-fvp.bash

@@ -1,64 +1,3 @@
 #!/bin/bash
-
-export ARTIFACTS="/workspaces/artifacts"
-export LINUX_BASE_ADDR="0x84000000"
-export INITRD_BASE_ADDR="0x83000000"
-
-export HOSTNAME=`hostname`
-export VOLUME_NAME=`docker inspect $HOSTNAME | jq -r '.[].Mounts[] | select(.Destination | startswith("/workspaces")) | .Name'`
-docker run --rm --name fvp -i -t \
-		-v $VOLUME_NAME:/workspaces \
-		-p 17010:17010 \
-		cca-cpu/fvp ./fvp \
-		-C bp.refcounter.non_arch_start_at_default=1 \
-		-C bp.refcounter.use_real_time=0 \
-		-C bp.ve_sysregs.exit_on_shutdown=1 \
-		-C cache_state_modelled=0 \
-		-C bp.dram_size=2 \
-		-C bp.dram_metadata.is_enabled=1 \
-		-C bp.secure_memory=1 \
-		-C cluster0.NUM_CORES=4 \
-		-C cluster0.PA_SIZE=48 \
-		-C cluster0.ecv_support_level=2 \
-		-C cluster0.gicv3.cpuintf-mmap-access-level=2 \
-		-C cluster0.gicv3.without-DS-support=1 \
-		-C cluster0.gicv4.mask-virtual-interrupt=1 \
-		-C cluster0.has_arm_v8-6=1 \
-		-C cluster0.has_amu=1 \
-		-C cluster0.has_branch_target_exception=1 \
-		-C cluster0.rme_support_level=2 \
-		-C cluster0.has_rndr=1 \
-		-C cluster0.has_v8_7_pmu_extension=2 \
-		-C cluster0.max_32bit_el=-1 \
-		-C cluster0.restriction_on_speculative_execution=2 \
-		-C cluster0.restriction_on_speculative_execution_aarch32=2 \
-		-C cluster0.stage12_tlb_size=1024 \
-		-C cluster0.check_memory_attributes=0 \
-		-C pci.pci_smmuv3.mmu.SMMU_AIDR=2 \
-		-C pci.pci_smmuv3.mmu.SMMU_IDR0=0x0046123B \
-		-C pci.pci_smmuv3.mmu.SMMU_IDR1=0x00600002 \
-		-C pci.pci_smmuv3.mmu.SMMU_IDR3=0x1714 \
-		-C pci.pci_smmuv3.mmu.SMMU_IDR5=0xFFFF0475 \
-		-C pci.pci_smmuv3.mmu.SMMU_S_IDR1=0xA0000002 \
-		-C pci.pci_smmuv3.mmu.SMMU_S_IDR2=0 \
-		-C pci.pci_smmuv3.mmu.SMMU_S_IDR3=0 \
-		-C pci.pci_smmuv3.mmu.SMMU_ROOT_IDR0=3 \
-		-C pci.pci_smmuv3.mmu.SMMU_ROOT_IIDR=0x43B \
-		-C pci.pci_smmuv3.mmu.root_register_page_offset=0x20000 \
-		-C pctl.startup=0.0.0.0 \
-		-C bp.smsc_91c111.enabled=1 \
-		-C bp.hostbridge.userNetworking=1 \
-		-C bp.hostbridge.userNetPorts=17010=17010 \
-		-C bp.pl011_uart0.uart_enable=1 \
-		-C bp.pl011_uart1.uart_enable=1 \
-		-C bp.pl011_uart2.uart_enable=1 \
-		-C bp.pl011_uart3.uart_enable=1 \
-		-C bp.terminal_0.terminal_command="screen -dmS Linux  -t 'Linux' telnet localhost %port" \
-		-C bp.terminal_1.terminal_command="screen -dmS RMM -t 'RMM' telnet localhost %port" \
-		-C bp.terminal_2.terminal_command="screen -dmS Debug -t 'Debug' telnet localhost %port" \
-		-C bp.terminal_3.terminal_command="screen -dmS Unknown -t 'Unknown' telnet localhost %port" \
-		-C bp.secureflashloader.fname=$ARTIFACTS/bl1-linux.bin \
-		-C bp.flashloader0.fname=$ARTIFACTS/fip-linux.bin \
-		--data cluster0.cpu0=$ARTIFACTS/Image@$LINUX_BASE_ADDR \
-        --data cluster0.cpu0=$ARTIFACTS/initramfs.cpio@$INITRD_BASE_ADDR \
-		-C bp.virtiop9device.root_path=/work
+echo Starting FVP...hit ^a-d to exit
+docker run -i -t --rm --privileged --volumes-from `hostname` --name fvp -p 17010:17010 cca-cpu/fvp /workspaces/cca-cpu/bin/start-fvp-stage2.bash

bin/test-lkvm.bash

@@ -1,6 +1,8 @@
-#!/bin/bash
-FVP_IP=`docker inspect fvp | jq -r '.[].NetworkSettings.IPAddress'`
-ln -f -s /tmp/local/tmp $HOME/.lkvm
-PWD=/ /workspaces/artifacts/cpu -namespace "/workspaces:/lib:/lib64:/usr:/bin:/etc:/home" $FVP_IP /workspaces/artifacts/lkvm run --realm --irqchip=gicv3-its --console=virtio \
-    -c 1 -m 256 \
-    -k /workspaces/artifacts/Image.guest -i /workspaces/artifacts/initramfs.cpio
+#!/bbin/elvish
+#FVP_IP=`docker inspect fvp | jq -r '.[].NetworkSettings.IPAddress'`
+#ln -f -s /tmp/local/tmp $HOME/.lkvm
+#PWD=/ /workspaces/artifacts/cpu -namespace "/workspaces:/lib:/lib64:/usr:/bin:/etc:/home" $FVP_IP /workspaces/artifacts/lkvm run --realm --irqchip=gicv3-its --console=virtio \
+#    -c 1 -m 256 \
+#    -k /workspaces/artifacts/Image.guest -i /workspaces/artifacts/initramfs.cpio
+
+/mnt/workspaces/artifacts/lkvm run --realm --irqchip=gicv3-its --console=virtio -c 1 -m 256 -k /mnt/workspaces/artifacts/Image.guest -i /mnt/workspaces/artifacts/initramfs.cpio --9p /mnt,FM --rng

fvp/Dockerfile

@@ -16,3 +16,5 @@ RUN if [ `uname -m` = "aarch64" ] ; then wget https://developer.arm.com/-/media/
                                     else wget https://developer.arm.com/-/media/Files/downloads/ecosystem-models/FVP_Base_RevC-2xAEMvA_11.21_15_Linux64.tgz ; fi \
     && tar xf F*.tgz && rm *.tgz
 RUN ln -s Base_RevC_AEMvA_pkg/models/Linux64*/FVP_Base_RevC-2xAEMvA fvp
+ADD cpud /usr/bin/cpud
+ADD identity.pub key.pub

u-root-initramfs/Makefile

@@ -6,11 +6,13 @@ include ../common.mk
 
 $(SRC_DIR)/cpu/.git:
 	git clone --depth 1 https://github.com/u-root/cpu.git $(SRC_DIR)/cpu
-	patch -N -p 1 -d $(SRC_DIR)/cpu -i $(ROOTDIR)/cca-cpu/u-root-initramfs/remove-init-tasks.patch
 
 $(ARTIFACTS)/cpu: $(SRC_DIR)/cpu/.git
 	cd $(SRC_DIR)/cpu && go build -o $(ARTIFACTS)/cpu $(SRC_DIR)/cpu/cmds/cpu/.
 
+$(ARTIFACTS)/cpud: $(SRC_DIR)/cpu/.git
+	cd $(SRC_DIR)/cpu && go build -o $(ARTIFACTS)/cpud $(SRC_DIR)/cpu/cmds/cpud/.
+
 $(SRC_DIR)/u-root/.git:
 	git clone --depth 1 https://github.com/u-root/u-root.git $(SRC_DIR)/u-root
 
@@ -22,14 +24,18 @@ ${HOME}/.ssh/identity:
 	mkdir -p ${HOME}/.ssh
 	ssh-keygen -N "" -f ${HOME}/.ssh/identity
 
+$(ARTIFACTS)/identity.pub: ${HOME}/.ssh/identity
+	cp ${HOME}/.ssh/identity.pub $(ARTIFACTS)
+
 $(ARTIFACTS)/initramfs.cpio: $(SRC_DIR)/cpu/.git $(BUILDS_DIR)/u-root/u-root ${HOME}/.ssh/identity
-	cd $(SRC_DIR)/cpu && $(BUILDS_DIR)/u-root/u-root -o $(ARTIFACTS)/initramfs.cpio -files ${HOME}/.ssh/identity.pub:key.pub -files /mnt -uroot-source $(SRC_DIR)/u-root -initcmd=/bbin/cpud $(SRC_DIR)/u-root/cmds/core/elvish cmds/cpud
+	cd $(SRC_DIR)/u-root && $(BUILDS_DIR)/u-root/u-root -o $(ARTIFACTS)/initramfs.cpio -files ${HOME}/.ssh/identity.pub:key.pub -files /mnt -files /workspaces/cca-cpu/bin/namespace.elv:/bbin/namespace.elv -uroot-source $(SRC_DIR)/u-root -uinitcmd "/bbin/namespace.elv" ./cmds/core/elvish ./cmds/core/mount ./cmds/core/init ./cmds/core/mkdir
 
 .PHONY: build
 build: $(ARTIFACTS)/initramfs.cpio $(ARTIFACTS)/cpu
 
-.PHONY: cpu
+.PHONY: cpu cpud
 cpu: $(ARTIFACTS)/cpu
+cpud: $(ARTIFACTS)/cpud $(ARTIFACTS)/identity.pub
 
 .PHONY: initramfs.cpio
 initramfs.cpio: $(ARTIFACTS)/initramfs.cpio