Bump Checkpointing Script + Docs

.github/scripts/defaults.sh

@@ -36,6 +36,8 @@ grouping["group-fpga"]="arty35t arty100t nexysvideo vc707 vcu118"
 # key value store to get the build strings
 declare -A mapping
 mapping["chipyard-rocket"]=" CONFIG=QuadChannelRocketConfig"
+# TODO: Verilator chokes on cospikeCheckpointingRocketConfig
+# mapping["chipyard-dmirocket"]=" CONFIG=dmiCospikeCheckpointingRocketConfig"
 mapping["chipyard-dmirocket"]=" CONFIG=dmiRocketConfig"
 mapping["chipyard-mempress"]=" CONFIG=MempressRocketConfig"
 mapping["chipyard-compressacc"]=" CONFIG=ZstdCompressorRocketConfig"
@@ -45,9 +47,9 @@ mapping["chipyard-manymmioaccels"]=" CONFIG=ManyMMIOAcceleratorRocketConfig"
 mapping["chipyard-nvdla"]=" CONFIG=SmallNVDLARocketConfig verilog"
 mapping["chipyard-hetero"]=" CONFIG=LargeBoomAndRocketConfig"
 mapping["chipyard-boomv3"]=" CONFIG=MediumBoomV3CosimConfig"
-mapping["chipyard-dmiboomv3"]=" CONFIG=dmiMediumBoomV3CosimConfig"
+mapping["chipyard-dmiboomv3"]=" CONFIG=dmiCheckpointingMediumBoomV3Config"
 mapping["chipyard-boomv4"]=" CONFIG=MediumBoomV4CosimConfig"
-mapping["chipyard-dmiboomv4"]=" CONFIG=dmiMediumBoomV4CosimConfig"
+mapping["chipyard-dmiboomv4"]=" CONFIG=dmiCheckpointingMediumBoomV4Config"
 mapping["chipyard-spike"]=" CONFIG=SpikeConfig EXTRA_SIM_FLAGS='+spike-ipc=10'"
 mapping["chipyard-gemmini"]=" CONFIG=GemminiRocketConfig"
 mapping["chipyard-cva6"]=" CONFIG=CVA6Config"

.github/scripts/run-tests.sh

@@ -42,9 +42,12 @@ case $1 in
         run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/hello.riscv
         ;;
     chipyard-dmirocket)
-        # Test checkpoint-restore
-        $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -i 10000
-        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch
+        # Test checkpoint-restore without cospike
+        # TODO: This is broken on verilator for some reason
+        # $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -i 10000
+        # run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.unused.10000.defaultspikedts.loadarch EXTRA_SIM_FLAGS="+cospike-enable=0"
+        # Test cospike without checkpoint-restore
+        run_binary BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv LOADMEM=1
         ;;
     chipyard-boomv3)
         run_bmark
@@ -58,12 +61,12 @@ case $1 in
     chipyard-dmiboomv3)
         # Test checkpoint-restore
         $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -i 10000
-        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch
+        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.unused.10000.defaultspikedts.loadarch
         ;;
     chipyard-dmiboomv4)
         # Test checkpoint-restore
         $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -i 10000
-        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch
+        run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.unused.10000.defaultspikedts.loadarch
         ;;
     chipyard-spike)
         run_bmark

docs/Advanced-Concepts/Architectural-Checkpoints.rst

@@ -9,16 +9,16 @@ The checkpoints include the contents of cacheable memory, core architectural reg
 RTL simulations of SoCs can resume execution from checkpoints after restoring the architectural state.
 
 .. note::
-   Currently, only checkpoints of single-core systems are supported
+   Currently, only checkpoints of single-core systems are supported.
 
 Generating Checkpoints
 ------------------------
 
-``scripts/generate-ckpt.sh`` is a script that runs spike with the right commands to generate an architectural checkpoint
+``scripts/generate-ckpt.sh`` is a script that runs Spike with the right commands to generate an architectural checkpoint.
 ``scripts/generate-ckpt.sh -h`` lists options for checkpoint generation.
 
 Example: run the ``hello.riscv`` binary for 1000 instructions before generating a checkpoint.
-This should produce a directory named ``hello.riscv.0x80000000.1000.loadarch``
+This should produce a directory named ``hello.riscv.*.loadarch``
 
 .. code::
 
@@ -29,10 +29,34 @@ Loading Checkpoints in RTL Simulation
 --------------------------------------
 
 Checkpoints can be loaded in RTL simulations with the ``LOADARCH`` flag.
-The target config **MUST** use dmi-based bringup (as opposed to the default TSI-based bringup), and support fast ``LOADMEM``.
-The target config should also match the architectural configuration of however spike was configured when generating the checkpoint.
+The target config needs the following properties:
+
+- **MUST** use DMI-based bringup (as opposed to the default TSI-based bringup)
+- **MUST** support fast ``LOADMEM``
+- Should match the architectural configuration of however Spike was configured when generating the checkpoint (i.e. same ISA, no PMPs, etc).
 
 .. code::
 
    cd sims/vcs
-   make CONFIG=dmiRocketConfig run-binary LOADARCH=../../hello.riscv.0x80000000.1000.loadarch
+   make CONFIG=dmiRocketConfig run-binary LOADARCH=../../hello.riscv.*.loadarch
+
+Checkpointing Linux Binaries
+----------------------------
+
+Checkpoints can be used to run Linux binaries with the following caveats:
+
+- The binary must only use the HTIF console and should be non-interactive (i.e no stdin available)
+- The target config must be built without a serial device (i.e. the Rocket Chip Blocks UART can't be used)
+- The binary must only use an initramfs (i.e. no block device)
+- The target config must be built without a block device (i.e. the IceBlk block device can't be used).
+- The binary size must be smaller than the size of the target configs memory region (for example if FireMarshal's ``rootfs-size`` is 1GB, and OpenSBI is 350KB then you must have at least 1G + 350KB of space)
+
+This means that you most likely need to do the following:
+
+- By default Spike has a default UART device that is used during most Linux boot's.
+  This can be bypassed by creating a DTS without a serial device then passing it to the ``generate-ckpt.sh`` script.
+  You can copy the DTS of the design you want to checkpoint into - located in Chipyards ``sims/<simulator>/generated-src/`` - and modify it to pass to the checkpointing script (needs to be stripped down of extra devices and nodes).
+  An example of a config made for checkpointing is ``dmiCospikeCheckpointingRocketConfig`` or ``dmiCheckpointingSpikeUltraFastConfig``.
+- Additionally, you need to change your Linux config in FireMarshal to default to only use HTIF during OpenSBI and force Linux to use the OpenSBI HTIF console.
+  This can be done by the following in the ``linux-config``: changing to ``CONFIG_CMDLINE="console=hvc0 earlycon=sbi"``, adding ``CONFIG_RISCV_SBI_V01=y``, adding ``CONFIG_HVC_RISCV_SBI=y``, and adding ``CONFIG_SERIAL_EARLYCON_RISCV_SBI=y``.
+  An example workload with these changes can be found at ``<firemarshal>/example-workloads/br-base-htif-only-serial.yaml``.

generators/chipyard/src/main/resources/csrc/spiketile.cc

@@ -185,6 +185,7 @@ context_t *host;
 std::map<int, tile_t*> tiles;
 std::ostream sout(nullptr);
 log_file_t* log_file;
+#define DEFAULT_PRIV_ "MSU"
 
 extern "C" void spike_tile_reset(int hartid)
 {
@@ -295,15 +296,16 @@ extern "C" void spike_tile(int hartid, char* isa,
   }
   if (tiles.find(hartid) == tiles.end()) {
     printf("Constructing spike processor_t\n");
-    isa_parser_t *isa_parser = new isa_parser_t(isa, "MSU");
+    isa_parser_t *isa_parser = new isa_parser_t(isa, DEFAULT_PRIV_);
     std::string* isastr = new std::string(isa);
     chipyard_simif_t* simif = new chipyard_simif_t(icache_ways, icache_sets,
                                                    dcache_ways, dcache_sets,
                                                    cacheable, uncacheable, readonly_uncacheable, executable,
                                                    icache_sourceids, dcache_sourceids,
                                                    tcm_base, tcm_size,
                                                    isastr->c_str(), pmpregions);
-    processor_t* p = new processor_t(isa_parser,
+    processor_t* p = new processor_t(isa,
+                                     DEFAULT_PRIV_,
                                      &simif->get_cfg(),
                                      simif,
                                      hartid,
@@ -461,7 +463,7 @@ chipyard_simif_t::chipyard_simif_t(size_t icache_ways,
   cfg.initrd_bounds = std::make_pair(0, 0);
   cfg.bootargs = nullptr;
   cfg.isa = isastr;
-  cfg.priv = "MSU";
+  cfg.priv = DEFAULT_PRIV_;
   cfg.misaligned = false;
   cfg.endianness = endianness_little;
   cfg.pmpregions = pmpregions;

generators/chipyard/src/main/scala/SpikeTile.scala

@@ -16,7 +16,9 @@ import freechips.rocketchip.util._
 import freechips.rocketchip.tile._
 import freechips.rocketchip.prci._
 
-case class SpikeCoreParams() extends CoreParams {
+case class SpikeCoreParams(
+  nPMPs: Int = 16
+) extends CoreParams {
   val xLen = 64
   val pgLevels = 5
   val useVM = true
@@ -35,7 +37,6 @@ case class SpikeCoreParams() extends CoreParams {
   val nLocalInterrupts = 0
   val useNMI = false
   val nPTECacheEntries = 0
-  val nPMPs = 16
   val pmpGranularity = 4
   val nBreakpoints = 0
   val useBPWatch = false
@@ -119,6 +120,7 @@ class SpikeTile(
   val masterNode = visibilityNode
   val slaveNode = TLIdentityNode()
 
+  // Note: Rocket doesn't support zicntr but Spike does (err on the side of having Rocket's ISA)
   override def isaDTS = "rv64imafdcv_zicsr_zifencei_zihpm_zvl128b_zve64d"
 
   // Required entry of CPU device in the device tree for interrupt purpose

generators/chipyard/src/main/scala/clocking/ClockGroupCombiner.scala

@@ -51,9 +51,9 @@ class ClockGroupCombiner(implicit p: Parameters, v: ValName) extends LazyModule
       val names = g.map(_.name.getOrElse("unamed"))
       val takes = g.map(_.take).flatten
       require(takes.distinct.size <= 1,
-        s"Clock group $name has non-homogeneous requested ClockParameters ${names.zip(takes)}")
+        s"Clock group '$name' has non-homogeneous requested ClockParameters ${names.zip(takes)}")
       require(takes.size > 0,
-        s"Clock group $name has no inheritable frequencies")
+        s"Clock group '$name' has no inheritable frequencies")
       (grouped ++ Seq(ClockSinkParameters(take = takes.headOption, name = Some(name))), r)
     }
 

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -121,7 +121,6 @@ class AbstractConfig extends Config(
 
   // Bus/interconnect settings
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++     /** hierarchical buses including sbus/mbus/pbus/fbus/cbus/l2 */
-  new chipyard.config.WithSV48IfPossible ++                         /** use sv48 if possible */
 
   // ================================================
   //   Set up power, reset and clocking

generators/chipyard/src/main/scala/config/BoomConfigs.scala

@@ -48,7 +48,8 @@ class MediumBoomV3CosimConfig extends Config(
   new boom.v3.common.WithNMediumBooms(1) ++
   new chipyard.config.AbstractConfig)
 
-class dmiMediumBoomV3Config extends Config(
+class dmiCheckpointingMediumBoomV3Config extends Config(
+  new chipyard.config.WithNPMPs(0) ++                            // remove PMPs (reduce non-core arch state)
   new chipyard.harness.WithSerialTLTiedOff ++                    // don't attach anything to serial-tl
   new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
   new boom.v3.common.WithNMediumBooms(1) ++
@@ -109,7 +110,8 @@ class MediumBoomV4CosimConfig extends Config(
   new boom.v4.common.WithNMediumBooms(1) ++
   new chipyard.config.AbstractConfig)
 
-class dmiMediumBoomV4Config extends Config(
+class dmiCheckpointingMediumBoomV4Config extends Config(
+  new chipyard.config.WithNPMPs(0) ++                            // remove PMPs (reduce non-core arch state)
   new chipyard.harness.WithSerialTLTiedOff ++                    // don't attach anything to serial-tl
   new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
   new boom.v4.common.WithNMediumBooms(1) ++

generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala

@@ -67,6 +67,18 @@ class dmiRocketConfig extends Config(
   new chipyard.config.AbstractConfig)
 // DOC include end: DmiRocket
 
+class dmiCospikeCheckpointingRocketConfig extends Config(
+  new chipyard.harness.WithSerialTLTiedOff ++                    // don't attach anything to serial-tl
+  new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
+  new chipyard.harness.WithCospike ++                   // attach spike-cosim
+  new chipyard.config.WithTraceIO ++                    // enable the traceio
+  new chipyard.config.WithNPMPs(0) ++                   // remove PMPs (reduce non-core arch state)
+  new freechips.rocketchip.rocket.WithDebugROB ++       // cospike needs wdata given by the unsynth. debug rom
+  new freechips.rocketchip.rocket.WithCease(false) ++   // remove xrocket ISA extension
+  new freechips.rocketchip.rocket.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)
+
+
 class ManyPeripheralsRocketConfig extends Config(
   new testchipip.iceblk.WithBlockDevice ++                   // add block-device module to peripherybus
   new testchipip.soc.WithOffchipBusClient(MBUS) ++           // OBUS provides backing memory to the MBUS

generators/chipyard/src/main/scala/config/RocketConfigs.scala

@@ -53,8 +53,13 @@ class ScratchpadOnlyRocketConfig extends Config(
 class MMIOScratchpadOnlyRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithDefaultMMIOPort ++  // add default external master port
   new freechips.rocketchip.subsystem.WithDefaultSlavePort ++ // add default external slave port
-  new ScratchpadOnlyRocketConfig
-)
+  new chipyard.config.WithL2TLBs(0) ++
+  new testchipip.soc.WithNoScratchpads ++                      // remove subsystem scratchpads, confusingly named, does not remove the L1D$ scratchpads
+  new freechips.rocketchip.subsystem.WithNBanks(0) ++
+  new freechips.rocketchip.subsystem.WithNoMemPort ++          // remove offchip mem port
+  new freechips.rocketchip.rocket.WithScratchpadsOnly ++       // use rocket l1 DCache scratchpad as base phys mem
+  new freechips.rocketchip.rocket.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)
 
 class L1ScratchpadRocketConfig extends Config(
   new chipyard.config.WithRocketICacheScratchpad ++         // use rocket ICache scratchpad
@@ -103,4 +108,10 @@ class ClusteredRocketConfig extends Config(
 
 class FastRTLSimRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithoutTLMonitors ++
-  new chipyard.RocketConfig)
+  new freechips.rocketchip.rocket.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)
+
+class SV48RocketConfig extends Config(
+  new freechips.rocketchip.rocket.WithSV48 ++
+  new freechips.rocketchip.rocket.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)

generators/chipyard/src/main/scala/config/SpikeConfigs.scala

@@ -18,40 +18,36 @@ class dmiSpikeConfig extends Config(
 // Avoids polling on the UART registers
 class SpikeFastUARTConfig extends Config(
   new chipyard.WithNSpikeCores(1) ++
-  new chipyard.config.WithUART(txEntries=128, rxEntries=128) ++ // Spike sim requires a larger UART FIFO buffer, 
-  new chipyard.config.WithNoUART() ++                           // so we overwrite the default one
+  new chipyard.config.WithUART(txEntries=128, rxEntries=128) ++   // Spike sim requires a larger UART FIFO buffer,
+  new chipyard.config.WithNoUART() ++                             // so we overwrite the default one
+  new chipyard.config.WithPeripheryBusFrequency(2) ++             // configured to be as fast as possible
   new chipyard.config.WithMemoryBusFrequency(2) ++
-  new chipyard.config.WithPeripheryBusFrequency(2) ++
+  new chipyard.config.WithControlBusFrequency(2) ++
+  new chipyard.config.WithSystemBusFrequency(2) ++
+  new chipyard.config.WithFrontBusFrequency(2) ++
+  new chipyard.config.WithOffchipBusFrequency(2) ++
   new chipyard.config.AbstractConfig)
 
-// Makes the UART fast, also builds no L2 and a ludicrous L1D
+// No L2 and a ludicrous L1D
 class SpikeUltraFastConfig extends Config(
+  new testchipip.soc.WithNoScratchpads ++
   new chipyard.WithSpikeTCM ++
-  new chipyard.WithNSpikeCores(1) ++
-  new chipyard.config.WithUART(txEntries=128, rxEntries=128) ++ // Spike sim requires a larger UART FIFO buffer, 
-  new chipyard.config.WithNoUART() ++                           // so we overwrite the default one
-  new chipyard.config.WithMemoryBusFrequency(2) ++
-  new chipyard.config.WithPeripheryBusFrequency(2) ++
   new chipyard.config.WithBroadcastManager ++
-  new chipyard.config.AbstractConfig)
+  new SpikeFastUARTConfig)
 
 class dmiSpikeUltraFastConfig extends Config(
   new chipyard.harness.WithSerialTLTiedOff ++                    // don't attach anything to serial-tilelink
   new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
   new SpikeUltraFastConfig)
 
+class dmiCheckpointingSpikeUltraFastConfig extends Config(
+  new chipyard.config.WithNPMPs(0) ++                            // remove PMPs (reduce non-core arch state)
+  new dmiSpikeUltraFastConfig)
+
 // Add the default firechip devices
 class SpikeUltraFastDevicesConfig extends Config(
   new chipyard.harness.WithSimBlockDevice ++
   new chipyard.harness.WithLoopbackNIC ++
   new icenet.WithIceNIC ++
   new testchipip.iceblk.WithBlockDevice ++
-
-  new chipyard.WithSpikeTCM ++
-  new chipyard.WithNSpikeCores(1) ++
-  new chipyard.config.WithUART(txEntries=128, rxEntries=128) ++ // Spike sim requires a larger UART FIFO buffer, 
-  new chipyard.config.WithNoUART() ++                           // so we overwrite the default one
-  new chipyard.config.WithMemoryBusFrequency(2) ++
-  new chipyard.config.WithPeripheryBusFrequency(2) ++
-  new chipyard.config.WithBroadcastManager ++
-  new chipyard.config.AbstractConfig)
+  new SpikeUltraFastConfig)

generators/chipyard/src/main/scala/config/fragments/TileFragments.scala

@@ -71,6 +71,8 @@ class WithNPMPs(n: Int = 8) extends Config((site, here, up) => {
       core = tp.tileParams.core.copy(nPMPs = n)))
     case tp: boom.v4.common.BoomTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
       core = tp.tileParams.core.copy(nPMPs = n)))
+    case tp: chipyard.SpikeTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(
+      core = tp.tileParams.core.copy(nPMPs = n)))
     case other => other
   }
 })
@@ -109,8 +111,8 @@ class WithTilePrefetchers extends Config((site, here, up) => {
   }
 })
 
-// Uses SV48 if possible, otherwise default to the Rocket Chip core default
-class WithSV48IfPossible extends Config((site, here, up) => {
+// Use SV48
+class WithSV48 extends Config((site, here, up) => {
   case TilesLocated(loc) => up(TilesLocated(loc), site) map {
     case tp: RocketTileAttachParams => tp.copy(tileParams = tp.tileParams.copy(core =
       tp.tileParams.core.copy(pgLevels = 4)))