Cap the maximum basic block padding in Cranelift

cranelift/codegen/src/machinst/vcode.rs

@@ -1059,13 +1059,13 @@ impl<I: VCodeInst> VCode<I> {
             // Padding can get quite large during fuzzing though so place a
             // total cap on it where when a per-function threshold is exceeded
             // the padding is turned back down to zero. This avoids a small-ish
-            // test case generating a 2GB memory footprint in Cranelift for
+            // test case generating a GB+ memory footprint in Cranelift for
             // example.
             if !bb_padding.is_empty() {
                 buffer.put_data(&bb_padding);
                 buffer.align_to(I::LabelUse::ALIGN);
                 total_bb_padding += bb_padding.len();
-                if total_bb_padding > (1 << 20) {
+                if total_bb_padding > (64 << 20) {
                     bb_padding = Vec::new();
                 }
             }

crates/fuzzing/src/generators/config.rs

@@ -207,15 +207,32 @@ impl Config {
             }
         }
 
-        // Allow at most 1<<16 == 64k of padding between basic blocks. If this
-        // gets too large then functions actually grow to 4G+ sizes which is not
-        // really that interesting to test as it's pretty unrealistic at this
-        // time.
-        unsafe {
-            cfg.cranelift_flag_set(
-                "bb_padding_log2_minus_one",
-                &(self.wasmtime.bb_padding_log2 & 0xf).to_string(),
-            );
+        // Try to configure cranelift to insert padding between basic blocks to
+        // stress code layout mechanisms within Cranelift. This padding can get
+        // unwieldy quickly, however, generating a 4G+ function which isn't
+        // particularly interesting at this time.
+        //
+        // To keep this setting under control there are a few limits in place:
+        //
+        // * Cranelift will generate at most 64M of padding per-function,
+        //   regardless of how many basic blocks there are.
+        // * Here it's limited to enable this setting only when there's at most
+        //   10 functions to ensure that the overhead for all functions is <1G
+        //   ish or otherwise dosn't run away.
+        // * The `bb_padding_log2_minus_one` setting isn't allowed to be
+        //   larger than 26 as that allows for `1<<25` maximum padding size
+        //   which is 32M.
+        //
+        // With all that combined this is intended to still be enabled,
+        // although perhaps not all the time, and stress enough interesting test
+        // cases in cranelift.
+        if self.module_config.config.max_funcs < 10 {
+            unsafe {
+                cfg.cranelift_flag_set(
+                    "bb_padding_log2_minus_one",
+                    &(self.wasmtime.bb_padding_log2 % 26).to_string(),
+                );
+            }
         }
 
         // Vary the memory configuration, but only if threads are not enabled.