cli: throwaway work disks go to tmpfs on non-reflink filesystems

A plain run's work rootfs is unlinked while the VM holds it open and never
read back, yet on ext4 data dirs every boot wrote ~300 MB through the page
cache and teardown evicted those extents again — multi-second writeback
stalls under repeated runs (spark: 0.55-3.3 s wall for the same command).
prepare_vm now learns the disk's fate up front: discard disks clone to
/dev/shm when reflink is unavailable and the tmpfs has room, so the copy is
a memory copy, nothing is written back, and teardown frees pages. prune
sweeps crash leftovers there too. Checkpoint creation and stdio mode keep
their disk in the instance dir as before.

Co-authored-by: Hanzo Dev <[email protected]>

crates/vm-cli/src/boot.rs

@@ -1,5 +1,4 @@
 use std::collections::HashMap;
-#[cfg(target_os = "macos")]
 use std::ffi::CString;
 use std::io::IsTerminal;
 
@@ -71,7 +70,15 @@ pub(crate) struct PreparedVm {
     pub mounts: Vec<MountConfig>,
 }
 
-pub(crate) fn prepare_vm(vm: &VmArgs, cfg: &Config, from: Option<&str>) -> Result<PreparedVm> {
+/// Prepare a VM instance. `discard` marks a throwaway disk (plain `run`):
+/// it is unlinked while the VM holds it open, never read back, and on Linux
+/// may live on tmpfs when the data dir cannot reflink.
+pub(crate) fn prepare_vm(
+    vm: &VmArgs,
+    cfg: &Config,
+    from: Option<&str>,
+    discard: bool,
+) -> Result<PreparedVm> {
     trace_boot("prepare_vm start");
     let cpus = vm.cpus.or(cfg.cpus).unwrap_or(2);
     let memory = vm.memory.or(cfg.memory).unwrap_or(2048);
@@ -218,7 +225,7 @@ pub(crate) fn prepare_vm(vm: &VmArgs, cfg: &Config, from: Option<&str>) -> Resul
     let instance_dir = format!("{}/instances/{}", data_dir, std::process::id());
     let _ = std::fs::remove_dir_all(&instance_dir);
     std::fs::create_dir_all(&instance_dir)?;
-    let work_rootfs = format!("{}/rootfs.ext4", instance_dir);
+    let mut work_rootfs = format!("{}/rootfs.ext4", instance_dir);
     trace_boot("prepare: instance dir ready");
 
     // CAS checkpoints don't need a file copy — the NBD server reads from the chunk store.
@@ -228,7 +235,7 @@ pub(crate) fn prepare_vm(vm: &VmArgs, cfg: &Config, from: Option<&str>) -> Resul
         if verbose {
             eprintln!("hanzo-vm: creating working copy...");
         }
-        clone_file(&source, &work_rootfs)?;
+        clone_rootfs(&source, &mut work_rootfs, discard, disk_size * 1024 * 1024)?;
         // The clone is a fresh vnode: its pages are cold even when the source
         // is cached, and the guest's root mount pays for that in disk reads.
         // Warm it sequentially while the VM is being configured and started.
@@ -275,13 +282,71 @@ pub(crate) fn prepare_vm(vm: &VmArgs, cfg: &Config, from: Option<&str>) -> Resul
         disk_size,
         proxy_config,
         sync_before_stop: false,
-        discard_disk: false,
+        discard_disk: discard,
         verbose,
         forwards,
         mounts,
     })
 }
 
+/// Clone the base image to the working copy. macOS: APFS clonefile,
+/// always free. (`&mut String` matches the Linux variant, which may point
+/// the working copy at tmpfs.)
+#[cfg(target_os = "macos")]
+#[allow(clippy::ptr_arg)]
+fn clone_rootfs(source: &str, work_rootfs: &mut String, _discard: bool, _bytes: u64) -> Result<()> {
+    clone_file(source, work_rootfs)
+}
+
+/// Clone the base image to the working copy. Reflink when the filesystem
+/// offers it (XFS/btrfs). Otherwise a throwaway disk goes to tmpfs when it
+/// fits: the sparse copy runs at page-cache speed, nothing is ever written
+/// back to storage, and teardown frees pages instead of evicting hundreds
+/// of MB of freshly written ext4 extents — on ext4 data dirs this is the
+/// difference between a steady 0.4 s boot and multi-second writeback
+/// stalls. Disks that are read back after the VM stops (checkpoint
+/// creation, stdio mode) stay in the instance dir.
+#[cfg(target_os = "linux")]
+fn clone_rootfs(source: &str, work_rootfs: &mut String, discard: bool, bytes: u64) -> Result<()> {
+    if vm::reflink_file(source, work_rootfs).is_ok() {
+        return Ok(());
+    }
+    if discard {
+        if let Some(path) = tmpfs_work_path(bytes) {
+            if clone_file(source, &path).is_ok() {
+                *work_rootfs = path;
+                return Ok(());
+            }
+            let _ = std::fs::remove_file(&path);
+        }
+    }
+    clone_file(source, work_rootfs)
+}
+
+/// A per-pid path on /dev/shm, only when the tmpfs has room for the full
+/// disk. `prune` sweeps leftovers of crashed runs.
+#[cfg(target_os = "linux")]
+fn tmpfs_work_path(bytes: u64) -> Option<String> {
+    let dir = "/dev/shm/hanzo-vm";
+    std::fs::create_dir_all(dir).ok()?;
+    let stat = nix_statvfs(dir)?;
+    if stat < bytes {
+        return None;
+    }
+    Some(format!("{}/{}.ext4", dir, std::process::id()))
+}
+
+/// Available bytes on the filesystem holding `path`.
+#[cfg(target_os = "linux")]
+fn nix_statvfs(path: &str) -> Option<u64> {
+    let c_path = CString::new(path).ok()?;
+    let mut st: libc::statvfs = unsafe { std::mem::zeroed() };
+    if unsafe { libc::statvfs(c_path.as_ptr(), &mut st) } != 0 {
+        return None;
+    }
+    Some(st.f_bavail as u64 * st.f_frsize as u64)
+}
+
 pub(crate) fn build_sandbox(
     prepared: &PreparedVm,
     console: bool,

crates/vm-cli/src/checkpoint.rs

@@ -31,7 +31,7 @@ pub(crate) fn create(
         bail!("checkpoint '{}' already exists, delete it first", name);
     }
 
-    let mut prepared = boot::prepare_vm(vm_args, &cfg, from)?;
+    let mut prepared = boot::prepare_vm(vm_args, &cfg, from, false)?;
     prepared.sync_before_stop = true;
     let result = boot::run_command(&prepared, &command)?;
 

crates/vm-cli/src/main.rs

@@ -44,8 +44,7 @@ fn main() -> Result<()> {
                 vec!["/bin/sh".to_string()]
             };
 
-            let mut prepared = boot::prepare_vm(&vm, &cfg, from.as_deref())?;
-            prepared.discard_disk = !stdio && !console;
+            let prepared = boot::prepare_vm(&vm, &cfg, from.as_deref(), !stdio && !console)?;
 
             let result = if stdio {
                 stdio::run_stdio(&prepared)
@@ -55,6 +54,9 @@ fn main() -> Result<()> {
                 boot::run_command(&prepared, &command).map(|r| r.exit_code)
             };
 
+            // The work rootfs may live outside the instance dir (tmpfs);
+            // a no-op when the discard path already unlinked it.
+            let _ = std::fs::remove_file(&prepared.work_rootfs);
             let _ = std::fs::remove_dir_all(&prepared.instance_dir);
             boot::trace_boot("instance dir removed");
             process::exit(result?);
@@ -104,6 +106,25 @@ fn main() -> Result<()> {
                 }
             }
 
+            // Throwaway work disks of crashed runs may live on tmpfs.
+            #[cfg(target_os = "linux")]
+            if let Ok(entries) = std::fs::read_dir("/dev/shm/hanzo-vm") {
+                for entry in entries.flatten() {
+                    let name = entry.file_name();
+                    let Some(pid) = name
+                        .to_str()
+                        .and_then(|s| s.strip_suffix(".ext4"))
+                        .and_then(|s| s.parse::<i32>().ok())
+                    else {
+                        continue;
+                    };
+                    if unsafe { libc::kill(pid, 0) } != 0 {
+                        let _ = std::fs::remove_file(entry.path());
+                        removed += 1;
+                    }
+                }
+            }
+
             if removed == 0 {
                 eprintln!("hanzo-vm: no orphaned instances found");
             } else {