Optimize anonymous mmap and page-fault paths

Makefile

@@ -197,6 +197,28 @@ $(BUILD_DIR)/test-pthread: tests/test-pthread.c | $(BUILD_DIR)
 	@echo "  CROSS   $< (with -lpthread)"
 	$(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread
 
+# test-mmap-lazy races concurrent first touch from several threads.
+$(BUILD_DIR)/test-mmap-lazy: tests/test-mmap-lazy.c | $(BUILD_DIR)
+	@echo "  CROSS   $< (with -lpthread)"
+	$(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread
+
+.PHONY: test-mmap-lazy
+test-mmap-lazy: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy
+	@$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy
+	@sh tests/test-mmap-dirty-stats.sh $(ELFUSE_BIN) \
+		$(BUILD_DIR)/test-mmap-lazy
+
+# EL1 consumer-mmap integration/stress test.
+$(BUILD_DIR)/test-mmap-fastpath: tests/test-mmap-fastpath.c | $(BUILD_DIR)
+	@echo "  CROSS   $< (with -lpthread)"
+	$(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread
+
+.PHONY: test-mmap-fastpath
+test-mmap-fastpath: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath
+	@$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath
+	@sh tests/test-mmap-fastpath-stats.sh $(ELFUSE_BIN) \
+		$(BUILD_DIR)/test-mmap-fastpath
+
 # test-thread-churn creates >64 threads to force thread-table slot reuse.
 $(BUILD_DIR)/test-thread-churn: tests/test-thread-churn.c | $(BUILD_DIR)
 	@echo "  CROSS   $< (with -lpthread)"

docs/usage.md

@@ -29,6 +29,15 @@ only bounds a single `hv_vcpu_run()` iteration before the host regains control,
 which is what allows host-side timers and signals to be observed promptly.
 Setting `--timeout 0` disables this watchdog for long-running CPU-bound guests.
 
+### mmap call fast path
+
+The aarch64 EL1 consumer fast path is enabled by default for
+`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)`.
+Set `ELFUSE_MMAP_FASTPATH=0` to disable it. Unsupported mmap shapes, exhausted
+arenas, and full consumption rings fall back to the normal host syscall path.
+Verbose tracing, the syscall histogram, GDB, and Rosetta keep mmap on the host
+path so observability and translated-guest behavior are unchanged.
+
 ## Common Launch Patterns
 
 Run a statically linked guest binary:

src/core/bootstrap.c

@@ -36,6 +36,7 @@
 #include "syscall/signal.h"
 
 #include "debug/log.h"
+#include "core/mmap-fastpath.h"
 
 /* Worst case: 7 fixed regions (shim, shim-data, vDSO, brk, stack, mmap RX, mmap
  * RW) plus up to ELF_MAX_SEGMENTS for both the executable and the interpreter.
@@ -136,6 +137,26 @@ static void register_runtime_regions(guest_t *g, size_t shim_bin_len)
     guest_invalidate_ptes(g, 0, 0x1000);
 }
 
+/* ELF/shim/stack bytes are populated directly in the slab before their page
+ * tables become live. Mark their semantic backing regardless of requested
+ * permissions: a read-only file segment is still nonzero and must be scrubbed
+ * if a later MAP_FIXED lazy-anonymous mapping reuses the same slab block.
+ * Synthetic page-table coverage for unallocated mmap space has no semantic
+ * region and therefore remains clean.
+ */
+static void mark_registered_backing_dirty(guest_t *g)
+{
+    for (int i = 0; i < g->nregions; i++) {
+        const guest_region_t *r = &g->regions[i];
+        if (r->end <= r->start)
+            continue;
+        uint64_t len = r->end - r->start;
+        if (r->gpa_base > g->guest_size || len > g->guest_size - r->gpa_base)
+            continue;
+        guest_dirty_mark_range(g, r->gpa_base, r->gpa_base + len);
+    }
+}
+
 int guest_bootstrap_probe_elf(const char *elf_path, elf_info_t *info)
 {
     memset(info, 0, sizeof(*info));
@@ -547,6 +568,7 @@ int guest_bootstrap_prepare(guest_t *g,
     }
 
     register_runtime_regions(g, shim_bin_len);
+    mark_registered_backing_dirty(g);
     startup_trace_step("register_regions", t0);
 
     log_debug("TTBR0=0x%llx, IPA base=0x%llx", (unsigned long long) boot->ttbr0,
@@ -732,6 +754,13 @@ int guest_bootstrap_create_vcpu(guest_t *g,
      */
     shim_globals_set_singleton(g);
 
+    /* Publish the main vCPU's first arena only after shim_globals_init has
+     * cleared every recycled control slot. Verbose tracing keeps all shim
+     * syscall fast paths on HVC so the trace remains complete.
+     */
+    if (!verbose)
+        mmap_fastpath_prepare_vcpu(g, current_thread);
+
     /* CNTKCTL_EL1.EL0VCTEN | EL0PCTEN: allow EL0 to read {CNTVCT,CNTPCT}_EL0.
      * Required by the vDSO clock_gettime fast path (and is the default on
      * native Linux), without which the guest gets 0 back from MRS.
@@ -843,6 +872,7 @@ int guest_bootstrap_rosetta_post_reset(guest_t *g,
                                  g->rosetta_guest_base - g->rosetta_va_base,
                                  ROSETTA_PATH);
     register_runtime_regions(g, shim_bin_len);
+    mark_registered_backing_dirty(g);
 
     int rosetta_argc = 0;
     const char **rosetta_argv = NULL;