fix(cni): quiesce host NICs on stop to end the idle-TAP softirq storm

cmd/vm/hibernate.go

@@ -47,14 +47,14 @@ func (h Handler) Hibernate(cmd *cobra.Command, args []string) error {
 	logger.Infof(ctx, "hibernating VM %s ...", vmRef)
 	// persist runs inside the pause window; the VMM dies only after it succeeds.
 	var snapID string
-	err = hib.Hibernate(ctx, vm.ID, func(cfg *types.SnapshotConfig, srcDir string) error {
+	if err := hib.Hibernate(ctx, vm.ID, func(cfg *types.SnapshotConfig, srcDir string) error {
 		id, pErr := cmdcore.PersistSnapshotDir(ctx, snapBackend, cfg, srcDir, name, description)
 		snapID = id
 		return pErr
-	})
-	if err != nil {
+	}); err != nil {
 		return err
 	}
+	h.quiesceNetwork(ctx, conf, hyper, []string{vm.ID})
 	logger.Infof(ctx, "VM hibernated; snapshot %s (resume: cocoon vm restore %s %s)", snapID, vmRef, cmp.Or(name, snapID))
 	return nil
 }

cmd/vm/lifecycle.go

@@ -83,7 +83,7 @@ func (h Handler) Stop(cmd *cobra.Command, args []string) error {
 		return err
 	}
 	return batchRoutedCmd(ctx, cmd, "stop", "stopped", routed, func(hyper hypervisor.Hypervisor, refs []string) ([]string, error) {
-		return hyper.Stop(ctx, refs)
+		return h.stopAndQuiesce(ctx, conf, hyper, refs)
 	})
 }
 
@@ -204,6 +204,12 @@ func (h Handler) RM(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	if force {
+		for hyper, refs := range routed {
+			_, _ = h.stopAndQuiesce(ctx, conf, hyper, refs)
+		}
+	}
+
 	const logTag = "cmd.vm.rm"
 	allDeleted, lastErr := runRoutedLoop(ctx, logTag, "deleted", cliutil.WantJSON(cmd), routed,
 		func(hyper hypervisor.Hypervisor, refs []string) ([]string, error) {
@@ -278,6 +284,9 @@ func (h Handler) recoverNetwork(ctx context.Context, conf *config.Config, hyper
 		}
 		recoverOne := func(vm *types.VM) {
 			if netProvider.Verify(ctx, vm.ID, vm.NetworkConfigs) == nil {
+				if err := netProvider.Unquiesce(ctx, vm.ID); err != nil {
+					logger.Warnf(ctx, "unquiesce network for VM %s: %v", vm.ID, err)
+				}
 				return
 			}
 			logger.Warnf(ctx, "network missing for VM %s, recovering", vm.ID)
@@ -310,6 +319,48 @@ func (h Handler) recoverNetwork(ctx context.Context, conf *config.Config, hyper
 	}
 }
 
+func (h Handler) stopAndQuiesce(ctx context.Context, conf *config.Config, hyper hypervisor.Hypervisor, refs []string) ([]string, error) {
+	stopped, err := hyper.Stop(ctx, refs)
+	h.quiesceNetwork(ctx, conf, hyper, stopped)
+	return stopped, err
+}
+
+// quiesceNetwork brings each stopped VM's host NICs down — Stop's counterpart to Start's recoverNetwork — so an idle TAP's TC redirect can't storm softirqs. Best-effort: a quiesce failure never blocks the stop.
+func (h Handler) quiesceNetwork(ctx context.Context, conf *config.Config, hyper hypervisor.Hypervisor, ids []string) {
+	if len(ids) == 0 {
+		return
+	}
+	logger := log.WithFunc("cmd.vm.quiesceNetwork")
+
+	var cniProvider network.Network
+	if p, err := cmdcore.InitNetwork(conf); err == nil {
+		cniProvider = p
+	}
+	bridgeProviders := map[string]network.Network{}
+
+	for _, id := range ids {
+		vm, err := hyper.Inspect(ctx, id)
+		if err != nil {
+			logger.Warnf(ctx, "inspect VM %s for quiesce: %v", id, err)
+			continue
+		}
+		if vm == nil {
+			continue
+		}
+		if vm.ResolvedNetBackend() == "" || len(vm.NetworkConfigs) == 0 {
+			continue
+		}
+		netProvider, provErr := providerForVM(conf, cniProvider, bridgeProviders, vm)
+		if provErr != nil {
+			logger.Warnf(ctx, "skip quiesce for VM %s: %v", id, provErr)
+			continue
+		}
+		if err := netProvider.Quiesce(ctx, id); err != nil {
+			logger.Warnf(ctx, "quiesce network for VM %s: %v", id, err)
+		}
+	}
+}
+
 // applyStopFlags maps --force (-1 = immediate kill; #82: FC guests without i8042 never answer CtrlAltDel) and --timeout onto the stop window; rm has no --timeout flag, that read no-ops.
 func applyStopFlags(conf *config.Config, cmd *cobra.Command) {
 	force, _ := cmd.Flags().GetBool("force")

network/bridge/bridge_linux.go

@@ -151,6 +151,10 @@ func (b *Bridge) Remove(_ context.Context, vmID string, indices ...int) error {
 	return tearDownTAPs(vmID, indices, false)
 }
 
+// Quiesce and Unquiesce are no-ops: bridge TAPs sit directly on the host bridge, with no TC redirect to storm when the VM stops.
+func (b *Bridge) Quiesce(_ context.Context, _ string) error   { return nil }
+func (b *Bridge) Unquiesce(_ context.Context, _ string) error { return nil }
+
 func (b *Bridge) Delete(_ context.Context, vmIDs []string) ([]string, error) {
 	return CleanupTAPs(vmIDs), nil
 }

network/bridge/bridge_other.go

@@ -30,7 +30,9 @@ func (b *Bridge) Verify(_ context.Context, _ string, _ []*types.NetworkConfig) e
 func (b *Bridge) Remove(_ context.Context, _ string, _ ...int) error {
 	return errUnsupported
 }
-func (b *Bridge) RegisterGC(_ *gc.Orchestrator) {}
+func (b *Bridge) Quiesce(_ context.Context, _ string) error   { return nil }
+func (b *Bridge) Unquiesce(_ context.Context, _ string) error { return nil }
+func (b *Bridge) RegisterGC(_ *gc.Orchestrator)               {}
 
 func (b *Bridge) Prepare(_ context.Context, _ string, _ *types.VMConfig) (string, error) {
 	return "", errUnsupported

network/cni/cni.go

@@ -36,6 +36,7 @@ var (
 	setupTCRedirectFn = setupTCRedirect
 	tapPresentFn      = tapPresentInNetns
 	statNetnsFn       = os.Stat
+	setLinkStateFn    = setLinkStateInNetns
 )
 
 // CNI implements network.Network using CNI plugins with per-VM netns + bridge + tap.
@@ -130,6 +131,16 @@ func (c *CNI) List(ctx context.Context) ([]*types.Network, error) {
 	})
 }
 
+// Quiesce brings the VM's host-side veths down so a stopped VM's TC redirect stops storming softirqs against its now-carrier-less TAP (mirred-to-down-device). The netns and TAP are kept for a fast restart, which Unquiesce re-enables.
+func (c *CNI) Quiesce(ctx context.Context, vmID string) error {
+	return c.setLinkState(ctx, vmID, false)
+}
+
+// Unquiesce restores the veths Quiesce brought down, run on start before the VMM re-opens the TAP.
+func (c *CNI) Unquiesce(ctx context.Context, vmID string) error {
+	return c.setLinkState(ctx, vmID, true)
+}
+
 // Delete tears down all NICs for each VM and removes the netns. Best-effort.
 func (c *CNI) Delete(ctx context.Context, vmIDs []string) ([]string, error) {
 	result := utils.ForEach(ctx, vmIDs, func(ctx context.Context, vmID string) error {
@@ -223,6 +234,24 @@ func (c *CNI) deleteRecords(ctx context.Context, ids []string) error {
 	})
 }
 
+func (c *CNI) setLinkState(ctx context.Context, vmID string, up bool) error {
+	var records []networkRecord
+	if err := c.store.With(ctx, func(idx *networkIndex) error {
+		records = idx.byVMID(vmID)
+		return nil
+	}); err != nil {
+		return fmt.Errorf("read network index: %w", err)
+	}
+	if len(records) == 0 {
+		return nil
+	}
+	ifNames := make([]string, 0, len(records))
+	for _, rec := range records {
+		ifNames = append(ifNames, rec.IfName)
+	}
+	return setLinkStateFn(netnsPath(vmID), ifNames, up)
+}
+
 // confListByName resolves a conflist by name; empty name returns the default (first alphabetically).
 func (c *CNI) confListByName(name string) (*libcni.NetworkConfigList, error) {
 	if len(c.confLists) == 0 {

network/cni/cni_test.go

@@ -301,6 +301,56 @@ func TestAddFailsClosedOnStaleReclaim(t *testing.T) {
 	}
 }
 
+func TestQuiesceUnquiesceTogglesEveryNIC(t *testing.T) {
+	c, _ := newTestCNIWithStore(t)
+	var gotNS string
+	var gotIfs []string
+	var gotUp []bool
+	origSet := setLinkStateFn
+	setLinkStateFn = func(nsPath string, ifNames []string, up bool) error {
+		gotNS, gotIfs = nsPath, ifNames
+		gotUp = append(gotUp, up)
+		return nil
+	}
+	t.Cleanup(func() { setLinkStateFn = origSet })
+
+	ctx := t.Context()
+	seedRecords(t, c, "vm1", "eth0", "eth1")
+
+	if err := c.Quiesce(ctx, "vm1"); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+	if gotNS != netnsPath("vm1") {
+		t.Fatalf("nsPath = %q, want %q", gotNS, netnsPath("vm1"))
+	}
+	slices.Sort(gotIfs)
+	if !slices.Equal(gotIfs, []string{"eth0", "eth1"}) {
+		t.Fatalf("ifNames = %v, want every NIC [eth0 eth1]", gotIfs)
+	}
+	if err := c.Unquiesce(ctx, "vm1"); err != nil {
+		t.Fatalf("Unquiesce: %v", err)
+	}
+	if !slices.Equal(gotUp, []bool{false, true}) {
+		t.Fatalf("state sequence = %v, want [false true] (Quiesce down, Unquiesce up)", gotUp)
+	}
+}
+
+func TestQuiesceNoRecordsSkipsNetns(t *testing.T) {
+	c, _ := newTestCNIWithStore(t)
+	called := false
+	origSet := setLinkStateFn
+	setLinkStateFn = func(string, []string, bool) error { called = true; return nil }
+	t.Cleanup(func() { setLinkStateFn = origSet })
+
+	// A VM with no NIC records owns no plumbing to storm: never enter its netns.
+	if err := c.Quiesce(t.Context(), "ghost"); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+	if called {
+		t.Fatal("setLinkStateFn called for a VM with no records")
+	}
+}
+
 // newTestCNIWithStore builds a CNI over a real JSON store and a recordingExec-backed libcni.
 func newTestCNIWithStore(t *testing.T) (*CNI, *recordingExec) {
 	t.Helper()
@@ -322,12 +372,15 @@ func newTestCNIWithStore(t *testing.T) (*CNI, *recordingExec) {
 func stubLifecycleSeams(t *testing.T) {
 	t.Helper()
 	origTAP, origNetns, origEnsure, origTC := deleteTAPFn, deleteNetnsFn, ensureNetnsFn, setupTCRedirectFn
+	origSet := setLinkStateFn
 	deleteTAPFn = func(string, string) error { return nil }
 	deleteNetnsFn = func(context.Context, string) error { return nil }
 	ensureNetnsFn = func(string, string) (bool, error) { return false, nil }
 	setupTCRedirectFn = func(_, _, _ string, _ int, _ string) (string, error) { return "aa:bb:cc:dd:ee:01", nil }
+	setLinkStateFn = func(string, []string, bool) error { return nil }
 	t.Cleanup(func() {
 		deleteTAPFn, deleteNetnsFn, ensureNetnsFn, setupTCRedirectFn = origTAP, origNetns, origEnsure, origTC
+		setLinkStateFn = origSet
 	})
 }
 

network/cni/lifecycle_darwin.go

@@ -23,6 +23,10 @@ func deleteTAPInNetns(_, _ string) error {
 	return errNotSupported
 }
 
+func setLinkStateInNetns(_ string, _ []string, _ bool) error {
+	return errNotSupported
+}
+
 func tapPresentInNetns(_, _ string) error {
 	return errNotSupported
 }

network/cni/lifecycle_linux.go

@@ -79,6 +79,33 @@ func deleteTAPInNetns(nsPath, tapName string) error {
 	return err
 }
 
+// setLinkStateInNetns brings ifNames up or down inside nsPath. A missing netns or link is success: Quiesce/Unquiesce run across stop/restart and partial teardown, where the plumbing may already be gone.
+func setLinkStateInNetns(nsPath string, ifNames []string, up bool) error {
+	transition := netlink.LinkSetDown
+	if up {
+		transition = netlink.LinkSetUp
+	}
+	err := cns.WithNetNSPath(nsPath, func(_ cns.NetNS) error {
+		for _, name := range ifNames {
+			link, err := netlink.LinkByName(name)
+			if err != nil {
+				if _, ok := errors.AsType[netlink.LinkNotFoundError](err); ok {
+					continue
+				}
+				return fmt.Errorf("find %s: %w", name, err)
+			}
+			if err := transition(link); err != nil {
+				return fmt.Errorf("set %s state: %w", name, err)
+			}
+		}
+		return nil
+	})
+	if _, ok := errors.AsType[cns.NSPathNotExistErr](err); ok {
+		return nil
+	}
+	return err
+}
+
 // setupTCRedirect wires ifName <-> tapName inside target netns, returns MAC.
 func setupTCRedirect(nsPath, ifName, tapName string, queues int, overrideMAC string) (string, error) {
 	var mac string

network/network.go

@@ -12,21 +12,20 @@ var ErrNotConfigured = errors.New("network provider not configured")
 
 // AddSpec is one NIC's add request; Existing != nil reuses MAC/IP for recovery.
 type AddSpec struct {
-	Index int
-	// Queues is the TAP queue count; 0 derives NetNumQueues(vmCfg.CPU), backend-specific callers (FC opens single-queue) set it explicitly.
+	Index    int
 	Queues   int
 	Existing *types.NetworkConfig
 }
 
 // Network is the per-VM host-side networking provider (CNI, bridge, ...).
 type Network interface {
 	Type() string
-	// Verify checks the VM's plumbing against its recorded NICs (netns and each TAP), not just container existence: a half-rolled-back recovery leaves the netns without TAPs.
 	Verify(ctx context.Context, vmID string, expected []*types.NetworkConfig) error
-	// Prepare provisions per-VM state; returns the netns path or "".
 	Prepare(ctx context.Context, vmID string, vmCfg *types.VMConfig) (string, error)
 	Add(ctx context.Context, vmID string, vmCfg *types.VMConfig, specs ...AddSpec) ([]*types.NetworkConfig, error)
 	Remove(ctx context.Context, vmID string, indices ...int) error
+	Quiesce(ctx context.Context, vmID string) error
+	Unquiesce(ctx context.Context, vmID string) error
 	Delete(context.Context, []string) ([]string, error)
 	Inspect(context.Context, string) (*types.Network, error)
 	List(context.Context) ([]*types.Network, error)