From 34864a7cee1c03c2bf4d046d066aa3cc02d3c088 Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 14 Jul 2026 01:32:56 +0800 Subject: [PATCH 1/4] fix(cni): quiesce host NICs on stop to end the idle-TAP softirq storm A stopped CNI VM keeps its netns/TAP for a fast restart, but the VMM exit drops the TAP's carrier while its veth stays up on the host bridge; the tc mirred redirect then fires against the down TAP for every LAN broadcast packet ("mirred to Houston: device is down"), storming softirqs (NOHZ tick-stop) until the host soft-locks and reboots. Teardown only ran on vm rm, so any stopped CNI VM was a host-crash landmine. Bring the veths down on stop (Quiesce) and back up on start (Unquiesce) so the redirect never targets a carrier-less TAP. --- cmd/vm/lifecycle.go | 47 ++++++++++++++++++++++++++++- network/bridge/bridge_linux.go | 4 +++ network/bridge/bridge_other.go | 4 ++- network/cni/cni.go | 29 ++++++++++++++++++ network/cni/cni_test.go | 53 +++++++++++++++++++++++++++++++++ network/cni/lifecycle_darwin.go | 4 +++ network/cni/lifecycle_linux.go | 27 +++++++++++++++++ network/network.go | 3 ++ 8 files changed, 169 insertions(+), 2 deletions(-) diff --git a/cmd/vm/lifecycle.go b/cmd/vm/lifecycle.go index 14ad0dc8..092cfee0 100644 --- a/cmd/vm/lifecycle.go +++ b/cmd/vm/lifecycle.go @@ -83,7 +83,11 @@ func (h Handler) Stop(cmd *cobra.Command, args []string) error { return err } return batchRoutedCmd(ctx, cmd, "stop", "stopped", routed, func(hyper hypervisor.Hypervisor, refs []string) ([]string, error) { - return hyper.Stop(ctx, refs) + stopped, err := hyper.Stop(ctx, refs) + // The TAP loses carrier when the VMM exits; bring the host NICs down so the + // kept-for-restart TC redirect can't storm softirqs against it (recoverNetwork undoes it). + h.quiesceNetwork(ctx, conf, hyper, stopped) + return stopped, err }) } @@ -278,6 +282,11 @@ func (h Handler) recoverNetwork(ctx context.Context, conf *config.Config, hyper } recoverOne := func(vm *types.VM) { if netProvider.Verify(ctx, vm.ID, vm.NetworkConfigs) == nil { + // Plumbing intact (fast-restart path): undo Stop's quiesce so the host + // NICs forward again once the VMM re-opens the TAP. + if err := netProvider.Unquiesce(ctx, vm.ID); err != nil { + logger.Warnf(ctx, "unquiesce network for VM %s: %v", vm.ID, err) + } return } logger.Warnf(ctx, "network missing for VM %s, recovering", vm.ID) @@ -310,6 +319,42 @@ func (h Handler) recoverNetwork(ctx context.Context, conf *config.Config, hyper } } +// quiesceNetwork brings each stopped VM's host NICs down — Stop's counterpart to Start's recoverNetwork — so an idle TAP's TC redirect can't storm softirqs. Best-effort: a quiesce failure never blocks the stop. +func (h Handler) quiesceNetwork(ctx context.Context, conf *config.Config, hyper hypervisor.Hypervisor, ids []string) { + if len(ids) == 0 { + return + } + logger := log.WithFunc("cmd.vm.quiesceNetwork") + + var cniProvider network.Network + if p, err := cmdcore.InitNetwork(conf); err == nil { + cniProvider = p + } + bridgeProviders := map[string]network.Network{} + + for _, id := range ids { + vm, err := hyper.Inspect(ctx, id) + if err != nil { + logger.Warnf(ctx, "inspect VM %s for quiesce: %v", id, err) + continue + } + if vm == nil { + continue + } + if vm.ResolvedNetBackend() == "" || len(vm.NetworkConfigs) == 0 { + continue + } + netProvider, provErr := providerForVM(conf, cniProvider, bridgeProviders, vm) + if provErr != nil { + logger.Warnf(ctx, "skip quiesce for VM %s: %v", id, provErr) + continue + } + if err := netProvider.Quiesce(ctx, id); err != nil { + logger.Warnf(ctx, "quiesce network for VM %s: %v", id, err) + } + } +} + // applyStopFlags maps --force (-1 = immediate kill; #82: FC guests without i8042 never answer CtrlAltDel) and --timeout onto the stop window; rm has no --timeout flag, that read no-ops. func applyStopFlags(conf *config.Config, cmd *cobra.Command) { force, _ := cmd.Flags().GetBool("force") diff --git a/network/bridge/bridge_linux.go b/network/bridge/bridge_linux.go index 78b91f3a..8e64e2d6 100644 --- a/network/bridge/bridge_linux.go +++ b/network/bridge/bridge_linux.go @@ -151,6 +151,10 @@ func (b *Bridge) Remove(_ context.Context, vmID string, indices ...int) error { return tearDownTAPs(vmID, indices, false) } +// Quiesce and Unquiesce are no-ops: bridge TAPs sit directly on the host bridge, with no TC redirect to storm when the VM stops. +func (b *Bridge) Quiesce(_ context.Context, _ string) error { return nil } +func (b *Bridge) Unquiesce(_ context.Context, _ string) error { return nil } + func (b *Bridge) Delete(_ context.Context, vmIDs []string) ([]string, error) { return CleanupTAPs(vmIDs), nil } diff --git a/network/bridge/bridge_other.go b/network/bridge/bridge_other.go index cc37f812..3146eac9 100644 --- a/network/bridge/bridge_other.go +++ b/network/bridge/bridge_other.go @@ -30,7 +30,9 @@ func (b *Bridge) Verify(_ context.Context, _ string, _ []*types.NetworkConfig) e func (b *Bridge) Remove(_ context.Context, _ string, _ ...int) error { return errUnsupported } -func (b *Bridge) RegisterGC(_ *gc.Orchestrator) {} +func (b *Bridge) Quiesce(_ context.Context, _ string) error { return nil } +func (b *Bridge) Unquiesce(_ context.Context, _ string) error { return nil } +func (b *Bridge) RegisterGC(_ *gc.Orchestrator) {} func (b *Bridge) Prepare(_ context.Context, _ string, _ *types.VMConfig) (string, error) { return "", errUnsupported diff --git a/network/cni/cni.go b/network/cni/cni.go index af53035e..8028c49f 100644 --- a/network/cni/cni.go +++ b/network/cni/cni.go @@ -36,6 +36,7 @@ var ( setupTCRedirectFn = setupTCRedirect tapPresentFn = tapPresentInNetns statNetnsFn = os.Stat + setLinkStateFn = setLinkStateInNetns ) // CNI implements network.Network using CNI plugins with per-VM netns + bridge + tap. @@ -223,6 +224,34 @@ func (c *CNI) deleteRecords(ctx context.Context, ids []string) error { }) } +// Quiesce brings the VM's host-side veths down so a stopped VM's TC redirect stops storming softirqs against its now-carrier-less TAP (mirred-to-down-device). The netns and TAP are kept for a fast restart, which Unquiesce re-enables. +func (c *CNI) Quiesce(ctx context.Context, vmID string) error { + return c.setLinkState(ctx, vmID, false) +} + +// Unquiesce restores the veths Quiesce brought down, run on start before the VMM re-opens the TAP. +func (c *CNI) Unquiesce(ctx context.Context, vmID string) error { + return c.setLinkState(ctx, vmID, true) +} + +func (c *CNI) setLinkState(ctx context.Context, vmID string, up bool) error { + var records []networkRecord + if err := c.store.With(ctx, func(idx *networkIndex) error { + records = idx.byVMID(vmID) + return nil + }); err != nil { + return fmt.Errorf("read network index: %w", err) + } + if len(records) == 0 { + return nil + } + ifNames := make([]string, 0, len(records)) + for _, rec := range records { + ifNames = append(ifNames, rec.IfName) + } + return setLinkStateFn(netnsPath(vmID), ifNames, up) +} + // confListByName resolves a conflist by name; empty name returns the default (first alphabetically). func (c *CNI) confListByName(name string) (*libcni.NetworkConfigList, error) { if len(c.confLists) == 0 { diff --git a/network/cni/cni_test.go b/network/cni/cni_test.go index bdb00a55..3fae9cee 100644 --- a/network/cni/cni_test.go +++ b/network/cni/cni_test.go @@ -301,6 +301,56 @@ func TestAddFailsClosedOnStaleReclaim(t *testing.T) { } } +func TestQuiesceUnquiesceTogglesEveryNIC(t *testing.T) { + c, _ := newTestCNIWithStore(t) + var gotNS string + var gotIfs []string + var gotUp []bool + origSet := setLinkStateFn + setLinkStateFn = func(nsPath string, ifNames []string, up bool) error { + gotNS, gotIfs = nsPath, ifNames + gotUp = append(gotUp, up) + return nil + } + t.Cleanup(func() { setLinkStateFn = origSet }) + + ctx := t.Context() + seedRecords(t, c, "vm1", "eth0", "eth1") + + if err := c.Quiesce(ctx, "vm1"); err != nil { + t.Fatalf("Quiesce: %v", err) + } + if gotNS != netnsPath("vm1") { + t.Fatalf("nsPath = %q, want %q", gotNS, netnsPath("vm1")) + } + slices.Sort(gotIfs) + if !slices.Equal(gotIfs, []string{"eth0", "eth1"}) { + t.Fatalf("ifNames = %v, want every NIC [eth0 eth1]", gotIfs) + } + if err := c.Unquiesce(ctx, "vm1"); err != nil { + t.Fatalf("Unquiesce: %v", err) + } + if !slices.Equal(gotUp, []bool{false, true}) { + t.Fatalf("state sequence = %v, want [false true] (Quiesce down, Unquiesce up)", gotUp) + } +} + +func TestQuiesceNoRecordsSkipsNetns(t *testing.T) { + c, _ := newTestCNIWithStore(t) + called := false + origSet := setLinkStateFn + setLinkStateFn = func(string, []string, bool) error { called = true; return nil } + t.Cleanup(func() { setLinkStateFn = origSet }) + + // A VM with no NIC records owns no plumbing to storm: never enter its netns. + if err := c.Quiesce(t.Context(), "ghost"); err != nil { + t.Fatalf("Quiesce: %v", err) + } + if called { + t.Fatal("setLinkStateFn called for a VM with no records") + } +} + // newTestCNIWithStore builds a CNI over a real JSON store and a recordingExec-backed libcni. func newTestCNIWithStore(t *testing.T) (*CNI, *recordingExec) { t.Helper() @@ -322,12 +372,15 @@ func newTestCNIWithStore(t *testing.T) (*CNI, *recordingExec) { func stubLifecycleSeams(t *testing.T) { t.Helper() origTAP, origNetns, origEnsure, origTC := deleteTAPFn, deleteNetnsFn, ensureNetnsFn, setupTCRedirectFn + origSet := setLinkStateFn deleteTAPFn = func(string, string) error { return nil } deleteNetnsFn = func(context.Context, string) error { return nil } ensureNetnsFn = func(string, string) (bool, error) { return false, nil } setupTCRedirectFn = func(_, _, _ string, _ int, _ string) (string, error) { return "aa:bb:cc:dd:ee:01", nil } + setLinkStateFn = func(string, []string, bool) error { return nil } t.Cleanup(func() { deleteTAPFn, deleteNetnsFn, ensureNetnsFn, setupTCRedirectFn = origTAP, origNetns, origEnsure, origTC + setLinkStateFn = origSet }) } diff --git a/network/cni/lifecycle_darwin.go b/network/cni/lifecycle_darwin.go index aaa0329b..961fbf7d 100644 --- a/network/cni/lifecycle_darwin.go +++ b/network/cni/lifecycle_darwin.go @@ -23,6 +23,10 @@ func deleteTAPInNetns(_, _ string) error { return errNotSupported } +func setLinkStateInNetns(_ string, _ []string, _ bool) error { + return errNotSupported +} + func tapPresentInNetns(_, _ string) error { return errNotSupported } diff --git a/network/cni/lifecycle_linux.go b/network/cni/lifecycle_linux.go index 306296d3..d982314a 100644 --- a/network/cni/lifecycle_linux.go +++ b/network/cni/lifecycle_linux.go @@ -79,6 +79,33 @@ func deleteTAPInNetns(nsPath, tapName string) error { return err } +// setLinkStateInNetns brings ifNames up or down inside nsPath. A missing netns or link is success: Quiesce/Unquiesce run across stop/restart and partial teardown, where the plumbing may already be gone. +func setLinkStateInNetns(nsPath string, ifNames []string, up bool) error { + transition := netlink.LinkSetDown + if up { + transition = netlink.LinkSetUp + } + err := cns.WithNetNSPath(nsPath, func(_ cns.NetNS) error { + for _, name := range ifNames { + link, err := netlink.LinkByName(name) + if err != nil { + if _, ok := errors.AsType[netlink.LinkNotFoundError](err); ok { + continue + } + return fmt.Errorf("find %s: %w", name, err) + } + if err := transition(link); err != nil { + return fmt.Errorf("set %s state: %w", name, err) + } + } + return nil + }) + if _, ok := errors.AsType[cns.NSPathNotExistErr](err); ok { + return nil + } + return err +} + // setupTCRedirect wires ifName <-> tapName inside target netns, returns MAC. func setupTCRedirect(nsPath, ifName, tapName string, queues int, overrideMAC string) (string, error) { var mac string diff --git a/network/network.go b/network/network.go index 196f7819..22dc35f1 100644 --- a/network/network.go +++ b/network/network.go @@ -27,6 +27,9 @@ type Network interface { Prepare(ctx context.Context, vmID string, vmCfg *types.VMConfig) (string, error) Add(ctx context.Context, vmID string, vmCfg *types.VMConfig, specs ...AddSpec) ([]*types.NetworkConfig, error) Remove(ctx context.Context, vmID string, indices ...int) error + // Quiesce brings the VM's host-side NICs down when it stops so an idle TAP's TC redirect cannot storm softirqs (mirred-to-down-device); Unquiesce restores them before restart. + Quiesce(ctx context.Context, vmID string) error + Unquiesce(ctx context.Context, vmID string) error Delete(context.Context, []string) ([]string, error) Inspect(context.Context, string) (*types.Network, error) List(context.Context) ([]*types.Network, error) From f5956a4ea1a740eb4589250b49109f3ac03ec473 Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 14 Jul 2026 01:43:04 +0800 Subject: [PATCH 2/4] clean --- cmd/vm/lifecycle.go | 4 ---- network/cni/cni.go | 20 ++++++++++---------- network/network.go | 6 +----- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/cmd/vm/lifecycle.go b/cmd/vm/lifecycle.go index 092cfee0..cf9bf672 100644 --- a/cmd/vm/lifecycle.go +++ b/cmd/vm/lifecycle.go @@ -84,8 +84,6 @@ func (h Handler) Stop(cmd *cobra.Command, args []string) error { } return batchRoutedCmd(ctx, cmd, "stop", "stopped", routed, func(hyper hypervisor.Hypervisor, refs []string) ([]string, error) { stopped, err := hyper.Stop(ctx, refs) - // The TAP loses carrier when the VMM exits; bring the host NICs down so the - // kept-for-restart TC redirect can't storm softirqs against it (recoverNetwork undoes it). h.quiesceNetwork(ctx, conf, hyper, stopped) return stopped, err }) @@ -282,8 +280,6 @@ func (h Handler) recoverNetwork(ctx context.Context, conf *config.Config, hyper } recoverOne := func(vm *types.VM) { if netProvider.Verify(ctx, vm.ID, vm.NetworkConfigs) == nil { - // Plumbing intact (fast-restart path): undo Stop's quiesce so the host - // NICs forward again once the VMM re-opens the TAP. if err := netProvider.Unquiesce(ctx, vm.ID); err != nil { logger.Warnf(ctx, "unquiesce network for VM %s: %v", vm.ID, err) } diff --git a/network/cni/cni.go b/network/cni/cni.go index 8028c49f..a4b1673d 100644 --- a/network/cni/cni.go +++ b/network/cni/cni.go @@ -131,6 +131,16 @@ func (c *CNI) List(ctx context.Context) ([]*types.Network, error) { }) } +// Quiesce brings the VM's host-side veths down so a stopped VM's TC redirect stops storming softirqs against its now-carrier-less TAP (mirred-to-down-device). The netns and TAP are kept for a fast restart, which Unquiesce re-enables. +func (c *CNI) Quiesce(ctx context.Context, vmID string) error { + return c.setLinkState(ctx, vmID, false) +} + +// Unquiesce restores the veths Quiesce brought down, run on start before the VMM re-opens the TAP. +func (c *CNI) Unquiesce(ctx context.Context, vmID string) error { + return c.setLinkState(ctx, vmID, true) +} + // Delete tears down all NICs for each VM and removes the netns. Best-effort. func (c *CNI) Delete(ctx context.Context, vmIDs []string) ([]string, error) { result := utils.ForEach(ctx, vmIDs, func(ctx context.Context, vmID string) error { @@ -224,16 +234,6 @@ func (c *CNI) deleteRecords(ctx context.Context, ids []string) error { }) } -// Quiesce brings the VM's host-side veths down so a stopped VM's TC redirect stops storming softirqs against its now-carrier-less TAP (mirred-to-down-device). The netns and TAP are kept for a fast restart, which Unquiesce re-enables. -func (c *CNI) Quiesce(ctx context.Context, vmID string) error { - return c.setLinkState(ctx, vmID, false) -} - -// Unquiesce restores the veths Quiesce brought down, run on start before the VMM re-opens the TAP. -func (c *CNI) Unquiesce(ctx context.Context, vmID string) error { - return c.setLinkState(ctx, vmID, true) -} - func (c *CNI) setLinkState(ctx context.Context, vmID string, up bool) error { var records []networkRecord if err := c.store.With(ctx, func(idx *networkIndex) error { diff --git a/network/network.go b/network/network.go index 22dc35f1..713ba0c2 100644 --- a/network/network.go +++ b/network/network.go @@ -12,8 +12,7 @@ var ErrNotConfigured = errors.New("network provider not configured") // AddSpec is one NIC's add request; Existing != nil reuses MAC/IP for recovery. type AddSpec struct { - Index int - // Queues is the TAP queue count; 0 derives NetNumQueues(vmCfg.CPU), backend-specific callers (FC opens single-queue) set it explicitly. + Index int Queues int Existing *types.NetworkConfig } @@ -21,13 +20,10 @@ type AddSpec struct { // Network is the per-VM host-side networking provider (CNI, bridge, ...). type Network interface { Type() string - // Verify checks the VM's plumbing against its recorded NICs (netns and each TAP), not just container existence: a half-rolled-back recovery leaves the netns without TAPs. Verify(ctx context.Context, vmID string, expected []*types.NetworkConfig) error - // Prepare provisions per-VM state; returns the netns path or "". Prepare(ctx context.Context, vmID string, vmCfg *types.VMConfig) (string, error) Add(ctx context.Context, vmID string, vmCfg *types.VMConfig, specs ...AddSpec) ([]*types.NetworkConfig, error) Remove(ctx context.Context, vmID string, indices ...int) error - // Quiesce brings the VM's host-side NICs down when it stops so an idle TAP's TC redirect cannot storm softirqs (mirred-to-down-device); Unquiesce restores them before restart. Quiesce(ctx context.Context, vmID string) error Unquiesce(ctx context.Context, vmID string) error Delete(context.Context, []string) ([]string, error) From 8b9310f2e3dcf94f996368dfff4d114b25f3396d Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 14 Jul 2026 02:02:42 +0800 Subject: [PATCH 3/4] clean --- cmd/vm/hibernate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/vm/hibernate.go b/cmd/vm/hibernate.go index 37a7062c..9e18f2be 100644 --- a/cmd/vm/hibernate.go +++ b/cmd/vm/hibernate.go @@ -47,14 +47,14 @@ func (h Handler) Hibernate(cmd *cobra.Command, args []string) error { logger.Infof(ctx, "hibernating VM %s ...", vmRef) // persist runs inside the pause window; the VMM dies only after it succeeds. var snapID string - err = hib.Hibernate(ctx, vm.ID, func(cfg *types.SnapshotConfig, srcDir string) error { + if err := hib.Hibernate(ctx, vm.ID, func(cfg *types.SnapshotConfig, srcDir string) error { id, pErr := cmdcore.PersistSnapshotDir(ctx, snapBackend, cfg, srcDir, name, description) snapID = id return pErr - }) - if err != nil { + }); err != nil { return err } + h.quiesceNetwork(ctx, conf, hyper, []string{vm.ID}) logger.Infof(ctx, "VM hibernated; snapshot %s (resume: cocoon vm restore %s %s)", snapID, vmRef, cmp.Or(name, snapID)) return nil } From 7db02f6b2ecd00281e37baf790db84f8c03d34e9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 14 Jul 2026 02:13:56 +0800 Subject: [PATCH 4/4] fix(cni): also quiesce on vm rm --force, not only vm stop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vm rm --force kills the VMM (dropping the TAP's carrier) but only removes the netns/TC redirect afterwards via netProvider.Delete, leaving the same idle-TAP softirq storm window vm stop had — long enough to crash a host on a busy bridge. Extract stopAndQuiesce and reuse it from rm --force before the delete. --- cmd/vm/lifecycle.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/vm/lifecycle.go b/cmd/vm/lifecycle.go index cf9bf672..6f562dad 100644 --- a/cmd/vm/lifecycle.go +++ b/cmd/vm/lifecycle.go @@ -83,9 +83,7 @@ func (h Handler) Stop(cmd *cobra.Command, args []string) error { return err } return batchRoutedCmd(ctx, cmd, "stop", "stopped", routed, func(hyper hypervisor.Hypervisor, refs []string) ([]string, error) { - stopped, err := hyper.Stop(ctx, refs) - h.quiesceNetwork(ctx, conf, hyper, stopped) - return stopped, err + return h.stopAndQuiesce(ctx, conf, hyper, refs) }) } @@ -206,6 +204,12 @@ func (h Handler) RM(cmd *cobra.Command, args []string) error { return err } + if force { + for hyper, refs := range routed { + _, _ = h.stopAndQuiesce(ctx, conf, hyper, refs) + } + } + const logTag = "cmd.vm.rm" allDeleted, lastErr := runRoutedLoop(ctx, logTag, "deleted", cliutil.WantJSON(cmd), routed, func(hyper hypervisor.Hypervisor, refs []string) ([]string, error) { @@ -315,6 +319,12 @@ func (h Handler) recoverNetwork(ctx context.Context, conf *config.Config, hyper } } +func (h Handler) stopAndQuiesce(ctx context.Context, conf *config.Config, hyper hypervisor.Hypervisor, refs []string) ([]string, error) { + stopped, err := hyper.Stop(ctx, refs) + h.quiesceNetwork(ctx, conf, hyper, stopped) + return stopped, err +} + // quiesceNetwork brings each stopped VM's host NICs down — Stop's counterpart to Start's recoverNetwork — so an idle TAP's TC redirect can't storm softirqs. Best-effort: a quiesce failure never blocks the stop. func (h Handler) quiesceNetwork(ctx context.Context, conf *config.Config, hyper hypervisor.Hypervisor, ids []string) { if len(ids) == 0 {