diff --git a/CHANGELOG.md b/CHANGELOG.md index 79153c990dc..75912fedbd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to rescan the PCI bus after hotplug and remove the device before unplug since no automatic notification mechanism is implemented yet. More information can be found in the [Device Hotplugging](docs/device-hotplug.md) documentation page. +- [#5891](https://github.com/firecracker-microvm/firecracker/pull/5891): Added + support for virtio device reset. - [#5323](https://github.com/firecracker-microvm/firecracker/pull/5323): Add support for Vsock Unix domain socket path overriding on snapshot restore. More information can be found in the diff --git a/src/vmm/src/device_manager/mmio.rs b/src/vmm/src/device_manager/mmio.rs index e84784a3d5f..d405c9eef11 100644 --- a/src/vmm/src/device_manager/mmio.rs +++ b/src/vmm/src/device_manager/mmio.rs @@ -587,6 +587,12 @@ pub(crate) mod tests { fn is_activated(&self) -> bool { false } + + fn deactivate(&mut self) {} + + fn _reset(&mut self) -> bool { + false + } } #[test] diff --git a/src/vmm/src/devices/virtio/balloon/device.rs b/src/vmm/src/devices/virtio/balloon/device.rs index 1e962e828d7..cb25a20861b 100644 --- a/src/vmm/src/devices/virtio/balloon/device.rs +++ b/src/vmm/src/devices/virtio/balloon/device.rs @@ -979,6 +979,17 @@ impl VirtioDevice for Balloon { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + + fn _reset(&mut self) -> bool { + self.stats_timer.arm(Duration::ZERO, None); + self.stats_desc_index = None; + self.hinting_state = Default::default(); + true + } + fn kick(&mut self) { if self.is_activated() { if self.free_page_hinting() { diff --git a/src/vmm/src/devices/virtio/block/device.rs b/src/vmm/src/devices/virtio/block/device.rs index 31bea2771d8..8afb40a075d 100644 --- a/src/vmm/src/devices/virtio/block/device.rs +++ b/src/vmm/src/devices/virtio/block/device.rs @@ -207,6 +207,20 @@ impl VirtioDevice for Block { } } + fn deactivate(&mut self) { + match self { + Self::Virtio(b) => b.deactivate(), + Self::VhostUser(b) => b.deactivate(), + } + } + + fn _reset(&mut self) -> bool { + match self { + Self::Virtio(b) => b._reset(), + Self::VhostUser(b) => b._reset(), + } + } + fn prepare_save(&mut self) { match self { Self::Virtio(b) => b.prepare_save(), diff --git a/src/vmm/src/devices/virtio/block/vhost_user/device.rs b/src/vmm/src/devices/virtio/block/vhost_user/device.rs index 7df600722ab..f4dc506053a 100644 --- a/src/vmm/src/devices/virtio/block/vhost_user/device.rs +++ b/src/vmm/src/devices/virtio/block/vhost_user/device.rs @@ -379,6 +379,14 @@ where fn is_activated(&self) -> bool { self.device_state.is_activated() } + + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + + fn _reset(&mut self) -> bool { + false + } } #[cfg(test)] diff --git a/src/vmm/src/devices/virtio/block/virtio/device.rs b/src/vmm/src/devices/virtio/block/virtio/device.rs index ec7ec468505..1fb2e372938 100644 --- a/src/vmm/src/devices/virtio/block/virtio/device.rs +++ b/src/vmm/src/devices/virtio/block/virtio/device.rs @@ -674,6 +674,15 @@ impl VirtioDevice for VirtioBlock { fn is_activated(&self) -> bool { self.device_state.is_activated() } + + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + + fn _reset(&mut self) -> bool { + self.is_io_engine_throttled = false; + true + } } impl Drop for VirtioBlock { diff --git a/src/vmm/src/devices/virtio/device.rs b/src/vmm/src/devices/virtio/device.rs index 1cb591d7a3a..2f95d1c891b 100644 --- a/src/vmm/src/devices/virtio/device.rs +++ b/src/vmm/src/devices/virtio/device.rs @@ -178,12 +178,24 @@ pub trait VirtioDevice: AsAny + MutEventSubscriber + Send { /// Checks if the resources of this device are activated. fn is_activated(&self) -> bool; - /// Optionally deactivates this device and returns ownership of the guest memory map, interrupt - /// event, and queue events. - fn reset(&mut self) -> Option<(Arc, Vec)> { - None + /// Set the device state to Inactive + fn deactivate(&mut self); + + /// Reset the device. Returns true on success, false otherwise. + /// It must not be overridden. + fn reset(&mut self) -> bool { + self.deactivate(); + self.set_acked_features(0); + for queue in self.queues_mut() { + *queue = Queue::new(queue.max_size); + } + self._reset() } + /// Backend-specific reset logic. Returns true on success, false if the + /// backend does not support reset. + fn _reset(&mut self) -> bool; + /// Mark pages used by queues as dirty. fn mark_queue_memory_dirty(&mut self, mem: &GuestMemoryMmap) -> Result<(), QueueError> { for queue in self.queues_mut() { @@ -310,6 +322,14 @@ pub(crate) mod tests { fn is_activated(&self) -> bool { todo!() } + + fn deactivate(&mut self) { + todo!() + } + + fn _reset(&mut self) -> bool { + todo!() + } } #[test] diff --git a/src/vmm/src/devices/virtio/mem/device.rs b/src/vmm/src/devices/virtio/mem/device.rs index c6c0ea443e4..ffc4e22ad23 100644 --- a/src/vmm/src/devices/virtio/mem/device.rs +++ b/src/vmm/src/devices/virtio/mem/device.rs @@ -656,6 +656,21 @@ impl VirtioDevice for VirtioMem { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + + fn _reset(&mut self) -> bool { + // Virtio spec, section 5.15.5.2: + // The device MUST NOT change the state of memory blocks during device + // reset. The device MUST NOT modify memory or memory properties of + // plugged memory blocks during device reset. + // + // Note: the Linux virtio-mem driver does not support rebinding when + // memory is plugged + true + } + fn activate( &mut self, mem: GuestMemoryMmap, diff --git a/src/vmm/src/devices/virtio/net/device.rs b/src/vmm/src/devices/virtio/net/device.rs index 22b223b383b..625302e4afd 100644 --- a/src/vmm/src/devices/virtio/net/device.rs +++ b/src/vmm/src/devices/virtio/net/device.rs @@ -135,6 +135,15 @@ impl RxBuffers { }) } + /// Reset the RX buffers to their initial state. + fn clear(&mut self) { + self.iovec.clear(); + self.parsed_descriptors.clear(); + self.used_descriptors = 0; + self.used_bytes = 0; + self.min_buffer_size = 0; + } + /// Add a new `DescriptorChain` that we received from the RX queue in the buffer. /// /// SAFETY: The `DescriptorChain` cannot be referencing the same memory location as any other @@ -1088,6 +1097,16 @@ impl VirtioDevice for Net { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + + fn _reset(&mut self) -> bool { + self.rx_buffer.clear(); + self.tx_buffer.clear(); + true + } + /// Prepare saving state fn prepare_save(&mut self) { // We shouldn't be messing with the queue if the device is not activated. @@ -2601,4 +2620,16 @@ pub mod tests { assert!(queues[RX_INDEX].uses_notif_suppression); assert!(queues[TX_INDEX].uses_notif_suppression); } + + #[test] + fn test_reset() { + let mem = single_region_mem(2 * MAX_BUFFER_SIZE); + let mut th = TestHelper::get_default(&mem); + th.activate_net(); + + assert!(th.net().is_activated()); + assert!(th.net().reset()); + assert!(!th.net().is_activated()); + assert_eq!(th.net().acked_features(), 0); + } } diff --git a/src/vmm/src/devices/virtio/pmem/device.rs b/src/vmm/src/devices/virtio/pmem/device.rs index 4d4da40a8e9..1ae19c25bcb 100644 --- a/src/vmm/src/devices/virtio/pmem/device.rs +++ b/src/vmm/src/devices/virtio/pmem/device.rs @@ -588,6 +588,14 @@ impl VirtioDevice for Pmem { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + + fn _reset(&mut self) -> bool { + true + } + fn kick(&mut self) { if self.is_activated() { info!("kick pmem {}.", self.config.id); diff --git a/src/vmm/src/devices/virtio/queue.rs b/src/vmm/src/devices/virtio/queue.rs index 7fd862f45ca..79c635e5c4d 100644 --- a/src/vmm/src/devices/virtio/queue.rs +++ b/src/vmm/src/devices/virtio/queue.rs @@ -669,19 +669,6 @@ impl Queue { new - used_event - Wrapping(1) < new - old } - - /// Resets the Virtio Queue - pub(crate) fn reset(&mut self) { - self.ready = false; - self.size = self.max_size; - self.desc_table_address = GuestAddress(0); - self.avail_ring_address = GuestAddress(0); - self.used_ring_address = GuestAddress(0); - self.next_avail = Wrapping(0); - self.next_used = Wrapping(0); - self.num_added = Wrapping(0); - self.uses_notif_suppression = false; - } } #[cfg(kani)] diff --git a/src/vmm/src/devices/virtio/rng/device.rs b/src/vmm/src/devices/virtio/rng/device.rs index a6fee702628..e0da1f08f53 100644 --- a/src/vmm/src/devices/virtio/rng/device.rs +++ b/src/vmm/src/devices/virtio/rng/device.rs @@ -307,6 +307,14 @@ impl VirtioDevice for Entropy { self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + + fn _reset(&mut self) -> bool { + true + } + fn activate( &mut self, mem: GuestMemoryMmap, diff --git a/src/vmm/src/devices/virtio/transport/mmio.rs b/src/vmm/src/devices/virtio/transport/mmio.rs index 9c519604a96..e45198b97e2 100644 --- a/src/vmm/src/devices/virtio/transport/mmio.rs +++ b/src/vmm/src/devices/virtio/transport/mmio.rs @@ -153,9 +153,6 @@ impl MmioTransport { // . Keep interrupt_evt and queue_evts as is. There may be pending notifications in those // eventfds, but nothing will happen other than supurious wakeups. // . Do not reset config_generation and keep it monotonically increasing - for queue in self.locked_device().queues_mut() { - *queue = Queue::new(queue.max_size); - } } /// Update device status according to the state machine defined by VirtIO Spec 1.0. @@ -165,7 +162,6 @@ impl MmioTransport { /// of the driver initialization sequence specified in 3.1. The driver MUST NOT clear /// a device status bit. If the driver sets the FAILED bit, the driver MUST later reset /// the device before attempting to re-initialize. - #[allow(unused_assignments)] fn set_device_status(&mut self, status: u32) { use device_status::*; @@ -183,25 +179,11 @@ impl MmioTransport { // TODO: notify backend driver to stop the device self.device_status |= FAILED; } else if status == INIT { - { - let mut locked_device = self.device.lock().expect("Poisoned lock"); - if locked_device.is_activated() { - let mut device_status = self.device_status; - let reset_result = locked_device.reset(); - match reset_result { - Some((_interrupt_evt, mut _queue_evts)) => {} - None => { - device_status |= FAILED; - } - } - self.device_status = device_status; - } - } - - // If the backend device driver doesn't support reset, - // just leave the device marked as FAILED. - if self.device_status & FAILED == 0 { + if self.device_status != INIT { self.reset(); + if !self.device.lock().expect("Poisoned lock").reset() { + self.device_status |= FAILED; + } } } else if VALID_TRANSITIONS .iter() @@ -506,6 +488,7 @@ pub(crate) mod tests { device_activated: bool, config_bytes: [u8; 0xeff], activate_should_error: bool, + reset_should_fail: bool, } impl DummyDevice { @@ -522,6 +505,7 @@ pub(crate) mod tests { device_activated: false, config_bytes: [0; 0xeff], activate_should_error: false, + reset_should_fail: false, } } @@ -600,6 +584,14 @@ pub(crate) mod tests { fn is_activated(&self) -> bool { self.device_activated } + + fn deactivate(&mut self) { + self.device_activated = false; + } + + fn _reset(&mut self) -> bool { + !self.reset_should_fail + } } fn set_device_status(d: &mut MmioTransport, status: u32) { @@ -613,8 +605,7 @@ pub(crate) mod tests { let m = single_region_mem(0x1000); let interrupt = Arc::new(IrqTrigger::new()); let mut dummy = DummyDevice::new(); - // Validate reset is no-op. - assert!(dummy.reset().is_none()); + assert!(dummy.reset()); let mut d = MmioTransport::new(m, interrupt, Arc::new(Mutex::new(dummy)), false); // We just make sure here that the implementation of a mmio device behaves as we expect, @@ -1144,11 +1135,32 @@ pub(crate) mod tests { assert_eq!(d.device_status, 0x8f); assert!(d.locked_device().is_activated()); - // Nothing happens when backend driver doesn't support reset + // Resetting the device should deactivate it write_le_u32(&mut buf[..], 0x0); d.write(0x0, 0x70, &buf[..]); - assert_eq!(d.device_status, 0x8f); + assert_eq!(d.device_status, device_status::INIT); + assert!(!d.locked_device().is_activated()); + assert_eq!(d.locked_device().acked_features(), 0); + } + + #[test] + fn test_bus_device_reset_failure() { + let m = single_region_mem(0x1000); + let interrupt = Arc::new(IrqTrigger::new()); + let device = DummyDevice { + reset_should_fail: true, + ..DummyDevice::new() + }; + let mut d = MmioTransport::new(m, interrupt, Arc::new(Mutex::new(device)), false); + + activate_device(&mut d); assert!(d.locked_device().is_activated()); + + // A backend that doesn't support reset must set FAILED. + let mut buf = [0; 4]; + write_le_u32(&mut buf[..], 0x0); + d.write(0x0, 0x70, &buf[..]); + assert_ne!(d.device_status & device_status::FAILED, 0); } #[test] diff --git a/src/vmm/src/devices/virtio/transport/pci/device.rs b/src/vmm/src/devices/virtio/transport/pci/device.rs index 58735c1b723..23f262420a1 100644 --- a/src/vmm/src/devices/virtio/transport/pci/device.rs +++ b/src/vmm/src/devices/virtio/transport/pci/device.rs @@ -944,41 +944,29 @@ impl PciDevice for VirtioPciDevice { // Device has been reset by the driver if self.device_activated.load(Ordering::SeqCst) && self.is_driver_init() { let mut device = self.device.lock().unwrap(); - let reset_result = device.reset(); - match reset_result { - Some(_) => { - // Upon reset the device returns its interrupt EventFD - self.virtio_interrupt = None; - self.device_activated.store(false, Ordering::SeqCst); - - // Reset queue readiness (changes queue_enable), queue sizes - // and selected_queue as per spec for reset - self.virtio_device() - .lock() - .unwrap() - .queues_mut() - .iter_mut() - .for_each(Queue::reset); - self.common_config.queue_select = 0; - } - None => { - error!("Attempt to reset device when not implemented in underlying device"); - // The virtio spec does not specify what to do if reset fails. - // - // Our MMIO transport sets FAILED in this case, but we must NOT do that for PCI. - // During shutdown, the Linux kernel issues a reset to each virtio device. The - // virtio PCI driver then polls device_status until it reads back 0, unlike the - // virtio MMIO driver which simply writes 0 and returns. Setting FAILED would - // cause the poll to spin forever, breaking reboot command and Ctrl-Alt-Del. - // - PCI: https://elixir.bootlin.com/linux/v6.19.8/source/drivers/virtio/virtio_pci_modern.c#L546-L565 - // - MMIO: https://elixir.bootlin.com/linux/v6.19.8/source/drivers/virtio/virtio_mmio.c#L251-L258 - // - // Since device_status was already set to INIT by set_device_status(), we don't - // need to set it again here. However, the backend device is still active since - // reset() is unimplemented. The combination of device_activated == true and - // device_status == INIT will cause set_device_status() to block any - // re-initialization attempts. - } + if device.reset() { + self.device_activated.store(false, Ordering::SeqCst); + + self.common_config.queue_select = 0; + self.common_config.device_feature_select = 0; + self.common_config.driver_feature_select = 0; + } else { + error!("Attempt to reset device when not implemented in underlying device"); + // The virtio spec does not specify what to do if reset fails. + // + // Our MMIO transport sets FAILED in this case, but we must NOT do that for PCI. + // During shutdown, the Linux kernel issues a reset to each virtio device. The + // virtio PCI driver then polls device_status until it reads back 0, unlike the + // virtio MMIO driver which simply writes 0 and returns. Setting FAILED would + // cause the poll to spin forever, breaking reboot command and Ctrl-Alt-Del. + // - PCI: https://elixir.bootlin.com/linux/v6.19.8/source/drivers/virtio/virtio_pci_modern.c#L546-L565 + // - MMIO: https://elixir.bootlin.com/linux/v6.19.8/source/drivers/virtio/virtio_mmio.c#L251-L258 + // + // Since device_status was already set to INIT by set_device_status(), we don't + // need to set it again here. However, the backend device is still active since + // reset() is unimplemented. The combination of device_activated == true and + // device_status == INIT will cause set_device_status() to block any + // re-initialization attempts. } } None @@ -1672,7 +1660,7 @@ mod tests { } #[test] - fn test_failed_reset_blocks_reinitialization() { + fn test_reset_and_reinitialization() { let mut vmm = create_vmm_with_virtio_pci_device(); let device = get_virtio_device(&vmm); let mut locked = device.lock().unwrap(); @@ -1688,40 +1676,20 @@ mod tests { assert!(locked.device_activated.load(Ordering::SeqCst)); // Write 0 to device_status to request a reset. - // Entropy's reset() returns None (unimplemented), so the reset fails. write_driver_status(&mut locked, 0); assert_eq!(read_driver_status(&mut locked), 0); - // device_activated stays true because the backend was not actually reset. - assert!(locked.device_activated.load(Ordering::SeqCst)); + // Device should be deactivated after successful reset. + assert!(!locked.device_activated.load(Ordering::SeqCst)); - // Attempt to re-initialize should be rejected because device_activated is - // still true while driver_status is INIT. + // Re-initialization should succeed after a successful reset. write_driver_status(&mut locked, ACKNOWLEDGE); - assert_eq!(read_driver_status(&mut locked), 0); - - // Save state and restore into a new device -- the combination of - // device_activated == true and driver_status == INIT is preserved in the - // snapshot, so the blocking behavior survives restore. - let saved_state = locked.state(); - drop(locked); - - // Fully drop the original device before constructing the restored copy: the restored - // copy reuses the same MSI-X GSIs, so the two `MsixVectorGroup` instances must never - // exist concurrently. This is how it will happen in real scenario anyway. - let kvm_vm = vmm.vm.as_kvm().unwrap().clone(); - let saved_allocator = kvm_vm.resource_allocator().clone(); - drop(device); - drop(vmm); - // Restore the allocator state so the restored group's GSIs are marked allocated and - // its `MsixVectorGroup::drop` will succeed when the test ends. - *kvm_vm.resource_allocator() = saved_allocator; - - let new_entropy = Arc::new(Mutex::new(Entropy::new(RateLimiter::default()).unwrap())); - let restored = - VirtioPciDevice::new_from_state("rng".to_string(), &kvm_vm, new_entropy, saved_state) - .unwrap(); - - assert!(restored.device_activated.load(Ordering::SeqCst)); - assert_eq!(restored.common_config.driver_status, 0); + assert_eq!(read_driver_status(&mut locked), ACKNOWLEDGE); + write_driver_status(&mut locked, ACKNOWLEDGE | DRIVER); + let features = read_device_features(&mut locked); + write_driver_features(&mut locked, features); + write_driver_status(&mut locked, ACKNOWLEDGE | DRIVER | FEATURES_OK); + setup_queues(&mut locked); + write_driver_status(&mut locked, ACKNOWLEDGE | DRIVER | FEATURES_OK | DRIVER_OK); + assert!(locked.device_activated.load(Ordering::SeqCst)); } } diff --git a/src/vmm/src/devices/virtio/vsock/device.rs b/src/vmm/src/devices/virtio/vsock/device.rs index 8c8c51eb45e..dcd39f14514 100644 --- a/src/vmm/src/devices/virtio/vsock/device.rs +++ b/src/vmm/src/devices/virtio/vsock/device.rs @@ -389,6 +389,17 @@ where self.device_state.is_activated() } + fn deactivate(&mut self) { + self.device_state = DeviceState::Inactive; + } + + fn _reset(&mut self) -> bool { + self.backend.reset(); + self.rx_packet.clear(); + self.tx_packet.clear(); + true + } + fn kick(&mut self) { // Vsock has complicated protocol that isn't resilient to any packet loss, // so for Vsock we don't support connection persistence through snapshot. diff --git a/src/vmm/src/devices/virtio/vsock/mod.rs b/src/vmm/src/devices/virtio/vsock/mod.rs index cc9f7746580..1ec87534af1 100644 --- a/src/vmm/src/devices/virtio/vsock/mod.rs +++ b/src/vmm/src/devices/virtio/vsock/mod.rs @@ -179,4 +179,7 @@ pub trait VsockChannel { /// The vsock backend, which is basically an epoll-event-driven vsock channel. /// Currently, the only implementation we have is `crate::devices::virtio::unix::muxer::VsockMuxer`, /// which translates guest-side vsock connections to host-side Unix domain socket connections. -pub trait VsockBackend: VsockChannel + VsockEpollListener + Send {} +pub trait VsockBackend: VsockChannel + VsockEpollListener + Send { + /// Reset the backend, dropping all active connections. + fn reset(&mut self); +} diff --git a/src/vmm/src/devices/virtio/vsock/packet.rs b/src/vmm/src/devices/virtio/vsock/packet.rs index 7253f41ce76..baac307e773 100644 --- a/src/vmm/src/devices/virtio/vsock/packet.rs +++ b/src/vmm/src/devices/virtio/vsock/packet.rs @@ -197,6 +197,12 @@ pub struct VsockPacketTx { } impl VsockPacketTx { + /// Clear the packet buffer. + pub fn clear(&mut self) { + self.hdr = Default::default(); + self.buffer.clear(); + } + /// Create the packet wrapper from a TX virtq chain head. /// /// ## Errors @@ -290,6 +296,12 @@ pub struct VsockPacketRx { } impl VsockPacketRx { + /// Clear the packet buffer. + pub fn clear(&mut self) { + self.hdr = Default::default(); + self.buffer.clear(); + } + /// Creates new VsockPacketRx. pub fn new() -> Result { let buffer = IoVecBufferMut::new().map_err(VsockError::IovDeque)?; diff --git a/src/vmm/src/devices/virtio/vsock/test_utils.rs b/src/vmm/src/devices/virtio/vsock/test_utils.rs index 3d4ab704975..30c86fad169 100644 --- a/src/vmm/src/devices/virtio/vsock/test_utils.rs +++ b/src/vmm/src/devices/virtio/vsock/test_utils.rs @@ -113,7 +113,9 @@ impl VsockEpollListener for TestBackend { self.evset = Some(evset); } } -impl VsockBackend for TestBackend {} +impl VsockBackend for TestBackend { + fn reset(&mut self) {} +} #[derive(Debug)] pub struct TestContext { diff --git a/src/vmm/src/devices/virtio/vsock/unix/muxer.rs b/src/vmm/src/devices/virtio/vsock/unix/muxer.rs index e2c28033ec7..7946ec1e7bb 100644 --- a/src/vmm/src/devices/virtio/vsock/unix/muxer.rs +++ b/src/vmm/src/devices/virtio/vsock/unix/muxer.rs @@ -305,7 +305,33 @@ impl VsockEpollListener for VsockMuxer { } } -impl VsockBackend for VsockMuxer {} +impl VsockBackend for VsockMuxer { + fn reset(&mut self) { + // Remove all connections and their epoll listeners. + let keys: Vec = self.conn_map.keys().copied().collect(); + for key in keys { + self.remove_connection(key); + } + + // Remove any pending LocalStream listeners (host sockets that + // connected but haven't sent a CONNECT command yet). Keep only + // the HostSock listener. + let stale_fds: Vec = self + .listener_map + .iter() + .filter(|(_, listener)| !matches!(listener, EpollListener::HostSock)) + .map(|(fd, _)| *fd) + .collect(); + for fd in stale_fds { + self.remove_listener(fd); + } + + self.rxq = MuxerRxQ::new(); + self.killq = MuxerKillQ::new(); + self.local_port_set.clear(); + self.local_port_last = (1u32 << 30) - 1; + } +} impl VsockMuxer { /// Muxer constructor. diff --git a/tests/integration_tests/functional/test_balloon.py b/tests/integration_tests/functional/test_balloon.py index 44d9b9b6f37..2be3261f181 100644 --- a/tests/integration_tests/functional/test_balloon.py +++ b/tests/integration_tests/functional/test_balloon.py @@ -622,3 +622,46 @@ def test_memory_scrub(uvm_plain_any, method): microvm.ssh.check_output("/usr/local/bin/readmem {} {}".format(60, 1)) check_guest_dmesg_for_stalls(microvm.ssh) + + +def test_device_reset(uvm_plain_any): + """ + Test that virtio-balloon device reset works. + """ + vm = uvm_plain_any + vm.spawn() + vm.basic_config() + vm.add_net_iface() + vm.api.balloon.put(amount_mib=0, deflate_on_oom=True, stats_polling_interval_s=0) + vm.start() + + # Find the virtio balloon device. + virtio_dev = vm.ssh.check_output( + "ls -d /sys/bus/virtio/drivers/virtio_balloon/virtio* | xargs -n1 basename" + ).stdout.strip() + + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_balloon/unbind" + ) + + # Verify the balloon is gone. + ret = vm.ssh.run("ls /sys/bus/virtio/drivers/virtio_balloon/virtio*") + assert ret.returncode != 0 + + # Rebind and verify it's back. + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_balloon/bind" + ) + + # Verify the balloon is functional by inflating it and checking that guest + # free memory decreases. + meminfo = MeminfoGuest(vm) + free_before = meminfo.get().mem_free.kib() + + vm.api.balloon.patch(amount_mib=64) + _ = get_stable_rss_mem(vm) + + free_after = meminfo.get().mem_free.kib() + # Inflating 64 MiB should reclaim at least 85% of that from guest free + # memory. The 15% slack accounts for kernel accounting overhead. + assert free_after <= free_before - 64 * 1024 * 85 // 100 diff --git a/tests/integration_tests/functional/test_drive_virtio.py b/tests/integration_tests/functional/test_drive_virtio.py index 9c61ead56a9..f2720e73260 100644 --- a/tests/integration_tests/functional/test_drive_virtio.py +++ b/tests/integration_tests/functional/test_drive_virtio.py @@ -383,3 +383,38 @@ def _check_mount(ssh_connection, dev_path): assert stderr == "" _, _, stderr = ssh_connection.run("umount /tmp", timeout=30.0) assert stderr == "" + + +def test_device_reset(uvm_plain_any): + """ + Test that virtio-block device reset works. + """ + vm = uvm_plain_any + vm.spawn() + vm.basic_config() + vm.add_net_iface() + + fs = drive_tools.FilesystemFile(os.path.join(vm.fsfiles, "scratch"), size=2) + vm.add_drive("scratch", fs.path) + vm.start() + + # Verify the scratch drive is accessible. + vm.ssh.check_output("mount /dev/vdb /tmp && umount /tmp") + + # Find the virtio device backing vdb and unbind it. + virtio_dev = vm.ssh.check_output( + "basename $(readlink /sys/block/vdb/device)" + ).stdout.strip() + + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_blk/unbind" + ) + + # Verify the drive is gone. + ret = vm.ssh.run("ls /dev/vdb") + assert ret.returncode != 0 + + # Rebind and verify the drive is back. + vm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_blk/bind") + vm.ssh.check_output("ls /dev/vdb") + vm.ssh.check_output("mount /dev/vdb /tmp && umount /tmp") diff --git a/tests/integration_tests/functional/test_net.py b/tests/integration_tests/functional/test_net.py index 3830c30ef6b..1b00f07f7ac 100644 --- a/tests/integration_tests/functional/test_net.py +++ b/tests/integration_tests/functional/test_net.py @@ -175,3 +175,46 @@ def test_tap_mtu_advertised_to_guest(uvm_plain_any): f"{iface_name} (guest: {guest_if}): VIRTIO_NET_F_MTU (bit {VIRTIO_NET_F_MTU_BIT})" f" not set in negotiated features: {features!r}" ) + + +def test_device_reset(uvm_plain_any): + """ + Test that virtio-net device reset works. + """ + vm = uvm_plain_any + vm.spawn() + vm.basic_config() + vm.add_net_iface() # eth0 - used for SSH + iface2 = vm.add_net_iface() # eth1 - will be reset + vm.start() + + guest_ip2 = iface2.guest_ip + host_ip2 = iface2.host_ip + virtio_dev = vm.ssh.check_output( + "basename $(readlink /sys/class/net/eth1/device)" + ).stdout.strip() + + def configure_eth1(): + vm.ssh.check_output( + f"ip addr flush dev eth1 && " + f"ip addr add {guest_ip2}/30 dev eth1 && " + f"ip link set eth1 up" + ) + + configure_eth1() + vm.ssh.check_output(f"ping -c 1 {host_ip2}") + + # Reset eth1 by unbinding and rebinding its virtio driver. + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_net/unbind" + ) + + # Verify that ping fails after unbind. + ret = vm.ssh.run(f"ping -c 1 -W 1 {host_ip2}") + assert ret.returncode != 0 + + # Re-bind and check again + vm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_net/bind") + + configure_eth1() + vm.ssh.check_output(f"ping -c 1 {host_ip2}") diff --git a/tests/integration_tests/functional/test_pmem.py b/tests/integration_tests/functional/test_pmem.py index ad7fc35020e..904438c61e4 100644 --- a/tests/integration_tests/functional/test_pmem.py +++ b/tests/integration_tests/functional/test_pmem.py @@ -226,3 +226,40 @@ def test_pmem_dax_memory_saving( assert ( pmem_rss_usage < block_rss_usage ), f"{block_cache_usage} <= {pmem_cache_usage}" + + +def test_device_reset(uvm_plain_any): + """ + Test that virtio-pmem device reset works. + """ + vm = uvm_plain_any + vm.spawn() + vm.basic_config(add_root_device=True) + vm.add_net_iface() + + fs = drive_tools.FilesystemFile(os.path.join(vm.fsfiles, "scratch"), size=2) + vm.add_pmem("pmem_scratch", fs.path, False, False) + vm.start() + + # Verify the pmem device is accessible. + vm.ssh.check_output("ls /dev/pmem0") + + virtio_dev = vm.ssh.check_output( + "basename $(realpath /sys/block/pmem0/device/../../..)" + ).stdout.strip() + + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_pmem/unbind" + ) + + # Verify the device is gone. + ret = vm.ssh.run("ls /dev/pmem0") + assert ret.returncode != 0 + + # Rebind and verify the device is functional. + vm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_pmem/bind") + vm.ssh.check_output("mount /dev/pmem0 /tmp") + vm.ssh.check_output("echo reset_test > /tmp/testfile") + ret = vm.ssh.check_output("cat /tmp/testfile") + assert "reset_test" in ret.stdout + vm.ssh.check_output("umount /tmp") diff --git a/tests/integration_tests/functional/test_rng.py b/tests/integration_tests/functional/test_rng.py index 7eb26f95c50..5cdac9d8e7c 100644 --- a/tests/integration_tests/functional/test_rng.py +++ b/tests/integration_tests/functional/test_rng.py @@ -231,3 +231,35 @@ def test_rng_bw_rate_limiter(uvm_any): # Check the rate limiter using a request size equal to the size # of the token bucket. _check_entropy_rate_limited(vm.ssh, size, expected_kbps) + + +def test_device_reset(uvm_plain_any): + """ + Test that virtio-rng device reset works. + """ + vm = uvm_plain_any + vm.spawn() + vm.basic_config() + vm.add_net_iface() + vm.api.entropy.put() + vm.start() + + # Verify the rng device works. + vm.ssh.check_output("dd if=/dev/hwrng of=/dev/null bs=32 count=1") + + # Find the virtio rng device. + virtio_dev = vm.ssh.check_output( + "ls -d /sys/bus/virtio/drivers/virtio_rng/virtio* | xargs -n1 basename" + ).stdout.strip() + + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_rng/unbind" + ) + + # Verify the rng device is gone. + ret = vm.ssh.run("ls /sys/bus/virtio/drivers/virtio_rng/virtio*") + assert ret.returncode != 0 + + # Rebind and verify it works again. + vm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_rng/bind") + vm.ssh.check_output("dd if=/dev/hwrng of=/dev/null bs=32 count=1") diff --git a/tests/integration_tests/functional/test_vsock.py b/tests/integration_tests/functional/test_vsock.py index 7dd96ba7fc4..3a544d97c7e 100644 --- a/tests/integration_tests/functional/test_vsock.py +++ b/tests/integration_tests/functional/test_vsock.py @@ -367,3 +367,44 @@ def test_vsock_override_fails_without_device(uvm_plain_any, microvm_factory): ) vm2.mark_killed() + + +def test_device_reset(uvm_plain_any): + """ + Test that virtio-vsock device reset works. + """ + vm = uvm_plain_any + vm.spawn() + vm.basic_config() + vm.add_net_iface() + vm.api.vsock.put(guest_cid=3, vsock_id="vsock0", uds_path="/" + VSOCK_UDS_PATH) + vm.start() + + # Establish a vsock connection before reset so there is active state in the + # backend that must be cleaned up. + uds_path = start_guest_echo_server(vm) + blob_path, blob_hash = make_blob(vm.path) + check_host_connections(uds_path, blob_path, blob_hash) + + # Find the virtio vsock device. + virtio_dev = vm.ssh.check_output( + "ls -d /sys/bus/virtio/drivers/vmw_vsock_virtio_transport/virtio*" + " | xargs -n1 basename" + ).stdout.strip() + + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/vmw_vsock_virtio_transport/unbind" + ) + + # Verify the vsock device is gone. + ret = vm.ssh.run("ls /sys/bus/virtio/drivers/vmw_vsock_virtio_transport/virtio*") + assert ret.returncode != 0 + + # Rebind and verify the data path works by starting a guest echo server + # and sending data from the host through the vsock. + vm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/vmw_vsock_virtio_transport/bind" + ) + uds_path = start_guest_echo_server(vm) + blob_path, blob_hash = make_blob(vm.path) + check_host_connections(uds_path, blob_path, blob_hash) diff --git a/tests/integration_tests/performance/test_hotplug_memory.py b/tests/integration_tests/performance/test_hotplug_memory.py index d6b6cee8da6..8108f560eaa 100644 --- a/tests/integration_tests/performance/test_hotplug_memory.py +++ b/tests/integration_tests/performance/test_hotplug_memory.py @@ -499,3 +499,29 @@ def test_memory_hotplug_latency( timed_memory_hotplug(uvm, 0, metrics, "hotunplug", "unplug_agg") timed_memory_hotplug(uvm, hotplug_size, metrics, "hotplug_2nd", "plug_agg") uvm.kill() + + +def test_device_reset(uvm_plain): + """ + Test that virtio-mem device reset works. + + Note: the Linux virtio-mem driver does not support rebinding when memory is + plugged (the resource region can't be re-registered), so we reset without + any plugged memory and verify the device is functional afterwards. + """ + config = {"total_size_mib": 1024, "slot_size_mib": 128, "block_size_mib": 2} + uvm = uvm_booted_memhp(uvm_plain, None, None, False, config, None, None, None) + + # Reset the device via driver unbind/bind. + virtio_dev = uvm.ssh.check_output( + "ls -d /sys/bus/virtio/drivers/virtio_mem/virtio* | xargs -n1 basename" + ).stdout.strip() + + uvm.ssh.check_output( + f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_mem/unbind" + ) + uvm.ssh.check_output(f"echo {virtio_dev} > /sys/bus/virtio/drivers/virtio_mem/bind") + + # Verify the device is functional after reset by hotplugging memory. + # check_hotplug() asserts that guest mem_total reflects the new size. + check_hotplug(uvm, 256)