vsock listener cannot accept a second connection

[archevel]

After a vsock server accepts and closes one connection, accept() returns EBADF (-9) on all subsequent calls. Any new connection attempt from the host is immediately reset (ENODEV on the client side).

Expected behaviour

A vsock listener should accept multiple sequential connections, the same as TCP. This works correctly on Linux (see comparison below).

Actual behaviour

• Connection 1: accepted, data exchanged, connection closed — works correctly.
• Connection 2: accept() returns -9 (EBADF) immediately. The host client gets ENODEV (No such device, os error 19) when attempting to connect.

Reproduction

The essential main logic in the vsock "server" compiled as a hermit-os application:

#[cfg(target_os = "hermit")]
use hermit as _;

/// Minimal vsock ping/pong server for hermit-os.
///
/// Listens on vsock port 1234. For each connection, reads one line and
/// writes "pong\n" back. Expected to handle multiple sequential connections.
///
/// Bug: after the first connection is closed, accept() returns EBADF (-9)
/// forever and any new connection attempt from the host gets RST immediately.
use hermit_abi::{
    accept, bind, listen, read, sa_family_t, sockaddr, sockaddr_vm, socket, socklen_t, write,
    AF_VSOCK, SOCK_STREAM, VMADDR_CID_ANY,
};
use std::mem::size_of;

const PORT: u32 = 1234;

fn main() {
    eprintln!("[pingpong] binding vsock port {PORT}");

    // Retry bind/listen until the vsock driver is ready (async PCI init).
    let listener_fd = 'bind: {
        for _ in 0..100 {
        let saddr = sockaddr_vm {
            svm_len: size_of::<sockaddr_vm>() as u8,
            svm_reserved1: 0,
            svm_family: AF_VSOCK as sa_family_t,
            svm_cid: VMADDR_CID_ANY,
            svm_port: PORT,
            svm_zero: [0; 4],
        };

        let fd = unsafe { socket(AF_VSOCK, SOCK_STREAM, 0) };
        if fd < 0 { std::thread::yield_now(); continue; }

        let r = unsafe { bind(fd, &saddr as *const _ as *const sockaddr, size_of::<sockaddr_vm>() as socklen_t) };
        if r < 0 { unsafe { hermit_abi::close(fd) }; std::thread::yield_now(); continue; }

        let r = unsafe { listen(fd, 128) };
        if r < 0 { unsafe { hermit_abi::close(fd) }; std::thread::yield_now(); continue; }

        eprintln!("[pingpong] listener fd={fd}");
        break 'bind fd;
        }
        eprintln!("[pingpong] failed to bind vsock port {PORT} after 100 attempts");
        std::process::exit(1);
    };

    let mut conn_count = 0u32;
    loop {
        eprintln!("[pingpong] waiting for connection #{}", conn_count + 1);

        let mut peer_addr = sockaddr_vm {
            svm_len: size_of::<sockaddr_vm>() as u8,
            svm_reserved1: 0,
            svm_family: AF_VSOCK as sa_family_t,
            svm_cid: 0,
            svm_port: 0,
            svm_zero: [0; 4],
        };
        let mut peer_len = size_of::<sockaddr_vm>() as socklen_t;

        let conn_fd = unsafe {
            accept(listener_fd, &mut peer_addr as *mut _ as *mut sockaddr, &mut peer_len)
        };

        if conn_fd < 0 {
            eprintln!("[pingpong] accept failed: {conn_fd} — listener fd={listener_fd} is broken after first connection");
            eprintln!("[pingpong] BUG: vsock listener cannot accept a second connection");
            std::process::exit(1);
        }

        conn_count += 1;
        eprintln!("[pingpong] accepted connection #{conn_count} fd={conn_fd} from cid={}", peer_addr.svm_cid);

        let mut buf = [0u8; 64];
        let n = unsafe { read(conn_fd, buf.as_mut_ptr(), buf.len()) };
        if n > 0 {
            let msg = std::str::from_utf8(&buf[..n as usize]).unwrap_or("<invalid utf8>");
            eprintln!("[pingpong] received ({n} bytes): {msg:?}");
        } else {
            eprintln!("[pingpong] read returned {n}");
        }

        let pong = b"pong\n";
        let w = unsafe { write(conn_fd, pong.as_ptr(), pong.len()) };
        eprintln!("[pingpong] wrote pong: {w} bytes");

        eprintln!("[pingpong] closing conn fd={conn_fd}");
        unsafe { hermit_abi::close(conn_fd) };
        eprintln!("[pingpong] conn fd={conn_fd} closed");
    }
}

Then a small client that I've compiled as a regular linux binary:

/// Minimal vsock ping tool for testing vsock-pingpong unikernel.
///
/// Usage: vsock-ping <cid> <port> [count]
use std::io::{Read, Write};
use vsock::VsockStream;

fn main() {
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 3 {
        eprintln!("usage: vsock-ping <cid> <port> [count]");
        std::process::exit(1);
    }
    let cid: u32 = args[1].parse().expect("cid must be u32");
    let port: u32 = args[2].parse().expect("port must be u32");
    let count: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(2);

    let mut all_ok = true;
    for i in 1..=count {
        eprintln!("--- ping {i}/{count} ---");
        match do_ping(cid, port) {
            Ok(response) => {
                println!("response: {response:?}");
                eprintln!("--- ok ---");
            }
            Err(e) => {
                eprintln!("--- FAILED: {e} ---");
                all_ok = false;
            }
        }
        if i < count {
            std::thread::sleep(std::time::Duration::from_secs(1));
        }
    }

    if !all_ok {
        std::process::exit(1);
    }
}

fn do_ping(cid: u32, port: u32) -> Result<String, Box<dyn std::error::Error>> {
    eprintln!("connecting vsock cid={cid} port={port}...");
    let mut stream = VsockStream::connect_with_cid_port(cid, port)?;
    eprintln!("connected, sending ping...");
    stream.write_all(b"ping\n")?;
    eprintln!("sent, reading response...");
    let mut buf = [0u8; 64];
    let n = stream.read(&mut buf)?;
    Ok(String::from_utf8_lossy(&buf[..n]).into_owned())
}

Steps

Terminal 1 — build and boot the unikernel. I have used QEMU with vhost-vsock-pci:

cargo build --release;
qemu-system-x86_64 -enable-kvm -cpu host  -m 64M -kernel "$LOADER"  -initrd "target/x86_64-unknown-hermit/release/vsock-pingpong" -device "vhost-vsock-pci,guest-cid=99"  -nographic -serial stdio -monitor none  -no-reboot

Terminal 2 — build and run the vsock-client that send two sequential pings once the VM prints waiting for connection #1:

cargo build;
target/debug/vsock-ping 99 1234 2

Observed output (after boot output)

VM:

[pingpong] listener fd=3
[pingpong] waiting for connection #1
[pingpong] accepted connection #1 fd=4 from cid=2
[pingpong] received (5 bytes): "ping\n"
[pingpong] wrote pong: 5 bytes
[pingpong] closing conn fd=4
[pingpong] conn fd=4 closed
[pingpong] waiting for connection #2
[pingpong] accept failed: -9 — listener fd=3 is broken after first connection
[pingpong] BUG: vsock listener cannot accept a second connection
Number of interrupts
[0][FPU]: 1
[0][virtio]: 3
exit status 1

Client output:

--- ping 1/2 ---
connecting vsock cid=99 port=1234...
connected, sending ping...
sent, reading response...
response: "pong\n"
--- ok ---
--- ping 2/2 ---
connecting vsock cid=99 port=1234...
--- FAILED: No such device (os error 19) ---

---

I did setup a similar ping server that I compiled against linux and it manages to handle the second accept:

[pingpong] listener fd=3
[pingpong] waiting for connection #1
[pingpong] accepted connection #1 fd=4 from cid=1
[pingpong] received (5 bytes): "ping\n"
[pingpong] wrote pong: 5 bytes
[pingpong] closing conn fd=4
[pingpong] conn fd=4 closed
[pingpong] waiting for connection #2
[pingpong] accepted connection #2 fd=4 from cid=1
[pingpong] received (5 bytes): "ping\n"
[pingpong] wrote pong: 5 bytes
[pingpong] closing conn fd=4
[pingpong] conn fd=4 closed
[pingpong] waiting for connection #3