From d298609b6ed70322a901d17f41941b38e2160fe8 Mon Sep 17 00:00:00 2001 From: Xalestar Date: Wed, 15 Jul 2026 16:06:51 +0800 Subject: [PATCH] Raise host NOFILE capacity for the guest FD table elfuse exposes a 1024-entry guest FD table but inherits the host RLIMIT_NOFILE. With a host soft limit of 256, guests hit host-side EMFILE while the guest table still has free entries. Require host capacity for FD_TABLE_SIZE plus a 256-descriptor reserve. The reserve budgets the bounded two-reference blocking-I/O cost across MAX_THREADS and leaves headroom for elfuse runtime descriptors. Raise only the host soft limit, preserve its hard limit, and fail at startup with a diagnostic when the required capacity cannot be established. Keep this internal capacity separate from a guest-visible 1024/1024 NOFILE limit used by resource-limit syscalls, proc reporting, and FD allocation. Carry the guest limit through fork IPC, restore inherited descriptors before applying a lowered limit, and recheck host capacity in fork helpers. Update the fork protocol identity and add regressions for hidden reserve reporting and fork after lowering the hard limit. PR #148 addresses the same low-host-limit failure with a different startup policy and a host-side regression wrapper. Close #68 --- src/main.c | 51 +++++++++++++ src/runtime/fork-state.h | 8 ++- src/runtime/forkipc.c | 22 ++++++ src/syscall/fdtable.c | 5 -- src/syscall/internal.h | 3 - src/syscall/sys.c | 107 +++++++++++++++++++++------- src/syscall/sys.h | 7 ++ tests/test-fd-lifecycle.c | 69 ++++++++++++++++++ tests/test-fork-ipc-protocol-host.c | 7 +- 9 files changed, 241 insertions(+), 38 deletions(-) diff --git a/src/main.c b/src/main.c index 62fac804..46380027 100644 --- a/src/main.c +++ b/src/main.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "utils.h" @@ -157,6 +158,48 @@ _Static_assert((INFRA_PT_POOL_OFF & 0xFFF) == 0 && /* Build-time version string (generated by make into build/version.h) */ #include "version.h" +/* Host descriptors used outside the guest FD table. Blocking I/O may hold two + * duplicated descriptors per guest thread (for example, copy_file_range()), + * requiring up to 2 * MAX_THREADS descriptors. Keep another 128 descriptors + * for runtime pipes, fork IPC, debugger sockets, sysroot/FUSE plumbing, and + * similar bounded overhead. Operations whose descriptor use grows with their + * input set, such as ppoll(), require separate accounting. + */ +#define HOST_FD_RESERVE 256 + +static int host_nofile_ensure_capacity(void) +{ + const rlim_t required = (rlim_t) FD_TABLE_SIZE + HOST_FD_RESERVE; + struct rlimit limit; + + if (getrlimit(RLIMIT_NOFILE, &limit) < 0) { + log_error("failed to read host RLIMIT_NOFILE: %s", strerror(errno)); + return -1; + } + if (limit.rlim_cur >= required) + return 0; + + if (limit.rlim_max != RLIM_INFINITY && limit.rlim_max < required) { + log_error( + "host RLIMIT_NOFILE hard limit %llu is below the required %llu " + "(%d guest FDs + %d elfuse reserve)", + (unsigned long long) limit.rlim_max, (unsigned long long) required, + FD_TABLE_SIZE, HOST_FD_RESERVE); + return -1; + } + + rlim_t old_soft = limit.rlim_cur; + limit.rlim_cur = required; + if (setrlimit(RLIMIT_NOFILE, &limit) < 0) { + log_error("failed to raise host RLIMIT_NOFILE from %llu to %llu: %s", + (unsigned long long) old_soft, (unsigned long long) required, + strerror(errno)); + return -1; + } + + return 0; +} + /* Verify the host CPU's DC ZVA granule matches the shim's hardcoded value. * * DCZID_EL0 is readable from EL0 without trapping, so guest libc reads the @@ -365,6 +408,14 @@ int main(int argc, char **argv) } proc_set_fakeroot_enabled(fakeroot); + /* Top-level processes establish the capacity; fork helpers normally + * inherit it, but recheck before receiving the parent's FD table. Guest + * RLIMIT_NOFILE state is virtualized separately and cannot lower this + * internal host reserve. + */ + if (host_nofile_ensure_capacity() < 0) + return 1; + /* Block the vCPU-preemption signals and start the sigwait thread before any * vCPU thread exists, so both the normal path and the fork-child path below * inherit the block on every thread they spawn. diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index 87a59555..4f1b77cb 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C464CU /* "ELFL" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C464DU /* "ELFM" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ @@ -72,6 +72,12 @@ typedef struct { */ uint32_t shm_is_clone; uint32_t _pad2; + /* Guest-visible NOFILE state is separate from the host capacity reserved + * by elfuse. The child restores this after installing inherited FDs so + * descriptors above a subsequently lowered soft limit survive fork. + */ + uint64_t nofile_cur; + uint64_t nofile_max; } ipc_header_t; typedef struct { diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index d00acf70..c4727b59 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -46,6 +46,7 @@ #include "syscall/proc.h" #include "syscall/proc-pidfd.h" #include "syscall/signal.h" +#include "syscall/sys.h" #include "debug/log.h" #include "debug/syscall-hist.h" @@ -115,6 +116,12 @@ int fork_child_main(int ipc_fd, log_error("fork-child: bad magic 0x%x", hdr.magic); return 1; } + if (hdr.nofile_cur > hdr.nofile_max || hdr.nofile_max > FD_TABLE_SIZE) { + log_error("fork-child: invalid RLIMIT_NOFILE %llu/%llu", + (unsigned long long) hdr.nofile_cur, + (unsigned long long) hdr.nofile_max); + return 1; + } log_debug("fork-child: pid=%lld ppid=%lld", (long long) hdr.child_pid, (long long) hdr.parent_pid); @@ -266,6 +273,16 @@ int fork_child_main(int ipc_fd, return 1; } + /* Linux preserves already-open descriptors above a newly lowered soft + * limit across fork. Install the inherited table under the default table + * limit first, then restore the parent's guest-visible limit. + */ + if (sys_nofile_restore(hdr.nofile_cur, hdr.nofile_max) < 0) { + log_error("fork-child: failed to restore RLIMIT_NOFILE"); + guest_destroy(&g); + return 1; + } + /* Must follow fork_ipc_recv_fd_table: the keepalive recv resolves each * payload guest_fd to its (now installed) child-side host master fd. */ @@ -1646,6 +1663,9 @@ int64_t sys_clone(hv_vcpu_t vcpu, * but before sibling vCPUs resume. Declared up front so all goto paths to * fail_snapshot can free it unconditionally. Header */ + uint64_t nofile_cur, nofile_max; + sys_nofile_snapshot(&nofile_cur, &nofile_max); + ipc_header_t hdr = { .magic = IPC_MAGIC_HEADER, .ipa_bits = g->ipa_bits, @@ -1685,6 +1705,8 @@ int64_t sys_clone(hv_vcpu_t vcpu, flags & (LINUX_CLONE_CHILD_SETTID | LINUX_CLONE_CHILD_CLEARTID), .ctid_gva = ctid_gva, .shm_is_clone = (snapshot_shm_fd >= 0) ? 1 : 0, + .nofile_cur = nofile_cur, + .nofile_max = nofile_max, }; proc_registry_publish_self(); if (fork_ipc_write_all(ipc_sock, &hdr, sizeof(hdr)) < 0) { diff --git a/src/syscall/fdtable.c b/src/syscall/fdtable.c index a4c1e469..32231764 100644 --- a/src/syscall/fdtable.c +++ b/src/syscall/fdtable.c @@ -54,11 +54,6 @@ void fd_set_rlimit_nofile(int cur) rlimit_nofile_cur = cur; } -int fd_get_rlimit_nofile(void) -{ - return rlimit_nofile_cur; -} - /* Bitmap for O(1) lowest-free-FD allocation. A set bit means the FD is free * (FD_CLOSED). bit_ctz64 on each word finds the lowest free FD in O(1) per * word, vs O(FD_TABLE_SIZE) linear scan. diff --git a/src/syscall/internal.h b/src/syscall/internal.h index 35882e9c..d5943f10 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -328,9 +328,6 @@ int64_t sys_mmap_anon(guest_t *g, uint64_t addr, uint64_t length, int prot); */ void fd_set_rlimit_nofile(int cur); -/* Get the current guest RLIMIT_NOFILE soft limit. */ -int fd_get_rlimit_nofile(void); - /* Borrowed-or-owned host fd reference. * * Single-threaded guests borrow the raw host fd directly (no dup, no close). diff --git a/src/syscall/sys.c b/src/syscall/sys.c index 418f56ed..09ebe7dd 100644 --- a/src/syscall/sys.c +++ b/src/syscall/sys.c @@ -68,6 +68,10 @@ static linux_sysinfo_t cached_sysinfo; static pthread_mutex_t rlimit_lock = PTHREAD_MUTEX_INITIALIZER; static linux_rlimit64_t cached_linux_rlimits[10]; static uint8_t cached_linux_rlimit_valid[10]; +static linux_rlimit64_t guest_nofile_limit = { + .rlim_cur = FD_TABLE_SIZE, + .rlim_max = FD_TABLE_SIZE, +}; #define RLIMIT_CACHE_SIZE ((int) (ARRAY_SIZE(cached_linux_rlimits))) _Static_assert(sizeof(struct rusage) == sizeof(linux_rusage_t), @@ -659,6 +663,63 @@ static void rlimit_cache_set(int resource, linux_rlimit64_t lim) pthread_mutex_unlock(&rlimit_lock); } +void sys_nofile_snapshot(uint64_t *cur, uint64_t *max) +{ + pthread_mutex_lock(&rlimit_lock); + if (cur) + *cur = guest_nofile_limit.rlim_cur; + if (max) + *max = guest_nofile_limit.rlim_max; + pthread_mutex_unlock(&rlimit_lock); +} + +int sys_nofile_restore(uint64_t cur, uint64_t max) +{ + if (cur > max || max > FD_TABLE_SIZE) + return -1; + + pthread_mutex_lock(&rlimit_lock); + guest_nofile_limit.rlim_cur = cur; + guest_nofile_limit.rlim_max = max; + fd_set_rlimit_nofile((int) cur); + pthread_mutex_unlock(&rlimit_lock); + return 0; +} + +static int64_t sys_prlimit64_nofile(guest_t *g, + uint64_t new_gva, + uint64_t old_gva) +{ + linux_rlimit64_t new_lim = {0}; + if (new_gva != 0 && + guest_read_small(g, new_gva, &new_lim, sizeof(new_lim)) < 0) + return -LINUX_EFAULT; + + pthread_mutex_lock(&rlimit_lock); + linux_rlimit64_t old_lim = guest_nofile_limit; + + int error = 0; + if (new_gva != 0) { + if (new_lim.rlim_cur > new_lim.rlim_max) { + error = LINUX_EINVAL; + } else if (new_lim.rlim_max > guest_nofile_limit.rlim_max) { + /* The guest has no capability path for raising its hard limit. */ + error = LINUX_EPERM; + } else { + guest_nofile_limit = new_lim; + fd_set_rlimit_nofile((int) new_lim.rlim_cur); + } + } + pthread_mutex_unlock(&rlimit_lock); + + if (error) + return -error; + if (old_gva != 0 && + guest_write_small(g, old_gva, &old_lim, sizeof(old_lim)) < 0) + return -LINUX_EFAULT; + return 0; +} + int64_t sys_prlimit64(guest_t *g, int pid, int resource, @@ -670,6 +731,8 @@ int64_t sys_prlimit64(guest_t *g, int mac_res = translate_rlimit_resource(resource); if (mac_res < 0) return -LINUX_EINVAL; + if (resource == 7 /* RLIMIT_NOFILE */) + return sys_prlimit64_nofile(g, new_gva, old_gva); /* Get old limits BEFORE setting new ones (Linux prlimit64 atomically * returns old values and sets new ones; approximate by get-then-set). @@ -704,15 +767,6 @@ int64_t sys_prlimit64(guest_t *g, return linux_errno(); rlimit_cache_set(resource, new_lim); - - /* Track RLIMIT_NOFILE in the guest FD table so fd_alloc enforces the - * limit and returns -EMFILE. - */ - if (resource == 7 /* RLIMIT_NOFILE */) { - int cur = (new_lim.rlim_cur == UINT64_MAX) ? FD_TABLE_SIZE - : (int) new_lim.rlim_cur; - fd_set_rlimit_nofile(cur); - } } /* Write old limits to guest after set (so guest sees pre-set values) */ @@ -763,30 +817,29 @@ int sys_format_limits(char *buf, size_t bufsz) char soft[24], hard[24]; int res = rows[i].linux_res; - if (RANGE_CHECK(res, 0, RLIMIT_CACHE_SIZE)) { + if (res == 7 /* RLIMIT_NOFILE */) { + uint64_t cur, max; + sys_nofile_snapshot(&cur, &max); + linux_rlimit64_t lim = {.rlim_cur = cur, .rlim_max = max}; + snprintf(soft, sizeof(soft), "%llu", + (unsigned long long) lim.rlim_cur); + snprintf(hard, sizeof(hard), "%llu", + (unsigned long long) lim.rlim_max); + } else if (RANGE_CHECK(res, 0, RLIMIT_CACHE_SIZE)) { linux_rlimit64_t lim; if (!rlimit_cache_get(res, &lim)) { - /* RLIMIT_NOFILE (Linux 7): use the guest FD table limit rather - * than host getrlimit, which may return RLIM_INFINITY on macOS. - */ - if (res == 7) { - int cur = fd_get_rlimit_nofile(); - lim.rlim_cur = (uint64_t) cur; - lim.rlim_max = (uint64_t) FD_TABLE_SIZE; - } else { - int mac_res = translate_rlimit_resource(res); - if (mac_res >= 0) { - struct rlimit rl; - if (getrlimit(mac_res, &rl) == 0) { - lim = translate_host_rlimit(res, rl); - } else { - lim.rlim_cur = UINT64_MAX; - lim.rlim_max = UINT64_MAX; - } + int mac_res = translate_rlimit_resource(res); + if (mac_res >= 0) { + struct rlimit rl; + if (getrlimit(mac_res, &rl) == 0) { + lim = translate_host_rlimit(res, rl); } else { lim.rlim_cur = UINT64_MAX; lim.rlim_max = UINT64_MAX; } + } else { + lim.rlim_cur = UINT64_MAX; + lim.rlim_max = UINT64_MAX; } } if (lim.rlim_cur == UINT64_MAX) diff --git a/src/syscall/sys.h b/src/syscall/sys.h index 355399c6..6f6a8d92 100644 --- a/src/syscall/sys.h +++ b/src/syscall/sys.h @@ -50,6 +50,13 @@ int64_t sys_prlimit64(guest_t *g, uint64_t new_gva, uint64_t old_gva); +/* Snapshot/restore the guest-visible RLIMIT_NOFILE across the posix_spawn + * fork boundary. The host limit is internal capacity and is deliberately not + * exposed through these helpers. + */ +void sys_nofile_snapshot(uint64_t *cur, uint64_t *max); +int sys_nofile_restore(uint64_t cur, uint64_t max); + /* Format /proc/self/limits content into buf. * * Returns bytes written (excluding NUL) or -1 on error. diff --git a/tests/test-fd-lifecycle.c b/tests/test-fd-lifecycle.c index 2c89edac..c4f28519 100644 --- a/tests/test-fd-lifecycle.c +++ b/tests/test-fd-lifecycle.c @@ -11,8 +11,11 @@ #include #include #include +#include #include #include +#include +#include #include #include "test-harness.h" @@ -229,6 +232,69 @@ static void test_dup3_above_rlimit_fails(void) close(pipefd[1]); } +static void test_nofile_hides_host_reserve(void) +{ + TEST("RLIMIT_NOFILE hides host reserve"); + + struct rlimit limit; + struct utsname uts; + if (getrlimit(RLIMIT_NOFILE, &limit) != 0 || uname(&uts) != 0) { + FAIL("getrlimit or uname failed"); + return; + } + + if (strcmp(uts.nodename, "elfuse") == 0) { + EXPECT_TRUE(limit.rlim_cur == 1024 && limit.rlim_max == 1024, + "elfuse exposed its host FD reserve"); + } else { + EXPECT_TRUE(limit.rlim_cur > 0 && limit.rlim_cur <= limit.rlim_max, + "native RLIMIT_NOFILE is invalid"); + } +} + +static void test_nofile_lowered_before_fork(void) +{ + TEST("fork preserves FDs above lowered RLIMIT_NOFILE"); + + int pipefd[2]; + if (pipe(pipefd) != 0) { + FAIL("pipe setup failed"); + return; + } + + struct rlimit low_limit = {.rlim_cur = 3, .rlim_max = 3}; + if (setrlimit(RLIMIT_NOFILE, &low_limit) != 0) { + FAIL("setrlimit failed"); + close(pipefd[0]); + close(pipefd[1]); + return; + } + + pid_t pid = fork(); + if (pid == 0) { + char byte = 'x'; + ssize_t n = write(pipefd[1], &byte, 1); + _exit(n == 1 ? 0 : 1); + } + if (pid < 0) { + FAIL("fork failed after lowering RLIMIT_NOFILE"); + close(pipefd[0]); + close(pipefd[1]); + return; + } + + close(pipefd[1]); + char byte = 0; + ssize_t n = read(pipefd[0], &byte, 1); + close(pipefd[0]); + + int status = 0; + pid_t waited = waitpid(pid, &status, 0); + EXPECT_TRUE(n == 1 && byte == 'x' && waited == pid && WIFEXITED(status) && + WEXITSTATUS(status) == 0, + "fork child did not preserve the inherited pipe"); +} + int main(void) { printf("test-fd-lifecycle: fd metadata lifecycle tests\n\n"); @@ -239,6 +305,9 @@ int main(void) test_memfd_accepts_valid_linux_flags(); test_rlimit_nofile_reports_emfile(); test_dup3_above_rlimit_fails(); + test_nofile_hides_host_reserve(); + /* This permanently lowers the hard limit, so it must remain last. */ + test_nofile_lowered_before_fork(); SUMMARY("test-fd-lifecycle"); return fails > 0 ? 1 : 0; diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 115ce16a..7a59dc81 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -17,14 +17,17 @@ #include "runtime/fork-state.h" #define LEGACY_ELFK_MAGIC 0x454C464BU +#define PREVIOUS_ELFL_MAGIC 0x454C464CU -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464CU, - "fork IPC protocol magic must remain ELFL until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464DU, + "fork IPC protocol magic must remain ELFM until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != LEGACY_ELFK_MAGIC, "current protocol must reject old ELFK children/parents"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFL_MAGIC, + "NOFILE header fields require rejecting ELFL peers"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol");