fix: harden media server under load

apps/media-server/Dockerfile

@@ -1,4 +1,4 @@
-FROM oven/bun:1
+FROM oven/bun:1.3.14
 
 RUN apt-get update \
 	&& apt-get install -y --no-install-recommends ffmpeg \
@@ -15,6 +15,6 @@ ENV PORT=3456
 EXPOSE 3456
 
 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
-	CMD ["bun", "-e", "const response = await fetch(`http://localhost:${process.env.PORT || 3456}/health`); process.exit(response.ok ? 0 : 1)"]
+	CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"]
 
 CMD ["bun", "run", "src/index.ts"]

apps/media-server/Dockerfile.standalone

@@ -1,4 +1,4 @@
-FROM oven/bun:1
+FROM oven/bun:1.3.14
 
 RUN apt-get update \
 	&& apt-get install -y --no-install-recommends ffmpeg \
@@ -15,6 +15,6 @@ ENV PORT=3456
 EXPOSE 3456
 
 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
-	CMD ["bun", "-e", "const response = await fetch(`http://localhost:${process.env.PORT || 3456}/health`); process.exit(response.ok ? 0 : 1)"]
+	CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"]
 
 CMD ["bun", "run", "src/index.ts"]

apps/media-server/src/__tests__/lib/container-cpu.test.ts

@@ -0,0 +1,52 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+	getContainerCpuLimit,
+	getContainerCpuUsageMicros,
+} from "../../lib/container-cpu";
+
+const tempDirs: string[] = [];
+
+afterEach(async () => {
+	await Promise.all(
+		tempDirs
+			.splice(0)
+			.map((path) => rm(path, { recursive: true, force: true })),
+	);
+});
+
+describe("container CPU metrics", () => {
+	test("reads a cgroup v2 CPU quota", async () => {
+		const dir = await mkdtemp(join(tmpdir(), "cap-container-cpu-"));
+		tempDirs.push(dir);
+		const cpuMaxPath = join(dir, "cpu.max");
+		await writeFile(cpuMaxPath, "200000 100000");
+
+		expect(
+			getContainerCpuLimit({
+				cpuMaxPath,
+				cpuQuotaPath: join(dir, "missing-quota"),
+				cpuPeriodPath: join(dir, "missing-period"),
+			}),
+		).toBe(2);
+	});
+
+	test("reads cgroup v2 aggregate CPU usage", async () => {
+		const dir = await mkdtemp(join(tmpdir(), "cap-container-cpu-"));
+		tempDirs.push(dir);
+		const cpuStatPath = join(dir, "cpu.stat");
+		await writeFile(
+			cpuStatPath,
+			"usage_usec 1234567\nuser_usec 1000000\nsystem_usec 234567\n",
+		);
+
+		expect(
+			getContainerCpuUsageMicros({
+				cpuStatPath,
+				cpuUsagePath: join(dir, "missing-usage"),
+			}),
+		).toBe(1234567);
+	});
+});

apps/media-server/src/__tests__/lib/container-memory.test.ts

@@ -0,0 +1,53 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { getContainerMemoryMetrics } from "../../lib/container-memory";
+
+const tempDirs: string[] = [];
+
+afterEach(async () => {
+	await Promise.all(
+		tempDirs
+			.splice(0)
+			.map((path) => rm(path, { recursive: true, force: true })),
+	);
+});
+
+describe("container memory metrics", () => {
+	test("reads cgroup usage and limit values", async () => {
+		const dir = await mkdtemp(join(tmpdir(), "cap-container-memory-"));
+		tempDirs.push(dir);
+		const limitPath = join(dir, "memory.max");
+		const usagePath = join(dir, "memory.current");
+		await writeFile(limitPath, String(1024 * 1024 * 1000));
+		await writeFile(usagePath, String(1024 * 1024 * 750));
+
+		const metrics = getContainerMemoryMetrics({
+			limitPaths: [limitPath],
+			usagePaths: [usagePath],
+			configuredLimitMB: 0,
+		});
+
+		expect(metrics.limitMB).toBe(1000);
+		expect(metrics.usageMB).toBe(750);
+		expect(metrics.pressure).toBe(0.75);
+	});
+
+	test("prefers an explicit limit while retaining cgroup usage", async () => {
+		const dir = await mkdtemp(join(tmpdir(), "cap-container-memory-"));
+		tempDirs.push(dir);
+		const usagePath = join(dir, "memory.current");
+		await writeFile(usagePath, String(1024 * 1024 * 900));
+
+		const metrics = getContainerMemoryMetrics({
+			limitPaths: [],
+			usagePaths: [usagePath],
+			configuredLimitMB: 1200,
+		});
+
+		expect(metrics.limitMB).toBe(1200);
+		expect(metrics.usageMB).toBe(900);
+		expect(metrics.pressure).toBe(0.75);
+	});
+});

apps/media-server/src/__tests__/lib/media-audio.integration.test.ts

@@ -1,4 +1,6 @@
 import { describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
 	checkHasAudioTrack,
@@ -11,6 +13,41 @@ const FIXTURES_DIR = join(import.meta.dir, "..", "fixtures");
 const TEST_VIDEO_WITH_AUDIO = `file://${join(FIXTURES_DIR, "test-with-audio.mp4")}`;
 const TEST_VIDEO_NO_AUDIO = `file://${join(FIXTURES_DIR, "test-no-audio.mp4")}`;
 
+async function createHlsFixture(): Promise<string> {
+	const dirPath = await mkdtemp(join(tmpdir(), "cap-audio-hls-"));
+	const proc = Bun.spawn({
+		cmd: [
+			"ffmpeg",
+			"-hide_banner",
+			"-loglevel",
+			"error",
+			"-y",
+			"-i",
+			join(FIXTURES_DIR, "test-with-audio.mp4"),
+			"-c",
+			"copy",
+			"-hls_time",
+			"0.25",
+			"-hls_list_size",
+			"0",
+			"-hls_segment_filename",
+			join(dirPath, "segment-%03d.ts"),
+			join(dirPath, "manifest.m3u8"),
+		],
+		stdout: "ignore",
+		stderr: "pipe",
+	});
+	const [stderr, exitCode] = await Promise.all([
+		new Response(proc.stderr).text(),
+		proc.exited,
+	]);
+	if (exitCode !== 0) {
+		await rm(dirPath, { recursive: true, force: true });
+		throw new Error(`Failed to create HLS fixture: ${stderr}`);
+	}
+	return dirPath;
+}
+
 async function readStream(stream: ReadableStream<Uint8Array>) {
 	const reader = stream.getReader();
 	const chunks: Uint8Array[] = [];
@@ -66,6 +103,53 @@ describe("mediaAudio integration tests", () => {
 			const hasAudio = await checkHasAudioTrack(TEST_VIDEO_NO_AUDIO);
 			expect(hasAudio).toBe(false);
 		});
+
+		test("inherits signed queries for relative HLS segments", async () => {
+			const dirPath = await createHlsFixture();
+			const requests: string[] = [];
+			const server = Bun.serve({
+				port: 0,
+				fetch(request) {
+					const url = new URL(request.url);
+					requests.push(`${url.pathname}${url.search}`);
+					if (
+						url.pathname !== "/manifest.m3u8" &&
+						url.searchParams.get("token") !== "signed"
+					) {
+						return new Response("Forbidden", { status: 403 });
+					}
+					return new Response(Bun.file(join(dirPath, url.pathname.slice(1))));
+				},
+			});
+
+			try {
+				const hasAudio = await checkHasAudioTrack(
+					`${server.url}manifest.m3u8?token=signed`,
+				);
+				expect(hasAudio).toBe(true);
+				expect(requests).toContain("/segment-000.ts?token=signed");
+			} finally {
+				await server.stop(true);
+				await rm(dirPath, { recursive: true, force: true });
+			}
+		});
+
+		test("contains upstream failures without leaking the active operation", async () => {
+			const beforeCount = getActiveProcessCount();
+			const server = Bun.serve({
+				port: 0,
+				fetch: () => new Response("Unavailable", { status: 503 }),
+			});
+
+			try {
+				await expect(
+					checkHasAudioTrack(`${server.url}video.mp4`),
+				).rejects.toThrow("FFprobe exited");
+				await waitForAudioOperations(beforeCount);
+			} finally {
+				await server.stop(true);
+			}
+		});
 	});
 
 	describe("extractAudio", () => {

apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts

@@ -33,6 +33,9 @@ let baseUrl = "";
 let tempDir = "";
 
 const uploadedArtifacts = new Map<string, Uint8Array>();
+let transientFixtureFailures = 0;
+let permanentFixtureFailures = 0;
+let slowFixtureCancellations = 0;
 
 function fileUrl(path: string) {
 	return pathToFileURL(path).toString();
@@ -138,12 +141,38 @@ beforeAll(async () => {
 			const url = new URL(request.url);
 
 			if (request.method === "GET" || request.method === "HEAD") {
+				if (url.pathname === "/fixtures/permanent-unavailable.m4s") {
+					permanentFixtureFailures++;
+					return new Response("Unavailable", { status: 503 });
+				}
+				if (url.pathname === "/fixtures/slow-segment.m4s") {
+					return new Response(
+						new ReadableStream({
+							start(controller) {
+								controller.enqueue(new Uint8Array(1024));
+							},
+							cancel() {
+								slowFixtureCancellations++;
+							},
+						}),
+					);
+				}
+				if (
+					request.method === "GET" &&
+					url.pathname === "/fixtures/transient-no-audio.mp4" &&
+					transientFixtureFailures < 2
+				) {
+					transientFixtureFailures++;
+					return new Response("Unavailable", { status: 503 });
+				}
 				const fixturePath =
 					url.pathname === "/fixtures/test-no-audio.mp4"
 						? TEST_VIDEO_NO_AUDIO
-						: url.pathname === "/fixtures/test-with-audio.mp4"
-							? TEST_VIDEO_WITH_AUDIO
-							: null;
+						: url.pathname === "/fixtures/transient-no-audio.mp4"
+							? TEST_VIDEO_NO_AUDIO
+							: url.pathname === "/fixtures/test-with-audio.mp4"
+								? TEST_VIDEO_WITH_AUDIO
+								: null;
 
 				if (fixturePath) {
 					const fixture = Bun.file(fixturePath);
@@ -169,7 +198,7 @@ beforeAll(async () => {
 				};
 				return request.method === "HEAD"
 					? new Response(null, { headers })
-					: new Response(bytes, { headers });
+					: new Response(Uint8Array.from(bytes).buffer, { headers });
 			}
 
 			if (request.method === "PUT" && url.pathname.startsWith("/uploads/")) {
@@ -196,6 +225,9 @@ beforeAll(async () => {
 beforeEach(() => {
 	mock.restore();
 	uploadedArtifacts.clear();
+	transientFixtureFailures = 0;
+	permanentFixtureFailures = 0;
+	slowFixtureCancellations = 0;
 });
 
 afterAll(() => {
@@ -320,6 +352,65 @@ describe("media routes real-world integration tests", () => {
 		}
 	}, 90000);
 
+	test("retries transient segment downloads and completes a real mux job", async () => {
+		const response = await app.fetch(
+			mediaPostRequest("/video/mux-segments", {
+				videoId: "real-mux-video",
+				userId: "real-mux-user",
+				outputPresignedUrl: uploadUrl("mux-output.mp4"),
+				videoInitUrl: `${baseUrl}/fixtures/transient-no-audio.mp4`,
+				videoSegmentUrls: [],
+			}),
+		);
+
+		expect(response.status).toBe(200);
+		const data = (await response.json()) as { jobId: string };
+		const job = await waitForTerminalJob(data.jobId);
+		try {
+			expect(job.phase).toBe("complete");
+			expect(job.error).toBeUndefined();
+			expect(transientFixtureFailures).toBe(2);
+
+			const bytes = uploadedBytes("/uploads/mux-output.mp4");
+			expectMp4(bytes);
+			const metadata = await probeBytesAsMp4(bytes, "mux-output.mp4");
+			expect(metadata.videoCodec).toBe("h264");
+			expect(metadata.audioCodec).toBeNull();
+		} finally {
+			deleteJob(data.jobId);
+		}
+	}, 90000);
+
+	test("fails a mux job when a segment stays unavailable after retries", async () => {
+		const response = await app.fetch(
+			mediaPostRequest("/video/mux-segments", {
+				videoId: "failed-segment-mux-video",
+				userId: "failed-segment-mux-user",
+				outputPresignedUrl: uploadUrl("failed-segment-output.mp4"),
+				videoInitUrl: fixtureUrl("test-no-audio.mp4"),
+				videoSegmentUrls: [
+					`${baseUrl}/fixtures/permanent-unavailable.m4s`,
+					`${baseUrl}/fixtures/slow-segment.m4s`,
+				],
+			}),
+		);
+
+		expect(response.status).toBe(200);
+		const data = (await response.json()) as { jobId: string };
+		const job = await waitForTerminalJob(data.jobId);
+		try {
+			expect(job.phase).toBe("error");
+			expect(job.error).toContain("503");
+			expect(permanentFixtureFailures).toBe(3);
+			expect(slowFixtureCancellations).toBe(1);
+			expect(uploadedArtifacts.has("/uploads/failed-segment-output.mp4")).toBe(
+				false,
+			);
+		} finally {
+			deleteJob(data.jobId);
+		}
+	}, 90000);
+
 	test("edits and uploads a real video job through the async route", async () => {
 		const response = await app.fetch(
 			mediaPostRequest("/video/edit", {