From 570a56f74bf79aa3fa383e708d4741043c1e4f6f Mon Sep 17 00:00:00 2001 From: Yash Mandaviya Date: Thu, 23 Jul 2026 23:14:47 -0400 Subject: [PATCH 1/3] Add stream-based copyIn/copyOut for tar stream cp support Adds file-handle/stream-based copy entrypoints so callers can transfer a tar stream directly to/from a container's filesystem without staging the contents on the host, enabling true `docker cp -` / `podman cp -` parity in downstream CLIs (apple/container#1908). - LinuxContainer.copyIn(archive:to:) splices a tar stream from a file handle straight to the guest over vsock; the guest extracts it in place, honoring the ownership/mode/symlink metadata in the tar headers. - LinuxContainer.copyOut(from:to:) streams the guest-produced tar archive to a file handle; it always archives, even a single regular file, to match `docker cp CONTAINER:/path -`. - ArchiveReader.init(fileHandle:) auto-detects archive format and compression filter, so an externally supplied tar stream (uncompressed or gzip/bzip2/xz) is accepted, not only the internal pax+gzip form. - vminitd copy handlers use the auto-detecting reader for copyIn and honor CopyRequest.is_archive to force single-file archiving on copyOut. No host-side temp staging, so host permission and path-length limits no longer constrain the transfer. Existing path-based copyIn/copyOut behavior is unchanged (the new is_archive-on-copyOut path is opt-in). Co-Authored-By: Claude Opus 4.8 (1M context) --- Sources/Containerization/LinuxContainer.swift | 173 ++++++++++++++++++ .../ArchiveReader.swift | 23 +++ .../ArchiveReaderTests.swift | 60 ++++++ vminitd/Sources/VminitdCore/Server+GRPC.swift | 21 ++- 4 files changed, 274 insertions(+), 3 deletions(-) diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index e24bdf67a..68357a057 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -1439,6 +1439,179 @@ extension LinuxContainer { } } } + + /// Copy the contents of a tar stream into the container. + /// + /// The `archive` file handle is expected to yield a tar stream, either + /// uncompressed or compressed (gzip/bzip2/xz), matching `docker cp -` / + /// `podman cp -` input semantics. The bytes are streamed directly to the + /// guest, which extracts them in place honoring the ownership, mode and + /// symlink metadata recorded in the tar headers. No intermediate files are + /// created on the host, so host-side permission and path-length constraints + /// do not apply. + /// + /// - Parameters: + /// - archive: A file handle yielding the tar stream to extract. + /// - destination: The guest directory to extract the archive into. + /// - createParents: Create parent directories of `destination` if missing. + /// - chunkSize: The transfer chunk size in bytes. + public func copyIn( + archive: FileHandle, + to destination: URL, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { + let state = try $0.startedState("copyIn") + + let guestPath = URL(filePath: self.root).appending(path: destination.path) + let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue + let listener = try state.vm.listen(port) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await state.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyIn, + guestPath: guestPath, + vsockPort: port, + createParents: createParents, + isArchive: true + ) + } + } + + group.addTask { + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.copyQueue.async { + do { + defer { conn.closeFile() } + try Self.spliceFd( + from: archive.fileDescriptor, + to: conn.fileDescriptor, + chunkSize: chunkSize, + label: "copyIn" + ) + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + } + + /// Stream the contents of a container path as a tar archive to a file handle. + /// + /// Matches `docker cp CONTAINER:/path -` / `podman cp` output semantics: the + /// output is always a tar archive, even for a single regular file, and + /// preserves the ownership and mode recorded in the guest filesystem. The + /// guest performs the archiving natively and streams the result directly; + /// no intermediate files are created on the host. + /// + /// - Parameters: + /// - source: The guest path (file or directory) to archive. + /// - archive: A file handle the tar stream is written to. + /// - chunkSize: The transfer chunk size in bytes. + public func copyOut( + from source: URL, + to archive: FileHandle, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { + let state = try $0.startedState("copyOut") + + let guestPath = URL(filePath: self.root).appending(path: source.path) + let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue + let listener = try state.vm.listen(port) + + let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await state.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyOut, + guestPath: guestPath, + vsockPort: port, + isArchive: true, + onMetadata: { meta in + metadataCont.yield(meta) + metadataCont.finish() + } + ) + } + } + + group.addTask { + // Wait for the guest to report metadata (and thus that the + // source exists) before accepting the data connection. + _ = await metadataStream.first(where: { _ in true }) + + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.copyQueue.async { + do { + defer { conn.closeFile() } + try Self.spliceFd( + from: conn.fileDescriptor, + to: archive.fileDescriptor, + chunkSize: chunkSize, + label: "copyOut" + ) + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + } + + /// Copy raw bytes from one file descriptor to another until EOF. + private static func spliceFd(from srcFd: Int32, to dstFd: Int32, chunkSize: Int, label: String) throws { + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(srcFd, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError(.internalError, message: "\(label): read error: \(String(cString: strerror(errno)))") + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(dstFd, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError(.internalError, message: "\(label): vsock write error: \(String(cString: strerror(errno)))") + } + written += w + } + } + } } extension VirtualMachineInstance { diff --git a/Sources/ContainerizationArchive/ArchiveReader.swift b/Sources/ContainerizationArchive/ArchiveReader.swift index 0477d8e4a..01ac8d674 100644 --- a/Sources/ContainerizationArchive/ArchiveReader.swift +++ b/Sources/ContainerizationArchive/ArchiveReader.swift @@ -106,6 +106,29 @@ public final class ArchiveReader { .checkOk(elseThrow: { .unableToOpenArchive($0) }) } + /// Initializes an `ArchiveReader` to read from the provided file descriptor, + /// auto-detecting the archive `Format` and compression `Filter`. + /// + /// Use this when the incoming stream's format and compression are not known + /// ahead of time, e.g. a tar stream supplied on stdin that may be + /// uncompressed or compressed (gzip, bzip2, xz, ...), matching + /// `docker cp -` / `podman cp -` input semantics. zstd is not auto-detected + /// from a stream because it requires seeking; use + /// ``init(format:filter:fileHandle:)`` with ``Filter/zstd`` for that case. + public init(fileHandle: FileHandle) throws { + self.underlying = archive_read_new() + self.fileHandle = fileHandle + + try archive_read_support_filter_all(underlying) + .checkOk(elseThrow: .failedToDetectFilter) + try archive_read_support_format_all(underlying) + .checkOk(elseThrow: .failedToDetectFormat) + + let fd = fileHandle.fileDescriptor + try archive_read_open_fd(underlying, fd, 4096) + .checkOk(elseThrow: { .unableToOpenArchive($0) }) + } + /// Initialize the `ArchiveReader` to read from a specified file URL /// by trying to auto determine the archives `Format` and `Filter`. public init(file: URL) throws { diff --git a/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift b/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift index c7c0e563a..83aff4cbe 100644 --- a/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift +++ b/Tests/ContainerizationArchiveTests/ArchiveReaderTests.swift @@ -658,6 +658,66 @@ struct ArchiveReaderTests { } } + // MARK: - FileHandle Stream Auto-Detect Tests + + /// The stream-based auto-detecting reader must accept an uncompressed tar + /// stream, matching `docker cp -` / `podman cp -` input semantics. + @Test func readUncompressedTarFromFileHandle() throws { + let archiveURL = try createTestArchive( + name: "stream-plain", + entries: [ + ("dir/", .directory, nil), + ("dir/file.txt", .regular("streamed plain"), nil), + ]) + defer { try? FileManager.default.removeItem(at: archiveURL.deletingLastPathComponent()) } + + let extractDir = try createExtractionDirectory(name: "stream-plain") + defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) } + + let fh = try FileHandle(forReadingFrom: archiveURL) + defer { try? fh.close() } + + let reader = try ArchiveReader(fileHandle: fh) + let rejected = try reader.extractContents(to: extractDir) + + #expect(rejected.isEmpty) + let content = try String(contentsOf: extractDir.appendingPathComponent("dir/file.txt"), encoding: .utf8) + #expect(content == "streamed plain") + } + + /// The stream-based auto-detecting reader must also accept a gzip-compressed + /// tar stream (the format the guest emits internally for directory copies), + /// without being told the filter up front. + @Test func readGzipTarFromFileHandle() throws { + let testDirectory = createTemporaryDirectory(baseName: "ArchiveReaderTests")! + defer { try? FileManager.default.removeItem(at: testDirectory) } + let archiveURL = testDirectory.appendingPathComponent("stream-gzip.tar.gz") + + let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) + try writer.open(file: archiveURL) + let entry = WriteEntry() + entry.path = "hello.txt" + entry.fileType = .regular + entry.permissions = 0o644 + let data = "streamed gzip".data(using: .utf8)! + entry.size = numericCast(data.count) + try writer.writeEntry(entry: entry, data: data) + try writer.finishEncoding() + + let extractDir = try createExtractionDirectory(name: "stream-gzip") + defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) } + + let fh = try FileHandle(forReadingFrom: archiveURL) + defer { try? fh.close() } + + let reader = try ArchiveReader(fileHandle: fh) + let rejected = try reader.extractContents(to: extractDir) + + #expect(rejected.isEmpty) + let content = try String(contentsOf: extractDir.appendingPathComponent("hello.txt"), encoding: .utf8) + #expect(content == "streamed gzip") + } + // MARK: - Zstd Compression Tests @Test func readZstdCompressedArchive() throws { diff --git a/vminitd/Sources/VminitdCore/Server+GRPC.swift b/vminitd/Sources/VminitdCore/Server+GRPC.swift index dd07ef54f..feff5307f 100644 --- a/vminitd/Sources/VminitdCore/Server+GRPC.swift +++ b/vminitd/Sources/VminitdCore/Server+GRPC.swift @@ -535,7 +535,11 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ try FileManager.default.createDirectory(at: destURL, withIntermediateDirectories: true) let fileHandle = FileHandle(fileDescriptor: sockFd, closeOnDealloc: false) - let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fileHandle) + // Auto-detect format and compression filter so both the internal + // pax+gzip archives (directory copyIn) and externally supplied tar + // streams (uncompressed or compressed, e.g. `docker cp -`) are + // accepted. + let reader = try ArchiveReader(fileHandle: fileHandle) return try reader.extractContents(to: destURL) } @@ -562,7 +566,10 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ guard FileManager.default.fileExists(atPath: path, isDirectory: &isDirectory) else { throw RPCError(code: .notFound, message: "copy: path not found '\(path)'") } - let isArchive = isDirectory.boolValue + // `request.isArchive` forces archive output even for a single regular + // file, matching `docker cp CONTAINER:/path -` which always emits a tar + // stream. Directories are always archived. + let isArchive = isDirectory.boolValue || request.isArchive // Determine total size for single files. var totalSize: UInt64 = 0 @@ -593,7 +600,15 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ let fileURL = URL(fileURLWithPath: path) let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) try writer.open(fileDescriptor: sock.fileDescriptor) - try writer.archiveDirectory(fileURL) + if isDirectory.boolValue { + try writer.archiveDirectory(fileURL) + } else { + // Forced single-file archive: emit one entry named after the + // file's basename, relative to its parent directory. + let filePath = FilePath(path) + let base = filePath.removingLastComponent() + try writer.archive([filePath], base: base) + } try writer.finishEncoding() } else { let srcFd = open(path, O_RDONLY) From 420d8023293a98a391680309b5d09e9c9c3e4d03 Mon Sep 17 00:00:00 2001 From: Yash Mandaviya Date: Sat, 1 Aug 2026 19:15:51 -0400 Subject: [PATCH 2/3] Match docker cp entry naming and compression for tar streams The stream copyOut path reused archiveDirectory for directories, which emits a leading "./" and names entries relative to the directory itself, and always compressed with gzip. Docker and podman name entries relative to the source's parent, so the source's own basename is the top level entry, and emit an uncompressed tar. The difference is visible in a round trip: with contents-relative naming, `cp ctr:/dir - | cp - other:/dest` scatters dir's children into /dest instead of producing /dest/dir. The gzip framing also breaks consumers that read the stream with a plain tar reader rather than shelling out to tar. Use the parent-relative form for both files and directories whenever request.isArchive marks the stream path, with no compression filter. The internal directory copyOut, where isArchive is derived from the path being a directory, keeps shipping pax+gzip of the directory's contents, which is what the host side extracts into the destination directory. --- vminitd/Sources/VminitdCore/Server+GRPC.swift | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/vminitd/Sources/VminitdCore/Server+GRPC.swift b/vminitd/Sources/VminitdCore/Server+GRPC.swift index feff5307f..d740b21c0 100644 --- a/vminitd/Sources/VminitdCore/Server+GRPC.swift +++ b/vminitd/Sources/VminitdCore/Server+GRPC.swift @@ -597,17 +597,21 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ defer { try? sock.close() } if isArchive { - let fileURL = URL(fileURLWithPath: path) - let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) + // `request.isArchive` marks the `docker cp CONTAINER:/path -` + // stream path. Docker emits an uncompressed tar whose entries are + // named relative to the source's parent, so the source's own + // basename is the top level entry, for both files and directories. + // The internal directory copyOut instead ships pax+gzip of the + // directory's contents, which is what the host extracts into the + // destination directory. + let filter: Filter = request.isArchive ? Filter.none : Filter.gzip + let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: filter)) try writer.open(fileDescriptor: sock.fileDescriptor) - if isDirectory.boolValue { - try writer.archiveDirectory(fileURL) - } else { - // Forced single-file archive: emit one entry named after the - // file's basename, relative to its parent directory. + if request.isArchive { let filePath = FilePath(path) - let base = filePath.removingLastComponent() - try writer.archive([filePath], base: base) + try writer.archive([filePath], base: filePath.removingLastComponent()) + } else { + try writer.archiveDirectory(URL(fileURLWithPath: path)) } try writer.finishEncoding() } else { From 20e16657c362b796bcd038c72e0ef217079df210 Mon Sep 17 00:00:00 2001 From: Yash Mandaviya Date: Sat, 1 Aug 2026 19:24:41 -0400 Subject: [PATCH 3/3] cleanup --- vminitd/Sources/VminitdCore/Server+GRPC.swift | 7 ------- 1 file changed, 7 deletions(-) diff --git a/vminitd/Sources/VminitdCore/Server+GRPC.swift b/vminitd/Sources/VminitdCore/Server+GRPC.swift index d740b21c0..91909ee43 100644 --- a/vminitd/Sources/VminitdCore/Server+GRPC.swift +++ b/vminitd/Sources/VminitdCore/Server+GRPC.swift @@ -597,13 +597,6 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ defer { try? sock.close() } if isArchive { - // `request.isArchive` marks the `docker cp CONTAINER:/path -` - // stream path. Docker emits an uncompressed tar whose entries are - // named relative to the source's parent, so the source's own - // basename is the top level entry, for both files and directories. - // The internal directory copyOut instead ships pax+gzip of the - // directory's contents, which is what the host extracts into the - // destination directory. let filter: Filter = request.isArchive ? Filter.none : Filter.gzip let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: filter)) try writer.open(fileDescriptor: sock.fileDescriptor)