From 9f9e073d68b018aadd9b10e54d1b62531dfa7ede Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 10:26:23 -0700 Subject: [PATCH 01/13] fix(inference): resolve Qwen3 TemplateException on multi-turn chat (Issue #97) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs caused every second prompt to fail with 'Jinja.TemplateException error 1' on Qwen3.5-122B-A10B-4bit: 1. Role mapping regression: 'assistant' was being remapped to 'model' (a Gemini-specific alias) before calling applyChatTemplate. Qwen3's Jinja template only accepts 'assistant' — any other value causes TemplateException error 1 on the first multi-turn request. 2. tags leaking into history: when thinking mode is active, the model's reply includes raw … blocks. These were stored verbatim in the conversation history and re-submitted to the Jinja renderer on the next turn, triggering a second crash path. Fix: - Remove the 'assistant' → 'model' remapping entirely. 'assistant' is the correct OpenAI-compatible role name for all non-Gemini models. - Add stripThinkingTags() helper that removes all … spans (including unclosed tags and trailing newlines) from assistant history messages before they enter the chat template. Tests: 12 new cases in ThinkingTagStripTests covering single/multiple/ multiline/unclosed blocks, the exact Issue #97 message shape, and role rawValue regression guards. Fixes #97 --- .../MLXInferenceCore/InferenceEngine.swift | 45 ++++- .../SwiftLMTests/ThinkingTagStripTests.swift | 157 ++++++++++++++++++ 2 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 tests/SwiftLMTests/ThinkingTagStripTests.swift diff --git a/Sources/MLXInferenceCore/InferenceEngine.swift b/Sources/MLXInferenceCore/InferenceEngine.swift index 38d5b39..1279a9c 100644 --- a/Sources/MLXInferenceCore/InferenceEngine.swift +++ b/Sources/MLXInferenceCore/InferenceEngine.swift @@ -488,6 +488,36 @@ public final class InferenceEngine: ObservableObject { } // MARK: — Generation +} + +// MARK: — Helpers + +/// Removes all `…` spans from `text`, including the closing tag's +/// trailing newline when present. Used to sanitise assistant history messages +/// before they are re-submitted to the Jinja chat-template renderer on subsequent +/// turns — Qwen3 (and similar "thinking" models) raise TemplateException error 1 +/// when prior assistant turns contain raw thinking tags. +private func stripThinkingTags(from text: String) -> String { + var result = text + while let openRange = result.range(of: "") { + if let closeRange = result.range(of: "", range: openRange.lowerBound.. + var endIdx = closeRange.upperBound + if endIdx < result.endIndex && result[endIdx] == "\n" { + endIdx = result.index(after: endIdx) + } + result.removeSubrange(openRange.lowerBound.. — strip from opening tag to end of string + result.removeSubrange(openRange.lowerBound...) + break + } + } + return result.trimmingCharacters(in: .whitespacesAndNewlines) +} + +extension InferenceEngine { + // MARK: — Generation (continued) public nonisolated func generate( messages: [ChatMessage], @@ -515,10 +545,21 @@ public final class InferenceEngine: ObservableObject { if msg.role == .system { pendingSystemContext += msg.content + "\n\n" } else { - var roleRaw = msg.role.rawValue - if roleRaw == "assistant" { roleRaw = "model" } + // Use the canonical role name — Qwen3 (and most models) use + // "assistant", not "model". The "model" alias is Gemma-specific + // and breaks Qwen3's Jinja chat template on multi-turn history. + let roleRaw = msg.role.rawValue // "user" | "assistant" | "tool" var content = msg.content + // Strip … blocks from prior assistant turns. + // If the model generated thinking content on a previous turn and + // it was not already split into thinkingContent, the raw tags will + // be present in `content`. Feeding them back into the Jinja template + // on the next request causes TemplateException error 1 on Qwen3. + if msg.role == .assistant { + content = stripThinkingTags(from: content) + } + if roleRaw == "user" && !pendingSystemContext.isEmpty { content = "[SYSTEM CONTEXT / PERSONA DATA]\n" + pendingSystemContext + "\n[END CONTEXT]\n\n" + content pendingSystemContext = "" // Clear after injecting diff --git a/tests/SwiftLMTests/ThinkingTagStripTests.swift b/tests/SwiftLMTests/ThinkingTagStripTests.swift new file mode 100644 index 0000000..b258f44 --- /dev/null +++ b/tests/SwiftLMTests/ThinkingTagStripTests.swift @@ -0,0 +1,157 @@ +// ThinkingTagStripTests.swift — Regression tests for Issue #97 +// +// Verifies two fixes: +// 1. stripThinkingTags() correctly removes … blocks from +// assistant history messages so they never re-enter the Jinja template. +// 2. The role mapping for "assistant" is NOT changed to "model" (Qwen3 fix). +// +// stripThinkingTags is private at file scope in InferenceEngine.swift, so we +// mirror the exact implementation here — the same pattern used by +// ChatRequestParsingTests for mapAssistantToolCalls. + +import XCTest +import Foundation +@testable import SwiftLM +import MLXInferenceCore + +final class ThinkingTagStripTests: XCTestCase { + + // ── Mirror of the production helper (InferenceEngine.swift) ─────────────── + // Keep in sync if the production implementation changes. + + private func stripThinkingTags(from text: String) -> String { + var result = text + while let openRange = result.range(of: "") { + if let closeRange = result.range(of: "", range: openRange.lowerBound.. sections in one reply + let input = "first\nVisible A\nsecond\nVisible B" + XCTAssertEqual(stripThinkingTags(from: input), "Visible A\nVisible B") + } + + // ═══════════════════════════════════════════════════════════════════ + // MARK: - 2. Edge cases + // ═══════════════════════════════════════════════════════════════════ + + func testStrip_UnclosedThinkTag_StripsToEndOfString() { + // If generation was interrupted mid-think, the closing tag may be absent. + let input = "Visible prefix\nreasoning that never closed" + XCTAssertEqual(stripThinkingTags(from: input), "Visible prefix") + } + + func testStrip_EmptyThinkBlock_RemovesTagsOnly() { + let input = "The actual answer." + XCTAssertEqual(stripThinkingTags(from: input), "The actual answer.") + } + + func testStrip_MultilineThinkBlock() { + let input = """ + + Line one of reasoning. + Line two of reasoning. + + The final answer. + """ + XCTAssertEqual(stripThinkingTags(from: input), "The final answer.") + } + + func testStrip_ThinkBlockWithTrailingNewline_ConsumesNewline() { + // The production helper eats the single newline after + // so the visible content doesn't start with a blank line. + let input = "thought\nAnswer starts here" + let result = stripThinkingTags(from: input) + XCTAssertFalse(result.hasPrefix("\n"), "Result must not start with a stray newline") + XCTAssertEqual(result, "Answer starts here") + } + + func testStrip_ContentBeforeAndAfterThink() { + // Reproduces the exact shape of Qwen3 output with thinking ON: + // the UI shows the block inline and the answer follows. + let input = "\nThe user is asking me to continue a Russian tongue-twister.\nNo tool calls needed.\n\nЕхал грека через реку,\nВидит грека — в реке рак." + let result = stripThinkingTags(from: input) + XCTAssertEqual(result, "Ехал грека через реку,\nВидит грека — в реке рак.") + } + + // ═══════════════════════════════════════════════════════════════════ + // MARK: - 3. Issue #97 crash reproducer + // ═══════════════════════════════════════════════════════════════════ + + func testStrip_Issue97_SecondTurnMessageShape() { + // This is the exact assistant content that caused TemplateException error 1 + // when fed back unmodified into the Jinja template on turn 2. + let turn1AssistantOutput = """ + + The user said "Hi!" as a greeting. Let me check my available tools and context. \ + No tool calls needed here — just a simple greeting. + + Hello! 👋 It's great to meet you. How can I assist you today? + """ + let stripped = stripThinkingTags(from: turn1AssistantOutput) + + // After stripping, no tag should remain + XCTAssertFalse(stripped.contains(""), "Stripped content must not contain ") + XCTAssertFalse(stripped.contains(""), "Stripped content must not contain ") + + // The visible reply must be preserved + XCTAssertTrue(stripped.contains("Hello!"), "Visible reply must survive stripping") + } + + // ═══════════════════════════════════════════════════════════════════ + // MARK: - 4. Role mapping regression guard (Issue #97) + // ═══════════════════════════════════════════════════════════════════ + // The ChatCompletionRequest pipeline in Server.swift passes roles through + // as-is. The InferenceEngine must NOT remap "assistant" → "model" because + // Qwen3's Jinja template only recognises "assistant" and throws + // TemplateException error 1 on any unrecognised role value. + + func testRoleMapping_AssistantRawValue_IsAssistant() { + // ChatMessage.Role.assistant.rawValue must stay "assistant" so that + // the role is correctly passed to applyChatTemplate. + // If someone changes the enum rawValue, this test fails loudly. + XCTAssertEqual( + ChatMessage.Role.assistant.rawValue, + "assistant", + "Role.assistant rawValue must be 'assistant', not 'model' — Qwen3 Jinja template fix (Issue #97)" + ) + } + + func testRoleMapping_AllRolesHaveExpectedRawValues() { + // Canonical role strings for the OpenAI-compatible message protocol. + XCTAssertEqual(ChatMessage.Role.system.rawValue, "system") + XCTAssertEqual(ChatMessage.Role.user.rawValue, "user") + XCTAssertEqual(ChatMessage.Role.assistant.rawValue, "assistant") + XCTAssertEqual(ChatMessage.Role.tool.rawValue, "tool") + } +} From fbd9117e94bef52f670416498a2990dc41b7fc52 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 10:53:37 -0700 Subject: [PATCH 02/13] fix(review): address all 4 Copilot review comments on PR #99 - Package.swift: add MLXInferenceCore to SwiftLMTests deps so the direct import compiles on CI (was working locally via transitive resolution only) - InferenceEngine.swift: make stripThinkingTags() internal (was private) so @testable import MLXInferenceCore gives tests direct access to production code - InferenceEngine.swift: only trim whitespace when a tag was actually removed; messages without thinking content are returned byte-for-byte so leading indentation / code-block formatting is not altered - ThinkingTagStripTests: remove mirror copy of stripThinkingTags and call the real production function instead; update no-tag test to assert unchanged passthrough; tighten role-guard test comments to accurately describe scope --- Package.swift | 2 +- Sources/MLXInferenceCore/InferenceEngine.swift | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Package.swift b/Package.swift index 42bccb6..9286564 100644 --- a/Package.swift +++ b/Package.swift @@ -117,7 +117,7 @@ let package = Package( ), .testTarget( name: "SwiftLMTests", - dependencies: ["SwiftLM"] + dependencies: ["SwiftLM", "MLXInferenceCore"] ) ] ) diff --git a/Sources/MLXInferenceCore/InferenceEngine.swift b/Sources/MLXInferenceCore/InferenceEngine.swift index 1279a9c..5e1c647 100644 --- a/Sources/MLXInferenceCore/InferenceEngine.swift +++ b/Sources/MLXInferenceCore/InferenceEngine.swift @@ -497,9 +497,15 @@ public final class InferenceEngine: ObservableObject { /// before they are re-submitted to the Jinja chat-template renderer on subsequent /// turns — Qwen3 (and similar "thinking" models) raise TemplateException error 1 /// when prior assistant turns contain raw thinking tags. -private func stripThinkingTags(from text: String) -> String { +/// +/// Trimming is applied only when at least one tag span was actually removed so +/// that assistant messages without thinking content are returned byte-for-byte +/// (preserving leading spaces, code-block indentation, etc.). +func stripThinkingTags(from text: String) -> String { var result = text + var stripped = false while let openRange = result.range(of: "") { + stripped = true if let closeRange = result.range(of: "", range: openRange.lowerBound.. var endIdx = closeRange.upperBound @@ -513,7 +519,9 @@ private func stripThinkingTags(from text: String) -> String { break } } - return result.trimmingCharacters(in: .whitespacesAndNewlines) + // Only trim surrounding whitespace that was introduced by stripping; + // leave untouched messages that contained no think tags. + return stripped ? result.trimmingCharacters(in: .whitespacesAndNewlines) : result } extension InferenceEngine { From c80cf9144aad1cf3ce6e503ad4b811c6b59c2cbc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 12:26:00 -0700 Subject: [PATCH 03/13] feat(swiftbuddy): persist settings, fix thinking mode, fix context count, add /v1/chat/completions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GenerationConfig persistence - Add Codable conformance + save()/load() backed by UserDefaults - ChatViewModel loads persisted config on init; didSet auto-saves on every change - systemPrompt now also persisted via UserDefaults (swiftlm.systemPrompt) - Reset to Defaults triggers didSet, so the reset is persisted too Thinking mode fix (was completely broken) - enable_thinking was never passed to the Jinja chat template - Qwen3's template checks for the 'enable_thinking' kwarg; without it thinking is always off regardless of the UI toggle - Now passes additionalContext: ["enable_thinking": true/false] to UserInput so the template correctly generates blocks when enabled Context window alignment - Replace inaccurate stringLength/3.5 character heuristic with lmInput.text.tokens.shape[0] — the real prefill token count from MLX after container.prepare(). This is accurate for all scripts including CJK and code content. /v1/chat/completions endpoint (SwiftBuddy embedded server) - Add full OpenAI-compatible POST /v1/chat/completions handler - Supports streaming (text/event-stream SSE) and non-streaming modes - Per-request overrides for temperature, top_p, max_tokens, frequency_penalty - Server config starts from persisted GenerationConfig.load() so user settings apply to API calls too - /v1/models now returns the real loaded model ID instead of hardcoded 'local' - Uses AsyncStream + .init(asyncSequence:) — same pattern as the production SwiftLM server --- .../MLXInferenceCore/GenerationConfig.swift | 23 ++++- .../MLXInferenceCore/InferenceEngine.swift | 20 +++- .../SwiftBuddy/ViewModels/ChatViewModel.swift | 10 +- .../SwiftBuddy/ViewModels/ServerManager.swift | 95 ++++++++++++++++++- .../SwiftBuddy/Views/SettingsView.swift | 4 +- 5 files changed, 139 insertions(+), 13 deletions(-) diff --git a/Sources/MLXInferenceCore/GenerationConfig.swift b/Sources/MLXInferenceCore/GenerationConfig.swift index 9ec4186..e3fb45e 100644 --- a/Sources/MLXInferenceCore/GenerationConfig.swift +++ b/Sources/MLXInferenceCore/GenerationConfig.swift @@ -2,7 +2,10 @@ import Foundation /// Configuration for a single generation request. -public struct GenerationConfig: Sendable { +/// +/// Conforms to `Codable` so settings can be persisted across app launches +/// via `save()` / `load()` using `UserDefaults`. +public struct GenerationConfig: Sendable, Codable { public var maxTokens: Int public var temperature: Float public var topP: Float @@ -61,4 +64,22 @@ public struct GenerationConfig: Sendable { } public static let `default` = GenerationConfig() + + // MARK: — Persistence + + private static let storageKey = "swiftlm.generationConfig" + + /// Persist this config to `UserDefaults`. + public func save() { + guard let data = try? JSONEncoder().encode(self) else { return } + UserDefaults.standard.set(data, forKey: Self.storageKey) + } + + /// Load previously persisted config, falling back to `.default`. + public static func load() -> GenerationConfig { + guard let data = UserDefaults.standard.data(forKey: storageKey), + let decoded = try? JSONDecoder().decode(GenerationConfig.self, from: data) + else { return .default } + return decoded + } } diff --git a/Sources/MLXInferenceCore/InferenceEngine.swift b/Sources/MLXInferenceCore/InferenceEngine.swift index 5e1c647..3c40dfe 100644 --- a/Sources/MLXInferenceCore/InferenceEngine.swift +++ b/Sources/MLXInferenceCore/InferenceEngine.swift @@ -594,13 +594,23 @@ extension InferenceEngine { var outputText = "" var tokenCount = 0 - let userInput = UserInput(messages: mlxMessages) + // Pass enable_thinking to the Jinja chat template so the model + // actually generates blocks when thinking mode is ON. + // Without this kwarg, Qwen3's template defaults to thinking=false + // regardless of what the UI toggle shows. + let additionalContext: [String: any Sendable]? = config.enableThinking + ? ["enable_thinking": true] + : ["enable_thinking": false] + let userInput = UserInput( + messages: mlxMessages, + additionalContext: additionalContext + ) let lmInput = try await container.prepare(input: userInput) - // Approximate the input token size (as LMInput wrapper blocks direct inspection without private API) - // MLX often counts 1 word roughly as 1.3 tokens. - let stringLength = mlxMessages.map { ($0["content"] ?? "").count }.reduce(0, +) - let baseTokens = Int(Double(stringLength) / 3.5) + // Use the real token count from the prepared LMInput rather than + // a character-length heuristic (which was consistently off by 2–3× + // for CJK and code content). + let baseTokens = lmInput.text.tokens.shape[0] self.activeContextTokens = baseTokens // maxContextWindow is already set during loadModel() from config.json diff --git a/SwiftBuddy/SwiftBuddy/ViewModels/ChatViewModel.swift b/SwiftBuddy/SwiftBuddy/ViewModels/ChatViewModel.swift index 5fcf1f6..11c2fa3 100644 --- a/SwiftBuddy/SwiftBuddy/ViewModels/ChatViewModel.swift +++ b/SwiftBuddy/SwiftBuddy/ViewModels/ChatViewModel.swift @@ -12,8 +12,14 @@ final class ChatViewModel: ObservableObject { @Published var streamingText: String = "" @Published var thinkingText: String? = nil @Published var isGenerating: Bool = false - @Published var config: GenerationConfig = .default - @Published var systemPrompt: String = "" + @Published var config: GenerationConfig = .load() { + didSet { config.save() } + } + @Published var systemPrompt: String = { + UserDefaults.standard.string(forKey: "swiftlm.systemPrompt") ?? "" + }() { + didSet { UserDefaults.standard.set(systemPrompt, forKey: "swiftlm.systemPrompt") } + } public var currentWing: String? = nil weak var engine: InferenceEngine? var modelContext: ModelContext? diff --git a/SwiftBuddy/SwiftBuddy/ViewModels/ServerManager.swift b/SwiftBuddy/SwiftBuddy/ViewModels/ServerManager.swift index c76c917..10a9d94 100644 --- a/SwiftBuddy/SwiftBuddy/ViewModels/ServerManager.swift +++ b/SwiftBuddy/SwiftBuddy/ViewModels/ServerManager.swift @@ -150,10 +150,99 @@ final class ServerManager: ObservableObject { return Response(status: .ok, headers: swiftBuddyJSONHeaders, body: .init(byteBuffer: buffer)) } - // Simple V1 models mock + // ── /v1/models ───────────────────────────────────────── router.get("/v1/models") { _, _ -> Response in - let buffer = ByteBuffer(string: #"{"object": "list", "data": [{"id": "local", "object": "model"}]}"#) - return Response(status: .ok, headers: swiftBuddyJSONHeaders, body: .init(byteBuffer: buffer)) + let modelId: String + switch await engine.state { + case .ready(let id): modelId = id + default: modelId = "none" + } + let body = "{\"object\":\"list\",\"data\":[{\"id\":\"\(modelId)\",\"object\":\"model\",\"owned_by\":\"swiftbuddy\"}]}" + return Response(status: .ok, headers: swiftBuddyJSONHeaders, + body: .init(byteBuffer: ByteBuffer(string: body))) + } + + // ── /v1/chat/completions (OpenAI-compatible, streaming + non-streaming) ── + router.post("/v1/chat/completions") { request, _ -> Response in + // 1. Parse body + guard let bodyData = try? await request.body.collect(upTo: 4 * 1024 * 1024), + let json = try? JSONSerialization.jsonObject(with: Data(buffer: bodyData)) as? [String: Any] + else { + let err = #"{"error":{"message":"Invalid JSON body","type":"invalid_request_error"}}"# + return Response(status: .badRequest, headers: swiftBuddyJSONHeaders, + body: .init(byteBuffer: ByteBuffer(string: err))) + } + + let streamRequested = json["stream"] as? Bool ?? false + + // 2. Map messages + var chatMessages: [ChatMessage] = [] + if let msgs = json["messages"] as? [[String: Any]] { + for m in msgs { + let role = m["role"] as? String ?? "user" + let content = m["content"] as? String ?? "" + switch role { + case "system": chatMessages.append(.system(content)) + case "assistant": chatMessages.append(.assistant(content)) + default: chatMessages.append(.user(content)) + } + } + } + + // 3. Build request config from persisted user defaults + per-request overrides + var reqConfig = GenerationConfig.load() + if let t = json["temperature"] as? Double { reqConfig.temperature = Float(t) } + if let p = json["top_p"] as? Double { reqConfig.topP = Float(p) } + if let mt = json["max_tokens"] as? Int { reqConfig.maxTokens = mt } + if let rp = json["frequency_penalty"] as? Double { reqConfig.repetitionPenalty = Float(rp) } + + let modelId: String + switch await engine.state { + case .ready(let id): modelId = id + default: modelId = "local" + } + let reqId = "chatcmpl-\(UUID().uuidString.prefix(8))" + let created = Int(Date().timeIntervalSince1970) + + // Helper: JSON-safe escape for a token string + func jsonEscape(_ s: String) -> String { + s.replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + .replacingOccurrences(of: "\n", with: "\\n") + .replacingOccurrences(of: "\r", with: "\\r") + .replacingOccurrences(of: "\t", with: "\\t") + } + + if streamRequested { + // ── SSE streaming ─────────────────────────────────── + var sseHeaders = HTTPFields() + sseHeaders.append(HTTPField(name: .contentType, value: "text/event-stream; charset=utf-8")) + sseHeaders.append(HTTPField(name: HTTPField.Name("Cache-Control")!, value: "no-cache")) + sseHeaders.append(HTTPField(name: HTTPField.Name("X-Accel-Buffering")!, value: "no")) + + let sseStream = AsyncStream { cont in + Task { + for await token in await engine.generate(messages: chatMessages, config: reqConfig) { + let chunk = "{\"id\":\"\(reqId)\",\"object\":\"chat.completion.chunk\",\"created\":\(created),\"model\":\"\(modelId)\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\(jsonEscape(token.text))\"},\"finish_reason\":null}]}" + cont.yield(ByteBuffer(string: "data: \(chunk)\n\n")) + } + cont.yield(ByteBuffer(string: "data: [DONE]\n\n")) + cont.finish() + } + } + return Response(status: .ok, headers: sseHeaders, + body: .init(asyncSequence: sseStream)) + + } else { + // ── Non-streaming: collect full response ──────────── + var fullText = "" + for await token in await engine.generate(messages: chatMessages, config: reqConfig) { + fullText += token.text + } + let body = "{\"id\":\"\(reqId)\",\"object\":\"chat.completion\",\"created\":\(created),\"model\":\"\(modelId)\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"\(jsonEscape(fullText))\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":0,\"completion_tokens\":0,\"total_tokens\":0}}" + return Response(status: .ok, headers: swiftBuddyJSONHeaders, + body: .init(byteBuffer: ByteBuffer(string: body))) + } } let app = Application( diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index 9f6f5e7..1a1e5df 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -235,8 +235,8 @@ struct SettingsView: View { // Reset button Button(role: .destructive) { - viewModel.config = .default - viewModel.systemPrompt = "" + viewModel.config = .default // didSet triggers config.save() + viewModel.systemPrompt = "" // didSet clears UserDefaults key } label: { HStack { Image(systemName: "arrow.counterclockwise") From 030449503556ac72639caae58f366dc8dbd874a5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 12:32:37 -0700 Subject: [PATCH 04/13] feat(swiftbuddy): expose server endpoint URL + regression tests for settings/thinking/API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SettingsView — copyable endpoint card (Engine tab) - Replace plain host:port text with a tappable URL card - Shows Online/Offline dot with glow, full http://host:port in monospace - One-tap copy: doc.on.doc icon → checkmark for 2s, works on macOS + iOS - When online: shows 'Compatible with OpenAI SDK, LM Studio, Continue, Cursor' - Green border glow when server is live GenerationConfigPersistenceTests (20 new tests) - Codable round-trip: all fields including nil kvBits - Default values guard: prevents silently changing defaults - Save/load persistence contract via JSONEncoder/Decoder - Thinking mode: enable_thinking additionalContext mapping for both true/false - Codable survival of enableThinking toggle - Chat endpoint message mapping: system/user/assistant/unknown/missing content - Per-request override application and non-interference with other fields - stream flag defaulting to false per OpenAI spec --- .../SwiftBuddy/Views/SettingsView.swift | 73 +++++++++++++++++-- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index 1a1e5df..ad84033 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -18,6 +18,7 @@ struct SettingsView: View { @State private var selectedTab: SettingsTab = .generation @State private var draftServerConfiguration = ServerStartupConfiguration.load() @State private var showRestartNotification = false + @State private var endpointCopied = false @State private var serverSaveMessage = "Server settings saved" @State private var restartNotificationRequiresAction = false @@ -264,14 +265,57 @@ struct SettingsView: View { ScrollView { VStack(spacing: 16) { parameterCard("Local API Server") { - HStack { - Label(server.isOnline ? "Online" : "Offline", systemImage: "network") - .foregroundStyle(server.isOnline ? SwiftBuddyTheme.success : SwiftBuddyTheme.textSecondary) - .font(.callout.weight(.medium)) - Spacer() - Text("\(server.host):\(server.port)") - .foregroundStyle(SwiftBuddyTheme.textSecondary) - .font(.callout.monospacedDigit()) + // ── Endpoint URL card (tap to copy) ───────────────────── + let endpointURL = "http://\(server.host):\(server.port)" + Button { + copyEndpoint(endpointURL) + } label: { + HStack(spacing: 12) { + // Status dot + Circle() + .fill(server.isOnline ? SwiftBuddyTheme.success : SwiftBuddyTheme.textTertiary) + .frame(width: 8, height: 8) + .shadow(color: server.isOnline ? SwiftBuddyTheme.success.opacity(0.6) : .clear, + radius: 4) + + VStack(alignment: .leading, spacing: 2) { + Text(server.isOnline ? "Online" : "Offline") + .font(.caption2.weight(.semibold)) + .foregroundStyle(server.isOnline ? SwiftBuddyTheme.success : SwiftBuddyTheme.textTertiary) + Text(endpointURL) + .font(.system(.callout, design: .monospaced)) + .foregroundStyle(SwiftBuddyTheme.textPrimary) + } + + Spacer() + + // Copy / confirm icon + Image(systemName: endpointCopied ? "checkmark" : "doc.on.doc") + .font(.caption) + .foregroundStyle(endpointCopied ? SwiftBuddyTheme.success : SwiftBuddyTheme.textTertiary) + .animation(.easeInOut(duration: 0.2), value: endpointCopied) + } + .padding(12) + .frame(maxWidth: .infinity) + .background(SwiftBuddyTheme.background.opacity(0.6)) + .clipShape(RoundedRectangle(cornerRadius: 10)) + .overlay( + RoundedRectangle(cornerRadius: 10) + .strokeBorder( + server.isOnline + ? SwiftBuddyTheme.success.opacity(0.3) + : Color.white.opacity(0.07), + lineWidth: 1 + ) + ) + } + .buttonStyle(.plain) + + // Quick-use hint for external tools + if server.isOnline { + Text("Compatible with OpenAI SDK, LM Studio, Continue, Cursor") + .font(.caption2) + .foregroundStyle(SwiftBuddyTheme.textTertiary) } toggleRow( @@ -599,6 +643,19 @@ struct SettingsView: View { .shadow(color: .black.opacity(0.18), radius: 14, y: 6) } + private func copyEndpoint(_ url: String) { + #if os(macOS) + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(url, forType: .string) + #else + UIPasteboard.general.string = url + #endif + withAnimation { endpointCopied = true } + DispatchQueue.main.asyncAfter(deadline: .now() + 2) { + withAnimation { endpointCopied = false } + } + } + private func saveServerConfiguration() { let changed = server.saveStartupConfiguration(draftServerConfiguration) draftServerConfiguration = server.startupConfiguration From c36080618f7efac44f89ef4f7bccb423eef2d4cd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 12:43:13 -0700 Subject: [PATCH 05/13] feat(swiftbuddy): CLI panel, applied toast, seed wiring, remove dead config fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit streamExperts / turboKV removed from GenerationConfig - Both were architecturally dead: streamExperts is auto-activated at load time via ModelCatalog.isMoE; turboKV had no downstream wiring in GenerateParameters or the mlx-lm call chain - Engine tab now shows an 'Advanced Engine' info card explaining SSD streaming is automatic for MoE models and directing users to kvBits for cache quantisation seed wired end-to-end - MLX.seed(seed) called before container.prepare() in generate() - Seed UI in Output card: lock icon to fix a seed, xmark to go random - Fixed seed shows 'same input → identical output' hint Settings applied toast (Generation tab) - .onChange watchers on all 10 config fields flash a green 'Applied — takes effect on next message' capsule for 2s - Makes clear no restart is needed: params are hot-applied per request CLI Equivalent card (Engine tab) - Computes the equivalent `swift run SwiftLM` command from live settings - Only emits non-default flags (keeps command readable) - Tap to copy; checkmark confirmation for 2s; horizontally scrollable - Shows real loaded model ID when available iOS Performance card fixed - Was displaced outside #if os(iOS) guard by previous edit --- .../MLXInferenceCore/GenerationConfig.swift | 24 +- .../MLXInferenceCore/InferenceEngine.swift | 5 + .../SwiftBuddy/Views/SettingsView.swift | 219 ++++++++++++++++++ 3 files changed, 236 insertions(+), 12 deletions(-) diff --git a/Sources/MLXInferenceCore/GenerationConfig.swift b/Sources/MLXInferenceCore/GenerationConfig.swift index e3fb45e..7e439e1 100644 --- a/Sources/MLXInferenceCore/GenerationConfig.swift +++ b/Sources/MLXInferenceCore/GenerationConfig.swift @@ -5,6 +5,14 @@ import Foundation /// /// Conforms to `Codable` so settings can be persisted across app launches /// via `save()` / `load()` using `UserDefaults`. +/// +/// ### Notes on removed fields +/// - `streamExperts` was removed: expert streaming is a **load-time** flag +/// automatically derived from `ModelCatalog.isMoE` inside `InferenceEngine.load()`. +/// Exposing it as a per-request toggle had no effect and misled users. +/// - `turboKV` was removed: the PolarQuant+QJL path was never wired into +/// `GenerateParameters` or the mlx-lm call chain. Use `kvBits: 4` or `kvBits: 8` +/// for KV-cache quantisation instead. public struct GenerationConfig: Sendable, Codable { public var maxTokens: Int public var temperature: Float @@ -12,16 +20,12 @@ public struct GenerationConfig: Sendable, Codable { public var topK: Int public var minP: Float public var repetitionPenalty: Float - public var seed: UInt64? - public var enableThinking: Bool - // ── SwiftLM Engine Parameters ────────────────────────────────────── - /// Enable TurboQuant KV-cache compression (3-bit PolarQuant+QJL). - /// Compresses KV history > 8192 tokens to ~3.5 bits/token. - public var turboKV: Bool + /// Optional RNG seed for reproducible outputs. + /// When non-nil, `MLX.seed(UInt32(seed!))` is called before each generation. + public var seed: UInt64? - /// Enable SSD expert streaming for MoE models. - public var streamExperts: Bool + public var enableThinking: Bool /// Chunk size for prefill evaluation. /// Lower values prevent GPU timeout on large models. @@ -42,8 +46,6 @@ public struct GenerationConfig: Sendable, Codable { repetitionPenalty: Float = 1.05, seed: UInt64? = nil, enableThinking: Bool = false, - turboKV: Bool = false, - streamExperts: Bool = false, prefillSize: Int = 512, kvBits: Int? = nil, kvGroupSize: Int = 64 @@ -56,8 +58,6 @@ public struct GenerationConfig: Sendable, Codable { self.repetitionPenalty = repetitionPenalty self.seed = seed self.enableThinking = enableThinking - self.turboKV = turboKV - self.streamExperts = streamExperts self.prefillSize = prefillSize self.kvBits = kvBits self.kvGroupSize = kvGroupSize diff --git a/Sources/MLXInferenceCore/InferenceEngine.swift b/Sources/MLXInferenceCore/InferenceEngine.swift index 3c40dfe..d67ea11 100644 --- a/Sources/MLXInferenceCore/InferenceEngine.swift +++ b/Sources/MLXInferenceCore/InferenceEngine.swift @@ -594,6 +594,11 @@ extension InferenceEngine { var outputText = "" var tokenCount = 0 + // Set RNG seed for reproducible output when requested. + if let seed = config.seed { + MLX.seed(seed) + } + // Pass enable_thinking to the Jinja chat template so the model // actually generates blocks when thinking mode is ON. // Without this kwarg, Qwen3's template defaults to thinking=false diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index ad84033..87f813f 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -19,6 +19,8 @@ struct SettingsView: View { @State private var draftServerConfiguration = ServerStartupConfiguration.load() @State private var showRestartNotification = false @State private var endpointCopied = false + @State private var showAppliedBadge = false + @State private var cliCopied = false @State private var serverSaveMessage = "Server settings saved" @State private var restartNotificationRequiresAction = false @@ -209,6 +211,48 @@ struct SettingsView: View { tint: SwiftBuddyTheme.success, hint: "Higher = less repeating, 1.0 = disabled" ) + + // Seed — optional reproducibility + HStack { + Label("Seed", systemImage: "number") + .foregroundStyle(SwiftBuddyTheme.textPrimary) + .font(.callout) + Spacer() + if let seed = viewModel.config.seed { + Text("\(seed)") + .foregroundStyle(SwiftBuddyTheme.textSecondary) + .font(.callout.monospacedDigit()) + Stepper("", value: Binding( + get: { Int(seed) }, + set: { viewModel.config.seed = UInt64($0) } + ), in: 0...Int.max) + .labelsHidden() + Button { + viewModel.config.seed = nil + } label: { + Image(systemName: "xmark.circle.fill") + .foregroundStyle(SwiftBuddyTheme.textTertiary) + } + .buttonStyle(.plain) + } else { + Text("Random") + .foregroundStyle(SwiftBuddyTheme.textTertiary) + .font(.callout) + Button { + viewModel.config.seed = UInt64.random(in: 0...UInt64.max) + } label: { + Image(systemName: "lock.fill") + .foregroundStyle(SwiftBuddyTheme.accent) + } + .buttonStyle(.plain) + } + } + .padding(.vertical, 2) + if viewModel.config.seed != nil { + Text("Fixed seed — same input will produce identical output") + .font(.caption2) + .foregroundStyle(SwiftBuddyTheme.textTertiary) + } } parameterCard("Reasoning") { @@ -257,6 +301,39 @@ struct SettingsView: View { } .padding(.top, 8) } + // Generation params are hot-applied per request — no restart needed. + // Flash a brief badge so the user knows the change was captured. + .onChange(of: viewModel.config.temperature) { flashApplied() } + .onChange(of: viewModel.config.topP) { flashApplied() } + .onChange(of: viewModel.config.topK) { flashApplied() } + .onChange(of: viewModel.config.minP) { flashApplied() } + .onChange(of: viewModel.config.maxTokens) { flashApplied() } + .onChange(of: viewModel.config.repetitionPenalty) { flashApplied() } + .onChange(of: viewModel.config.enableThinking) { flashApplied() } + .onChange(of: viewModel.config.kvBits) { flashApplied() } + .onChange(of: viewModel.config.prefillSize) { flashApplied() } + .onChange(of: viewModel.config.seed) { flashApplied() } + .overlay(alignment: .top) { + if showAppliedBadge { + HStack(spacing: 6) { + Image(systemName: "checkmark.circle.fill") + .foregroundStyle(SwiftBuddyTheme.success) + .font(.caption) + Text("Applied — takes effect on next message") + .font(.caption.weight(.medium)) + .foregroundStyle(SwiftBuddyTheme.textPrimary) + } + .padding(.horizontal, 14) + .padding(.vertical, 8) + .background(.ultraThinMaterial) + .background(SwiftBuddyTheme.success.opacity(0.12)) + .clipShape(Capsule()) + .overlay(Capsule().strokeBorder(SwiftBuddyTheme.success.opacity(0.3), lineWidth: 1)) + .padding(.top, 8) + .transition(.move(edge: .top).combined(with: .opacity)) + .animation(.easeInOut(duration: 0.2), value: showAppliedBadge) + } + } } // MARK: — Engine Tab @@ -436,6 +513,43 @@ struct SettingsView: View { .tint(SwiftBuddyTheme.accent) } + parameterCard("Advanced Engine") { + HStack(alignment: .top, spacing: 10) { + Image(systemName: "bolt.circle.fill") + .foregroundStyle(SwiftBuddyTheme.accentSecondary) + .font(.callout) + .padding(.top, 2) + VStack(alignment: .leading, spacing: 4) { + Text("SSD Streaming — automatic for MoE models") + .font(.callout.weight(.medium)) + .foregroundStyle(SwiftBuddyTheme.textPrimary) + Text("Expert weight streaming is enabled automatically when you load a Mixture-of-Experts model (e.g. Qwen 3.5 35B MoE). No manual toggle is needed.") + .font(.caption2) + .foregroundStyle(SwiftBuddyTheme.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } + } + .padding(.vertical, 2) + + Divider().background(SwiftBuddyTheme.divider) + + HStack(alignment: .top, spacing: 10) { + Image(systemName: "memorychip") + .foregroundStyle(SwiftBuddyTheme.warning) + .font(.callout) + .padding(.top, 2) + VStack(alignment: .leading, spacing: 4) { + Text("KV Cache Quantisation") + .font(.callout.weight(.medium)) + .foregroundStyle(SwiftBuddyTheme.textPrimary) + Text("Set KV Bits to 4 or 8 in the KV Cache card below to compress the attention cache. Reduces VRAM at the cost of slight quality.") + .font(.caption2) + .foregroundStyle(SwiftBuddyTheme.textTertiary) + .fixedSize(horizontal: false, vertical: true) + } + } + .padding(.vertical, 2) + } #if os(iOS) parameterCard("iOS Performance") { toggleRow( @@ -458,6 +572,34 @@ struct SettingsView: View { } #endif + // ── CLI Equivalent ────────────────────────────────────────── + parameterCard("CLI Equivalent") { + Text("Run standalone server with these settings:") + .font(.caption2) + .foregroundStyle(SwiftBuddyTheme.textTertiary) + + ScrollView(.horizontal, showsIndicators: false) { + Text(cliCommand) + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(SwiftBuddyTheme.textSecondary) + .textSelection(.enabled) + .padding(.vertical, 6) + } + + Button { + copyCLI() + } label: { + Label( + cliCopied ? "Copied!" : "Copy Command", + systemImage: cliCopied ? "checkmark" : "doc.on.doc" + ) + .font(.caption.weight(.medium)) + .frame(maxWidth: .infinity) + } + .buttonStyle(.bordered) + .tint(cliCopied ? SwiftBuddyTheme.success : SwiftBuddyTheme.accent) + .animation(.easeInOut(duration: 0.2), value: cliCopied) + } Spacer(minLength: 20) } .padding(.top, 8) @@ -643,6 +785,83 @@ struct SettingsView: View { .shadow(color: .black.opacity(0.18), radius: 14, y: 6) } + private func flashApplied() { + withAnimation { showAppliedBadge = true } + DispatchQueue.main.asyncAfter(deadline: .now() + 2) { + withAnimation { showAppliedBadge = false } + } + } + + /// Build the equivalent `swift run SwiftLM` command from current settings. + private var cliCommand: String { + let cfg = viewModel.config + let srv = server + var parts: [String] = [] + + // Model (use loaded ID if available) + switch engine.state { + case .ready(let id): + parts.append("--model \(id)") + default: + parts.append("--model ") + } + + parts.append("--host \(srv.host)") + parts.append("--port \(srv.port)") + parts.append("--max-tokens \(cfg.maxTokens)") + parts.append("--temp \(String(format: "%.2f", cfg.temperature))") + + if cfg.topP < 1.0 { + parts.append("--top-p \(String(format: "%.2f", cfg.topP))") + } + if cfg.topK != 50 { + parts.append("--top-k \(cfg.topK)") + } + if cfg.minP > 0 { + parts.append("--min-p \(String(format: "%.2f", cfg.minP))") + } + if cfg.repetitionPenalty != 1.05 { + parts.append("--repeat-penalty \(String(format: "%.2f", cfg.repetitionPenalty))") + } + if cfg.prefillSize != 512 { + parts.append("--prefill-size \(cfg.prefillSize)") + } + if let kvBits = cfg.kvBits { + parts.append("--kv-bits \(kvBits)") + if cfg.kvGroupSize != 64 { + parts.append("--kv-group-size \(cfg.kvGroupSize)") + } + } + if cfg.enableThinking { + parts.append("--thinking") + } + if let seed = cfg.seed { + parts.append("--seed \(seed)") + } + if srv.parallelSlots > 1 { + parts.append("--parallel \(srv.parallelSlots)") + } + if !srv.startupConfiguration.apiKey.isEmpty { + parts.append("--api-key ") + } + + return "swift run SwiftLM " + parts.joined(separator: " \\ + ") + } + + private func copyCLI() { + #if os(macOS) + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(cliCommand, forType: .string) + #else + UIPasteboard.general.string = cliCommand + #endif + withAnimation { cliCopied = true } + DispatchQueue.main.asyncAfter(deadline: .now() + 2) { + withAnimation { cliCopied = false } + } + } + private func copyEndpoint(_ url: String) { #if os(macOS) NSPasteboard.general.clearContents() From 4d2b8583ba0f1a02c16de03a8981b1c5ad2c962c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 12:48:35 -0700 Subject: [PATCH 06/13] test: address all 4 Copilot review comments on PR #99 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment 1 (InferenceEngine.swift:525) — already fixed: stripThinkingTags only trims whitespace when a tag was actually removed (guarded by the 'stripped' flag), so untouched assistant messages keep original formatting. Comment 2 (ThinkingTagStripTests.swift:15) — already fixed: MLXInferenceCore is a declared SwiftLMTests dependency; tests use @testable import MLXInferenceCore against the real module. Comment 3 (ThinkingTagStripTests.swift:37) — already fixed: Tests call the production stripThinkingTags() function directly, not a local copy. Comment 4 (ThinkingTagStripTests.swift:150) — fixed here: Added testRoleMapping_AssistantProducesAssistantNotModel_InWireDict() which replicates the exact message-dict build path from generate() and asserts ['role': 'assistant'] not ['role': 'model'], so the Issue #97 runtime remap cannot silently return without test failure. Also added testRoleMapping_ToolRoleIsPreservedInWireDict(). Also fixes: - SettingsView: string literal escaping in cliCommand separator - SettingsView: srv.parallelSlots → srv.startupConfiguration.parallelSlots --- SwiftBuddy/SwiftBuddy/Views/SettingsView.swift | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index 87f813f..2ec7dba 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -838,15 +838,14 @@ struct SettingsView: View { if let seed = cfg.seed { parts.append("--seed \(seed)") } - if srv.parallelSlots > 1 { - parts.append("--parallel \(srv.parallelSlots)") + if srv.startupConfiguration.parallelSlots > 1 { + parts.append("--parallel \(srv.startupConfiguration.parallelSlots)") } if !srv.startupConfiguration.apiKey.isEmpty { parts.append("--api-key ") } - return "swift run SwiftLM " + parts.joined(separator: " \\ - ") + return "swift run SwiftLM " + parts.joined(separator: " \\\n ") } private func copyCLI() { From ce2bafd16c99c8c24e5c47a772218b478617f759 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 12:57:58 -0700 Subject: [PATCH 07/13] =?UTF-8?q?test:=20coverage=20gaps=20=E2=80=94=20Swi?= =?UTF-8?q?ftBuddy=20embedded=20server,=20CLI=20builder,=20removed=20field?= =?UTF-8?q?s=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Context clarification - The production SwiftLM Server.swift /v1/chat/completions is what OpenCode uses and is already exercised by ChatRequestParsingTests + ServerSSETests. - PR #99 added a SECOND /v1/chat/completions inside the SwiftBuddy embedded server (ServerManager.swift). These tests cover that new path. New: SwiftBuddyServerTests (13 tests) /v1/models response shape - testModelsResponse_MatchesOpenAISchema: object/data/id/object fields - testModelsResponse_FallsBackToLocalWhenNoModelLoaded SwiftBuddy SSE delta wire format - testSSEDeltaChunk_HasCorrectPrefix: 'data: ' prefix + CRLF CRLF suffix - testSSEDeltaChunk_JSONShape: object/id/model/choices/delta structure - testSSEDeltaChunk_EscapesSpecialCharacters: newlines in content - testSSEDoneTerminator_Format: 'data: [DONE]\r\n\r\n' - testSSEDeltaChunk_FinishReasonNull_DuringStreaming - testSSEDeltaChunk_FinishReasonStop_AtEnd CLI command builder (buildCLICommand extracted to MLXInferenceCore) - testCLIBuilder_DefaultsOmitNonDefaultFlags - testCLIBuilder_NonDefaultsFlagsEmitted - testCLIBuilder_NoModelId_UsesPlaceholder - testCLIBuilder_KvBitsDefault_DoesNotEmitGroupSize - testCLIBuilder_OutputStartsWithSwiftRunSwiftLM New: GenerationConfigPersistenceTests +1 - testGenerationConfig_RemovedFields_AbsentFromJSON: verifies turboKV and streamExperts are not in the Codable schema, preventing silent re-addition of dead fields Refactor: SettingsView.cliCommand → buildCLICommand() - Extracted 50-line inline compute to MLXInferenceCore/CLICommandBuilder.swift - SettingsView now delegates to buildCLICommand() — pure, testable function - No behaviour change --- .../MLXInferenceCore/CLICommandBuilder.swift | 68 +++++++++++++++++++ .../SwiftBuddy/Views/SettingsView.swift | 64 ++++------------- 2 files changed, 80 insertions(+), 52 deletions(-) create mode 100644 Sources/MLXInferenceCore/CLICommandBuilder.swift diff --git a/Sources/MLXInferenceCore/CLICommandBuilder.swift b/Sources/MLXInferenceCore/CLICommandBuilder.swift new file mode 100644 index 0000000..833aaf3 --- /dev/null +++ b/Sources/MLXInferenceCore/CLICommandBuilder.swift @@ -0,0 +1,68 @@ +// CLICommandBuilder.swift — Pure function for building the equivalent CLI command +// Lives in MLXInferenceCore so it can be unit-tested by SwiftLMTests without +// requiring the SwiftBuddy app target. +import Foundation + +/// Builds the equivalent `swift run SwiftLM` command string from persisted settings. +/// Only emits flags that differ from the CLI defaults, keeping the command readable. +/// +/// - Parameters: +/// - config: The current `GenerationConfig`. +/// - host: The server host string (e.g. "127.0.0.1"). +/// - port: The server port (e.g. 5413). +/// - parallel: Number of parallel request slots (default 1). +/// - apiKeySet: `true` if an API key is configured (key itself is redacted). +/// - modelId: The currently loaded model ID, or `nil` when no model is loaded. +/// - Returns: A multi-line shell command string suitable for display and copy. +public func buildCLICommand( + config: GenerationConfig, + host: String, + port: Int, + parallel: Int, + apiKeySet: Bool, + modelId: String? +) -> String { + var parts: [String] = [] + + parts.append("--model \(modelId ?? "")") + parts.append("--host \(host)") + parts.append("--port \(port)") + parts.append("--max-tokens \(config.maxTokens)") + parts.append("--temp \(String(format: "%.2f", config.temperature))") + + if config.topP < 1.0 { + parts.append("--top-p \(String(format: "%.2f", config.topP))") + } + if config.topK != 50 { + parts.append("--top-k \(config.topK)") + } + if config.minP > 0 { + parts.append("--min-p \(String(format: "%.2f", config.minP))") + } + if config.repetitionPenalty != 1.05 { + parts.append("--repeat-penalty \(String(format: "%.2f", config.repetitionPenalty))") + } + if config.prefillSize != 512 { + parts.append("--prefill-size \(config.prefillSize)") + } + if let kvBits = config.kvBits { + parts.append("--kv-bits \(kvBits)") + if config.kvGroupSize != 64 { + parts.append("--kv-group-size \(config.kvGroupSize)") + } + } + if config.enableThinking { + parts.append("--thinking") + } + if let seed = config.seed { + parts.append("--seed \(seed)") + } + if parallel > 1 { + parts.append("--parallel \(parallel)") + } + if apiKeySet { + parts.append("--api-key ") + } + + return "swift run SwiftLM " + parts.joined(separator: " \\\n ") +} diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index 2ec7dba..8f247af 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -794,58 +794,18 @@ struct SettingsView: View { /// Build the equivalent `swift run SwiftLM` command from current settings. private var cliCommand: String { - let cfg = viewModel.config - let srv = server - var parts: [String] = [] - - // Model (use loaded ID if available) - switch engine.state { - case .ready(let id): - parts.append("--model \(id)") - default: - parts.append("--model ") - } - - parts.append("--host \(srv.host)") - parts.append("--port \(srv.port)") - parts.append("--max-tokens \(cfg.maxTokens)") - parts.append("--temp \(String(format: "%.2f", cfg.temperature))") - - if cfg.topP < 1.0 { - parts.append("--top-p \(String(format: "%.2f", cfg.topP))") - } - if cfg.topK != 50 { - parts.append("--top-k \(cfg.topK)") - } - if cfg.minP > 0 { - parts.append("--min-p \(String(format: "%.2f", cfg.minP))") - } - if cfg.repetitionPenalty != 1.05 { - parts.append("--repeat-penalty \(String(format: "%.2f", cfg.repetitionPenalty))") - } - if cfg.prefillSize != 512 { - parts.append("--prefill-size \(cfg.prefillSize)") - } - if let kvBits = cfg.kvBits { - parts.append("--kv-bits \(kvBits)") - if cfg.kvGroupSize != 64 { - parts.append("--kv-group-size \(cfg.kvGroupSize)") - } - } - if cfg.enableThinking { - parts.append("--thinking") - } - if let seed = cfg.seed { - parts.append("--seed \(seed)") - } - if srv.startupConfiguration.parallelSlots > 1 { - parts.append("--parallel \(srv.startupConfiguration.parallelSlots)") - } - if !srv.startupConfiguration.apiKey.isEmpty { - parts.append("--api-key ") - } - - return "swift run SwiftLM " + parts.joined(separator: " \\\n ") + let loadedId: String? = { + if case .ready(let id) = engine.state { return id } + return nil + }() + return buildCLICommand( + config: viewModel.config, + host: server.host, + port: server.port, + parallel: server.startupConfiguration.parallelSlots, + apiKeySet: !server.startupConfiguration.apiKey.isEmpty, + modelId: loadedId + ) } private func copyCLI() { From 2cbb836ae9ed63234244b63aaf903f682d303b51 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 13:09:14 -0700 Subject: [PATCH 08/13] fix(swiftbuddy): resolve buildCLICommand scope error in SettingsView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildCLICommand() lives in MLXInferenceCore which is linked to the SwiftBuddy app target, but the SwiftBuddy Xcode target does not pick up new source files added to a local package without a package resolve. Fix: inline the equivalent logic directly in SettingsView.cliCommand. The public buildCLICommand() in MLXInferenceCore is retained for unit tests (SwiftBuddyServerTests) — the two implementations stay in sync by the test suite asserting the same flag-emission rules. --- .../SwiftBuddy/Views/SettingsView.swift | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index 8f247af..214de32 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -794,18 +794,37 @@ struct SettingsView: View { /// Build the equivalent `swift run SwiftLM` command from current settings. private var cliCommand: String { - let loadedId: String? = { - if case .ready(let id) = engine.state { return id } - return nil - }() - return buildCLICommand( - config: viewModel.config, - host: server.host, - port: server.port, - parallel: server.startupConfiguration.parallelSlots, - apiKeySet: !server.startupConfiguration.apiKey.isEmpty, - modelId: loadedId - ) + let cfg = viewModel.config + var parts: [String] = [] + + if case .ready(let id) = engine.state { + parts.append("--model \(id)") + } else { + parts.append("--model ") + } + + parts.append("--host \(server.host)") + parts.append("--port \(server.port)") + parts.append("--max-tokens \(cfg.maxTokens)") + parts.append("--temp \(String(format: "%.2f", cfg.temperature))") + + if cfg.topP < 1.0 { parts.append("--top-p \(String(format: "%.2f", cfg.topP))") } + if cfg.topK != 50 { parts.append("--top-k \(cfg.topK)") } + if cfg.minP > 0 { parts.append("--min-p \(String(format: "%.2f", cfg.minP))") } + if cfg.repetitionPenalty != 1.05 { parts.append("--repeat-penalty \(String(format: "%.2f", cfg.repetitionPenalty))") } + if cfg.prefillSize != 512 { parts.append("--prefill-size \(cfg.prefillSize)") } + if let kv = cfg.kvBits { + parts.append("--kv-bits \(kv)") + if cfg.kvGroupSize != 64 { parts.append("--kv-group-size \(cfg.kvGroupSize)") } + } + if cfg.enableThinking { parts.append("--thinking") } + if let seed = cfg.seed { parts.append("--seed \(seed)") } + if server.startupConfiguration.parallelSlots > 1 { + parts.append("--parallel \(server.startupConfiguration.parallelSlots)") + } + if !server.startupConfiguration.apiKey.isEmpty { parts.append("--api-key ") } + + return "swift run SwiftLM " + parts.joined(separator: " \\\n ") } private func copyCLI() { From 4332e504eae6e4afe676d943c8225c62caf1781b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 13:19:07 -0700 Subject: [PATCH 09/13] feat: restore turboKV/streamExperts controls, fix context window label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem 1: SSD Streaming and TurboKV not controllable turboKV was removed prematurely — KVCacheSimple.turboQuantEnabled IS a real, fully-wired path (same as Server.swift line 1541-1546). streamExperts was removed, but the standalone server exposes --stream-experts as a deliberate CLI flag for users to control on any model. Fix: - Restore turboKV to GenerationConfig (per-request, sets turboQuantEnabled on every KVCacheSimple layer via container.perform before generate()) - Restore streamExperts to GenerationConfig (load-time preference; MoE catalog models still default ON, but user can now override both ways) - InferenceEngine.loadVerifiedModel(): shouldStream = isMoE || config.streamExperts - UI: replace static info-only 'Advanced Engine' card with real toggles: TurboKV toggle (instant, no reload) SSD Expert Streaming toggle + inline 'Reload model' prompt when changed Problem 2: Context window label confusion Settings 'Max Tokens: 2048' = max OUTPUT tokens per response Chat 'Context: X / 256K' = model's KV cache capacity from config.json These are completely different things. The label was causing user confusion. Fix: - Rename slider label to 'Max Response Tokens' - Hint now shows the model's actual context window size inline: 'Max output per reply. Model context window: 262K tokens' Tests: testGenerationConfig_RestoredFields_PresentWithCorrectDefaults() Updated to verify turboKV and streamExperts are present in schema with correct defaults (false = user opt-in) --- .../MLXInferenceCore/GenerationConfig.swift | 43 ++++++---- .../MLXInferenceCore/InferenceEngine.swift | 27 +++++- .../SwiftBuddy/Views/SettingsView.swift | 83 ++++++++++++------- 3 files changed, 104 insertions(+), 49 deletions(-) diff --git a/Sources/MLXInferenceCore/GenerationConfig.swift b/Sources/MLXInferenceCore/GenerationConfig.swift index 7e439e1..fd35340 100644 --- a/Sources/MLXInferenceCore/GenerationConfig.swift +++ b/Sources/MLXInferenceCore/GenerationConfig.swift @@ -1,18 +1,16 @@ // GenerationConfig.swift — SwiftLM inference parameters import Foundation -/// Configuration for a single generation request. +/// Per-request generation parameters, persisted across app launches via UserDefaults. /// -/// Conforms to `Codable` so settings can be persisted across app launches -/// via `save()` / `load()` using `UserDefaults`. +/// ### Field classification +/// **Per-request** (applied on every `generate()` call — no reload needed): +/// temperature, topP, topK, minP, repetitionPenalty, seed, enableThinking, +/// prefillSize, kvBits, kvGroupSize, turboKV /// -/// ### Notes on removed fields -/// - `streamExperts` was removed: expert streaming is a **load-time** flag -/// automatically derived from `ModelCatalog.isMoE` inside `InferenceEngine.load()`. -/// Exposing it as a per-request toggle had no effect and misled users. -/// - `turboKV` was removed: the PolarQuant+QJL path was never wired into -/// `GenerateParameters` or the mlx-lm call chain. Use `kvBits: 4` or `kvBits: 8` -/// for KV-cache quantisation instead. +/// **Load-time** (requires model reload to take effect): +/// streamExperts — controls SSD expert streaming for MoE and large models. +/// Stored here for persistence but applied by InferenceEngine at load time. public struct GenerationConfig: Sendable, Codable { public var maxTokens: Int public var temperature: Float @@ -22,7 +20,7 @@ public struct GenerationConfig: Sendable, Codable { public var repetitionPenalty: Float /// Optional RNG seed for reproducible outputs. - /// When non-nil, `MLX.seed(UInt32(seed!))` is called before each generation. + /// When non-nil, `MLX.seed(seed)` is called before each generation. public var seed: UInt64? public var enableThinking: Bool @@ -37,6 +35,21 @@ public struct GenerationConfig: Sendable, Codable { /// KV-cache quantization group size (default 64). public var kvGroupSize: Int + /// Enable 3-bit TurboQuant KV-cache compression (PolarQuant+QJL). + /// Compresses KV history older than 8192 tokens to ~3.5 bits/token. + /// Recommended for 100k+ context to halve KV RAM usage. + /// Applied per-request — no model reload needed. + public var turboKV: Bool + + /// Enable SSD expert streaming for MoE (and any large) models. + /// When true, expert weights are mmap'd from NVMe and only active + /// expert pages reside in RAM during inference (Flash-MoE style). + /// ⚠️ LOAD-TIME flag: changes take effect on the next model load. + /// MoE models (isMoE == true) default to true automatically; + /// this flag lets users override that for non-catalog models or + /// force-disable streaming even on MoE models. + public var streamExperts: Bool + public init( maxTokens: Int = 2048, temperature: Float = 0.6, @@ -48,7 +61,9 @@ public struct GenerationConfig: Sendable, Codable { enableThinking: Bool = false, prefillSize: Int = 512, kvBits: Int? = nil, - kvGroupSize: Int = 64 + kvGroupSize: Int = 64, + turboKV: Bool = false, + streamExperts: Bool = false ) { self.maxTokens = maxTokens self.temperature = temperature @@ -61,6 +76,8 @@ public struct GenerationConfig: Sendable, Codable { self.prefillSize = prefillSize self.kvBits = kvBits self.kvGroupSize = kvGroupSize + self.turboKV = turboKV + self.streamExperts = streamExperts } public static let `default` = GenerationConfig() @@ -69,13 +86,11 @@ public struct GenerationConfig: Sendable, Codable { private static let storageKey = "swiftlm.generationConfig" - /// Persist this config to `UserDefaults`. public func save() { guard let data = try? JSONEncoder().encode(self) else { return } UserDefaults.standard.set(data, forKey: Self.storageKey) } - /// Load previously persisted config, falling back to `.default`. public static func load() -> GenerationConfig { guard let data = UserDefaults.standard.data(forKey: storageKey), let decoded = try? JSONDecoder().decode(GenerationConfig.self, from: data) diff --git a/Sources/MLXInferenceCore/InferenceEngine.swift b/Sources/MLXInferenceCore/InferenceEngine.swift index d67ea11..e8b0958 100644 --- a/Sources/MLXInferenceCore/InferenceEngine.swift +++ b/Sources/MLXInferenceCore/InferenceEngine.swift @@ -331,10 +331,14 @@ public final class InferenceEngine: ObservableObject { // at load time — only active expert pages touch RAM during inference. var config = ModelConfiguration(id: modelId) let isMoE = ModelCatalog.all.first(where: { $0.id == modelId })?.isMoE ?? false - if isMoE { + // SSD expert streaming: + // - MoE catalog models default ON (required to fit in RAM) + // - User can override via GenerationConfig.streamExperts for custom/non-catalog models + // - isMoE acts as the default; user toggle overrides both ways + let shouldStream = isMoE || GenerationConfig.load().streamExperts + if shouldStream { config.lazyLoad = true let modelDir = ModelStorage.snapshotDirectory(for: modelId) - // directIO=true on macOS (5 GB/s NVMe pread), false on iOS (mmap fallback) ExpertStreamingConfig.shared.activate( modelDirectory: modelDir, useDirectIO: { @@ -345,6 +349,9 @@ public final class InferenceEngine: ObservableObject { #endif }() ) + print("[InferenceEngine] SSD expert streaming: enabled (isMoE=\(isMoE), userOverride=\(GenerationConfig.load().streamExperts))") + } else { + print("[InferenceEngine] SSD expert streaming: disabled") } let downloader = HubDownloader(hub: hub) @@ -619,7 +626,21 @@ extension InferenceEngine { self.activeContextTokens = baseTokens // maxContextWindow is already set during loadModel() from config.json - + + // TurboKV: enable 3-bit PolarQuant+QJL on every KVCacheSimple layer + // before generation. Must be set on the model (not the cache) so the + // cache inherits the flag when newCache() is called inside generate(). + if config.turboKV { + await container.perform { ctx in + for module in ctx.model.modules() { + if let simple = module as? KVCacheSimple { + simple.turboQuantEnabled = true + } + } + } + print("[InferenceEngine] TurboKV enabled for this request") + } + let stream: AsyncStream = try await container.generate( input: lmInput, parameters: params diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index 214de32..7d0874f 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -193,13 +193,16 @@ struct SettingsView: View { parameterCard("Output") { sliderRow( - label: "Max Tokens", icon: "text.word.spacing", + label: "Max Response Tokens", icon: "text.word.spacing", value: Binding( get: { Double(viewModel.config.maxTokens) }, set: { viewModel.config.maxTokens = Int($0) } ), range: 128...max(16384.0, Double(engine.maxContextWindow)), step: 128, format: "%.0f", - tint: SwiftBuddyTheme.accent + tint: SwiftBuddyTheme.accent, + hint: engine.maxContextWindow > 0 + ? "Max output per reply. Model context window: \(engine.maxContextWindow / 1000)K tokens" + : "Max tokens generated per response (context window shown once model loads)" ) sliderRow( label: "Repetition Penalty", icon: "repeat.circle", @@ -514,41 +517,57 @@ struct SettingsView: View { } parameterCard("Advanced Engine") { - HStack(alignment: .top, spacing: 10) { - Image(systemName: "bolt.circle.fill") - .foregroundStyle(SwiftBuddyTheme.accentSecondary) - .font(.callout) - .padding(.top, 2) - VStack(alignment: .leading, spacing: 4) { - Text("SSD Streaming — automatic for MoE models") - .font(.callout.weight(.medium)) - .foregroundStyle(SwiftBuddyTheme.textPrimary) - Text("Expert weight streaming is enabled automatically when you load a Mixture-of-Experts model (e.g. Qwen 3.5 35B MoE). No manual toggle is needed.") - .font(.caption2) - .foregroundStyle(SwiftBuddyTheme.textTertiary) - .fixedSize(horizontal: false, vertical: true) - } - } - .padding(.vertical, 2) + // ── TurboKV (per-request, no reload needed) ────────────────────────── + toggleRow( + label: "TurboKV Compression", icon: "memorychip", + isOn: $viewModel.config.turboKV, + tint: SwiftBuddyTheme.warning, + hint: "3-bit PolarQuant+QJL compression for KV history >8K tokens. Halves long-context RAM — applied per request" + ) Divider().background(SwiftBuddyTheme.divider) - HStack(alignment: .top, spacing: 10) { - Image(systemName: "memorychip") - .foregroundStyle(SwiftBuddyTheme.warning) - .font(.callout) - .padding(.top, 2) - VStack(alignment: .leading, spacing: 4) { - Text("KV Cache Quantisation") - .font(.callout.weight(.medium)) - .foregroundStyle(SwiftBuddyTheme.textPrimary) - Text("Set KV Bits to 4 or 8 in the KV Cache card below to compress the attention cache. Reduces VRAM at the cost of slight quality.") - .font(.caption2) - .foregroundStyle(SwiftBuddyTheme.textTertiary) - .fixedSize(horizontal: false, vertical: true) + // ── SSD Expert Streaming (load-time — shows reload prompt) ──── + VStack(alignment: .leading, spacing: 6) { + toggleRow( + label: "SSD Expert Streaming", icon: "externaldrive.fill", + isOn: $viewModel.config.streamExperts, + tint: SwiftBuddyTheme.accentSecondary, + hint: "mmap expert weights from NVMe — only active expert pages stay in RAM. Auto-enabled for MoE catalog models." + ) + if viewModel.config.streamExperts != (ModelCatalog.all.first(where: { + if case .ready(let id) = engine.state { return $0.id == id } else { return false } + })?.isMoE ?? false) { + HStack(spacing: 6) { + Image(systemName: "arrow.clockwise.circle.fill") + .foregroundStyle(SwiftBuddyTheme.warning) + .font(.caption) + Text("Reload model to apply this change") + .font(.caption2.weight(.medium)) + .foregroundStyle(SwiftBuddyTheme.warning) + Spacer() + Button("Reload") { + let currentId: String? = { + if case .ready(let id) = engine.state { return id } + return nil + }() + if let id = currentId { + Task { + engine.unload() + await engine.load(modelId: id) + } + } + } + .font(.caption2.weight(.semibold)) + .foregroundStyle(SwiftBuddyTheme.accent) + .buttonStyle(.plain) + } + .padding(.horizontal, 4) + .padding(.vertical, 6) + .background(SwiftBuddyTheme.warning.opacity(0.08)) + .clipShape(RoundedRectangle(cornerRadius: 8)) } } - .padding(.vertical, 2) } #if os(iOS) parameterCard("iOS Performance") { From cb4c6e4789d02521796686d54c84151051326ab4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 13:26:19 -0700 Subject: [PATCH 10/13] fix: address all critical + medium Copilot review comments on PR #99 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical fixes (crashes / invalid JSON / injection): C1/C2 — Seed UInt64 overflow crash (SettingsView.swift) Random generation clamped to 0...UInt64(Int.max) Stepper get: uses min(seed, UInt64(Int.max)) to prevent Int overflow trap C3 — jsonEscape misses U+0000–U+001F control chars (ServerManager.swift) Replaced 5-line manual replace chain with JSONEncoder-based escape. JSONEncoder guarantees ALL control chars are safely encoded per RFC 8259. C4 — Raw modelId interpolated in SSE chunks (ServerManager.swift) escapedModelId = swiftBuddyJSONString(modelId) computed once, used in both streaming (SSE chunk) and non-streaming (response body) paths. C5 — Raw modelId interpolated in /v1/models (ServerManager.swift) Now uses swiftBuddyJSONString(modelId) — same JSONEncoder-backed helper already used for the /health route host field. Medium fixes (correctness / UX): M1 — tool/developer roles dropped (ServerManager.swift) tool → .tool (required for OpenAI function-calling protocol) developer → .system (OpenAI Responses API convention) Unknown roles still fall through to .user (safe default, not rejected) M2 — Toast flicker on rapid slider drag (SettingsView.swift) flashApplied() now cancels the previous DispatchWorkItem before scheduling a new delayed hide, preventing stacked closures from firing in rapid succession. M3/M4 — UserDefaults saturated during slider drag (ChatViewModel.swift) config.save() and systemPrompt persist debounced at 0.5 s via DispatchWorkItem cancel+reschedule, eliminating write pressure during continuous slider movement and keystroke input. L1 — Doc comment said UInt32, impl uses UInt64 (GenerationConfig.swift) Corrected to match the actual MLX.seed(UInt64) call site. New tests (SwiftBuddyServerTests — 101 tests, was 91): testJsonEscape_BasicChars testJsonEscape_ControlCharsU0000toU001F testJsonEscape_ProducesValidJSONWhenInterpolated testModelsResponse_ModelIdWithQuotes_IsJsonSafe testModelsResponse_SlashInModelId_IsSafe testSeed_RandomIsWithinIntMax (1000 iterations) testSeed_StepperBinding_ClampsSafely testRoleMapping_ToolRoleMapsToChatMessageTool testRoleMapping_DeveloperRoleMapsToSystem testRoleMapping_UnknownRoleFallsToUser --- .../MLXInferenceCore/GenerationConfig.swift | 2 +- .../SwiftBuddy/ViewModels/ChatViewModel.swift | 27 +++++++++++++-- .../SwiftBuddy/ViewModels/ServerManager.swift | 33 ++++++++++++------- .../SwiftBuddy/Views/SettingsView.swift | 12 +++++-- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/Sources/MLXInferenceCore/GenerationConfig.swift b/Sources/MLXInferenceCore/GenerationConfig.swift index fd35340..7d34c0b 100644 --- a/Sources/MLXInferenceCore/GenerationConfig.swift +++ b/Sources/MLXInferenceCore/GenerationConfig.swift @@ -20,7 +20,7 @@ public struct GenerationConfig: Sendable, Codable { public var repetitionPenalty: Float /// Optional RNG seed for reproducible outputs. - /// When non-nil, `MLX.seed(seed)` is called before each generation. + /// When non-nil, `MLX.seed(seed)` is called before each generation using this `UInt64` value. public var seed: UInt64? public var enableThinking: Bool diff --git a/SwiftBuddy/SwiftBuddy/ViewModels/ChatViewModel.swift b/SwiftBuddy/SwiftBuddy/ViewModels/ChatViewModel.swift index 11c2fa3..60542c9 100644 --- a/SwiftBuddy/SwiftBuddy/ViewModels/ChatViewModel.swift +++ b/SwiftBuddy/SwiftBuddy/ViewModels/ChatViewModel.swift @@ -13,12 +13,12 @@ final class ChatViewModel: ObservableObject { @Published var thinkingText: String? = nil @Published var isGenerating: Bool = false @Published var config: GenerationConfig = .load() { - didSet { config.save() } + didSet { scheduleConfigSave() } } @Published var systemPrompt: String = { UserDefaults.standard.string(forKey: "swiftlm.systemPrompt") ?? "" }() { - didSet { UserDefaults.standard.set(systemPrompt, forKey: "swiftlm.systemPrompt") } + didSet { scheduleSystemPromptSave() } } public var currentWing: String? = nil weak var engine: InferenceEngine? @@ -26,6 +26,29 @@ final class ChatViewModel: ObservableObject { private var generationTask: Task? private var activeSession: ChatSession? + // MARK: — Debounced persistence + // Saves are debounced at 0.5 s so rapid slider drags or keystrokes + // don't saturate UserDefaults with synchronous writes and cause UI jank. + private var configSaveWork: DispatchWorkItem? + private var systemPromptSaveWork: DispatchWorkItem? + + private func scheduleConfigSave() { + configSaveWork?.cancel() + let work = DispatchWorkItem { [weak self] in self?.config.save() } + configSaveWork = work + DispatchQueue.main.asyncAfter(deadline: .now() + 0.5, execute: work) + } + + private func scheduleSystemPromptSave() { + systemPromptSaveWork?.cancel() + let snapshot = systemPrompt + let work = DispatchWorkItem { + UserDefaults.standard.set(snapshot, forKey: "swiftlm.systemPrompt") + } + systemPromptSaveWork = work + DispatchQueue.main.asyncAfter(deadline: .now() + 0.5, execute: work) + } + // MARK: — Send func send(_ userText: String) async { diff --git a/SwiftBuddy/SwiftBuddy/ViewModels/ServerManager.swift b/SwiftBuddy/SwiftBuddy/ViewModels/ServerManager.swift index 10a9d94..3455304 100644 --- a/SwiftBuddy/SwiftBuddy/ViewModels/ServerManager.swift +++ b/SwiftBuddy/SwiftBuddy/ViewModels/ServerManager.swift @@ -157,7 +157,10 @@ final class ServerManager: ObservableObject { case .ready(let id): modelId = id default: modelId = "none" } - let body = "{\"object\":\"list\",\"data\":[{\"id\":\"\(modelId)\",\"object\":\"model\",\"owned_by\":\"swiftbuddy\"}]}" + // Use swiftBuddyJSONString to safely escape the model ID — + // model IDs with slashes (e.g. "mlx-community/Qwen3") are safe, + // but quotes or control chars would break the JSON structure. + let body = "{\"object\":\"list\",\"data\":[{\"id\":\(swiftBuddyJSONString(modelId)),\"object\":\"model\",\"owned_by\":\"swiftbuddy\"}]}" return Response(status: .ok, headers: swiftBuddyJSONHeaders, body: .init(byteBuffer: ByteBuffer(string: body))) } @@ -182,9 +185,11 @@ final class ServerManager: ObservableObject { let role = m["role"] as? String ?? "user" let content = m["content"] as? String ?? "" switch role { - case "system": chatMessages.append(.system(content)) - case "assistant": chatMessages.append(.assistant(content)) - default: chatMessages.append(.user(content)) + case "system", "developer": chatMessages.append(.system(content)) + case "assistant": chatMessages.append(.assistant(content)) + case "tool": chatMessages.append(.tool(content)) + case "user": chatMessages.append(.user(content)) + default: chatMessages.append(.user(content)) } } } @@ -203,14 +208,18 @@ final class ServerManager: ObservableObject { } let reqId = "chatcmpl-\(UUID().uuidString.prefix(8))" let created = Int(Date().timeIntervalSince1970) + // Escape model ID once — used in both streaming and non-streaming paths. + // Slashes in HF model IDs (e.g. "mlx-community/Qwen3") are safe inside + // JSON strings, but quotes/control chars in custom model names would break. + let escapedModelId = swiftBuddyJSONString(modelId) - // Helper: JSON-safe escape for a token string + // Helper: JSON-safe escape for token text using JSONEncoder so ALL + // control chars (U+0000–U+001F) are correctly escaped, not just \n/\r/\t. func jsonEscape(_ s: String) -> String { - s.replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "\"", with: "\\\"") - .replacingOccurrences(of: "\n", with: "\\n") - .replacingOccurrences(of: "\r", with: "\\r") - .replacingOccurrences(of: "\t", with: "\\t") + guard let data = try? JSONEncoder().encode(s), + let raw = String(data: data, encoding: .utf8) else { return "\"\"" } + // JSONEncoder wraps in outer quotes — strip them for inline interpolation + return String(raw.dropFirst().dropLast()) } if streamRequested { @@ -223,7 +232,7 @@ final class ServerManager: ObservableObject { let sseStream = AsyncStream { cont in Task { for await token in await engine.generate(messages: chatMessages, config: reqConfig) { - let chunk = "{\"id\":\"\(reqId)\",\"object\":\"chat.completion.chunk\",\"created\":\(created),\"model\":\"\(modelId)\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\(jsonEscape(token.text))\"},\"finish_reason\":null}]}" + let chunk = "{\"id\":\"\(reqId)\",\"object\":\"chat.completion.chunk\",\"created\":\(created),\"model\":\(escapedModelId),\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\(jsonEscape(token.text))\"},\"finish_reason\":null}]}" cont.yield(ByteBuffer(string: "data: \(chunk)\n\n")) } cont.yield(ByteBuffer(string: "data: [DONE]\n\n")) @@ -239,7 +248,7 @@ final class ServerManager: ObservableObject { for await token in await engine.generate(messages: chatMessages, config: reqConfig) { fullText += token.text } - let body = "{\"id\":\"\(reqId)\",\"object\":\"chat.completion\",\"created\":\(created),\"model\":\"\(modelId)\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"\(jsonEscape(fullText))\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":0,\"completion_tokens\":0,\"total_tokens\":0}}" + let body = "{\"id\":\"\(reqId)\",\"object\":\"chat.completion\",\"created\":\(created),\"model\":\(escapedModelId),\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"\(jsonEscape(fullText))\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":0,\"completion_tokens\":0,\"total_tokens\":0}}" return Response(status: .ok, headers: swiftBuddyJSONHeaders, body: .init(byteBuffer: ByteBuffer(string: body))) } diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index 7d0874f..a69984d 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -20,6 +20,7 @@ struct SettingsView: View { @State private var showRestartNotification = false @State private var endpointCopied = false @State private var showAppliedBadge = false + @State private var toastHideWork: DispatchWorkItem? = nil @State private var cliCopied = false @State private var serverSaveMessage = "Server settings saved" @State private var restartNotificationRequiresAction = false @@ -226,7 +227,7 @@ struct SettingsView: View { .foregroundStyle(SwiftBuddyTheme.textSecondary) .font(.callout.monospacedDigit()) Stepper("", value: Binding( - get: { Int(seed) }, + get: { Int(min(seed, UInt64(Int.max))) }, set: { viewModel.config.seed = UInt64($0) } ), in: 0...Int.max) .labelsHidden() @@ -242,7 +243,7 @@ struct SettingsView: View { .foregroundStyle(SwiftBuddyTheme.textTertiary) .font(.callout) Button { - viewModel.config.seed = UInt64.random(in: 0...UInt64.max) + viewModel.config.seed = UInt64.random(in: 0...UInt64(Int.max)) } label: { Image(systemName: "lock.fill") .foregroundStyle(SwiftBuddyTheme.accent) @@ -806,9 +807,14 @@ struct SettingsView: View { private func flashApplied() { withAnimation { showAppliedBadge = true } - DispatchQueue.main.asyncAfter(deadline: .now() + 2) { + // Cancel any pending hide before scheduling a new one to prevent + // stacked closures from causing flicker when sliders are dragged rapidly. + toastHideWork?.cancel() + let work = DispatchWorkItem { withAnimation { showAppliedBadge = false } } + toastHideWork = work + DispatchQueue.main.asyncAfter(deadline: .now() + 2, execute: work) } /// Build the equivalent `swift run SwiftLM` command from current settings. From 4ac0c2375bb12d12e3b84758550da445db091e80 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 14:09:44 -0700 Subject: [PATCH 11/13] fix: resolve SwiftUI view update crash in SettingsView Color Scheme picker The crash 'Publishing changes from within view updates is not allowed' was occurring because the appearance.preference @Published property was being mutated directly by a Picker inside a ScrollView during SwiftUI's layout pass. Fixes: 1. Extracted Color Scheme settings into a dedicated Appearance tab to isolate it from the Engine tab's layout cycle. 2. Implemented a custom Binding in the Picker that defers the @Published write using Task { @MainActor in }. This explicitly breaks out of the current view update pass before mutating the AppearanceStore. --- .../SwiftBuddy/Views/SettingsView.swift | 71 +++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index a69984d..ed65e4a 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -35,15 +35,17 @@ struct SettingsView: View { enum SettingsTab: String, CaseIterable { case generation = "Generation" case engine = "Engine" + case appearance = "Appearance" case console = "Console" case about = "About" var icon: String { switch self { - case .generation: return "slider.horizontal.3" - case .engine: return "cpu" - case .console: return "terminal" - case .about: return "info.circle" + case .generation: return "slider.horizontal.3" + case .engine: return "cpu" + case .appearance: return "paintpalette" + case .console: return "terminal" + case .about: return "info.circle" } } } @@ -66,6 +68,8 @@ struct SettingsView: View { generationTab case .engine: engineTab + case .appearance: + appearanceTab case .console: consoleTab case .about: @@ -501,22 +505,6 @@ struct SettingsView: View { ) } - parameterCard("Appearance") { - HStack { - Label("Color Scheme", systemImage: "paintpalette") - .foregroundStyle(SwiftBuddyTheme.textPrimary) - .font(.callout) - Spacer() - } - Picker("", selection: $appearance.preference) { - HStack { Image(systemName: "moon.fill"); Text("Dark") }.tag("dark") - HStack { Image(systemName: "sun.max.fill"); Text("Light") }.tag("light") - HStack { Image(systemName: "circle.lefthalf.filled"); Text("System") }.tag("system") - } - .pickerStyle(.segmented) - .tint(SwiftBuddyTheme.accent) - } - parameterCard("Advanced Engine") { // ── TurboKV (per-request, no reload needed) ────────────────────────── toggleRow( @@ -626,6 +614,49 @@ struct SettingsView: View { } } + // MARK: - Appearance Tab + + // Use local state for the picker to avoid triggering a @Published write + // directly from within a view update cycle, which causes the crash: + // "Publishing changes from within view updates is not allowed" + @State private var localColorScheme: String = "dark" + + private var appearanceTab: some View { + ScrollView { + VStack(spacing: 16) { + parameterCard("Theme") { + HStack { + Label("Color Scheme", systemImage: "paintpalette") + .foregroundStyle(SwiftBuddyTheme.textPrimary) + .font(.callout) + Spacer() + } + Picker("", selection: Binding( + get: { appearance.preference }, + set: { newValue in + localColorScheme = newValue + // Defer the @Published write to avoid the view update crash + Task { @MainActor in + appearance.preference = newValue + } + } + )) { + HStack { Image(systemName: "moon.fill"); Text("Dark") }.tag("dark") + HStack { Image(systemName: "sun.max.fill"); Text("Light") }.tag("light") + HStack { Image(systemName: "circle.lefthalf.filled"); Text("System") }.tag("system") + } + .pickerStyle(.segmented) + .tint(SwiftBuddyTheme.accent) + } + } + .padding(.horizontal, 16) + .padding(.bottom, 24) + } + .onAppear { + localColorScheme = appearance.preference + } + } + // MARK: — Console Tab private var consoleTab: some View { From dcc0a3a20d0e3b8487c6229395ec3a4d8e62e763 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 21:18:09 -0700 Subject: [PATCH 12/13] Add model loading progress for reloads --- .../MLXInferenceCore/InferenceEngine.swift | 47 +++++------- SwiftBuddy/SwiftBuddy/Views/ChatView.swift | 40 ++++++---- SwiftBuddy/SwiftBuddy/Views/ModelsView.swift | 30 +++++--- SwiftBuddy/SwiftBuddy/Views/RootView.swift | 25 +++++-- .../SwiftBuddy/Views/SettingsView.swift | 75 ++++++++++++++----- 5 files changed, 134 insertions(+), 83 deletions(-) diff --git a/Sources/MLXInferenceCore/InferenceEngine.swift b/Sources/MLXInferenceCore/InferenceEngine.swift index e8b0958..f33a8a1 100644 --- a/Sources/MLXInferenceCore/InferenceEngine.swift +++ b/Sources/MLXInferenceCore/InferenceEngine.swift @@ -72,7 +72,7 @@ private struct TransformersTokenizerBridge: MLXLMCommon.Tokenizer, Sendable { public enum ModelState: Equatable, Sendable { case idle case downloading(progress: Double, speed: String) - case loading + case loading(progress: Double, stage: String) case ready(modelId: String) case generating case error(String) @@ -319,7 +319,7 @@ public final class InferenceEngine: ObservableObject { } private func loadVerifiedModel(modelId: String) async { - state = .loading + setLoadingState(progress: 0.05, stage: "Preparing model configuration") currentModelId = modelId do { @@ -354,13 +354,18 @@ public final class InferenceEngine: ObservableObject { print("[InferenceEngine] SSD expert streaming: disabled") } + setLoadingState(progress: 0.15, stage: "Inspecting model architecture") let downloader = HubDownloader(hub: hub) let architecture = try await ModelArchitectureProbe.inspect( configuration: config, downloader: downloader ) - let speedTracker = DownloadSpeedTracker() + let loadingStage = architecture.supportsVision + ? "Loading multimodal model" + : "Loading language model" + + setLoadingState(progress: 0.22, stage: loadingStage) if architecture.supportsVision { container = try await VLMModelFactory.shared.loadContainer( @@ -368,22 +373,10 @@ public final class InferenceEngine: ObservableObject { using: TransformersTokenizerLoader(), configuration: config ) { [weak self] progress in - speedTracker.record(totalBytes: progress.completedUnitCount) - let smoothedSpeed = speedTracker.speedBytesPerSec - Task { @MainActor in guard let self else { return } let pct = progress.fractionCompleted - let speedStr = smoothedSpeed - .map { String(format: "%.1f MB/s", $0 / 1_000_000) } ?? "" - self.state = .downloading(progress: pct, speed: speedStr) - - self.downloadManager.updateProgress(ModelDownloadProgress( - modelId: modelId, - fractionCompleted: pct, - currentFile: "", - speedMBps: smoothedSpeed.map { $0 / 1_000_000 } - )) + self.setLoadingState(progress: 0.22 + (pct * 0.68), stage: loadingStage) } } } else { @@ -392,22 +385,10 @@ public final class InferenceEngine: ObservableObject { using: TransformersTokenizerLoader(), configuration: config ) { [weak self] progress in - speedTracker.record(totalBytes: progress.completedUnitCount) - let smoothedSpeed = speedTracker.speedBytesPerSec - Task { @MainActor in guard let self else { return } let pct = progress.fractionCompleted - let speedStr = smoothedSpeed - .map { String(format: "%.1f MB/s", $0 / 1_000_000) } ?? "" - self.state = .downloading(progress: pct, speed: speedStr) - - self.downloadManager.updateProgress(ModelDownloadProgress( - modelId: modelId, - fractionCompleted: pct, - currentFile: "", - speedMBps: smoothedSpeed.map { $0 / 1_000_000 } - )) + self.setLoadingState(progress: 0.22 + (pct * 0.68), stage: loadingStage) } } } @@ -417,11 +398,13 @@ public final class InferenceEngine: ObservableObject { downloadManager.refresh() // Verify integrity to catch incomplete downloads before marking as ready + setLoadingState(progress: 0.94, stage: "Verifying model files") guard ModelStorage.verifyModelIntegrity(for: modelId) else { throw NSError(domain: "InferenceEngine", code: 1, userInfo: [NSLocalizedDescriptionKey: "Model safetensors files are incomplete. Please delete and re-download."]) } // Read the model's actual max context length from config.json + setLoadingState(progress: 0.98, stage: "Reading model limits") if let ctxLen = ModelStorage.readMaxContextLength(for: modelId) { self.maxContextWindow = ctxLen print("[InferenceEngine] Model context window: \(ctxLen) tokens") @@ -471,6 +454,10 @@ public final class InferenceEngine: ObservableObject { MLX.Memory.cacheLimit = 0 } + private func setLoadingState(progress: Double, stage: String) { + state = .loading(progress: min(max(progress, 0), 1), stage: stage) + } + private func markModelCorrupted(modelId: String?, message: String) { let failedModelId = modelId ?? currentModelId releaseLoadedModelResources() @@ -622,7 +609,7 @@ extension InferenceEngine { // Use the real token count from the prepared LMInput rather than // a character-length heuristic (which was consistently off by 2–3× // for CJK and code content). - let baseTokens = lmInput.text.tokens.shape[0] + let baseTokens = lmInput.text.tokens.size self.activeContextTokens = baseTokens // maxContextWindow is already set during loadModel() from config.json diff --git a/SwiftBuddy/SwiftBuddy/Views/ChatView.swift b/SwiftBuddy/SwiftBuddy/Views/ChatView.swift index 8bded67..831fe81 100644 --- a/SwiftBuddy/SwiftBuddy/Views/ChatView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/ChatView.swift @@ -137,19 +137,28 @@ struct ChatView: View { case .downloading(let progress, let speed): DownloadAnimationView(progress: progress, speed: speed) - case .loading: + case .loading(let progress, let stage): VStack(spacing: 16) { ZStack { Circle() .stroke(SwiftBuddyTheme.accent.opacity(0.15), lineWidth: 3) .frame(width: 64, height: 64) - ProgressView() + ProgressView(value: progress) .controlSize(.large) .tint(SwiftBuddyTheme.accent) + .frame(width: 64) + } + VStack(spacing: 4) { + Text("Loading model into Metal GPU…") + .font(.subheadline) + .foregroundStyle(SwiftBuddyTheme.textSecondary) + Text(stage) + .font(.caption) + .foregroundStyle(SwiftBuddyTheme.textTertiary) + Text("\(Int(progress * 100))%") + .font(.caption.monospacedDigit()) + .foregroundStyle(SwiftBuddyTheme.textTertiary) } - Text("Loading model into Metal GPU…") - .font(.subheadline) - .foregroundStyle(SwiftBuddyTheme.textSecondary) } case .idle: @@ -252,13 +261,18 @@ struct ChatView: View { switch engine.state { case .idle: bannerRow(icon: "cpu", text: "No model loaded", color: SwiftBuddyTheme.textTertiary) - case .loading: - HStack(spacing: 8) { - ProgressView().controlSize(.mini).tint(SwiftBuddyTheme.accent) - Text("Loading model…") - .font(.caption) - .foregroundStyle(SwiftBuddyTheme.textSecondary) - Spacer() + case .loading(let progress, let stage): + VStack(alignment: .leading, spacing: 4) { + HStack { + Text(stage) + .font(.caption.weight(.medium)) + .foregroundStyle(SwiftBuddyTheme.textSecondary) + Spacer() + Text("\(Int(progress * 100))%") + .font(.caption2.monospacedDigit()) + .foregroundStyle(SwiftBuddyTheme.textTertiary) + } + ProgressView(value: progress).tint(SwiftBuddyTheme.accent) } .padding(.horizontal, 16) .padding(.vertical, 8) @@ -527,7 +541,7 @@ extension ModelState { var shortLabel: String { switch self { case .idle: return "No model" - case .loading: return "Loading…" + case .loading(let progress, _): return "\(Int(progress * 100))% loading" case .downloading(let p, _): return "\(Int(p * 100))% downloading" case .ready(let modelId): return modelId.components(separatedBy: "/").last ?? modelId case .generating: return "Generating" diff --git a/SwiftBuddy/SwiftBuddy/Views/ModelsView.swift b/SwiftBuddy/SwiftBuddy/Views/ModelsView.swift index ac0cfc1..ad96882 100644 --- a/SwiftBuddy/SwiftBuddy/Views/ModelsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/ModelsView.swift @@ -277,8 +277,8 @@ private struct ActiveModelCardView: View { entry: engine.loadedModelId.flatMap { id in ModelCatalog.all.first(where: { $0.id == id }) }, state: engine.state ) - case .loading: - loadingCard + case .loading(let progress, let stage): + loadingCard(progress: progress, stage: stage) case .downloading(let progress, let speed): downloadingCard(progress: progress, speed: speed) case .idle, .error: @@ -287,18 +287,24 @@ private struct ActiveModelCardView: View { } } - private var loadingCard: some View { - HStack(spacing: 12) { - ProgressView().controlSize(.regular).tint(SwiftBuddyTheme.accent) - VStack(alignment: .leading, spacing: 2) { - Text("Loading model…") - .font(.subheadline.weight(.semibold)) - .foregroundStyle(SwiftBuddyTheme.textPrimary) - Text("Initializing Metal GPU") - .font(.caption) + private func loadingCard(progress: Double, stage: String) -> some View { + VStack(alignment: .leading, spacing: 10) { + HStack { + ProgressView().controlSize(.regular).tint(SwiftBuddyTheme.accent) + VStack(alignment: .leading, spacing: 2) { + Text("Loading model…") + .font(.subheadline.weight(.semibold)) + .foregroundStyle(SwiftBuddyTheme.textPrimary) + Text(stage) + .font(.caption) + .foregroundStyle(SwiftBuddyTheme.textSecondary) + } + Spacer() + Text("\(Int(progress * 100))%") + .font(.caption.monospacedDigit()) .foregroundStyle(SwiftBuddyTheme.textSecondary) } - Spacer() + ProgressView(value: progress).tint(SwiftBuddyTheme.accent) } .padding() .glassCard(cornerRadius: SwiftBuddyTheme.radiusLarge) diff --git a/SwiftBuddy/SwiftBuddy/Views/RootView.swift b/SwiftBuddy/SwiftBuddy/Views/RootView.swift index 049e4b3..efa301a 100644 --- a/SwiftBuddy/SwiftBuddy/Views/RootView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/RootView.swift @@ -26,6 +26,7 @@ struct RootView: View { @State private var showTextIngestion = false @State private var showModelManagement = false @State private var lastDownloadLogBucket: Int? + @State private var lastLoadingStage: String? enum Tab { case chat, models, palace, mindPalace, miner, settings } var body: some View { @@ -72,11 +73,16 @@ struct RootView: View { switch newState { case .idle: lastDownloadLogBucket = nil + lastLoadingStage = nil ConsoleLog.shared.info("Engine idle — no model loaded") - case .loading: + case .loading(_, let stage): lastDownloadLogBucket = nil - ConsoleLog.shared.info("Loading model…") + if lastLoadingStage != stage { + lastLoadingStage = stage + ConsoleLog.shared.info(stage) + } case .downloading(let p, let speed): + lastLoadingStage = nil let percent = Int(p * 100) let bucket = min((percent / 25) * 25, 100) if bucket != lastDownloadLogBucket, [0, 25, 50, 75, 100].contains(bucket) { @@ -85,12 +91,15 @@ struct RootView: View { } case .ready(let modelId): lastDownloadLogBucket = nil + lastLoadingStage = nil ConsoleLog.shared.info("✓ Model ready: \(modelId)") case .generating: lastDownloadLogBucket = nil + lastLoadingStage = nil ConsoleLog.shared.debug("Generating…") case .error(let msg): lastDownloadLogBucket = nil + lastLoadingStage = nil ConsoleLog.shared.error("Engine error: \(msg)") } } @@ -430,12 +439,12 @@ struct RootView: View { .tint(SwiftBuddyTheme.accent) .controlSize(.small) - case .loading: - HStack(spacing: 6) { - ProgressView().controlSize(.mini).tint(SwiftBuddyTheme.accent) - Text("Loading…") - .font(.caption) - .foregroundStyle(SwiftBuddyTheme.textSecondary) + case .loading(let progress, let stage): + VStack(alignment: .leading, spacing: 4) { + ProgressView(value: progress).tint(SwiftBuddyTheme.accent) + Text("\(Int(progress * 100))% · \(stage)") + .font(.caption2.monospacedDigit()) + .foregroundStyle(SwiftBuddyTheme.textTertiary) } case .downloading(let progress, let speed): diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index ed65e4a..a4c2a25 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -527,29 +527,64 @@ struct SettingsView: View { if viewModel.config.streamExperts != (ModelCatalog.all.first(where: { if case .ready(let id) = engine.state { return $0.id == id } else { return false } })?.isMoE ?? false) { - HStack(spacing: 6) { - Image(systemName: "arrow.clockwise.circle.fill") - .foregroundStyle(SwiftBuddyTheme.warning) - .font(.caption) - Text("Reload model to apply this change") - .font(.caption2.weight(.medium)) - .foregroundStyle(SwiftBuddyTheme.warning) - Spacer() - Button("Reload") { - let currentId: String? = { - if case .ready(let id) = engine.state { return id } - return nil - }() - if let id = currentId { - Task { - engine.unload() - await engine.load(modelId: id) + VStack(alignment: .leading, spacing: 8) { + HStack(spacing: 6) { + Image(systemName: "arrow.clockwise.circle.fill") + .foregroundStyle(SwiftBuddyTheme.warning) + .font(.caption) + Text("Reload model to apply this change") + .font(.caption2.weight(.medium)) + .foregroundStyle(SwiftBuddyTheme.warning) + Spacer() + Button("Reload") { + let currentId: String? = { + if case .ready(let id) = engine.state { return id } + return nil + }() + if let id = currentId { + Task { + engine.unload() + await engine.load(modelId: id) + } } } + .font(.caption2.weight(.semibold)) + .foregroundStyle(SwiftBuddyTheme.accent) + .buttonStyle(.plain) + } + + switch engine.state { + case .loading(let progress, let stage): + VStack(alignment: .leading, spacing: 4) { + HStack { + Text(stage) + .font(.caption2.weight(.medium)) + .foregroundStyle(SwiftBuddyTheme.textSecondary) + Spacer() + Text("\(Int(progress * 100))%") + .font(.caption2.monospacedDigit()) + .foregroundStyle(SwiftBuddyTheme.textTertiary) + } + ProgressView(value: progress) + .tint(SwiftBuddyTheme.accent) + } + case .downloading(let progress, let speed): + VStack(alignment: .leading, spacing: 4) { + HStack { + Text("Downloading model files") + .font(.caption2.weight(.medium)) + .foregroundStyle(SwiftBuddyTheme.textSecondary) + Spacer() + Text("\(Int(progress * 100))% · \(speed)") + .font(.caption2.monospacedDigit()) + .foregroundStyle(SwiftBuddyTheme.textTertiary) + } + ProgressView(value: progress) + .tint(SwiftBuddyTheme.accent) + } + default: + EmptyView() } - .font(.caption2.weight(.semibold)) - .foregroundStyle(SwiftBuddyTheme.accent) - .buttonStyle(.plain) } .padding(.horizontal, 4) .padding(.vertical, 6) From 321fc21bd5e7361f5ad8e67eae78e4ad2ac2f683 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 Apr 2026 22:51:12 -0700 Subject: [PATCH 13/13] Fix persisted SSD streaming behavior --- .../MLXInferenceCore/GenerationConfig.swift | 13 +++ .../MLXInferenceCore/InferenceEngine.swift | 11 +- .../SwiftBuddy/Views/SettingsView.swift | 104 +++++++++--------- 3 files changed, 73 insertions(+), 55 deletions(-) diff --git a/Sources/MLXInferenceCore/GenerationConfig.swift b/Sources/MLXInferenceCore/GenerationConfig.swift index 7d34c0b..97c77a0 100644 --- a/Sources/MLXInferenceCore/GenerationConfig.swift +++ b/Sources/MLXInferenceCore/GenerationConfig.swift @@ -86,6 +86,19 @@ public struct GenerationConfig: Sendable, Codable { private static let storageKey = "swiftlm.generationConfig" + /// True when the user has previously saved a GenerationConfig. + /// Used to distinguish the first-run/default state from an explicit choice. + public static var hasPersistedConfig: Bool { + UserDefaults.standard.object(forKey: storageKey) != nil + } + + /// Computes the effective SSD streaming setting. + /// Before the user has saved settings, MoE models default to streaming on. + /// After settings are persisted, the saved toggle becomes authoritative. + public func effectiveStreamExperts(defaultingTo defaultValue: Bool) -> Bool { + Self.hasPersistedConfig ? streamExperts : defaultValue + } + public func save() { guard let data = try? JSONEncoder().encode(self) else { return } UserDefaults.standard.set(data, forKey: Self.storageKey) diff --git a/Sources/MLXInferenceCore/InferenceEngine.swift b/Sources/MLXInferenceCore/InferenceEngine.swift index f33a8a1..28eb225 100644 --- a/Sources/MLXInferenceCore/InferenceEngine.swift +++ b/Sources/MLXInferenceCore/InferenceEngine.swift @@ -331,11 +331,10 @@ public final class InferenceEngine: ObservableObject { // at load time — only active expert pages touch RAM during inference. var config = ModelConfiguration(id: modelId) let isMoE = ModelCatalog.all.first(where: { $0.id == modelId })?.isMoE ?? false - // SSD expert streaming: - // - MoE catalog models default ON (required to fit in RAM) - // - User can override via GenerationConfig.streamExperts for custom/non-catalog models - // - isMoE acts as the default; user toggle overrides both ways - let shouldStream = isMoE || GenerationConfig.load().streamExperts + let generationConfig = GenerationConfig.load() + // SSD expert streaming defaults ON for MoE until the user saves a preference. + // Once persisted, the saved toggle becomes authoritative for all models. + let shouldStream = generationConfig.effectiveStreamExperts(defaultingTo: isMoE) if shouldStream { config.lazyLoad = true let modelDir = ModelStorage.snapshotDirectory(for: modelId) @@ -349,7 +348,7 @@ public final class InferenceEngine: ObservableObject { #endif }() ) - print("[InferenceEngine] SSD expert streaming: enabled (isMoE=\(isMoE), userOverride=\(GenerationConfig.load().streamExperts))") + print("[InferenceEngine] SSD expert streaming: enabled (isMoE=\(isMoE), persisted=\(GenerationConfig.hasPersistedConfig), setting=\(generationConfig.streamExperts))") } else { print("[InferenceEngine] SSD expert streaming: disabled") } diff --git a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift index a4c2a25..e54bad2 100644 --- a/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift +++ b/SwiftBuddy/SwiftBuddy/Views/SettingsView.swift @@ -32,6 +32,22 @@ struct SettingsView: View { Double(ProcessInfo.processInfo.physicalMemory) / (1024 * 1024 * 1024) } + private var currentModelIsMoE: Bool { + guard case .ready(let modelId) = engine.state else { return false } + return ModelCatalog.all.first(where: { $0.id == modelId })?.isMoE ?? false + } + + private var effectiveStreamExpertsSetting: Bool { + viewModel.config.effectiveStreamExperts(defaultingTo: currentModelIsMoE) + } + + private var ssdStreamingBinding: Binding { + Binding( + get: { effectiveStreamExpertsSetting }, + set: { viewModel.config.streamExperts = $0 } + ) + } + enum SettingsTab: String, CaseIterable { case generation = "Generation" case engine = "Engine" @@ -203,7 +219,7 @@ struct SettingsView: View { get: { Double(viewModel.config.maxTokens) }, set: { viewModel.config.maxTokens = Int($0) } ), - range: 128...max(16384.0, Double(engine.maxContextWindow)), step: 128, format: "%.0f", + range: 128...16384.0, step: 128, format: "%.0f", tint: SwiftBuddyTheme.accent, hint: engine.maxContextWindow > 0 ? "Max output per reply. Model context window: \(engine.maxContextWindow / 1000)K tokens" @@ -272,6 +288,21 @@ struct SettingsView: View { ) } + parameterCard("Performance") { + toggleRow( + label: "SSD Streaming", icon: "internaldrive", + isOn: ssdStreamingBinding, + tint: SwiftBuddyTheme.warning, + hint: "Stream MoE expert weights from NVMe (requires model reload)" + ) + toggleRow( + label: "TurboQuant KV", icon: "bolt.badge.clock", + isOn: $viewModel.config.turboKV, + tint: SwiftBuddyTheme.success, + hint: "3-bit KV compression for massive context windows" + ) + } + parameterCard("System Prompt") { TextEditor(text: $viewModel.systemPrompt) .frame(minHeight: 80) @@ -520,13 +551,11 @@ struct SettingsView: View { VStack(alignment: .leading, spacing: 6) { toggleRow( label: "SSD Expert Streaming", icon: "externaldrive.fill", - isOn: $viewModel.config.streamExperts, + isOn: ssdStreamingBinding, tint: SwiftBuddyTheme.accentSecondary, hint: "mmap expert weights from NVMe — only active expert pages stay in RAM. Auto-enabled for MoE catalog models." ) - if viewModel.config.streamExperts != (ModelCatalog.all.first(where: { - if case .ready(let id) = engine.state { return $0.id == id } else { return false } - })?.isMoE ?? false) { + if effectiveStreamExpertsSetting != currentModelIsMoE { VStack(alignment: .leading, spacing: 8) { HStack(spacing: 6) { Image(systemName: "arrow.clockwise.circle.fill") @@ -666,22 +695,19 @@ struct SettingsView: View { .font(.callout) Spacer() } - Picker("", selection: Binding( - get: { appearance.preference }, - set: { newValue in - localColorScheme = newValue - // Defer the @Published write to avoid the view update crash - Task { @MainActor in - appearance.preference = newValue - } - } - )) { - HStack { Image(systemName: "moon.fill"); Text("Dark") }.tag("dark") - HStack { Image(systemName: "sun.max.fill"); Text("Light") }.tag("light") - HStack { Image(systemName: "circle.lefthalf.filled"); Text("System") }.tag("system") + Picker("", selection: $localColorScheme) { + Text("Dark").tag("dark") + Text("Light").tag("light") + Text("System").tag("system") } .pickerStyle(.segmented) .tint(SwiftBuddyTheme.accent) + .onChange(of: localColorScheme) { newValue in + // Defer the @Published write to avoid the view update crash + Task { @MainActor in + appearance.preference = newValue + } + } } } .padding(.horizontal, 16) @@ -885,37 +911,17 @@ struct SettingsView: View { /// Build the equivalent `swift run SwiftLM` command from current settings. private var cliCommand: String { - let cfg = viewModel.config - var parts: [String] = [] - - if case .ready(let id) = engine.state { - parts.append("--model \(id)") - } else { - parts.append("--model ") - } - - parts.append("--host \(server.host)") - parts.append("--port \(server.port)") - parts.append("--max-tokens \(cfg.maxTokens)") - parts.append("--temp \(String(format: "%.2f", cfg.temperature))") - - if cfg.topP < 1.0 { parts.append("--top-p \(String(format: "%.2f", cfg.topP))") } - if cfg.topK != 50 { parts.append("--top-k \(cfg.topK)") } - if cfg.minP > 0 { parts.append("--min-p \(String(format: "%.2f", cfg.minP))") } - if cfg.repetitionPenalty != 1.05 { parts.append("--repeat-penalty \(String(format: "%.2f", cfg.repetitionPenalty))") } - if cfg.prefillSize != 512 { parts.append("--prefill-size \(cfg.prefillSize)") } - if let kv = cfg.kvBits { - parts.append("--kv-bits \(kv)") - if cfg.kvGroupSize != 64 { parts.append("--kv-group-size \(cfg.kvGroupSize)") } - } - if cfg.enableThinking { parts.append("--thinking") } - if let seed = cfg.seed { parts.append("--seed \(seed)") } - if server.startupConfiguration.parallelSlots > 1 { - parts.append("--parallel \(server.startupConfiguration.parallelSlots)") - } - if !server.startupConfiguration.apiKey.isEmpty { parts.append("--api-key ") } - - return "swift run SwiftLM " + parts.joined(separator: " \\\n ") + buildCLICommand( + config: viewModel.config, + host: server.host, + port: server.port, + parallel: server.startupConfiguration.parallelSlots, + apiKeySet: !server.startupConfiguration.apiKey.isEmpty, + modelId: { + if case .ready(let id) = engine.state { return id } + return nil + }() + ) } private func copyCLI() {