feat: add OpenAI-compatible local endpoint backend

Cotabby/App/Coordinators/SettingsCoordinator.swift

@@ -15,6 +15,7 @@ final class SettingsCoordinator: NSObject, NSWindowDelegate {
     private let permissionManager: PermissionManager
     private let permissionGuidanceController: PermissionGuidanceController
     private let suggestionSettings: SuggestionSettingsModel
+    private let openAICompatibleConnectionModel: OpenAICompatibleConnectionModel
     private let foundationModelAvailabilityService: FoundationModelAvailabilityService
     private let runtimeModel: RuntimeBootstrapModel
     private let modelDownloadManager: ModelDownloadManager
@@ -39,6 +40,7 @@ final class SettingsCoordinator: NSObject, NSWindowDelegate {
         permissionManager: PermissionManager,
         permissionGuidanceController: PermissionGuidanceController,
         suggestionSettings: SuggestionSettingsModel,
+        openAICompatibleConnectionModel: OpenAICompatibleConnectionModel,
         foundationModelAvailabilityService: FoundationModelAvailabilityService,
         runtimeModel: RuntimeBootstrapModel,
         modelDownloadManager: ModelDownloadManager,
@@ -53,6 +55,7 @@ final class SettingsCoordinator: NSObject, NSWindowDelegate {
         self.permissionManager = permissionManager
         self.permissionGuidanceController = permissionGuidanceController
         self.suggestionSettings = suggestionSettings
+        self.openAICompatibleConnectionModel = openAICompatibleConnectionModel
         self.foundationModelAvailabilityService = foundationModelAvailabilityService
         self.runtimeModel = runtimeModel
         self.modelDownloadManager = modelDownloadManager
@@ -81,6 +84,7 @@ final class SettingsCoordinator: NSObject, NSWindowDelegate {
                     permissionManager: permissionManager,
                     permissionGuidanceController: permissionGuidanceController,
                     suggestionSettings: suggestionSettings,
+                    openAICompatibleConnectionModel: openAICompatibleConnectionModel,
                     foundationModelAvailabilityService: foundationModelAvailabilityService,
                     runtimeModel: runtimeModel,
                     modelDownloadManager: modelDownloadManager,

Cotabby/App/Coordinators/SuggestionCoordinator+Prediction.swift

@@ -27,8 +27,10 @@ extension SuggestionCoordinator {
         // fast, calmer when it is slow (fewer doomed generations to cancel). The configured value
         // is the fallback until a first latency exists.
         let debounceMilliseconds = DebouncePolicy.milliseconds(
-            lastGenerationLatencyMilliseconds: latestLatencyMilliseconds,
-            fallback: settingsSnapshot.debounceMilliseconds
+            lastGenerationLatencyMilliseconds:
+                lastLatencyByEngine[settingsSnapshot.selectedEngine],
+            fallback: settingsSnapshot.debounceMilliseconds,
+            engine: settingsSnapshot.selectedEngine
         )
         // The debounce clock starts at the keystroke, not here. The host-publish poll has already
         // consumed real wall time waiting for the host to publish the keystroke to AX, and that
@@ -641,6 +643,12 @@ extension SuggestionCoordinator {
             return
         }
 
+        // Record every completed result, including output later suppressed by normalization or seam
+        // checks. Those requests still consumed backend time and are exactly the evidence the next
+        // debounce needs to avoid another burst of doomed HTTP work.
+        lastLatencyByEngine[settingsSnapshot.selectedEngine] =
+            Int((result.latency * 1_000).rounded())
+
         // The free-running focus poll keeps capturing while the engine generates, so a fresh
         // capture often already exists here; only pay a synchronous AX walk when it does not.
         // Any keystroke during generation bumped the work id (checked above), and non-keyboard

Cotabby/App/Coordinators/SuggestionCoordinator.swift

@@ -77,6 +77,11 @@ final class SuggestionCoordinator: ObservableObject {
     // Async work and active-session storage now live in dedicated collaborators below.
     var cancellables = Set<AnyCancellable>()
     var settingsSnapshot: SuggestionSettingsSnapshot
+    /// Last completed round trip per backend, retained across keystrokes. `latestLatencyMilliseconds`
+    /// belongs to the currently displayed suggestion and is cleared as soon as the user types, so it
+    /// cannot drive adaptive scheduling. Keeping this separate lets a slow HTTP backend retain an
+    /// appropriate debounce without leaking that latency into the in-process engines.
+    var lastLatencyByEngine: [SuggestionEngineKind: Int] = [:]
     // Synchronous input/focus callbacks cannot directly `await`, so resets are represented as a
     // barrier task that the next generation must cross before it can ask the runtime for output.
     var cacheResetSequence: UInt64 = 0

Cotabby/App/Core/AppDelegate.swift

@@ -264,8 +264,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
         switch suggestionSettings.selectedEngine {
         case .llamaOpenSource:
             runtimeModel.startIfNeeded()
-        case .appleIntelligence:
-            break
+        case .appleIntelligence, .openAICompatible:
+            // Switching away must release Metal buffers and the mapped GGUF. Otherwise an Ollama
+            // user still pays the duplicate memory cost the external endpoint is meant to avoid.
+            runtimeModel.stop()
         }
     }
 

Cotabby/App/Core/CotabbyAppEnvironment.swift

@@ -21,6 +21,7 @@ final class CotabbyAppEnvironment {
     let appUpdateManager: AppUpdateManager
     let permissionGuidanceController: PermissionGuidanceController
     let suggestionSettings: SuggestionSettingsModel
+    let openAICompatibleConnectionModel: OpenAICompatibleConnectionModel
     let foundationModelAvailabilityService: FoundationModelAvailabilityService
     let powerSourceMonitor: PowerSourceMonitor
     /// Detects when a composing input method (Japanese kana, Chinese pinyin, Korean hangul, ...) is
@@ -53,7 +54,15 @@ final class CotabbyAppEnvironment {
         let runtimeManager = LlamaRuntimeManager()
         let runtimeModel = RuntimeBootstrapModel(runtimeManager: runtimeManager)
         let modelDownloadManager = ModelDownloadManager()
-        let suggestionSettings = SuggestionSettingsModel(configuration: configuration)
+        let endpointCredentialStore = KeychainOpenAICompatibleCredentialStore()
+        let suggestionSettings = SuggestionSettingsModel(
+            configuration: configuration,
+            endpointCredentialStore: endpointCredentialStore
+        )
+        let openAICompatibleClient = OpenAICompatibleAPIClient()
+        let openAICompatibleConnectionModel = OpenAICompatibleConnectionModel(
+            client: openAICompatibleClient
+        )
         let foundationModelAvailabilityService = FoundationModelAvailabilityService()
         let powerSourceMonitor = PowerSourceMonitor()
         let keyboardInputSourceMonitor = KeyboardInputSourceMonitor()
@@ -179,6 +188,21 @@ final class CotabbyAppEnvironment {
             qualityMetricsStore: qualityMetricsStore,
             llamaModelNameProvider: { [weak runtimeManager] in
                 runtimeManager?.currentModelFilename
+            },
+            openAICompatibleEngine: OpenAICompatibleSuggestionEngine(
+                client: openAICompatibleClient,
+                configurationProvider: { [weak suggestionSettings] in
+                    guard let suggestionSettings else {
+                        throw OpenAICompatibleClientError.invalidResponse
+                    }
+                    return try suggestionSettings.openAICompatibleConfiguration
+                },
+                apiKeyProvider: { [weak suggestionSettings] in
+                    try suggestionSettings?.openAICompatibleAPIKey()
+                }
+            ),
+            endpointModelNameProvider: { [weak suggestionSettings] in
+                suggestionSettings?.openAICompatibleModelName.nonEmpty
             }
         )
 
@@ -191,6 +215,7 @@ final class CotabbyAppEnvironment {
             permissionManager: permissionManager,
             permissionGuidanceController: permissionGuidanceController,
             suggestionSettings: suggestionSettings,
+            openAICompatibleConnectionModel: openAICompatibleConnectionModel,
             foundationModelAvailabilityService: foundationModelAvailabilityService,
             runtimeModel: runtimeModel,
             modelDownloadManager: modelDownloadManager,
@@ -287,6 +312,7 @@ final class CotabbyAppEnvironment {
         self.appUpdateManager = appUpdateManager
         self.permissionGuidanceController = permissionGuidanceController
         self.suggestionSettings = suggestionSettings
+        self.openAICompatibleConnectionModel = openAICompatibleConnectionModel
         self.foundationModelAvailabilityService = foundationModelAvailabilityService
         self.powerSourceMonitor = powerSourceMonitor
         self.keyboardInputSourceMonitor = keyboardInputSourceMonitor
@@ -328,6 +354,7 @@ final class CotabbyAppEnvironment {
             .store(in: &cancellables)
 
         observePowerSourceProfileSwitching()
+        observeOpenAICompatibleSelection()
     }
 
     /// Applies the user's per-power-source profile (engine + model) whenever anything that could
@@ -342,8 +369,10 @@ final class CotabbyAppEnvironment {
             suggestionSettings.$isPowerBasedModelSwitchingEnabled.map { _ in () }.eraseToAnyPublisher(),
             suggestionSettings.$batteryEngine.map { _ in () }.eraseToAnyPublisher(),
             suggestionSettings.$batteryModelFilename.map { _ in () }.eraseToAnyPublisher(),
+            suggestionSettings.$batteryEndpointModelName.map { _ in () }.eraseToAnyPublisher(),
             suggestionSettings.$pluggedInEngine.map { _ in () }.eraseToAnyPublisher(),
             suggestionSettings.$pluggedInModelFilename.map { _ in () }.eraseToAnyPublisher(),
+            suggestionSettings.$pluggedInEndpointModelName.map { _ in () }.eraseToAnyPublisher(),
             runtimeModel.$availableModels.map { _ in () }.eraseToAnyPublisher()
         ]
 
@@ -400,6 +429,53 @@ final class CotabbyAppEnvironment {
             Task {
                 await runtimeModel.selectModel(filename)
             }
+        case .openAICompatible(let modelName):
+            guard !modelName.isEmpty else { return }
+            suggestionSettings.setOpenAICompatibleModelName(modelName)
+            suggestionSettings.selectEngine(.openAICompatible)
         }
     }
+
+    /// Connect once when the endpoint engine becomes active. Configuration edits only invalidate
+    /// discovery so status never describes a different URL, mode, credential, or active model;
+    /// Settings still owns explicit Connect/Return refreshes and avoids network traffic per keystroke.
+    private func observeOpenAICompatibleSelection() {
+        suggestionSettings.$selectedEngine
+            .removeDuplicates()
+            .sink { [weak self] engine in
+                guard engine == .openAICompatible, let self else { return }
+                Task { [weak self] in
+                    guard let self else { return }
+                    do {
+                        let configuration = try self.suggestionSettings.openAICompatibleConfiguration
+                        let apiKey = try self.suggestionSettings.openAICompatibleAPIKey()
+                        await self.openAICompatibleConnectionModel.refresh(
+                            configuration: configuration,
+                            apiKey: apiKey
+                        )
+                    } catch {
+                        self.openAICompatibleConnectionModel.invalidate()
+                    }
+                }
+            }
+            .store(in: &cancellables)
+
+        let configurationChanges: [AnyPublisher<Void, Never>] = [
+            suggestionSettings.$openAICompatibleBaseURL.dropFirst().map { _ in () }.eraseToAnyPublisher(),
+            suggestionSettings.$openAICompatibleModelName.dropFirst().map { _ in () }.eraseToAnyPublisher(),
+            suggestionSettings.$openAICompatibleAPIMode.dropFirst().map { _ in () }.eraseToAnyPublisher(),
+            suggestionSettings.$endpointCredentialRevision.dropFirst().map { _ in () }.eraseToAnyPublisher()
+        ]
+
+        Publishers.MergeMany(configurationChanges)
+            .sink { [weak self] _ in
+                guard let self, self.suggestionSettings.selectedEngine == .openAICompatible else { return }
+                self.openAICompatibleConnectionModel.invalidate()
+            }
+            .store(in: &cancellables)
+    }
+}
+
+private extension String {
+    var nonEmpty: String? { isEmpty ? nil : self }
 }