fix: cap compaction and completion max_tokens to provider limits

Compaction requests sent max_tokens equal to the model's full context
window whenever maxOutputSize was not configured, which strict
OpenAI-compatible providers reject with 400 invalid_request_error
(e.g. "Invalid max_tokens value, the valid range of max_tokens is
[1, 393216]"). Cap compaction output at 128k by default and size
chat-completions caps to the remaining context window.

Author: elkaix

.changeset/compaction-max-tokens-cap.md

@@ -0,0 +1,5 @@
+---
+"@pythoughts/pythinker-code": patch
+---
+
+Fix context compaction failing with provider "Invalid max_tokens" errors by capping requested completion tokens to the remaining context window and a safe output ceiling instead of the full context window size.

packages/agent-core/src/agent/compaction/full.ts

@@ -44,6 +44,15 @@ import {
 
 export const MAX_COMPACTION_RETRY_ATTEMPTS = 5;
 
+/**
+ * Default hard cap on compaction output tokens when `maxOutputSize` is not
+ * configured on the model alias. Without this, compaction falls back to the
+ * full context window size, which exceeds the `max_tokens` ceiling enforced
+ * by many OpenAI-compatible providers. 128k matches the chat-completions
+ * ceiling applied by the OpenAI Legacy provider.
+ */
+const DEFAULT_COMPACTION_MAX_COMPLETION_TOKENS = 128 * 1024;
+
 class CompactionTruncatedError extends Error {
   constructor() {
     super('Compaction response was truncated before producing a complete summary.');
@@ -299,12 +308,25 @@ export class FullCompaction {
       await this.triggerPreCompactHook(data, tokensBefore, signal);
 
       const model = this.agent.config.model;
+      const capability = this.agent.config.modelCapabilities;
+      const maxContextTokens = capability.max_context_tokens;
+      // When the model's context window is known and the user has not set
+      // `maxOutputSize`, cap compaction output to a safe default so a large
+      // context window does not push `max_tokens` past the provider's ceiling.
+      // When the window is unknown (maxContextTokens === 0), leave
+      // `maxOutputSize` unset so `resolveCompletionBudget` falls back to the
+      // conservative unknown-context fallback.
+      const defaultCompactionCap =
+        maxContextTokens > 0
+          ? Math.min(maxContextTokens, DEFAULT_COMPACTION_MAX_COMPLETION_TOKENS)
+          : undefined;
       const provider = applyCompletionBudget({
         provider: this.agent.config.provider,
         budget: resolveCompletionBudget({
+          maxOutputSize: this.agent.config.maxOutputSize ?? defaultCompactionCap,
           reservedContextSize: this.agent.pythinkerConfig?.loopControl?.reservedContextSize,
         }),
-        capability: this.agent.config.modelCapabilities,
+        capability,
       });
 
       const delays = retryBackoffDelays(MAX_COMPACTION_RETRY_ATTEMPTS);

packages/agent-core/src/agent/index.ts

@@ -311,6 +311,7 @@ export class Agent {
       capability: this.config.modelCapabilities,
       generate: this.generate,
       completionBudgetConfig,
+      usedContextTokens: () => this.context.tokenCount,
     });
   }
 

packages/agent-core/src/agent/turn/kosong-llm.ts

@@ -55,6 +55,12 @@ export interface KosongLLMConfig {
    * final cap is applied to each request.
    */
   readonly completionBudgetConfig?: CompletionBudgetConfig | undefined;
+  /**
+   * Returns the number of context tokens already consumed by the latest
+   * completed step (API-reported input + output). Used by chat-completions
+   * providers to size the completion budget to the remaining context window.
+   */
+  readonly usedContextTokens?: (() => number) | undefined;
 }
 
 export class KosongLLM implements LLM {
@@ -65,6 +71,7 @@ export class KosongLLM implements LLM {
   private readonly provider: ChatProvider;
   private readonly generate: GenerateFn;
   private readonly completionBudgetConfig: CompletionBudgetConfig | undefined;
+  private readonly usedContextTokens: (() => number) | undefined;
 
   constructor(config: KosongLLMConfig) {
     this.provider = config.provider;
@@ -73,6 +80,7 @@ export class KosongLLM implements LLM {
     this.capability = config.capability;
     this.generate = config.generate ?? kosongGenerate;
     this.completionBudgetConfig = config.completionBudgetConfig;
+    this.usedContextTokens = config.usedContextTokens;
   }
 
   async chat(params: LLMChatParams): Promise<LLMChatResponse> {
@@ -98,6 +106,7 @@ export class KosongLLM implements LLM {
       provider: this.provider,
       budget: this.completionBudgetConfig,
       capability: this.capability,
+      usedContextTokens: this.usedContextTokens?.(),
     });
     const options: GenerateOptionsWithRequestLogFields = {
       signal: params.signal,

packages/agent-core/src/utils/completion-budget.ts

@@ -79,12 +79,16 @@ export function applyCompletionBudget(args: {
   readonly provider: ChatProvider;
   readonly budget: CompletionBudgetConfig | undefined;
   readonly capability: ModelCapability | undefined;
+  readonly usedContextTokens?: number;
 }): ChatProvider {
   if (args.budget === undefined) return args.provider;
   if (args.provider.withMaxCompletionTokens === undefined) return args.provider;
   const cap = computeCompletionBudgetCap({
     budget: args.budget,
     capability: args.capability,
   });
-  return args.provider.withMaxCompletionTokens(cap);
+  return args.provider.withMaxCompletionTokens(cap, {
+    usedContextTokens: args.usedContextTokens,
+    maxContextTokens: args.capability?.max_context_tokens,
+  });
 }

packages/agent-core/test/agent/compaction/full.test.ts

@@ -1841,6 +1841,39 @@ describe('FullCompaction', () => {
     expect(compactionMaxCompletionTokens).toEqual([undefined]);
   });
 
+  it('uses default 128k hardCap when maxOutputSize is not configured', async () => {
+    let callCount = 0;
+    const compactionMaxCompletionTokens: unknown[] = [];
+    const generate: GenerateFn = async (provider, _system, _tools, _history, callbacks) => {
+      callCount += 1;
+      if (callCount === 1) {
+        throw new APIContextOverflowError(400, 'Context length exceeded', 'req-default-cap');
+      }
+      if (callCount === 2) {
+        compactionMaxCompletionTokens.push(providerMaxCompletionTokens(provider));
+        return textResult('Default cap compacted summary.');
+      }
+      await callbacks?.onMessagePart?.({
+        type: 'text',
+        text: 'Recovered with default cap.',
+      });
+      return textResult('Recovered with default cap.');
+    };
+    const ctx = testAgent({ generate });
+    ctx.configure({
+      provider: CATALOGUED_PROVIDER,
+      modelCapabilities: CATALOGUED_MODEL_CAPABILITIES,
+    });
+    ctx.appendExchange(1, 'old user one', 'old assistant one', 20);
+    ctx.newEvents();
+
+    await ctx.rpc.prompt({ input: [{ type: 'text', text: 'Retry with default cap' }] });
+    await ctx.untilTurnEnd();
+
+    expect(callCount).toBe(3);
+    expect(compactionMaxCompletionTokens).toEqual([128 * 1024]);
+  });
+
   it('ignores filtered assistant placeholders when checking the retained overflow suffix', async () => {
     let callCount = 0;
     const generate: GenerateFn = async (_provider, _system, _tools, _history, callbacks) => {

packages/agent-core/test/agent/config-state.test.ts

@@ -74,7 +74,7 @@ describe('ConfigState model capabilities', () => {
     });
   });
 
-  it('uses model max output size as the LLM completion cap', async () => {
+  it('clamps the LLM completion cap to 128k for openai-compatible providers', async () => {
     let requestMaxTokens: unknown;
     const ctx = testAgent({
       generate: async (provider) => {
@@ -121,7 +121,9 @@ describe('ConfigState model capabilities', () => {
       signal: new AbortController().signal,
     });
 
-    expect(requestMaxTokens).toBe(384000);
+    // maxOutputSize (384000) is clamped to the 128k ceiling applied to
+    // OpenAI-compatible chat-completions providers.
+    expect(requestMaxTokens).toBe(131072);
   });
 
   it('uses session id as a provider prompt cache hint without storing it on Agent', () => {

packages/kosong/src/provider.ts

@@ -13,6 +13,22 @@ import type { TokenUsage } from './usage';
  */
 export type ThinkingEffort = 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
 
+/**
+ * Optional context passed to {@link ChatProvider.withMaxCompletionTokens} so a
+ * provider can tighten the caller-supplied cap to its own transport
+ * constraints.
+ */
+export interface MaxCompletionTokensOptions {
+  /**
+   * Tokens already consumed by the current context (API-reported input +
+   * output of the latest completed step). Chat-completions providers use it
+   * to size the cap to the remaining context window.
+   */
+  readonly usedContextTokens?: number;
+  /** Model context-window size in tokens (`max_context_size`). */
+  readonly maxContextTokens?: number;
+}
+
 /**
  * Normalized finish-reason signal indicating why a generation stopped.
  *
@@ -161,11 +177,19 @@ export interface ChatProvider {
    * budget clamped to `maxCompletionTokens`. Optional because not every
    * backend benefits from a client-computed cap.
    *
+   * When `options` are provided, implementations may further tighten the cap
+   * based on their own transport constraints — e.g. chat-completions
+   * endpoints size the cap to the remaining context window
+   * (`maxContextTokens - usedContextTokens`) and/or clamp to a fixed ceiling.
+   *
    * Implementations MUST NOT mutate or replace internal HTTP clients on the
    * returned clone — the clone is expected to share transport state with the
    * original. See `PythinkerChatProvider._clone()` for the rationale.
    */
-  withMaxCompletionTokens?(maxCompletionTokens: number): ChatProvider;
+  withMaxCompletionTokens?(
+    maxCompletionTokens: number,
+    options?: MaxCompletionTokensOptions,
+  ): ChatProvider;
   /** Upload a video and return a content part that can be sent to this provider. */
   uploadVideo?(input: string | VideoUploadInput, options?: GenerateOptions): Promise<VideoURLPart>;
 }

packages/kosong/src/providers/openai-legacy.ts

@@ -3,6 +3,7 @@ import type {
   ChatProvider,
   FinishReason,
   GenerateOptions,
+  MaxCompletionTokensOptions,
   ProviderRequestAuth,
   StreamedMessage,
   ThinkingEffort,
@@ -48,6 +49,13 @@ import {
 // arms can be overridden by an explicit `reasoningKey` on the provider config.
 const KNOWN_REASONING_KEYS = ['reasoning_content', 'reasoning_details', 'reasoning'] as const;
 const DEFAULT_OUTBOUND_REASONING_KEY = KNOWN_REASONING_KEYS[0];
+
+/**
+ * Hard upper bound on `max_tokens` for OpenAI-compatible chat-completions
+ * endpoints. Many third-party providers reject `max_tokens` above this limit
+ * (the documented range is `[1, 131072]`).
+ */
+const CHAT_COMPLETIONS_MAX_OUTPUT_TOKENS_CEILING = 128 * 1024;
 const OPENAI_CHAT_TOOL_CALL_ID_POLICY: ToolCallIdPolicy = {
   normalize: (id) => sanitizeToolCallId(id, 64),
   maxLength: 64,
@@ -645,8 +653,20 @@ export class OpenAILegacyChatProvider implements ChatProvider {
     return clone;
   }
 
-  withMaxCompletionTokens(maxCompletionTokens: number): OpenAILegacyChatProvider {
-    return this.withGenerationKwargs(completionTokenKwargs(this._model, maxCompletionTokens));
+  withMaxCompletionTokens(
+    maxCompletionTokens: number,
+    options?: MaxCompletionTokensOptions,
+  ): OpenAILegacyChatProvider {
+    let cap = maxCompletionTokens;
+    if (
+      options?.usedContextTokens !== undefined &&
+      options?.maxContextTokens !== undefined &&
+      options.maxContextTokens > 0
+    ) {
+      cap = Math.min(cap, options.maxContextTokens - options.usedContextTokens);
+    }
+    cap = Math.min(cap, CHAT_COMPLETIONS_MAX_OUTPUT_TOKENS_CEILING);
+    return this.withGenerationKwargs(completionTokenKwargs(this._model, Math.max(1, cap)));
   }
 
   private _clone(): OpenAILegacyChatProvider {

packages/kosong/src/providers/pythinker.ts

@@ -4,6 +4,7 @@ import type {
   ChatProvider,
   FinishReason,
   GenerateOptions,
+  MaxCompletionTokensOptions,
   ProviderRequestAuth,
   StreamedMessage,
   ThinkingEffort,
@@ -551,8 +552,19 @@ export class PythinkerChatProvider implements ChatProvider {
     return this._withGenerationKwargs(kwargs);
   }
 
-  withMaxCompletionTokens(maxCompletionTokens: number): PythinkerChatProvider {
-    return this._withGenerationKwargs({ max_completion_tokens: maxCompletionTokens });
+  withMaxCompletionTokens(
+    maxCompletionTokens: number,
+    options?: MaxCompletionTokensOptions,
+  ): PythinkerChatProvider {
+    let cap = maxCompletionTokens;
+    if (
+      options?.usedContextTokens !== undefined &&
+      options?.maxContextTokens !== undefined &&
+      options.maxContextTokens > 0
+    ) {
+      cap = Math.min(cap, options.maxContextTokens - options.usedContextTokens);
+    }
+    return this._withGenerationKwargs({ max_completion_tokens: Math.max(1, cap) });
   }
 
   withExtraBody(extraBody: ExtraBody): PythinkerChatProvider {