feat: per-model max output tokens and a conversation-time thinking level backed by agent settings (#28)

Co-authored-by: Alice <alice@prismshadow.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/core/src/agent.ts

@@ -117,6 +117,16 @@ export function effectiveMaxContextLength(configured: number, contextWindow: unk
   return Math.min(configured, Math.floor(contextWindow * 0.75));
 }
 
+/**
+ * Output cap for meta requests (title generation / vision describing): these carry their own
+ * small hardcoded budget, tightened further by the entry's per-model `max_tokens` when that is
+ * smaller — a cap the user pinned below the budget must bind every request to that model. The
+ * budget is never raised.
+ */
+export function metaMaxTokens(budget: number, modelCap: number | undefined): number {
+  return modelCap !== undefined ? Math.min(budget, modelCap) : budget;
+}
+
 /** Create or load an Agent. */
 export async function createAgent(opts: CreateAgentOptions = {}): Promise<Agent> {
   const state = await loadOrInitAgentState(opts);
@@ -570,7 +580,8 @@ export class Agent {
                 : {}),
               tools: [],
               thinkingLevel: "none",
-              maxTokens: 2048,
+              // The describing budget, tightened by the vision entry's own pinned cap when smaller.
+              maxTokens: metaMaxTokens(2048, visionEntry.max_tokens),
               requestTimeoutMs: 60_000,
             }),
         };
@@ -592,6 +603,12 @@ export class Agent {
     });
     const tools = await environment.listTools();
 
+    // Effective output cap: the entry's per-model annotation wins over the Agent's
+    // system_config value — the fit is a model trait: the seeded per-Agent default (32000)
+    // cannot fit into e.g. a 32768-token context window together with any prompt, so a
+    // small-window model needs its own pinned cap. Unset inherits the Agent value.
+    const maxTokens = modelEntry.max_tokens ?? this.state.systemConfig.model?.max_tokens;
+
     // LLM constructor args are extracted into a constant so they can be reused as-is when
     // rebuilding a new LLM object after compaction (with a fresh model context) — the system
     // prompt and tool definitions aren't part of the compacted history, so the new object keeps
@@ -612,9 +629,7 @@ export class Agent {
       ...(modelEntry.context_window !== undefined
         ? { contextWindow: modelEntry.context_window }
         : {}),
-      ...(this.state.systemConfig.model?.max_tokens !== undefined
-        ? { maxTokens: this.state.systemConfig.model.max_tokens }
-        : {}),
+      ...(maxTokens !== undefined ? { maxTokens } : {}),
       ...(this.state.systemConfig.model?.thinking_level !== undefined
         ? { thinkingLevel: this.state.systemConfig.model.thinking_level }
         : {}),
@@ -640,7 +655,8 @@ export class Agent {
         ...(modelEntry.client_type !== undefined ? { clientType: modelEntry.client_type } : {}),
         tools: [],
         thinkingLevel: "none",
-        maxTokens: 300,
+        // The meta budget, tightened by the entry's pinned per-model cap when smaller.
+        maxTokens: metaMaxTokens(300, modelEntry.max_tokens),
         requestTimeoutMs: 30_000,
       });
 

packages/core/src/index.ts

@@ -40,13 +40,13 @@ export type {
 } from "./engine/context-engine.js";
 export { Session } from "./session.js";
 export type { SessionConfig } from "./session.js";
-export {
-  buildTitlePrompt,
-  generateTitleWithLLM,
-  sanitizeTitle,
-  stripConversationMarkers,
-} from "./session-title.js";
-export type { SessionTitleResult } from "./session-title.js";
+// Session-title generation lives in internal/ (an assembly detail of Session.generateTitle);
+// only its narrow public surface is re-exported: the result type (part of
+// Session.generateTitle's signature) and the sanitation helpers the Web server's title
+// fallback builds on (stripConversationMarkers / sanitizeTitle). The prompt/request
+// internals (buildTitlePrompt / generateTitleWithLLM) are deliberately not public.
+export { sanitizeTitle, stripConversationMarkers } from "./internal/session-title.js";
+export type { SessionTitleResult } from "./internal/session-title.js";
 export { Agent, createAgent } from "./agent.js";
 export type { CreateAgentOptions, CreateSessionOptions, ResumeSessionOptions } from "./agent.js";
 

packages/core/src/internal/session-title.ts

@@ -1,21 +1,25 @@
 /**
  * Session title generation: an **out-of-band, one-off request** that generates
- * a short title from the first-turn conversation text.
+ * a short title from the first-turn conversation text (used by `Session.generateTitle`
+ * for assembly, not exported wholesale via the barrel).
  *
  * Called by `session.generateTitle()`: sends one request using the bare LLM for the session's
  * Model (no tools, no system prompt, thinking off), without writing history or Trace. Material
  * defaults to what the Session self-captures during run (see session.ts); this module is only
  * responsible for the prompt format, driving the one-off request, and sanitizing the result —
  * when to generate a title and where to store it is decided by the host (Web server / CLI).
+ * The narrow public surface — `SessionTitleResult` (part of `Session.generateTitle`'s
+ * signature) and the sanitation helpers the host's title fallback builds on — is re-exported
+ * by the barrel; the prompt/request internals are not.
  */
-import { userText } from "./omnimessage/index.js";
+import { userText } from "../omnimessage/index.js";
 import type {
   OmniMessage,
   TextPayload,
   TokenCounts,
   TokenUsagePayload,
-} from "./omnimessage/index.js";
-import type { LLMInterface } from "./interfaces.js";
+} from "../omnimessage/index.js";
+import type { LLMInterface } from "../interfaces.js";
 
 /** Cap on conversation text spliced into the title request (user/model each truncated separately, to control cost). */
 const EXCERPT_MAX_CHARS = 2000;

packages/core/src/session.ts

@@ -21,8 +21,8 @@ import { sessionMeta } from "./omnimessage/index.js";
 import type { OmniMessage, SessionMetaPayload, TokenCounts } from "./omnimessage/index.js";
 import { imagesToScratchpadPaths } from "./internal/session-support.js";
 import type { EnvironmentInterface, LLMInterface, ToolPermission } from "./interfaces.js";
-import { generateTitleWithLLM } from "./session-title.js";
-import type { SessionTitleResult } from "./session-title.js";
+import { generateTitleWithLLM } from "./internal/session-title.js";
+import type { SessionTitleResult } from "./internal/session-title.js";
 import { ContextEngine } from "./engine/context-engine.js";
 import type {
   CompactAvailability,

packages/core/src/state/project-config.ts

@@ -88,6 +88,14 @@ export interface ModelEntry {
    * enters that session's history.
    */
   vision?: boolean;
+  /**
+   * Per-model max output tokens (the request's output cap, i.e. GenerativeModelConfig.maxTokens):
+   * when set it wins over the Agent's `system_config.model.max_tokens` — the fit is a model trait
+   * (the seeded per-Agent default of 32000 cannot fit into e.g. a 32768-token context window
+   * together with any prompt, and the upstream rejects the request outright). Unset = inherit
+   * the Agent value. User-only, never preset by the builtin catalog.
+   */
+  max_tokens?: number;
   /** Pricing info; absent means this Model's cost isn't counted. */
   pricing?: ModelPricing;
   /** API key (inlined credential); left empty falls back to the vendor's environment variable. */
@@ -275,6 +283,8 @@ export async function addModel(
     client_type?: string;
     /** Whether image input is supported (vision/multimodal); keeps the existing value by default (treated as supported if never set). */
     vision?: boolean;
+    /** Per-model max output tokens (wins over the Agent config); keeps the existing value by default (unset = inherit the Agent value). */
+    max_tokens?: number;
     /** Price input may cover only some buckets; merged and written as a complete `ModelPricing`. */
     pricing?: Partial<ModelPricing>;
     api_key?: string;
@@ -310,6 +320,10 @@ export async function addModel(
   if (vision !== undefined) {
     modelEntry.vision = vision;
   }
+  const maxTokens = entry.max_tokens ?? existing?.max_tokens;
+  if (maxTokens !== undefined) {
+    modelEntry.max_tokens = maxTokens;
+  }
   // The three price buckets are merged field by field: an unspecified bucket keeps its existing
   // value (the same policy as context_window/credential); the unit is fixed to usd_per_mtok, and
   // the complete pricing is written as long as any bucket is present.

packages/core/test/agent.test.ts

@@ -22,7 +22,7 @@ import {
   installSkill,
   setVaultEntry,
 } from "../src/index.js";
-import { effectiveMaxContextLength } from "../src/agent.js";
+import { effectiveMaxContextLength, metaMaxTokens } from "../src/agent.js";
 import { stubProviderKeys } from "./provider-keys.js";
 
 let tmpRoot: string;
@@ -53,6 +53,15 @@ describe("effectiveMaxContextLength (compaction threshold clamped to the model w
   });
 });
 
+describe("metaMaxTokens (meta-request budget tightened by the per-model cap)", () => {
+  it("keeps the budget unless the per-model cap is smaller; never raises it", () => {
+    expect(metaMaxTokens(300, undefined)).toBe(300); // no per-model cap: the budget as-is
+    expect(metaMaxTokens(300, 8000)).toBe(300); // ample cap: the small budget stays
+    expect(metaMaxTokens(300, 128)).toBe(128); // pinned below the budget: the cap binds
+    expect(metaMaxTokens(2048, 1024)).toBe(1024); // vision-describer budget, same rule
+  });
+});
+
 describe("Agent.createSession workspace handling", () => {
   it("throws a clear error when the given workspace does not exist (no auto-create)", async () => {
     const agent = await createAgent();
@@ -192,6 +201,93 @@ describe("Agent.createSession model reference ((provider, model_id) pair)", () =
   });
 });
 
+describe("Agent.createSession max output tokens (per-model cap wins over the Agent config)", () => {
+  // Reads a constructed GenerativeModel's request config (private; runtime-accessible for assertion).
+  const uniConfigOf = (llm: unknown) =>
+    (llm as { uniConfig?: { max_tokens?: number } }).uniConfig ?? {};
+
+  it("uses the entry's max_tokens in llmConfig, and inherits the seeded 32000 when unset", async () => {
+    // A 32k-context local model: the seeded per-Agent default (32000 output tokens) cannot fit
+    // into its window together with any prompt — the pinned per-model cap must win.
+    await addModel(tmpRoot, DEFAULT_PROJECT_ID, {
+      provider: "custom",
+      model_id: "local-32k",
+      client_type: "openai",
+      context_window: 32768,
+      max_tokens: 8000,
+    });
+    const agent = await createAgent();
+    expect(agent.state.systemConfig.model?.max_tokens).toBe(32000);
+    const ws = path.join(tmpRoot, "ws-max-tokens");
+    await fs.mkdir(ws, { recursive: true });
+
+    const pinned = await agent.createSession({
+      workspaceDir: ws,
+      modelId: "local-32k",
+      provider: "custom",
+    });
+    try {
+      const llm = (pinned as unknown as { engine: { deps: { llm: unknown } } }).engine.deps.llm;
+      expect(uniConfigOf(llm).max_tokens).toBe(8000);
+    } finally {
+      pinned.dispose();
+    }
+
+    // An unannotated entry (the default model) inherits the Agent value, as before.
+    const inherited = await agent.createSession({ workspaceDir: ws });
+    try {
+      const llm = (inherited as unknown as { engine: { deps: { llm: unknown } } }).engine.deps.llm;
+      expect(uniConfigOf(llm).max_tokens).toBe(32000);
+    } finally {
+      inherited.dispose();
+    }
+  });
+
+  it("meta requests keep their small budget, tightened when the per-model cap is even smaller", async () => {
+    await addModel(tmpRoot, DEFAULT_PROJECT_ID, {
+      provider: "custom",
+      model_id: "local-32k",
+      client_type: "openai",
+      max_tokens: 8000,
+    });
+    await addModel(tmpRoot, DEFAULT_PROJECT_ID, {
+      provider: "custom",
+      model_id: "tiny-cap",
+      client_type: "openai",
+      max_tokens: 128,
+    });
+    const agent = await createAgent();
+    const ws = path.join(tmpRoot, "ws-meta-cap");
+    await fs.mkdir(ws, { recursive: true });
+
+    // Ample per-model cap: the title one-shot keeps its own 300 budget (never raised to the cap).
+    const ample = await agent.createSession({
+      workspaceDir: ws,
+      modelId: "local-32k",
+      provider: "custom",
+    });
+    try {
+      const bare = (ample as unknown as { createBareLLM?: () => unknown }).createBareLLM?.();
+      expect(uniConfigOf(bare).max_tokens).toBe(300);
+    } finally {
+      ample.dispose();
+    }
+
+    // Cap pinned below the budget: the meta request must respect it too.
+    const tiny = await agent.createSession({
+      workspaceDir: ws,
+      modelId: "tiny-cap",
+      provider: "custom",
+    });
+    try {
+      const bare = (tiny as unknown as { createBareLLM?: () => unknown }).createBareLLM?.();
+      expect(uniConfigOf(bare).max_tokens).toBe(128);
+    } finally {
+      tiny.dispose();
+    }
+  });
+});
+
 describe("Agent.createSession vault injection", () => {
   it("injects vault key names (never values) into the assembled system prompt", async () => {
     // Write the Agent vault to disk first; createSession reads that Agent's own .vault.toml.

packages/core/test/session-title.test.ts

@@ -5,9 +5,7 @@
 import { describe, it, expect } from "vitest";
 import {
   assistantText,
-  buildTitlePrompt,
   emptyTokenCounts,
-  generateTitleWithLLM,
   sanitizeTitle,
   Session,
   stripConversationMarkers,
@@ -15,6 +13,8 @@ import {
   tokenUsage,
   userText,
 } from "../src/index.js";
+// Prompt/request internals are no longer exported via the barrel: imported directly from the internal module.
+import { buildTitlePrompt, generateTitleWithLLM } from "../src/internal/session-title.js";
 import type {
   EnvironmentInterface,
   LLMInterface,

packages/core/test/state.test.ts

@@ -531,6 +531,7 @@ describe("project-config round trip", () => {
         provider: "custom",
         model_id: "gpt-test",
         context_window: 128000,
+        max_tokens: 8192,
         api_key: "sk-abc",
         base_url: "https://example.com/v1",
       },
@@ -549,6 +550,7 @@ describe("project-config round trip", () => {
       provider: "custom",
       model_id: "gpt-test",
       context_window: 128000,
+      max_tokens: 8192,
       api_key: "sk-abc",
       base_url: "https://example.com/v1",
     });
@@ -644,6 +646,32 @@ describe("project-config round trip", () => {
     expect(m?.vision).toBe(true);
   });
 
+  it("addModel persists max_tokens and upsert preserves it when not re-specified", async () => {
+    await addModel(tmpRoot, DEFAULT_PROJECT_ID, {
+      provider: "custom",
+      model_id: "small-window",
+      max_tokens: 4096,
+    });
+    // Only supplements context_window, without max_tokens: the original annotation is kept.
+    await addModel(tmpRoot, DEFAULT_PROJECT_ID, {
+      provider: "custom",
+      model_id: "small-window",
+      context_window: 32768,
+    });
+    const ref = { provider: "custom", model_id: "small-window" };
+    let m = getModel(await loadProjectConfig(tmpRoot, DEFAULT_PROJECT_ID), ref);
+    expect(m?.max_tokens).toBe(4096);
+    expect(m?.context_window).toBe(32768);
+    // Explicitly re-pins the cap.
+    await addModel(tmpRoot, DEFAULT_PROJECT_ID, {
+      provider: "custom",
+      model_id: "small-window",
+      max_tokens: 2048,
+    });
+    m = getModel(await loadProjectConfig(tmpRoot, DEFAULT_PROJECT_ID), ref);
+    expect(m?.max_tokens).toBe(2048);
+  });
+
   it("setVisionModel persists and validates the target", async () => {
     await addModel(tmpRoot, DEFAULT_PROJECT_ID, {
       provider: "custom",