fix(core,web): retry transport and quota LLM errors, disable dead sessions on auth failure (#82)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/server/src/api/types.ts

@@ -234,6 +234,13 @@ export interface ModelsResponse {
   defaultModel?: ModelRefDto;
   /** Vision model used as a proxy reader for read_image (describes images when the session model has vision=false). */
   visionModel?: ModelRefDto;
+  /**
+   * When the Project's model/credential config last changed (ISO; the config file's mtime,
+   * so it survives restarts). The web's auth-dead gate compares it against the last auth
+   * abort: an abort OLDER than the last credential update no longer disables the composer
+   * (the key was fixed since). Absent when the Project has no config file yet.
+   */
+  updatedAt?: string;
   models: ModelInfo[];
 }
 
@@ -609,6 +616,16 @@ export interface ApprovalDecisionRequest {
   decision: "allow" | "deny";
 }
 
+/**
+ * POST /api/sessions/:sessionId/retry-now — skip the in-progress reconnect backoff and
+ * fire the next retry immediately (the "retry now" button on the reconnect countdown).
+ * `skipped: false` is the benign "no reconnect wait in progress" case (idle session, or
+ * the wait elapsed in a timing race), not an error.
+ */
+export interface RetryNowResponse {
+  skipped: boolean;
+}
+
 // ---------------------------------------------------------------------------
 // SSE server events (OmniMessage uses the default event, only server_event here)
 // ---------------------------------------------------------------------------
@@ -626,6 +643,14 @@ export type ServerEvent =
   | { type: "session_title"; sessionId: string; title: string }
   /** Last-Event-ID has been evicted from the buffer: the frontend should re-fetch the history endpoint before continuing to consume this connection. */
   | { type: "resync_required" }
+  /**
+   * The Project's model credentials changed (PUT /models): cached runtimes have been
+   * invalidated server-side, so an auth-dead Session can continue — the frontend clears
+   * its auth-dead composer state immediately. Published to every existing Session channel
+   * of the Project; tabs without a live channel learn the same fact from the models
+   * response's `updatedAt` on their next load.
+   */
+  | { type: "credentials_updated" }
   /** Placeholder handshake on the user channel (reserved for automated task notifications). */
   | { type: "hello" }
   /** New session registered (pushed over the parent session's channel for subagent sessions): frontend refreshes the list in place. */

packages/server/src/http/routes/models.ts

@@ -10,11 +10,25 @@ import type {
   ModelsUpdateRequest,
   ModelTestRequest,
   ModelUpdateEntry,
+  ServerEvent,
 } from "../../api/types.js";
 import type { AppEnv } from "../../auth/middleware.js";
 import { badRequest, readJson, requireString, requireValidId } from "../validate.js";
 import type { AppDeps } from "../../app.js";
 
+/**
+ * Live unlock for auth-dead composers: after a models/credential update, publish
+ * `credentials_updated` to every EXISTING channel of this Project's Sessions (`peek` —
+ * never creates channels: a tab that isn't subscribed learns the same fact from the models
+ * response's `updatedAt` when it next loads).
+ */
+function publishCredentialsUpdated(deps: AppDeps, projectId: string): void {
+  const event: ServerEvent = { type: "credentials_updated" };
+  for (const row of deps.sessionsRepo.listByProject(projectId)) {
+    deps.channels.peek(row.sessionId)?.publish(event, "server_event");
+  }
+}
+
 /** Validate a paired reference object ({ provider, modelId }); shape mismatch throws 400. */
 function parseRef(value: unknown, label: string): ModelRefDto {
   if (value === null || typeof value !== "object" || Array.isArray(value)) {
@@ -143,7 +157,15 @@ export function modelsRoutes(deps: AppDeps): Hono<AppEnv> {
     const projectId = requireValidId(c, "projectId");
     deps.projectService.requireProjectOwner(c.var.user.userId, projectId);
     const req = parseModelsUpdate(await readJson(c));
-    return c.json(await deps.projectConfigService.updateModels(projectId, req));
+    const res = await deps.projectConfigService.updateModels(projectId, req);
+    // Effective-value semantics (mirrors the vault route): no hot swap into a Task already
+    // in flight, but every cached runtime in this Project is invalidated — the next Task on
+    // any of its Sessions re-resumes and reads the new api_key / base_url. Without this, a
+    // key edit would not reach a loaded Session until the 30-minute idle sweep or a restart.
+    deps.manager.invalidateProjectRuntimes(projectId);
+    // Live unlock: open tabs clear their auth-dead composer immediately (no reload needed).
+    publishCredentialsUpdated(deps, projectId);
+    return c.json(res);
   });
 
   // Connectivity test (owner): the model reference `(provider, modelId)` is sent as a pair

packages/server/src/http/routes/sessions.ts

@@ -21,6 +21,7 @@ import type {
   SessionCreateResponse,
   SessionResponse,
   SessionsResponse,
+  RetryNowResponse,
   TaskCreateResponse,
 } from "../../api/types.js";
 import { PREVIEW_TOKEN_TTL_MS, resolvePreviewTarget } from "../../services/preview-token.js";
@@ -420,6 +421,16 @@ export function sessionsRoutes(deps: AppDeps): Hono<AppEnv> {
     return c.body(null, aborted ? 202 : 204);
   });
 
+  // "Retry now" on the reconnect countdown: skip the remaining backoff wait and fire the
+  // next retry immediately (attempt counter unchanged). Benign either way — 200 with
+  // skipped:false when no reconnect wait is in progress, so a timing race (the wait
+  // elapsed just before the click) never surfaces as an error.
+  app.post("/:sessionId/retry-now", (c) => {
+    const row = resolveSession(c);
+    const skipped = deps.manager.retryNow(row.sessionId);
+    return c.json({ skipped } satisfies RetryNowResponse);
+  });
+
   app.post("/:sessionId/compact", async (c) => {
     const row = resolveSession(c);
     const { sessionId } = await deps.manager.startCompact(row.sessionId);

packages/server/src/runtime/session-manager.ts

@@ -78,6 +78,8 @@ export interface RuntimeSession {
   compactability(): CompactAvailability;
   /** Queues a mid-run steering message (core `Session.steer`); false when no Task is running. */
   steer(text: string): boolean;
+  /** Skips the in-progress reconnect backoff, firing the next retry immediately (core `Session.skipReconnectWait`); false when no wait is in progress. */
+  skipReconnectWait(): boolean;
   toolPermission(name: string): "r" | "rw" | undefined;
   /**
    * Out-of-band one-shot request for title generation (core `Session.generateTitle`,
@@ -396,6 +398,22 @@ export class SessionManager {
     this.agentGenerations.set(key, this.generationOf(projectId, agentId) + 1);
   }
 
+  /**
+   * After a Project's models/credentials change: invalidate every cached runtime in this
+   * Project, so the next Task re-resumes with the new api_key / base_url. Same
+   * effective-value semantics as invalidateAgentRuntimes — no hot swap into a Task already
+   * in flight. Iterating the active table is complete here, not a shortcut: the generation
+   * map only matters for runtimes that are already cached, and an Agent with no cached
+   * entry builds fresh through the loader anyway.
+   */
+  invalidateProjectRuntimes(projectId: string): void {
+    const agentIds = new Set<string>();
+    for (const e of this.entries.values()) {
+      if (e.projectId === projectId) agentIds.add(e.agentId);
+    }
+    for (const agentId of agentIds) this.invalidateAgentRuntimes(projectId, agentId);
+  }
+
   // —— Task / compaction drive ——
 
   /**
@@ -558,6 +576,23 @@ export class SessionManager {
     entry.lastActivityMs = Date.now();
   }
 
+  /**
+   * "Retry now" on the reconnect countdown: skip the in-progress backoff wait and fire
+   * the next retry immediately (the attempt counter is unchanged — the skipped wait does
+   * not consume an extra attempt). Returns false as a benign no-op when the session has
+   * no active runtime or no reconnect wait is in progress — timing races (the wait
+   * elapsed right before the click) must not surface as errors. Mirrors the steer seam:
+   * manager → RuntimeSession → core Session → engine.
+   */
+  retryNow(sessionId: string): boolean {
+    const entry = this.entries.get(sessionId);
+    // Both running Tasks and compactions can be parked in a reconnect backoff.
+    if (!entry || entry.status === "idle") return false;
+    const skipped = entry.session.skipReconnectWait();
+    if (skipped) entry.lastActivityMs = Date.now();
+    return skipped;
+  }
+
   /**
    * Interrupt the current Task/compaction: pending approvals converge to deny first,
    * then the AbortSignal fires. Returns false if nothing is in progress (the route

packages/server/src/runtime/stream-error-watcher.ts

@@ -9,21 +9,23 @@
  * (its state wraps up accordingly, see close).
  *
  * LLM (source = `llm`): reads the status of `request_end` —
- * - `failed` → unexpected (not retryable: auth failure, invalid params, etc., needs a human);
+ * - `failed` / `auth` → unexpected (not retryable: credentials rejected, invalid params,
+ *   etc., needs a human; both share the llm_failed code — no separate taxonomy);
  * - `timeout` / `malformed` → expected (the engine already reconnects and retries, part
  *   of normal operation);
  * - `aborted` / `completed` are not recorded (the former is a user-initiated interrupt,
  *   not an error).
  *
- * The message uses the real reason: `request_end` only carries status, and **the only
- * place core carries the actual failure-reason text is the `abort` event's reason**
- * (e.g. `llm request error: 401 …` / `malformed response failed after N retries`). So a
- * `request_end` failure is first held pending, not persisted immediately, and is
- * resolved at the next request boundary:
+ * The message uses the real reason: `request_end` carries the failure detail on
+ * non-completed statuses (`message`, from LLMOutcome), and the `abort` event's reason is
+ * core's failure-reason prose (e.g. `llm request error: 401 …` / `malformed response
+ * failed after N retries`). A `request_end` failure is first held pending (status + its
+ * own detail), not persisted immediately, and is resolved at the next request boundary:
  * - Immediately followed by `abort` → use its reason as the message (the real reason);
- * - Immediately followed by `request_begin` (the engine is retrying) → no reason text
- *   left to wait for, use the status text;
- * - Still unresolved when the run ends → close persists it as a fallback.
+ * - Immediately followed by `request_begin` (the engine is retrying — no abort will ever
+ *   arrive) → use the staged request_end's own `message` (the real detail, e.g. a quota
+ *   code), falling back to the generic status text when it carried none;
+ * - Still unresolved when the run ends → close persists it the same way.
  * Exception: when reason is a user-interrupt message (`aborted …`), it's not trusted —
  * "the user clicked stop during backoff" isn't the reason for this timeout, so the
  * status text is used instead (that timeout is a genuine failure and is still recorded).
@@ -71,11 +73,14 @@ export const TOOL_NAMES_MAX = 1000;
 export const ORIGIN_CTX_MAX = 200;
 
 /** Recorded LLM failure states (`aborted` / `completed` are not errors and aren't included here). */
-type LlmFailure = "failed" | "timeout" | "malformed";
+type LlmFailure = "failed" | "timeout" | "malformed" | "auth";
 
 /** LLM failure state → error code, classification, and fallback message (used when the abort reason isn't available). */
 const LLM_FAILURES: Record<LlmFailure, { code: string; kind: ErrorKind; text: string }> = {
   failed: { code: "llm_failed", kind: "unexpected", text: "LLM request failed (not retryable)." },
+  // Credentials rejection shares the failed bucket (no new taxonomy): same code/kind, the
+  // real reason arrives via the abort reason or the event's own message as usual.
+  auth: { code: "llm_failed", kind: "unexpected", text: "LLM request failed (not retryable)." },
   timeout: {
     code: "llm_timeout",
     kind: "expected",
@@ -92,7 +97,7 @@ const LLM_FAILURES: Record<LlmFailure, { code: string; kind: ErrorKind; text: st
 type ToolFailure = "failed" | "timeout";
 
 function isLlmFailure(s: unknown): s is LlmFailure {
-  return s === "failed" || s === "timeout" || s === "malformed";
+  return s === "failed" || s === "timeout" || s === "malformed" || s === "auth";
 }
 
 function isToolFailure(s: unknown): s is ToolFailure {
@@ -123,8 +128,8 @@ function toolFailureText(output: string): string {
 }
 
 export class StreamErrorWatcher {
-  /** LLM failures awaiting a real reason: origin → failure state (see file header; each session has at most one in-flight Request). */
-  private readonly pending = new Map<string, LlmFailure>();
+  /** LLM failures awaiting a real reason: origin → failure state + the request_end's own detail (see file header; each session has at most one in-flight Request). */
+  private readonly pending = new Map<string, { status: LlmFailure; message?: string }>();
   /** Names of in-flight tool calls: `origin \0 tool_call_id` → tool name (dequeued once output arrives). */
   private readonly toolNames = new Map<string, string>();
   /** Subagent identity: origin → that subagent's `{agentId, sessionId}` (see the file header's attribution section). */
@@ -196,11 +201,23 @@ export class StreamErrorWatcher {
   // —— LLM ——
 
   private observeLlm(msg: OmniMessage): void {
-    const p = msg.payload as { type?: string; status?: StopReason; reason?: string | null };
+    const p = msg.payload as {
+      type?: string;
+      status?: StopReason;
+      reason?: string | null;
+      message?: string;
+    };
     const key = originKey(msg);
     if (p.type === "request_end") {
       this.flush(key); // Defensive: if a previous failure is still pending (normally resolved by request_begin), persist it first
-      if (isLlmFailure(p.status)) this.pending.set(key, p.status);
+      if (isLlmFailure(p.status)) {
+        this.pending.set(key, {
+          status: p.status,
+          // The event's own failure detail (LLMOutcome.message): the message of record when
+          // no abort follows (the retry path).
+          ...(typeof p.message === "string" && p.message.trim() ? { message: p.message } : {}),
+        });
+      }
       return;
     }
     // A new attempt begins (the engine is retrying): no reason text left to wait for the previous failure, persist using the status text.
@@ -221,13 +238,15 @@ export class StreamErrorWatcher {
    * up from it (see file header).
    */
   private flush(key: string, reason?: string | null): void {
-    const status = this.pending.get(key);
-    if (status === undefined) return;
+    const entry = this.pending.get(key);
+    if (entry === undefined) return;
     this.pending.delete(key);
-    const spec = LLM_FAILURES[status];
+    const spec = LLM_FAILURES[entry.status];
     const trimmed = reason?.trim();
-    // A user-interrupt message isn't a failure reason (see file header); fall back to the status text.
-    const message = trimmed && !isUserAbortReason(trimmed) ? trimmed : spec.text;
+    // Message priority: the abort reason (core's failure prose) → the staged request_end's
+    // own detail (the retry path: no abort ever arrives) → the generic status text. A
+    // user-interrupt message isn't a failure reason (see file header).
+    const message = trimmed && !isUserAbortReason(trimmed) ? trimmed : (entry.message ?? spec.text);
     this.errors.record({
       source: "llm",
       err: message,

packages/server/src/services/project-config-service.ts

@@ -121,6 +121,20 @@ export class ProjectConfigService {
     return projectConfigPath(this.root, projectId);
   }
 
+  /**
+   * When the Project config (models/credentials) last changed: the config file's mtime as
+   * ISO — an honest persistent source that survives server restarts (every models update
+   * rewrites the file). undefined when the file doesn't exist yet.
+   */
+  private async configUpdatedAt(projectId: string): Promise<string | undefined> {
+    try {
+      const st = await fs.stat(this.filePath(projectId));
+      return st.mtime.toISOString();
+    } catch {
+      return undefined;
+    }
+  }
+
   /** Reads the raw TOML object; returns an empty object if the file doesn't exist (does not write to disk). */
   async readRaw(projectId: string): Promise<RawTable> {
     let raw: string;
@@ -390,9 +404,11 @@ export class ProjectConfigService {
       provider: ref.provider,
       modelId: ref.model_id,
     });
+    const updatedAt = await this.configUpdatedAt(projectId);
     return {
       ...(defaultRef !== undefined ? { defaultModel: toDto(defaultRef) } : {}),
       ...(visionRef !== undefined ? { visionModel: toDto(visionRef) } : {}),
+      ...(updatedAt !== undefined ? { updatedAt } : {}),
       models,
     };
   }