fix(core,web): retry transport and quota LLM errors, disable dead sessions on auth failure (#82)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/core/src/engine/context-engine.ts

@@ -152,10 +152,29 @@ export interface ContextEngineDeps {
   initialState?: EngineInitialState;
   /** Maximum LLM turns for a single Task. Defaults to 100; -1 removes the cap. */
   maxTurns?: number;
-  /** Maximum automatic retries for LLM timeout/reconnect within a single run. Defaults to 2. */
+  /**
+   * Maximum automatic retries for LLM timeout/reconnect within a single run. Defaults
+   * to 5: with the default backoff (250ms base, 30s ceiling) that is 250+500+1000+2000+
+   * 4000 ≈ 7.75s of total patience — the first attempts stay fast enough for transport
+   * blips while the tail still gives transient provider rejections a window.
+   */
   maxReconnects?: number;
-  /** Linear backoff base (ms) before each reconnect retry; actual backoff = base × retry number. Defaults to 250. */
+  /**
+   * Exponential backoff base (ms): the wait before reconnect retry N is
+   * `base × 2^(N−1)`, capped at `reconnectBackoffMaxMs` (see reconnectDelayMs).
+   * Defaults to 250.
+   */
   reconnectBackoffMs?: number;
+  /** Ceiling (ms) for a single reconnect backoff wait. Defaults to 30000. */
+  reconnectBackoffMaxMs?: number;
+  /**
+   * Maximum retries for a failing compaction request (instead of the shared
+   * `maxReconnects`). Compaction failure already has graceful semantics — the original
+   * context is kept and compaction retries on the next trigger — so it fails fast
+   * (~1.75s of backoff) rather than stalling the session through the full reconnect
+   * ladder. Defaults to 3.
+   */
+  compactionMaxReconnects?: number;
   /**
    * Creates a new LLM object after compaction (a fresh model context); the argument is the
    * current Session cumulative token counts, for the new object to carry forward
@@ -232,10 +251,23 @@ class MergeQueue {
   }
 }
 
+/**
+ * Delay before reconnect attempt N (1-based): exponential growth from `base` with a hard
+ * ceiling `max` — `min(base × 2^(N−1), max)`. With the defaults (250ms base, 30s ceiling,
+ * 5 reconnects) the ladder is 250, 500, 1000, 2000, 4000 ≈ 7.75s of total patience: one
+ * shared schedule serves every retryable class, growing toward the slower ones (transient
+ * provider quota errors) while the first steps stay as fast as a transport blip needs.
+ */
+export function reconnectDelayMs(base: number, max: number, attempt: number): number {
+  return Math.min(base * 2 ** (attempt - 1), max);
+}
+
 export class ContextEngine {
   private readonly maxTurns: number;
   private readonly maxReconnects: number;
   private readonly reconnectBackoffMs: number;
+  private readonly reconnectBackoffMaxMs: number;
+  private readonly compactionMaxReconnects: number;
   /** Interruption cleanup: content to resend generated when the previous run was aborted, held on the engine across runs. */
   private pendingCarryOver: OmniMessage[] = [];
   /** Current LLM object; swapped for a new one created by `createLLM` after a successful compaction (a fresh model context). */
@@ -274,8 +306,10 @@ export class ContextEngine {
 
   constructor(private readonly deps: ContextEngineDeps) {
     this.maxTurns = deps.maxTurns ?? 100;
-    this.maxReconnects = deps.maxReconnects ?? 2;
+    this.maxReconnects = deps.maxReconnects ?? 5;
     this.reconnectBackoffMs = deps.reconnectBackoffMs ?? 250;
+    this.reconnectBackoffMaxMs = deps.reconnectBackoffMaxMs ?? 30_000;
+    this.compactionMaxReconnects = deps.compactionMaxReconnects ?? 3;
     this.llm = deps.llm;
     // Session resumption: apply the initial state derived from replay.
     const init = deps.initialState;
@@ -420,8 +454,10 @@ export class ContextEngine {
       for (;;) {
         // Both LLM and Environment handle errors internally and guarantee a complete, closed
         // output with no thrown exceptions; the engine doesn't handle exceptions —
-        // it decides retry/resend purely from `outcome`.
-        turn = yield* this.runTurn(attemptInput, approve, signal, thinkingLevel);
+        // it decides retry/resend purely from `outcome`. The retry count so far is threaded
+        // in so the turn's request_end can announce the planned backoff (retry_in_ms) —
+        // the counter lives in this loop while the event is built inside the turn.
+        turn = yield* this.runTurn(attemptInput, approve, signal, thinkingLevel, reconnects);
 
         // User interruption (the LLM stream was aborted, outcome=aborted, or `signal` fired
         // during tool execution): stop and hand control back to the user.
@@ -430,9 +466,12 @@ export class ContextEngine {
           yield* this.emitAbort("aborted by user");
           return;
         }
-        // Non-retryable error (auth/parameter etc.): stop and hand control back to the user;
-        // the failure reason is written to the abort event / Trace.
-        if (turn.outcome.status === "failed") {
+        // Non-retryable error: stop and hand control back to the user; the failure reason
+        // is written to the abort event / Trace. `auth` (credentials rejected) behaves
+        // exactly like `failed` here — direct stop, never the retry loop below (the
+        // classifier already refuses to mark auth retryable; this branch is the second
+        // belt) — hosts read the status from this turn's request_end to gate input.
+        if (turn.outcome.status === "failed" || turn.outcome.status === "auth") {
           this.pendingCarryOver = this.buildCarryOver(attemptInput, turn);
           yield* this.emitAbort(`llm request error: ${turn.outcome.message ?? "unknown"}`);
           return;
@@ -587,26 +626,77 @@ export class ContextEngine {
   }
 
   /**
-   * Linear backoff before a reconnect retry (base × retry number, numbering starts at 1);
-   * returns false if the user interrupts during the backoff, so the caller can proceed to
-   * interruption cleanup.
+   * The wait the engine WILL apply before retrying this failure in-run, or undefined when
+   * it won't (a non-retryable status, or `reconnectsSoFar` has reached `cap` — an abort
+   * follows instead). Announced on the failure's `request_end` as `retry_in_ms` so the
+   * frontend can render a live countdown; shares `reconnectDelayMs` with `backoff`, so the
+   * announced wait and the actual sleep cannot drift. The caps differ by loop: the turn
+   * loop passes `maxReconnects`, the compaction loop `compactionMaxReconnects`.
+   */
+  private plannedRetryDelayMs(
+    status: StopReason,
+    reconnectsSoFar: number,
+    cap: number,
+  ): number | undefined {
+    if (status !== "timeout" && status !== "malformed") return undefined;
+    if (reconnectsSoFar >= cap) return undefined;
+    return reconnectDelayMs(
+      this.reconnectBackoffMs,
+      this.reconnectBackoffMaxMs,
+      reconnectsSoFar + 1,
+    );
+  }
+
+  /** Resolves the in-progress backoff wait early ("retry now"); null when no wait is in progress. */
+  private wakeBackoff: (() => void) | null = null;
+
+  /**
+   * Skips the in-progress reconnect backoff and fires the next retry immediately (the
+   * user's "retry now" on the countdown): resolves the current wait as if its timer had
+   * elapsed — the attempt counter is untouched, so the skipped wait never consumes an
+   * extra attempt. Returns false (a benign no-op) when no reconnect wait is in progress;
+   * idempotent under races — the wait settles exactly once whether the timer, a user
+   * abort, or this skip lands first. Wakes whichever loop is waiting (the turn loop's
+   * reconnect backoff, or a compaction retry's). Mirrors `steer` as the second
+   * mid-run nudge hosts can reach through the Session.
+   */
+  skipReconnectWait(): boolean {
+    const wake = this.wakeBackoff;
+    if (!wake) return false;
+    wake();
+    return true;
+  }
+
+  /**
+   * Exponential backoff before a reconnect retry (`reconnectDelayMs` of the configured
+   * base/ceiling; attempt numbering starts at 1); returns false if the user interrupts
+   * during the backoff — the abort listener also fires mid-wait, so even a 30s ceiling
+   * wait hands control back immediately — letting the caller proceed to interruption
+   * cleanup. A "retry now" skip (`skipReconnectWait`) resolves the wait early as true,
+   * proceeding straight to the retry.
    */
   private backoff(attempt: number, signal?: AbortSignal): Promise<boolean> {
-    const ms = this.reconnectBackoffMs * attempt;
+    const ms = reconnectDelayMs(this.reconnectBackoffMs, this.reconnectBackoffMaxMs, attempt);
     return new Promise<boolean>((resolve) => {
       if (signal?.aborted) {
         resolve(false);
         return;
       }
-      const onAbort = (): void => {
+      // Settles exactly once: the timer, a user abort, and a "retry now" skip may race —
+      // whichever lands first wins, the rest become no-ops (no double-fire).
+      let settled = false;
+      const settle = (proceed: boolean): void => {
+        if (settled) return;
+        settled = true;
         clearTimeout(timer);
-        resolve(false);
-      };
-      const timer = setTimeout(() => {
         signal?.removeEventListener("abort", onAbort);
-        resolve(true);
-      }, ms);
+        this.wakeBackoff = null;
+        resolve(proceed);
+      };
+      const onAbort = (): void => settle(false);
+      const timer = setTimeout(() => settle(true), ms);
       signal?.addEventListener("abort", onAbort, { once: true });
+      this.wakeBackoff = () => settle(true);
     });
   }
 
@@ -623,6 +713,8 @@ export class ContextEngine {
     approve: ApproveFn,
     signal?: AbortSignal,
     thinkingLevel?: ThinkingLevelName,
+    /** Retries already performed for this turn (from the caller's reconnect loop): lets request_end announce the NEXT attempt's planned backoff. */
+    reconnectsSoFar = 0,
   ): AsyncGenerator<OmniMessage, TurnResult> {
     const queue = new MergeQueue();
     // Tool outputs are collected in **completion order** (for streaming yield to the frontend);
@@ -659,7 +751,16 @@ export class ContextEngine {
           const res = await gen.next();
           if (res.done) {
             outcome = res.value;
-            const stopEvt = requestEnd(outcome.status);
+            // Non-completed outcomes carry the failure detail onto the event: a retried
+            // request never produces an abort, so this is the only place observability
+            // (the errors panel) can learn the real reason (e.g. a quota code). When the
+            // engine will retry in-run, the planned backoff rides along as retry_in_ms
+            // (the frontend's live countdown); absent on final failures and completions.
+            const stopEvt = requestEnd(
+              outcome.status,
+              outcome.message,
+              this.plannedRetryDelayMs(outcome.status, reconnectsSoFar, this.maxReconnects),
+            );
             queue.push(stopEvt);
             await this.write(stopEvt);
             break;
@@ -687,7 +788,7 @@ export class ContextEngine {
           // was never committed to history by AgentHub, so there's nothing to pair. This turn
           // must then end with a non-completed outcome (only interruption closure produces such
           // a tool_call): timeout/malformed is cleaned up by reconnect resending the flatten
-          // carry-over, failed/aborted exits directly.
+          // carry-over, failed/auth/aborted exits directly.
           if (isCompleteModelMessage(msg) && msg.payload.type === "tool_call") {
             const tc = msg as OmniMessage<ToolCallPayload>;
             if (tc.payload.stop_reason !== "completed") continue;
@@ -909,9 +1010,10 @@ export class ContextEngine {
    * compaction request's streamed output is not pushed to the Human output stream (it emits
    * paired compaction events, plus the compaction request's `token_usage` — positioned between
    * the two events, so the frontend can count compaction cost into its stats), but it is written
-   * to the old Trace. timeout/malformed reconnect via the existing retry mechanism, collapsing
-   * to failed once retries are exhausted; on failure/abort, the original context and Trace index
-   * are kept — it does not fall back to discard.
+   * to the old Trace. timeout/malformed reconnect via the existing retry mechanism under the
+   * compaction-specific cap (`compactionMaxReconnects`, tighter than the turn loop's ladder),
+   * collapsing to failed once retries are exhausted; on failure/abort, the original context and
+   * Trace index are kept — it does not fall back to discard.
    * Docs: /docs/agent-loop § "Compaction".
    */
   private async *summarizeContext(
@@ -937,7 +1039,7 @@ export class ContextEngine {
         yield* this.emitCompactionEnd(reason, "summarize", "aborted");
         return { status: "aborted" };
       }
-      const attempt = await this.runCompactionRequest(input, signal);
+      const attempt = await this.runCompactionRequest(input, signal, reconnects);
       if (attempt.status === "completed") {
         // The compaction request's token_usage is pushed to the Human output stream (already
         // written to Trace in runCompactionRequest, so here it's only yielded, not rewritten);
@@ -951,13 +1053,24 @@ export class ContextEngine {
         await this.startNewContext();
         return { status: "completed", summary };
       }
-      if (attempt.status === "aborted" || attempt.status === "failed") {
-        yield* this.emitCompactionEnd(reason, "summarize", attempt.status);
-        return { status: attempt.status };
+      if (attempt.status === "aborted") {
+        yield* this.emitCompactionEnd(reason, "summarize", "aborted");
+        return { status: "aborted" };
+      }
+      if (attempt.status === "failed" || attempt.status === "auth") {
+        // `auth` folds into `failed` here: the compaction event pair keeps its
+        // completed/failed/aborted set, the original context is kept, and the host learns
+        // about the credential problem from the request's own terminal status (a turn-loop
+        // request will surface it; the compaction request_end is Trace-only).
+        yield* this.emitCompactionEnd(reason, "summarize", "failed");
+        return { status: "failed" };
       }
       // timeout / malformed: retried via reconnect. The compaction request was never committed
-      // by AgentHub (case B), so the original input is resent unchanged.
-      if (reconnects >= this.maxReconnects) {
+      // by AgentHub (case B), so the original input is resent unchanged. Compaction uses its
+      // own, tighter cap (not the shared maxReconnects): a failed compaction keeps the
+      // original context and retries on the next trigger, so failing fast beats holding the
+      // session through the full exponential ladder.
+      if (reconnects >= this.compactionMaxReconnects) {
         yield* this.emitCompactionEnd(reason, "summarize", "failed");
         return { status: "failed" };
       }
@@ -981,6 +1094,8 @@ export class ContextEngine {
   private async runCompactionRequest(
     input: OmniMessage[],
     signal?: AbortSignal,
+    /** Retries already performed by the compaction loop (its request_end announces the next planned backoff too). */
+    reconnectsSoFar = 0,
   ): Promise<{ status: StopReason; text: string; usage: OmniMessage | null }> {
     // The compaction request is itself an ordinary Request, emitting paired request events —
     // written to the (old) Trace only, not pushed to the stream, keeping the compaction process
@@ -995,7 +1110,22 @@ export class ContextEngine {
     for (;;) {
       const res = await gen.next();
       if (res.done) {
-        await this.write(requestEnd(res.value.status));
+        // Same failure-detail + planned-backoff pass-through as the turn loop's
+        // request_end, under the compaction cap. Compaction request events are written to
+        // the old Trace only (never streamed), so retry_in_ms lands in the Trace record —
+        // no live countdown renders for compaction; the frontend only sees the
+        // compaction event pair.
+        await this.write(
+          requestEnd(
+            res.value.status,
+            res.value.message,
+            this.plannedRetryDelayMs(
+              res.value.status,
+              reconnectsSoFar,
+              this.compactionMaxReconnects,
+            ),
+          ),
+        );
         return { status: res.value.status, text, usage };
       }
       const msg = res.value;
@@ -1061,7 +1191,7 @@ export class ContextEngine {
   /**
    * Builds the interruption resend content (carry-over, interruption cleanup)
    * based on the LLM's terminal state. Used only for the **exit** cleanup of
-   * aborted / failed (reconnect retry doesn't go through here — retry input is assembled by
+   * aborted / failed / auth (reconnect retry doesn't go through here — retry input is assembled by
    * withRetriedTurns, appending `[turn_retried]` with the failed attempt's output, distinct
    * from the user-interruption `[turn_aborted]`):
    * - Model output completed (case A, outcome=completed): AgentHub already committed an

packages/core/src/index.ts

@@ -29,7 +29,7 @@ export * from "./environment/index.js";
 export * from "./trace/index.js";
 
 // Runtime entry points
-export { ContextEngine } from "./engine/context-engine.js";
+export { ContextEngine, reconnectDelayMs } from "./engine/context-engine.js";
 export type {
   CompactAvailability,
   CompactionSettings,

packages/core/src/interfaces.ts

@@ -139,7 +139,7 @@ export interface GenerativeModelParameters {
 
 /**
  * The terminal state of an LLM request, returned as the **return value** of the `streamGenerate`
- * async generator (not a yielded message). The status values share the same five-value protocol
+ * async generator (not a yielded message). The status values share the same six-value protocol
  * as OmniMessage `stop_reason`:
  *   - `completed`: finished normally (already produced `token_usage`);
  *   - `timeout`: LLM timed out or lost connection, needs reconnect — retried by `context_engine`
@@ -147,12 +147,23 @@ export interface GenerativeModelParameters {
  *   - `malformed`: AgentHub response failed JSON parsing, needs reconnect — also retried by
  *     `context_engine`;
  *   - `aborted`: user-initiated interruption — stop and hand back to the user;
- *   - `failed`: other non-retryable errors (auth/params, etc.) — stop and hand back to the user
- *     (`message` provides the display text).
+ *   - `failed`: other non-retryable errors (params, etc.) — stop and hand back to the user
+ *     (`message` provides the display text);
+ *   - `auth`: the provider rejected the credentials (see `isAuthenticationError`) — behaves
+ *     like `failed` in the engine (direct stop, never retried), but hosts key on the status
+ *     to disable input until the model's API key is updated (only the model reference is
+ *     fixed at Session creation; credentials come from the current Project config, so a key
+ *     update lets the Session continue).
  * Docs: /docs/interfaces § "LLMOutcome semantics".
  */
 export interface LLMOutcome {
   status: StopReason;
+  /**
+   * Failure detail (`describeError` text): present on `failed` / `auth`, and on `timeout` /
+   * `malformed` when a concrete transport/provider error was caught (a plain idle timeout
+   * has none). Carried onto the `request_end` event so observability (the Cost center's
+   * errors panel) can show the real reason behind a retried request.
+   */
   message?: string;
 }
 

packages/core/src/llm/generative-model.ts

@@ -13,10 +13,12 @@
  *   3. Interruption/error handling: `finishInterrupted` first closes any open
  *      streaming segments and backfills the complete message, then the output ends — never
  *      leaking a malformed structure. This interface **never retries internally** — retryable
- *      errors (network/timeout/429/5xx, see `isRetryableError`) end with `timeout`; AgentHub
- *      JSON parse errors end with `malformed`; both are handed to `context_engine` to reconnect
- *      within the same run. User interruption ends with `aborted`; non-retryable errors
- *      (auth/parameters) end with `failed`.
+ *      errors (network/transport drops, timeouts, 429/5xx, provider quota exhaustion, see
+ *      `isRetryableError`) end with `timeout`; AgentHub JSON parse errors end with `malformed`;
+ *      both are handed to `context_engine` to reconnect within the same run. User interruption
+ *      ends with `aborted`; non-retryable errors (parameters etc.) end with `failed`;
+ *      credentials failures end with their own terminal status `auth` (see
+ *      `isAuthenticationError`) — same engine behavior as `failed`, but hosts key on it.
  *
  * `context_engine` only consumes OmniMessage; all Uni* protocol details are encapsulated here.
  * Docs: /docs/interfaces § "The built-in implementation: GenerativeModel".
@@ -651,7 +653,7 @@ export class EventTranslator {
   }
 
   /**
-   * Converts an AgentHub finish_reason into an OmniMessage stop_reason (the five-value protocol).
+   * Converts an AgentHub finish_reason into an OmniMessage stop_reason (the six-value protocol).
    * "stop", "tool_call", and null are treated as completed; length or unknown reasons map to failed.
    */
   private omniStopReason(): StopReason {
@@ -794,11 +796,138 @@ export function isIncompleteStreamError(error: unknown): boolean {
   return false;
 }
 
+/**
+ * Retryable network/transport error codes: the classic Node socket codes plus undici's
+ * transport failures (`UND_ERR_*`). Node's `fetch` surfaces a dropped connection as
+ * `TypeError: terminated` whose `cause` carries the real code (e.g. "other side closed"
+ * with `code: "UND_ERR_SOCKET"`) — a transport disconnect, not a provider verdict, so it
+ * reconnects like any network drop.
+ */
+const RETRYABLE_NETWORK_CODES: ReadonlySet<string> = new Set([
+  "ECONNRESET",
+  "ECONNREFUSED",
+  "ETIMEDOUT",
+  "EPIPE",
+  "EAI_AGAIN",
+  "ENOTFOUND",
+  "ECONNABORTED",
+  // undici (Node fetch) transport failures
+  "UND_ERR_SOCKET",
+  "UND_ERR_CONNECT_TIMEOUT",
+  "UND_ERR_HEADERS_TIMEOUT",
+  "UND_ERR_BODY_TIMEOUT",
+]);
+
+/** Provider "quota / subscription exhausted" error codes (OpenAI-compatible bodies). */
+const QUOTA_CODES: ReadonlySet<string> = new Set(["insufficient_user_quota", "insufficient_quota"]);
+
+/** Credentials/authentication error codes and types (OpenAI-compatible bodies / SDK errors). */
+const AUTH_CODES: ReadonlySet<string> = new Set([
+  "invalid_api_key",
+  "authentication_error",
+  "unauthorized",
+]);
+
+/**
+ * Walks the error's `cause` chain (cycle-safe, same approach as `describeError`) applying
+ * `probe` to each level; true as soon as one level matches. Higher layers routinely wrap the
+ * real failure (Node fetch puts the transport error on `cause`), so single-level checks miss it.
+ */
+function anyInCauseChain(error: unknown, probe: (level: object) => boolean): boolean {
+  const seen = new Set<unknown>();
+  let cur: unknown = error;
+  while (cur != null && typeof cur === "object" && !seen.has(cur)) {
+    seen.add(cur);
+    if (probe(cur)) return true;
+    cur = (cur as { cause?: unknown }).cause;
+  }
+  return false;
+}
+
+/**
+ * Provider error-code signals at one level of an error: `code` on the error itself (the
+ * OpenAI SDK exposes the parsed body's code directly), plus the parsed body under `error` —
+ * both the OpenAI shape (`err.error.code`) and the Anthropic SDK shape, whose `error`
+ * property holds the whole response body (`err.error.error.code`). With `includeType`, the
+ * matching `type` fields are collected too (auth signals ride on `type` in some bodies).
+ */
+function providerSignals(level: object, includeType: boolean): string[] {
+  const err = level as {
+    code?: unknown;
+    type?: unknown;
+    error?: { code?: unknown; type?: unknown; error?: { code?: unknown; type?: unknown } };
+  };
+  const body = typeof err.error === "object" && err.error !== null ? err.error : undefined;
+  const inner = typeof body?.error === "object" && body.error !== null ? body.error : undefined;
+  const signals = [err.code, body?.code, inner?.code];
+  if (includeType) signals.push(err.type, body?.type, inner?.type);
+  return signals.filter((v): v is string => typeof v === "string");
+}
+
+/**
+ * Determines whether an error is the provider's "quota / subscription exhausted" rejection
+ * (e.g. a gateway 403 whose body carries `insufficient_user_quota`). Topping up or the
+ * billing cycle rolling over fixes it without touching the Session, so it is transient and
+ * goes through the engine's reconnect flow like a network problem. Deliberately tight:
+ * only a payment/permission status (402/403 — or no status at all) combined with a known
+ * quota code (own, `cause` chain, or parsed provider body), or a 403 whose message names
+ * quota/subscription; a plain 403 (permission denied) stays non-retryable.
+ */
+export function isQuotaExhaustedError(error: unknown): boolean {
+  if (error == null || typeof error !== "object") return false;
+  // Belt to the auth-first ordering in isRetryableError: an error carrying a definitive
+  // auth signal is never quota, no matter what its message says — a 403 whose body codes
+  // `invalid_api_key` but whose message mentions a subscription must not classify as
+  // retryable-quota and burn through the reconnect ladder against a dead credential.
+  if (isAuthenticationError(error)) return false;
+  const err = error as { status?: number; statusCode?: number; message?: unknown };
+  const status = err.status ?? err.statusCode;
+  // Any other status keeps its own classification (401 auth, 429 rate limit, …).
+  if (typeof status === "number" && status !== 402 && status !== 403) return false;
+  if (
+    anyInCauseChain(error, (level) => providerSignals(level, false).some((c) => QUOTA_CODES.has(c)))
+  ) {
+    return true;
+  }
+  // 额度 covers Chinese-language gateway messages such as 订阅额度不足 ("subscription quota
+  // exhausted") that carry no machine-readable code.
+  return (
+    status === 403 &&
+    typeof err.message === "string" &&
+    /quota|subscription|额度/i.test(err.message)
+  );
+}
+
+/**
+ * Determines whether an error is a credentials/authentication failure — the one class an
+ * in-run retry can never fix: the request keeps going out with the same dead credential.
+ * Only the model REFERENCE is fixed at Session creation; the credential is read from the
+ * current Project config whenever the Session loads, so the fix is updating that model's
+ * API key (Models page) — after which the Session can continue — not retrying. Signals:
+ * HTTP 401 (any), a known auth code/type (on the error, its `cause` chain, or the parsed
+ * provider body), or the SDK error class name `AuthenticationError` (OpenAI / Anthropic
+ * SDKs). A bare 403 carries no such signal and stays a plain failure (usually
+ * permission/policy, not credentials).
+ */
+export function isAuthenticationError(error: unknown): boolean {
+  return anyInCauseChain(error, (level) => {
+    const err = level as { status?: unknown; statusCode?: unknown; name?: unknown };
+    if (err.status === 401 || err.statusCode === 401) return true;
+    if (providerSignals(level, true).some((c) => AUTH_CODES.has(c))) return true;
+    const name = typeof err.name === "string" ? err.name : "";
+    return name === "AuthenticationError" || level.constructor?.name === "AuthenticationError";
+  });
+}
+
 /**
  * Determines whether an error is retryable.
  *
- * Retryable: network errors, timeouts, connection reset, HTTP 429 / 5xx.
- * Not retryable: HTTP 4xx auth/parameter errors (401/403/400/404, etc.).
+ * Retryable: network/transport errors (including undici's `UND_ERR_*` on the `cause` chain),
+ * timeouts, connection reset, HTTP 429 / 5xx, and provider quota/subscription exhaustion
+ * (see `isQuotaExhaustedError`).
+ * Not retryable: authentication errors (checked FIRST — a definitive credential signal is
+ * never reclassified by the heuristics below, see `isAuthenticationError`) and other HTTP
+ * 4xx auth/parameter errors (401/403/400/404, etc.).
  * JSON parse errors are classified separately as `malformed` by `isMalformedJsonParseError`.
  *
  * Since AgentHub doesn't guarantee the shape of error objects, this uses a lenient check: first
@@ -808,10 +937,15 @@ export function isIncompleteStreamError(error: unknown): boolean {
 export function isRetryableError(error: unknown): boolean {
   if (error == null) return false;
 
+  // 0. Authentication is definitive and terminal: retrying a dead credential can never
+  //    succeed, and no heuristic below (quota keywords, message vocabulary) may reclassify
+  //    it — this ordering is the only thing keeping a dead credential out of the retry
+  //    ladder now that quota errors are deliberately retryable.
+  if (isAuthenticationError(error)) return false;
+
   const err = error as {
     status?: number;
     statusCode?: number;
-    code?: string;
     name?: string;
     message?: string;
   };
@@ -821,31 +955,43 @@ export function isRetryableError(error: unknown): boolean {
   if (typeof status === "number") {
     if (status === 429 || status === 408) return true; // Rate limited / request timeout (transient)
     if (status >= 500 && status <= 599) return true; // Server error
+    // Provider quota/subscription exhaustion (402/403 + explicit signals): transient —
+    // checked before the blanket 4xx rule below would discard it.
+    if (isQuotaExhaustedError(error)) return true;
     if (status >= 400 && status <= 499) return false; // Other 4xx auth/parameter errors, not retryable
   }
 
-  // 2. Network-layer error codes.
-  const retryableCodes = new Set([
-    "ECONNRESET",
-    "ECONNREFUSED",
-    "ETIMEDOUT",
-    "EPIPE",
-    "EAI_AGAIN",
-    "ENOTFOUND",
-    "ECONNABORTED",
-  ]);
-  if (err.code && retryableCodes.has(err.code)) return true;
+  // 2. Network/transport error codes, probing the `cause` chain (Node fetch wraps the real
+  //    transport failure as `TypeError: terminated` with the code on `cause`, see describeError).
+  if (
+    anyInCauseChain(error, (level) => {
+      const code = (level as { code?: unknown }).code;
+      return typeof code === "string" && RETRYABLE_NETWORK_CODES.has(code);
+    })
+  ) {
+    return true;
+  }
 
-  // 3. Error name / message keywords (timeout, network, rate limit).
+  // 2b. Status-less quota signals (some gateways surface only the provider code).
+  if (isQuotaExhaustedError(error)) return true;
+
+  // 3. Error name / message keywords (timeout, network, rate limit, undici disconnects).
   if (err.name === "AbortError") return false; // User interruption, not retryable
   const text = `${err.name ?? ""} ${err.message ?? ""}`.toLowerCase();
   if (
-    /timeout|timed out|network|socket hang up|econnreset|connection reset|too many requests|rate limit|temporarily unavailable|503|502|504/.test(
+    /timeout|timed out|network|socket hang up|econnreset|connection reset|other side closed|fetch failed|too many requests|rate limit|temporarily unavailable|503|502|504/.test(
       text,
     )
   ) {
     return true;
   }
+  // `terminated` alone is ambiguous — providers use the word in verdict copy too ("Request
+  // terminated by content filter"), which must NOT retry — so it only counts alongside
+  // transport vocabulary. The real socket case is already caught by the cause-chain
+  // `UND_ERR_*` codes in step 2; this is a last-resort fallback for cause-less wrappers.
+  if (/terminated/.test(text) && /socket|connection|closed/.test(text)) {
+    return true;
+  }
 
   return false;
 }
@@ -923,14 +1069,18 @@ export class GenerativeModel implements LLMInterface {
    * and the terminal state is then returned as `LLMOutcome`:
    *   - **Normal completion**: `finish()` closes out and produces `token_usage` (usage is only
    *     produced in this case) → `completed`;
-   *   - **Idle timeout / network drop** (retryable errors like network/429/5xx):
-   *     `finishInterrupted("timeout")` closes out, produces no usage → `timeout`, reconnected by
+   *   - **Idle timeout / network drop** (retryable errors like network/429/5xx/quota):
+   *     `finishInterrupted("timeout")` closes out, produces no usage → `timeout` (carrying
+   *     the error detail as `message` when a concrete error was caught), reconnected by
    *     `context_engine` within the same run;
    *   - **AgentHub JSON parse error**: `finishInterrupted("malformed")` closes out, produces no
    *     usage → `malformed`, likewise reconnected by `context_engine` within the same run;
    *   - **User interruption**: `finishInterrupted("aborted")` closes out, produces no usage →
    *     `aborted`;
-   *   - **Other non-retryable errors** (auth/parameters etc.): `finishInterrupted("failed")`
+   *   - **Credentials failure**: `finishInterrupted("auth")` closes out, produces no usage →
+   *     `auth` (carrying `message`) — the engine stops exactly like `failed`, hosts key on
+   *     the status to gate input until the model's API key is updated;
+   *   - **Other non-retryable errors** (parameters etc.): `finishInterrupted("failed")`
    *     closes out, produces no usage → `failed` (carrying `message`), handed to
    *     `context_engine` to stop and return control to the user.
    *
@@ -1037,15 +1187,23 @@ export class GenerativeModel implements LLMInterface {
           status: "malformed",
           message: describeError(error),
         };
+      } else if (isAuthenticationError(error)) {
+        // Credentials failure: its own terminal status so hosts can tell "update this
+        // model's API key, then this Session can continue" (only the model reference is
+        // fixed at Session creation; the credential is read from the current Project
+        // config on load) apart from a one-off failure. Checked before the retryable
+        // branch as a belt — isRetryableError itself already refuses auth signals.
+        outcome = { status: "auth", message: describeError(error) };
       } else if (isRetryableError(error)) {
-        outcome = { status: "timeout" }; // Network drop/network error -> needs reconnection
+        // Network drop / transient provider rejection -> needs reconnection. The detail
+        // (e.g. "403 … (insufficient_user_quota)") rides on the outcome so observability
+        // (request_end -> the Cost center's errors panel) shows the real reason behind a
+        // retried request, not just "timeout".
+        outcome = { status: "timeout", message: describeError(error) };
       } else if ((error as { name?: string })?.name === "AbortError") {
         outcome = { status: "aborted" }; // Fallback: an unexpected abort (neither timeout nor user)
       } else {
-        outcome = {
-          status: "failed",
-          message: describeError(error),
-        };
+        outcome = { status: "failed", message: describeError(error) };
       }
     } finally {
       clearTimer();

packages/core/src/llm/index.ts

@@ -15,6 +15,8 @@ export {
   isMalformedJsonParseError,
   isIncompleteStreamError,
   isRetryableError,
+  isQuotaExhaustedError,
+  isAuthenticationError,
   mapThinkingLevel,
   toolDefinitionsToSchemas,
   buildUniConfig,

packages/core/src/omnimessage/builders.ts

@@ -269,9 +269,23 @@ export function requestBegin(): OmniMessage<RequestBeginPayload> {
   return event({ type: "request_begin" });
 }
 
-/** request end event: carries the terminal state (`completed` means this turn was already committed to AgentHub). */
-export function requestEnd(status: StopReason): OmniMessage<RequestEndPayload> {
-  return event({ type: "request_end", status });
+/**
+ * request end event: carries the terminal state (`completed` means this turn was already
+ * committed to AgentHub), plus — on non-completed statuses — the failure detail (from
+ * LLMOutcome.message) and, when the engine will retry in-run, the planned backoff wait
+ * (`retry_in_ms`, rendered by the Web App as a live countdown).
+ */
+export function requestEnd(
+  status: StopReason,
+  message?: string,
+  retryInMs?: number,
+): OmniMessage<RequestEndPayload> {
+  return event({
+    type: "request_end",
+    status,
+    ...(message !== undefined ? { message } : {}),
+    ...(retryInMs !== undefined ? { retry_in_ms: retryInMs } : {}),
+  });
 }
 
 /** compaction begin event: carries the trigger reason, mode, current context usage, and cumulative Session turn count. */

packages/core/src/omnimessage/types.ts

@@ -23,7 +23,7 @@ export type OmniMessageType = "session_meta" | "model_msg" | "event_msg";
 export type Role = "user" | "assistant";
 
 /**
- * The reason a model response or message generation ended. Only five protocol values are
+ * The reason a model response or message generation ended. Only six protocol values are
  * allowed:
  *   - `completed`: finished normally, including completed text, thinking, tool requests, or
  *     tool output;
@@ -31,10 +31,15 @@ export type Role = "user" | "assistant";
  *   - `aborted`: user-initiated interruption or cancellation;
  *   - `timeout`: LLM request timed out;
  *   - `malformed`: the LLM response was malformed (e.g. AgentHub JSON parsing exception).
- *     Only LLM timeout / malformed trigger a context_engine reconnect.
+ *     Only LLM timeout / malformed trigger a context_engine reconnect;
+ *   - `auth`: the LLM rejected the credentials (see `isAuthenticationError`) — a
+ *     `failed`-shaped stop that no in-run retry can fix; hosts disable input until the
+ *     model's API key is updated (only the model reference is fixed at Session creation —
+ *     credentials come from the current Project config), after which the Session
+ *     continues.
  * Docs: /docs/omni-message § "stop_reason".
  */
-export type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed";
+export type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed" | "auth";
 
 /** The event phase of a streaming fragment. `stop` marks the end of a fragment and usually carries no incremental content. */
 export type StreamEventType = "start" | "delta" | "stop";
@@ -277,8 +282,25 @@ export interface RequestBeginPayload {
 
 export interface RequestEndPayload {
   type: "request_end";
-  /** Terminal state of this Request (reuses the five StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome). */
+  /** Terminal state of this Request (reuses the six StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome; `auth` is the credentials-failure signal hosts key on). */
   status: StopReason;
+  /**
+   * Failure detail from `LLMOutcome.message`, present only on non-completed statuses: the
+   * real reason behind a retried/failed Request (e.g. `403 … (insufficient_user_quota)`),
+   * for observability — the server's error records / Cost center read it here because a
+   * retried request never produces an abort event. Additive: old Traces without it replay
+   * unchanged.
+   */
+  message?: string;
+  /**
+   * Planned in-run retry wait (ms) — present ONLY when the engine will retry this failure
+   * within the same run (status `timeout`/`malformed` with attempts remaining under the
+   * applicable cap). Computed by the same formula as the actual backoff sleep
+   * (`reconnectDelayMs`), so the announced wait and the real one cannot drift; the Web App
+   * renders it as a live countdown to the next attempt. Absent on final failures (an abort
+   * follows instead) and on completed requests. Additive: old Traces replay unchanged.
+   */
+  retry_in_ms?: number;
 }
 
 /** Compaction trigger reason: context threshold / turn-count threshold / user-initiated request. */
@@ -291,7 +313,7 @@ export type CompactionMode = "summarize" | "discard";
  * Compaction boundary event: the compaction process exposes
  * only this event pair to Human, produced **in pairs** by `context_engine`. Both `reason` and
  * `mode` are carried on both events, for stateless frontend rendering; `status` reuses the
- * five-value `StopReason` protocol (compaction converges to a terminal state, taking
+ * six-value `StopReason` protocol (compaction converges to a terminal state, taking
  * `completed` / `failed` / `aborted` in practice — `timeout` / `malformed` are handled internally
  * by the compaction request's existing retry mechanism, collapsing to `failed` once retries are
  * exhausted).

packages/core/src/session.ts

@@ -200,6 +200,18 @@ export class Session {
     return this.engine.steer(text);
   }
 
+  /**
+   * Skips the in-progress reconnect backoff and fires the next retry immediately (the
+   * user's "retry now" on the reconnect countdown): the attempt counter is unchanged —
+   * the skipped wait does not consume an extra attempt. Returns false (a benign no-op)
+   * when no reconnect wait is in progress; idempotent under races with the timer/abort.
+   * Mirrors `steer` as a mid-run nudge (no message of its own — the effect surfaces as
+   * the next `request_begin` arriving early).
+   */
+  skipReconnectWait(): boolean {
+    return this.engine.skipReconnectWait();
+  }
+
   /**
    * User-initiated request to compact context (e.g. a CLI command): reuses the automatic
    * compaction flow but skips the threshold check (reason=manual). Only callable at Task

packages/core/src/state/default-config.ts

@@ -112,7 +112,7 @@ Communicate with the user precisely and concisely, yet with warmth. Do not repea
 # System markers
 Some messages contain system-synthesized blocks written as \`[tag]...[/tag]\`, not user text to answer directly:
 - \`[turn_aborted]\`: the previous round was interrupted. Inside are the original request, your partial thinking/text, and the tool calls already issued with their results. Continue from where it left off; do not re-run tools whose results are already included.
-- \`[turn_retried]\`: the previous attempt of this round failed on a transport error (timeout or malformed response) — the user did NOT interrupt — and this request is the automatic retry. Inside are your partial thinking/text and the tool calls already executed with their results. Continue from them; do not re-run tools whose results are already included.
+- \`[turn_retried]\`: the previous attempt of this round failed on a transient error (timeout, disconnect, malformed response, or a temporary provider error) — the user did NOT interrupt — and this request is the automatic retry. Inside are your partial thinking/text and the tool calls already executed with their results. Continue from them; do not re-run tools whose results are already included.
 - \`[context_summary]\`: earlier conversation was compacted. This summary replaced the raw transcript and is its only record; treat it as established context and continue the task from it.
 - \`[user_steering]\`: a user message sent while you were still working, delivered between turns alongside tool results. It is not a new task: incorporate it immediately and adjust course within the current task.