diff --git a/packages/core/src/engine/context-engine.ts b/packages/core/src/engine/context-engine.ts
index 0287494..8812916 100644
--- a/packages/core/src/engine/context-engine.ts
+++ b/packages/core/src/engine/context-engine.ts
@@ -152,10 +152,29 @@ export interface ContextEngineDeps {
initialState?: EngineInitialState;
/** Maximum LLM turns for a single Task. Defaults to 100; -1 removes the cap. */
maxTurns?: number;
- /** Maximum automatic retries for LLM timeout/reconnect within a single run. Defaults to 2. */
+ /**
+ * Maximum automatic retries for LLM timeout/reconnect within a single run. Defaults
+ * to 5: with the default backoff (250ms base, 30s ceiling) that is 250+500+1000+2000+
+ * 4000 ≈ 7.75s of total patience — the first attempts stay fast enough for transport
+ * blips while the tail still gives transient provider rejections a window.
+ */
maxReconnects?: number;
- /** Linear backoff base (ms) before each reconnect retry; actual backoff = base × retry number. Defaults to 250. */
+ /**
+ * Exponential backoff base (ms): the wait before reconnect retry N is
+ * `base × 2^(N−1)`, capped at `reconnectBackoffMaxMs` (see reconnectDelayMs).
+ * Defaults to 250.
+ */
reconnectBackoffMs?: number;
+ /** Ceiling (ms) for a single reconnect backoff wait. Defaults to 30000. */
+ reconnectBackoffMaxMs?: number;
+ /**
+ * Maximum retries for a failing compaction request (instead of the shared
+ * `maxReconnects`). Compaction failure already has graceful semantics — the original
+ * context is kept and compaction retries on the next trigger — so it fails fast
+ * (~1.75s of backoff) rather than stalling the session through the full reconnect
+ * ladder. Defaults to 3.
+ */
+ compactionMaxReconnects?: number;
/**
* Creates a new LLM object after compaction (a fresh model context); the argument is the
* current Session cumulative token counts, for the new object to carry forward
@@ -232,10 +251,23 @@ class MergeQueue {
}
}
+/**
+ * Delay before reconnect attempt N (1-based): exponential growth from `base` with a hard
+ * ceiling `max` — `min(base × 2^(N−1), max)`. With the defaults (250ms base, 30s ceiling,
+ * 5 reconnects) the ladder is 250, 500, 1000, 2000, 4000 ≈ 7.75s of total patience: one
+ * shared schedule serves every retryable class, growing toward the slower ones (transient
+ * provider quota errors) while the first steps stay as fast as a transport blip needs.
+ */
+export function reconnectDelayMs(base: number, max: number, attempt: number): number {
+ return Math.min(base * 2 ** (attempt - 1), max);
+}
+
export class ContextEngine {
private readonly maxTurns: number;
private readonly maxReconnects: number;
private readonly reconnectBackoffMs: number;
+ private readonly reconnectBackoffMaxMs: number;
+ private readonly compactionMaxReconnects: number;
/** Interruption cleanup: content to resend generated when the previous run was aborted, held on the engine across runs. */
private pendingCarryOver: OmniMessage[] = [];
/** Current LLM object; swapped for a new one created by `createLLM` after a successful compaction (a fresh model context). */
@@ -274,8 +306,10 @@ export class ContextEngine {
constructor(private readonly deps: ContextEngineDeps) {
this.maxTurns = deps.maxTurns ?? 100;
- this.maxReconnects = deps.maxReconnects ?? 2;
+ this.maxReconnects = deps.maxReconnects ?? 5;
this.reconnectBackoffMs = deps.reconnectBackoffMs ?? 250;
+ this.reconnectBackoffMaxMs = deps.reconnectBackoffMaxMs ?? 30_000;
+ this.compactionMaxReconnects = deps.compactionMaxReconnects ?? 3;
this.llm = deps.llm;
// Session resumption: apply the initial state derived from replay.
const init = deps.initialState;
@@ -420,8 +454,10 @@ export class ContextEngine {
for (;;) {
// Both LLM and Environment handle errors internally and guarantee a complete, closed
// output with no thrown exceptions; the engine doesn't handle exceptions —
- // it decides retry/resend purely from `outcome`.
- turn = yield* this.runTurn(attemptInput, approve, signal, thinkingLevel);
+ // it decides retry/resend purely from `outcome`. The retry count so far is threaded
+ // in so the turn's request_end can announce the planned backoff (retry_in_ms) —
+ // the counter lives in this loop while the event is built inside the turn.
+ turn = yield* this.runTurn(attemptInput, approve, signal, thinkingLevel, reconnects);
// User interruption (the LLM stream was aborted, outcome=aborted, or `signal` fired
// during tool execution): stop and hand control back to the user.
@@ -430,9 +466,12 @@ export class ContextEngine {
yield* this.emitAbort("aborted by user");
return;
}
- // Non-retryable error (auth/parameter etc.): stop and hand control back to the user;
- // the failure reason is written to the abort event / Trace.
- if (turn.outcome.status === "failed") {
+ // Non-retryable error: stop and hand control back to the user; the failure reason
+ // is written to the abort event / Trace. `auth` (credentials rejected) behaves
+ // exactly like `failed` here — direct stop, never the retry loop below (the
+ // classifier already refuses to mark auth retryable; this branch is the second
+ // belt) — hosts read the status from this turn's request_end to gate input.
+ if (turn.outcome.status === "failed" || turn.outcome.status === "auth") {
this.pendingCarryOver = this.buildCarryOver(attemptInput, turn);
yield* this.emitAbort(`llm request error: ${turn.outcome.message ?? "unknown"}`);
return;
@@ -587,26 +626,77 @@ export class ContextEngine {
}
/**
- * Linear backoff before a reconnect retry (base × retry number, numbering starts at 1);
- * returns false if the user interrupts during the backoff, so the caller can proceed to
- * interruption cleanup.
+ * The wait the engine WILL apply before retrying this failure in-run, or undefined when
+ * it won't (a non-retryable status, or `reconnectsSoFar` has reached `cap` — an abort
+ * follows instead). Announced on the failure's `request_end` as `retry_in_ms` so the
+ * frontend can render a live countdown; shares `reconnectDelayMs` with `backoff`, so the
+ * announced wait and the actual sleep cannot drift. The caps differ by loop: the turn
+ * loop passes `maxReconnects`, the compaction loop `compactionMaxReconnects`.
+ */
+ private plannedRetryDelayMs(
+ status: StopReason,
+ reconnectsSoFar: number,
+ cap: number,
+ ): number | undefined {
+ if (status !== "timeout" && status !== "malformed") return undefined;
+ if (reconnectsSoFar >= cap) return undefined;
+ return reconnectDelayMs(
+ this.reconnectBackoffMs,
+ this.reconnectBackoffMaxMs,
+ reconnectsSoFar + 1,
+ );
+ }
+
+ /** Resolves the in-progress backoff wait early ("retry now"); null when no wait is in progress. */
+ private wakeBackoff: (() => void) | null = null;
+
+ /**
+ * Skips the in-progress reconnect backoff and fires the next retry immediately (the
+ * user's "retry now" on the countdown): resolves the current wait as if its timer had
+ * elapsed — the attempt counter is untouched, so the skipped wait never consumes an
+ * extra attempt. Returns false (a benign no-op) when no reconnect wait is in progress;
+ * idempotent under races — the wait settles exactly once whether the timer, a user
+ * abort, or this skip lands first. Wakes whichever loop is waiting (the turn loop's
+ * reconnect backoff, or a compaction retry's). Mirrors `steer` as the second
+ * mid-run nudge hosts can reach through the Session.
+ */
+ skipReconnectWait(): boolean {
+ const wake = this.wakeBackoff;
+ if (!wake) return false;
+ wake();
+ return true;
+ }
+
+ /**
+ * Exponential backoff before a reconnect retry (`reconnectDelayMs` of the configured
+ * base/ceiling; attempt numbering starts at 1); returns false if the user interrupts
+ * during the backoff — the abort listener also fires mid-wait, so even a 30s ceiling
+ * wait hands control back immediately — letting the caller proceed to interruption
+ * cleanup. A "retry now" skip (`skipReconnectWait`) resolves the wait early as true,
+ * proceeding straight to the retry.
*/
private backoff(attempt: number, signal?: AbortSignal): Promise {
- const ms = this.reconnectBackoffMs * attempt;
+ const ms = reconnectDelayMs(this.reconnectBackoffMs, this.reconnectBackoffMaxMs, attempt);
return new Promise((resolve) => {
if (signal?.aborted) {
resolve(false);
return;
}
- const onAbort = (): void => {
+ // Settles exactly once: the timer, a user abort, and a "retry now" skip may race —
+ // whichever lands first wins, the rest become no-ops (no double-fire).
+ let settled = false;
+ const settle = (proceed: boolean): void => {
+ if (settled) return;
+ settled = true;
clearTimeout(timer);
- resolve(false);
- };
- const timer = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
- resolve(true);
- }, ms);
+ this.wakeBackoff = null;
+ resolve(proceed);
+ };
+ const onAbort = (): void => settle(false);
+ const timer = setTimeout(() => settle(true), ms);
signal?.addEventListener("abort", onAbort, { once: true });
+ this.wakeBackoff = () => settle(true);
});
}
@@ -623,6 +713,8 @@ export class ContextEngine {
approve: ApproveFn,
signal?: AbortSignal,
thinkingLevel?: ThinkingLevelName,
+ /** Retries already performed for this turn (from the caller's reconnect loop): lets request_end announce the NEXT attempt's planned backoff. */
+ reconnectsSoFar = 0,
): AsyncGenerator {
const queue = new MergeQueue();
// Tool outputs are collected in **completion order** (for streaming yield to the frontend);
@@ -659,7 +751,16 @@ export class ContextEngine {
const res = await gen.next();
if (res.done) {
outcome = res.value;
- const stopEvt = requestEnd(outcome.status);
+ // Non-completed outcomes carry the failure detail onto the event: a retried
+ // request never produces an abort, so this is the only place observability
+ // (the errors panel) can learn the real reason (e.g. a quota code). When the
+ // engine will retry in-run, the planned backoff rides along as retry_in_ms
+ // (the frontend's live countdown); absent on final failures and completions.
+ const stopEvt = requestEnd(
+ outcome.status,
+ outcome.message,
+ this.plannedRetryDelayMs(outcome.status, reconnectsSoFar, this.maxReconnects),
+ );
queue.push(stopEvt);
await this.write(stopEvt);
break;
@@ -687,7 +788,7 @@ export class ContextEngine {
// was never committed to history by AgentHub, so there's nothing to pair. This turn
// must then end with a non-completed outcome (only interruption closure produces such
// a tool_call): timeout/malformed is cleaned up by reconnect resending the flatten
- // carry-over, failed/aborted exits directly.
+ // carry-over, failed/auth/aborted exits directly.
if (isCompleteModelMessage(msg) && msg.payload.type === "tool_call") {
const tc = msg as OmniMessage;
if (tc.payload.stop_reason !== "completed") continue;
@@ -909,9 +1010,10 @@ export class ContextEngine {
* compaction request's streamed output is not pushed to the Human output stream (it emits
* paired compaction events, plus the compaction request's `token_usage` — positioned between
* the two events, so the frontend can count compaction cost into its stats), but it is written
- * to the old Trace. timeout/malformed reconnect via the existing retry mechanism, collapsing
- * to failed once retries are exhausted; on failure/abort, the original context and Trace index
- * are kept — it does not fall back to discard.
+ * to the old Trace. timeout/malformed reconnect via the existing retry mechanism under the
+ * compaction-specific cap (`compactionMaxReconnects`, tighter than the turn loop's ladder),
+ * collapsing to failed once retries are exhausted; on failure/abort, the original context and
+ * Trace index are kept — it does not fall back to discard.
* Docs: /docs/agent-loop § "Compaction".
*/
private async *summarizeContext(
@@ -937,7 +1039,7 @@ export class ContextEngine {
yield* this.emitCompactionEnd(reason, "summarize", "aborted");
return { status: "aborted" };
}
- const attempt = await this.runCompactionRequest(input, signal);
+ const attempt = await this.runCompactionRequest(input, signal, reconnects);
if (attempt.status === "completed") {
// The compaction request's token_usage is pushed to the Human output stream (already
// written to Trace in runCompactionRequest, so here it's only yielded, not rewritten);
@@ -951,13 +1053,24 @@ export class ContextEngine {
await this.startNewContext();
return { status: "completed", summary };
}
- if (attempt.status === "aborted" || attempt.status === "failed") {
- yield* this.emitCompactionEnd(reason, "summarize", attempt.status);
- return { status: attempt.status };
+ if (attempt.status === "aborted") {
+ yield* this.emitCompactionEnd(reason, "summarize", "aborted");
+ return { status: "aborted" };
+ }
+ if (attempt.status === "failed" || attempt.status === "auth") {
+ // `auth` folds into `failed` here: the compaction event pair keeps its
+ // completed/failed/aborted set, the original context is kept, and the host learns
+ // about the credential problem from the request's own terminal status (a turn-loop
+ // request will surface it; the compaction request_end is Trace-only).
+ yield* this.emitCompactionEnd(reason, "summarize", "failed");
+ return { status: "failed" };
}
// timeout / malformed: retried via reconnect. The compaction request was never committed
- // by AgentHub (case B), so the original input is resent unchanged.
- if (reconnects >= this.maxReconnects) {
+ // by AgentHub (case B), so the original input is resent unchanged. Compaction uses its
+ // own, tighter cap (not the shared maxReconnects): a failed compaction keeps the
+ // original context and retries on the next trigger, so failing fast beats holding the
+ // session through the full exponential ladder.
+ if (reconnects >= this.compactionMaxReconnects) {
yield* this.emitCompactionEnd(reason, "summarize", "failed");
return { status: "failed" };
}
@@ -981,6 +1094,8 @@ export class ContextEngine {
private async runCompactionRequest(
input: OmniMessage[],
signal?: AbortSignal,
+ /** Retries already performed by the compaction loop (its request_end announces the next planned backoff too). */
+ reconnectsSoFar = 0,
): Promise<{ status: StopReason; text: string; usage: OmniMessage | null }> {
// The compaction request is itself an ordinary Request, emitting paired request events —
// written to the (old) Trace only, not pushed to the stream, keeping the compaction process
@@ -995,7 +1110,22 @@ export class ContextEngine {
for (;;) {
const res = await gen.next();
if (res.done) {
- await this.write(requestEnd(res.value.status));
+ // Same failure-detail + planned-backoff pass-through as the turn loop's
+ // request_end, under the compaction cap. Compaction request events are written to
+ // the old Trace only (never streamed), so retry_in_ms lands in the Trace record —
+ // no live countdown renders for compaction; the frontend only sees the
+ // compaction event pair.
+ await this.write(
+ requestEnd(
+ res.value.status,
+ res.value.message,
+ this.plannedRetryDelayMs(
+ res.value.status,
+ reconnectsSoFar,
+ this.compactionMaxReconnects,
+ ),
+ ),
+ );
return { status: res.value.status, text, usage };
}
const msg = res.value;
@@ -1061,7 +1191,7 @@ export class ContextEngine {
/**
* Builds the interruption resend content (carry-over, interruption cleanup)
* based on the LLM's terminal state. Used only for the **exit** cleanup of
- * aborted / failed (reconnect retry doesn't go through here — retry input is assembled by
+ * aborted / failed / auth (reconnect retry doesn't go through here — retry input is assembled by
* withRetriedTurns, appending `[turn_retried]` with the failed attempt's output, distinct
* from the user-interruption `[turn_aborted]`):
* - Model output completed (case A, outcome=completed): AgentHub already committed an
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 098b450..4ddf735 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -29,7 +29,7 @@ export * from "./environment/index.js";
export * from "./trace/index.js";
// Runtime entry points
-export { ContextEngine } from "./engine/context-engine.js";
+export { ContextEngine, reconnectDelayMs } from "./engine/context-engine.js";
export type {
CompactAvailability,
CompactionSettings,
diff --git a/packages/core/src/interfaces.ts b/packages/core/src/interfaces.ts
index b418674..83f99c3 100644
--- a/packages/core/src/interfaces.ts
+++ b/packages/core/src/interfaces.ts
@@ -139,7 +139,7 @@ export interface GenerativeModelParameters {
/**
* The terminal state of an LLM request, returned as the **return value** of the `streamGenerate`
- * async generator (not a yielded message). The status values share the same five-value protocol
+ * async generator (not a yielded message). The status values share the same six-value protocol
* as OmniMessage `stop_reason`:
* - `completed`: finished normally (already produced `token_usage`);
* - `timeout`: LLM timed out or lost connection, needs reconnect — retried by `context_engine`
@@ -147,12 +147,23 @@ export interface GenerativeModelParameters {
* - `malformed`: AgentHub response failed JSON parsing, needs reconnect — also retried by
* `context_engine`;
* - `aborted`: user-initiated interruption — stop and hand back to the user;
- * - `failed`: other non-retryable errors (auth/params, etc.) — stop and hand back to the user
- * (`message` provides the display text).
+ * - `failed`: other non-retryable errors (params, etc.) — stop and hand back to the user
+ * (`message` provides the display text);
+ * - `auth`: the provider rejected the credentials (see `isAuthenticationError`) — behaves
+ * like `failed` in the engine (direct stop, never retried), but hosts key on the status
+ * to disable input until the model's API key is updated (only the model reference is
+ * fixed at Session creation; credentials come from the current Project config, so a key
+ * update lets the Session continue).
* Docs: /docs/interfaces § "LLMOutcome semantics".
*/
export interface LLMOutcome {
status: StopReason;
+ /**
+ * Failure detail (`describeError` text): present on `failed` / `auth`, and on `timeout` /
+ * `malformed` when a concrete transport/provider error was caught (a plain idle timeout
+ * has none). Carried onto the `request_end` event so observability (the Cost center's
+ * errors panel) can show the real reason behind a retried request.
+ */
message?: string;
}
diff --git a/packages/core/src/llm/generative-model.ts b/packages/core/src/llm/generative-model.ts
index bb88f46..6407b55 100644
--- a/packages/core/src/llm/generative-model.ts
+++ b/packages/core/src/llm/generative-model.ts
@@ -13,10 +13,12 @@
* 3. Interruption/error handling: `finishInterrupted` first closes any open
* streaming segments and backfills the complete message, then the output ends — never
* leaking a malformed structure. This interface **never retries internally** — retryable
- * errors (network/timeout/429/5xx, see `isRetryableError`) end with `timeout`; AgentHub
- * JSON parse errors end with `malformed`; both are handed to `context_engine` to reconnect
- * within the same run. User interruption ends with `aborted`; non-retryable errors
- * (auth/parameters) end with `failed`.
+ * errors (network/transport drops, timeouts, 429/5xx, provider quota exhaustion, see
+ * `isRetryableError`) end with `timeout`; AgentHub JSON parse errors end with `malformed`;
+ * both are handed to `context_engine` to reconnect within the same run. User interruption
+ * ends with `aborted`; non-retryable errors (parameters etc.) end with `failed`;
+ * credentials failures end with their own terminal status `auth` (see
+ * `isAuthenticationError`) — same engine behavior as `failed`, but hosts key on it.
*
* `context_engine` only consumes OmniMessage; all Uni* protocol details are encapsulated here.
* Docs: /docs/interfaces § "The built-in implementation: GenerativeModel".
@@ -651,7 +653,7 @@ export class EventTranslator {
}
/**
- * Converts an AgentHub finish_reason into an OmniMessage stop_reason (the five-value protocol).
+ * Converts an AgentHub finish_reason into an OmniMessage stop_reason (the six-value protocol).
* "stop", "tool_call", and null are treated as completed; length or unknown reasons map to failed.
*/
private omniStopReason(): StopReason {
@@ -794,11 +796,138 @@ export function isIncompleteStreamError(error: unknown): boolean {
return false;
}
+/**
+ * Retryable network/transport error codes: the classic Node socket codes plus undici's
+ * transport failures (`UND_ERR_*`). Node's `fetch` surfaces a dropped connection as
+ * `TypeError: terminated` whose `cause` carries the real code (e.g. "other side closed"
+ * with `code: "UND_ERR_SOCKET"`) — a transport disconnect, not a provider verdict, so it
+ * reconnects like any network drop.
+ */
+const RETRYABLE_NETWORK_CODES: ReadonlySet = new Set([
+ "ECONNRESET",
+ "ECONNREFUSED",
+ "ETIMEDOUT",
+ "EPIPE",
+ "EAI_AGAIN",
+ "ENOTFOUND",
+ "ECONNABORTED",
+ // undici (Node fetch) transport failures
+ "UND_ERR_SOCKET",
+ "UND_ERR_CONNECT_TIMEOUT",
+ "UND_ERR_HEADERS_TIMEOUT",
+ "UND_ERR_BODY_TIMEOUT",
+]);
+
+/** Provider "quota / subscription exhausted" error codes (OpenAI-compatible bodies). */
+const QUOTA_CODES: ReadonlySet = new Set(["insufficient_user_quota", "insufficient_quota"]);
+
+/** Credentials/authentication error codes and types (OpenAI-compatible bodies / SDK errors). */
+const AUTH_CODES: ReadonlySet = new Set([
+ "invalid_api_key",
+ "authentication_error",
+ "unauthorized",
+]);
+
+/**
+ * Walks the error's `cause` chain (cycle-safe, same approach as `describeError`) applying
+ * `probe` to each level; true as soon as one level matches. Higher layers routinely wrap the
+ * real failure (Node fetch puts the transport error on `cause`), so single-level checks miss it.
+ */
+function anyInCauseChain(error: unknown, probe: (level: object) => boolean): boolean {
+ const seen = new Set();
+ let cur: unknown = error;
+ while (cur != null && typeof cur === "object" && !seen.has(cur)) {
+ seen.add(cur);
+ if (probe(cur)) return true;
+ cur = (cur as { cause?: unknown }).cause;
+ }
+ return false;
+}
+
+/**
+ * Provider error-code signals at one level of an error: `code` on the error itself (the
+ * OpenAI SDK exposes the parsed body's code directly), plus the parsed body under `error` —
+ * both the OpenAI shape (`err.error.code`) and the Anthropic SDK shape, whose `error`
+ * property holds the whole response body (`err.error.error.code`). With `includeType`, the
+ * matching `type` fields are collected too (auth signals ride on `type` in some bodies).
+ */
+function providerSignals(level: object, includeType: boolean): string[] {
+ const err = level as {
+ code?: unknown;
+ type?: unknown;
+ error?: { code?: unknown; type?: unknown; error?: { code?: unknown; type?: unknown } };
+ };
+ const body = typeof err.error === "object" && err.error !== null ? err.error : undefined;
+ const inner = typeof body?.error === "object" && body.error !== null ? body.error : undefined;
+ const signals = [err.code, body?.code, inner?.code];
+ if (includeType) signals.push(err.type, body?.type, inner?.type);
+ return signals.filter((v): v is string => typeof v === "string");
+}
+
+/**
+ * Determines whether an error is the provider's "quota / subscription exhausted" rejection
+ * (e.g. a gateway 403 whose body carries `insufficient_user_quota`). Topping up or the
+ * billing cycle rolling over fixes it without touching the Session, so it is transient and
+ * goes through the engine's reconnect flow like a network problem. Deliberately tight:
+ * only a payment/permission status (402/403 — or no status at all) combined with a known
+ * quota code (own, `cause` chain, or parsed provider body), or a 403 whose message names
+ * quota/subscription; a plain 403 (permission denied) stays non-retryable.
+ */
+export function isQuotaExhaustedError(error: unknown): boolean {
+ if (error == null || typeof error !== "object") return false;
+ // Belt to the auth-first ordering in isRetryableError: an error carrying a definitive
+ // auth signal is never quota, no matter what its message says — a 403 whose body codes
+ // `invalid_api_key` but whose message mentions a subscription must not classify as
+ // retryable-quota and burn through the reconnect ladder against a dead credential.
+ if (isAuthenticationError(error)) return false;
+ const err = error as { status?: number; statusCode?: number; message?: unknown };
+ const status = err.status ?? err.statusCode;
+ // Any other status keeps its own classification (401 auth, 429 rate limit, …).
+ if (typeof status === "number" && status !== 402 && status !== 403) return false;
+ if (
+ anyInCauseChain(error, (level) => providerSignals(level, false).some((c) => QUOTA_CODES.has(c)))
+ ) {
+ return true;
+ }
+ // 额度 covers Chinese-language gateway messages such as 订阅额度不足 ("subscription quota
+ // exhausted") that carry no machine-readable code.
+ return (
+ status === 403 &&
+ typeof err.message === "string" &&
+ /quota|subscription|额度/i.test(err.message)
+ );
+}
+
+/**
+ * Determines whether an error is a credentials/authentication failure — the one class an
+ * in-run retry can never fix: the request keeps going out with the same dead credential.
+ * Only the model REFERENCE is fixed at Session creation; the credential is read from the
+ * current Project config whenever the Session loads, so the fix is updating that model's
+ * API key (Models page) — after which the Session can continue — not retrying. Signals:
+ * HTTP 401 (any), a known auth code/type (on the error, its `cause` chain, or the parsed
+ * provider body), or the SDK error class name `AuthenticationError` (OpenAI / Anthropic
+ * SDKs). A bare 403 carries no such signal and stays a plain failure (usually
+ * permission/policy, not credentials).
+ */
+export function isAuthenticationError(error: unknown): boolean {
+ return anyInCauseChain(error, (level) => {
+ const err = level as { status?: unknown; statusCode?: unknown; name?: unknown };
+ if (err.status === 401 || err.statusCode === 401) return true;
+ if (providerSignals(level, true).some((c) => AUTH_CODES.has(c))) return true;
+ const name = typeof err.name === "string" ? err.name : "";
+ return name === "AuthenticationError" || level.constructor?.name === "AuthenticationError";
+ });
+}
+
/**
* Determines whether an error is retryable.
*
- * Retryable: network errors, timeouts, connection reset, HTTP 429 / 5xx.
- * Not retryable: HTTP 4xx auth/parameter errors (401/403/400/404, etc.).
+ * Retryable: network/transport errors (including undici's `UND_ERR_*` on the `cause` chain),
+ * timeouts, connection reset, HTTP 429 / 5xx, and provider quota/subscription exhaustion
+ * (see `isQuotaExhaustedError`).
+ * Not retryable: authentication errors (checked FIRST — a definitive credential signal is
+ * never reclassified by the heuristics below, see `isAuthenticationError`) and other HTTP
+ * 4xx auth/parameter errors (401/403/400/404, etc.).
* JSON parse errors are classified separately as `malformed` by `isMalformedJsonParseError`.
*
* Since AgentHub doesn't guarantee the shape of error objects, this uses a lenient check: first
@@ -808,10 +937,15 @@ export function isIncompleteStreamError(error: unknown): boolean {
export function isRetryableError(error: unknown): boolean {
if (error == null) return false;
+ // 0. Authentication is definitive and terminal: retrying a dead credential can never
+ // succeed, and no heuristic below (quota keywords, message vocabulary) may reclassify
+ // it — this ordering is the only thing keeping a dead credential out of the retry
+ // ladder now that quota errors are deliberately retryable.
+ if (isAuthenticationError(error)) return false;
+
const err = error as {
status?: number;
statusCode?: number;
- code?: string;
name?: string;
message?: string;
};
@@ -821,31 +955,43 @@ export function isRetryableError(error: unknown): boolean {
if (typeof status === "number") {
if (status === 429 || status === 408) return true; // Rate limited / request timeout (transient)
if (status >= 500 && status <= 599) return true; // Server error
+ // Provider quota/subscription exhaustion (402/403 + explicit signals): transient —
+ // checked before the blanket 4xx rule below would discard it.
+ if (isQuotaExhaustedError(error)) return true;
if (status >= 400 && status <= 499) return false; // Other 4xx auth/parameter errors, not retryable
}
- // 2. Network-layer error codes.
- const retryableCodes = new Set([
- "ECONNRESET",
- "ECONNREFUSED",
- "ETIMEDOUT",
- "EPIPE",
- "EAI_AGAIN",
- "ENOTFOUND",
- "ECONNABORTED",
- ]);
- if (err.code && retryableCodes.has(err.code)) return true;
+ // 2. Network/transport error codes, probing the `cause` chain (Node fetch wraps the real
+ // transport failure as `TypeError: terminated` with the code on `cause`, see describeError).
+ if (
+ anyInCauseChain(error, (level) => {
+ const code = (level as { code?: unknown }).code;
+ return typeof code === "string" && RETRYABLE_NETWORK_CODES.has(code);
+ })
+ ) {
+ return true;
+ }
- // 3. Error name / message keywords (timeout, network, rate limit).
+ // 2b. Status-less quota signals (some gateways surface only the provider code).
+ if (isQuotaExhaustedError(error)) return true;
+
+ // 3. Error name / message keywords (timeout, network, rate limit, undici disconnects).
if (err.name === "AbortError") return false; // User interruption, not retryable
const text = `${err.name ?? ""} ${err.message ?? ""}`.toLowerCase();
if (
- /timeout|timed out|network|socket hang up|econnreset|connection reset|too many requests|rate limit|temporarily unavailable|503|502|504/.test(
+ /timeout|timed out|network|socket hang up|econnreset|connection reset|other side closed|fetch failed|too many requests|rate limit|temporarily unavailable|503|502|504/.test(
text,
)
) {
return true;
}
+ // `terminated` alone is ambiguous — providers use the word in verdict copy too ("Request
+ // terminated by content filter"), which must NOT retry — so it only counts alongside
+ // transport vocabulary. The real socket case is already caught by the cause-chain
+ // `UND_ERR_*` codes in step 2; this is a last-resort fallback for cause-less wrappers.
+ if (/terminated/.test(text) && /socket|connection|closed/.test(text)) {
+ return true;
+ }
return false;
}
@@ -923,14 +1069,18 @@ export class GenerativeModel implements LLMInterface {
* and the terminal state is then returned as `LLMOutcome`:
* - **Normal completion**: `finish()` closes out and produces `token_usage` (usage is only
* produced in this case) → `completed`;
- * - **Idle timeout / network drop** (retryable errors like network/429/5xx):
- * `finishInterrupted("timeout")` closes out, produces no usage → `timeout`, reconnected by
+ * - **Idle timeout / network drop** (retryable errors like network/429/5xx/quota):
+ * `finishInterrupted("timeout")` closes out, produces no usage → `timeout` (carrying
+ * the error detail as `message` when a concrete error was caught), reconnected by
* `context_engine` within the same run;
* - **AgentHub JSON parse error**: `finishInterrupted("malformed")` closes out, produces no
* usage → `malformed`, likewise reconnected by `context_engine` within the same run;
* - **User interruption**: `finishInterrupted("aborted")` closes out, produces no usage →
* `aborted`;
- * - **Other non-retryable errors** (auth/parameters etc.): `finishInterrupted("failed")`
+ * - **Credentials failure**: `finishInterrupted("auth")` closes out, produces no usage →
+ * `auth` (carrying `message`) — the engine stops exactly like `failed`, hosts key on
+ * the status to gate input until the model's API key is updated;
+ * - **Other non-retryable errors** (parameters etc.): `finishInterrupted("failed")`
* closes out, produces no usage → `failed` (carrying `message`), handed to
* `context_engine` to stop and return control to the user.
*
@@ -1037,15 +1187,23 @@ export class GenerativeModel implements LLMInterface {
status: "malformed",
message: describeError(error),
};
+ } else if (isAuthenticationError(error)) {
+ // Credentials failure: its own terminal status so hosts can tell "update this
+ // model's API key, then this Session can continue" (only the model reference is
+ // fixed at Session creation; the credential is read from the current Project
+ // config on load) apart from a one-off failure. Checked before the retryable
+ // branch as a belt — isRetryableError itself already refuses auth signals.
+ outcome = { status: "auth", message: describeError(error) };
} else if (isRetryableError(error)) {
- outcome = { status: "timeout" }; // Network drop/network error -> needs reconnection
+ // Network drop / transient provider rejection -> needs reconnection. The detail
+ // (e.g. "403 … (insufficient_user_quota)") rides on the outcome so observability
+ // (request_end -> the Cost center's errors panel) shows the real reason behind a
+ // retried request, not just "timeout".
+ outcome = { status: "timeout", message: describeError(error) };
} else if ((error as { name?: string })?.name === "AbortError") {
outcome = { status: "aborted" }; // Fallback: an unexpected abort (neither timeout nor user)
} else {
- outcome = {
- status: "failed",
- message: describeError(error),
- };
+ outcome = { status: "failed", message: describeError(error) };
}
} finally {
clearTimer();
diff --git a/packages/core/src/llm/index.ts b/packages/core/src/llm/index.ts
index c4cbd03..82defa5 100644
--- a/packages/core/src/llm/index.ts
+++ b/packages/core/src/llm/index.ts
@@ -15,6 +15,8 @@ export {
isMalformedJsonParseError,
isIncompleteStreamError,
isRetryableError,
+ isQuotaExhaustedError,
+ isAuthenticationError,
mapThinkingLevel,
toolDefinitionsToSchemas,
buildUniConfig,
diff --git a/packages/core/src/omnimessage/builders.ts b/packages/core/src/omnimessage/builders.ts
index e476b17..325fdc8 100644
--- a/packages/core/src/omnimessage/builders.ts
+++ b/packages/core/src/omnimessage/builders.ts
@@ -269,9 +269,23 @@ export function requestBegin(): OmniMessage {
return event({ type: "request_begin" });
}
-/** request end event: carries the terminal state (`completed` means this turn was already committed to AgentHub). */
-export function requestEnd(status: StopReason): OmniMessage {
- return event({ type: "request_end", status });
+/**
+ * request end event: carries the terminal state (`completed` means this turn was already
+ * committed to AgentHub), plus — on non-completed statuses — the failure detail (from
+ * LLMOutcome.message) and, when the engine will retry in-run, the planned backoff wait
+ * (`retry_in_ms`, rendered by the Web App as a live countdown).
+ */
+export function requestEnd(
+ status: StopReason,
+ message?: string,
+ retryInMs?: number,
+): OmniMessage {
+ return event({
+ type: "request_end",
+ status,
+ ...(message !== undefined ? { message } : {}),
+ ...(retryInMs !== undefined ? { retry_in_ms: retryInMs } : {}),
+ });
}
/** compaction begin event: carries the trigger reason, mode, current context usage, and cumulative Session turn count. */
diff --git a/packages/core/src/omnimessage/types.ts b/packages/core/src/omnimessage/types.ts
index 2df80e7..2093c53 100644
--- a/packages/core/src/omnimessage/types.ts
+++ b/packages/core/src/omnimessage/types.ts
@@ -23,7 +23,7 @@ export type OmniMessageType = "session_meta" | "model_msg" | "event_msg";
export type Role = "user" | "assistant";
/**
- * The reason a model response or message generation ended. Only five protocol values are
+ * The reason a model response or message generation ended. Only six protocol values are
* allowed:
* - `completed`: finished normally, including completed text, thinking, tool requests, or
* tool output;
@@ -31,10 +31,15 @@ export type Role = "user" | "assistant";
* - `aborted`: user-initiated interruption or cancellation;
* - `timeout`: LLM request timed out;
* - `malformed`: the LLM response was malformed (e.g. AgentHub JSON parsing exception).
- * Only LLM timeout / malformed trigger a context_engine reconnect.
+ * Only LLM timeout / malformed trigger a context_engine reconnect;
+ * - `auth`: the LLM rejected the credentials (see `isAuthenticationError`) — a
+ * `failed`-shaped stop that no in-run retry can fix; hosts disable input until the
+ * model's API key is updated (only the model reference is fixed at Session creation —
+ * credentials come from the current Project config), after which the Session
+ * continues.
* Docs: /docs/omni-message § "stop_reason".
*/
-export type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed";
+export type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed" | "auth";
/** The event phase of a streaming fragment. `stop` marks the end of a fragment and usually carries no incremental content. */
export type StreamEventType = "start" | "delta" | "stop";
@@ -277,8 +282,25 @@ export interface RequestBeginPayload {
export interface RequestEndPayload {
type: "request_end";
- /** Terminal state of this Request (reuses the five StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome). */
+ /** Terminal state of this Request (reuses the six StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome; `auth` is the credentials-failure signal hosts key on). */
status: StopReason;
+ /**
+ * Failure detail from `LLMOutcome.message`, present only on non-completed statuses: the
+ * real reason behind a retried/failed Request (e.g. `403 … (insufficient_user_quota)`),
+ * for observability — the server's error records / Cost center read it here because a
+ * retried request never produces an abort event. Additive: old Traces without it replay
+ * unchanged.
+ */
+ message?: string;
+ /**
+ * Planned in-run retry wait (ms) — present ONLY when the engine will retry this failure
+ * within the same run (status `timeout`/`malformed` with attempts remaining under the
+ * applicable cap). Computed by the same formula as the actual backoff sleep
+ * (`reconnectDelayMs`), so the announced wait and the real one cannot drift; the Web App
+ * renders it as a live countdown to the next attempt. Absent on final failures (an abort
+ * follows instead) and on completed requests. Additive: old Traces replay unchanged.
+ */
+ retry_in_ms?: number;
}
/** Compaction trigger reason: context threshold / turn-count threshold / user-initiated request. */
@@ -291,7 +313,7 @@ export type CompactionMode = "summarize" | "discard";
* Compaction boundary event: the compaction process exposes
* only this event pair to Human, produced **in pairs** by `context_engine`. Both `reason` and
* `mode` are carried on both events, for stateless frontend rendering; `status` reuses the
- * five-value `StopReason` protocol (compaction converges to a terminal state, taking
+ * six-value `StopReason` protocol (compaction converges to a terminal state, taking
* `completed` / `failed` / `aborted` in practice — `timeout` / `malformed` are handled internally
* by the compaction request's existing retry mechanism, collapsing to `failed` once retries are
* exhausted).
diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts
index d62e55f..1c49c31 100644
--- a/packages/core/src/session.ts
+++ b/packages/core/src/session.ts
@@ -200,6 +200,18 @@ export class Session {
return this.engine.steer(text);
}
+ /**
+ * Skips the in-progress reconnect backoff and fires the next retry immediately (the
+ * user's "retry now" on the reconnect countdown): the attempt counter is unchanged —
+ * the skipped wait does not consume an extra attempt. Returns false (a benign no-op)
+ * when no reconnect wait is in progress; idempotent under races with the timer/abort.
+ * Mirrors `steer` as a mid-run nudge (no message of its own — the effect surfaces as
+ * the next `request_begin` arriving early).
+ */
+ skipReconnectWait(): boolean {
+ return this.engine.skipReconnectWait();
+ }
+
/**
* User-initiated request to compact context (e.g. a CLI command): reuses the automatic
* compaction flow but skips the threshold check (reason=manual). Only callable at Task
diff --git a/packages/core/src/state/default-config.ts b/packages/core/src/state/default-config.ts
index 2857dba..4b11f04 100644
--- a/packages/core/src/state/default-config.ts
+++ b/packages/core/src/state/default-config.ts
@@ -112,7 +112,7 @@ Communicate with the user precisely and concisely, yet with warmth. Do not repea
# System markers
Some messages contain system-synthesized blocks written as \`[tag]...[/tag]\`, not user text to answer directly:
- \`[turn_aborted]\`: the previous round was interrupted. Inside are the original request, your partial thinking/text, and the tool calls already issued with their results. Continue from where it left off; do not re-run tools whose results are already included.
-- \`[turn_retried]\`: the previous attempt of this round failed on a transport error (timeout or malformed response) — the user did NOT interrupt — and this request is the automatic retry. Inside are your partial thinking/text and the tool calls already executed with their results. Continue from them; do not re-run tools whose results are already included.
+- \`[turn_retried]\`: the previous attempt of this round failed on a transient error (timeout, disconnect, malformed response, or a temporary provider error) — the user did NOT interrupt — and this request is the automatic retry. Inside are your partial thinking/text and the tool calls already executed with their results. Continue from them; do not re-run tools whose results are already included.
- \`[context_summary]\`: earlier conversation was compacted. This summary replaced the raw transcript and is its only record; treat it as established context and continue the task from it.
- \`[user_steering]\`: a user message sent while you were still working, delivered between turns alongside tool results. It is not a new task: incorporate it immediately and adjust course within the current task.
diff --git a/packages/core/test/compaction.test.ts b/packages/core/test/compaction.test.ts
index 8d3f236..bbd8c54 100644
--- a/packages/core/test/compaction.test.ts
+++ b/packages/core/test/compaction.test.ts
@@ -399,7 +399,7 @@ describe("context compaction", () => {
environment: fakeEnvironment,
compaction: settings(),
createLLM: () => llm1,
- maxReconnects: 1,
+ compactionMaxReconnects: 1,
reconnectBackoffMs: 1,
});
@@ -411,6 +411,52 @@ describe("context compaction", () => {
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
});
+ it("the compaction loop uses its own cap, not the shared maxReconnects ladder", async () => {
+ // Compaction failure has graceful semantics (the original context is kept, compaction
+ // retries on the next trigger), so it fails fast: with compactionMaxReconnects=2, the
+ // compaction request runs 1+2 attempts and gives up — even though maxReconnects is far
+ // larger and would have kept the session stalled through the full exponential ladder.
+ const llm1 = new ScriptedLLM(
+ [
+ { messages: [assistantText("answer"), usage(150, 150)] },
+ { messages: [], outcome: { status: "timeout" } },
+ { messages: [], outcome: { status: "timeout" } },
+ { messages: [], outcome: { status: "timeout" } },
+ // Never reached: the compaction cap stops at 3 compaction attempts in total.
+ { messages: [assistantText("[summary]late[/summary]")] },
+ ],
+ "llm1",
+ );
+ const recorded: OmniMessage[] = [];
+ const engine = new ContextEngine({
+ llm: llm1,
+ environment: fakeEnvironment,
+ compaction: settings(),
+ createLLM: () => llm1,
+ maxReconnects: 8,
+ compactionMaxReconnects: 2,
+ reconnectBackoffMs: 1,
+ trace: {
+ write: async (m) => {
+ recorded.push(m);
+ },
+ },
+ });
+
+ const out = await collect(engine.run([userText("go")], { approve: allowAll }));
+ const events = compactionEvents(out);
+ expect(events[1]).toMatchObject({ type: "compaction_end", status: "failed" });
+ // 1 turn request + 3 compaction attempts (initial + compactionMaxReconnects retries).
+ expect(llm1.calls).toHaveLength(4);
+ // The Trace-written compaction request_ends announce the planned backoff under the
+ // COMPACTION cap (base 1ms: 1, then 2), with none on the final failure — these events
+ // are never streamed, so the value lands in the Trace record only.
+ const retryPlans = recorded
+ .filter((m) => (m.payload as { type?: string }).type === "request_end")
+ .map((m) => (m.payload as { retry_in_ms?: number }).retry_in_ms);
+ expect(retryPlans).toEqual([undefined, 1, 2, undefined]);
+ });
+
it("session turns reaching (==) the threshold compact at task end — no waiting for the next task", async () => {
const llm1 = new ScriptedLLM(
[
diff --git a/packages/core/test/engine.test.ts b/packages/core/test/engine.test.ts
index eece206..ffcbb41 100644
--- a/packages/core/test/engine.test.ts
+++ b/packages/core/test/engine.test.ts
@@ -30,7 +30,7 @@ import type { GenerativeModelParameters, LLMInterface, LLMOutcome } from "../src
import type { OmniMessage, TextPayload, ToolCallPayload } from "../src/omnimessage/index.js";
import { Environment } from "../src/environment/index.js";
import { Writer, readTrace } from "../src/trace/index.js";
-import { ContextEngine } from "../src/engine/context-engine.js";
+import { ContextEngine, reconnectDelayMs } from "../src/engine/context-engine.js";
import type { ApproveFn, EnvironmentInterface, ToolPermission } from "../src/interfaces.js";
/** Deterministic fake LLM: the first turn yields a tool_call, the second yields the final reply. */
@@ -1488,12 +1488,14 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
const inputs: OmniMessage[][] = [];
const llm: LLMInterface = {
// The LLM must never throw an exception at the engine: a non-retryable error resolves
- // by returning a failed outcome after closing the structure.
+ // by returning a failed outcome after closing the structure. A genuinely non-retryable
+ // failure nowadays is a parameter error (quota 403s retry as timeout, 401s carry
+ // code "auth" — both covered by their own tests below).
// eslint-disable-next-line require-yield
async *streamGenerate(params) {
calls += 1;
inputs.push(params.newMessages);
- return { status: "failed", message: "invalid api key" };
+ return { status: "failed", message: "400 unknown parameter: max_output_tokens" };
},
};
const environment = new Environment({
@@ -1509,7 +1511,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
expect(abort).toBeDefined();
const reason = (abort!.payload as { reason?: string }).reason ?? "";
expect(reason).toContain("llm request error");
- expect(reason).toContain("invalid api key");
+ expect(reason).toContain("unknown parameter");
// The failed turn's input is flattened and stashed; the next run resends it merged with
// the new input.
@@ -1519,6 +1521,232 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
expect(text).toContain("next");
});
+ it("an auth outcome stops immediately like failed: no retry, request_end carries status auth", async () => {
+ let calls = 0;
+ const llm: LLMInterface = {
+ // GenerativeModel classifies a 401/invalid_api_key as status "auth" (see
+ // llm.test.ts); the engine must stop directly — the auth/failed branch is the second
+ // belt keeping a dead credential out of the retry loop (the classifier is the first).
+ // eslint-disable-next-line require-yield
+ async *streamGenerate() {
+ calls += 1;
+ return { status: "auth", message: "401 invalid x-api-key" };
+ },
+ };
+ const environment = new Environment({
+ workspaceDir: workspace,
+ toolConfig: execCommandToolConfig(),
+ });
+ const engine = new ContextEngine({ llm, environment, reconnectBackoffMs: 1 });
+
+ const all = await collectRun(engine, [userText("go")], allowAll);
+ expect(calls).toBe(1); // Auth behaves like failed: never enters the reconnect loop.
+ // The request's own terminal status is the host signal (streams to the web).
+ const end = all.find((m) => (m.payload as { type?: string }).type === "request_end");
+ expect((end!.payload as { status?: string }).status).toBe("auth");
+ expect((end!.payload as { message?: string }).message).toBe("401 invalid x-api-key");
+ // No planned retry is announced for a terminal failure.
+ expect((end!.payload as { retry_in_ms?: number }).retry_in_ms).toBeUndefined();
+ const abort = all.find((m) => (m.payload as { type?: string }).type === "abort");
+ expect(abort).toBeDefined();
+ expect((abort!.payload as { reason?: string }).reason).toContain("llm request error");
+ });
+
+ it("a quota-403 (classified timeout) retries within the default cap and succeeds", async () => {
+ let calls = 0;
+ const llm: LLMInterface = {
+ async *streamGenerate() {
+ calls += 1;
+ // Two quota rejections (GenerativeModel classifies them as timeout), then success —
+ // attempt 3 is within the default cap of 5.
+ if (calls <= 2) return { status: "timeout" };
+ yield assistantText("recovered");
+ yield tokenUsage(emptyTokenCounts(), {
+ cache_read: 0,
+ cache_write: 0,
+ output: 1,
+ total: 1,
+ });
+ return { status: "completed" };
+ },
+ };
+ const environment = new Environment({
+ workspaceDir: workspace,
+ toolConfig: execCommandToolConfig(),
+ });
+ const engine = new ContextEngine({ llm, environment, reconnectBackoffMs: 1 });
+
+ const all = await collectRun(engine, [userText("go")], allowAll);
+ expect(calls).toBe(3);
+ expect(
+ all.some(
+ (m) =>
+ isCompleteModelMessage(m) && m.payload.type === "text" && m.payload.text === "recovered",
+ ),
+ ).toBe(true);
+ expect(all.map((m) => (m.payload as { type?: string }).type)).not.toContain("abort");
+ });
+
+ it("default reconnect cap is 5: the exhaustion abort message says so", async () => {
+ let calls = 0;
+ const llm: LLMInterface = {
+ // eslint-disable-next-line require-yield
+ async *streamGenerate() {
+ calls += 1;
+ return { status: "timeout" }; // Always needs a reconnect.
+ },
+ };
+ const environment = new Environment({
+ workspaceDir: workspace,
+ toolConfig: execCommandToolConfig(),
+ });
+ // No maxReconnects override: exercises the default cap (tiny base keeps the
+ // exponential waits at 1+2+4+8+16 = 31ms total).
+ const engine = new ContextEngine({ llm, environment, reconnectBackoffMs: 1 });
+
+ const all = await collectRun(engine, [userText("go")], allowAll);
+ expect(calls).toBe(6); // Initial attempt + 5 retries.
+ const abort = all.find((m) => (m.payload as { type?: string }).type === "abort");
+ expect((abort!.payload as { reason?: string }).reason).toBe("reconnect failed after 5 retries");
+ });
+
+ it("skipReconnectWait wakes the backoff early ('retry now'): attempt numbering unchanged; no-op without a wait", async () => {
+ let calls = 0;
+ const llm: LLMInterface = {
+ async *streamGenerate() {
+ calls += 1;
+ if (calls === 1) return { status: "timeout" };
+ yield assistantText("ok");
+ yield tokenUsage(emptyTokenCounts(), {
+ cache_read: 0,
+ cache_write: 0,
+ output: 1,
+ total: 1,
+ });
+ return { status: "completed" };
+ },
+ };
+ const environment = new Environment({
+ workspaceDir: workspace,
+ toolConfig: execCommandToolConfig(),
+ });
+ // A wait long enough that the run could only finish in time if the skip woke it.
+ const engine = new ContextEngine({
+ llm,
+ environment,
+ maxReconnects: 2,
+ reconnectBackoffMs: 60_000,
+ });
+
+ // No reconnect wait in progress yet: skip is a benign no-op.
+ expect(engine.skipReconnectWait()).toBe(false);
+
+ const started = Date.now();
+ const runP = collectRun(engine, [userText("go")], allowAll);
+ // Poll the skip itself: it keeps returning false until the engine parks in the
+ // backoff, and consumes the wait exactly once when it does.
+ while (!engine.skipReconnectWait()) {
+ await new Promise((r) => setTimeout(r, 5));
+ }
+ const all = await runP;
+ expect(Date.now() - started).toBeLessThan(10_000); // woke early: the scheduled wait was 60s
+ // Exactly the one retry — the skipped wait consumed no extra attempt.
+ expect(calls).toBe(2);
+ const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end");
+ expect(ends.map((m) => (m.payload as { status?: string }).status)).toEqual([
+ "timeout",
+ "completed",
+ ]);
+ expect(all.map((m) => (m.payload as { type?: string }).type)).not.toContain("abort");
+ // The wait settled (skip won the race): further skips are no-ops again.
+ expect(engine.skipReconnectWait()).toBe(false);
+ });
+
+ it("reconnectDelayMs: exponential-with-ceiling ladder (defaults: 250ms base, 30s cap)", () => {
+ // The default cap (5) walks the first five steps — 250+500+1000+2000+4000 ≈ 7.75s of
+ // total patience; the formula keeps climbing to the 30s ceiling for larger caps.
+ const ladder = [1, 2, 3, 4, 5].map((n) => reconnectDelayMs(250, 30_000, n));
+ expect(ladder).toEqual([250, 500, 1000, 2000, 4000]);
+ expect(ladder.reduce((a, b) => a + b, 0)).toBe(7750);
+ expect([6, 7, 8].map((n) => reconnectDelayMs(250, 30_000, n))).toEqual([8000, 16000, 30000]);
+ // Past the ceiling the delay stays pinned (no overflow, no further growth).
+ expect(reconnectDelayMs(250, 30_000, 12)).toBe(30_000);
+ // The cap also applies when the base itself exceeds it.
+ expect(reconnectDelayMs(50_000, 30_000, 1)).toBe(30_000);
+ });
+
+ it("request_end carries the outcome's failure detail on non-completed statuses only", async () => {
+ let calls = 0;
+ const llm: LLMInterface = {
+ async *streamGenerate() {
+ calls += 1;
+ if (calls === 1) {
+ // A retryable provider rejection: the detail must reach observability via the
+ // event — a retried request never produces an abort to carry it.
+ return { status: "timeout", message: "403 quota exceeded (insufficient_user_quota)" };
+ }
+ yield assistantText("ok");
+ yield tokenUsage(emptyTokenCounts(), {
+ cache_read: 0,
+ cache_write: 0,
+ output: 1,
+ total: 1,
+ });
+ return { status: "completed" };
+ },
+ };
+ const environment = new Environment({
+ workspaceDir: workspace,
+ toolConfig: execCommandToolConfig(),
+ });
+ const engine = new ContextEngine({ llm, environment, maxReconnects: 1, reconnectBackoffMs: 0 });
+
+ const all = await collectRun(engine, [userText("go")], allowAll);
+ const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end") as {
+ payload: { status?: string; message?: string; retry_in_ms?: number };
+ }[];
+ expect(ends).toHaveLength(2);
+ expect(ends[0]!.payload.status).toBe("timeout");
+ expect(ends[0]!.payload.message).toBe("403 quota exceeded (insufficient_user_quota)");
+ // The engine will retry: the planned backoff is announced (base 0 here -> 0ms).
+ expect(ends[0]!.payload.retry_in_ms).toBe(0);
+ expect(ends[1]!.payload.status).toBe("completed");
+ expect(ends[1]!.payload.message).toBeUndefined();
+ expect(ends[1]!.payload.retry_in_ms).toBeUndefined();
+ });
+
+ it("request_end announces the planned backoff (retry_in_ms) from the shared ladder; absent once the cap is reached", async () => {
+ let calls = 0;
+ const llm: LLMInterface = {
+ // eslint-disable-next-line require-yield
+ async *streamGenerate() {
+ calls += 1;
+ return { status: "timeout" }; // Always needs a reconnect.
+ },
+ };
+ const environment = new Environment({
+ workspaceDir: workspace,
+ toolConfig: execCommandToolConfig(),
+ });
+ const engine = new ContextEngine({
+ llm,
+ environment,
+ maxReconnects: 2,
+ reconnectBackoffMs: 10,
+ reconnectBackoffMaxMs: 15,
+ });
+
+ const all = await collectRun(engine, [userText("go")], allowAll);
+ const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end") as {
+ payload: { retry_in_ms?: number };
+ }[];
+ // Announced waits follow the same exponential-with-ceiling formula the sleep uses
+ // (10, then min(20, 15) = 15); the FINAL failure carries none — no retry follows,
+ // the exhaustion abort does.
+ expect(ends.map((e) => e.payload.retry_in_ms)).toEqual([10, 15, undefined]);
+ expect(calls).toBe(3); // Initial attempt + 2 retries.
+ });
+
it("LLM timeout after a tool already executed: retry carries the call/result via [turn_retried] (tool runs once)", async () => {
let calls = 0;
const inputs: OmniMessage[][] = [];
diff --git a/packages/core/test/llm.test.ts b/packages/core/test/llm.test.ts
index 0502146..054b4ad 100644
--- a/packages/core/test/llm.test.ts
+++ b/packages/core/test/llm.test.ts
@@ -23,8 +23,10 @@ import {
GenerativeModel,
ToolCallIdAllocator,
buildUniConfig,
+ isAuthenticationError,
isIncompleteStreamError,
isMalformedJsonParseError,
+ isQuotaExhaustedError,
isRetryableError,
mapThinkingLevel,
mergeOmniToUniMessage,
@@ -1097,6 +1099,85 @@ describe("isRetryableError", () => {
expect(isRetryableError(new Error("request timeout"))).toBe(true);
});
+ it("treats undici transport failures as retryable, probing the cause chain", () => {
+ // Node fetch wraps a dropped connection as TypeError("terminated") with the real code on
+ // `cause` — exactly the field observed: "terminated: other side closed (UND_ERR_SOCKET)".
+ expect(
+ isRetryableError(
+ new TypeError("terminated", {
+ cause: Object.assign(new Error("other side closed"), { code: "UND_ERR_SOCKET" }),
+ }),
+ ),
+ ).toBe(true);
+ expect(isRetryableError({ code: "UND_ERR_SOCKET" })).toBe(true);
+ expect(isRetryableError({ code: "UND_ERR_CONNECT_TIMEOUT" })).toBe(true);
+ expect(isRetryableError({ code: "UND_ERR_HEADERS_TIMEOUT" })).toBe(true);
+ expect(isRetryableError({ code: "UND_ERR_BODY_TIMEOUT" })).toBe(true);
+ // A cause-less "fetch failed" still counts via the message keywords; "terminated" only
+ // counts alongside transport vocabulary — the real socket case always carries the
+ // UND_ERR_* code on its cause, and bare "terminated" appears in unrelated provider
+ // copy too (see the content-filter counter-case below).
+ expect(isRetryableError(new TypeError("fetch failed"))).toBe(true);
+ expect(isRetryableError(new TypeError("terminated: other side closed"))).toBe(true);
+ expect(isRetryableError(new TypeError("terminated"))).toBe(false);
+ // Provider verdict copy that merely contains the word must NOT retry.
+ expect(isRetryableError(new Error("Request terminated by content filter"))).toBe(false);
+ // Classic Node codes wrapped one level down are found too.
+ expect(
+ isRetryableError(
+ new Error("request failed", {
+ cause: Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" }),
+ }),
+ ),
+ ).toBe(true);
+ });
+
+ it("treats provider quota/subscription exhaustion as retryable (tight allowlist)", () => {
+ // The field case: a gateway 403 with an OpenAI-compatible body code.
+ expect(isRetryableError({ status: 403, code: "insufficient_user_quota" })).toBe(true);
+ expect(isRetryableError({ status: 403, error: { code: "insufficient_user_quota" } })).toBe(
+ true,
+ );
+ // Anthropic SDK shape: `error` holds the whole response body.
+ expect(
+ isRetryableError({ status: 403, error: { error: { code: "insufficient_quota" } } }),
+ ).toBe(true);
+ expect(isRetryableError({ status: 402, code: "insufficient_quota" })).toBe(true);
+ // Status-less quota code (some gateways surface only the provider code).
+ expect(isRetryableError({ code: "insufficient_user_quota" })).toBe(true);
+ // 403 with a quota/subscription message but no machine-readable code.
+ expect(isRetryableError({ status: 403, message: "no active subscription" })).toBe(true);
+ expect(isRetryableError({ status: 403, message: "订阅额度不足" })).toBe(true);
+ // A plain 403/400/404 without quota signals stays non-retryable.
+ expect(isRetryableError({ status: 403, message: "permission denied" })).toBe(false);
+ expect(isRetryableError({ status: 400, code: "insufficient_user_quota" })).toBe(false);
+ expect(isRetryableError({ status: 404 })).toBe(false);
+ // 401 keeps its auth classification even with a quota-looking message.
+ expect(isRetryableError({ status: 401, message: "quota" })).toBe(false);
+ });
+
+ it("auth wins over the quota heuristic: a definitive credential signal is never retried", () => {
+ // The adversarial case: a 403 whose BODY carries a definitive auth code but whose
+ // MESSAGE mentions a subscription (SDKs routinely put the body's message on
+ // err.message). The quota keyword fallback must not swallow it — auth is checked
+ // first, so it classifies failed (dead credential) instead of burning every reconnect.
+ const err = Object.assign(new Error("subscription key invalid"), {
+ status: 403,
+ error: { code: "invalid_api_key" },
+ });
+ expect(isAuthenticationError(err)).toBe(true);
+ expect(isQuotaExhaustedError(err)).toBe(false); // belt: an auth signal is never quota
+ expect(isRetryableError(err)).toBe(false); // never retried, never reclassified
+ // Same with the auth signal on the cause chain.
+ expect(
+ isRetryableError(
+ new Error("request failed: quota subscription issue", {
+ cause: { status: 403, error: { type: "authentication_error" } },
+ }),
+ ),
+ ).toBe(false);
+ });
+
it("does not retry abort or unknown local errors", () => {
const abort = new Error("aborted");
abort.name = "AbortError";
@@ -1107,6 +1188,76 @@ describe("isRetryableError", () => {
});
});
+describe("isQuotaExhaustedError", () => {
+ it("matches only 402/403 (or status-less) errors carrying a known quota code", () => {
+ expect(isQuotaExhaustedError({ status: 403, code: "insufficient_user_quota" })).toBe(true);
+ expect(isQuotaExhaustedError({ status: 402, error: { code: "insufficient_quota" } })).toBe(
+ true,
+ );
+ expect(isQuotaExhaustedError({ code: "insufficient_quota" })).toBe(true);
+ // The code may sit on the cause chain (wrapped by a higher layer).
+ expect(
+ isQuotaExhaustedError(
+ new Error("request failed", {
+ cause: { status: 403, code: "insufficient_user_quota" },
+ }),
+ ),
+ ).toBe(true);
+ expect(
+ isQuotaExhaustedError(
+ Object.assign(new Error("request failed"), {
+ status: 403,
+ cause: { code: "insufficient_user_quota" },
+ }),
+ ),
+ ).toBe(true);
+ // Any other status keeps its own classification.
+ expect(isQuotaExhaustedError({ status: 429, code: "insufficient_quota" })).toBe(false);
+ expect(isQuotaExhaustedError({ status: 401, code: "insufficient_quota" })).toBe(false);
+ });
+
+ it("matches a 403 whose message names quota/subscription, but not a plain 403", () => {
+ expect(isQuotaExhaustedError({ status: 403, message: "monthly quota exceeded" })).toBe(true);
+ expect(isQuotaExhaustedError({ status: 403, message: "订阅额度不足" })).toBe(true);
+ expect(isQuotaExhaustedError({ status: 403, message: "forbidden" })).toBe(false);
+ // The message shortcut is 403-only: a status-less error needs the explicit code.
+ expect(isQuotaExhaustedError({ message: "quota exceeded" })).toBe(false);
+ expect(isQuotaExhaustedError(null)).toBe(false);
+ });
+});
+
+describe("isAuthenticationError", () => {
+ it("classifies HTTP 401 as auth, own or wrapped", () => {
+ expect(isAuthenticationError({ status: 401 })).toBe(true);
+ expect(isAuthenticationError({ statusCode: 401 })).toBe(true);
+ expect(isAuthenticationError(new Error("request failed", { cause: { status: 401 } }))).toBe(
+ true,
+ );
+ });
+
+ it("classifies known auth codes/types and the SDK AuthenticationError class", () => {
+ expect(isAuthenticationError({ code: "invalid_api_key" })).toBe(true);
+ expect(isAuthenticationError({ status: 403, error: { type: "authentication_error" } })).toBe(
+ true,
+ );
+ // Anthropic SDK shape: `error` holds the whole response body.
+ expect(isAuthenticationError({ error: { error: { code: "unauthorized" } } })).toBe(true);
+ class AuthenticationError extends Error {}
+ expect(isAuthenticationError(new AuthenticationError("bad key"))).toBe(true);
+ expect(
+ isAuthenticationError(new Error("request failed", { cause: { code: "invalid_api_key" } })),
+ ).toBe(true);
+ });
+
+ it("does not classify a bare 403 or unrelated failures as auth", () => {
+ expect(isAuthenticationError({ status: 403 })).toBe(false);
+ expect(isAuthenticationError({ status: 400, message: "invalid param" })).toBe(false);
+ expect(isAuthenticationError({ status: 403, code: "insufficient_user_quota" })).toBe(false);
+ expect(isAuthenticationError(new Error("socket hang up"))).toBe(false);
+ expect(isAuthenticationError(null)).toBe(false);
+ });
+});
+
describe("isMalformedJsonParseError", () => {
it("detects JSON.parse SyntaxError by exception type, including the cause chain", () => {
// AgentHub uses JSON.parse internally; a parse failure throws a SyntaxError, so it can be
@@ -1433,14 +1584,80 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
expect(outcome2.status).toBe("malformed");
});
- it("classifies a non-retryable error as failed with message, no token_usage", async () => {
+ it("classifies a credentials failure as its own terminal status auth; params stay failed", async () => {
const model = new SeamModel(() => authError());
const { messages, outcome } = await drain(
model.streamGenerate({ newMessages: [userText("go")] }),
);
- expect(outcome.status).toBe("failed");
+ // 401 = credentials failure: its own stop reason so hosts can gate input until the
+ // model's API key is updated (same engine behavior as failed).
+ expect(outcome.status).toBe("auth");
expect(outcome.message).toContain("invalid api key");
expect(messages.map(typeOf)).not.toContain("token_usage");
+
+ // A genuinely non-retryable parameter error stays a plain failure.
+ async function* paramError(): AsyncGenerator {
+ throw Object.assign(new Error("unknown parameter: max_output_tokens"), { status: 400 });
+ }
+ const model2 = new SeamModel(() => paramError());
+ const { outcome: outcome2 } = await drain(
+ model2.streamGenerate({ newMessages: [userText("go")] }),
+ );
+ expect(outcome2.status).toBe("failed");
+ });
+
+ it("an auth error dressed in quota language still funnels to status auth", async () => {
+ // The adversarial case from the review: 403 + body code invalid_api_key + a message
+ // mentioning the subscription. Must NOT classify as retryable quota (timeout): the
+ // ordering fix keeps the dead credential out of the reconnect loop entirely.
+ async function* dressedAuthError(): AsyncGenerator {
+ throw Object.assign(new Error("subscription key invalid"), {
+ status: 403,
+ error: { code: "invalid_api_key" },
+ });
+ }
+ const model = new SeamModel(() => dressedAuthError());
+ const { messages, outcome } = await drain(
+ model.streamGenerate({ newMessages: [userText("go")] }),
+ );
+ expect(outcome.status).toBe("auth");
+ expect(outcome.message).toContain("subscription key invalid");
+ expect(messages.map(typeOf)).not.toContain("token_usage");
+ });
+
+ it("classifies provider quota exhaustion (403 + code) as timeout — the reconnect path", async () => {
+ async function* quotaError(): AsyncGenerator {
+ throw Object.assign(new Error("订阅额度不足 no active subscription"), {
+ status: 403,
+ code: "insufficient_user_quota",
+ });
+ }
+ const model = new SeamModel(() => quotaError());
+ const { messages, outcome } = await drain(
+ model.streamGenerate({ newMessages: [userText("go")] }),
+ );
+ expect(outcome.status).toBe("timeout");
+ // The real reason rides on the outcome (request_end -> the Cost center's errors panel
+ // shows it): a retried quota rejection must not surface as a bare "timeout".
+ expect(outcome.message).toContain("insufficient_user_quota");
+ expect(messages.map(typeOf)).not.toContain("token_usage");
+ });
+
+ it("classifies an undici transport drop (TypeError terminated, cause UND_ERR_SOCKET) as timeout", async () => {
+ async function* socketDrop(): AsyncGenerator {
+ yield ev({ content_items: [{ type: "text", text: "hi" }] });
+ throw new TypeError("terminated", {
+ cause: Object.assign(new Error("other side closed"), { code: "UND_ERR_SOCKET" }),
+ });
+ }
+ const model = new SeamModel(() => socketDrop());
+ const { messages, outcome } = await drain(
+ model.streamGenerate({ newMessages: [userText("go")] }),
+ );
+ expect(outcome.status).toBe("timeout");
+ const complete = messages.find((m) => typeOf(m) === "text");
+ expect((complete!.payload as TextPayload).stop_reason).toBe("timeout");
+ expect(messages.map(typeOf)).not.toContain("token_usage");
});
it("classifies a user abort (mid idle) as aborted, not timeout", async () => {
diff --git a/packages/docs/content/agent-loop.en.md b/packages/docs/content/agent-loop.en.md
index 7401bb1..2ba7d9b 100644
--- a/packages/docs/content/agent-loop.en.md
+++ b/packages/docs/content/agent-loop.en.md
@@ -26,7 +26,7 @@ session.run(newMessages, { approve, signal })
│ │ output streams back │
│ └─ LLMOutcome: │
│ timeout / malformed ──► reconnect within the turn │
-│ (≤2, with [turn_retried]; tools not rerun) │
+│ (≤5, with [turn_retried]; tools not rerun) │
│ token_usage + request_end (at LLM-stream end; not waiting │
│ for tools) │
│ │
@@ -91,7 +91,7 @@ While a Task is running, the host can queue a user message with `session.steer(t
## Automatic reconnect
-Only LLM-side `timeout` (network timeouts, rate limits, 5xx) and `malformed` (truncated streams, JSON parse failures) trigger an in-run reconnect: the engine re-sends the original input plus a `[turn_retried]` block carrying the previous partial output, so tools are never re-executed. Default limit is 2 reconnects with linear backoff (base 250ms); beyond that the turn settles as `failed`. Tool errors are never retried — they are fed back to the model as `tool_call_output` and the model decides what to do next.
+Only LLM-side `timeout` (network timeouts, transport disconnects such as a dropped socket, rate limits, 5xx, and transient provider quota/subscription errors like `insufficient_user_quota`) and `malformed` (truncated streams, JSON parse failures) trigger an in-run reconnect: the engine re-sends the original input plus a `[turn_retried]` block carrying the previous partial output, so tools are never re-executed. Default limit is 5 reconnects with exponential backoff under a ceiling (base 250ms, cap 30s: 250ms, 500ms, 1s, 2s, 4s ≈ 7.75s of total patience — one shared schedule growing toward the slower retryable classes, with the first steps as fast as a transport blip needs); beyond that the turn settles as `failed`. Each failure's `request_end` announces the planned wait as `retry_in_ms` (same formula as the sleep), which the Web App renders as a live countdown with "retry now" (skips the remaining wait via `Session.skipReconnectWait` — the attempt counter is unchanged) and "give up" (the ordinary abort; the engine's abort-during-backoff path ends the turn) controls. Compaction requests use their own tighter cap (3 retries): a failed compaction keeps the original context and retries on the next trigger, so failing fast beats stalling the session. Authentication errors are classified before any retry heuristic and never retry: the request ends with its own terminal status `auth` (only the model reference is fixed at Session creation — credentials are read from the current Project config when the Session loads), and the Web App disables that Session's composer until the model's credential is updated (which auto-unlocks it) or the notice is dismissed for a retry. Tool errors are never retried — they are fed back to the model as `tool_call_output` and the model decides what to do next.
## Compaction
diff --git a/packages/docs/content/agent-loop.zh.md b/packages/docs/content/agent-loop.zh.md
index 179237f..05fa9cf 100644
--- a/packages/docs/content/agent-loop.zh.md
+++ b/packages/docs/content/agent-loop.zh.md
@@ -24,7 +24,7 @@ session.run(newMessages, { approve, signal })
│ │ ▼ │
│ │ Environment.executeTool ──► 并发执行,输出流式回传 │
│ └─ LLMOutcome: │
-│ timeout / malformed ──► 同轮自动重连(≤2 次, │
+│ timeout / malformed ──► 同轮自动重连(≤5 次, │
│ 附 [turn_retried],工具不重跑) │
│ token_usage + request_end(LLM 流结束即产出,不等工具) │
│ │
@@ -88,7 +88,7 @@ Task 运行期间,宿主可通过 `session.steer(text)` 排队一条用户消
## 自动重连
-只有 LLM 侧的 `timeout`(网络超时、限流、5xx)与 `malformed`(流截断、JSON 解析失败)会触发引擎内自动重连:同一次 `run` 内重发原始输入,并附加 `[turn_retried]` 块携带上一次的部分输出,避免工具重复执行。默认最多重连 2 次,线性退避(基数 250ms);超限后该轮以 `failed` 收场。工具错误从不重试——它们作为 `tool_call_output` 反馈给模型,由模型决定下一步。
+只有 LLM 侧的 `timeout`(网络超时、传输层断连如 socket 被对端关闭、限流、5xx,以及 `insufficient_user_quota` 这类瞬时的供应商额度/订阅错误)与 `malformed`(流截断、JSON 解析失败)会触发引擎内自动重连:同一次 `run` 内重发原始输入,并附加 `[turn_retried]` 块携带上一次的部分输出,避免工具重复执行。默认最多重连 5 次,指数退避并设上限(基数 250ms、上限 30s:250ms、500ms、1s、2s、4s,总耐心约 7.75s——所有可重试类别共用一张时间表,向较慢的类别递增,头几步仍与传输层抖动所需的一样快);超限后该轮以 `failed` 收场。每次失败的 `request_end` 会以 `retry_in_ms` 宣告计划中的等待(与实际休眠同一公式),Web App 据此实时倒计时,并提供「立即重试」(经 `Session.skipReconnectWait` 跳过剩余等待——重试计数不变)与「放弃」(普通中断;引擎的退避中中断路径结束本轮)两个内联按钮。压缩请求使用独立的更小上限(重试 3 次):压缩失败本就有优雅语义——保留原上下文、等下一次触发再试——快速失败好过让会话干等。鉴权错误在任何重试启发式之前判定、从不重试:请求以专属终态 `auth` 收场(Session 锁定的只是模型引用,凭据在会话装载时取自当前 Project 配置),Web App 据此禁用该 Session 的输入框,直到该模型的凭据被更新(更新后自动解锁)或用户点击「重试」。工具错误从不重试——它们作为 `tool_call_output` 反馈给模型,由模型决定下一步。
## 上下文压缩(Compaction)
diff --git a/packages/docs/content/architecture.en.md b/packages/docs/content/architecture.en.md
index 24f8be2..2bf5b8e 100644
--- a/packages/docs/content/architecture.en.md
+++ b/packages/docs/content/architecture.en.md
@@ -129,7 +129,7 @@ The Server keeps an additional SQLite index (users, authorization, usage stats)
## Key design decisions
- **One protocol, three jobs**: OmniMessage is simultaneously the SDK's external interface, the Trace on-disk format and the engine's internal currency — what streams, what is stored and what the model sees are the same thing.
-- **Errors converge into messages**: the LLM and Environment never throw into the engine; results carry a five-value `stop_reason` (`completed | failed | aborted | timeout | malformed`), and only LLM-side `timeout / malformed` trigger an in-run reconnect.
+- **Errors converge into messages**: the LLM and Environment never throw into the engine; results carry a six-value `stop_reason` (`completed | failed | aborted | timeout | malformed | auth`), and only LLM-side `timeout / malformed` trigger an in-run reconnect (up to 5 times with an exponential-with-ceiling backoff — `timeout` covers network timeouts, transport disconnects and transient provider quota errors; `auth` stops like `failed`).
- **A thin model layer**: core defines only `LLMInterface`; provider adaptation lives entirely in AgentHub (`@prismshadow/agenthub`), which is what makes any OpenAI-compatible endpoint reachable. See [Models & Providers](/models).
Source entry points: `packages/core/src/engine/context-engine.ts`, `packages/core/src/interfaces.ts`.
diff --git a/packages/docs/content/architecture.zh.md b/packages/docs/content/architecture.zh.md
index 3cd7ed6..0d9b8b9 100644
--- a/packages/docs/content/architecture.zh.md
+++ b/packages/docs/content/architecture.zh.md
@@ -129,7 +129,7 @@ Server 额外维护一个 SQLite 索引库(用户、授权、用量统计),但
## 关键设计决策
- **一个协议,三种职责**:OmniMessage 同时是 SDK 对外接口、Trace 落盘格式与引擎内部通货——「流出去的」「存下来的」「模型看到的」是同一种东西。
-- **错误收敛为消息**:LLM 与 Environment 从不向引擎抛异常;结果携带五值 `stop_reason`(`completed | failed | aborted | timeout | malformed`),仅 LLM 侧的 `timeout / malformed` 触发引擎内重连。
+- **错误收敛为消息**:LLM 与 Environment 从不向引擎抛异常;结果携带六值 `stop_reason`(`completed | failed | aborted | timeout | malformed | auth`),仅 LLM 侧的 `timeout / malformed` 触发引擎内重连(至多 5 次、指数退避设上限——`timeout` 涵盖网络超时、传输层断连与瞬时的供应商额度错误;`auth` 与 `failed` 同样直接停止)。
- **薄模型层**:core 只定义 `LLMInterface`,Provider 适配全部下沉到 AgentHub(`@prismshadow/agenthub`),因此支持任意 OpenAI 兼容端点,见[模型与 Provider](/models)。
源码入口:`packages/core/src/engine/context-engine.ts`、`packages/core/src/interfaces.ts`。
diff --git a/packages/docs/content/interfaces.en.md b/packages/docs/content/interfaces.en.md
index 4788f36..2c96209 100644
--- a/packages/docs/content/interfaces.en.md
+++ b/packages/docs/content/interfaces.en.md
@@ -50,18 +50,21 @@ The generator yields `partial_*` fragments and complete messages, emits Token us
```ts
interface LLMOutcome {
- status: StopReason; // completed | timeout | malformed | aborted | failed
- message?: string; // display text when failed
+ status: StopReason; // completed | timeout | malformed | aborted | failed | auth
+ message?: string; // failure detail: on failed/auth, and on timeout/malformed when a
+ // concrete error was caught — carried onto request_end so the
+ // errors panel shows the real reason behind a retried request
}
```
| status | Meaning | Engine reaction |
| --- | --- | --- |
| `completed` | finished normally (token_usage already emitted) | proceed |
-| `timeout` | timeout / lost connection | auto-reconnect within the run |
+| `timeout` | timeout / transport disconnect / transient provider quota error | auto-reconnect within the run |
| `malformed` | response parse failure | auto-reconnect within the run |
| `aborted` | user interrupt | stop, hand back to the user |
-| `failed` | non-retryable (auth/params, …) | stop, hand back to the user |
+| `failed` | non-retryable (params, …) | stop, hand back to the user |
+| `auth` | credentials rejected | stop like `failed`; hosts gate input until the model's API key is updated |
Implementation constraints: never throw; no internal retries — reconnecting is the engine's job (see [The Agent Loop](/agent-loop)).
diff --git a/packages/docs/content/interfaces.zh.md b/packages/docs/content/interfaces.zh.md
index 7214d4f..60754a5 100644
--- a/packages/docs/content/interfaces.zh.md
+++ b/packages/docs/content/interfaces.zh.md
@@ -50,18 +50,21 @@ interface GenerativeModelParameters {
```ts
interface LLMOutcome {
- status: StopReason; // completed | timeout | malformed | aborted | failed
- message?: string; // failed 时的展示文案
+ status: StopReason; // completed | timeout | malformed | aborted | failed | auth
+ message?: string; // 失败详情:failed/auth 时携带;timeout/malformed 捕获到具体
+ // 错误时也携带——透传到 request_end,错误面板据此展示被重试
+ // 请求背后的真实原因
}
```
| status | 含义 | 引擎的反应 |
| --- | --- | --- |
| `completed` | 正常完成(已产出 token_usage) | 继续下一步 |
-| `timeout` | 超时/断连 | 同一 run 内自动重连 |
+| `timeout` | 超时/传输层断连/瞬时的供应商额度错误 | 同一 run 内自动重连 |
| `malformed` | 响应解析失败 | 同一 run 内自动重连 |
| `aborted` | 用户中断 | 停止交还用户 |
-| `failed` | 鉴权/参数等不可重试错误 | 停止交还用户 |
+| `failed` | 参数等不可重试错误 | 停止交还用户 |
+| `auth` | 凭据被拒绝 | 与 `failed` 同样停止;宿主据此禁用输入,直到该模型的 API key 被更新 |
实现约束:从不抛异常;不做内部重试(重连是引擎的职责,见 [Agent 运行循环](/agent-loop))。
diff --git a/packages/docs/content/message-flow.en.md b/packages/docs/content/message-flow.en.md
index 25d0e56..2ae3045 100644
--- a/packages/docs/content/message-flow.en.md
+++ b/packages/docs/content/message-flow.en.md
@@ -102,7 +102,7 @@ Therefore **a renderer must never reconstruct the context from arrival order**
| Case | Observable order on the stream |
| --- | --- |
| User interrupt | (messages produced so far) → the `abort` event — the last message before `run` returns; carry-over goes to the model context only, never streamed, never written to Trace |
-| Automatic reconnect | `request_end(timeout \| malformed)` → a fresh `request_begin`; the `[turn_retried]` block is model-visible only |
+| Automatic reconnect | `request_end(timeout \| malformed)` → a fresh `request_begin` (up to 5 times, exponential backoff with a 30s ceiling; `timeout` covers transport disconnects and transient provider quota errors too); the `[turn_retried]` block is model-visible only |
| Compaction | `compaction_begin` → the compaction request runs against the old context (its streamed output is **not** forwarded, only written to Trace) → that request's `token_usage` → `compaction_end(status)` |
| max_turns reached | a length notice → the run ends; unsubmitted input is kept as carry-over |
| The Prompt itself | written to Trace, not echoed back onto the stream (the caller already has it) |
diff --git a/packages/docs/content/message-flow.zh.md b/packages/docs/content/message-flow.zh.md
index 80d954e..82cb61e 100644
--- a/packages/docs/content/message-flow.zh.md
+++ b/packages/docs/content/message-flow.zh.md
@@ -101,7 +101,7 @@ Human ──run(newMessages)──► engine
| 情形 | 流上可见的顺序 |
| --- | --- |
| 用户中断 | (已产出的消息)→ `abort` 事件——`run` 返回前的最后一条;补发内容只进模型上下文,不上流、不进 Trace |
-| 自动重连 | `request_end(timeout \| malformed)` → 新的 `request_begin`;`[turn_retried]` 块仅模型可见 |
+| 自动重连 | `request_end(timeout \| malformed)` → 新的 `request_begin`(至多 5 次,指数退避、上限 30s;`timeout` 同样涵盖传输层断连与瞬时的供应商额度错误);`[turn_retried]` 块仅模型可见 |
| 上下文压缩 | `compaction_begin` → 压缩请求在旧上下文中执行(其流式输出**不上行**,只写 Trace)→ 该请求的 `token_usage` → `compaction_end(status)` |
| 达到 max_turns | 长度提示消息 → 结束;未提交的输入按补发保留 |
| Prompt 本身 | 写入 Trace,但不回流(输入方已有) |
diff --git a/packages/docs/content/omni-message.en.md b/packages/docs/content/omni-message.en.md
index 22129bc..df6752d 100644
--- a/packages/docs/content/omni-message.en.md
+++ b/packages/docs/content/omni-message.en.md
@@ -1,6 +1,6 @@
---
title: The OmniMessage Protocol
-description: One envelope, three message types, a five-value stop_reason — the unified protocol behind the SDK, the Trace and SSE, field by field.
+description: One envelope, three message types, a six-value stop_reason — the unified protocol behind the SDK, the Trace and SSE, field by field.
---
OmniMessage is PenguinHarness's unified message protocol: the SDK yields it, the Trace stores it line by line, and the Server pushes it verbatim over SSE. What streams, what is stored and what the model sees are one structure — there is no second format between front end, back end and storage.
@@ -190,6 +190,10 @@ interface RequestBeginPayload {
interface RequestEndPayload {
type: "request_end";
status: StopReason; // "completed" is the mechanical commit criterion for replay
+ message?: string; // failure detail (LLMOutcome.message), non-completed only:
+ // the real reason behind a retried/failed Request (e.g. a
+ // provider quota code) — read by the Cost center's errors
+ // panel; additive, old Traces replay unchanged
}
interface ApprovalDecisionPayload {
@@ -242,19 +246,20 @@ interface SubagentPayload {
## stop_reason
-A five-value enum used across messages and interface results (`LLMOutcome.status` uses the same set — see [Core Interfaces](/interfaces)):
+A six-value enum used across messages and interface results (`LLMOutcome.status` uses the same set — see [Core Interfaces](/interfaces)):
```ts
-type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed";
+type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed" | "auth";
```
| Value | Meaning | Engine reaction |
| --- | --- | --- |
| `completed` | finished normally | continue |
| `aborted` | user interrupt | stop, hand back to the user |
-| `timeout` | LLM timeout / lost connection | LLM side only: auto-reconnect within the run |
+| `timeout` | LLM timeout / transport disconnect / transient provider quota error | LLM side only: auto-reconnect within the run |
| `malformed` | parse failure / truncated stream | LLM side only: auto-reconnect within the run |
| `failed` | other non-retryable error | stop, hand back to the user |
+| `auth` | the provider rejected the credentials | stop like `failed`; hosts gate input until the model's API key is updated (credentials come from the current Project config) |
Errors never cross an interface boundary as exceptions — they *are* messages. See [The Agent Loop](/agent-loop).
diff --git a/packages/docs/content/omni-message.zh.md b/packages/docs/content/omni-message.zh.md
index c9a9b1b..d5609da 100644
--- a/packages/docs/content/omni-message.zh.md
+++ b/packages/docs/content/omni-message.zh.md
@@ -1,6 +1,6 @@
---
title: OmniMessage 协议
-description: 一个信封、三类消息、五值 stop_reason——贯穿 SDK、Trace 与 SSE 的统一消息协议,逐字段定义。
+description: 一个信封、三类消息、六值 stop_reason——贯穿 SDK、Trace 与 SSE 的统一消息协议,逐字段定义。
---
OmniMessage 是 PenguinHarness 的统一消息协议:SDK 对外产出它,Trace 逐行存储它,Server 经 SSE 原样推送它。「流出去的」「存下来的」「模型看到的」是同一种结构,前后端与存储之间不存在第二套格式。
@@ -189,6 +189,9 @@ interface RequestBeginPayload {
interface RequestEndPayload {
type: "request_end";
status: StopReason; // completed 是回放判定「该轮已提交」的机械标准
+ message?: string; // 失败详情(LLMOutcome.message),仅非 completed 携带:
+ // 被重试/失败的 Request 背后的真实原因(如供应商额度码),
+ // 供成本中心错误面板读取;增量字段,旧 Trace 回放不受影响
}
interface ApprovalDecisionPayload {
@@ -241,19 +244,20 @@ interface SubagentPayload {
## stop_reason
-五值枚举,贯穿消息与接口返回(`LLMOutcome.status` 使用同一集合,见[接口契约](/interfaces)):
+六值枚举,贯穿消息与接口返回(`LLMOutcome.status` 使用同一集合,见[接口契约](/interfaces)):
```ts
-type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed";
+type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed" | "auth";
```
| 值 | 语义 | 引擎的反应 |
| --- | --- | --- |
| `completed` | 正常完成 | 继续 |
| `aborted` | 用户中断 | 停止并交还用户 |
-| `timeout` | LLM 超时/断连 | 仅 LLM 侧:同一 run 内自动重连 |
+| `timeout` | LLM 超时/传输层断连/瞬时的供应商额度错误 | 仅 LLM 侧:同一 run 内自动重连 |
| `malformed` | 响应解析失败/流截断 | 仅 LLM 侧:同一 run 内自动重连 |
| `failed` | 其他不可重试错误 | 停止并交还用户 |
+| `auth` | 供应商拒绝了凭据 | 与 `failed` 同样停止;宿主据此禁用输入,直到该模型的 API key 被更新(凭据取自当前 Project 配置) |
错误从不以异常形式穿过接口边界——它们就是消息,见 [Agent 运行循环](/agent-loop)。
diff --git a/packages/docs/content/server-api.en.md b/packages/docs/content/server-api.en.md
index bbf2a52..391c21d 100644
--- a/packages/docs/content/server-api.en.md
+++ b/packages/docs/content/server-api.en.md
@@ -89,6 +89,8 @@ Member writes are owner-only.
Every endpoint that names a model takes the complete `(provider, modelId)` pair. Nothing is inferred: a request carrying only one half is a 400, never a lookup. Where the reference itself is optional (Session creation, Schedules), omitting both halves selects the Project's default model.
+`PUT /models` also invalidates the Project's cached Session runtimes (same effective-value semantics as a vault update): no hot swap into a run already in flight, but the next Task on any Session of the Project re-resumes and reads the new `api_key` / `base_url`. It additionally publishes a `credentials_updated` event to the Project's open Session channels (see Streaming below), and the models response carries `updatedAt` (the config file's mtime) — the Web App compares it against the last auth failure to decide whether an auth-dead composer should stay disabled.
+
### Agents
The paths below omit the `/api/projects/:projectId` prefix.
@@ -148,6 +150,7 @@ The paths below omit the `/api/sessions/:sessionId` prefix. For the storage mode
| POST | /steer | Mid-run steering: `{text}` queues a message for the running Task (delivered between turns as a standalone `[user_steering]` user message) → 202; 409 `not_running` when no Task is in progress |
| POST | /approvals/:toolCallId | Approval decision: `{decision}` is `allow` or `deny` → 204 |
| POST | /abort | Interrupt the current Task: 202 when triggered, 204 when idle |
+| POST | /retry-now | "Retry now" on the reconnect countdown: skips the in-progress backoff wait, firing the next retry immediately (attempt counter unchanged) → 200 `{skipped}` — `skipped:false` is the benign "no wait in progress" case, never an error |
| POST | /compact | Trigger context compaction: 202; 409 `nothing_to_compact` when there is nothing to compact |
| GET | /files?path= | Browse the Workspace directory |
| GET | /files/content?path=&download=&preview= | Read a Workspace file (`download=1` serves it as an attachment, `preview=1` renders it in a sandbox — see below) |
@@ -230,6 +233,7 @@ export type ServerEvent =
| { type: "task_state"; state: "idle" | "running" | "compacting" }
| { type: "session_title"; sessionId: string; title: string }
| { type: "resync_required" }
+ | { type: "credentials_updated" }
| { type: "hello" }
| { type: "session_created"; projectId: string; agentId: string; sessionId: string; source: SessionSource }
| { type: "schedule_fired"; projectId: string; agentId: string; name: string; sessionId: string }
@@ -242,6 +246,7 @@ export type ServerEvent =
| task_state | The Session's run state flips (idle / running / compacting) |
| session_title | The model-generated title after the first turn has been persisted |
| resync_required | The Last-Event-ID was evicted from the buffer; the client must refetch history |
+| credentials_updated | The Project's model credentials changed (`PUT /models`): cached runtimes were invalidated, so the client clears any auth-dead composer state |
| hello | Handshake on the user channel |
| session_created | A new Session was registered (e.g. a subagent session) |
| schedule_fired | A scheduled task fired and was delivered |
diff --git a/packages/docs/content/server-api.zh.md b/packages/docs/content/server-api.zh.md
index 9a4738f..2fdb861 100644
--- a/packages/docs/content/server-api.zh.md
+++ b/packages/docs/content/server-api.zh.md
@@ -89,6 +89,8 @@ curl -c cookies.txt -H "Content-Type: application/json" \
所有涉及模型的接口都要求完整的 `(provider, modelId)` 二元组,不做任何推断:只带一半的请求一律 400,绝不会退化为一次查找。模型引用本身可省略的场景(创建 Session、定时任务)省略的是整对,两半都不给即选用 Project 默认模型。
+`PUT /models` 同时会使该 Project 已缓存的 Session 运行时失效(与 vault 更新同一套生效语义):进行中的运行不做热替换,但该 Project 下任何 Session 的下一个 Task 都会重新装载并读到新的 `api_key` / `base_url`。它还会向该 Project 已打开的 Session 通道发布 `credentials_updated` 事件(见下文「流式推送」),且模型响应携带 `updatedAt`(配置文件 mtime)——Web App 用它与最近一次鉴权失败的时间比较,决定鉴权失败的输入框是否继续禁用。
+
### Agent
以下路径均省略前缀 `/api/projects/:projectId`。
@@ -148,6 +150,7 @@ Schedule 写操作仅限 Owner。新建 Session 模式的任务,`modelId` 与
| POST | /steer | 运行中插话:`{text}` 为运行中的 Task 排队一条消息(作为独立的 `[user_steering]` 用户消息随下一轮送达)→ 202;无 Task 运行返回 409 `not_running` |
| POST | /approvals/:toolCallId | 审批决定:`{decision}` 取 `allow` 或 `deny` → 204 |
| POST | /abort | 中断当前 Task:已触发返回 202,无任务返回 204 |
+| POST | /retry-now | 重连倒计时上的「立即重试」:跳过进行中的退避等待、立刻发起下一次重试(重试计数不变)→ 200 `{skipped}`——`skipped:false` 表示当前没有等待可跳过(良性空操作,非错误) |
| POST | /compact | 触发上下文压缩:202;无可压缩内容返回 409 `nothing_to_compact` |
| GET | /files?path= | 浏览 Workspace 目录 |
| GET | /files/content?path=&download=&preview= | 读取 Workspace 文件(`download=1` 时作为附件下载,`preview=1` 以沙箱方式预览 —— 见下) |
@@ -229,6 +232,7 @@ export type ServerEvent =
| { type: "task_state"; state: "idle" | "running" | "compacting" }
| { type: "session_title"; sessionId: string; title: string }
| { type: "resync_required" }
+ | { type: "credentials_updated" }
| { type: "hello" }
| { type: "session_created"; projectId: string; agentId: string; sessionId: string; source: SessionSource }
| { type: "schedule_fired"; projectId: string; agentId: string; name: string; sessionId: string }
@@ -241,6 +245,7 @@ export type ServerEvent =
| task_state | Session 运行状态翻转(idle / running / compacting) |
| session_title | 首轮后模型生成的标题已持久化 |
| resync_required | Last-Event-ID 已被缓冲区淘汰,客户端须重新拉取历史 |
+| credentials_updated | Project 模型凭据已变更(`PUT /models`):缓存运行时已失效,客户端应清除鉴权失败的输入框禁用态 |
| hello | 用户通道连接握手 |
| session_created | 新 Session 注册(如子 Agent 会话) |
| schedule_fired | 定时任务已触发并发送 |
diff --git a/packages/docs/content/web-app.en.md b/packages/docs/content/web-app.en.md
index 42fe68e..cfc4fc7 100644
--- a/packages/docs/content/web-app.en.md
+++ b/packages/docs/content/web-app.en.md
@@ -50,7 +50,9 @@ There are four approval modes: `allow-all`, `deny-all`, `read-only` (only read-o
- Typing `/` opens the slash menu: trigger context compaction (`/compact`) or toggle installed Skills — chosen Skills are sent along with the message in a `[use_skills]` block;
- While a Task is running the input stays live and the toolbar keeps a single action button: an empty composer shows **Stop**, and typing turns it into **Send**, whose behavior follows the **mid-run send mode** from the toolbar's More-settings popover (a compact extensible settings panel, also available in draft state; the choice is remembered): **Steer** (default) delivers the text mid-run as a `[user_steering]` user message with the next turn, **Queue** holds the whole message server-side as a follow-up and auto-sends it as an ordinary new message when the run finishes (an "N queued" hint shows near the input until then; the queue survives page reloads);
- Typing `@` mentions another Agent to hand the conversation over to it;
-- When human approval is required, tool calls show inline allow/deny buttons in the message stream; the approval mode can be changed mid-Session.
+- When human approval is required, tool calls show inline allow/deny buttons in the message stream; the approval mode can be changed mid-Session;
+- While the engine waits out a reconnect backoff (≥2s), the retry line shows a live countdown to the next attempt with inline **Retry now** (skips the remaining wait) and **Give up** (the ordinary abort) controls;
+- When the model API rejects the Session's credentials (an authentication failure), the composer grays out and disables — recoverably: the Session pins only the model reference, and credentials come from the current Project config. The notice's primary button opens the Models page; saving a new API key there unlocks the composer by itself (open tabs unlock live via a `credentials_updated` event, and after a reload the composer stays unlocked because the credential update is newer than the recorded auth failure). A "Retry" button clears the state manually for another attempt (it re-arms if the key is still bad), a completed request always clears it, and "New Session" remains as the escape to a fresh draft. The disabled composer keeps its draft selectable, so a long message that failed to send can still be copied out.
### Files Panel
diff --git a/packages/docs/content/web-app.zh.md b/packages/docs/content/web-app.zh.md
index 208af69..44fba67 100644
--- a/packages/docs/content/web-app.zh.md
+++ b/packages/docs/content/web-app.zh.md
@@ -50,7 +50,9 @@ penguin web
- 输入 `/` 打开快捷菜单:触发上下文压缩(`/compact`),或勾选已安装的 Skill——所选 Skill 会以 `[use_skills]` 块随消息发送;
- Task 运行期间输入框保持可用,工具条只保留一个操作按钮:输入框为空时是**停止**,一旦输入内容即变为**发送**,其行为遵循工具条「更多设置」弹出分组中的**运行中发送方式**(一个可扩展的设置面板,草稿态同样可设,选择会被记忆):**插话**(默认)把文字以 `[user_steering]` 用户消息随下一轮送达运行中的 Agent;**排队** 把整条消息暂存在服务端,本轮结束后自动作为普通新消息发出(期间在输入框附近显示「N 条已排队」提示;队列存放在服务端,刷新页面不丢失);
- 输入 `@` 提及其他 Agent,将会话交接给它;
-- 需要人工审批时,工具调用在消息流中内联显示“允许 / 拒绝”按钮;审批模式在会话中途可随时调整。
+- 需要人工审批时,工具调用在消息流中内联显示“允许 / 拒绝”按钮;审批模式在会话中途可随时调整;
+- 引擎在重连退避等待(≥2 秒)期间,重试提示行会实时倒计时到下一次尝试,并内联提供**立即重试**(跳过剩余等待)与**放弃**(普通中断)两个按钮;
+- 模型 API 拒绝该 Session 的凭据(鉴权失败)时,输入框会置灰禁用——但可恢复:Session 锁定的只是模型引用,凭据取自当前 Project 配置。提示条的主按钮跳转到模型配置页;在那里保存新的 API key 后输入框会自动解锁(已打开的标签页经 `credentials_updated` 事件即时解锁;刷新后也保持解锁,因为凭据更新时间晚于记录的鉴权失败时间)。「重试」按钮可手动清除该状态再试一次(key 仍无效时会重新变灰),一次成功的请求总会清除该状态,「新建会话」仍作为跳到全新草稿的出口。禁用态的输入框保留草稿且可选中——发送失败的长消息仍能复制出来。
### 文件面板
diff --git a/packages/server/src/api/types.ts b/packages/server/src/api/types.ts
index d0e5943..1fb725e 100644
--- a/packages/server/src/api/types.ts
+++ b/packages/server/src/api/types.ts
@@ -234,6 +234,13 @@ export interface ModelsResponse {
defaultModel?: ModelRefDto;
/** Vision model used as a proxy reader for read_image (describes images when the session model has vision=false). */
visionModel?: ModelRefDto;
+ /**
+ * When the Project's model/credential config last changed (ISO; the config file's mtime,
+ * so it survives restarts). The web's auth-dead gate compares it against the last auth
+ * abort: an abort OLDER than the last credential update no longer disables the composer
+ * (the key was fixed since). Absent when the Project has no config file yet.
+ */
+ updatedAt?: string;
models: ModelInfo[];
}
@@ -609,6 +616,16 @@ export interface ApprovalDecisionRequest {
decision: "allow" | "deny";
}
+/**
+ * POST /api/sessions/:sessionId/retry-now — skip the in-progress reconnect backoff and
+ * fire the next retry immediately (the "retry now" button on the reconnect countdown).
+ * `skipped: false` is the benign "no reconnect wait in progress" case (idle session, or
+ * the wait elapsed in a timing race), not an error.
+ */
+export interface RetryNowResponse {
+ skipped: boolean;
+}
+
// ---------------------------------------------------------------------------
// SSE server events (OmniMessage uses the default event, only server_event here)
// ---------------------------------------------------------------------------
@@ -626,6 +643,14 @@ export type ServerEvent =
| { type: "session_title"; sessionId: string; title: string }
/** Last-Event-ID has been evicted from the buffer: the frontend should re-fetch the history endpoint before continuing to consume this connection. */
| { type: "resync_required" }
+ /**
+ * The Project's model credentials changed (PUT /models): cached runtimes have been
+ * invalidated server-side, so an auth-dead Session can continue — the frontend clears
+ * its auth-dead composer state immediately. Published to every existing Session channel
+ * of the Project; tabs without a live channel learn the same fact from the models
+ * response's `updatedAt` on their next load.
+ */
+ | { type: "credentials_updated" }
/** Placeholder handshake on the user channel (reserved for automated task notifications). */
| { type: "hello" }
/** New session registered (pushed over the parent session's channel for subagent sessions): frontend refreshes the list in place. */
diff --git a/packages/server/src/http/routes/models.ts b/packages/server/src/http/routes/models.ts
index 722c38a..2686e0a 100644
--- a/packages/server/src/http/routes/models.ts
+++ b/packages/server/src/http/routes/models.ts
@@ -10,11 +10,25 @@ import type {
ModelsUpdateRequest,
ModelTestRequest,
ModelUpdateEntry,
+ ServerEvent,
} from "../../api/types.js";
import type { AppEnv } from "../../auth/middleware.js";
import { badRequest, readJson, requireString, requireValidId } from "../validate.js";
import type { AppDeps } from "../../app.js";
+/**
+ * Live unlock for auth-dead composers: after a models/credential update, publish
+ * `credentials_updated` to every EXISTING channel of this Project's Sessions (`peek` —
+ * never creates channels: a tab that isn't subscribed learns the same fact from the models
+ * response's `updatedAt` when it next loads).
+ */
+function publishCredentialsUpdated(deps: AppDeps, projectId: string): void {
+ const event: ServerEvent = { type: "credentials_updated" };
+ for (const row of deps.sessionsRepo.listByProject(projectId)) {
+ deps.channels.peek(row.sessionId)?.publish(event, "server_event");
+ }
+}
+
/** Validate a paired reference object ({ provider, modelId }); shape mismatch throws 400. */
function parseRef(value: unknown, label: string): ModelRefDto {
if (value === null || typeof value !== "object" || Array.isArray(value)) {
@@ -143,7 +157,15 @@ export function modelsRoutes(deps: AppDeps): Hono {
const projectId = requireValidId(c, "projectId");
deps.projectService.requireProjectOwner(c.var.user.userId, projectId);
const req = parseModelsUpdate(await readJson(c));
- return c.json(await deps.projectConfigService.updateModels(projectId, req));
+ const res = await deps.projectConfigService.updateModels(projectId, req);
+ // Effective-value semantics (mirrors the vault route): no hot swap into a Task already
+ // in flight, but every cached runtime in this Project is invalidated — the next Task on
+ // any of its Sessions re-resumes and reads the new api_key / base_url. Without this, a
+ // key edit would not reach a loaded Session until the 30-minute idle sweep or a restart.
+ deps.manager.invalidateProjectRuntimes(projectId);
+ // Live unlock: open tabs clear their auth-dead composer immediately (no reload needed).
+ publishCredentialsUpdated(deps, projectId);
+ return c.json(res);
});
// Connectivity test (owner): the model reference `(provider, modelId)` is sent as a pair
diff --git a/packages/server/src/http/routes/sessions.ts b/packages/server/src/http/routes/sessions.ts
index 7787ee1..f87eeb9 100644
--- a/packages/server/src/http/routes/sessions.ts
+++ b/packages/server/src/http/routes/sessions.ts
@@ -21,6 +21,7 @@ import type {
SessionCreateResponse,
SessionResponse,
SessionsResponse,
+ RetryNowResponse,
TaskCreateResponse,
} from "../../api/types.js";
import { PREVIEW_TOKEN_TTL_MS, resolvePreviewTarget } from "../../services/preview-token.js";
@@ -420,6 +421,16 @@ export function sessionsRoutes(deps: AppDeps): Hono {
return c.body(null, aborted ? 202 : 204);
});
+ // "Retry now" on the reconnect countdown: skip the remaining backoff wait and fire the
+ // next retry immediately (attempt counter unchanged). Benign either way — 200 with
+ // skipped:false when no reconnect wait is in progress, so a timing race (the wait
+ // elapsed just before the click) never surfaces as an error.
+ app.post("/:sessionId/retry-now", (c) => {
+ const row = resolveSession(c);
+ const skipped = deps.manager.retryNow(row.sessionId);
+ return c.json({ skipped } satisfies RetryNowResponse);
+ });
+
app.post("/:sessionId/compact", async (c) => {
const row = resolveSession(c);
const { sessionId } = await deps.manager.startCompact(row.sessionId);
diff --git a/packages/server/src/runtime/session-manager.ts b/packages/server/src/runtime/session-manager.ts
index b767212..d323b81 100644
--- a/packages/server/src/runtime/session-manager.ts
+++ b/packages/server/src/runtime/session-manager.ts
@@ -78,6 +78,8 @@ export interface RuntimeSession {
compactability(): CompactAvailability;
/** Queues a mid-run steering message (core `Session.steer`); false when no Task is running. */
steer(text: string): boolean;
+ /** Skips the in-progress reconnect backoff, firing the next retry immediately (core `Session.skipReconnectWait`); false when no wait is in progress. */
+ skipReconnectWait(): boolean;
toolPermission(name: string): "r" | "rw" | undefined;
/**
* Out-of-band one-shot request for title generation (core `Session.generateTitle`,
@@ -396,6 +398,22 @@ export class SessionManager {
this.agentGenerations.set(key, this.generationOf(projectId, agentId) + 1);
}
+ /**
+ * After a Project's models/credentials change: invalidate every cached runtime in this
+ * Project, so the next Task re-resumes with the new api_key / base_url. Same
+ * effective-value semantics as invalidateAgentRuntimes — no hot swap into a Task already
+ * in flight. Iterating the active table is complete here, not a shortcut: the generation
+ * map only matters for runtimes that are already cached, and an Agent with no cached
+ * entry builds fresh through the loader anyway.
+ */
+ invalidateProjectRuntimes(projectId: string): void {
+ const agentIds = new Set();
+ for (const e of this.entries.values()) {
+ if (e.projectId === projectId) agentIds.add(e.agentId);
+ }
+ for (const agentId of agentIds) this.invalidateAgentRuntimes(projectId, agentId);
+ }
+
// —— Task / compaction drive ——
/**
@@ -558,6 +576,23 @@ export class SessionManager {
entry.lastActivityMs = Date.now();
}
+ /**
+ * "Retry now" on the reconnect countdown: skip the in-progress backoff wait and fire
+ * the next retry immediately (the attempt counter is unchanged — the skipped wait does
+ * not consume an extra attempt). Returns false as a benign no-op when the session has
+ * no active runtime or no reconnect wait is in progress — timing races (the wait
+ * elapsed right before the click) must not surface as errors. Mirrors the steer seam:
+ * manager → RuntimeSession → core Session → engine.
+ */
+ retryNow(sessionId: string): boolean {
+ const entry = this.entries.get(sessionId);
+ // Both running Tasks and compactions can be parked in a reconnect backoff.
+ if (!entry || entry.status === "idle") return false;
+ const skipped = entry.session.skipReconnectWait();
+ if (skipped) entry.lastActivityMs = Date.now();
+ return skipped;
+ }
+
/**
* Interrupt the current Task/compaction: pending approvals converge to deny first,
* then the AbortSignal fires. Returns false if nothing is in progress (the route
diff --git a/packages/server/src/runtime/stream-error-watcher.ts b/packages/server/src/runtime/stream-error-watcher.ts
index d360171..344d988 100644
--- a/packages/server/src/runtime/stream-error-watcher.ts
+++ b/packages/server/src/runtime/stream-error-watcher.ts
@@ -9,21 +9,23 @@
* (its state wraps up accordingly, see close).
*
* LLM (source = `llm`): reads the status of `request_end` —
- * - `failed` → unexpected (not retryable: auth failure, invalid params, etc., needs a human);
+ * - `failed` / `auth` → unexpected (not retryable: credentials rejected, invalid params,
+ * etc., needs a human; both share the llm_failed code — no separate taxonomy);
* - `timeout` / `malformed` → expected (the engine already reconnects and retries, part
* of normal operation);
* - `aborted` / `completed` are not recorded (the former is a user-initiated interrupt,
* not an error).
*
- * The message uses the real reason: `request_end` only carries status, and **the only
- * place core carries the actual failure-reason text is the `abort` event's reason**
- * (e.g. `llm request error: 401 …` / `malformed response failed after N retries`). So a
- * `request_end` failure is first held pending, not persisted immediately, and is
- * resolved at the next request boundary:
+ * The message uses the real reason: `request_end` carries the failure detail on
+ * non-completed statuses (`message`, from LLMOutcome), and the `abort` event's reason is
+ * core's failure-reason prose (e.g. `llm request error: 401 …` / `malformed response
+ * failed after N retries`). A `request_end` failure is first held pending (status + its
+ * own detail), not persisted immediately, and is resolved at the next request boundary:
* - Immediately followed by `abort` → use its reason as the message (the real reason);
- * - Immediately followed by `request_begin` (the engine is retrying) → no reason text
- * left to wait for, use the status text;
- * - Still unresolved when the run ends → close persists it as a fallback.
+ * - Immediately followed by `request_begin` (the engine is retrying — no abort will ever
+ * arrive) → use the staged request_end's own `message` (the real detail, e.g. a quota
+ * code), falling back to the generic status text when it carried none;
+ * - Still unresolved when the run ends → close persists it the same way.
* Exception: when reason is a user-interrupt message (`aborted …`), it's not trusted —
* "the user clicked stop during backoff" isn't the reason for this timeout, so the
* status text is used instead (that timeout is a genuine failure and is still recorded).
@@ -71,11 +73,14 @@ export const TOOL_NAMES_MAX = 1000;
export const ORIGIN_CTX_MAX = 200;
/** Recorded LLM failure states (`aborted` / `completed` are not errors and aren't included here). */
-type LlmFailure = "failed" | "timeout" | "malformed";
+type LlmFailure = "failed" | "timeout" | "malformed" | "auth";
/** LLM failure state → error code, classification, and fallback message (used when the abort reason isn't available). */
const LLM_FAILURES: Record = {
failed: { code: "llm_failed", kind: "unexpected", text: "LLM request failed (not retryable)." },
+ // Credentials rejection shares the failed bucket (no new taxonomy): same code/kind, the
+ // real reason arrives via the abort reason or the event's own message as usual.
+ auth: { code: "llm_failed", kind: "unexpected", text: "LLM request failed (not retryable)." },
timeout: {
code: "llm_timeout",
kind: "expected",
@@ -92,7 +97,7 @@ const LLM_FAILURES: Record();
+ /** LLM failures awaiting a real reason: origin → failure state + the request_end's own detail (see file header; each session has at most one in-flight Request). */
+ private readonly pending = new Map();
/** Names of in-flight tool calls: `origin \0 tool_call_id` → tool name (dequeued once output arrives). */
private readonly toolNames = new Map();
/** Subagent identity: origin → that subagent's `{agentId, sessionId}` (see the file header's attribution section). */
@@ -196,11 +201,23 @@ export class StreamErrorWatcher {
// —— LLM ——
private observeLlm(msg: OmniMessage): void {
- const p = msg.payload as { type?: string; status?: StopReason; reason?: string | null };
+ const p = msg.payload as {
+ type?: string;
+ status?: StopReason;
+ reason?: string | null;
+ message?: string;
+ };
const key = originKey(msg);
if (p.type === "request_end") {
this.flush(key); // Defensive: if a previous failure is still pending (normally resolved by request_begin), persist it first
- if (isLlmFailure(p.status)) this.pending.set(key, p.status);
+ if (isLlmFailure(p.status)) {
+ this.pending.set(key, {
+ status: p.status,
+ // The event's own failure detail (LLMOutcome.message): the message of record when
+ // no abort follows (the retry path).
+ ...(typeof p.message === "string" && p.message.trim() ? { message: p.message } : {}),
+ });
+ }
return;
}
// A new attempt begins (the engine is retrying): no reason text left to wait for the previous failure, persist using the status text.
@@ -221,13 +238,15 @@ export class StreamErrorWatcher {
* up from it (see file header).
*/
private flush(key: string, reason?: string | null): void {
- const status = this.pending.get(key);
- if (status === undefined) return;
+ const entry = this.pending.get(key);
+ if (entry === undefined) return;
this.pending.delete(key);
- const spec = LLM_FAILURES[status];
+ const spec = LLM_FAILURES[entry.status];
const trimmed = reason?.trim();
- // A user-interrupt message isn't a failure reason (see file header); fall back to the status text.
- const message = trimmed && !isUserAbortReason(trimmed) ? trimmed : spec.text;
+ // Message priority: the abort reason (core's failure prose) → the staged request_end's
+ // own detail (the retry path: no abort ever arrives) → the generic status text. A
+ // user-interrupt message isn't a failure reason (see file header).
+ const message = trimmed && !isUserAbortReason(trimmed) ? trimmed : (entry.message ?? spec.text);
this.errors.record({
source: "llm",
err: message,
diff --git a/packages/server/src/services/project-config-service.ts b/packages/server/src/services/project-config-service.ts
index f2f2c32..8339f7a 100644
--- a/packages/server/src/services/project-config-service.ts
+++ b/packages/server/src/services/project-config-service.ts
@@ -121,6 +121,20 @@ export class ProjectConfigService {
return projectConfigPath(this.root, projectId);
}
+ /**
+ * When the Project config (models/credentials) last changed: the config file's mtime as
+ * ISO — an honest persistent source that survives server restarts (every models update
+ * rewrites the file). undefined when the file doesn't exist yet.
+ */
+ private async configUpdatedAt(projectId: string): Promise {
+ try {
+ const st = await fs.stat(this.filePath(projectId));
+ return st.mtime.toISOString();
+ } catch {
+ return undefined;
+ }
+ }
+
/** Reads the raw TOML object; returns an empty object if the file doesn't exist (does not write to disk). */
async readRaw(projectId: string): Promise {
let raw: string;
@@ -390,9 +404,11 @@ export class ProjectConfigService {
provider: ref.provider,
modelId: ref.model_id,
});
+ const updatedAt = await this.configUpdatedAt(projectId);
return {
...(defaultRef !== undefined ? { defaultModel: toDto(defaultRef) } : {}),
...(visionRef !== undefined ? { visionModel: toDto(visionRef) } : {}),
+ ...(updatedAt !== undefined ? { updatedAt } : {}),
models,
};
}
diff --git a/packages/server/test/errors.test.ts b/packages/server/test/errors.test.ts
index d037e2f..6683441 100644
--- a/packages/server/test/errors.test.ts
+++ b/packages/server/test/errors.test.ts
@@ -406,6 +406,66 @@ describe("stream-error-watcher (LLM / Environment errors)", () => {
});
});
+ it("request_end(auth) shares the llm_failed/unexpected bucket; message takes the abort reason", () => {
+ // Credentials rejection is its own stop reason but no new error taxonomy: same code
+ // and kind as failed, resolved by the abort that follows like any failed exit.
+ const got = feed([
+ requestBegin(),
+ requestEnd("auth", "401 invalid x-api-key (invalid_api_key)"),
+ abortEvent("llm request error: 401 invalid x-api-key (invalid_api_key)"),
+ ]);
+ expect(got).toHaveLength(1);
+ expect(got[0]).toMatchObject({
+ source: "llm",
+ kind: "unexpected",
+ code: "llm_failed",
+ message: "llm request error: 401 invalid x-api-key (invalid_api_key)",
+ });
+ });
+
+ it("a retried failure keeps its real detail: request_end(timeout).message lands in the record", () => {
+ // The retry path: the engine reconnects (request_begin) and eventually succeeds, so no
+ // abort ever arrives for the staged failure — the request_end's own failure detail
+ // (LLMOutcome.message, e.g. a quota code) is the message of record, not the generic
+ // "timed out" status text. This is what the Cost center shows for a retried quota 403.
+ const got = feed([
+ requestBegin(),
+ requestEnd("timeout", "403 no active subscription (insufficient_user_quota)"),
+ requestBegin(),
+ requestEnd("completed"),
+ ]);
+ expect(got).toHaveLength(1);
+ expect(got[0]).toMatchObject({
+ source: "llm",
+ kind: "expected",
+ code: "llm_timeout",
+ message: "403 no active subscription (insufficient_user_quota)",
+ });
+ });
+
+ it("an abort reason still outranks the staged request_end detail (failed exit path)", () => {
+ const got = feed([
+ requestBegin(),
+ requestEnd("failed", "401 invalid x-api-key (invalid_api_key)"),
+ abortEvent("llm request error: 401 invalid x-api-key (invalid_api_key)"),
+ ]);
+ expect(got).toHaveLength(1);
+ // The abort's prose (with core's "llm request error" framing) wins over the raw detail.
+ expect(got[0]!.message).toBe("llm request error: 401 invalid x-api-key (invalid_api_key)");
+ });
+
+ it("interrupt during backoff with a staged detail: the detail wins over the status text", () => {
+ // The interrupt message is distrusted (not the failure's reason), but the staged
+ // request_end detail IS the failure's reason — prefer it over the generic status text.
+ const got = feed([
+ requestBegin(),
+ requestEnd("timeout", "403 quota exceeded (insufficient_user_quota)"),
+ abortEvent("aborted during reconnect backoff"),
+ ]);
+ expect(got).toHaveLength(1);
+ expect(got[0]!.message).toBe("403 quota exceeded (insufficient_user_quota)");
+ });
+
it("aborted (user clicked Stop) is not an error: not recorded; neither is completed", () => {
expect(
feed([
diff --git a/packages/server/test/followup.test.ts b/packages/server/test/followup.test.ts
index aa345d0..1ba062c 100644
--- a/packages/server/test/followup.test.ts
+++ b/packages/server/test/followup.test.ts
@@ -25,6 +25,7 @@ function parkingFakeSession(sessionId: string, runs: string[][]): RuntimeSession
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run(input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
runs.push(input.map((m) => (m.payload as { text?: string }).text ?? ""));
const tc = toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-fu" });
diff --git a/packages/server/test/models.test.ts b/packages/server/test/models.test.ts
index f19e8cf..2b1d1c5 100644
--- a/packages/server/test/models.test.ts
+++ b/packages/server/test/models.test.ts
@@ -13,7 +13,7 @@ import type { AddressInfo } from "node:net";
import { readFile, stat } from "node:fs/promises";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { MODEL_CATALOG } from "@prismshadow/penguin-core";
+import { MODEL_CATALOG, userText } from "@prismshadow/penguin-core";
import type {
ModelsResponse,
ModelTestResponse,
@@ -21,7 +21,9 @@ import type {
SessionCreateResponse,
} from "../src/api/types.js";
import { ProjectConfigService } from "../src/services/project-config-service.js";
-import { apiClient, createTestApp, loginAdmin, provisionUser } from "./helpers.js";
+import type { RuntimeSession } from "../src/runtime/session-manager.js";
+import type { ChannelEvent } from "../src/runtime/channel.js";
+import { apiClient, createTestApp, loginAdmin, provisionUser, waitFor } from "./helpers.js";
import type { TestApp } from "./helpers.js";
/** Catalog paired refs (config primary key = (provider, model_id)). */
@@ -586,3 +588,134 @@ describe("模型引用改键与连通性测试", () => {
}
});
});
+
+describe("models update reaches loaded sessions (invalidation + live unlock)", () => {
+ let t: TestApp;
+ let api: ReturnType;
+ let projectId: string;
+ /** Loader call count: how many times the manager (re)built a runtime from the index. */
+ let loads: number;
+
+ /** Minimal fake runtime Session (no LLM calls; mirrors vault.test.ts). */
+ const fakeRuntimeSession = (sessionId: string): RuntimeSession => ({
+ sessionId,
+ toolPermission: () => "rw",
+ generateTitle: async () => ({ title: null, usage: null }),
+ compactability: () => "ok" as const,
+ steer: () => false,
+ skipReconnectWait: () => false,
+ async *run() {},
+ async *compact() {},
+ });
+
+ const insertSession = (sessionId: string, project: string): void => {
+ t.deps.sessionsRepo.insert({
+ sessionId,
+ projectId: project,
+ agentId: "default_agent",
+ modelId: "m1",
+ provider: "custom",
+ workspace: t.root,
+ approvalMode: "allow-all",
+ title: null,
+ createdAt: new Date().toISOString(),
+ });
+ };
+
+ const putModels = (apiKey: string) =>
+ api.put(`/api/projects/${projectId}/models`, {
+ defaultModel: { provider: "custom", modelId: "m-inv" },
+ models: [{ provider: "custom", modelId: "m-inv", apiKey }],
+ });
+
+ beforeEach(async () => {
+ loads = 0;
+ t = await createTestApp({
+ loader: {
+ load: async (row) => {
+ loads++;
+ return fakeRuntimeSession(row.sessionId);
+ },
+ },
+ });
+ const { cookie } = await provisionUser(t.app, "inv_owner");
+ api = apiClient(t.app, cookie);
+ const created = (await (
+ await api.post("/api/projects", { projectId: "inv_owner-models", name: "invalidation" })
+ ).json()) as ProjectCreateResponse;
+ projectId = created.project.projectId;
+ });
+ afterEach(async () => {
+ await t.cleanup();
+ });
+
+ it("PUT invalidates the Project's cached Session runtimes: the next Task re-resumes with the new key", async () => {
+ insertSession("models-sess-1", projectId);
+ const idle = () => t.deps.manager.statusOf("models-sess-1") === "idle";
+
+ // First Task builds the runtime (load #1); the second reuses the active-table entry.
+ await t.deps.manager.startTask("models-sess-1", [userText("a")]);
+ await waitFor(idle);
+ await t.deps.manager.startTask("models-sess-1", [userText("b")]);
+ await waitFor(idle);
+ expect(loads).toBe(1);
+
+ // Key update via the API: the cached runtime is stale, so the next Task re-resumes
+ // (the loader re-reads the Project config — the new api_key reaches the next Task).
+ expect((await putModels("sk-fresh-key-000111")).status).toBe(200);
+ await t.deps.manager.startTask("models-sess-1", [userText("c")]);
+ await waitFor(idle);
+ expect(loads).toBe(2);
+
+ // Reads don't invalidate: the rebuilt runtime is reused.
+ expect((await api.get(`/api/projects/${projectId}/models`)).status).toBe(200);
+ await t.deps.manager.startTask("models-sess-1", [userText("d")]);
+ await waitFor(idle);
+ expect(loads).toBe(2);
+ });
+
+ it("PUT publishes credentials_updated to the Project's existing session channels only", async () => {
+ insertSession("models-sess-live", projectId);
+ insertSession("models-sess-cold", projectId);
+ // A session of ANOTHER project must not receive the event.
+ const other = (await (
+ await api.post("/api/projects", { projectId: "inv_owner-other", name: "other" })
+ ).json()) as ProjectCreateResponse;
+ insertSession("other-sess", other.project.projectId);
+
+ // Only the "live" session has an open channel (a subscribed tab); "cold" has none.
+ const events: ChannelEvent[] = [];
+ t.deps.channels.get("models-sess-live").subscribe((e) => events.push(e));
+ const otherEvents: ChannelEvent[] = [];
+ t.deps.channels.get("other-sess").subscribe((e) => otherEvents.push(e));
+
+ expect((await putModels("sk-live-key-000222")).status).toBe(200);
+
+ const types = events
+ .filter((e) => e.event === "server_event")
+ .map((e) => (JSON.parse(e.data) as { type: string }).type);
+ expect(types).toContain("credentials_updated");
+ // Cross-project channels stay silent, and no channel is created for unsubscribed sessions.
+ expect(
+ otherEvents
+ .filter((e) => e.event === "server_event")
+ .map((e) => (JSON.parse(e.data) as { type: string }).type),
+ ).not.toContain("credentials_updated");
+ expect(t.deps.channels.peek("models-sess-cold")).toBeUndefined();
+ });
+
+ it("GET/PUT responses expose updatedAt (config file mtime) for the web's auth-dead gate", async () => {
+ const before = Date.now() - 60_000;
+ const put = await putModels("sk-ts-key-000333");
+ expect(put.status).toBe(200);
+ const putBody = (await put.json()) as ModelsResponse;
+ expect(typeof putBody.updatedAt).toBe("string");
+ expect(Date.parse(putBody.updatedAt!)).toBeGreaterThan(before);
+
+ const get = await api.get(`/api/projects/${projectId}/models`);
+ const getBody = (await get.json()) as ModelsResponse;
+ expect(typeof getBody.updatedAt).toBe("string");
+ // Reads don't bump it: still the PUT's write time (same file mtime).
+ expect(Date.parse(getBody.updatedAt!)).toBeGreaterThanOrEqual(Date.parse(putBody.updatedAt!));
+ });
+});
diff --git a/packages/server/test/retry-now.test.ts b/packages/server/test/retry-now.test.ts
new file mode 100644
index 0000000..646afad
--- /dev/null
+++ b/packages/server/test/retry-now.test.ts
@@ -0,0 +1,99 @@
+/**
+ * Integration tests for POST /api/sessions/:id/retry-now ("retry now" on the reconnect
+ * countdown — skips the in-progress backoff wait):
+ * - 200 {skipped:true} while the runtime reports a wait was skipped;
+ * - 200 {skipped:false} as a benign no-op when the session is idle or no wait is in
+ * progress (timing races must not surface as errors);
+ * - 404 for foreign/unknown sessions (via the shared resolveSession lookup).
+ */
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { approvalDecision, assistantText, toolCall, userText } from "@prismshadow/penguin-core";
+import type { ApproveFn, OmniMessage } from "@prismshadow/penguin-core";
+import type { RetryNowResponse } from "../src/api/types.js";
+import type { SessionRow } from "../src/db/repos/sessions.js";
+import type { RuntimeSession } from "../src/runtime/session-manager.js";
+import { apiClient, createTestApp, provisionUser, waitFor } from "./helpers.js";
+import type { TestApp } from "./helpers.js";
+
+const SID = "session-2026-07-27-10-00-00-ccdd0002";
+
+/** Fake Session that parks on one approval (keeps the Task running) and records skip calls. */
+function backoffFakeSession(sessionId: string, skips: number[]): RuntimeSession {
+ return {
+ sessionId,
+ toolPermission: () => "rw",
+ generateTitle: async () => ({ title: null, usage: null }),
+ compactability: () => "ok" as const,
+ steer: () => false,
+ skipReconnectWait: () => {
+ skips.push(Date.now());
+ return true; // pretend a reconnect wait was in progress and got skipped
+ },
+ async *run(_input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
+ const tc = toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-retry" });
+ yield tc;
+ const decision = await opts.approve(tc);
+ yield approvalDecision(decision, "tc-retry");
+ yield assistantText("done");
+ },
+ async *compact() {},
+ };
+}
+
+describe("retry-now route", () => {
+ let t: TestApp;
+ let api: ReturnType;
+ let outsider: ReturnType;
+ let skips: number[];
+
+ beforeEach(async () => {
+ t = await createTestApp();
+ const { cookie } = await provisionUser(t.app, "retrier");
+ const other = await provisionUser(t.app, "outsider_r");
+ api = apiClient(t.app, cookie);
+ outsider = apiClient(t.app, other.cookie);
+ const row: SessionRow = {
+ sessionId: SID,
+ projectId: "retrier-default_project",
+ agentId: "default_agent",
+ modelId: "m1",
+ provider: "custom",
+ workspace: "/tmp/w",
+ approvalMode: "always-ask",
+ title: null,
+ createdAt: new Date().toISOString(),
+ };
+ t.deps.sessionsRepo.insert(row);
+ skips = [];
+ t.deps.manager.adopt(row, backoffFakeSession(SID, skips));
+ });
+ afterEach(async () => {
+ await t.cleanup();
+ });
+
+ it("idle → benign 200 {skipped:false}; the runtime is never poked", async () => {
+ const res = await api.post(`/api/sessions/${SID}/retry-now`, {});
+ expect(res.status).toBe(200);
+ expect((await res.json()) as RetryNowResponse).toEqual({ skipped: false });
+ expect(skips).toEqual([]);
+ });
+
+ it("running (parked in a wait) → 200 {skipped:true}, the skip reaches the runtime", async () => {
+ await t.deps.manager.startTask(SID, [userText("go")]);
+ await waitFor(() => t.deps.manager.pendingApprovalCount(SID) === 1);
+
+ const res = await api.post(`/api/sessions/${SID}/retry-now`, {});
+ expect(res.status).toBe(200);
+ expect((await res.json()) as RetryNowResponse).toEqual({ skipped: true });
+ expect(skips).toHaveLength(1);
+
+ t.deps.manager.decideApproval(SID, "tc-retry", "allow");
+ await waitFor(() => t.deps.manager.statusOf(SID) === "idle");
+ });
+
+ it("foreign and unknown sessions → 404 (same auth semantics as the other session routes)", async () => {
+ expect((await outsider.post(`/api/sessions/${SID}/retry-now`, {})).status).toBe(404);
+ expect((await api.post(`/api/sessions/session-ghost/retry-now`, {})).status).toBe(404);
+ expect(skips).toEqual([]);
+ });
+});
diff --git a/packages/server/test/session-manager.test.ts b/packages/server/test/session-manager.test.ts
index e24646d..b791d9c 100644
--- a/packages/server/test/session-manager.test.ts
+++ b/packages/server/test/session-manager.test.ts
@@ -58,6 +58,7 @@ function approvalFakeSession(sessionId: string, toolName = "exec_command"): Runt
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run(_input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
const tc = toolCall({ name: toolName, arguments: "{}", toolCallId: "tc-1" });
yield tc;
@@ -165,6 +166,7 @@ describe("session-manager", () => {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run(_input: OmniMessage[], opts: { thinkingLevel?: string }) {
seen.push(opts.thinkingLevel);
yield assistantText("ok");
@@ -192,6 +194,7 @@ describe("session-manager", () => {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run(): AsyncGenerator {
yield requestBegin();
yield toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-1" });
@@ -335,6 +338,7 @@ describe("session-manager", () => {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
// eslint-disable-next-line require-yield
async *run(input: OmniMessage[]): AsyncGenerator {
runInputs.push(input.map((m) => (m.payload as { text?: string }).text ?? ""));
@@ -417,6 +421,7 @@ describe("session-manager", () => {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run() {
// The parent-level run_subagent call (no origin): its prompt becomes the sub-session title.
yield toolCall({
@@ -493,6 +498,7 @@ describe("session-manager", () => {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run(_input, opts) {
const tc1 = toolCall({ name: "t1", arguments: "{}", toolCallId: "tc-1" });
yield tc1;
@@ -699,6 +705,41 @@ describe("session-manager", () => {
expect(loads).toBe(2);
});
+ it("invalidateProjectRuntimes: every cached entry in the Project is rebuilt; other Projects unaffected", async () => {
+ // Two Agents with cached runtimes in p1, one in p2 — a models/credential update to p1
+ // must reach BOTH of p1's Agents (collected from the active table) and leave p2 alone.
+ const rowA2: SessionRow = { ...ROW, sessionId: "session-2", agentId: "a2" };
+ const rowP2: SessionRow = { ...ROW, sessionId: "session-3", projectId: "p2" };
+ sessions.insert(rowA2);
+ sessions.insert(rowP2);
+ let loads = 0;
+ const loader: SessionLoader = {
+ load: async (row) => {
+ loads++;
+ return approvalFakeSession(row.sessionId);
+ },
+ };
+ const manager = makeManager(loader);
+ for (const sid of ["session-1", "session-2", "session-3"]) {
+ sessions.updateApprovalMode(sid, "allow-all");
+ }
+ manager.adopt(ROW, approvalFakeSession("session-1"));
+ manager.adopt(rowA2, approvalFakeSession("session-2"));
+ manager.adopt(rowP2, approvalFakeSession("session-3"));
+
+ manager.invalidateProjectRuntimes("p1");
+ // Both p1 entries are stale and rebuild through the loader on their next Task.
+ await manager.startTask("session-1", [userText("a")]);
+ await waitFor(() => manager.statusOf("session-1") === "idle");
+ await manager.startTask("session-2", [userText("b")]);
+ await waitFor(() => manager.statusOf("session-2") === "idle");
+ expect(loads).toBe(2);
+ // The p2 entry keeps its cached runtime.
+ await manager.startTask("session-3", [userText("c")]);
+ await waitFor(() => manager.statusOf("session-3") === "idle");
+ expect(loads).toBe(2);
+ });
+
it("sweepIdle: entries that are running / have pending approvals are not evicted", async () => {
const manager = makeManager(loaderOf(approvalFakeSession("session-1")));
await manager.startTask("session-1", [userText("go")]);
@@ -731,6 +772,7 @@ describe("session-manager", () => {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run() {
yield thinkingMessage("thinking");
yield assistantText("answer A");
@@ -785,6 +827,7 @@ describe("session-manager", () => {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run() {
yield assistantText("x".repeat(600));
// Sub-session text doesn't count toward the main-session body threshold
@@ -836,6 +879,7 @@ describe("session-manager", () => {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run() {
yield toolCall({
name: "run_subagent",
diff --git a/packages/server/test/sse-stream.test.ts b/packages/server/test/sse-stream.test.ts
index 61e646e..30d48d9 100644
--- a/packages/server/test/sse-stream.test.ts
+++ b/packages/server/test/sse-stream.test.ts
@@ -64,6 +64,7 @@ function approvalFakeSession(sessionId: string): RuntimeSession {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run(_input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
const tc = toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-sse" });
yield tc;
diff --git a/packages/server/test/steer.test.ts b/packages/server/test/steer.test.ts
index 4903a0f..7336052 100644
--- a/packages/server/test/steer.test.ts
+++ b/packages/server/test/steer.test.ts
@@ -27,6 +27,7 @@ function steeringFakeSession(sessionId: string, steered: string[]): RuntimeSessi
steered.push(text);
return true;
},
+ skipReconnectWait: () => false,
async *run(_input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
const tc = toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-steer" });
yield tc;
diff --git a/packages/server/test/vault.test.ts b/packages/server/test/vault.test.ts
index cb3a8cb..7a2c0f9 100644
--- a/packages/server/test/vault.test.ts
+++ b/packages/server/test/vault.test.ts
@@ -22,6 +22,7 @@ function fakeRuntimeSession(sessionId: string): RuntimeSession {
generateTitle: async () => ({ title: null, usage: null }),
compactability: () => "ok" as const,
steer: () => false,
+ skipReconnectWait: () => false,
async *run() {},
async *compact() {},
};
diff --git a/packages/web/e2e/llm-errors.spec.mjs b/packages/web/e2e/llm-errors.spec.mjs
new file mode 100644
index 0000000..2a43de3
--- /dev/null
+++ b/packages/web/e2e/llm-errors.spec.mjs
@@ -0,0 +1,239 @@
+/**
+ * LLM request-failure recovery:
+ *
+ * 1. Provider quota exhaustion (403 insufficient_user_quota) is retryable: the mock rejects
+ * the first five requests, GenerativeModel classifies them as timeout, the engine
+ * reconnects with exponential backoff (250/500/1000/2000/4000ms — engine deps are not
+ * env-configurable, so the mock's failure count is chosen to open a ≥2s countdown
+ * window instead of injecting knobs; five failures + five retries sits exactly at the
+ * default cap of 5, so attempt 6 is the last allowed and succeeds). The 4s wait before
+ * retry #5 shows a live countdown whose seconds tick DOWN; clicking "retry now"
+ * (立即重试) skips the rest of the wait and the turn completes normally — no abort.
+ * 2. Give-up: a conversation whose quota rejections never stop — clicking 放弃 on the
+ * countdown fires the ordinary abort; the engine's abort-during-backoff path ends the
+ * turn and the composer is immediately usable again.
+ * 3. An authentication failure (401 invalid_api_key) marks the Session auth-dead but
+ * RECOVERABLE: only the model reference is fixed at creation — credentials come from the
+ * current Project config — so the notice points at the Models page, updating the key
+ * auto-unlocks the composer (live via the credentials_updated event; across reloads via
+ * the credentials-updated-vs-abort time gate), Retry is the manual escape hatch (and the
+ * dead state re-arms if the key is still bad), and New Session stays as the way out.
+ */
+import { test, expect } from "@playwright/test";
+import { provisionAndLogin } from "./auth.mjs";
+
+const BASE = process.env.BASE_URL;
+const MOCK = process.env.MOCK_URL;
+
+/** Provision a user with a configured mock model and one session; returns ids for later config updates. */
+async function makeSession(page, userId, apiKey = "sk-mock") {
+ await provisionAndLogin(page.request, userId, "password123");
+ const projects = await (await page.request.get(`${BASE}/api/projects`)).json();
+ const projectId = projects.projects[0].projectId;
+ await page.request.put(`${BASE}/api/projects/${projectId}/models`, {
+ data: {
+ defaultModel: { provider: "custom", modelId: "claude-4-8" },
+ models: [
+ {
+ provider: "custom",
+ modelId: "claude-4-8",
+ apiKey,
+ baseUrl: MOCK,
+ contextWindow: 200000,
+ },
+ ],
+ },
+ });
+ const sess = await (
+ await page.request.post(`${BASE}/api/projects/${projectId}/agents/default_agent/sessions`, {
+ data: { provider: "custom", modelId: "claude-4-8" },
+ })
+ ).json();
+ return { sessionId: sess.session.sessionId, projectId };
+}
+
+test("a quota-403 retries with a live countdown; 'retry now' skips the wait and the turn completes", async ({
+ page,
+}) => {
+ const { sessionId } = await makeSession(page, "quotauser");
+
+ await page.goto(`${BASE}/chat/${sessionId}`);
+ await page.getByPlaceholder(/输入消息/).fill("quota retry test");
+ await page.getByRole("button", { name: "发送" }).click();
+
+ // Early retries flip fast (250/500ms waits — below the 2s countdown floor they keep the
+ // plain waiting/retried text).
+ await expect(page.locator("p.text-amber-600", { hasText: "已发起第 1 次重试" })).toBeVisible({
+ timeout: 20000,
+ });
+ await expect(page.locator("p.text-amber-600", { hasText: "已发起第 2 次重试" })).toBeVisible();
+
+ // The 4s wait before retry #5: a live countdown (whole seconds, ticking down).
+ const countdown = page.locator("p.text-amber-600", { hasText: /第 5 次重试,\d+ 秒后发起/ });
+ await expect(countdown).toBeVisible({ timeout: 20000 });
+ const readSecs = async () => {
+ const txt = await countdown.textContent({ timeout: 500 }).catch(() => null);
+ const m = txt === null ? null : /第 5 次重试,(\d+) 秒后发起/.exec(txt);
+ return m ? parseInt(m[1], 10) : null;
+ };
+ const first = await readSecs();
+ expect(first).not.toBeNull();
+ // Poll until the displayed seconds DECREASE (a live ticker, not a static label).
+ let second = first;
+ for (let i = 0; i < 12 && second !== null && second >= first; i++) {
+ await page.waitForTimeout(300);
+ second = await readSecs();
+ }
+ expect(second).not.toBeNull();
+ expect(second).toBeLessThan(first);
+
+ // "Retry now" skips the remaining wait: the retry fires promptly — well before the
+ // scheduled 4s would have elapsed (the 1.5s expectation window is the proof: without
+ // the skip, the natural timer still had >2s to go).
+ await page.getByRole("button", { name: "立即重试" }).click();
+ await expect(page.locator("p.text-amber-600", { hasText: "已发起第 5 次重试" })).toBeVisible({
+ timeout: 1500,
+ });
+
+ // Attempt 6 succeeds: the final answer streams in.
+ await expect(page.getByText("Quota recovered; the answer is 42.")).toBeVisible({
+ timeout: 20000,
+ });
+
+ // No abort: the run recovered, the composer stays usable.
+ await expect(page.getByText(/已中断/)).toHaveCount(0);
+ await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();
+
+ // Trace: all five quota rejections recorded as request_end(timeout) — the reconnect
+ // path — carrying the real failure detail (the Cost center's errors panel reads it from
+ // here) and the announced backoff ladder; no abort event.
+ const msgs = await (await page.request.get(`${BASE}/api/sessions/${sessionId}/messages`)).json();
+ const timeouts = msgs.messages.filter(
+ (m) => m.payload.type === "request_end" && m.payload.status === "timeout",
+ );
+ expect(timeouts.length).toBe(5);
+ for (const t of timeouts) expect(t.payload.message).toContain("insufficient_user_quota");
+ expect(timeouts.map((t) => t.payload.retry_in_ms)).toEqual([250, 500, 1000, 2000, 4000]);
+ expect(msgs.messages.some((m) => m.payload.type === "abort")).toBe(false);
+});
+
+test("'give up' on the countdown aborts the backoff: the turn ends and the composer is usable again", async ({
+ page,
+}) => {
+ const { sessionId } = await makeSession(page, "giveupuser");
+
+ await page.goto(`${BASE}/chat/${sessionId}`);
+ await page.getByPlaceholder(/输入消息/).fill("quota giveup test");
+ await page.getByRole("button", { name: "发送" }).click();
+
+ // The mock rejects every request: by the 2s wait before retry #4 the countdown (and its
+ // inline controls) are on screen.
+ await expect(page.getByRole("button", { name: "放弃" })).toBeVisible({ timeout: 20000 });
+ await page.getByRole("button", { name: "放弃" }).click();
+
+ // The ordinary abort lands mid-backoff: the engine's abort-during-backoff path ends the
+ // turn (abort line + the waiting notice flips to "stopped"), and the composer is
+ // immediately usable again.
+ await expect(page.getByText(/已中断/)).toBeVisible({ timeout: 10000 });
+ await expect(page.locator("p.text-amber-600", { hasText: "已停止重试" })).toBeVisible();
+ await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();
+});
+
+test("an auth-401 marks the Session dead but recoverable: Models CTA, key update auto-unlocks, Retry re-arms", async ({
+ page,
+}) => {
+ // The mock rejects the provisioned key (`sk-auth-bad`) with a 401 and accepts any other.
+ const { sessionId, projectId } = await makeSession(page, "authuser", "sk-auth-bad");
+
+ await page.goto(`${BASE}/chat/${sessionId}`);
+ await page.getByPlaceholder(/输入消息/).fill("auth dead test");
+ await page.getByRole("button", { name: "发送" }).click();
+
+ // The existing abort line renders unchanged (the notice is additional).
+ await expect(page.getByText(/已中断/)).toBeVisible({ timeout: 20000 });
+ await expect(page.getByText(/llm request error/)).toBeVisible();
+
+ // Action-only notice (per review: tell the user what to do, no explanations) and the
+ // composer disabled with the matching placeholder.
+ await expect(page.getByText(/请在模型配置页更新该模型的 API key/)).toBeVisible();
+ await expect(page.getByPlaceholder(/模型认证失败/)).toBeDisabled();
+
+ // Trace: the credentials failure is the request's own terminal status (no abort code —
+ // "auth" is a stop reason now); the abort event only carries the prose reason.
+ const msgs = await (await page.request.get(`${BASE}/api/sessions/${sessionId}/messages`)).json();
+ const authEnd = msgs.messages.find(
+ (m) => m.payload.type === "request_end" && m.payload.status === "auth",
+ );
+ expect(authEnd).toBeTruthy();
+ expect(authEnd.payload.message).toContain("invalid x-api-key");
+ const abort = msgs.messages.find((m) => m.payload.type === "abort");
+ expect(abort.payload.code).toBeUndefined();
+
+ // Reload: the state is rebuilt from Trace replay (the abort event is persisted) and the
+ // key has not changed, so the session stays dead.
+ await page.reload();
+ await expect(page.getByText(/模型 API 认证失败/)).toBeVisible({ timeout: 15000 });
+ await expect(page.getByPlaceholder(/模型认证失败/)).toBeDisabled();
+
+ // Primary CTA targets the Models page — where the credential is actually fixed.
+ await page.getByRole("button", { name: "打开模型配置" }).click();
+ await expect(page).toHaveURL(/\/models$/);
+ await page.goBack();
+ await expect(page.getByText(/模型 API 认证失败/)).toBeVisible({ timeout: 15000 });
+
+ // Secondary escape: New Session still jumps to a usable fresh draft.
+ await page.getByRole("button", { name: "新建会话" }).click();
+ await expect(page).toHaveURL(/\/chat\/new$/);
+ const draftInput = page.getByPlaceholder(/输入消息/);
+ await expect(draftInput).toBeVisible();
+ await expect(draftInput).toBeEnabled();
+ await page.goBack();
+ await expect(page.getByText(/模型 API 认证失败/)).toBeVisible({ timeout: 15000 });
+
+ // Retry (escape hatch) WITHOUT fixing the key: the composer re-enables for one more
+ // attempt, the mock rejects again, and the dead state re-arms.
+ await page.getByRole("button", { name: "重试", exact: true }).click();
+ await expect(page.getByText(/模型 API 认证失败/)).toHaveCount(0);
+ const input = page.getByPlaceholder(/输入消息/);
+ await expect(input).toBeEnabled();
+ await input.fill("auth dead test again");
+ await page.getByRole("button", { name: "发送" }).click();
+ await expect(page.getByText(/模型 API 认证失败/)).toBeVisible({ timeout: 20000 });
+ await expect(page.getByPlaceholder(/模型认证失败/)).toBeDisabled();
+
+ // PRIMARY PATH: update the model's key (as the Models page would). The server
+ // invalidates the Project's cached runtimes and publishes credentials_updated to this
+ // open tab — the composer unlocks WITHOUT a reload and WITHOUT clicking Retry.
+ const put = await page.request.put(`${BASE}/api/projects/${projectId}/models`, {
+ data: {
+ defaultModel: { provider: "custom", modelId: "claude-4-8" },
+ models: [
+ {
+ provider: "custom",
+ modelId: "claude-4-8",
+ apiKey: "sk-auth-good",
+ baseUrl: MOCK,
+ contextWindow: 200000,
+ },
+ ],
+ },
+ });
+ expect(put.status()).toBe(200);
+ await expect(page.getByText(/模型 API 认证失败/)).toHaveCount(0, { timeout: 15000 });
+ await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();
+
+ // The SAME conversation continues on the new key (the rebuilt runtime re-reads the
+ // Project config): the send completes and the notice stays gone.
+ await page.getByPlaceholder(/输入消息/).fill("auth dead test after fix");
+ await page.getByRole("button", { name: "发送" }).click();
+ await expect(page.getByText("Auth restored; hello again.")).toBeVisible({ timeout: 20000 });
+ await expect(page.getByText(/模型 API 认证失败/)).toHaveCount(0);
+
+ // Reload AFTER the success: replay still contains the auth aborts, but they are followed
+ // by a completed request AND predate the credential update (the time gate) — the dead
+ // state must not resurrect.
+ await page.reload();
+ await expect(page.getByText("Auth restored; hello again.")).toBeVisible({ timeout: 15000 });
+ await expect(page.getByText(/模型 API 认证失败/)).toHaveCount(0);
+ await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();
+});
diff --git a/packages/web/e2e/mock-llm.mjs b/packages/web/e2e/mock-llm.mjs
index d81c1e9..0a070ae 100644
--- a/packages/web/e2e/mock-llm.mjs
+++ b/packages/web/e2e/mock-llm.mjs
@@ -18,6 +18,9 @@ const PORT = Number(process.env.MOCK_PORT || 8931);
/** Count of non-replay requests seen in the "bad stream" conversation: the 1st is cut off (malformed), later ones are retries that get a full tool call. */
let malformedTurns = 0;
+/** Count of requests seen in the "quota retry" conversation: the first 5 are rejected 403 (insufficient_user_quota), the 6th streams normally. */
+let quotaTurns = 0;
+
function sse(res, event, data) {
res.write(`event: ${event}\n`);
res.write(`data: ${JSON.stringify(data)}\n\n`);
@@ -97,6 +100,78 @@ const server = http.createServer((req, res) => {
// first request; the mock can only tell them apart by request count (see the malformedTurns counter).
const wantsMalformed = flat.includes("bad stream test");
+ // "Auth dead" test case: KEY-BASED — requests carrying the bad key (`sk-auth-bad`,
+ // the Anthropic protocol sends it as the x-api-key header) are rejected with a 401 +
+ // OpenAI-compatible body code; any other key succeeds with a plain text answer. This
+ // exercises the recoverable flow end to end: GenerativeModel classifies the 401 as
+ // failed + code "auth" (never retried) and the composer goes dead; once the spec
+ // updates the model's key via PUT /models, the very same conversation continues
+ // (live unlock via credentials_updated + runtime invalidation re-reading the config).
+ // Gated on !isTitle so a stray title request doesn't hit this branch.
+ if (flat.includes("auth dead test") && !isTitle) {
+ if (req.headers["x-api-key"] === "sk-auth-bad") {
+ res.writeHead(401, { "content-type": "application/json" });
+ res.end(
+ JSON.stringify({ error: { code: "invalid_api_key", message: "invalid x-api-key" } }),
+ );
+ return;
+ }
+ res.writeHead(200, {
+ "content-type": "text/event-stream",
+ "cache-control": "no-cache",
+ connection: "keep-alive",
+ });
+ messageStart(res, msgCount);
+ block(res, 0, { type: "text", text: "" }, [
+ { type: "text_delta", text: "Auth restored; hello again." },
+ ]);
+ messageStop(res, "end_turn", 8);
+ return;
+ }
+
+ // "Quota give-up" test case: EVERY request is rejected 403 with the quota code — the
+ // spec clicks the reconnect countdown's give-up button mid-wait, so the conversation
+ // must never recover on its own (the abort ends it instead).
+ if (flat.includes("quota giveup test") && !isTitle) {
+ res.writeHead(403, { "content-type": "application/json" });
+ res.end(
+ JSON.stringify({
+ error: { code: "insufficient_user_quota", message: "no active subscription" },
+ }),
+ );
+ return;
+ }
+
+ // "Quota retry" test case: the first 5 requests of the conversation are rejected 403
+ // with the provider's quota-exhaustion code (as OpenAI-compatible gateways do).
+ // GenerativeModel classifies them retryable (timeout) and the engine reconnects with
+ // exponential backoff (250/500/1000/2000/4000ms) — the 4s wait before retry #5 is the
+ // window the spec uses to observe the live countdown and click "retry now"; the 6th
+ // attempt streams a normal final answer.
+ if (flat.includes("quota retry test") && !isTitle) {
+ quotaTurns += 1;
+ if (quotaTurns <= 5) {
+ res.writeHead(403, { "content-type": "application/json" });
+ res.end(
+ JSON.stringify({
+ error: { code: "insufficient_user_quota", message: "no active subscription" },
+ }),
+ );
+ return;
+ }
+ res.writeHead(200, {
+ "content-type": "text/event-stream",
+ "cache-control": "no-cache",
+ connection: "keep-alive",
+ });
+ messageStart(res, msgCount);
+ block(res, 0, { type: "text", text: "" }, [
+ { type: "text_delta", text: "Quota recovered; the answer is 42." },
+ ]);
+ messageStop(res, "end_turn", 10);
+ return;
+ }
+
res.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-cache",
diff --git a/packages/web/src/api/endpoints.ts b/packages/web/src/api/endpoints.ts
index f45528e..58c4271 100644
--- a/packages/web/src/api/endpoints.ts
+++ b/packages/web/src/api/endpoints.ts
@@ -51,6 +51,7 @@ import type {
SessionTracesResponse,
SkillInstallRequest,
SkillLibraryResponse,
+ RetryNowResponse,
SteerRequest,
TaskCreateRequest,
TaskCreateResponse,
@@ -265,6 +266,13 @@ export const postAbort = (sessionId: string) =>
body: {},
});
+/** "Retry now" on the reconnect countdown: skips the remaining backoff wait server-side (skipped:false is the benign "no wait in progress" case — e.g. the timer elapsed in a race — never an error). */
+export const postRetryNow = (sessionId: string) =>
+ apiFetch(`/api/sessions/${encodeURIComponent(sessionId)}/retry-now`, {
+ method: "POST",
+ body: {},
+ });
+
/** Mid-run steering: queues a message for the running Task (delivered between turns as a standalone `[user_steering]` user message); 409 not_running when no Task is in progress. */
export const postSteer = (sessionId: string, body: SteerRequest) =>
apiFetch(`/api/sessions/${encodeURIComponent(sessionId)}/steer`, {
diff --git a/packages/web/src/api/sse.ts b/packages/web/src/api/sse.ts
index e2f3e5c..2c20654 100644
--- a/packages/web/src/api/sse.ts
+++ b/packages/web/src/api/sse.ts
@@ -3,7 +3,8 @@
*
* - OmniMessage uses the default event (no `event:` line); data is the message envelope as
* raw JSON;
- * - Server events use `event: server_event` (approval_request / task_state / resync_required / hello);
+ * - Server events use `event: server_event` (approval_request / task_state / resync_required /
+ * credentials_updated / hello);
* - EventSource can't set custom request headers, so auth relies on same-origin cookies; on
* disconnect, the browser auto-reconnects and attaches a `Last-Event-ID` header (the server
* replays from its ring buffer; if the event was already evicted, it pushes resync_required
diff --git a/packages/web/src/features/chat/chat-input.tsx b/packages/web/src/features/chat/chat-input.tsx
index 4344156..ccc2d8a 100644
--- a/packages/web/src/features/chat/chat-input.tsx
+++ b/packages/web/src/features/chat/chat-input.tsx
@@ -856,6 +856,10 @@ export function ChatInput({
onTextChange,
initialHandoffTargetId,
onHandoffTargetChange,
+ modelAuthDead = false,
+ onOpenModels,
+ onRetryModelAuth,
+ onNewSession,
}: {
status: SessionStatus;
/** Returns whether it succeeded: on failure the input draft is kept (not cleared). */
@@ -979,6 +983,26 @@ export function ChatInput({
initialHandoffTargetId?: string;
/** Callback when the @ handoff target changes (selected/removed; the clear after a successful send does not call back, same as onTextChange). */
onHandoffTargetChange?: (agentId: string | null) => void;
+ /**
+ * Session state: the Session's model credentials failed authentication (abort with
+ * status "auth") and the Project's credentials have not been updated since (the parent
+ * computes the time gate — see isModelAuthDead). Recoverable: only the model reference
+ * is fixed at creation, credentials come from the current Project config — so the
+ * composer disables and grays itself with a notice pointing at the Models page (primary),
+ * a Retry affordance, and a New Session escape. Updating the key auto-unlocks (live via
+ * the credentials_updated event, and across reloads via the time gate).
+ */
+ modelAuthDead?: boolean;
+ /** Navigates to the Models page (where the credential is actually fixed); renders the notice's primary button when supplied. */
+ onOpenModels?: () => void;
+ /**
+ * Clears the auth-dead state and re-enables the composer for another attempt (the state
+ * re-arms on the next auth failure). The escape hatch for credential changes the
+ * timestamps miss (e.g. edited outside the UI).
+ */
+ onRetryModelAuth?: () => void;
+ /** Navigates to a fresh draft (`/chat/new`); renders the notice's "New Session" button when supplied. */
+ onNewSession?: () => void;
}) {
const { locale } = useLocale();
const [text, setText] = useState(initialText ?? "");
@@ -1021,13 +1045,19 @@ export function ChatInput({
!running &&
!compacting &&
!busy &&
+ !modelAuthDead &&
(text.trim().length > 0 || images.length > 0 || target !== null || selectedSkills.length > 0);
// Mid-run steering: while running, Enter/send queues plain text for the running agent
// (delivered between turns as a [user_steering] user message). Text only — images / skills /
// @ target stay in the draft for a later normal send (an @ target also blocks steering: a
// leading mention means a handoff, not a message to this agent).
const canSteer =
- running && !busy && onSteer !== undefined && target === null && text.trim().length > 0;
+ running &&
+ !busy &&
+ !modelAuthDead &&
+ onSteer !== undefined &&
+ target === null &&
+ text.trim().length > 0;
// Mid-run send mode (owner directive): the user chooses between "steer" (delivered
// mid-run as a [user_steering] input) and "follow-up" (held server-side and auto-sent as
// an ordinary next task once this run finishes). Set from the More-settings popover on
@@ -1045,6 +1075,7 @@ export function ChatInput({
const canFollowUp =
running &&
!busy &&
+ !modelAuthDead &&
followUpMode &&
(text.trim().length > 0 || images.length > 0 || target !== null || selectedSkills.length > 0);
// The single action button's mode: while running, an **empty** composer means Stop
@@ -1631,13 +1662,62 @@ export function ChatInput({
)}
+ {/* Auth-dead session (model credentials failed): slim notice above the composer, same
+ width — the abort line in the stream keeps the raw reason, this adds the "what now"
+ guidance. Recoverable: primary CTA opens the Models page (fixing the key there
+ auto-unlocks the session), Retry clears the state for another attempt (re-arms on
+ the next auth failure), New Session stays as the escape. The composer below is
+ disabled and hazed. */}
+ {modelAuthDead && (
+
+ )}
+
{/* Unified input card: the multi-line text body occupies the top area, with all controls
collected onto a single bottom row that never shares a line with the text.
@container: the bottom toolbar row collapses based on the **card's actual width**
(help text/button text visibility uses @md/@lg container breakpoints) — because the
card's width changes with the viewport and the Files panel squeezing it, viewport
breakpoints wouldn't judge it accurately. */}
-
+
{/* Chip row above the text body: the @ handoff target (fixed at the front — send-time
@ semantics stay leading-only) followed by the selected skills, mirroring the
agent chip's look. Remove buttons recolor the x on hover (no background wash). */}
@@ -1724,18 +1804,21 @@ export function ChatInput({
onSelect={(e) => setCaret(e.currentTarget.selectionStart ?? 0)}
onKeyDown={onKeyDown}
onPaste={onPaste}
+ disabled={modelAuthDead}
placeholder={
- running && followUpMode
- ? narrow
- ? S.chat.followUpPlaceholderShort
- : S.chat.followUpPlaceholder
- : running && onSteer
+ modelAuthDead
+ ? S.chat.modelAuthDeadPlaceholder
+ : running && followUpMode
? narrow
- ? S.chat.steerPlaceholderShort
- : S.chat.steerPlaceholder
- : narrow
- ? S.chat.inputPlaceholderShort
- : S.chat.inputPlaceholder
+ ? S.chat.followUpPlaceholderShort
+ : S.chat.followUpPlaceholder
+ : running && onSteer
+ ? narrow
+ ? S.chat.steerPlaceholderShort
+ : S.chat.steerPlaceholder
+ : narrow
+ ? S.chat.inputPlaceholderShort
+ : S.chat.inputPlaceholder
}
className="block max-h-44 min-h-[60px] w-full resize-none bg-transparent px-1 py-0.5 text-base leading-6 placeholder:text-gray-400 focus:outline-none disabled:cursor-not-allowed disabled:opacity-60 dark:placeholder:text-gray-500"
/>
diff --git a/packages/web/src/features/chat/chat-page.tsx b/packages/web/src/features/chat/chat-page.tsx
index 26638b4..5434ff2 100644
--- a/packages/web/src/features/chat/chat-page.tsx
+++ b/packages/web/src/features/chat/chat-page.tsx
@@ -29,7 +29,7 @@ import { apiErrorText } from "../../lib/api-error";
import { useDocumentTitle } from "../../lib/use-document-title";
import { formatDateTime, formatMoney, humanizeDuration, humanizeTokens } from "../../lib/format";
import { latestConversation } from "../../lib/session-grouping";
-import { approvalKey } from "../../lib/omni/stream-model";
+import { approvalKey, isModelAuthDead } from "../../lib/omni/stream-model";
import { useTheme } from "../../state/theme";
import { useProject } from "../../state/project";
import { useSessions } from "../../state/sessions";
@@ -595,6 +595,11 @@ export function ChatPage() {
navigate(`/chat/${DRAFT_SESSION_ID}`);
}, [navigate]);
+ // Auth-dead notice primary CTA: the Models page is where the credential is actually fixed.
+ const openModels = useCallback(() => {
+ navigate("/models");
+ }, [navigate]);
+
// Real-time cost for this turn: converts the Task's bucketed usage using the session Model's
// (paired reference) current pricing; null if no pricing is configured.
const modelPricing = models?.models.find((m) => sameModelRef(m, activeModelRef))?.pricing;
@@ -616,6 +621,15 @@ export function ChatPage() {
1e6
);
},
+ // Reconnect countdown controls (live waiting state only): retry-now skips the
+ // remaining backoff server-side (benign no-op on timing races), give-up is the
+ // ordinary session abort — the engine's abort-during-backoff path ends the turn.
+ onRetryNow: () => {
+ if (selected) void api.postRetryNow(selected.sessionId).catch(() => undefined);
+ },
+ onGiveUp: () => {
+ void onStop();
+ },
onOpenFile: (path) => {
// The file card has already normalized the text path to a Workspace-relative path
// (toWorkspaceRelative, including stripping absolute-path prefixes and converting Windows
@@ -643,6 +657,17 @@ export function ChatPage() {
const emptyChat =
selected !== null && !stream.loading && !stream.error && stream.model.items.length === 0;
+ // Auth-dead gate (recoverable): an auth failure is on record AND the Project's credentials
+ // have not been updated since — only the model reference is fixed at creation, credentials
+ // come from the current Project config, so a key update (Models page) unlocks the session
+ // (live via the credentials_updated event; across reloads via this time comparison against
+ // the models response's updatedAt).
+ const credsUpdatedMs = models?.updatedAt !== undefined ? Date.parse(models.updatedAt) : NaN;
+ const modelAuthDead = isModelAuthDead(
+ stream.lastAuthFailureMs,
+ Number.isFinite(credsUpdatedMs) ? credsUpdatedMs : null,
+ );
+
// Input area in session state: Agent / Workspace / Model are already locked by the Session
// (the model selector isn't rendered; models feeds the locked model's read-only display and
// the /model switch picker) — approval mode and the per-turn thinking level stay editable;
@@ -650,6 +675,10 @@ export function ChatPage() {
const input = selected && (
= COUNTDOWN_MIN_MS &&
+ item.arrivedAtMs !== undefined
+ ? item.arrivedAtMs + item.plannedDelayMs
+ : null;
+ const [now, setNow] = useState(() => Date.now());
+ const [acted, setActed] = useState(false);
+ useEffect(() => {
+ if (target === null || Date.now() >= target) return;
+ const timer = setInterval(() => {
+ setNow(Date.now());
+ if (Date.now() >= target) clearInterval(timer); // stop ticking once the wait has elapsed
+ }, 250);
+ return () => clearInterval(timer);
+ }, [target]);
+ const remainingMs = target !== null ? target - now : 0;
+ const live = target !== null && remainingMs > 0;
+ const seconds = live ? Math.ceil(remainingMs / 1000) : undefined;
+ const showControls =
+ live && ctx.origin.length === 0 && (ctx.onRetryNow !== undefined || ctx.onGiveUp !== undefined);
+ return (
+
- );
+ return ;
case "compaction":
return ;
case "task_stats":
diff --git a/packages/web/src/features/chat/message-stream.tsx b/packages/web/src/features/chat/message-stream.tsx
index 551f1bf..b8af456 100644
--- a/packages/web/src/features/chat/message-stream.tsx
+++ b/packages/web/src/features/chat/message-stream.tsx
@@ -32,6 +32,15 @@ export interface StreamRenderContext {
taskRunning: boolean;
/** Converts this turn's stats into cost (USD) using the current Model pricing; returns null when no price is configured (cost is hidden). */
taskCost?: (stats: TaskStats) => number | null;
+ /**
+ * "Retry now" on the live reconnect countdown: skips the remaining backoff wait
+ * server-side (POST /retry-now); the line flips to "retrying" when the request_begin
+ * arrives. Only honored on main-session items — a subagent's backoff belongs to the
+ * child session, which the route does not target.
+ */
+ onRetryNow?: () => void;
+ /** "Give up" on the live reconnect countdown: the ordinary session abort (same call as the Stop button). */
+ onGiveUp?: () => void;
/** Opens the Files panel and navigates to this file (triggered by clicking the file-summary card at the end of a message; takes a Workspace-relative path); the card doesn't render if this isn't wired up. */
onOpenFile?: (path: string) => void;
/** Absolute Workspace path of the current Session (used by the file-summary card to normalize body paths). */
diff --git a/packages/web/src/features/chat/use-session-stream.ts b/packages/web/src/features/chat/use-session-stream.ts
index fb9b0f8..8d82550 100644
--- a/packages/web/src/features/chat/use-session-stream.ts
+++ b/packages/web/src/features/chat/use-session-stream.ts
@@ -38,6 +38,21 @@ export interface SessionStreamState {
taskState: SessionStatus;
/** Queued follow-up count from the stream's task_state events (auto-sent once the session is idle). */
queuedFollowUps: number;
+ /**
+ * Timestamp (ms) of the last main-session auth failure (request_end with status "auth"),
+ * or null. Derived from the model, so it survives history replay and resets when
+ * switching sessions; cleared by a later completed request, a `credentials_updated`
+ * server event, or dismissModelAuthDead. The composer gates on it TOGETHER with the
+ * models response's `updatedAt` (see isModelAuthDead): an abort older than the last
+ * credential update no longer disables the composer.
+ */
+ lastAuthFailureMs: number | null;
+ /**
+ * Clears the auth-dead state (the user clicked Retry / dismissed the notice — e.g. the
+ * credential changed outside the UI in a way the timestamps miss); the state re-arms if
+ * the next request aborts on auth again.
+ */
+ dismissModelAuthDead: () => void;
/** approvalKey(origin, toolCallId) → pending approval. */
pendingApprovals: ReadonlyMap;
/** Recorded when this client clicks an approval decision (marks it as "manual"). */
@@ -185,6 +200,14 @@ export function useSessionStream(
void controllerRef.current?.retry();
}, []);
+ const dismissModelAuthDead = useCallback(() => {
+ const m = controllerRef.current?.model;
+ if (m && m.lastAuthFailureMs !== null) {
+ m.lastAuthFailureMs = null;
+ setVersion((v) => v + 1);
+ }
+ }, []);
+
// pendingTick participates in the render dependencies, ensuring pending-table changes trigger a re-render.
void pendingTick;
@@ -194,6 +217,8 @@ export function useSessionStream(
loading,
taskState,
queuedFollowUps,
+ lastAuthFailureMs: (controllerRef.current?.model ?? placeholderRef.current).lastAuthFailureMs,
+ dismissModelAuthDead,
pendingApprovals: controllerRef.current?.pendingApprovals ?? EMPTY_PENDING,
markLocalDecision,
resolveApproval,
diff --git a/packages/web/src/lib/omni/stream-controller.ts b/packages/web/src/lib/omni/stream-controller.ts
index a28fc6e..5a1b305 100644
--- a/packages/web/src/lib/omni/stream-controller.ts
+++ b/packages/web/src/lib/omni/stream-controller.ts
@@ -166,6 +166,16 @@ export function createStreamController(deps: StreamControllerDeps): StreamContro
void rebuild();
return;
}
+ case "credentials_updated": {
+ // The Project's model credentials changed (Models page save): the server has
+ // already invalidated cached runtimes, so any auth-dead state is stale — clear it
+ // and let the user simply continue (it re-arms if the new key fails too). Buffered
+ // during history load like other model-affecting events, so it replays AFTER the
+ // history that set the flag.
+ model.lastAuthFailureMs = null;
+ deps.onModelChange();
+ return;
+ }
case "hello":
return;
}
diff --git a/packages/web/src/lib/omni/stream-model.ts b/packages/web/src/lib/omni/stream-model.ts
index 2d1b409..2dad16f 100644
--- a/packages/web/src/lib/omni/stream-model.ts
+++ b/packages/web/src/lib/omni/stream-model.ts
@@ -205,6 +205,20 @@ export interface ReconnectItem {
retrying: boolean;
/** Retries exhausted (set true when an abort event arrives; the subsequent interruption marker item gives the reason). */
gaveUp?: boolean;
+ /**
+ * The engine's planned wait before the next attempt (request_end.retry_in_ms; absent
+ * when the event carried none — old Traces, or a final failure). Waits can reach the
+ * 30s backoff ceiling, so the view renders a live countdown for the waiting state when
+ * this is ≥2s (see ReconnectLine).
+ */
+ plannedDelayMs?: number;
+ /**
+ * CLIENT-clock arrival time of the request_end (the countdown anchor — client-local, so
+ * server clock skew cannot bend the ticker). On history replay the following
+ * request_begin/abort arrives immediately and flips retrying/gaveUp, so a replayed item
+ * never stays in the waiting state to tick.
+ */
+ arrivedAtMs?: number;
}
export interface CompactionItem {
@@ -300,6 +314,20 @@ export interface StreamModel {
openApprovalWaitMs: number;
/** Consecutive reconnect-failure count (incremented when request_end is timeout/malformed, reset to zero on any other terminal status). */
reconnectRun: number;
+ /**
+ * Timestamp (ms) of the most recent auth failure: a main-session `request_end` with
+ * status "auth" arrived on THIS model (a subagent's request events route to the nested
+ * model and never mark the parent). Only the model REFERENCE is fixed at Session
+ * creation — credentials are read from the current Project config on load — so this is
+ * recoverable: the composer disables itself and points at the Models page. Cleared by a
+ * later completed main-session request_end (live and history replay take the same path,
+ * so a Trace with an auth failure followed by a completed request does not resurrect the
+ * state on reload), by a `credentials_updated` server event, or by an explicit user
+ * retry/dismiss; a timestamp (not a boolean) so the composer gate can compare it against
+ * the Project's credentials-updated time after a reload (see isModelAuthDead). Re-arms
+ * on the next auth failure.
+ */
+ lastAuthFailureMs: number | null;
/** Task segmentation state. */
taskOpen: boolean;
taskStartLocalMs: number;
@@ -350,6 +378,7 @@ function newModel(nested: boolean, localDecisions: Set): StreamModel {
openRequestBeginMs: null,
openApprovalWaitMs: 0,
reconnectRun: 0,
+ lastAuthFailureMs: null,
taskOpen: false,
taskStartLocalMs: 0,
taskFirstTsMs: 0,
@@ -364,6 +393,23 @@ export function createStreamModel(localDecisions: Set = new Set()): Stre
return newModel(false, localDecisions);
}
+/**
+ * Composer auth-dead gate: an auth failure is on record AND the Project's credentials have
+ * not been updated since. `credentialsUpdatedMs` is the models response's `updatedAt`
+ * (null = unknown / not loaded — nothing proves the credential changed, so the notice
+ * stays). The time comparison is what keeps a reload honest: after a key fix the Trace
+ * still ends with the auth-failed request, but that failure predates the credential update, so the
+ * composer stays alive; a wrong replacement key re-arms naturally because its own auth
+ * abort is newer than the update.
+ */
+export function isModelAuthDead(
+ lastAuthFailureMs: number | null,
+ credentialsUpdatedMs: number | null,
+): boolean {
+ if (lastAuthFailureMs === null) return false;
+ return credentialsUpdatedMs === null || lastAuthFailureMs > credentialsUpdatedMs;
+}
+
function nextId(model: StreamModel): number {
return model.nextItemId++;
}
@@ -406,7 +452,7 @@ export function pushMessage(
}
if (isEventMessage(msg)) {
touchTask(model, msg.timestamp);
- handleEvent(model, msg.payload as EventPayload, tsOf(msg.timestamp));
+ handleEvent(model, msg.payload as EventPayload, tsOf(msg.timestamp), nowMs);
advanceLastTs(model, msg.timestamp);
return;
}
@@ -1018,7 +1064,7 @@ function createToolCard(
// Events
// ---------------------------------------------------------------------------
-function handleEvent(model: StreamModel, p: EventPayload, tsMs?: number): void {
+function handleEvent(model: StreamModel, p: EventPayload, tsMs?: number, nowMs?: number): void {
switch (p.type) {
case "approval_decision": {
const card = model.toolCards.get(p.tool_call_id);
@@ -1103,6 +1149,18 @@ function handleEvent(model: StreamModel, p: EventPayload, tsMs?: number): void {
// within a compaction range (only visible during history rebuild)
// are neither rendered nor counted — the compaction process only exposes the compaction event pair to the Human.
if (model.stats.compactionActive) return;
+ // Credentials lifecycle rides on the request's own terminal status:
+ // - "auth" records WHEN the failure happened (the event's envelope time), so the
+ // composer gate can compare it against the Project's credentials-updated time
+ // (see lastAuthFailureMs / isModelAuthDead). Origin routing already sends a
+ // subagent's request events to the nested model, so reaching here with "auth"
+ // always means the failure belongs to THIS session.
+ // - a completed request proves the credentials work (again): the auth-dead state
+ // must not outlive a success. Live and history replay share this path, so a Trace
+ // with an auth failure followed by a completed request does not resurrect the dead
+ // composer on reload.
+ if (p.status === "auth") model.lastAuthFailureMs = tsMs ?? Date.now();
+ if (p.status === "completed") model.lastAuthFailureMs = null;
// Pairs with request_begin to compute this Request's wall-clock
// duration, deducts the human approval wait, and adds the result to
// this Task's LLM time (for output TPS) — this duration includes tool
@@ -1126,13 +1184,20 @@ function handleEvent(model: StreamModel, p: EventPayload, tsMs?: number): void {
commitPendingCompaction(model.stats);
if (p.status === "timeout" || p.status === "malformed") {
model.reconnectRun += 1;
- model.items.push({
+ const item: ReconnectItem = {
kind: "reconnect",
id: nextId(model),
status: p.status,
attempt: model.reconnectRun,
retrying: false,
- });
+ };
+ // The engine announced its planned backoff: keep it with the CLIENT arrival time
+ // as the countdown anchor (skew-free — see ReconnectItem.arrivedAtMs).
+ if (typeof p.retry_in_ms === "number" && p.retry_in_ms > 0) {
+ item.plannedDelayMs = p.retry_in_ms;
+ item.arrivedAtMs = nowMs ?? Date.now();
+ }
+ model.items.push(item);
} else {
model.reconnectRun = 0;
}
diff --git a/packages/web/src/lib/strings-en.ts b/packages/web/src/lib/strings-en.ts
index 545dcef..a9accd7 100644
--- a/packages/web/src/lib/strings-en.ts
+++ b/packages/web/src/lib/strings-en.ts
@@ -642,10 +642,19 @@ When done, open index.html in a browser and self-test once.`,
subagent: "Subagent",
subagentRunning: "Running",
aborted: (reason?: string) => `[Aborted]${reason ? `: ${reason}` : ""}`,
+ /** Auth-dead notice (request_end status "auth"): action-only copy — updating the key on the Models page auto-unlocks this Session. */
+ modelAuthDead:
+ "Model API authentication failed: update this model's API key on the Models page, or start a new Session.",
+ modelAuthDeadOpenModels: "Open Models page",
+ modelAuthDeadRetry: "Retry",
+ modelAuthDeadCta: "New Session",
+ modelAuthDeadPlaceholder: "Model authentication failed — update the API key first",
+ /** Reconnect hint line; `secondsLeft` (waiting state only) switches to the live-countdown wording. */
reconnect: (
status: "timeout" | "malformed",
state: "waiting" | "retried" | "gaveUp",
attempt: number,
+ secondsLeft?: number,
) => {
const cause =
status === "timeout" ? "Connection timed out" : "Response incomplete or unparseable";
@@ -654,9 +663,15 @@ When done, open index.html in a browser and self-test once.`,
? "no further retries"
: state === "retried"
? `retry #${attempt} sent`
- : `starting retry #${attempt}…`;
+ : secondsLeft !== undefined
+ ? `retry #${attempt} in ${secondsLeft}s…`
+ : `starting retry #${attempt}…`;
return `[Retry] ${cause}; ${action}`;
},
+ /** "Retry now" on the reconnect countdown (skips the remaining backoff wait). */
+ reconnectRetryNow: "Retry now",
+ /** "Give up" on the reconnect countdown (the ordinary session abort). */
+ reconnectGiveUp: "Give up",
imageAlt: "Image uploaded by user",
toolImageAlt: "Image from tool output",
imagesAsPathHint:
diff --git a/packages/web/src/lib/strings.ts b/packages/web/src/lib/strings.ts
index 9af8b36..796ad94 100644
--- a/packages/web/src/lib/strings.ts
+++ b/packages/web/src/lib/strings.ts
@@ -626,10 +626,18 @@ Penguin 视觉风格(见 web-design 技能),深色/浅色主题( `[已中断]${reason ? `:${reason}` : ""}`,
+ /** Auth-dead notice (request_end status "auth"): action-only copy — updating the key on the Models page auto-unlocks this Session. */
+ modelAuthDead: "模型 API 认证失败:请在模型配置页更新该模型的 API key,或新建会话。",
+ modelAuthDeadOpenModels: "打开模型配置",
+ modelAuthDeadRetry: "重试",
+ modelAuthDeadCta: "新建会话",
+ modelAuthDeadPlaceholder: "模型认证失败,请先更新 API key",
+ /** Reconnect hint line; `secondsLeft` (waiting state only) switches to the live-countdown wording. */
reconnect: (
status: "timeout" | "malformed",
state: "waiting" | "retried" | "gaveUp",
attempt: number,
+ secondsLeft?: number,
) => {
const cause = status === "timeout" ? "连接超时或网络中断" : "响应不完整或无法解析";
const action =
@@ -637,9 +645,15 @@ Penguin 视觉风格(见 web-design 技能),深色/浅色主题( {
expect((items(m)[0] as ToolCallItem).decision).toBe("allow");
});
+ it("a main-session request_end(auth) records the failure timestamp; other aborts/events do not", () => {
+ const m = createStreamModel();
+ pushMessage(m, abortEvent("aborted by user"));
+ expect(m.lastAuthFailureMs).toBeNull();
+ const end = requestEnd("auth", "401 invalid x-api-key");
+ pushMessage(m, end);
+ // The recorded time is the event's envelope timestamp (so a reload can compare it
+ // against the Project's credentials-updated time).
+ expect(m.lastAuthFailureMs).toBe(Date.parse(end.timestamp));
+ // The abort that follows still renders its line (the notice is additional).
+ pushMessage(m, abortEvent("llm request error: 401 invalid x-api-key"));
+ expect(items(m)[1]).toMatchObject({
+ kind: "abort",
+ reason: "llm request error: 401 invalid x-api-key",
+ });
+ // Unrelated later messages don't clear it: only a COMPLETED request does (below) —
+ // request_begin alone proves nothing about the credential.
+ pushMessage(m, userText("hello?"));
+ pushMessage(m, requestBegin());
+ expect(m.lastAuthFailureMs).not.toBeNull();
+ });
+
+ it("a later completed request clears the auth-dead state; a new auth failure re-arms it", () => {
+ const m = createStreamModel();
+ pushMessage(m, requestEnd("auth", "401"));
+ expect(m.lastAuthFailureMs).not.toBeNull();
+ // The key was fixed and a request succeeded: the state must not outlive the success.
+ pushMessage(m, userText("again"));
+ pushMessage(m, requestBegin());
+ pushMessage(m, requestEnd("completed"));
+ expect(m.lastAuthFailureMs).toBeNull();
+ // A fresh auth failure re-arms (e.g. the replacement key is wrong too).
+ pushMessage(m, requestEnd("auth", "401"));
+ expect(m.lastAuthFailureMs).not.toBeNull();
+ });
+
+ it("history replay: order decides — auth failure then completed request stays alive; the reverse stays dead", () => {
+ // Replay of a Trace where the auth failure was followed by a successful request (the
+ // user fixed the key and continued): reload must NOT resurrect the dead composer.
+ const recovered = createStreamModel();
+ pushMessages(recovered, [
+ userText("go"),
+ requestBegin(),
+ requestEnd("auth", "401 invalid x-api-key"),
+ abortEvent("llm request error: 401 invalid x-api-key"),
+ userText("after fix"),
+ requestBegin(),
+ requestEnd("completed"),
+ ]);
+ finalizeHistory(recovered);
+ expect(recovered.lastAuthFailureMs).toBeNull();
+
+ // Replay where the auth failure is the LAST word: the dead state is rebuilt.
+ const dead = createStreamModel();
+ pushMessages(dead, [
+ userText("go"),
+ requestBegin(),
+ requestEnd("completed"),
+ userText("later"),
+ requestBegin(),
+ requestEnd("auth", "401 invalid x-api-key"),
+ abortEvent("llm request error: 401 invalid x-api-key"),
+ ]);
+ finalizeHistory(dead);
+ expect(dead.lastAuthFailureMs).not.toBeNull();
+ });
+
+ it("isModelAuthDead gates on the credentials-updated timestamp", () => {
+ expect(isModelAuthDead(null, null)).toBe(false); // no auth failure on record
+ expect(isModelAuthDead(1000, null)).toBe(true); // no update info -> nothing proves a fix
+ expect(isModelAuthDead(1000, 2000)).toBe(false); // key updated after the failure -> alive
+ expect(isModelAuthDead(3000, 2000)).toBe(true); // a fresh failure after the update re-arms
+ });
+
+ it("a subagent-origin auth failure does NOT kill the parent session's input", () => {
+ const m = createStreamModel();
+ pushMessage(m, withOrigin(requestEnd("auth", "401"), "child1"));
+ // The failure belongs to the child session: its nested model carries the state, the
+ // parent composer stays usable (the subagent simply surfaces as failed).
+ expect(m.lastAuthFailureMs).toBeNull();
+ expect(m.subagents.get("child1")!.lastAuthFailureMs).not.toBeNull();
+ });
+
it("abort events produce an abort marker item", () => {
const m = createStreamModel();
pushMessage(m, abortEvent("user abort"));
@@ -310,7 +394,7 @@ describe("approvals and events", () => {
const m = createStreamModel();
pushMessage(m, requestBegin());
pushMessage(m, requestEnd("timeout"));
- pushMessage(m, abortEvent("reconnect failed after 2 retries"));
+ pushMessage(m, abortEvent("reconnect failed after 5 retries"));
const retry = items(m)[0] as ReconnectItem;
expect(retry).toMatchObject({
kind: "reconnect",
@@ -326,6 +410,58 @@ describe("approvals and events", () => {
expect((items(m)[2] as ReconnectItem).attempt).toBe(1);
});
+ it("request_end.retry_in_ms lands on the waiting item with the CLIENT arrival anchor (the countdown's inputs)", () => {
+ const m = createStreamModel();
+ pushMessage(m, requestBegin());
+ // The engine announced a 4s backoff before retry #1; nowMs (the injected client clock)
+ // is the countdown anchor — NOT the envelope timestamp, so server clock skew cannot
+ // bend the ticker.
+ pushMessage(m, requestEnd("timeout", "403 quota (insufficient_user_quota)", 4000), 111_000);
+ const item = items(m)[0] as ReconnectItem;
+ expect(item).toMatchObject({
+ kind: "reconnect",
+ status: "timeout",
+ attempt: 1,
+ retrying: false,
+ plannedDelayMs: 4000,
+ arrivedAtMs: 111_000,
+ });
+ // An event without the field (old Traces / final failures) leaves the fields unset —
+ // the view keeps the plain waiting text.
+ pushMessage(m, requestBegin());
+ pushMessage(m, requestEnd("timeout"));
+ const plain = items(m)[1] as ReconnectItem;
+ expect(plain.plannedDelayMs).toBeUndefined();
+ expect(plain.arrivedAtMs).toBeUndefined();
+ });
+
+ it("history replay of a retried failure leaves no live countdown: the following request_begin/abort flips the state", () => {
+ // Replay delivers the whole Trace back-to-back: the waiting state (the only state the
+ // countdown and the retry-now/give-up controls render for) never persists.
+ const retried = createStreamModel();
+ pushMessages(retried, [
+ userText("go"),
+ requestBegin(),
+ requestEnd("timeout", "quota", 30_000),
+ requestBegin(), // the engine's retry — replayed immediately after
+ requestEnd("completed"),
+ ]);
+ finalizeHistory(retried);
+ const item = items(retried).find((i) => i.kind === "reconnect") as ReconnectItem;
+ expect(item.retrying).toBe(true); // not waiting -> no countdown, no controls
+
+ const aborted = createStreamModel();
+ pushMessages(aborted, [
+ userText("go"),
+ requestBegin(),
+ requestEnd("timeout", "quota", 30_000),
+ abortEvent("aborted during reconnect backoff"), // the user gave up mid-wait
+ ]);
+ finalizeHistory(aborted);
+ const gaveUp = items(aborted).find((i) => i.kind === "reconnect") as ReconnectItem;
+ expect(gaveUp.gaveUp).toBe(true); // not waiting -> no countdown, no controls
+ });
+
it("a new Task closes the previous Task's dangling retry notice (server died in the backoff window, no abort in the Trace)", () => {
const m = createStreamModel();
pushMessage(m, userText("go"));