fix(core,web): retry transport and quota LLM errors, disable dead sessions on auth failure (#82)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/core/src/omnimessage/builders.ts

@@ -269,9 +269,23 @@ export function requestBegin(): OmniMessage<RequestBeginPayload> {
   return event({ type: "request_begin" });
 }
 
-/** request end event: carries the terminal state (`completed` means this turn was already committed to AgentHub). */
-export function requestEnd(status: StopReason): OmniMessage<RequestEndPayload> {
-  return event({ type: "request_end", status });
+/**
+ * request end event: carries the terminal state (`completed` means this turn was already
+ * committed to AgentHub), plus — on non-completed statuses — the failure detail (from
+ * LLMOutcome.message) and, when the engine will retry in-run, the planned backoff wait
+ * (`retry_in_ms`, rendered by the Web App as a live countdown).
+ */
+export function requestEnd(
+  status: StopReason,
+  message?: string,
+  retryInMs?: number,
+): OmniMessage<RequestEndPayload> {
+  return event({
+    type: "request_end",
+    status,
+    ...(message !== undefined ? { message } : {}),
+    ...(retryInMs !== undefined ? { retry_in_ms: retryInMs } : {}),
+  });
 }
 
 /** compaction begin event: carries the trigger reason, mode, current context usage, and cumulative Session turn count. */

packages/core/src/omnimessage/types.ts

@@ -23,7 +23,7 @@ export type OmniMessageType = "session_meta" | "model_msg" | "event_msg";
 export type Role = "user" | "assistant";
 
 /**
- * The reason a model response or message generation ended. Only five protocol values are
+ * The reason a model response or message generation ended. Only six protocol values are
  * allowed:
  *   - `completed`: finished normally, including completed text, thinking, tool requests, or
  *     tool output;
@@ -31,10 +31,15 @@ export type Role = "user" | "assistant";
  *   - `aborted`: user-initiated interruption or cancellation;
  *   - `timeout`: LLM request timed out;
  *   - `malformed`: the LLM response was malformed (e.g. AgentHub JSON parsing exception).
- *     Only LLM timeout / malformed trigger a context_engine reconnect.
+ *     Only LLM timeout / malformed trigger a context_engine reconnect;
+ *   - `auth`: the LLM rejected the credentials (see `isAuthenticationError`) — a
+ *     `failed`-shaped stop that no in-run retry can fix; hosts disable input until the
+ *     model's API key is updated (only the model reference is fixed at Session creation —
+ *     credentials come from the current Project config), after which the Session
+ *     continues.
  * Docs: /docs/omni-message § "stop_reason".
  */
-export type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed";
+export type StopReason = "completed" | "failed" | "aborted" | "timeout" | "malformed" | "auth";
 
 /** The event phase of a streaming fragment. `stop` marks the end of a fragment and usually carries no incremental content. */
 export type StreamEventType = "start" | "delta" | "stop";
@@ -277,8 +282,25 @@ export interface RequestBeginPayload {
 
 export interface RequestEndPayload {
   type: "request_end";
-  /** Terminal state of this Request (reuses the five StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome). */
+  /** Terminal state of this Request (reuses the six StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome; `auth` is the credentials-failure signal hosts key on). */
   status: StopReason;
+  /**
+   * Failure detail from `LLMOutcome.message`, present only on non-completed statuses: the
+   * real reason behind a retried/failed Request (e.g. `403 … (insufficient_user_quota)`),
+   * for observability — the server's error records / Cost center read it here because a
+   * retried request never produces an abort event. Additive: old Traces without it replay
+   * unchanged.
+   */
+  message?: string;
+  /**
+   * Planned in-run retry wait (ms) — present ONLY when the engine will retry this failure
+   * within the same run (status `timeout`/`malformed` with attempts remaining under the
+   * applicable cap). Computed by the same formula as the actual backoff sleep
+   * (`reconnectDelayMs`), so the announced wait and the real one cannot drift; the Web App
+   * renders it as a live countdown to the next attempt. Absent on final failures (an abort
+   * follows instead) and on completed requests. Additive: old Traces replay unchanged.
+   */
+  retry_in_ms?: number;
 }
 
 /** Compaction trigger reason: context threshold / turn-count threshold / user-initiated request. */
@@ -291,7 +313,7 @@ export type CompactionMode = "summarize" | "discard";
  * Compaction boundary event: the compaction process exposes
  * only this event pair to Human, produced **in pairs** by `context_engine`. Both `reason` and
  * `mode` are carried on both events, for stateless frontend rendering; `status` reuses the
- * five-value `StopReason` protocol (compaction converges to a terminal state, taking
+ * six-value `StopReason` protocol (compaction converges to a terminal state, taking
  * `completed` / `failed` / `aborted` in practice — `timeout` / `malformed` are handled internally
  * by the compaction request's existing retry mechanism, collapsing to `failed` once retries are
  * exhausted).