fix(core,web): retry transport and quota LLM errors, disable dead sessions on auth failure (#82)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/server/test/errors.test.ts

@@ -406,6 +406,66 @@ describe("stream-error-watcher (LLM / Environment errors)", () => {
     });
   });
 
+  it("request_end(auth) shares the llm_failed/unexpected bucket; message takes the abort reason", () => {
+    // Credentials rejection is its own stop reason but no new error taxonomy: same code
+    // and kind as failed, resolved by the abort that follows like any failed exit.
+    const got = feed([
+      requestBegin(),
+      requestEnd("auth", "401 invalid x-api-key (invalid_api_key)"),
+      abortEvent("llm request error: 401 invalid x-api-key (invalid_api_key)"),
+    ]);
+    expect(got).toHaveLength(1);
+    expect(got[0]).toMatchObject({
+      source: "llm",
+      kind: "unexpected",
+      code: "llm_failed",
+      message: "llm request error: 401 invalid x-api-key (invalid_api_key)",
+    });
+  });
+
+  it("a retried failure keeps its real detail: request_end(timeout).message lands in the record", () => {
+    // The retry path: the engine reconnects (request_begin) and eventually succeeds, so no
+    // abort ever arrives for the staged failure — the request_end's own failure detail
+    // (LLMOutcome.message, e.g. a quota code) is the message of record, not the generic
+    // "timed out" status text. This is what the Cost center shows for a retried quota 403.
+    const got = feed([
+      requestBegin(),
+      requestEnd("timeout", "403 no active subscription (insufficient_user_quota)"),
+      requestBegin(),
+      requestEnd("completed"),
+    ]);
+    expect(got).toHaveLength(1);
+    expect(got[0]).toMatchObject({
+      source: "llm",
+      kind: "expected",
+      code: "llm_timeout",
+      message: "403 no active subscription (insufficient_user_quota)",
+    });
+  });
+
+  it("an abort reason still outranks the staged request_end detail (failed exit path)", () => {
+    const got = feed([
+      requestBegin(),
+      requestEnd("failed", "401 invalid x-api-key (invalid_api_key)"),
+      abortEvent("llm request error: 401 invalid x-api-key (invalid_api_key)"),
+    ]);
+    expect(got).toHaveLength(1);
+    // The abort's prose (with core's "llm request error" framing) wins over the raw detail.
+    expect(got[0]!.message).toBe("llm request error: 401 invalid x-api-key (invalid_api_key)");
+  });
+
+  it("interrupt during backoff with a staged detail: the detail wins over the status text", () => {
+    // The interrupt message is distrusted (not the failure's reason), but the staged
+    // request_end detail IS the failure's reason — prefer it over the generic status text.
+    const got = feed([
+      requestBegin(),
+      requestEnd("timeout", "403 quota exceeded (insufficient_user_quota)"),
+      abortEvent("aborted during reconnect backoff"),
+    ]);
+    expect(got).toHaveLength(1);
+    expect(got[0]!.message).toBe("403 quota exceeded (insufficient_user_quota)");
+  });
+
   it("aborted (user clicked Stop) is not an error: not recorded; neither is completed", () => {
     expect(
       feed([

packages/server/test/followup.test.ts

@@ -25,6 +25,7 @@ function parkingFakeSession(sessionId: string, runs: string[][]): RuntimeSession
     generateTitle: async () => ({ title: null, usage: null }),
     compactability: () => "ok" as const,
     steer: () => false,
+    skipReconnectWait: () => false,
     async *run(input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
       runs.push(input.map((m) => (m.payload as { text?: string }).text ?? ""));
       const tc = toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-fu" });

packages/server/test/models.test.ts

@@ -13,7 +13,7 @@ import type { AddressInfo } from "node:net";
 import { readFile, stat } from "node:fs/promises";
 import path from "node:path";
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { MODEL_CATALOG } from "@prismshadow/penguin-core";
+import { MODEL_CATALOG, userText } from "@prismshadow/penguin-core";
 import type {
   ModelsResponse,
   ModelTestResponse,
@@ -21,7 +21,9 @@ import type {
   SessionCreateResponse,
 } from "../src/api/types.js";
 import { ProjectConfigService } from "../src/services/project-config-service.js";
-import { apiClient, createTestApp, loginAdmin, provisionUser } from "./helpers.js";
+import type { RuntimeSession } from "../src/runtime/session-manager.js";
+import type { ChannelEvent } from "../src/runtime/channel.js";
+import { apiClient, createTestApp, loginAdmin, provisionUser, waitFor } from "./helpers.js";
 import type { TestApp } from "./helpers.js";
 
 /** Catalog paired refs (config primary key = (provider, model_id)). */
@@ -586,3 +588,134 @@ describe("模型引用改键与连通性测试", () => {
     }
   });
 });
+
+describe("models update reaches loaded sessions (invalidation + live unlock)", () => {
+  let t: TestApp;
+  let api: ReturnType<typeof apiClient>;
+  let projectId: string;
+  /** Loader call count: how many times the manager (re)built a runtime from the index. */
+  let loads: number;
+
+  /** Minimal fake runtime Session (no LLM calls; mirrors vault.test.ts). */
+  const fakeRuntimeSession = (sessionId: string): RuntimeSession => ({
+    sessionId,
+    toolPermission: () => "rw",
+    generateTitle: async () => ({ title: null, usage: null }),
+    compactability: () => "ok" as const,
+    steer: () => false,
+    skipReconnectWait: () => false,
+    async *run() {},
+    async *compact() {},
+  });
+
+  const insertSession = (sessionId: string, project: string): void => {
+    t.deps.sessionsRepo.insert({
+      sessionId,
+      projectId: project,
+      agentId: "default_agent",
+      modelId: "m1",
+      provider: "custom",
+      workspace: t.root,
+      approvalMode: "allow-all",
+      title: null,
+      createdAt: new Date().toISOString(),
+    });
+  };
+
+  const putModels = (apiKey: string) =>
+    api.put(`/api/projects/${projectId}/models`, {
+      defaultModel: { provider: "custom", modelId: "m-inv" },
+      models: [{ provider: "custom", modelId: "m-inv", apiKey }],
+    });
+
+  beforeEach(async () => {
+    loads = 0;
+    t = await createTestApp({
+      loader: {
+        load: async (row) => {
+          loads++;
+          return fakeRuntimeSession(row.sessionId);
+        },
+      },
+    });
+    const { cookie } = await provisionUser(t.app, "inv_owner");
+    api = apiClient(t.app, cookie);
+    const created = (await (
+      await api.post("/api/projects", { projectId: "inv_owner-models", name: "invalidation" })
+    ).json()) as ProjectCreateResponse;
+    projectId = created.project.projectId;
+  });
+  afterEach(async () => {
+    await t.cleanup();
+  });
+
+  it("PUT invalidates the Project's cached Session runtimes: the next Task re-resumes with the new key", async () => {
+    insertSession("models-sess-1", projectId);
+    const idle = () => t.deps.manager.statusOf("models-sess-1") === "idle";
+
+    // First Task builds the runtime (load #1); the second reuses the active-table entry.
+    await t.deps.manager.startTask("models-sess-1", [userText("a")]);
+    await waitFor(idle);
+    await t.deps.manager.startTask("models-sess-1", [userText("b")]);
+    await waitFor(idle);
+    expect(loads).toBe(1);
+
+    // Key update via the API: the cached runtime is stale, so the next Task re-resumes
+    // (the loader re-reads the Project config — the new api_key reaches the next Task).
+    expect((await putModels("sk-fresh-key-000111")).status).toBe(200);
+    await t.deps.manager.startTask("models-sess-1", [userText("c")]);
+    await waitFor(idle);
+    expect(loads).toBe(2);
+
+    // Reads don't invalidate: the rebuilt runtime is reused.
+    expect((await api.get(`/api/projects/${projectId}/models`)).status).toBe(200);
+    await t.deps.manager.startTask("models-sess-1", [userText("d")]);
+    await waitFor(idle);
+    expect(loads).toBe(2);
+  });
+
+  it("PUT publishes credentials_updated to the Project's existing session channels only", async () => {
+    insertSession("models-sess-live", projectId);
+    insertSession("models-sess-cold", projectId);
+    // A session of ANOTHER project must not receive the event.
+    const other = (await (
+      await api.post("/api/projects", { projectId: "inv_owner-other", name: "other" })
+    ).json()) as ProjectCreateResponse;
+    insertSession("other-sess", other.project.projectId);
+
+    // Only the "live" session has an open channel (a subscribed tab); "cold" has none.
+    const events: ChannelEvent[] = [];
+    t.deps.channels.get("models-sess-live").subscribe((e) => events.push(e));
+    const otherEvents: ChannelEvent[] = [];
+    t.deps.channels.get("other-sess").subscribe((e) => otherEvents.push(e));
+
+    expect((await putModels("sk-live-key-000222")).status).toBe(200);
+
+    const types = events
+      .filter((e) => e.event === "server_event")
+      .map((e) => (JSON.parse(e.data) as { type: string }).type);
+    expect(types).toContain("credentials_updated");
+    // Cross-project channels stay silent, and no channel is created for unsubscribed sessions.
+    expect(
+      otherEvents
+        .filter((e) => e.event === "server_event")
+        .map((e) => (JSON.parse(e.data) as { type: string }).type),
+    ).not.toContain("credentials_updated");
+    expect(t.deps.channels.peek("models-sess-cold")).toBeUndefined();
+  });
+
+  it("GET/PUT responses expose updatedAt (config file mtime) for the web's auth-dead gate", async () => {
+    const before = Date.now() - 60_000;
+    const put = await putModels("sk-ts-key-000333");
+    expect(put.status).toBe(200);
+    const putBody = (await put.json()) as ModelsResponse;
+    expect(typeof putBody.updatedAt).toBe("string");
+    expect(Date.parse(putBody.updatedAt!)).toBeGreaterThan(before);
+
+    const get = await api.get(`/api/projects/${projectId}/models`);
+    const getBody = (await get.json()) as ModelsResponse;
+    expect(typeof getBody.updatedAt).toBe("string");
+    // Reads don't bump it: still the PUT's write time (same file mtime).
+    expect(Date.parse(getBody.updatedAt!)).toBeGreaterThanOrEqual(Date.parse(putBody.updatedAt!));
+  });
+});

packages/server/test/retry-now.test.ts

@@ -0,0 +1,99 @@
+/**
+ * Integration tests for POST /api/sessions/:id/retry-now ("retry now" on the reconnect
+ * countdown — skips the in-progress backoff wait):
+ *   - 200 {skipped:true} while the runtime reports a wait was skipped;
+ *   - 200 {skipped:false} as a benign no-op when the session is idle or no wait is in
+ *     progress (timing races must not surface as errors);
+ *   - 404 for foreign/unknown sessions (via the shared resolveSession lookup).
+ */
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { approvalDecision, assistantText, toolCall, userText } from "@prismshadow/penguin-core";
+import type { ApproveFn, OmniMessage } from "@prismshadow/penguin-core";
+import type { RetryNowResponse } from "../src/api/types.js";
+import type { SessionRow } from "../src/db/repos/sessions.js";
+import type { RuntimeSession } from "../src/runtime/session-manager.js";
+import { apiClient, createTestApp, provisionUser, waitFor } from "./helpers.js";
+import type { TestApp } from "./helpers.js";
+
+const SID = "session-2026-07-27-10-00-00-ccdd0002";
+
+/** Fake Session that parks on one approval (keeps the Task running) and records skip calls. */
+function backoffFakeSession(sessionId: string, skips: number[]): RuntimeSession {
+  return {
+    sessionId,
+    toolPermission: () => "rw",
+    generateTitle: async () => ({ title: null, usage: null }),
+    compactability: () => "ok" as const,
+    steer: () => false,
+    skipReconnectWait: () => {
+      skips.push(Date.now());
+      return true; // pretend a reconnect wait was in progress and got skipped
+    },
+    async *run(_input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
+      const tc = toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-retry" });
+      yield tc;
+      const decision = await opts.approve(tc);
+      yield approvalDecision(decision, "tc-retry");
+      yield assistantText("done");
+    },
+    async *compact() {},
+  };
+}
+
+describe("retry-now route", () => {
+  let t: TestApp;
+  let api: ReturnType<typeof apiClient>;
+  let outsider: ReturnType<typeof apiClient>;
+  let skips: number[];
+
+  beforeEach(async () => {
+    t = await createTestApp();
+    const { cookie } = await provisionUser(t.app, "retrier");
+    const other = await provisionUser(t.app, "outsider_r");
+    api = apiClient(t.app, cookie);
+    outsider = apiClient(t.app, other.cookie);
+    const row: SessionRow = {
+      sessionId: SID,
+      projectId: "retrier-default_project",
+      agentId: "default_agent",
+      modelId: "m1",
+      provider: "custom",
+      workspace: "/tmp/w",
+      approvalMode: "always-ask",
+      title: null,
+      createdAt: new Date().toISOString(),
+    };
+    t.deps.sessionsRepo.insert(row);
+    skips = [];
+    t.deps.manager.adopt(row, backoffFakeSession(SID, skips));
+  });
+  afterEach(async () => {
+    await t.cleanup();
+  });
+
+  it("idle → benign 200 {skipped:false}; the runtime is never poked", async () => {
+    const res = await api.post(`/api/sessions/${SID}/retry-now`, {});
+    expect(res.status).toBe(200);
+    expect((await res.json()) as RetryNowResponse).toEqual({ skipped: false });
+    expect(skips).toEqual([]);
+  });
+
+  it("running (parked in a wait) → 200 {skipped:true}, the skip reaches the runtime", async () => {
+    await t.deps.manager.startTask(SID, [userText("go")]);
+    await waitFor(() => t.deps.manager.pendingApprovalCount(SID) === 1);
+
+    const res = await api.post(`/api/sessions/${SID}/retry-now`, {});
+    expect(res.status).toBe(200);
+    expect((await res.json()) as RetryNowResponse).toEqual({ skipped: true });
+    expect(skips).toHaveLength(1);
+
+    t.deps.manager.decideApproval(SID, "tc-retry", "allow");
+    await waitFor(() => t.deps.manager.statusOf(SID) === "idle");
+  });
+
+  it("foreign and unknown sessions → 404 (same auth semantics as the other session routes)", async () => {
+    expect((await outsider.post(`/api/sessions/${SID}/retry-now`, {})).status).toBe(404);
+    expect((await api.post(`/api/sessions/session-ghost/retry-now`, {})).status).toBe(404);
+    expect(skips).toEqual([]);
+  });
+});

packages/server/test/session-manager.test.ts

@@ -58,6 +58,7 @@ function approvalFakeSession(sessionId: string, toolName = "exec_command"): Runt
     generateTitle: async () => ({ title: null, usage: null }),
     compactability: () => "ok" as const,
     steer: () => false,
+    skipReconnectWait: () => false,
     async *run(_input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
       const tc = toolCall({ name: toolName, arguments: "{}", toolCallId: "tc-1" });
       yield tc;
@@ -165,6 +166,7 @@ describe("session-manager", () => {
       generateTitle: async () => ({ title: null, usage: null }),
       compactability: () => "ok" as const,
       steer: () => false,
+      skipReconnectWait: () => false,
       async *run(_input: OmniMessage[], opts: { thinkingLevel?: string }) {
         seen.push(opts.thinkingLevel);
         yield assistantText("ok");
@@ -192,6 +194,7 @@ describe("session-manager", () => {
       generateTitle: async () => ({ title: null, usage: null }),
       compactability: () => "ok" as const,
       steer: () => false,
+      skipReconnectWait: () => false,
       async *run(): AsyncGenerator<OmniMessage> {
         yield requestBegin();
         yield toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-1" });
@@ -335,6 +338,7 @@ describe("session-manager", () => {
       generateTitle: async () => ({ title: null, usage: null }),
       compactability: () => "ok" as const,
       steer: () => false,
+      skipReconnectWait: () => false,
       // eslint-disable-next-line require-yield
       async *run(input: OmniMessage[]): AsyncGenerator<OmniMessage> {
         runInputs.push(input.map((m) => (m.payload as { text?: string }).text ?? ""));
@@ -417,6 +421,7 @@ describe("session-manager", () => {
       generateTitle: async () => ({ title: null, usage: null }),
       compactability: () => "ok" as const,
       steer: () => false,
+      skipReconnectWait: () => false,
       async *run() {
         // The parent-level run_subagent call (no origin): its prompt becomes the sub-session title.
         yield toolCall({
@@ -493,6 +498,7 @@ describe("session-manager", () => {
       generateTitle: async () => ({ title: null, usage: null }),
       compactability: () => "ok" as const,
       steer: () => false,
+      skipReconnectWait: () => false,
       async *run(_input, opts) {
         const tc1 = toolCall({ name: "t1", arguments: "{}", toolCallId: "tc-1" });
         yield tc1;
@@ -699,6 +705,41 @@ describe("session-manager", () => {
     expect(loads).toBe(2);
   });
 
+  it("invalidateProjectRuntimes: every cached entry in the Project is rebuilt; other Projects unaffected", async () => {
+    // Two Agents with cached runtimes in p1, one in p2 — a models/credential update to p1
+    // must reach BOTH of p1's Agents (collected from the active table) and leave p2 alone.
+    const rowA2: SessionRow = { ...ROW, sessionId: "session-2", agentId: "a2" };
+    const rowP2: SessionRow = { ...ROW, sessionId: "session-3", projectId: "p2" };
+    sessions.insert(rowA2);
+    sessions.insert(rowP2);
+    let loads = 0;
+    const loader: SessionLoader = {
+      load: async (row) => {
+        loads++;
+        return approvalFakeSession(row.sessionId);
+      },
+    };
+    const manager = makeManager(loader);
+    for (const sid of ["session-1", "session-2", "session-3"]) {
+      sessions.updateApprovalMode(sid, "allow-all");
+    }
+    manager.adopt(ROW, approvalFakeSession("session-1"));
+    manager.adopt(rowA2, approvalFakeSession("session-2"));
+    manager.adopt(rowP2, approvalFakeSession("session-3"));
+
+    manager.invalidateProjectRuntimes("p1");
+    // Both p1 entries are stale and rebuild through the loader on their next Task.
+    await manager.startTask("session-1", [userText("a")]);
+    await waitFor(() => manager.statusOf("session-1") === "idle");
+    await manager.startTask("session-2", [userText("b")]);
+    await waitFor(() => manager.statusOf("session-2") === "idle");
+    expect(loads).toBe(2);
+    // The p2 entry keeps its cached runtime.
+    await manager.startTask("session-3", [userText("c")]);
+    await waitFor(() => manager.statusOf("session-3") === "idle");
+    expect(loads).toBe(2);
+  });
+
   it("sweepIdle: entries that are running / have pending approvals are not evicted", async () => {
     const manager = makeManager(loaderOf(approvalFakeSession("session-1")));
     await manager.startTask("session-1", [userText("go")]);
@@ -731,6 +772,7 @@ describe("session-manager", () => {
       generateTitle: async () => ({ title: null, usage: null }),
       compactability: () => "ok" as const,
       steer: () => false,
+      skipReconnectWait: () => false,
       async *run() {
         yield thinkingMessage("thinking");
         yield assistantText("answer A");
@@ -785,6 +827,7 @@ describe("session-manager", () => {
       generateTitle: async () => ({ title: null, usage: null }),
       compactability: () => "ok" as const,
       steer: () => false,
+      skipReconnectWait: () => false,
       async *run() {
         yield assistantText("x".repeat(600));
         // Sub-session text doesn't count toward the main-session body threshold
@@ -836,6 +879,7 @@ describe("session-manager", () => {
       generateTitle: async () => ({ title: null, usage: null }),
       compactability: () => "ok" as const,
       steer: () => false,
+      skipReconnectWait: () => false,
       async *run() {
         yield toolCall({
           name: "run_subagent",

packages/server/test/sse-stream.test.ts

@@ -64,6 +64,7 @@ function approvalFakeSession(sessionId: string): RuntimeSession {
     generateTitle: async () => ({ title: null, usage: null }),
     compactability: () => "ok" as const,
     steer: () => false,
+    skipReconnectWait: () => false,
     async *run(_input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
       const tc = toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-sse" });
       yield tc;

packages/server/test/steer.test.ts

@@ -27,6 +27,7 @@ function steeringFakeSession(sessionId: string, steered: string[]): RuntimeSessi
       steered.push(text);
       return true;
     },
+    skipReconnectWait: () => false,
     async *run(_input: OmniMessage[], opts: { approve: ApproveFn; signal: AbortSignal }) {
       const tc = toolCall({ name: "exec_command", arguments: "{}", toolCallId: "tc-steer" });
       yield tc;

packages/server/test/vault.test.ts

@@ -22,6 +22,7 @@ function fakeRuntimeSession(sessionId: string): RuntimeSession {
     generateTitle: async () => ({ title: null, usage: null }),
     compactability: () => "ok" as const,
     steer: () => false,
+    skipReconnectWait: () => false,
     async *run() {},
     async *compact() {},
   };