fix(core,web): retry transport and quota LLM errors, disable dead sessions on auth failure (#82)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c369e089a7
commit
3d3495528c
54 files changed
+2278
-161
No files matched your search
@@ -399,7 +399,7 @@ describe("context compaction", () => {
|
||||
environment: fakeEnvironment,
|
||||
compaction: settings(),
|
||||
createLLM: () => llm1,
|
||||
maxReconnects: 1,
|
||||
compactionMaxReconnects: 1,
|
||||
reconnectBackoffMs: 1,
|
||||
});
|
||||
|
||||
@@ -411,6 +411,52 @@ describe("context compaction", () => {
|
||||
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
|
||||
});
|
||||
|
||||
it("the compaction loop uses its own cap, not the shared maxReconnects ladder", async () => {
|
||||
// Compaction failure has graceful semantics (the original context is kept, compaction
|
||||
// retries on the next trigger), so it fails fast: with compactionMaxReconnects=2, the
|
||||
// compaction request runs 1+2 attempts and gives up — even though maxReconnects is far
|
||||
// larger and would have kept the session stalled through the full exponential ladder.
|
||||
const llm1 = new ScriptedLLM(
|
||||
[
|
||||
{ messages: [assistantText("answer"), usage(150, 150)] },
|
||||
{ messages: [], outcome: { status: "timeout" } },
|
||||
{ messages: [], outcome: { status: "timeout" } },
|
||||
{ messages: [], outcome: { status: "timeout" } },
|
||||
// Never reached: the compaction cap stops at 3 compaction attempts in total.
|
||||
{ messages: [assistantText("[summary]late[/summary]")] },
|
||||
],
|
||||
"llm1",
|
||||
);
|
||||
const recorded: OmniMessage[] = [];
|
||||
const engine = new ContextEngine({
|
||||
llm: llm1,
|
||||
environment: fakeEnvironment,
|
||||
compaction: settings(),
|
||||
createLLM: () => llm1,
|
||||
maxReconnects: 8,
|
||||
compactionMaxReconnects: 2,
|
||||
reconnectBackoffMs: 1,
|
||||
trace: {
|
||||
write: async (m) => {
|
||||
recorded.push(m);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const out = await collect(engine.run([userText("go")], { approve: allowAll }));
|
||||
const events = compactionEvents(out);
|
||||
expect(events[1]).toMatchObject({ type: "compaction_end", status: "failed" });
|
||||
// 1 turn request + 3 compaction attempts (initial + compactionMaxReconnects retries).
|
||||
expect(llm1.calls).toHaveLength(4);
|
||||
// The Trace-written compaction request_ends announce the planned backoff under the
|
||||
// COMPACTION cap (base 1ms: 1, then 2), with none on the final failure — these events
|
||||
// are never streamed, so the value lands in the Trace record only.
|
||||
const retryPlans = recorded
|
||||
.filter((m) => (m.payload as { type?: string }).type === "request_end")
|
||||
.map((m) => (m.payload as { retry_in_ms?: number }).retry_in_ms);
|
||||
expect(retryPlans).toEqual([undefined, 1, 2, undefined]);
|
||||
});
|
||||
|
||||
it("session turns reaching (==) the threshold compact at task end — no waiting for the next task", async () => {
|
||||
const llm1 = new ScriptedLLM(
|
||||
[
|
||||
|
||||
@@ -30,7 +30,7 @@ import type { GenerativeModelParameters, LLMInterface, LLMOutcome } from "../src
|
||||
import type { OmniMessage, TextPayload, ToolCallPayload } from "../src/omnimessage/index.js";
|
||||
import { Environment } from "../src/environment/index.js";
|
||||
import { Writer, readTrace } from "../src/trace/index.js";
|
||||
import { ContextEngine } from "../src/engine/context-engine.js";
|
||||
import { ContextEngine, reconnectDelayMs } from "../src/engine/context-engine.js";
|
||||
import type { ApproveFn, EnvironmentInterface, ToolPermission } from "../src/interfaces.js";
|
||||
|
||||
/** Deterministic fake LLM: the first turn yields a tool_call, the second yields the final reply. */
|
||||
@@ -1488,12 +1488,14 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
const inputs: OmniMessage[][] = [];
|
||||
const llm: LLMInterface = {
|
||||
// The LLM must never throw an exception at the engine: a non-retryable error resolves
|
||||
// by returning a failed outcome after closing the structure.
|
||||
// by returning a failed outcome after closing the structure. A genuinely non-retryable
|
||||
// failure nowadays is a parameter error (quota 403s retry as timeout, 401s carry
|
||||
// code "auth" — both covered by their own tests below).
|
||||
// eslint-disable-next-line require-yield
|
||||
async *streamGenerate(params) {
|
||||
calls += 1;
|
||||
inputs.push(params.newMessages);
|
||||
return { status: "failed", message: "invalid api key" };
|
||||
return { status: "failed", message: "400 unknown parameter: max_output_tokens" };
|
||||
},
|
||||
};
|
||||
const environment = new Environment({
|
||||
@@ -1509,7 +1511,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
expect(abort).toBeDefined();
|
||||
const reason = (abort!.payload as { reason?: string }).reason ?? "";
|
||||
expect(reason).toContain("llm request error");
|
||||
expect(reason).toContain("invalid api key");
|
||||
expect(reason).toContain("unknown parameter");
|
||||
|
||||
// The failed turn's input is flattened and stashed; the next run resends it merged with
|
||||
// the new input.
|
||||
@@ -1519,6 +1521,232 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
expect(text).toContain("next");
|
||||
});
|
||||
|
||||
it("an auth outcome stops immediately like failed: no retry, request_end carries status auth", async () => {
|
||||
let calls = 0;
|
||||
const llm: LLMInterface = {
|
||||
// GenerativeModel classifies a 401/invalid_api_key as status "auth" (see
|
||||
// llm.test.ts); the engine must stop directly — the auth/failed branch is the second
|
||||
// belt keeping a dead credential out of the retry loop (the classifier is the first).
|
||||
// eslint-disable-next-line require-yield
|
||||
async *streamGenerate() {
|
||||
calls += 1;
|
||||
return { status: "auth", message: "401 invalid x-api-key" };
|
||||
},
|
||||
};
|
||||
const environment = new Environment({
|
||||
workspaceDir: workspace,
|
||||
toolConfig: execCommandToolConfig(),
|
||||
});
|
||||
const engine = new ContextEngine({ llm, environment, reconnectBackoffMs: 1 });
|
||||
|
||||
const all = await collectRun(engine, [userText("go")], allowAll);
|
||||
expect(calls).toBe(1); // Auth behaves like failed: never enters the reconnect loop.
|
||||
// The request's own terminal status is the host signal (streams to the web).
|
||||
const end = all.find((m) => (m.payload as { type?: string }).type === "request_end");
|
||||
expect((end!.payload as { status?: string }).status).toBe("auth");
|
||||
expect((end!.payload as { message?: string }).message).toBe("401 invalid x-api-key");
|
||||
// No planned retry is announced for a terminal failure.
|
||||
expect((end!.payload as { retry_in_ms?: number }).retry_in_ms).toBeUndefined();
|
||||
const abort = all.find((m) => (m.payload as { type?: string }).type === "abort");
|
||||
expect(abort).toBeDefined();
|
||||
expect((abort!.payload as { reason?: string }).reason).toContain("llm request error");
|
||||
});
|
||||
|
||||
it("a quota-403 (classified timeout) retries within the default cap and succeeds", async () => {
|
||||
let calls = 0;
|
||||
const llm: LLMInterface = {
|
||||
async *streamGenerate() {
|
||||
calls += 1;
|
||||
// Two quota rejections (GenerativeModel classifies them as timeout), then success —
|
||||
// attempt 3 is within the default cap of 5.
|
||||
if (calls <= 2) return { status: "timeout" };
|
||||
yield assistantText("recovered");
|
||||
yield tokenUsage(emptyTokenCounts(), {
|
||||
cache_read: 0,
|
||||
cache_write: 0,
|
||||
output: 1,
|
||||
total: 1,
|
||||
});
|
||||
return { status: "completed" };
|
||||
},
|
||||
};
|
||||
const environment = new Environment({
|
||||
workspaceDir: workspace,
|
||||
toolConfig: execCommandToolConfig(),
|
||||
});
|
||||
const engine = new ContextEngine({ llm, environment, reconnectBackoffMs: 1 });
|
||||
|
||||
const all = await collectRun(engine, [userText("go")], allowAll);
|
||||
expect(calls).toBe(3);
|
||||
expect(
|
||||
all.some(
|
||||
(m) =>
|
||||
isCompleteModelMessage(m) && m.payload.type === "text" && m.payload.text === "recovered",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(all.map((m) => (m.payload as { type?: string }).type)).not.toContain("abort");
|
||||
});
|
||||
|
||||
it("default reconnect cap is 5: the exhaustion abort message says so", async () => {
|
||||
let calls = 0;
|
||||
const llm: LLMInterface = {
|
||||
// eslint-disable-next-line require-yield
|
||||
async *streamGenerate() {
|
||||
calls += 1;
|
||||
return { status: "timeout" }; // Always needs a reconnect.
|
||||
},
|
||||
};
|
||||
const environment = new Environment({
|
||||
workspaceDir: workspace,
|
||||
toolConfig: execCommandToolConfig(),
|
||||
});
|
||||
// No maxReconnects override: exercises the default cap (tiny base keeps the
|
||||
// exponential waits at 1+2+4+8+16 = 31ms total).
|
||||
const engine = new ContextEngine({ llm, environment, reconnectBackoffMs: 1 });
|
||||
|
||||
const all = await collectRun(engine, [userText("go")], allowAll);
|
||||
expect(calls).toBe(6); // Initial attempt + 5 retries.
|
||||
const abort = all.find((m) => (m.payload as { type?: string }).type === "abort");
|
||||
expect((abort!.payload as { reason?: string }).reason).toBe("reconnect failed after 5 retries");
|
||||
});
|
||||
|
||||
it("skipReconnectWait wakes the backoff early ('retry now'): attempt numbering unchanged; no-op without a wait", async () => {
|
||||
let calls = 0;
|
||||
const llm: LLMInterface = {
|
||||
async *streamGenerate() {
|
||||
calls += 1;
|
||||
if (calls === 1) return { status: "timeout" };
|
||||
yield assistantText("ok");
|
||||
yield tokenUsage(emptyTokenCounts(), {
|
||||
cache_read: 0,
|
||||
cache_write: 0,
|
||||
output: 1,
|
||||
total: 1,
|
||||
});
|
||||
return { status: "completed" };
|
||||
},
|
||||
};
|
||||
const environment = new Environment({
|
||||
workspaceDir: workspace,
|
||||
toolConfig: execCommandToolConfig(),
|
||||
});
|
||||
// A wait long enough that the run could only finish in time if the skip woke it.
|
||||
const engine = new ContextEngine({
|
||||
llm,
|
||||
environment,
|
||||
maxReconnects: 2,
|
||||
reconnectBackoffMs: 60_000,
|
||||
});
|
||||
|
||||
// No reconnect wait in progress yet: skip is a benign no-op.
|
||||
expect(engine.skipReconnectWait()).toBe(false);
|
||||
|
||||
const started = Date.now();
|
||||
const runP = collectRun(engine, [userText("go")], allowAll);
|
||||
// Poll the skip itself: it keeps returning false until the engine parks in the
|
||||
// backoff, and consumes the wait exactly once when it does.
|
||||
while (!engine.skipReconnectWait()) {
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
}
|
||||
const all = await runP;
|
||||
expect(Date.now() - started).toBeLessThan(10_000); // woke early: the scheduled wait was 60s
|
||||
// Exactly the one retry — the skipped wait consumed no extra attempt.
|
||||
expect(calls).toBe(2);
|
||||
const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end");
|
||||
expect(ends.map((m) => (m.payload as { status?: string }).status)).toEqual([
|
||||
"timeout",
|
||||
"completed",
|
||||
]);
|
||||
expect(all.map((m) => (m.payload as { type?: string }).type)).not.toContain("abort");
|
||||
// The wait settled (skip won the race): further skips are no-ops again.
|
||||
expect(engine.skipReconnectWait()).toBe(false);
|
||||
});
|
||||
|
||||
it("reconnectDelayMs: exponential-with-ceiling ladder (defaults: 250ms base, 30s cap)", () => {
|
||||
// The default cap (5) walks the first five steps — 250+500+1000+2000+4000 ≈ 7.75s of
|
||||
// total patience; the formula keeps climbing to the 30s ceiling for larger caps.
|
||||
const ladder = [1, 2, 3, 4, 5].map((n) => reconnectDelayMs(250, 30_000, n));
|
||||
expect(ladder).toEqual([250, 500, 1000, 2000, 4000]);
|
||||
expect(ladder.reduce((a, b) => a + b, 0)).toBe(7750);
|
||||
expect([6, 7, 8].map((n) => reconnectDelayMs(250, 30_000, n))).toEqual([8000, 16000, 30000]);
|
||||
// Past the ceiling the delay stays pinned (no overflow, no further growth).
|
||||
expect(reconnectDelayMs(250, 30_000, 12)).toBe(30_000);
|
||||
// The cap also applies when the base itself exceeds it.
|
||||
expect(reconnectDelayMs(50_000, 30_000, 1)).toBe(30_000);
|
||||
});
|
||||
|
||||
it("request_end carries the outcome's failure detail on non-completed statuses only", async () => {
|
||||
let calls = 0;
|
||||
const llm: LLMInterface = {
|
||||
async *streamGenerate() {
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
// A retryable provider rejection: the detail must reach observability via the
|
||||
// event — a retried request never produces an abort to carry it.
|
||||
return { status: "timeout", message: "403 quota exceeded (insufficient_user_quota)" };
|
||||
}
|
||||
yield assistantText("ok");
|
||||
yield tokenUsage(emptyTokenCounts(), {
|
||||
cache_read: 0,
|
||||
cache_write: 0,
|
||||
output: 1,
|
||||
total: 1,
|
||||
});
|
||||
return { status: "completed" };
|
||||
},
|
||||
};
|
||||
const environment = new Environment({
|
||||
workspaceDir: workspace,
|
||||
toolConfig: execCommandToolConfig(),
|
||||
});
|
||||
const engine = new ContextEngine({ llm, environment, maxReconnects: 1, reconnectBackoffMs: 0 });
|
||||
|
||||
const all = await collectRun(engine, [userText("go")], allowAll);
|
||||
const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end") as {
|
||||
payload: { status?: string; message?: string; retry_in_ms?: number };
|
||||
}[];
|
||||
expect(ends).toHaveLength(2);
|
||||
expect(ends[0]!.payload.status).toBe("timeout");
|
||||
expect(ends[0]!.payload.message).toBe("403 quota exceeded (insufficient_user_quota)");
|
||||
// The engine will retry: the planned backoff is announced (base 0 here -> 0ms).
|
||||
expect(ends[0]!.payload.retry_in_ms).toBe(0);
|
||||
expect(ends[1]!.payload.status).toBe("completed");
|
||||
expect(ends[1]!.payload.message).toBeUndefined();
|
||||
expect(ends[1]!.payload.retry_in_ms).toBeUndefined();
|
||||
});
|
||||
|
||||
it("request_end announces the planned backoff (retry_in_ms) from the shared ladder; absent once the cap is reached", async () => {
|
||||
let calls = 0;
|
||||
const llm: LLMInterface = {
|
||||
// eslint-disable-next-line require-yield
|
||||
async *streamGenerate() {
|
||||
calls += 1;
|
||||
return { status: "timeout" }; // Always needs a reconnect.
|
||||
},
|
||||
};
|
||||
const environment = new Environment({
|
||||
workspaceDir: workspace,
|
||||
toolConfig: execCommandToolConfig(),
|
||||
});
|
||||
const engine = new ContextEngine({
|
||||
llm,
|
||||
environment,
|
||||
maxReconnects: 2,
|
||||
reconnectBackoffMs: 10,
|
||||
reconnectBackoffMaxMs: 15,
|
||||
});
|
||||
|
||||
const all = await collectRun(engine, [userText("go")], allowAll);
|
||||
const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end") as {
|
||||
payload: { retry_in_ms?: number };
|
||||
}[];
|
||||
// Announced waits follow the same exponential-with-ceiling formula the sleep uses
|
||||
// (10, then min(20, 15) = 15); the FINAL failure carries none — no retry follows,
|
||||
// the exhaustion abort does.
|
||||
expect(ends.map((e) => e.payload.retry_in_ms)).toEqual([10, 15, undefined]);
|
||||
expect(calls).toBe(3); // Initial attempt + 2 retries.
|
||||
});
|
||||
|
||||
it("LLM timeout after a tool already executed: retry carries the call/result via [turn_retried] (tool runs once)", async () => {
|
||||
let calls = 0;
|
||||
const inputs: OmniMessage[][] = [];
|
||||
|
||||
@@ -23,8 +23,10 @@ import {
|
||||
GenerativeModel,
|
||||
ToolCallIdAllocator,
|
||||
buildUniConfig,
|
||||
isAuthenticationError,
|
||||
isIncompleteStreamError,
|
||||
isMalformedJsonParseError,
|
||||
isQuotaExhaustedError,
|
||||
isRetryableError,
|
||||
mapThinkingLevel,
|
||||
mergeOmniToUniMessage,
|
||||
@@ -1097,6 +1099,85 @@ describe("isRetryableError", () => {
|
||||
expect(isRetryableError(new Error("request timeout"))).toBe(true);
|
||||
});
|
||||
|
||||
it("treats undici transport failures as retryable, probing the cause chain", () => {
|
||||
// Node fetch wraps a dropped connection as TypeError("terminated") with the real code on
|
||||
// `cause` — exactly the field observed: "terminated: other side closed (UND_ERR_SOCKET)".
|
||||
expect(
|
||||
isRetryableError(
|
||||
new TypeError("terminated", {
|
||||
cause: Object.assign(new Error("other side closed"), { code: "UND_ERR_SOCKET" }),
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(isRetryableError({ code: "UND_ERR_SOCKET" })).toBe(true);
|
||||
expect(isRetryableError({ code: "UND_ERR_CONNECT_TIMEOUT" })).toBe(true);
|
||||
expect(isRetryableError({ code: "UND_ERR_HEADERS_TIMEOUT" })).toBe(true);
|
||||
expect(isRetryableError({ code: "UND_ERR_BODY_TIMEOUT" })).toBe(true);
|
||||
// A cause-less "fetch failed" still counts via the message keywords; "terminated" only
|
||||
// counts alongside transport vocabulary — the real socket case always carries the
|
||||
// UND_ERR_* code on its cause, and bare "terminated" appears in unrelated provider
|
||||
// copy too (see the content-filter counter-case below).
|
||||
expect(isRetryableError(new TypeError("fetch failed"))).toBe(true);
|
||||
expect(isRetryableError(new TypeError("terminated: other side closed"))).toBe(true);
|
||||
expect(isRetryableError(new TypeError("terminated"))).toBe(false);
|
||||
// Provider verdict copy that merely contains the word must NOT retry.
|
||||
expect(isRetryableError(new Error("Request terminated by content filter"))).toBe(false);
|
||||
// Classic Node codes wrapped one level down are found too.
|
||||
expect(
|
||||
isRetryableError(
|
||||
new Error("request failed", {
|
||||
cause: Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" }),
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("treats provider quota/subscription exhaustion as retryable (tight allowlist)", () => {
|
||||
// The field case: a gateway 403 with an OpenAI-compatible body code.
|
||||
expect(isRetryableError({ status: 403, code: "insufficient_user_quota" })).toBe(true);
|
||||
expect(isRetryableError({ status: 403, error: { code: "insufficient_user_quota" } })).toBe(
|
||||
true,
|
||||
);
|
||||
// Anthropic SDK shape: `error` holds the whole response body.
|
||||
expect(
|
||||
isRetryableError({ status: 403, error: { error: { code: "insufficient_quota" } } }),
|
||||
).toBe(true);
|
||||
expect(isRetryableError({ status: 402, code: "insufficient_quota" })).toBe(true);
|
||||
// Status-less quota code (some gateways surface only the provider code).
|
||||
expect(isRetryableError({ code: "insufficient_user_quota" })).toBe(true);
|
||||
// 403 with a quota/subscription message but no machine-readable code.
|
||||
expect(isRetryableError({ status: 403, message: "no active subscription" })).toBe(true);
|
||||
expect(isRetryableError({ status: 403, message: "订阅额度不足" })).toBe(true);
|
||||
// A plain 403/400/404 without quota signals stays non-retryable.
|
||||
expect(isRetryableError({ status: 403, message: "permission denied" })).toBe(false);
|
||||
expect(isRetryableError({ status: 400, code: "insufficient_user_quota" })).toBe(false);
|
||||
expect(isRetryableError({ status: 404 })).toBe(false);
|
||||
// 401 keeps its auth classification even with a quota-looking message.
|
||||
expect(isRetryableError({ status: 401, message: "quota" })).toBe(false);
|
||||
});
|
||||
|
||||
it("auth wins over the quota heuristic: a definitive credential signal is never retried", () => {
|
||||
// The adversarial case: a 403 whose BODY carries a definitive auth code but whose
|
||||
// MESSAGE mentions a subscription (SDKs routinely put the body's message on
|
||||
// err.message). The quota keyword fallback must not swallow it — auth is checked
|
||||
// first, so it classifies failed (dead credential) instead of burning every reconnect.
|
||||
const err = Object.assign(new Error("subscription key invalid"), {
|
||||
status: 403,
|
||||
error: { code: "invalid_api_key" },
|
||||
});
|
||||
expect(isAuthenticationError(err)).toBe(true);
|
||||
expect(isQuotaExhaustedError(err)).toBe(false); // belt: an auth signal is never quota
|
||||
expect(isRetryableError(err)).toBe(false); // never retried, never reclassified
|
||||
// Same with the auth signal on the cause chain.
|
||||
expect(
|
||||
isRetryableError(
|
||||
new Error("request failed: quota subscription issue", {
|
||||
cause: { status: 403, error: { type: "authentication_error" } },
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does not retry abort or unknown local errors", () => {
|
||||
const abort = new Error("aborted");
|
||||
abort.name = "AbortError";
|
||||
@@ -1107,6 +1188,76 @@ describe("isRetryableError", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("isQuotaExhaustedError", () => {
|
||||
it("matches only 402/403 (or status-less) errors carrying a known quota code", () => {
|
||||
expect(isQuotaExhaustedError({ status: 403, code: "insufficient_user_quota" })).toBe(true);
|
||||
expect(isQuotaExhaustedError({ status: 402, error: { code: "insufficient_quota" } })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isQuotaExhaustedError({ code: "insufficient_quota" })).toBe(true);
|
||||
// The code may sit on the cause chain (wrapped by a higher layer).
|
||||
expect(
|
||||
isQuotaExhaustedError(
|
||||
new Error("request failed", {
|
||||
cause: { status: 403, code: "insufficient_user_quota" },
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isQuotaExhaustedError(
|
||||
Object.assign(new Error("request failed"), {
|
||||
status: 403,
|
||||
cause: { code: "insufficient_user_quota" },
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
// Any other status keeps its own classification.
|
||||
expect(isQuotaExhaustedError({ status: 429, code: "insufficient_quota" })).toBe(false);
|
||||
expect(isQuotaExhaustedError({ status: 401, code: "insufficient_quota" })).toBe(false);
|
||||
});
|
||||
|
||||
it("matches a 403 whose message names quota/subscription, but not a plain 403", () => {
|
||||
expect(isQuotaExhaustedError({ status: 403, message: "monthly quota exceeded" })).toBe(true);
|
||||
expect(isQuotaExhaustedError({ status: 403, message: "订阅额度不足" })).toBe(true);
|
||||
expect(isQuotaExhaustedError({ status: 403, message: "forbidden" })).toBe(false);
|
||||
// The message shortcut is 403-only: a status-less error needs the explicit code.
|
||||
expect(isQuotaExhaustedError({ message: "quota exceeded" })).toBe(false);
|
||||
expect(isQuotaExhaustedError(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isAuthenticationError", () => {
|
||||
it("classifies HTTP 401 as auth, own or wrapped", () => {
|
||||
expect(isAuthenticationError({ status: 401 })).toBe(true);
|
||||
expect(isAuthenticationError({ statusCode: 401 })).toBe(true);
|
||||
expect(isAuthenticationError(new Error("request failed", { cause: { status: 401 } }))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("classifies known auth codes/types and the SDK AuthenticationError class", () => {
|
||||
expect(isAuthenticationError({ code: "invalid_api_key" })).toBe(true);
|
||||
expect(isAuthenticationError({ status: 403, error: { type: "authentication_error" } })).toBe(
|
||||
true,
|
||||
);
|
||||
// Anthropic SDK shape: `error` holds the whole response body.
|
||||
expect(isAuthenticationError({ error: { error: { code: "unauthorized" } } })).toBe(true);
|
||||
class AuthenticationError extends Error {}
|
||||
expect(isAuthenticationError(new AuthenticationError("bad key"))).toBe(true);
|
||||
expect(
|
||||
isAuthenticationError(new Error("request failed", { cause: { code: "invalid_api_key" } })),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not classify a bare 403 or unrelated failures as auth", () => {
|
||||
expect(isAuthenticationError({ status: 403 })).toBe(false);
|
||||
expect(isAuthenticationError({ status: 400, message: "invalid param" })).toBe(false);
|
||||
expect(isAuthenticationError({ status: 403, code: "insufficient_user_quota" })).toBe(false);
|
||||
expect(isAuthenticationError(new Error("socket hang up"))).toBe(false);
|
||||
expect(isAuthenticationError(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isMalformedJsonParseError", () => {
|
||||
it("detects JSON.parse SyntaxError by exception type, including the cause chain", () => {
|
||||
// AgentHub uses JSON.parse internally; a parse failure throws a SyntaxError, so it can be
|
||||
@@ -1433,14 +1584,80 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
|
||||
expect(outcome2.status).toBe("malformed");
|
||||
});
|
||||
|
||||
it("classifies a non-retryable error as failed with message, no token_usage", async () => {
|
||||
it("classifies a credentials failure as its own terminal status auth; params stay failed", async () => {
|
||||
const model = new SeamModel(() => authError());
|
||||
const { messages, outcome } = await drain(
|
||||
model.streamGenerate({ newMessages: [userText("go")] }),
|
||||
);
|
||||
expect(outcome.status).toBe("failed");
|
||||
// 401 = credentials failure: its own stop reason so hosts can gate input until the
|
||||
// model's API key is updated (same engine behavior as failed).
|
||||
expect(outcome.status).toBe("auth");
|
||||
expect(outcome.message).toContain("invalid api key");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
|
||||
// A genuinely non-retryable parameter error stays a plain failure.
|
||||
async function* paramError(): AsyncGenerator<UniEvent> {
|
||||
throw Object.assign(new Error("unknown parameter: max_output_tokens"), { status: 400 });
|
||||
}
|
||||
const model2 = new SeamModel(() => paramError());
|
||||
const { outcome: outcome2 } = await drain(
|
||||
model2.streamGenerate({ newMessages: [userText("go")] }),
|
||||
);
|
||||
expect(outcome2.status).toBe("failed");
|
||||
});
|
||||
|
||||
it("an auth error dressed in quota language still funnels to status auth", async () => {
|
||||
// The adversarial case from the review: 403 + body code invalid_api_key + a message
|
||||
// mentioning the subscription. Must NOT classify as retryable quota (timeout): the
|
||||
// ordering fix keeps the dead credential out of the reconnect loop entirely.
|
||||
async function* dressedAuthError(): AsyncGenerator<UniEvent> {
|
||||
throw Object.assign(new Error("subscription key invalid"), {
|
||||
status: 403,
|
||||
error: { code: "invalid_api_key" },
|
||||
});
|
||||
}
|
||||
const model = new SeamModel(() => dressedAuthError());
|
||||
const { messages, outcome } = await drain(
|
||||
model.streamGenerate({ newMessages: [userText("go")] }),
|
||||
);
|
||||
expect(outcome.status).toBe("auth");
|
||||
expect(outcome.message).toContain("subscription key invalid");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
});
|
||||
|
||||
it("classifies provider quota exhaustion (403 + code) as timeout — the reconnect path", async () => {
|
||||
async function* quotaError(): AsyncGenerator<UniEvent> {
|
||||
throw Object.assign(new Error("订阅额度不足 no active subscription"), {
|
||||
status: 403,
|
||||
code: "insufficient_user_quota",
|
||||
});
|
||||
}
|
||||
const model = new SeamModel(() => quotaError());
|
||||
const { messages, outcome } = await drain(
|
||||
model.streamGenerate({ newMessages: [userText("go")] }),
|
||||
);
|
||||
expect(outcome.status).toBe("timeout");
|
||||
// The real reason rides on the outcome (request_end -> the Cost center's errors panel
|
||||
// shows it): a retried quota rejection must not surface as a bare "timeout".
|
||||
expect(outcome.message).toContain("insufficient_user_quota");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
});
|
||||
|
||||
it("classifies an undici transport drop (TypeError terminated, cause UND_ERR_SOCKET) as timeout", async () => {
|
||||
async function* socketDrop(): AsyncGenerator<UniEvent> {
|
||||
yield ev({ content_items: [{ type: "text", text: "hi" }] });
|
||||
throw new TypeError("terminated", {
|
||||
cause: Object.assign(new Error("other side closed"), { code: "UND_ERR_SOCKET" }),
|
||||
});
|
||||
}
|
||||
const model = new SeamModel(() => socketDrop());
|
||||
const { messages, outcome } = await drain(
|
||||
model.streamGenerate({ newMessages: [userText("go")] }),
|
||||
);
|
||||
expect(outcome.status).toBe("timeout");
|
||||
const complete = messages.find((m) => typeOf(m) === "text");
|
||||
expect((complete!.payload as TextPayload).stop_reason).toBe("timeout");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
});
|
||||
|
||||
it("classifies a user abort (mid idle) as aborted, not timeout", async () => {
|
||||
|
||||
Reference in new issue
Block a user