fix(core,docs): window-derived output caps and compaction threshold, slower retry ladder, retry-all-but-auth (#235)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yaowei Zheng
@@ -24,7 +24,7 @@ import {
|
||||
saveProjectConfig,
|
||||
setVaultEntry,
|
||||
} from "../src/index.js";
|
||||
import { effectiveMaxContextLength, metaMaxTokens } from "../src/agent.js";
|
||||
import { metaMaxTokens } from "../src/agent.js";
|
||||
import { mapThinkingLevel } from "../src/llm/index.js";
|
||||
import { stubProviderKeys } from "./provider-keys.js";
|
||||
import type { EnvironmentConfig, EnvironmentServices, SubagentRunner } from "../src/interfaces.js";
|
||||
@@ -79,15 +79,8 @@ afterEach(async () => {
|
||||
await fs.rm(tmpRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("effectiveMaxContextLength (compaction threshold clamped to the model window)", () => {
|
||||
it("clamps to 75% of a small model window; leaves big/unknown windows and off untouched", () => {
|
||||
expect(effectiveMaxContextLength(128000, 32768)).toBe(24576); // small window: clamp to 75%
|
||||
expect(effectiveMaxContextLength(128000, 200000)).toBe(128000); // ample window: unchanged
|
||||
expect(effectiveMaxContextLength(-1, 32768)).toBe(-1); // off: no clamping
|
||||
expect(effectiveMaxContextLength(0, 32768)).toBe(0); // off: no clamping
|
||||
expect(effectiveMaxContextLength(128000, "unknown")).toBe(128000); // unknown window: no clamping
|
||||
});
|
||||
});
|
||||
// effectiveMaxContextLength moved to llm/context-limits.ts; its derivation (window-derived
|
||||
// compaction threshold, issue #218) is covered in test/context-limits.test.ts.
|
||||
|
||||
describe("metaMaxTokens (meta-request budget tightened by the per-model cap)", () => {
|
||||
it("keeps the budget unless the per-model cap is smaller; never raises it", () => {
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
/**
|
||||
* Window-derived request limits (llm/context-limits.ts, issue #218): the per-request
|
||||
* output-cap arithmetic, the input-size estimator it feeds on, and the window-derived
|
||||
* compaction threshold. All pure functions — no timers, no network.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
COMPACTION_HEADROOM,
|
||||
DEFAULT_CONTEXT_WINDOW,
|
||||
MIN_OUTPUT_TOKENS,
|
||||
MIN_USABLE_CONTEXT_WINDOW,
|
||||
OUTPUT_SAFETY_MARGIN,
|
||||
approximateMessagesTokens,
|
||||
approximateTokens,
|
||||
effectiveMaxContextLength,
|
||||
effectiveMaxOutputTokens,
|
||||
resolveContextWindow,
|
||||
} from "../src/llm/context-limits.js";
|
||||
import { toolCallOutput, userText } from "../src/omnimessage/index.js";
|
||||
import type { OmniMessage } from "../src/omnimessage/index.js";
|
||||
|
||||
describe("constant derivation (one tunable: OUTPUT_SAFETY_MARGIN)", () => {
|
||||
it("derives the floor and headroom from the margin so their invariants cannot drift", () => {
|
||||
// Floor below the margin: a floored request still fits the window when the input
|
||||
// estimate is accurate. Headroom above the margin: at the compaction trigger the
|
||||
// summary request keeps a usable output budget (~1k), not just the floor.
|
||||
expect(MIN_OUTPUT_TOKENS).toBe(Math.floor(OUTPUT_SAFETY_MARGIN / 2));
|
||||
expect(COMPACTION_HEADROOM).toBe(OUTPUT_SAFETY_MARGIN * 2);
|
||||
expect(COMPACTION_HEADROOM - OUTPUT_SAFETY_MARGIN).toBeGreaterThanOrEqual(1000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveContextWindow", () => {
|
||||
it("takes a plausible configured window at face value, anything else is unconfigured", () => {
|
||||
expect(resolveContextWindow(32768)).toBe(32768);
|
||||
expect(resolveContextWindow(MIN_USABLE_CONTEXT_WINDOW)).toBe(MIN_USABLE_CONTEXT_WINDOW);
|
||||
expect(resolveContextWindow(undefined)).toBeUndefined();
|
||||
expect(resolveContextWindow("unknown")).toBeUndefined();
|
||||
expect(resolveContextWindow(0)).toBeUndefined();
|
||||
expect(resolveContextWindow(-1)).toBeUndefined();
|
||||
expect(resolveContextWindow(Number.NaN)).toBeUndefined();
|
||||
// Below the sane minimum = a typo'd window (no real model sits under 4096): treated as
|
||||
// unconfigured rather than clamping every request against a bogus number.
|
||||
expect(resolveContextWindow(2048)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("approximateTokens (character heuristic, not a tokenizer)", () => {
|
||||
it("counts ASCII at ~4 chars/token and non-ASCII at 1 token/char", () => {
|
||||
expect(approximateTokens("")).toBe(0);
|
||||
expect(approximateTokens("abcd")).toBe(1);
|
||||
expect(approximateTokens("abcde")).toBe(2); // ceil(5/4)
|
||||
// CJK errs high on purpose: underestimating input risks a provider 400.
|
||||
expect(approximateTokens("你好世界")).toBe(4);
|
||||
expect(approximateTokens("ab你好")).toBe(3); // ceil(2/4) + 2
|
||||
});
|
||||
});
|
||||
|
||||
describe("approximateMessagesTokens", () => {
|
||||
it("estimates text-bearing payloads from their serialized size", () => {
|
||||
const text = userText("x".repeat(400));
|
||||
const est = approximateMessagesTokens([text]);
|
||||
// ~100 tokens of body + the JSON envelope: a loose sanity band, not an exact figure
|
||||
// (the heuristic's contract is "close and erring high").
|
||||
expect(est).toBeGreaterThanOrEqual(100);
|
||||
expect(est).toBeLessThan(150);
|
||||
// Tool outputs go through the same serialized-payload path; a genuinely huge text
|
||||
// output still counts as text.
|
||||
const big = toolCallOutput({
|
||||
output: "y".repeat(120_000),
|
||||
toolCallId: "c1",
|
||||
stopReason: "completed",
|
||||
});
|
||||
expect(approximateMessagesTokens([big])).toBeGreaterThanOrEqual(30_000);
|
||||
});
|
||||
|
||||
it("gives image payloads a flat allowance instead of counting data-URI characters", () => {
|
||||
const image = {
|
||||
type: "model_msg",
|
||||
payload: {
|
||||
type: "image_url",
|
||||
role: "user",
|
||||
image_url: `data:image/png;base64,${"A".repeat(100_000)}`,
|
||||
},
|
||||
} as unknown as OmniMessage;
|
||||
const est = approximateMessagesTokens([image]);
|
||||
// 100k base64 chars would be ~25k "tokens" by the char heuristic; the flat allowance
|
||||
// stays in the low thousands so one pasted screenshot cannot floor the output cap.
|
||||
expect(est).toBeLessThan(2000);
|
||||
expect(est).toBeGreaterThan(1000);
|
||||
});
|
||||
|
||||
it("counts a tool output's `images` data URLs at the flat allowance, not as base64 text", () => {
|
||||
// read_image-style outputs carry the image as tool_call_output.images (data URLs). A
|
||||
// 1 MB base64 string serialized as text would estimate ~262k "tokens" (~163x over)
|
||||
// and floor the NEXT request's cap even on a 128k window.
|
||||
const withImage = toolCallOutput({
|
||||
output: "Read image OK (1024x768).",
|
||||
toolCallId: "c1",
|
||||
stopReason: "completed",
|
||||
images: [`data:image/png;base64,${"A".repeat(1_000_000)}`],
|
||||
});
|
||||
const est = approximateMessagesTokens([withImage]);
|
||||
expect(est).toBeLessThan(2000); // flat image allowance + a small text payload
|
||||
expect(est).toBeGreaterThan(1600 - 1);
|
||||
// Two images: two allowances.
|
||||
const twoImages = toolCallOutput({
|
||||
output: "ok",
|
||||
toolCallId: "c2",
|
||||
stopReason: "completed",
|
||||
images: [
|
||||
`data:image/png;base64,${"A".repeat(500_000)}`,
|
||||
`data:image/jpeg;base64,${"B".repeat(500_000)}`,
|
||||
],
|
||||
});
|
||||
expect(approximateMessagesTokens([twoImages])).toBeLessThan(3400);
|
||||
expect(approximateMessagesTokens([twoImages])).toBeGreaterThan(3200 - 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("effectiveMaxOutputTokens (per-request output clamp)", () => {
|
||||
it("is a no-op for big-window models: the configured cap comes back unchanged", () => {
|
||||
expect(effectiveMaxOutputTokens(32000, 1_000_000, 50_000)).toBe(32000);
|
||||
expect(effectiveMaxOutputTokens(32000, DEFAULT_CONTEXT_WINDOW, 10_000)).toBe(32000);
|
||||
});
|
||||
|
||||
it("keeps the no-explicit-cap contract: unset or non-positive stays undefined", () => {
|
||||
expect(effectiveMaxOutputTokens(undefined, 32768, 1000)).toBeUndefined();
|
||||
expect(effectiveMaxOutputTokens(-1, 32768, 1000)).toBeUndefined();
|
||||
expect(effectiveMaxOutputTokens(0, 32768, 1000)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not clamp without a configured window: no hard cap from an assumption", () => {
|
||||
// An entry without context_window used to derive a clamp from the assumed 128000;
|
||||
// with compaction disabled that pinned the cap to the floor past ~127k of real
|
||||
// context on a model that used to work. Unconfigured window = configured cap as-is.
|
||||
expect(effectiveMaxOutputTokens(32000, undefined, 500_000)).toBe(32000);
|
||||
expect(effectiveMaxOutputTokens(32000, undefined, 10)).toBe(32000);
|
||||
});
|
||||
|
||||
it("clamps so estimated input + cap + margin fits the window (the issue #218 report)", () => {
|
||||
// The reported failure: window 32768, configured cap 32000, prompt ~769 tokens —
|
||||
// the fixed cap overflowed the window on the very first request.
|
||||
const cap = effectiveMaxOutputTokens(32000, 32768, 769)!;
|
||||
expect(cap).toBe(32768 - 769 - OUTPUT_SAFETY_MARGIN);
|
||||
expect(769 + cap + OUTPUT_SAFETY_MARGIN).toBeLessThanOrEqual(32768);
|
||||
// The margin binds exactly: one token less input buys one token more cap.
|
||||
expect(effectiveMaxOutputTokens(32000, 32768, 768)).toBe(cap + 1);
|
||||
});
|
||||
|
||||
it("floors at MIN_OUTPUT_TOKENS instead of emitting a non-positive cap (degenerate case)", () => {
|
||||
// Remaining window smaller than the floor — compaction should have fired long before
|
||||
// this point; the deterministic floor beats sending max_tokens <= 0.
|
||||
expect(effectiveMaxOutputTokens(32000, 32768, 32_500)).toBe(MIN_OUTPUT_TOKENS);
|
||||
expect(effectiveMaxOutputTokens(32000, 32768, 99_999)).toBe(MIN_OUTPUT_TOKENS);
|
||||
// The floor stays below the safety margin, so a floored request still fits the window
|
||||
// whenever the input estimate is accurate.
|
||||
expect(MIN_OUTPUT_TOKENS).toBeLessThanOrEqual(OUTPUT_SAFETY_MARGIN);
|
||||
});
|
||||
|
||||
it("only ever lowers a cap: a configured cap below the floor is never raised", () => {
|
||||
// A pinned meta budget (metaMaxTokens can hand a cap as small as the user pinned it)
|
||||
// must come back verbatim — the clamp lowers caps, it never raises them.
|
||||
expect(effectiveMaxOutputTokens(128, 1_000_000, 100)).toBe(128);
|
||||
expect(effectiveMaxOutputTokens(128, 32768, 99_999)).toBe(128); // degenerate: still the configured cap
|
||||
});
|
||||
});
|
||||
|
||||
describe("effectiveMaxContextLength (window-derived compaction threshold)", () => {
|
||||
it("caps the configured threshold at context_window − COMPACTION_HEADROOM", () => {
|
||||
// 32k window: compaction now fires at ~30.7k instead of never (the 128000 default was
|
||||
// unreachable inside the window).
|
||||
expect(effectiveMaxContextLength(128000, 32768)).toBe(32768 - COMPACTION_HEADROOM);
|
||||
expect(effectiveMaxContextLength(128000, 200000)).toBe(128000); // ample window: unchanged
|
||||
expect(effectiveMaxContextLength(8000, 32768)).toBe(8000); // tighter user setting wins
|
||||
// A window at the sanity minimum still derives a positive threshold, so a usable
|
||||
// window can never flip the value into the "<=0 disables" contract.
|
||||
expect(effectiveMaxContextLength(128000, MIN_USABLE_CONTEXT_WINDOW)).toBe(
|
||||
MIN_USABLE_CONTEXT_WINDOW - COMPACTION_HEADROOM,
|
||||
);
|
||||
});
|
||||
|
||||
it("derives from the 128000 default when the window is unconfigured or implausible", () => {
|
||||
expect(effectiveMaxContextLength(128000, undefined)).toBe(
|
||||
DEFAULT_CONTEXT_WINDOW - COMPACTION_HEADROOM,
|
||||
);
|
||||
expect(effectiveMaxContextLength(128000, "unknown")).toBe(
|
||||
DEFAULT_CONTEXT_WINDOW - COMPACTION_HEADROOM,
|
||||
);
|
||||
// A typo'd tiny window counts as unconfigured (resolveContextWindow's sanity
|
||||
// threshold): the threshold derives from the assumption instead of collapsing to a
|
||||
// near-zero value that would compact after every request.
|
||||
expect(effectiveMaxContextLength(128000, 2048)).toBe(
|
||||
DEFAULT_CONTEXT_WINDOW - COMPACTION_HEADROOM,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps <=0 as 'compaction disabled'", () => {
|
||||
expect(effectiveMaxContextLength(-1, 32768)).toBe(-1); // off: no clamping
|
||||
expect(effectiveMaxContextLength(0, 32768)).toBe(0); // off: no clamping
|
||||
});
|
||||
});
|
||||
@@ -1910,14 +1910,15 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
expect((abort!.payload as { reason?: string }).reason).toContain("llm request error");
|
||||
});
|
||||
|
||||
it("a quota-403 (classified timeout) retries within the default cap and succeeds", async () => {
|
||||
it("a bare 403 (classified failed) retries within the default cap and succeeds", async () => {
|
||||
let calls = 0;
|
||||
const llm: LLMInterface = {
|
||||
async *streamGenerate() {
|
||||
calls += 1;
|
||||
// Two quota rejections (GenerativeModel classifies them as timeout), then success —
|
||||
// attempt 3 is within the default cap of 5.
|
||||
if (calls <= 2) return { status: "timeout" };
|
||||
// Two provider 403 rejections (GenerativeModel labels them failed — there is no
|
||||
// quota/message heuristic anymore), then success: every non-auth failure rides
|
||||
// the ladder, and attempt 3 is within the default cap of 5.
|
||||
if (calls <= 2) return { status: "failed", errorMessage: "403 forbidden" };
|
||||
yield assistantText("recovered");
|
||||
yield tokenUsage(emptyTokenCounts(), {
|
||||
cache_read: 0,
|
||||
@@ -2020,15 +2021,16 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
expect(engine.skipReconnectWait()).toBe(false);
|
||||
});
|
||||
|
||||
it("reconnectDelayMs: exponential-with-ceiling ladder (defaults: 250ms base, 30s cap)", () => {
|
||||
// The default cap (5) walks the first five steps — 250+500+1000+2000+4000 ≈ 7.75s of
|
||||
// total patience; the formula keeps climbing to the 30s ceiling for larger caps.
|
||||
const ladder = [1, 2, 3, 4, 5].map((n) => reconnectDelayMs(250, 30_000, n));
|
||||
expect(ladder).toEqual([250, 500, 1000, 2000, 4000]);
|
||||
expect(ladder.reduce((a, b) => a + b, 0)).toBe(7750);
|
||||
expect([6, 7, 8].map((n) => reconnectDelayMs(250, 30_000, n))).toEqual([8000, 16000, 30000]);
|
||||
it("reconnectDelayMs: exponential-with-ceiling ladder (defaults: 2s base, 30s cap)", () => {
|
||||
// The default cap (5) walks 2s/4s/8s/16s and hits the 30s ceiling on the fifth wait —
|
||||
// ≈ 60s of total patience (issue #218: the old 250ms base burned the whole ladder in
|
||||
// ~7.75s, faster than a provider restart or rate-limit window can recover). Every wait
|
||||
// sits at or above the hosts' 2s countdown floor, so each retry is visible.
|
||||
const ladder = [1, 2, 3, 4, 5].map((n) => reconnectDelayMs(2000, 30_000, n));
|
||||
expect(ladder).toEqual([2000, 4000, 8000, 16000, 30000]);
|
||||
expect(ladder.reduce((a, b) => a + b, 0)).toBe(60_000);
|
||||
// Past the ceiling the delay stays pinned (no overflow, no further growth).
|
||||
expect(reconnectDelayMs(250, 30_000, 12)).toBe(30_000);
|
||||
expect([6, 7].map((n) => reconnectDelayMs(2000, 30_000, n))).toEqual([30_000, 30_000]);
|
||||
// The cap also applies when the base itself exceeds it.
|
||||
expect(reconnectDelayMs(50_000, 30_000, 1)).toBe(30_000);
|
||||
});
|
||||
|
||||
+200
-94
@@ -21,12 +21,12 @@ import type { LLMOutcome, ThinkingLevelName } from "../src/interfaces.js";
|
||||
import {
|
||||
EventTranslator,
|
||||
GenerativeModel,
|
||||
MIN_OUTPUT_TOKENS,
|
||||
ToolCallIdAllocator,
|
||||
buildUniConfig,
|
||||
isAuthenticationError,
|
||||
isIncompleteStreamError,
|
||||
isMalformedJsonParseError,
|
||||
isQuotaExhaustedError,
|
||||
isRetryableError,
|
||||
mapThinkingLevel,
|
||||
mergeOmniToUniMessage,
|
||||
@@ -1077,29 +1077,37 @@ describe("config helpers", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("isRetryableError", () => {
|
||||
it("treats 429, 408 and 5xx as retryable", () => {
|
||||
describe("isRetryableError (the timeout-vs-failed label; the engine retries every non-auth error)", () => {
|
||||
it("labels 429, 408 and 5xx transport-shaped (timeout)", () => {
|
||||
expect(isRetryableError({ status: 429 })).toBe(true);
|
||||
expect(isRetryableError({ status: 408 })).toBe(true); // Request Timeout (transient)
|
||||
expect(isRetryableError({ status: 500 })).toBe(true);
|
||||
expect(isRetryableError({ statusCode: 503 })).toBe(true);
|
||||
});
|
||||
|
||||
it("treats 4xx auth/param errors as non-retryable", () => {
|
||||
it("labels provider 4xx rejections failed — including quota-coded and bare 403s", () => {
|
||||
// The label is taxonomy, not policy: `failed` is in the engine's RETRY_STATUSES, so
|
||||
// every one of these retries on the ladder and fails after exhaustion. There is no
|
||||
// quota detector anymore — a 402/403 with a quota code or subscription message
|
||||
// classifies exactly like a bare one.
|
||||
expect(isRetryableError({ status: 400 })).toBe(false);
|
||||
expect(isRetryableError({ status: 401 })).toBe(false);
|
||||
expect(isRetryableError({ status: 403 })).toBe(false);
|
||||
expect(isRetryableError({ status: 404 })).toBe(false);
|
||||
expect(isRetryableError({ status: 403, code: "insufficient_user_quota" })).toBe(false);
|
||||
expect(isRetryableError({ status: 402, code: "insufficient_quota" })).toBe(false);
|
||||
expect(isRetryableError({ status: 403, message: "no active subscription" })).toBe(false);
|
||||
// 401 is auth (terminal), not merely failed-shaped.
|
||||
expect(isRetryableError({ status: 401 })).toBe(false);
|
||||
});
|
||||
|
||||
it("treats network error codes as retryable", () => {
|
||||
it("labels network error codes transport-shaped", () => {
|
||||
expect(isRetryableError({ code: "ECONNRESET" })).toBe(true);
|
||||
expect(isRetryableError({ code: "ETIMEDOUT" })).toBe(true);
|
||||
expect(isRetryableError(new Error("socket hang up"))).toBe(true);
|
||||
expect(isRetryableError(new Error("request timeout"))).toBe(true);
|
||||
});
|
||||
|
||||
it("treats undici transport failures as retryable, probing the cause chain", () => {
|
||||
it("labels undici transport failures transport-shaped, probing the cause chain", () => {
|
||||
// Node fetch wraps a dropped connection as TypeError("terminated") with the real code on
|
||||
// `cause` — exactly the field observed: "terminated: other side closed (UND_ERR_SOCKET)".
|
||||
expect(
|
||||
@@ -1120,7 +1128,7 @@ describe("isRetryableError", () => {
|
||||
expect(isRetryableError(new TypeError("fetch failed"))).toBe(true);
|
||||
expect(isRetryableError(new TypeError("terminated: other side closed"))).toBe(true);
|
||||
expect(isRetryableError(new TypeError("terminated"))).toBe(false);
|
||||
// Provider verdict copy that merely contains the word must NOT retry.
|
||||
// Provider verdict copy that merely contains the word is failed-shaped, not transport.
|
||||
expect(isRetryableError(new Error("Request terminated by content filter"))).toBe(false);
|
||||
// Classic Node codes wrapped one level down are found too.
|
||||
expect(
|
||||
@@ -1132,42 +1140,16 @@ describe("isRetryableError", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("treats provider quota/subscription exhaustion as retryable (tight allowlist)", () => {
|
||||
// The field case: a gateway 403 with an OpenAI-compatible body code.
|
||||
expect(isRetryableError({ status: 403, code: "insufficient_user_quota" })).toBe(true);
|
||||
expect(isRetryableError({ status: 403, error: { code: "insufficient_user_quota" } })).toBe(
|
||||
true,
|
||||
);
|
||||
// Anthropic SDK shape: `error` holds the whole response body.
|
||||
expect(
|
||||
isRetryableError({ status: 403, error: { error: { code: "insufficient_quota" } } }),
|
||||
).toBe(true);
|
||||
expect(isRetryableError({ status: 402, code: "insufficient_quota" })).toBe(true);
|
||||
// Status-less quota code (some gateways surface only the provider code).
|
||||
expect(isRetryableError({ code: "insufficient_user_quota" })).toBe(true);
|
||||
// 403 with a quota/subscription message but no machine-readable code.
|
||||
expect(isRetryableError({ status: 403, message: "no active subscription" })).toBe(true);
|
||||
expect(isRetryableError({ status: 403, message: "订阅额度不足" })).toBe(true);
|
||||
// A plain 403/400/404 without quota signals stays non-retryable.
|
||||
expect(isRetryableError({ status: 403, message: "permission denied" })).toBe(false);
|
||||
expect(isRetryableError({ status: 400, code: "insufficient_user_quota" })).toBe(false);
|
||||
expect(isRetryableError({ status: 404 })).toBe(false);
|
||||
// 401 keeps its auth classification even with a quota-looking message.
|
||||
expect(isRetryableError({ status: 401, message: "quota" })).toBe(false);
|
||||
});
|
||||
|
||||
it("auth wins over the quota heuristic: a definitive credential signal is never retried", () => {
|
||||
// The adversarial case: a 403 whose BODY carries a definitive auth code but whose
|
||||
// MESSAGE mentions a subscription (SDKs routinely put the body's message on
|
||||
// err.message). The quota keyword fallback must not swallow it — auth is checked
|
||||
// first, so it classifies failed (dead credential) instead of burning every reconnect.
|
||||
it("auth always wins: a definitive credential signal is terminal, never merely a label", () => {
|
||||
// A 403 whose BODY carries a definitive auth code but whose MESSAGE mentions a
|
||||
// subscription (SDKs routinely put the body's message on err.message): the explicit
|
||||
// credential signal decides, keeping the dead credential off the retry ladder.
|
||||
const err = Object.assign(new Error("subscription key invalid"), {
|
||||
status: 403,
|
||||
error: { code: "invalid_api_key" },
|
||||
});
|
||||
expect(isAuthenticationError(err)).toBe(true);
|
||||
expect(isQuotaExhaustedError(err)).toBe(false); // belt: an auth signal is never quota
|
||||
expect(isRetryableError(err)).toBe(false); // never retried, never reclassified
|
||||
expect(isRetryableError(err)).toBe(false);
|
||||
// Same with the auth signal on the cause chain.
|
||||
expect(
|
||||
isRetryableError(
|
||||
@@ -1176,9 +1158,12 @@ describe("isRetryableError", () => {
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
// A bare 403 carries no credential signal: NOT auth (it rides the retry ladder as
|
||||
// failed; see the engine tests).
|
||||
expect(isAuthenticationError({ status: 403, message: "forbidden" })).toBe(false);
|
||||
});
|
||||
|
||||
it("does not retry abort or unknown local errors", () => {
|
||||
it("labels abort and unknown local errors failed-shaped", () => {
|
||||
const abort = new Error("aborted");
|
||||
abort.name = "AbortError";
|
||||
expect(isRetryableError(abort)).toBe(false);
|
||||
@@ -1188,44 +1173,6 @@ describe("isRetryableError", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("isQuotaExhaustedError", () => {
|
||||
it("matches only 402/403 (or status-less) errors carrying a known quota code", () => {
|
||||
expect(isQuotaExhaustedError({ status: 403, code: "insufficient_user_quota" })).toBe(true);
|
||||
expect(isQuotaExhaustedError({ status: 402, error: { code: "insufficient_quota" } })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(isQuotaExhaustedError({ code: "insufficient_quota" })).toBe(true);
|
||||
// The code may sit on the cause chain (wrapped by a higher layer).
|
||||
expect(
|
||||
isQuotaExhaustedError(
|
||||
new Error("request failed", {
|
||||
cause: { status: 403, code: "insufficient_user_quota" },
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isQuotaExhaustedError(
|
||||
Object.assign(new Error("request failed"), {
|
||||
status: 403,
|
||||
cause: { code: "insufficient_user_quota" },
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
// Any other status keeps its own classification.
|
||||
expect(isQuotaExhaustedError({ status: 429, code: "insufficient_quota" })).toBe(false);
|
||||
expect(isQuotaExhaustedError({ status: 401, code: "insufficient_quota" })).toBe(false);
|
||||
});
|
||||
|
||||
it("matches a 403 whose message names quota/subscription, but not a plain 403", () => {
|
||||
expect(isQuotaExhaustedError({ status: 403, message: "monthly quota exceeded" })).toBe(true);
|
||||
expect(isQuotaExhaustedError({ status: 403, message: "订阅额度不足" })).toBe(true);
|
||||
expect(isQuotaExhaustedError({ status: 403, message: "forbidden" })).toBe(false);
|
||||
// The message shortcut is 403-only: a status-less error needs the explicit code.
|
||||
expect(isQuotaExhaustedError({ message: "quota exceeded" })).toBe(false);
|
||||
expect(isQuotaExhaustedError(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isAuthenticationError", () => {
|
||||
it("classifies HTTP 401 as auth, own or wrapped", () => {
|
||||
expect(isAuthenticationError({ status: 401 })).toBe(true);
|
||||
@@ -1400,6 +1347,150 @@ describe("GenerativeModel per-request thinking level", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("GenerativeModel per-request output cap (window clamp, issue #218)", () => {
|
||||
// Same capturing pattern as the thinking-level suite: the openStream seam receives the
|
||||
// per-request resolved UniConfig, whose max_tokens is the value that would go on the
|
||||
// wire. The fake stream's usage_metadata drives lastRequestTotal between requests.
|
||||
function windowModel(opts: {
|
||||
maxTokens?: number;
|
||||
contextWindow?: number;
|
||||
promptTokens?: number;
|
||||
}): { model: GenerativeModel; configs: (UniConfig | undefined)[] } {
|
||||
const configs: (UniConfig | undefined)[] = [];
|
||||
class WindowModel extends GenerativeModel {
|
||||
protected override openStream(
|
||||
_uni: UniMessage,
|
||||
_signal: AbortSignal,
|
||||
config?: UniConfig,
|
||||
): AsyncIterable<UniEvent> {
|
||||
configs.push(config);
|
||||
return (async function* () {
|
||||
yield ev({
|
||||
content_items: [{ type: "text", text: "ok" }],
|
||||
finish_reason: "stop",
|
||||
usage_metadata: {
|
||||
cached_tokens: 0,
|
||||
prompt_tokens: opts.promptTokens ?? 1,
|
||||
thoughts_tokens: 0,
|
||||
response_tokens: 1,
|
||||
},
|
||||
});
|
||||
})();
|
||||
}
|
||||
}
|
||||
const model = new WindowModel({
|
||||
modelId: "claude-sonnet-4-6",
|
||||
tools: [],
|
||||
...(opts.maxTokens !== undefined ? { maxTokens: opts.maxTokens } : {}),
|
||||
...(opts.contextWindow !== undefined ? { contextWindow: opts.contextWindow } : {}),
|
||||
});
|
||||
return { model, configs };
|
||||
}
|
||||
|
||||
async function drainAll(gen: AsyncGenerator<OmniMessage, LLMOutcome | void>): Promise<void> {
|
||||
let res = await gen.next();
|
||||
while (!res.done) res = await gen.next();
|
||||
}
|
||||
|
||||
it("is a no-op for a big configured window: the configured cap goes out unchanged", async () => {
|
||||
const { model, configs } = windowModel({ maxTokens: 32000, contextWindow: 1_000_000 });
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("hi")] }));
|
||||
expect(configs[0]?.max_tokens).toBe(32000);
|
||||
});
|
||||
|
||||
it("never clamps without a configured window — even when the measured context is huge", async () => {
|
||||
// No contextWindow on the entry: a hard cap must not be derived from the 128000
|
||||
// assumption. A large-window model that omits context_window (and has compaction
|
||||
// disabled) would otherwise get its outputs floored past the assumed mark.
|
||||
const { model, configs } = windowModel({ maxTokens: 32000, promptTokens: 200_000 });
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("a")] }));
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("b")] }));
|
||||
expect(configs[0]?.max_tokens).toBe(32000);
|
||||
expect(configs[1]?.max_tokens).toBe(32000); // measured 200k context, still no clamp
|
||||
});
|
||||
|
||||
it("counts a tool output's base64 image at the flat allowance: the next request is not floored", async () => {
|
||||
// Regression for the review repro: a read_image-style tool output carrying a 1 MB
|
||||
// data-URL image used to be serialized into the estimate (~262k "tokens"), flooring
|
||||
// request 2's max_tokens to the minimum even on a 128k window.
|
||||
const { model, configs } = windowModel({
|
||||
maxTokens: 32000,
|
||||
contextWindow: 128_000,
|
||||
promptTokens: 1000,
|
||||
});
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("read the screenshot")] }));
|
||||
const imageOutput = toolCallOutput({
|
||||
output: "Read image OK (1024x768).",
|
||||
toolCallId: "c1",
|
||||
stopReason: "completed",
|
||||
images: [`data:image/png;base64,${"A".repeat(1_000_000)}`],
|
||||
});
|
||||
await drainAll(model.streamGenerate({ newMessages: [imageOutput] }));
|
||||
expect(configs[1]?.max_tokens).toBe(32000); // flat image allowance: nowhere near the window
|
||||
});
|
||||
|
||||
it("clamps the first request of a small-window model below the window (the vLLM 400)", async () => {
|
||||
// The issue #218 report: window 32768 with the seeded cap 32000 — the fixed cap
|
||||
// overflowed the window on the very first request. The clamped value must leave the
|
||||
// estimated input plus safety margin inside the window while staying positive.
|
||||
const { model, configs } = windowModel({ maxTokens: 32000, contextWindow: 32768 });
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("hello vllm")] }));
|
||||
const cap = configs[0]?.max_tokens ?? 0;
|
||||
expect(cap).toBeLessThan(32000);
|
||||
expect(cap).toBeGreaterThan(30000); // tiny prompt: only the estimate + margin is shaved off
|
||||
});
|
||||
|
||||
it("shrinks the cap as the measured context grows; floors instead of going non-positive", async () => {
|
||||
const { model, configs } = windowModel({
|
||||
maxTokens: 32000,
|
||||
contextWindow: 32768,
|
||||
promptTokens: 30_000,
|
||||
});
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("a")] }));
|
||||
// Request 2 reasons from request 1's real token_usage (total ≈ 30001): the remaining
|
||||
// window is ~1.7k, so the cap lands between the floor and 2k.
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("b")] }));
|
||||
const second = configs[1]?.max_tokens ?? 0;
|
||||
expect(second).toBeLessThan(2000);
|
||||
expect(second).toBeGreaterThanOrEqual(MIN_OUTPUT_TOKENS);
|
||||
expect(second).toBeLessThan(configs[0]?.max_tokens ?? 0);
|
||||
});
|
||||
|
||||
it("clamps to the deterministic floor once the window is exhausted (degenerate case)", async () => {
|
||||
// A measured context beyond the window (compaction disabled/misconfigured): the cap
|
||||
// pins at MIN_OUTPUT_TOKENS — never zero or negative, which providers reject outright.
|
||||
const { model, configs } = windowModel({
|
||||
maxTokens: 32000,
|
||||
contextWindow: 32768,
|
||||
promptTokens: 40_000,
|
||||
});
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("a")] }));
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("b")] }));
|
||||
expect(configs[1]?.max_tokens).toBe(MIN_OUTPUT_TOKENS);
|
||||
});
|
||||
|
||||
it("keeps the no-explicit-cap contract: without a configured cap nothing goes on the wire", async () => {
|
||||
// The provider's own default already bounds output by the remaining window; the clamp
|
||||
// must not invent a cap where the config said "none".
|
||||
const { model, configs } = windowModel({ contextWindow: 32768, promptTokens: 30_000 });
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("a")] }));
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("b")] }));
|
||||
expect(configs[0] !== undefined && "max_tokens" in configs[0]).toBe(false);
|
||||
expect(configs[1] !== undefined && "max_tokens" in configs[1]).toBe(false);
|
||||
});
|
||||
|
||||
it("setHistory seeds the input estimate, so a resumed session clamps its first request", async () => {
|
||||
const { model, configs } = windowModel({ maxTokens: 32000, contextWindow: 32768 });
|
||||
// ~30k estimated tokens of replayed history (120k ASCII chars): without the seed the
|
||||
// first request after resume would reason from an empty context and send ~31k.
|
||||
model.setHistory([userText("x".repeat(120_000))]);
|
||||
await drainAll(model.streamGenerate({ newMessages: [userText("continue")] }));
|
||||
const cap = configs[0]?.max_tokens ?? 0;
|
||||
expect(cap).toBeLessThan(2000);
|
||||
expect(cap).toBeGreaterThanOrEqual(MIN_OUTPUT_TOKENS);
|
||||
});
|
||||
});
|
||||
|
||||
describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () => {
|
||||
// Injects a controlled UniEvent stream through the protected openStream seam to verify the
|
||||
// outcome classification of timeout/network-drop/interrupt/error, without needing a real API.
|
||||
@@ -1637,7 +1728,9 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
|
||||
expect(outcome.errorMessage).toContain("invalid api key");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
|
||||
// A genuinely non-retryable parameter error stays a plain failure.
|
||||
// A parameter error (400) labels failed — honestly reported, and still retried by the
|
||||
// engine like every non-auth failure (see engine.test.ts: it burns the ladder, then
|
||||
// surfaces verbatim).
|
||||
async function* paramError(): AsyncGenerator<UniEvent> {
|
||||
throw Object.assign(new Error("unknown parameter: max_output_tokens"), { status: 400 });
|
||||
}
|
||||
@@ -1648,10 +1741,10 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
|
||||
expect(outcome2.status).toBe("failed");
|
||||
});
|
||||
|
||||
it("an auth error dressed in quota language still funnels to status auth", async () => {
|
||||
// The adversarial case from the review: 403 + body code invalid_api_key + a message
|
||||
// mentioning the subscription. Must NOT classify as retryable quota (timeout): the
|
||||
// ordering fix keeps the dead credential out of the reconnect loop entirely.
|
||||
it("an explicit auth signal wins whatever the message says: status auth, never retried", async () => {
|
||||
// 403 + body code invalid_api_key + a message mentioning the subscription: the
|
||||
// explicit credential signal decides — auth is the one terminal status, and no
|
||||
// message vocabulary may pull a dead credential back onto the retry ladder.
|
||||
async function* dressedAuthError(): AsyncGenerator<UniEvent> {
|
||||
throw Object.assign(new Error("subscription key invalid"), {
|
||||
status: 403,
|
||||
@@ -1667,22 +1760,35 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
});
|
||||
|
||||
it("classifies provider quota exhaustion (403 + code) as timeout — the reconnect path", async () => {
|
||||
async function* quotaError(): AsyncGenerator<UniEvent> {
|
||||
throw Object.assign(new Error("订阅额度不足 no active subscription"), {
|
||||
it("labels a bare 403 as failed — a retryable status, no quota/message heuristics involved", async () => {
|
||||
// Every LLM error retries except auth: a 403 without a credential signal lands in
|
||||
// `failed` (which the engine reconnects on) instead of being probed for quota or
|
||||
// subscription vocabulary. The former quota detector is gone; a quota-coded provider
|
||||
// rejection classifies exactly the same way, and its real message still rides on the
|
||||
// outcome for observability.
|
||||
async function* bare403(): AsyncGenerator<UniEvent> {
|
||||
throw Object.assign(new Error("forbidden"), { status: 403 });
|
||||
}
|
||||
const model = new SeamModel(() => bare403());
|
||||
const { messages, outcome } = await drain(
|
||||
model.streamGenerate({ newMessages: [userText("go")] }),
|
||||
);
|
||||
expect(outcome.status).toBe("failed");
|
||||
expect(outcome.errorMessage).toContain("forbidden");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
|
||||
async function* quotaCoded403(): AsyncGenerator<UniEvent> {
|
||||
throw Object.assign(new Error("no active subscription"), {
|
||||
status: 403,
|
||||
code: "insufficient_user_quota",
|
||||
});
|
||||
}
|
||||
const model = new SeamModel(() => quotaError());
|
||||
const { messages, outcome } = await drain(
|
||||
model.streamGenerate({ newMessages: [userText("go")] }),
|
||||
const model2 = new SeamModel(() => quotaCoded403());
|
||||
const { outcome: outcome2 } = await drain(
|
||||
model2.streamGenerate({ newMessages: [userText("go")] }),
|
||||
);
|
||||
expect(outcome.status).toBe("timeout");
|
||||
// The real reason rides on the outcome (request_end -> the Cost center's errors panel
|
||||
// shows it): a retried quota rejection must not surface as a bare "timeout".
|
||||
expect(outcome.errorMessage).toContain("insufficient_user_quota");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
expect(outcome2.status).toBe("failed");
|
||||
expect(outcome2.errorMessage).toContain("insufficient_user_quota");
|
||||
});
|
||||
|
||||
it("classifies an undici transport drop (TypeError terminated, cause UND_ERR_SOCKET) as timeout", async () => {
|
||||
|
||||
Reference in New Issue
Block a user