fix(core,server,web,cli): compaction tolerance, retry parity, and failure classification (#174)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yaowei ZhengandClaude Fable 5 authored and GitHub committed 2026-08-04 17:27:45 +08:00
1 parent 7015fb0153
commit 6f60b242bd
32 files changed
+741 -326

No files matched your search

+10 -5
View File
@@ -171,7 +171,12 @@ export interface Messages {
* line: total = Session cumulative, delta = consumed by this compaction, carrying its own * line: total = Session cumulative, delta = consumed by this compaction, carrying its own
* sign); when present it is appended at the end of the line, e.g. ` · tokens 14k (+6k)`. * sign); when present it is appended at the end of the line, e.g. ` · tokens 14k (+6k)`.
*/ */
compactionStop(mode: string, status: string, tokens?: { total: string; delta: string }): string; compactionStop(
mode: string,
status: string,
tokens?: { total: string; delta: string },
errorMessage?: string,
): string;
/** Prompt shown when `/compact` has nothing to compact (session just started / two consecutive compactions). */ /** Prompt shown when `/compact` has nothing to compact (session just started / two consecutive compactions). */
compactNothing(): string; compactNothing(): string;
/** Dim line announcing one goal round (printed before the round runs). */ /** Dim line announcing one goal round (printed before the round runs). */
@@ -406,12 +411,12 @@ const en: Messages = {
mode === "discard" mode === "discard"
? `[compaction] discarding context (${reason})…` ? `[compaction] discarding context (${reason})…`
: `[compaction] summarizing context (${reason})…`, : `[compaction] summarizing context (${reason})…`,
compactionStop: (mode, status, tokens) => compactionStop: (mode, status, tokens, errorMessage) =>
(status === "completed" (status === "completed"
? mode === "discard" ? mode === "discard"
? "[compaction] done; old context discarded" ? "[compaction] done; old context discarded"
: "[compaction] done; continuing with the summarized context" : "[compaction] done; continuing with the summarized context"
: `[compaction] ${status}; keeping the current context`) + : `[compaction] ${status}${errorMessage !== undefined ? ` (${errorMessage})` : ""}; keeping the current context`) +
(tokens ? ` · tokens ${tokens.total} (${tokens.delta})` : ""), (tokens ? ` · tokens ${tokens.total} (${tokens.delta})` : ""),
compactNothing: () => "[compaction] nothing to compact yet", compactNothing: () => "[compaction] nothing to compact yet",
goalRound: (round) => `[goal] round ${round}`, goalRound: (round) => `[goal] round ${round}`,
@@ -620,12 +625,12 @@ const zh: Messages = {
mode === "discard" mode === "discard"
? `[压缩] 正在丢弃旧上下文(${reason})……` ? `[压缩] 正在丢弃旧上下文(${reason})……`
: `[压缩] 正在总结压缩上下文(${reason})……`, : `[压缩] 正在总结压缩上下文(${reason})……`,
compactionStop: (mode, status, tokens) => compactionStop: (mode, status, tokens, errorMessage) =>
(status === "completed" (status === "completed"
? mode === "discard" ? mode === "discard"
? "[压缩] 完成,旧上下文已丢弃" ? "[压缩] 完成,旧上下文已丢弃"
: "[压缩] 完成,已切换到摘要后的新上下文" : "[压缩] 完成,已切换到摘要后的新上下文"
: `[压缩] ${status === "aborted" ? "已中断" : "失败"},保留当前上下文`) + : `[压缩] ${status === "aborted" ? "已中断" : `失败${errorMessage !== undefined ? `(${errorMessage})` : ""}`},保留当前上下文`) +
(tokens ? ` · tokens ${tokens.total} (${tokens.delta})` : ""), (tokens ? ` · tokens ${tokens.total} (${tokens.delta})` : ""),
compactNothing: () => "[压缩] 当前上下文为空,无需压缩", compactNothing: () => "[压缩] 当前上下文为空,无需压缩",
goalRound: (round) => `[目标] 第 ${round} 轮`, goalRound: (round) => `[目标] 第 ${round} 轮`,
+13 -7
View File
@@ -380,8 +380,8 @@ export class StreamRenderer {
private taskLastReqEndMs: number | null = null; private taskLastReqEndMs: number | null = null;
/** Retryable terminal state (failed/timeout/malformed) of the previous request: the next request_begin is a retry, at which point a notice is printed. */ /** Retryable terminal state (failed/timeout/malformed) of the previous request: the next request_begin is a retry, at which point a notice is printed. */
private pendingRetry: "failed" | "timeout" | "malformed" | null = null; private pendingRetry: "failed" | "timeout" | "malformed" | null = null;
/** Number of retries already initiated (increments on consecutive failures, reset once a request completes normally). */ /** The pending failure's attempt ordinal (request_end.attempt — the core's authoritative count, printed on the retry line). */
private reconnectRun = 0; private pendingRetryAttempt: number | undefined;
constructor(out: NodeJS.WritableStream = process.stdout, t: Messages = defaultMessages()) { constructor(out: NodeJS.WritableStream = process.stdout, t: Messages = defaultMessages()) {
this.out = out; this.out = out;
@@ -688,7 +688,7 @@ export class StreamRenderer {
} else if (payload.type === "abort") { } else if (payload.type === "abort") {
// Run ended (user interrupt / retries exhausted): clear any pending retry state so the next run doesn't mistakenly print a retry line. // Run ended (user interrupt / retries exhausted): clear any pending retry state so the next run doesn't mistakenly print a retry line.
this.pendingRetry = null; this.pendingRetry = null;
this.reconnectRun = 0; this.pendingRetryAttempt = undefined;
this.finishLine(); this.finishLine();
this.out.write(`${formatAbort(payload as AbortPayload, this.t)}\n`); this.out.write(`${formatAbort(payload as AbortPayload, this.t)}\n`);
this.lastLineKey = null; this.lastLineKey = null;
@@ -698,9 +698,12 @@ export class StreamRenderer {
// retries are exhausted, there's no retry after the last failure, only an abort // retries are exhausted, there's no retry after the last failure, only an abort
// explaining why). // explaining why).
if (this.pendingRetry) { if (this.pendingRetry) {
this.reconnectRun += 1; // The failed request_end stamped its authoritative attempt ordinal — print it
// (the CLI renders live streams only, which always come from a stamping core).
const attempt = this.pendingRetryAttempt ?? 1;
this.pendingRetryAttempt = undefined;
this.finishLine(); this.finishLine();
this.out.write(`${dim(this.t.reconnectLabel(this.pendingRetry, this.reconnectRun))}\n`); this.out.write(`${dim(this.t.reconnectLabel(this.pendingRetry, attempt))}\n`);
this.lastLineKey = null; this.lastLineKey = null;
this.pendingRetry = null; this.pendingRetry = null;
} }
@@ -725,9 +728,10 @@ export class StreamRenderer {
// reset the counter mid-ladder so a mixed run renumbers back to retry #1. // reset the counter mid-ladder so a mixed run renumbers back to retry #1.
if (p.status === "failed" || p.status === "timeout" || p.status === "malformed") { if (p.status === "failed" || p.status === "timeout" || p.status === "malformed") {
this.pendingRetry = p.status; this.pendingRetry = p.status;
this.pendingRetryAttempt = p.attempt;
} else { } else {
this.pendingRetry = null; this.pendingRetry = null;
this.reconnectRun = 0; this.pendingRetryAttempt = undefined;
} }
} else if (payload.type === "compaction_begin") { } else if (payload.type === "compaction_begin") {
// Paired compaction events: begin signals compaction is in progress. // Paired compaction events: begin signals compaction is in progress.
@@ -751,7 +755,9 @@ export class StreamRenderer {
} }
: undefined; : undefined;
this.compactionTokens = 0; this.compactionTokens = 0;
this.out.write(`${dim(this.t.compactionStop(p.mode, p.status, tokens))}\n`); this.out.write(
`${dim(this.t.compactionStop(p.mode, p.status, tokens, p.error_message))}\n`,
);
this.lastLineKey = null; this.lastLineKey = null;
} }
return; return;
+13 -12
View File
@@ -281,50 +281,51 @@ describe("StreamRenderer", () => {
expect(stripAnsi(text())).toBe("[tool-c3] -> line1\n"); expect(stripAnsi(text())).toBe("[tool-c3] -> line1\n");
}); });
it("prints the retry line only when the retry request actually begins", () => { it("prints the retry line, with the stamped attempt, only when the retry request actually begins", () => {
const { stream, text } = collector(); const { stream, text } = collector();
const r = new StreamRenderer(stream, t); const r = new StreamRenderer(stream, t);
r.handle(requestBegin()); r.handle(requestBegin());
r.handle(requestEnd("malformed")); r.handle(requestEnd("malformed", { attempt: 1 }));
expect(stripAnsi(text())).toBe(""); // the failure itself prints nothing; only the retry's start does expect(stripAnsi(text())).toBe(""); // the failure itself prints nothing; only the retry's start does
r.handle(requestBegin()); // retry #1 begins r.handle(requestBegin()); // retry #1 begins
expect(stripAnsi(text())).toContain("retry #1"); expect(stripAnsi(text())).toContain("retry #1");
r.handle(requestEnd("timeout")); r.handle(requestEnd("timeout", { attempt: 2 }));
r.handle(requestBegin()); // retry #2 begins r.handle(requestBegin()); // retry #2 begins
expect(stripAnsi(text())).toContain("retry #2"); expect(stripAnsi(text())).toContain("retry #2");
// Retry #2 fails again and retries are exhausted: no next request_begin, only abort — no retry #3 appears. // Retry #2 fails again and retries are exhausted: no next request_begin, only abort — no retry #3 appears.
r.handle(requestEnd("malformed")); r.handle(requestEnd("malformed", { attempt: 3 }));
r.handle(abortEvent("malformed response failed after 2 retries")); r.handle(abortEvent("malformed response failed after 2 retries"));
expect(stripAnsi(text())).not.toContain("retry #3"); expect(stripAnsi(text())).not.toContain("retry #3");
// The first request of the next run is not a retry, so it prints nothing; a new failure after it counts from 1 again. // The first request of the next run is not a retry, so it prints nothing; the next run's
// failures are stamped from 1 again.
r.handle(requestBegin()); r.handle(requestBegin());
r.handle(requestEnd("timeout")); r.handle(requestEnd("timeout", { attempt: 1 }));
r.handle(requestBegin()); r.handle(requestBegin());
const lines = stripAnsi(text()); const lines = stripAnsi(text());
expect(lines.match(/retry #1/g)).toHaveLength(2); expect(lines.match(/retry #1/g)).toHaveLength(2);
expect(lines).not.toContain("retry #3"); expect(lines).not.toContain("retry #3");
}); });
it("prints the retry line for a failed request too, and keeps counting across a mixed ladder", () => { it("prints the retry line for a failed request too, straight from the stamped ordinal", () => {
// The engine reconnects on `failed` as well, so the CLI has to say so — otherwise the // The engine reconnects on `failed` as well, so the CLI has to say so — otherwise the
// session goes quiet for the whole ladder. And because `failed` used to reset the // session goes quiet for the whole ladder. The number is the event's own attempt, so a
// counter, a mixed timeout → failed → timeout run renumbered back to retry #1. // mixed timeout → failed → timeout run keeps counting without any client-side state.
const { stream, text } = collector(); const { stream, text } = collector();
const r = new StreamRenderer(stream, t); const r = new StreamRenderer(stream, t);
r.handle(requestBegin()); r.handle(requestBegin());
r.handle(requestEnd("failed", "Upstream HTTP/2 stream failed")); r.handle(requestEnd("failed", { errorMessage: "Upstream HTTP/2 stream failed", attempt: 1 }));
r.handle(requestBegin()); // retry #1 begins r.handle(requestBegin()); // retry #1 begins
let lines = stripAnsi(text()); let lines = stripAnsi(text());
expect(lines).toContain("the model provider returned an error"); expect(lines).toContain("the model provider returned an error");
expect(lines).toContain("retry #1"); expect(lines).toContain("retry #1");
r.handle(requestEnd("timeout")); r.handle(requestEnd("timeout", { attempt: 2 }));
r.handle(requestBegin()); // retry #2 begins — the count does not restart r.handle(requestBegin()); // retry #2 begins — the count does not restart
lines = stripAnsi(text()); lines = stripAnsi(text());
expect(lines).toContain("connection timed out"); expect(lines).toContain("connection timed out");
expect(lines).toContain("retry #2"); expect(lines).toContain("retry #2");
// `auth` is terminal: the engine never retries it, so the next request_begin (a new run) // `auth` is terminal: the engine never retries it, so the next request_begin (a new run)
// must not be announced as a retry. // must not be announced as a retry.
r.handle(requestEnd("auth", "401 invalid x-api-key")); r.handle(requestEnd("auth", { errorMessage: "401 invalid x-api-key", attempt: 3 }));
r.handle(requestBegin()); r.handle(requestBegin());
expect(stripAnsi(text())).not.toContain("retry #3"); expect(stripAnsi(text())).not.toContain("retry #3");
}); });
+152 -101
View File
@@ -183,10 +183,14 @@ export interface ContextEngineDeps {
/** Ceiling (ms) for a single reconnect backoff wait. Defaults to 30000. */ /** Ceiling (ms) for a single reconnect backoff wait. Defaults to 30000. */
reconnectBackoffMaxMs?: number; reconnectBackoffMaxMs?: number;
/** /**
* Maximum retries for a failing compaction request (instead of the shared `maxReconnects`; * Maximum retries for a failing compaction request — one budget for every failure kind:
* same statuses, shorter budget — see RETRY_STATUSES). Compaction failure is already * the transport statuses (see RETRY_STATUSES) and a committed response that isn't a usable
* graceful — the original context is kept and compaction retries at the next trigger — so a * summary (empty, or tool calls) all draw from it; only `auth` stops without retrying.
* broken request fails fast (~1.75s of backoff). Defaults to 3. * Defaults to the shared `maxReconnects`: a compaction request is an ordinary LLM request
* and deserves the same patience (issue #170 — the earlier tighter budget made a
* struggling provider fail compaction fast, and a session whose every turn re-triggers
* compaction is stuck). Failure stays graceful either way: the original context is kept
* and compaction retries at the next trigger.
*/ */
compactionMaxReconnects?: number; compactionMaxReconnects?: number;
/** /**
@@ -225,17 +229,19 @@ const carriesSteering = (m: OmniMessage): boolean => {
export type CompactAvailability = "ok" | "unsupported" | "empty" | "just_compacted"; export type CompactAvailability = "ok" | "unsupported" | "empty" | "just_compacted";
/** /**
* Maximum summarize attempts when the compaction response is rejected as an invalid summary * Corrective note prepended to the re-sent compaction Prompt after a committed-but-unusable
* (empty extracted text, or the model answered with tool calls — issue #83/#84). Deliberately * response (empty summary, or tool calls — issues #83/#170). The unusable
* separate from (and larger than) `compactionMaxReconnects`: that cap governs the retryable * response is committed on the live LLM object and can only be *appended* to (rewriting the
* attempts that were never committed (failed/timeout/malformed) and back off exponentially, while a * prefix would invalidate the provider's prompt cache at the moment the context is largest —
* rejection is a well-formed committed response — the request itself works, the model just * the same invariant that pins the toolset, issue #84); without an explicit correction the
* didn't produce a summary, so the repaired input is resent immediately with no backoff. And * model sees its own bad output as the freshest example and copies it verbatim on every
* since the compaction request keeps the session's toolset (the prefix cache must stay valid, * retry (issue #170: deepseek-v4-flash kept writing the body after `[/summary]`).
* see summarizeContext), a model insisting on tools deserves several chances. * Exported for unit tests.
* Beyond this many rejected attempts the compaction fails (original context kept).
*/ */
const MAX_SUMMARY_REJECTIONS = 5; export const SUMMARY_RETRY_GUIDANCE =
"Your previous reply was not a usable summary. Reply again with text only, no tool " +
"calls, in exactly this format and nothing after it:\n\n" +
"[summary]put the summary text here...[/summary]";
/** Result of executing one LLM turn (the return value of runTurn). */ /** Result of executing one LLM turn (the return value of runTurn). */
interface TurnResult { interface TurnResult {
@@ -342,9 +348,11 @@ function downgradeCarriedGoalInput(msg: OmniMessage): OmniMessage {
* own way lands here; retrying a genuinely permanent error costs the ladder and ends the same * own way lands here; retrying a genuinely permanent error costs the ladder and ends the same
* way, while aborting a transient one destroys the turn. * way, while aborting a transient one destroys the turn.
* *
* Both loops retry the same set. What differs is the budget: compaction runs on * Both loops retry the same set on the same backoff ladder, and by default with the same
* `compactionMaxReconnects` (shorter than the turn loop's), because a compaction that gives * budget: compaction runs on `compactionMaxReconnects`, which follows `maxReconnects` unless
* up keeps the original context and tries again at the next trigger. * set explicitly (issue #170 — a compaction request is an ordinary LLM request and gets the
* turn loop's patience; compaction additionally routes a committed-but-unusable summary
* through the same budget, see summarizeContext).
*/ */
const RETRY_STATUSES: readonly StopReason[] = ["failed", "timeout", "malformed"]; const RETRY_STATUSES: readonly StopReason[] = ["failed", "timeout", "malformed"];
@@ -406,7 +414,7 @@ export class ContextEngine {
this.maxReconnects = deps.maxReconnects ?? 5; this.maxReconnects = deps.maxReconnects ?? 5;
this.reconnectBackoffMs = deps.reconnectBackoffMs ?? 250; this.reconnectBackoffMs = deps.reconnectBackoffMs ?? 250;
this.reconnectBackoffMaxMs = deps.reconnectBackoffMaxMs ?? 30_000; this.reconnectBackoffMaxMs = deps.reconnectBackoffMaxMs ?? 30_000;
this.compactionMaxReconnects = deps.compactionMaxReconnects ?? 3; this.compactionMaxReconnects = deps.compactionMaxReconnects ?? this.maxReconnects;
this.llm = deps.llm; this.llm = deps.llm;
// Session resumption: apply the initial state derived from replay. // Session resumption: apply the initial state derived from replay.
const init = deps.initialState; const init = deps.initialState;
@@ -619,7 +627,7 @@ export class ContextEngine {
// else — `failed` included — goes to the reconnect loop below. // else — `failed` included — goes to the reconnect loop below.
if (turn.outcome.status === "auth") { if (turn.outcome.status === "auth") {
this.pendingCarryOver = this.buildCarryOver(attemptInput, turn); this.pendingCarryOver = this.buildCarryOver(attemptInput, turn);
yield* this.emitAbort(`llm request error: ${turn.outcome.message ?? "unknown"}`); yield* this.emitAbort(`llm request error: ${turn.outcome.errorMessage ?? "unknown"}`);
return; return;
} }
// Completed normally. // Completed normally.
@@ -653,7 +661,7 @@ export class ContextEngine {
turn.outcome.status === "malformed" turn.outcome.status === "malformed"
? `malformed response failed after ${this.maxReconnects} retries` ? `malformed response failed after ${this.maxReconnects} retries`
: turn.outcome.status === "failed" : turn.outcome.status === "failed"
? `llm request failed after ${this.maxReconnects} retries: ${turn.outcome.message ?? "unknown"}` ? `llm request failed after ${this.maxReconnects} retries: ${turn.outcome.errorMessage ?? "unknown"}`
: `reconnect failed after ${this.maxReconnects} retries`; : `reconnect failed after ${this.maxReconnects} retries`;
yield* this.emitAbort(reason); yield* this.emitAbort(reason);
return; return;
@@ -967,16 +975,21 @@ export class ContextEngine {
// (the errors panel) can learn the real reason (e.g. a quota code). When the // (the errors panel) can learn the real reason (e.g. a quota code). When the
// engine will retry in-run, the planned backoff rides along as retry_in_ms // engine will retry in-run, the planned backoff rides along as retry_in_ms
// (the frontend's live countdown); absent on final failures and completions. // (the frontend's live countdown); absent on final failures and completions.
const stopEvt = requestEnd( const retryInMs = this.plannedRetryDelayMs(
outcome.status, outcome,
outcome.message, reconnectsSoFar,
this.plannedRetryDelayMs( this.maxReconnects,
outcome, RETRY_STATUSES,
reconnectsSoFar,
this.maxReconnects,
RETRY_STATUSES,
),
); );
const stopEvt = requestEnd(outcome.status, {
...(outcome.errorMessage !== undefined ? { errorMessage: outcome.errorMessage } : {}),
// The authoritative attempt ordinal (1-based, within this retry run); a clean
// first-try completion stays unstamped so the common case adds no noise.
...(outcome.status !== "completed" || reconnectsSoFar > 0
? { attempt: reconnectsSoFar + 1 }
: {}),
...(retryInMs !== undefined ? { retryInMs } : {}),
});
queue.push(stopEvt); queue.push(stopEvt);
await this.write(stopEvt); await this.write(stopEvt);
break; break;
@@ -1230,15 +1243,17 @@ export class ContextEngine {
* transcript uncached costs tens of times more (issue #84 — this is why tools are *not* * transcript uncached costs tens of times more (issue #84 — this is why tools are *not*
* omitted and no `tool_choice` override is used). The * omitted and no `tool_choice` override is used). The
* compaction request's streamed output is not pushed to the Human output stream (it emits * compaction request's streamed output is not pushed to the Human output stream (it emits
* paired compaction events, plus the compaction request's `token_usage` — positioned between * paired compaction events, plus every attempt's `token_usage` — positioned between the two
* the two events, so the frontend can count compaction cost into its stats), but it is written * events, so the frontend stats and the server's usage records count the compaction's true
* spend, rejected attempts included), but it is written
* to the old Trace. Compaction succeeds only with a **valid summary** — non-empty extracted * to the old Trace. Compaction succeeds only with a **valid summary** — non-empty extracted
* text and no tool calls in the response. An invalid summary is rejected: any tool calls the * text and no tool calls in the response. Everything short of that is one kind of failure,
* model issued are answered with synthesized failed outputs (pairing repair, see the loop * handled exactly like an ordinary LLM request's (issue #170): an unusable committed
* body) and the repaired input is resent immediately, up to MAX_SUMMARY_REJECTIONS attempts, * response (empty summary, or tool calls — answered with synthesized failed outputs and
* then the compaction fails. failed/timeout/malformed reconnect under the compaction-specific * retried behind a corrective note, see the loop body) and the transport failures
* cap (`compactionMaxReconnects` — a shorter budget than the turn loop's, not a narrower * (failed/timeout/malformed) all reconnect under the one `compactionMaxReconnects` budget
* set), collapsing to failed once retries are exhausted; on failure/abort, the original * (defaulting to the turn loop's budget and ladder — see RETRY_STATUSES); only `auth` stops
* without retrying. Once the budget is exhausted the compaction fails; on failure/abort, the original
* context and Trace index are kept — it does not fall back to discard. The first **committed** * context and Trace index are kept — it does not fall back to discard. The first **committed**
* attempt absorbs `pendingToolOutputs` into the old context's history (issue #85): later * attempt absorbs `pendingToolOutputs` into the old context's history (issue #85): later
* resends carry only the repairs and the Prompt, and the result's `committed` flag tells * resends carry only the repairs and the Prompt, and the result's `committed` flag tells
@@ -1271,24 +1286,40 @@ export class ContextEngine {
// clean end — none of those reach the stateful commit). Returned as `committed`: the // clean end — none of those reach the stateful commit). Returned as `committed`: the
// callers' two-case carry rule branches on it. // callers' two-case carry rule branches on it.
let committed = false; let committed = false;
// Synthesized outputs answering the latest rejected attempt's tool calls, not yet carried // Synthesized outputs answering the latest unusable attempt's tool calls, not yet carried
// by a committed request: prepended to the retry input, and stashed as carry-over should // by a committed request: prepended to the retry input, and stashed as carry-over should
// the compaction be abandoned first (see stashRepairs). // the compaction be abandoned first (see stashRepairs).
let pendingRepairs: OmniMessage[] = []; let pendingRepairs: OmniMessage[] = [];
// Two independent retry budgets: retryable attempts (failed/timeout/malformed, never // One retry budget for every failure: an unusable committed response (empty summary /
// committed) follow the compaction-specific reconnect cap with the exponential backoff // tool calls) counts exactly like a transport failure (issue #170) — same counter, same
// ladder; invalid-summary rejections (committed, well-formed responses that just aren't // exponential ladder — and only `auth` stops without retrying.
// summaries) get the larger dedicated cap and resend immediately — see
// MAX_SUMMARY_REJECTIONS and the rejection branch below.
let reconnects = 0; let reconnects = 0;
let rejections = 0; // The compaction_end event's share of the RetryDetail block (also what the server's
// error record carries): the final attempt ordinal, and the last failure's detail.
let attempts = 0;
let lastError: string | undefined;
for (;;) { for (;;) {
if (signal?.aborted) { if (signal?.aborted) {
this.stashRepairs(pendingRepairs); this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "aborted"); yield* this.emitCompactionEnd(
reason,
"summarize",
"aborted",
attempts > 0 ? { attempt: attempts } : undefined,
);
return { status: "aborted", committed }; return { status: "aborted", committed };
} }
const attempt = await this.runCompactionRequest(input, signal, reconnects); const attempt = await this.runCompactionRequest(input, signal, reconnects);
attempts += 1;
// Every attempt's token_usage is pushed to the Human output stream (already written to
// Trace in runCompactionRequest, so it's only yielded here, never rewritten): the frontend
// stats and the server's usage records then carry the compaction's true spend — failed
// attempts burn real tokens (issue #170), and surfacing only the adopted attempt's usage
// understated the cost center.
if (attempt.usage) yield attempt.usage;
// A committed-but-unusable response (empty summary or tool calls): its retry input must
// be rebuilt below — repairs + corrective note + Prompt — instead of resent unchanged.
let unusable = false;
if (attempt.status === "completed") { if (attempt.status === "completed") {
// The attempt was committed by AgentHub, so whatever its input carried — including // The attempt was committed by AgentHub, so whatever its input carried — including
// repairs synthesized for a previous rejection — is now in history and must not be // repairs synthesized for a previous rejection — is now in history and must not be
@@ -1306,30 +1337,26 @@ export class ContextEngine {
// live possibility, not just a hallucination (issue #83). // live possibility, not just a hallucination (issue #83).
const summaryText = extractSummary(attempt.text); const summaryText = extractSummary(attempt.text);
if (summaryText !== "" && attempt.toolCalls.length === 0) { if (summaryText !== "" && attempt.toolCalls.length === 0) {
// The compaction request's token_usage is pushed to the Human output stream (already
// written to Trace in runCompactionRequest, so here it's only yielded, not rewritten);
// the frontend uses this to count compaction cost into stats and display it on the
// compaction-complete line. Only the adopted attempt's usage is surfaced: rejected
// attempts still feed observeTokenUsage (Session cumulative cost and context
// tracking stay correct), so the displayed compaction cost deliberately understates
// the true spend when retries happened — chosen so the line reflects the attempt
// that produced the summary.
if (attempt.usage) yield attempt.usage;
const summary = userText(buildContextSummaryText(summaryText)); const summary = userText(buildContextSummaryText(summaryText));
yield* this.emitCompactionEnd(reason, "summarize", "completed"); yield* this.emitCompactionEnd(reason, "summarize", "completed", { attempt: attempts });
await this.startNewContext(); await this.startNewContext();
return { status: "completed", summary, committed }; return { status: "completed", summary, committed };
} }
// Rejected. Tool calls were never dispatched, yet the assistant turn holding them IS // Not a summary — one more failed attempt, sharing the reconnect budget below. Tool
// calls were never dispatched, yet the assistant turn holding them IS
// committed on the live LLM object — leaving them unanswered would get every // committed on the live LLM object — leaving them unanswered would get every
// subsequent request rejected by the provider (unanswered tool_use, issue #33): the // subsequent request rejected by the provider (unanswered tool_use, issue #33): the
// exact state this file's other safety nets exist to prevent. Answer each call with a // exact state this file's other safety nets exist to prevent. Answer each call with a
// synthesized failed output (the same shape executeOne uses), written to Trace so // synthesized failed output (the same shape executeOne uses), written to Trace so
// resume replays the identical pairing, and prepended to the retry input so the // resume replays the identical pairing, and prepended to the retry input so the
// provider sees tool_use/tool_result paired. The empty-text rejection needs no repair: // provider sees tool_use/tool_result paired. The empty-text case needs no repair:
// that committed turn is plain assistant text/thinking, and re-sending the compaction // that committed turn is plain assistant text/thinking, and re-sending the compaction
// Prompt on top of it is structurally sound. // Prompt on top of it is structurally sound.
rejections += 1; unusable = true;
lastError =
attempt.toolCalls.length > 0
? "the response called tools instead of writing a summary"
: "the response contained no usable summary";
pendingRepairs = attempt.toolCalls.map((tc) => pendingRepairs = attempt.toolCalls.map((tc) =>
toolCallOutput({ toolCallOutput({
output: "[tool error] the compaction request expects a summary, not tool calls", output: "[tool error] the compaction request expects a summary, not tool calls",
@@ -1338,53 +1365,61 @@ export class ContextEngine {
}), }),
); );
for (const repair of pendingRepairs) await this.write(repair); for (const repair of pendingRepairs) await this.write(repair);
// Rebuild from the (shrunken) base rather than appending: everything the rejected } else if (attempt.status === "aborted") {
// attempt's input carried is committed, so only the fresh repairs and the Prompt go
// out again.
input = pendingRepairs.length > 0 ? [...pendingRepairs, ...base] : base;
if (rejections >= MAX_SUMMARY_REJECTIONS) {
this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "failed");
return { status: "failed", committed };
}
// A rejection is model behavior, not a transport failure: the request pipeline is
// healthy, so the repaired input is resent immediately — no backoff and no
// retry_in_ms announcement (the rejected attempt's request_end carries status
// completed, for which plannedRetryDelayMs yields nothing). The exponential ladder
// below belongs to transport failures only.
continue;
}
if (attempt.status === "aborted") {
this.stashRepairs(pendingRepairs); this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "aborted"); yield* this.emitCompactionEnd(reason, "summarize", "aborted", { attempt: attempts });
return { status: "aborted", committed }; return { status: "aborted", committed };
} } else if (attempt.status === "auth") {
if (attempt.status === "auth") { // `auth` is the one status that never retries: credentials don't heal on a ladder.
// `auth` folds into `failed` here: the compaction event pair keeps its // It folds into `failed` here: the compaction event pair keeps its
// completed/failed/aborted set, the original context is kept, and the host learns // completed/failed/aborted set, the original context is kept, and the host learns
// about the credential problem from the request's own terminal status (a turn-loop // about the credential problem from the request's own terminal status (a turn-loop
// request will surface it; the compaction request_end is Trace-only). // request will surface it; the compaction request_end is Trace-only).
this.stashRepairs(pendingRepairs); this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "failed"); yield* this.emitCompactionEnd(reason, "summarize", "failed", {
attempt: attempts,
...(attempt.errorMessage !== undefined ? { errorMessage: attempt.errorMessage } : {}),
});
return { status: "failed", committed }; return { status: "failed", committed };
} else {
// Transport failure: keep its detail as the last error of record.
lastError = attempt.errorMessage;
} }
// failed / timeout / malformed: retried via reconnect — none of them is committed by // One failure path for everything else — unusable summaries and the transport statuses
// AgentHub (case B), so the input (any pending repairs included) is resent unchanged. // (failed / timeout / malformed, never committed by AgentHub) — treated like an
// Compaction uses its own, tighter cap (not the shared maxReconnects): a compaction // ordinary LLM request's failures: the same budget (defaulting to the shared
// that ends up failing keeps the original context and tries again on the next trigger, // maxReconnects, issue #170) and the same exponential ladder. An unusable attempt's
// so a short ladder here beats holding the session through the full one. // request_end carries status completed, for which no retry_in_ms is announced — the
// backoff wait still happens.
if (reconnects >= this.compactionMaxReconnects) { if (reconnects >= this.compactionMaxReconnects) {
this.stashRepairs(pendingRepairs); this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "failed"); yield* this.emitCompactionEnd(reason, "summarize", "failed", {
attempt: attempts,
...(lastError !== undefined ? { errorMessage: lastError } : {}),
});
return { status: "failed", committed }; return { status: "failed", committed };
} }
reconnects += 1; reconnects += 1;
const ok = await this.backoff(reconnects, signal); const ok = await this.backoff(reconnects, signal);
if (!ok) { if (!ok) {
this.stashRepairs(pendingRepairs); this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "aborted"); yield* this.emitCompactionEnd(reason, "summarize", "aborted", { attempt: attempts });
return { status: "aborted", committed }; return { status: "aborted", committed };
} }
if (unusable) {
// Rebuild from the (shrunken) base rather than appending: everything the unusable
// attempt's input carried is committed — the live object's history can only grow, so
// the retry appends the fresh repairs, a corrective note, and the Prompt. The note
// (written to Trace like the Prompt, and only when a retry actually follows) is what
// breaks the copy-my-own-mistake loop: the model's freshest example is its committed
// bad output, and rewriting history to hide it would invalidate the provider's
// prompt cache (issue #84) — correcting forward is the one cache-safe option
// (issue #170). Transport failures skip this: nothing was committed, so their input
// is resent unchanged (any pending repairs included).
const guidance = userText(SUMMARY_RETRY_GUIDANCE);
await this.write(guidance);
input = [...pendingRepairs, guidance, ...base];
}
} }
} }
@@ -1426,6 +1461,8 @@ export class ContextEngine {
text: string; text: string;
toolCalls: OmniMessage<ToolCallPayload>[]; toolCalls: OmniMessage<ToolCallPayload>[];
usage: OmniMessage | null; usage: OmniMessage | null;
/** Error detail (LLMOutcome.errorMessage) on non-completed statuses — becomes compaction_end.error_message when this failure ends the compaction. */
errorMessage?: string;
}> { }> {
// The compaction request is itself an ordinary Request, emitting paired request events — // The compaction request is itself an ordinary Request, emitting paired request events —
// written to the (old) Trace only, not pushed to the stream, keeping the compaction process // written to the (old) Trace only, not pushed to the stream, keeping the compaction process
@@ -1448,19 +1485,32 @@ export class ContextEngine {
// compaction event pair. A rejected summary ends `completed`, for which // compaction event pair. A rejected summary ends `completed`, for which
// plannedRetryDelayMs yields nothing — rejection resends are immediate (see // plannedRetryDelayMs yields nothing — rejection resends are immediate (see
// summarizeContext), so no wait is ever announced for them. // summarizeContext), so no wait is ever announced for them.
await this.write( const retryInMs = this.plannedRetryDelayMs(
requestEnd( res.value,
res.value.status, reconnectsSoFar,
res.value.message, this.compactionMaxReconnects,
this.plannedRetryDelayMs( RETRY_STATUSES,
res.value,
reconnectsSoFar,
this.compactionMaxReconnects,
RETRY_STATUSES,
),
),
); );
return { status: res.value.status, text, toolCalls, usage }; await this.write(
requestEnd(res.value.status, {
...(res.value.errorMessage !== undefined
? { errorMessage: res.value.errorMessage }
: {}),
// Same stamping rule as the turn loop; for compaction the ordinal counts every
// retry kind (transport and unusable-summary alike share one budget).
...(res.value.status !== "completed" || reconnectsSoFar > 0
? { attempt: reconnectsSoFar + 1 }
: {}),
...(retryInMs !== undefined ? { retryInMs } : {}),
}),
);
return {
status: res.value.status,
text,
toolCalls,
usage,
...(res.value.errorMessage !== undefined ? { errorMessage: res.value.errorMessage } : {}),
};
} }
const msg = res.value; const msg = res.value;
await this.write(msg); await this.write(msg);
@@ -1512,13 +1562,14 @@ export class ContextEngine {
await this.write(msg); await this.write(msg);
} }
/** Yields and records a compaction stop event (carrying the result status; non-completed means compaction was abandoned). */ /** Yields and records a compaction stop event (carrying the result status — non-completed means compaction was abandoned — plus its share of the RetryDetail block: final attempt ordinal, and the last error detail on failures). */
private async *emitCompactionEnd( private async *emitCompactionEnd(
reason: CompactionReason, reason: CompactionReason,
mode: CompactionMode, mode: CompactionMode,
status: StopReason, status: StopReason,
detail?: { attempt?: number; errorMessage?: string },
): AsyncGenerator<OmniMessage> { ): AsyncGenerator<OmniMessage> {
const msg = compactionEnd({ reason, mode, status }); const msg = compactionEnd({ reason, mode, status, ...detail });
yield msg; yield msg;
await this.write(msg); await this.write(msg);
} }
@@ -103,7 +103,9 @@ export function createDescribeImageTool(
if (signal?.aborted) return { stopReason: "aborted" }; if (signal?.aborted) return { stopReason: "aborted" };
if (!outcome || outcome.status !== "completed") { if (!outcome || outcome.status !== "completed") {
const detail = const detail =
outcome && "message" in outcome && outcome.message ? `: ${outcome.message}` : ""; outcome && "errorMessage" in outcome && outcome.errorMessage
? `: ${outcome.errorMessage}`
: "";
yield delta( yield delta(
`${streamedAny ? "\n" : ""}Vision model (${describer.modelId}) request ${outcome?.status ?? "failed"}${detail}`, `${streamedAny ? "\n" : ""}Vision model (${describer.modelId}) request ${outcome?.status ?? "failed"}${detail}`,
); );
+6 -5
View File
@@ -148,7 +148,7 @@ export interface GenerativeModelParameters {
* `context_engine`; * `context_engine`;
* - `aborted`: user-initiated interruption — stop and hand back to the user; * - `aborted`: user-initiated interruption — stop and hand back to the user;
* - `failed`: an error the retry classifier did not judge transient (params, etc.) — still * - `failed`: an error the retry classifier did not judge transient (params, etc.) — still
* retried by `context_engine` within the same run (`message` provides the display text). * retried by `context_engine` within the same run (`errorMessage` provides the display text).
* The classification stays honest — this is reported as `failed`, not relabelled a * The classification stays honest — this is reported as `failed`, not relabelled a
* timeout — while the *policy* retries it, because that classifier is an allowlist and a * timeout — while the *policy* retries it, because that classifier is an allowlist and a
* gateway phrasing a transient fault its own way lands here; * gateway phrasing a transient fault its own way lands here;
@@ -162,12 +162,13 @@ export interface GenerativeModelParameters {
export interface LLMOutcome { export interface LLMOutcome {
status: StopReason; status: StopReason;
/** /**
* Failure detail (`describeError` text): present on `failed` / `auth`, and on `timeout` / * Error detail (`describeError` text): present on `failed` / `auth`, and on `timeout` /
* `malformed` when a concrete transport/provider error was caught (a plain idle timeout * `malformed` when a concrete transport/provider error was caught (a plain idle timeout
* has none). Carried onto the `request_end` event so observability (the Cost center's * has none). Carried onto the `request_end` event as `error_message` — one name across
* errors panel) can show the real reason behind a retried request. * the internal outcome and the wire — so observability (the Cost center's errors panel)
* can show the real reason behind a retried request.
*/ */
message?: string; errorMessage?: string;
} }
/** /**
+5 -5
View File
@@ -1107,7 +1107,7 @@ export class GenerativeModel implements LLMInterface {
try { try {
uniMessage = mergeOmniToUniMessage(params.newMessages); uniMessage = mergeOmniToUniMessage(params.newMessages);
} catch (err) { } catch (err) {
return { status: "failed", message: describeError(err) }; return { status: "failed", errorMessage: describeError(err) };
} }
const translator = new EventTranslator(this.toolCallIds); const translator = new EventTranslator(this.toolCallIds);
@@ -1214,7 +1214,7 @@ export class GenerativeModel implements LLMInterface {
// malformed to reconnect and retry — must not be classified as failed. // malformed to reconnect and retry — must not be classified as failed.
outcome = { outcome = {
status: "malformed", status: "malformed",
message: describeError(error), errorMessage: describeError(error),
}; };
} else if (isAuthenticationError(error)) { } else if (isAuthenticationError(error)) {
// Credentials failure: its own terminal status so hosts can tell "update this // Credentials failure: its own terminal status so hosts can tell "update this
@@ -1222,17 +1222,17 @@ export class GenerativeModel implements LLMInterface {
// fixed at Session creation; the credential is read from the current Project // fixed at Session creation; the credential is read from the current Project
// config on load) apart from a one-off failure. Checked before the retryable // config on load) apart from a one-off failure. Checked before the retryable
// branch as a belt — isRetryableError itself already refuses auth signals. // branch as a belt — isRetryableError itself already refuses auth signals.
outcome = { status: "auth", message: describeError(error) }; outcome = { status: "auth", errorMessage: describeError(error) };
} else if (isRetryableError(error)) { } else if (isRetryableError(error)) {
// Network drop / transient provider rejection -> needs reconnection. The detail // Network drop / transient provider rejection -> needs reconnection. The detail
// (e.g. "403 … (insufficient_user_quota)") rides on the outcome so observability // (e.g. "403 … (insufficient_user_quota)") rides on the outcome so observability
// (request_end -> the Cost center's errors panel) shows the real reason behind a // (request_end -> the Cost center's errors panel) shows the real reason behind a
// retried request, not just "timeout". // retried request, not just "timeout".
outcome = { status: "timeout", message: describeError(error) }; outcome = { status: "timeout", errorMessage: describeError(error) };
} else if ((error as { name?: string })?.name === "AbortError") { } else if ((error as { name?: string })?.name === "AbortError") {
outcome = { status: "aborted" }; // Fallback: an unexpected abort (neither timeout nor user) outcome = { status: "aborted" }; // Fallback: an unexpected abort (neither timeout nor user)
} else { } else {
outcome = { status: "failed", message: describeError(error) }; outcome = { status: "failed", errorMessage: describeError(error) };
} }
} finally { } finally {
clearTimer(); clearTimer();
+13 -8
View File
@@ -273,20 +273,21 @@ export function requestBegin(): OmniMessage<RequestBeginPayload> {
/** /**
* request end event: carries the terminal state (`completed` means this turn was already * request end event: carries the terminal state (`completed` means this turn was already
* committed to AgentHub), plus — on non-completed statuses — the failure detail (from * committed to AgentHub), plus the unified retry detail block (see RequestRetryDetail):
* LLMOutcome.message) and, when the engine will retry in-run, the planned backoff wait * the error detail (from LLMOutcome.errorMessage), the 1-based attempt ordinal, and — when the
* (`retry_in_ms`, rendered by the Web App as a live countdown). * engine will retry in-run — the planned backoff wait (`retry_in_ms`, rendered by the Web
* App as a live countdown). This builder is the one place the block is stamped.
*/ */
export function requestEnd( export function requestEnd(
status: StopReason, status: StopReason,
message?: string, retry: { errorMessage?: string; attempt?: number; retryInMs?: number } = {},
retryInMs?: number,
): OmniMessage<RequestEndPayload> { ): OmniMessage<RequestEndPayload> {
return event({ return event({
type: "request_end", type: "request_end",
status, status,
...(message !== undefined ? { message } : {}), ...(retry.errorMessage !== undefined ? { error_message: retry.errorMessage } : {}),
...(retryInMs !== undefined ? { retry_in_ms: retryInMs } : {}), ...(retry.attempt !== undefined ? { attempt: retry.attempt } : {}),
...(retry.retryInMs !== undefined ? { retry_in_ms: retry.retryInMs } : {}),
}); });
} }
@@ -306,17 +307,21 @@ export function compactionBegin(args: {
}); });
} }
/** compaction end event: carries the compaction result (non-`completed` means compaction was abandoned and the original context is kept). */ /** compaction end event: carries the compaction result (non-`completed` means compaction was abandoned and the original context is kept), plus its share of the RetryDetail block (final attempt ordinal; last error_message detail on failures). */
export function compactionEnd(args: { export function compactionEnd(args: {
reason: CompactionReason; reason: CompactionReason;
mode: CompactionMode; mode: CompactionMode;
status: StopReason; status: StopReason;
attempt?: number;
errorMessage?: string;
}): OmniMessage<CompactionEndPayload> { }): OmniMessage<CompactionEndPayload> {
return event({ return event({
type: "compaction_end", type: "compaction_end",
reason: args.reason, reason: args.reason,
mode: args.mode, mode: args.mode,
status: args.status, status: args.status,
...(args.attempt !== undefined ? { attempt: args.attempt } : {}),
...(args.errorMessage !== undefined ? { error_message: args.errorMessage } : {}),
}); });
} }
@@ -4,7 +4,7 @@
* host — but their spelling belongs here with the rest of the markers so the engine and the * host — but their spelling belongs here with the rest of the markers so the engine and the
* resume path cannot drift apart (they used to duplicate the `[context_summary]` template). * resume path cannot drift apart (they used to duplicate the `[context_summary]` template).
*/ */
import { dualFormPatterns, markerBlock, matchDualForm } from "./block.js"; import { dualFormPatterns, markerBlock, matchDualForm, stripMarkerBlocks } from "./block.js";
import { MARKER_TAGS, TRANSCRIPT_TAGS } from "./tags.js"; import { MARKER_TAGS, TRANSCRIPT_TAGS } from "./tags.js";
/** Wraps a compaction summary as the new context's first input: `[context_summary]…[/context_summary]`. */ /** Wraps a compaction summary as the new context's first input: `[context_summary]…[/context_summary]`. */
@@ -12,19 +12,33 @@ export function buildContextSummaryText(summary: string): string {
return markerBlock(MARKER_TAGS.contextSummary, summary); return markerBlock(MARKER_TAGS.contextSummary, summary);
} }
const SUMMARY_PATTERNS = dualFormPatterns(MARKER_TAGS.summary, "([\\s\\S]*?)"); const SUMMARY_PATTERNS = dualFormPatterns(MARKER_TAGS.summary, "([\\s\\S]*?)", "g");
/** /**
* Extracts the summary the model wrote inside `[summary]…[/summary]` during compaction; when * Extracts the summary the model wrote during compaction, with a tolerance ladder (issue #170:
* the tag is missing, leniently uses the entire output as-is (not treated as a failure). * some models — deepseek-v4-flash observed — treat the tags as a "title" and write the body
* outside them):
* 1. the first `[summary]…[/summary]` pair whose content is non-empty once stray summary
* tags are stripped (so a nested `[summary]` line doesn't hide the text after it);
* 2. otherwise, everything left after stripping the (empty) summary blocks and stray tags —
* rescues output shaped `[summary]\n[/summary]\n<the actual summary>`;
* 3. tagless output stays used verbatim. An empty return means the output was genuinely
* empty — the caller treats that as a failed attempt.
* Step 1 preserves the historical result for every healthy output, so resume re-extracting
* from an old Trace reconstructs the same summary the live session used.
* *
* The legacy `<summary></summary>` form must stay accepted **indefinitely**: * The legacy `<summary></summary>` form must stay accepted **indefinitely**:
* `compaction.prompt` is persisted in every existing agent's `system_config.yaml`, so old * `compaction.prompt` is persisted in every existing agent's `system_config.yaml`, so old
* agents keep instructing the model to use the angle-bracket tags. * agents keep instructing the model to use the angle-bracket tags.
*/ */
export function extractSummary(raw: string): string { export function extractSummary(raw: string): string {
const match = matchDualForm(SUMMARY_PATTERNS, raw); for (const pattern of SUMMARY_PATTERNS) {
return (match ? match[1]! : raw).trim(); for (const match of raw.matchAll(pattern)) {
const inside = stripMarkerBlocks(match[1]!, MARKER_TAGS.summary).trim();
if (inside !== "") return inside;
}
}
return stripMarkerBlocks(raw, MARKER_TAGS.summary).trim();
} }
/** One transcribed line inside a turn block: an inner tag, optional attributes, and the body. */ /** One transcribed line inside a turn block: an inner tag, optional attributes, and the body. */
+37 -12
View File
@@ -280,29 +280,47 @@ export interface RequestBeginPayload {
type: "request_begin"; type: "request_begin";
} }
export interface RequestEndPayload { /**
type: "request_end"; * The unified retry/failure detail block shared by `request_end` and `compaction_end` —
/** Terminal state of this Request (reuses the six StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome; `auth` is the credentials-failure signal hosts key on). */ * one standard group of fields, stamped by the builders (the way `withOrigin` stamps
status: StopReason; * `origin`) rather than accreting as scattered ad-hoc parameters. Every field is optional
* and additive: old Traces replay unchanged.
*/
export interface RetryDetail {
/** /**
* Failure detail from `LLMOutcome.message`, present only on non-completed statuses: the * Error detail, one name across the stack (`LLMOutcome.errorMessage` internally): present
* real reason behind a retried/failed Request (e.g. `403 … (insufficient_user_quota)`), * only on non-completed statuses — the real reason behind a retried/failed Request (e.g.
* for observability — the server's error records / Cost center read it here because a * `403 … (insufficient_user_quota)`), for observability — the server's error records /
* retried request never produces an abort event. Additive: old Traces without it replay * Cost center read it here because a retried request never produces an abort event. Not
* unchanged. * plain `message`: in this protocol "message" means an OmniMessage / model output, and
* this field is neither. (Traces written before the rename carry it as `message`; nothing
* reads that field semantically after the fact, so no dual-read is kept.)
*/ */
message?: string; error_message?: string;
/**
* 1-based ordinal of this Request within its retry run — the authoritative retry count
* the CLI/Web display verbatim. Stamped on every non-completed request_end and on a
* completed one that needed retries; absent on a clean first-try completion (the common
* case stays noise-free) and in old Traces.
*/
attempt?: number;
/** /**
* Planned in-run retry wait (ms) — present ONLY when the engine will retry this failure * Planned in-run retry wait (ms) — present ONLY when the engine will retry this failure
* within the same run (status `timeout`/`malformed` with attempts remaining under the * within the same run (status `timeout`/`malformed` with attempts remaining under the
* applicable cap). Computed by the same formula as the actual backoff sleep * applicable cap). Computed by the same formula as the actual backoff sleep
* (`reconnectDelayMs`), so the announced wait and the real one cannot drift; the Web App * (`reconnectDelayMs`), so the announced wait and the real one cannot drift; the Web App
* renders it as a live countdown to the next attempt. Absent on final failures (an abort * renders it as a live countdown to the next attempt. Absent on final failures (an abort
* follows instead) and on completed requests. Additive: old Traces replay unchanged. * follows instead) and on completed requests.
*/ */
retry_in_ms?: number; retry_in_ms?: number;
} }
export interface RequestEndPayload extends RetryDetail {
type: "request_end";
/** Terminal state of this Request (reuses the six StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome; `auth` is the credentials-failure signal hosts key on). */
status: StopReason;
}
/** Compaction trigger reason: context threshold / turn-count threshold / user-initiated request. */ /** Compaction trigger reason: context threshold / turn-count threshold / user-initiated request. */
export type CompactionReason = "context" | "turns" | "manual"; export type CompactionReason = "context" | "turns" | "manual";
@@ -328,7 +346,14 @@ export interface CompactionBeginPayload {
turns: number; turns: number;
} }
export interface CompactionEndPayload { /**
* Inherits the shared RetryDetail block: `attempt` is the final attempt's 1-based ordinal
* (failed attempts and retries included — stamped by summarize mode), `error_message` is
* the last failure's detail (present only when `status` is `failed`), and `retry_in_ms` is
* never stamped here (compaction retries are announced on the compaction request's own
* request_end, which is Trace-only).
*/
export interface CompactionEndPayload extends RetryDetail {
type: "compaction_end"; type: "compaction_end";
reason: CompactionReason; reason: CompactionReason;
mode: CompactionMode; mode: CompactionMode;
+12 -10
View File
@@ -156,18 +156,20 @@ Skills are reusable instruction packages at <app_data_dir>/agents/<agent_id>/age
- Session ID: {{SESSION_ID}}`; - Session ID: {{SESSION_ID}}`;
/** /**
* Built-in default compaction Prompt (summarize mode): tells the model that after * Built-in default compaction Prompt (summarize mode): tells the model the summary will
* compaction the raw transcript is no longer visible and the * replace the transcript as its only record (so it must include everything needed to
* summary is the only record, so it must include everything needed to continue the task, * continue the task) and that no tools may be called. The format is shown as a concrete
* and no tools may be called while writing the summary. * example rather than described in prose — some models treat the tags as a "title" and
* write the body after the closing tag (issue #170); extraction salvages that shape, but
* generating it right beats repairing it. Persisted per-agent in system_config.yaml —
* existing agents keep their stored prompt.
*/ */
export const DEFAULT_COMPACTION_PROMPT = export const DEFAULT_COMPACTION_PROMPT =
"You have a partial transcript of the task above. Write a summary of it wrapped in " + "Summarize the task transcript above. The summary will replace the transcript as its " +
"`[summary][/summary]` tags. This summary will replace the transcript: in the next " + "only record, so include everything needed to continue the task — the original request, " +
"context window the raw transcript above will no longer be visible and this summary " + "current state, next steps, and any learnings. Do not call any tools; reply with text " +
"will be its only record, so include everything needed to continue the task — the " + "only, in exactly this format and nothing after it:\n\n" +
"original request, current state, next steps, and any learnings. Do not call any " + "[summary]put the summary text here...[/summary]";
"tools while writing the summary; respond with text only.";
/** /**
* Default built-in system tools: file reading/editing/writing first, then bash execution * Default built-in system tools: file reading/editing/writing first, then bash execution
+170 -58
View File
@@ -49,7 +49,7 @@ import type {
LLMInterface, LLMInterface,
LLMOutcome, LLMOutcome,
} from "../src/interfaces.js"; } from "../src/interfaces.js";
import { ContextEngine } from "../src/engine/context-engine.js"; import { ContextEngine, SUMMARY_RETRY_GUIDANCE } from "../src/engine/context-engine.js";
import type { CompactionSettings } from "../src/engine/context-engine.js"; import type { CompactionSettings } from "../src/engine/context-engine.js";
import { GenerativeModel } from "../src/llm/index.js"; import { GenerativeModel } from "../src/llm/index.js";
import type { UniConfig, UniEvent, UniMessage } from "@prismshadow/agenthub"; import type { UniConfig, UniEvent, UniMessage } from "@prismshadow/agenthub";
@@ -78,7 +78,7 @@ class ScriptedLLM implements LLMInterface {
this.calls.push(params.newMessages); this.calls.push(params.newMessages);
const next = this.responses.shift(); const next = this.responses.shift();
if (!next) { if (!next) {
return { status: "failed", message: `${this.label}: no scripted response` }; return { status: "failed", errorMessage: `${this.label}: no scripted response` };
} }
for (const msg of next.messages) yield msg; for (const msg of next.messages) yield msg;
return next.outcome ?? { status: "completed" }; return next.outcome ?? { status: "completed" };
@@ -308,7 +308,7 @@ describe("context compaction", () => {
messages: [toolCall({ name: "t", arguments: "{}", toolCallId: "c1" }), usage(150, 150)], messages: [toolCall({ name: "t", arguments: "{}", toolCallId: "c1" }), usage(150, 150)],
}, },
// Compaction request fails on the one status no ladder can fix (a rejected credential). // Compaction request fails on the one status no ladder can fix (a rejected credential).
{ messages: [], outcome: { status: "auth", message: "auth error" } }, { messages: [], outcome: { status: "auth", errorMessage: "auth error" } },
// Original context is kept: the task continues, tool outputs feed back into the old instance as usual (context usage keeps growing). // Original context is kept: the task continues, tool outputs feed back into the old instance as usual (context usage keeps growing).
{ messages: [assistantText("finished on old context"), usage(190, 340)] }, { messages: [assistantText("finished on old context"), usage(190, 340)] },
// Second trigger (context still over the limit) -> retries compaction at the boundary, this time succeeding. // Second trigger (context still over the limit) -> retries compaction at the boundary, this time succeeding.
@@ -415,12 +415,55 @@ describe("context compaction", () => {
const out = await collect(engine.run([userText("go")], { approve: allowAll })); const out = await collect(engine.run([userText("go")], { approve: allowAll }));
const events = compactionEvents(out); const events = compactionEvents(out);
expect(events[1]).toMatchObject({ type: "compaction_end", status: "failed" }); // The end event reports the attempts spent (issue #170's cost-center row message).
expect(events[1]).toMatchObject({
type: "compaction_end",
status: "failed",
attempt: 2,
});
// The retry resends the original input (tool results + prompt; here there are no tool results, just the prompt). // The retry resends the original input (tool results + prompt; here there are no tool results, just the prompt).
expect(llm1.calls).toHaveLength(3); expect(llm1.calls).toHaveLength(3);
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]); expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
}); });
it("compaction transport retries default to the shared maxReconnects budget", async () => {
// Issue #170: without an explicit compactionMaxReconnects, the compaction request gets
// the same retry budget as the turn loop (here 4) — under the old tighter default of 3
// this script would have failed before reaching the 5th, succeeding attempt.
const failing = (n: number): ScriptedResponse => ({
messages: [],
outcome: { status: "failed", errorMessage: `blip ${n}` },
});
const llm1 = new ScriptedLLM(
[
{ messages: [assistantText("answer"), usage(150, 150)] },
failing(1),
failing(2),
failing(3),
failing(4),
{ messages: [assistantText("[summary]after the shared ladder[/summary]"), usage(30, 300)] },
],
"llm1",
);
const llm2 = new ScriptedLLM([], "llm2");
const engine = new ContextEngine({
llm: llm1,
environment: fakeEnvironment,
compaction: settings(),
createLLM: () => llm2,
maxReconnects: 4,
reconnectBackoffMs: 1,
});
const out = await collect(engine.run([userText("go")], { approve: allowAll }));
expect(compactionEvents(out)[1]).toMatchObject({
type: "compaction_end",
status: "completed",
attempt: 5,
});
expect(llm1.calls).toHaveLength(6);
});
it("a failed compaction request takes the ladder and can recover on a later attempt", async () => { it("a failed compaction request takes the ladder and can recover on a later attempt", async () => {
// The compaction loop retries the same statuses the turn loop does: `failed` is where a // The compaction loop retries the same statuses the turn loop does: `failed` is where a
// transient fault lands whenever the classifier doesn't recognize the gateway's wording, // transient fault lands whenever the classifier doesn't recognize the gateway's wording,
@@ -429,7 +472,7 @@ describe("context compaction", () => {
const llm1 = new ScriptedLLM( const llm1 = new ScriptedLLM(
[ [
{ messages: [assistantText("answer"), usage(150, 150)] }, { messages: [assistantText("answer"), usage(150, 150)] },
{ messages: [], outcome: { status: "failed", message: "502 upstream" } }, { messages: [], outcome: { status: "failed", errorMessage: "502 upstream" } },
{ messages: [assistantText("[summary]recovered[/summary]")] }, { messages: [assistantText("[summary]recovered[/summary]")] },
], ],
"llm1", "llm1",
@@ -455,11 +498,11 @@ describe("context compaction", () => {
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]); expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
}); });
it("the compaction loop uses its own cap, not the shared maxReconnects ladder", async () => { it("an explicit compactionMaxReconnects overrides the shared maxReconnects budget", async () => {
// Compaction failure has graceful semantics (the original context is kept, compaction // The dep still allows a caller to bound compaction retries independently: with
// retries on the next trigger), so it fails fast: with compactionMaxReconnects=2, the // compactionMaxReconnects=2, the compaction request runs 1+2 attempts and gives up —
// compaction request runs 1+2 attempts and gives up — even though maxReconnects is far // even though maxReconnects is far larger (the default merely *follows* maxReconnects,
// larger and would have kept the session stalled through the full exponential ladder. // it does not override an explicit value).
const llm1 = new ScriptedLLM( const llm1 = new ScriptedLLM(
[ [
{ messages: [assistantText("answer"), usage(150, 150)] }, { messages: [assistantText("answer"), usage(150, 150)] },
@@ -495,26 +538,31 @@ describe("context compaction", () => {
// The Trace-written compaction request_ends announce the planned backoff under the // The Trace-written compaction request_ends announce the planned backoff under the
// COMPACTION cap (base 1ms: 1, then 2), with none on the final failure — these events // COMPACTION cap (base 1ms: 1, then 2), with none on the final failure — these events
// are never streamed, so the value lands in the Trace record only. // are never streamed, so the value lands in the Trace record only.
const retryPlans = recorded const compactionEnds = recorded.filter(
.filter((m) => (m.payload as { type?: string }).type === "request_end") (m) => (m.payload as { type?: string }).type === "request_end",
.map((m) => (m.payload as { retry_in_ms?: number }).retry_in_ms); );
const retryPlans = compactionEnds.map(
(m) => (m.payload as { retry_in_ms?: number }).retry_in_ms,
);
expect(retryPlans).toEqual([undefined, 1, 2, undefined]); expect(retryPlans).toEqual([undefined, 1, 2, undefined]);
// The attempt ordinal rides the same events: the turn's clean completion stays
// unstamped, the three compaction failures count 1..3.
const attempts = compactionEnds.map((m) => (m.payload as { attempt?: number }).attempt);
expect(attempts).toEqual([undefined, 1, 2, 3]);
}); });
it("an empty compaction response (thinking only, no text) is rejected: 5 attempts, then failed with the context kept", async () => { it("an empty compaction response (thinking only, no text) is retried like any failure: 5 attempts, then failed with the context kept", async () => {
// Issue #83: the compaction request completes but yields no text. Committing the empty // Issue #83/#170: the compaction request completes but yields no text. Committing the
// summary would discard the whole context and lose the task state — the response is // empty summary would discard the whole context and lose the task state — the response
// rejected and retried under the dedicated rejection cap (5 attempts, #84), then the // counts as one more failed attempt on the unified reconnect budget, and once the budget
// compaction fails while the original context and Trace file stay current. // is exhausted the compaction fails while the original context and Trace file stay current.
const empty = (n: number): ScriptedResponse => ({ const empty = (n: number): ScriptedResponse => ({
messages: [thinkingMessage(`pondering, attempt ${n}, no text`)], messages: [thinkingMessage(`pondering, attempt ${n}, no text`)],
}); });
const llm1 = new ScriptedLLM( const llm1 = new ScriptedLLM(
[ [
{ messages: [assistantText("answer one"), usage(150, 150)] }, { messages: [assistantText("answer one"), usage(150, 150)] },
// Five completed-but-empty compaction attempts: the dedicated cap allows exactly 5 // Five completed-but-empty compaction attempt: 4 retries of budget allow exactly 5.
// rejections. compactionMaxReconnects is 1 here on purpose — rejections must NOT
// consume the transport reconnect budget, or the loop would stop after 2 attempts.
empty(1), empty(1),
empty(2), empty(2),
empty(3), empty(3),
@@ -538,7 +586,7 @@ describe("context compaction", () => {
created += 1; created += 1;
return new ScriptedLLM([], "llm2"); return new ScriptedLLM([], "llm2");
}, },
compactionMaxReconnects: 1, compactionMaxReconnects: 4,
reconnectBackoffMs: 1, reconnectBackoffMs: 1,
}); });
const oldPath = trace.currentPath(); const oldPath = trace.currentPath();
@@ -546,12 +594,17 @@ describe("context compaction", () => {
const out1 = await collect(engine.run([userText("task one")], { approve: allowAll })); const out1 = await collect(engine.run([userText("task one")], { approve: allowAll }));
// Exactly one event pair, ending failed — an empty summary is never a completed compaction. // Exactly one event pair, ending failed — an empty summary is never a completed compaction.
// The end event reports the attempts spent (issue #170).
const events = compactionEvents(out1); const events = compactionEvents(out1);
expect( expect(
events.map((e) => `${e.type}:${(e as Partial<CompactionEndPayload>).status ?? ""}`), events.map((e) => `${e.type}:${(e as Partial<CompactionEndPayload>).status ?? ""}`),
).toEqual(["compaction_begin:", "compaction_end:failed"]); ).toEqual(["compaction_begin:", "compaction_end:failed"]);
// No rejected attempt's token_usage is surfaced (only a successful compaction yields expect(events[1]).toMatchObject({
// its usage between the paired events). attempt: 5,
error_message: "the response contained no usable summary",
});
// These scripted attempts carry no token_usage of their own, so none appears between the
// paired events (attempts that do carry usage surface it — see the 5th-attempt test).
const types1 = payloadTypes(out1); const types1 = payloadTypes(out1);
const between = out1.slice( const between = out1.slice(
types1.indexOf("compaction_begin") + 1, types1.indexOf("compaction_begin") + 1,
@@ -560,14 +613,16 @@ describe("context compaction", () => {
expect(between.filter((m) => (m.payload as { type?: string }).type === "token_usage")).toEqual( expect(between.filter((m) => (m.payload as { type?: string }).type === "token_usage")).toEqual(
[], [],
); );
// Turn + exactly five compaction attempts (the 5th rejection exhausts the cap, no 6th // Turn + exactly five compaction attempts (the 5th exhausts the budget, no 6th
// request), each resending the prompt unchanged — an empty rejection needs no repair. // request); each retry leads with the corrective note, then the prompt — an empty
// response needs no repair.
expect(llm1.calls).toHaveLength(6); expect(llm1.calls).toHaveLength(6);
for (let i = 1; i <= 5; i += 1) { expect(llm1.calls[1]!.map(textOf)).toEqual(["COMPACT NOW"]);
expect(llm1.calls[i]!.map(textOf)).toEqual(["COMPACT NOW"]); for (let i = 2; i <= 5; i += 1) {
expect(llm1.calls[i]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
} }
// Rejection resends are immediate, never announced: no compaction request_end carries a // An unusable attempt's request_end carries status `completed`, for which no retry_in_ms
// retry_in_ms (they all end `completed`, unlike the transport ladder's timeout ends). // is ever announced (the backoff wait still ran between attempts).
const rejectionPlans = (await readTrace(oldPath)) const rejectionPlans = (await readTrace(oldPath))
.filter((m) => (m.payload as { type?: string }).type === "request_end") .filter((m) => (m.payload as { type?: string }).type === "request_end")
.map((m) => (m.payload as { retry_in_ms?: number }).retry_in_ms); .map((m) => (m.payload as { retry_in_ms?: number }).retry_in_ms);
@@ -585,9 +640,9 @@ describe("context compaction", () => {
expect(await readdir(dirname(oldPath))).toEqual(["sess_empty_001.jsonl"]); expect(await readdir(dirname(oldPath))).toEqual(["sess_empty_001.jsonl"]);
}); });
it("tool-calling rejections exhaust the 5-attempt cap: every call is paired, and the next ordinary turn stays clean", async () => { it("tool-calling responses exhaust the retry budget: every call is paired, and the next ordinary turn stays clean", async () => {
// A tool-calling response is not a summary — even when it also carries plausible summary // A tool-calling response is not a summary — even when it also carries plausible summary
// text — but its assistant turn IS committed on the live LLM object. Each rejection's // text — but its assistant turn IS committed on the live LLM object. Each such attempt's
// calls are answered with synthesized failed outputs (written to Trace, prepended to the // calls are answered with synthesized failed outputs (written to Trace, prepended to the
// retried input), so no tool_use ever dangles: after the compaction fails, the same // retried input), so no tool_use ever dangles: after the compaction fails, the same
// object must still serve ordinary turns with a well-formed history (#84 review). // object must still serve ordinary turns with a well-formed history (#84 review).
@@ -621,7 +676,7 @@ describe("context compaction", () => {
created += 1; created += 1;
return new ScriptedLLM([], "llm2"); return new ScriptedLLM([], "llm2");
}, },
compactionMaxReconnects: 1, compactionMaxReconnects: 4,
reconnectBackoffMs: 1, reconnectBackoffMs: 1,
}); });
const oldPath = trace.currentPath(); const oldPath = trace.currentPath();
@@ -635,12 +690,18 @@ describe("context compaction", () => {
expect(payloadTypes(out)).not.toContain("tool_call_output"); expect(payloadTypes(out)).not.toContain("tool_call_output");
expect(created).toBe(0); expect(created).toBe(0);
// The end event reports the attempts spent and the last failure (issue #170's
// cost-center row message and the frontend banner detail).
expect(events[1]).toMatchObject({
attempt: 5,
error_message: "the response called tools instead of writing a summary",
});
// Five attempts; from the second on, the input leads with the repair answering the // Five attempts; from the second on, the input leads with the repair answering the
// previous rejection's call, then re-issues the prompt. // previous attempt's call, then the corrective note, then the prompt.
expect(llm1.calls).toHaveLength(6); expect(llm1.calls).toHaveLength(6);
for (let attempt = 2; attempt <= 5; attempt += 1) { for (let attempt = 2; attempt <= 5; attempt += 1) {
const retry = llm1.calls[attempt]!; const retry = llm1.calls[attempt]!;
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text"]); expect(payloadTypes(retry)).toEqual(["tool_call_output", "text", "text"]);
const repair = retry[0]!.payload as { const repair = retry[0]!.payload as {
tool_call_id: string; tool_call_id: string;
output: string; output: string;
@@ -651,7 +712,8 @@ describe("context compaction", () => {
"[tool error] the compaction request expects a summary, not tool calls", "[tool error] the compaction request expects a summary, not tool calls",
); );
expect(repair.stop_reason).toBe("failed"); expect(repair.stop_reason).toBe("failed");
expect(textOf(retry[1]!)).toBe("COMPACT NOW"); expect(textOf(retry[1]!)).toBe(SUMMARY_RETRY_GUIDANCE);
expect(textOf(retry[2]!)).toBe("COMPACT NOW");
} }
// All five synthesized repairs are written to the (old) Trace for replay to mirror. // All five synthesized repairs are written to the (old) Trace for replay to mirror.
@@ -681,8 +743,9 @@ describe("context compaction", () => {
}); });
it("a valid summary on the 5th and final allowed attempt completes the compaction", async () => { it("a valid summary on the 5th and final allowed attempt completes the compaction", async () => {
// Counting pin for the rejection cap: four rejected attempts spend the budget but the 5th // Counting pin for the unified budget: four failed attempts spend the 4 retries, but the
// attempt still gets its chance — a valid summary there succeeds (5 rejections would fail). // 5th (last allowed) attempt still gets its chance — a valid summary there succeeds
// (a 5th failure would exhaust the budget instead).
const llm1 = new ScriptedLLM( const llm1 = new ScriptedLLM(
[ [
{ messages: [assistantText("answer"), usage(150, 150)] }, { messages: [assistantText("answer"), usage(150, 150)] },
@@ -706,7 +769,7 @@ describe("context compaction", () => {
factoryTokens = tokens; factoryTokens = tokens;
return llm2; return llm2;
}, },
maxReconnects: 1, compactionMaxReconnects: 4,
reconnectBackoffMs: 1, reconnectBackoffMs: 1,
}); });
@@ -714,8 +777,10 @@ describe("context compaction", () => {
const events = compactionEvents(out); const events = compactionEvents(out);
expect(events).toHaveLength(2); expect(events).toHaveLength(2);
expect(events[1]).toMatchObject({ type: "compaction_end", status: "completed" }); expect(events[1]).toMatchObject({ type: "compaction_end", status: "completed" });
// Only the adopted attempt's token_usage is surfaced between the paired events; rejected // Every attempt's token_usage is surfaced between the paired events — rejected attempts
// attempts' usage still feeds the Session cumulative totals (see below) but is not shown. // burn real tokens, and hiding them understated the cost center (issue #170). Here the
// 1st (rejected) and 5th (adopted) attempts carry usage.
expect(events[1]).toMatchObject({ attempt: 5 });
const types = payloadTypes(out); const types = payloadTypes(out);
const between = out.slice( const between = out.slice(
types.indexOf("compaction_begin") + 1, types.indexOf("compaction_begin") + 1,
@@ -724,8 +789,9 @@ describe("context compaction", () => {
const usageBetween = between.filter( const usageBetween = between.filter(
(m) => (m.payload as { type?: string }).type === "token_usage", (m) => (m.payload as { type?: string }).type === "token_usage",
); );
expect(usageBetween).toHaveLength(1); expect(usageBetween.map((m) => (m.payload as TokenUsagePayload).request.total)).toEqual([
expect((usageBetween[0]!.payload as TokenUsagePayload).request.total).toBe(170); 160, 170,
]);
// Session cumulative tokens carried into the new instance include the rejected attempts' usage. // Session cumulative tokens carried into the new instance include the rejected attempts' usage.
expect(factoryTokens).toMatchObject({ total: 480 }); expect(factoryTokens).toMatchObject({ total: 480 });
@@ -778,10 +844,11 @@ describe("context compaction", () => {
status: "completed", status: "completed",
}); });
// The retried input answers the rejected attempt's call first, then re-issues the prompt. // The retried input answers the rejected attempt's call first, then carries the
// corrective note and re-issues the prompt.
expect(llm1.calls).toHaveLength(3); expect(llm1.calls).toHaveLength(3);
const retry = llm1.calls[2]!; const retry = llm1.calls[2]!;
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text"]); expect(payloadTypes(retry)).toEqual(["tool_call_output", "text", "text"]);
const repair = retry[0]!.payload as { const repair = retry[0]!.payload as {
tool_call_id: string; tool_call_id: string;
output: string; output: string;
@@ -792,7 +859,8 @@ describe("context compaction", () => {
"[tool error] the compaction request expects a summary, not tool calls", "[tool error] the compaction request expects a summary, not tool calls",
); );
expect(repair.stop_reason).toBe("failed"); expect(repair.stop_reason).toBe("failed");
expect(textOf(retry[1]!)).toBe("COMPACT NOW"); expect(textOf(retry[1]!)).toBe(SUMMARY_RETRY_GUIDANCE);
expect(textOf(retry[2]!)).toBe("COMPACT NOW");
// The repair belongs to the compaction dialogue: written to the old Trace (so replay // The repair belongs to the compaction dialogue: written to the old Trace (so replay
// mirrors the pairing), never pushed to the output stream. // mirrors the pairing), never pushed to the output stream.
@@ -897,6 +965,7 @@ describe("context compaction", () => {
created += 1; created += 1;
return new ScriptedLLM([], "llm2"); return new ScriptedLLM([], "llm2");
}, },
compactionMaxReconnects: 4,
reconnectBackoffMs: 1, reconnectBackoffMs: 1,
}); });
const oldPath = trace.currentPath(); const oldPath = trace.currentPath();
@@ -908,15 +977,16 @@ describe("context compaction", () => {
// is the whole story. // is the whole story.
expect(payloadTypes(out1)).not.toContain("abort"); expect(payloadTypes(out1)).not.toContain("abort");
expect(llm1.calls).toHaveLength(6); expect(llm1.calls).toHaveLength(6);
// Attempt 1 folds the turn's tool output in; attempt 2 carries the repair + Prompt but // Attempt 1 folds the turn's tool output in; attempt 2 carries the repair + note + Prompt
// NOT the absorbed output; attempts 3-5 are Prompt-only. // but NOT the absorbed output; attempts 3-5 are note + Prompt.
expect(payloadTypes(llm1.calls[1]!)).toEqual(["tool_call_output", "text"]); expect(payloadTypes(llm1.calls[1]!)).toEqual(["tool_call_output", "text"]);
expect((llm1.calls[1]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("ct"); expect((llm1.calls[1]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("ct");
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text"]); expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text", "text"]);
expect((llm1.calls[2]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("c1"); expect((llm1.calls[2]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("c1");
expect(textOf(llm1.calls[2]![1]!)).toBe("COMPACT NOW"); expect(textOf(llm1.calls[2]![1]!)).toBe(SUMMARY_RETRY_GUIDANCE);
expect(textOf(llm1.calls[2]![2]!)).toBe("COMPACT NOW");
for (let attempt = 3; attempt <= 5; attempt += 1) { for (let attempt = 3; attempt <= 5; attempt += 1) {
expect(llm1.calls[attempt]!.map(textOf)).toEqual(["COMPACT NOW"]); expect(llm1.calls[attempt]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
} }
// Original context kept: no LLM swap, no Trace rotation. // Original context kept: no LLM swap, no Trace rotation.
expect(created).toBe(0); expect(created).toBe(0);
@@ -955,16 +1025,17 @@ describe("context compaction", () => {
environment: fakeEnvironment, environment: fakeEnvironment,
compaction: settings(), compaction: settings(),
createLLM: () => new ScriptedLLM([], "llm2"), createLLM: () => new ScriptedLLM([], "llm2"),
compactionMaxReconnects: 4,
reconnectBackoffMs: 1, reconnectBackoffMs: 1,
}); });
const out = await collect(engine.run([userText("go")], { approve: allowAll })); const out = await collect(engine.run([userText("go")], { approve: allowAll }));
expect(compactionEvents(out)[1]).toMatchObject({ type: "compaction_end", status: "failed" }); expect(compactionEvents(out)[1]).toMatchObject({ type: "compaction_end", status: "failed" });
expect(llm1.calls).toHaveLength(7); expect(llm1.calls).toHaveLength(7);
// Attempt 1 folds the outputs; attempts 2-5 are Prompt-only (absorbed by the first commit). // Attempt 1 folds the outputs; attempts 2-5 are note + Prompt (absorbed by the first commit).
expect(payloadTypes(llm1.calls[1]!)).toEqual(["tool_call_output", "text"]); expect(payloadTypes(llm1.calls[1]!)).toEqual(["tool_call_output", "text"]);
for (let attempt = 2; attempt <= 5; attempt += 1) { for (let attempt = 2; attempt <= 5; attempt += 1) {
expect(llm1.calls[attempt]!.map(textOf)).toEqual(["COMPACT NOW"]); expect(llm1.calls[attempt]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
} }
// The continuation input is the repair answering c5 — nothing else: no absorbed outputs, // The continuation input is the repair answering c5 — nothing else: no absorbed outputs,
// no prompt. // no prompt.
@@ -1044,8 +1115,8 @@ describe("context compaction", () => {
expect(events[1]).toMatchObject({ type: "compaction_end", status: "aborted" }); expect(events[1]).toMatchObject({ type: "compaction_end", status: "aborted" });
expect(payloadTypes(out)).toContain("abort"); expect(payloadTypes(out)).toContain("abort");
expect(llm1.calls).toHaveLength(3); expect(llm1.calls).toHaveLength(3);
// Attempt 2 carried the repair + Prompt (not the absorbed outputs). // Attempt 2 carried the repair + note + Prompt (not the absorbed outputs).
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text"]); expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text", "text"]);
expect((llm1.calls[2]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("c1"); expect((llm1.calls[2]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("c1");
// The next run leads with the still-unanswered repair; the absorbed turn outputs are // The next run leads with the still-unanswered repair; the absorbed turn outputs are
@@ -1255,6 +1326,46 @@ describe("context compaction", () => {
); );
}); });
it("rescues a summary written outside the tags (issue #170) on the first attempt", async () => {
// deepseek-v4-flash writes `[summary]\n[/summary]` as a "title" and the body after the
// closing tag. Extraction salvages the outside text, so the very first attempt succeeds —
// no rejection loop, no burned retries.
const llm1 = new ScriptedLLM(
[
{ messages: [assistantText("answer"), usage(150, 150)] },
{
messages: [
assistantText("[summary]\n[/summary]\nRescued body outside the tags."),
usage(40, 200),
],
},
],
"llm1",
);
const llm2 = new ScriptedLLM([{ messages: [assistantText("fresh"), usage(10, 210)] }], "llm2");
const engine = new ContextEngine({
llm: llm1,
environment: fakeEnvironment,
compaction: settings(),
createLLM: () => llm2,
reconnectBackoffMs: 1,
});
const out = await collect(engine.run([userText("task one")], { approve: allowAll }));
expect(compactionEvents(out)[1]).toMatchObject({
type: "compaction_end",
status: "completed",
attempt: 1,
});
// No rejection retry: the turn request plus a single compaction attempt.
expect(llm1.calls).toHaveLength(2);
await collect(engine.run([userText("task two")], { approve: allowAll }));
expect(llm2.calls[0]!.map(textOf)).toEqual([
"[context_summary]\nRescued body outside the tags.\n[/context_summary]",
"task two",
]);
});
it("manual compaction skips threshold checks and reuses the same flow", async () => { it("manual compaction skips threshold checks and reuses the same flow", async () => {
const llm1 = new ScriptedLLM( const llm1 = new ScriptedLLM(
[ [
@@ -1356,6 +1467,7 @@ describe("context compaction", () => {
environment: fakeEnvironment, environment: fakeEnvironment,
compaction: settings(), compaction: settings(),
createLLM: () => new ScriptedLLM([], "llm2"), createLLM: () => new ScriptedLLM([], "llm2"),
compactionMaxReconnects: 4,
reconnectBackoffMs: 1, reconnectBackoffMs: 1,
}); });
await collect(engine.run([userText("start")], { approve: allowAll })); await collect(engine.run([userText("start")], { approve: allowAll }));
@@ -1368,11 +1480,11 @@ describe("context compaction", () => {
const out = await collect(engine.compact()); const out = await collect(engine.compact());
expect(compactionEvents(out)[1]).toMatchObject({ type: "compaction_end", status: "failed" }); expect(compactionEvents(out)[1]).toMatchObject({ type: "compaction_end", status: "failed" });
expect(llm1.calls).toHaveLength(6); expect(llm1.calls).toHaveLength(6);
// Attempt 1 folded the carry-over in (and committed it); later attempts resend the Prompt // Attempt 1 folded the carry-over in (and committed it); later attempts resend the note +
// alone — the absorbed carry-over never again. // Prompt — the absorbed carry-over never again.
expect(llm1.calls[1]!.map(textOf)).toEqual(["pending question", "COMPACT NOW"]); expect(llm1.calls[1]!.map(textOf)).toEqual(["pending question", "COMPACT NOW"]);
for (let attempt = 2; attempt <= 5; attempt += 1) { for (let attempt = 2; attempt <= 5; attempt += 1) {
expect(llm1.calls[attempt]!.map(textOf)).toEqual(["COMPACT NOW"]); expect(llm1.calls[attempt]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
} }
// The carry-over is consumed at the seam — the next run leads with the stashed repair // The carry-over is consumed at the seam — the next run leads with the stashed repair
+1 -1
View File
@@ -177,7 +177,7 @@ describe("describe_image (the text-only-model variant of read_image)", () => {
it("vision model request failure: failed with the status and message", async () => { it("vision model request failure: failed with the status and message", async () => {
await writeFile(path.join(tmp, "a.png"), PNG_1X1); await writeFile(path.join(tmp, "a.png"), PNG_1X1);
const { llm } = fakeLLM("", { status: "failed", message: "401 unauthorized" }); const { llm } = fakeLLM("", { status: "failed", errorMessage: "401 unauthorized" });
const { result, text } = await run({ source: "a.png" }, tmp, { const { result, text } = await run({ source: "a.png" }, tmp, {
modelId: "vis-1", modelId: "vis-1",
createLLM: () => llm, createLLM: () => llm,
+21 -11
View File
@@ -684,7 +684,7 @@ describe("ContextEngine ReAct loop (mock LLM, approve callback)", () => {
yield partialText("start", ""); yield partialText("start", "");
yield partialText("delta", "half a thought"); yield partialText("delta", "half a thought");
// `auth`: the one LLM status that still exits straight to the flatten path. // `auth`: the one LLM status that still exits straight to the flatten path.
return { status: "auth", message: "boom" }; return { status: "auth", errorMessage: "boom" };
} }
yield assistantText("ok"); yield assistantText("ok");
yield tokenUsage(emptyTokenCounts(), { yield tokenUsage(emptyTokenCounts(), {
@@ -1646,7 +1646,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
toolCallId: "tc-broken", toolCallId: "tc-broken",
stopReason: "malformed", stopReason: "malformed",
}); });
return { status: "malformed", message: "incomplete stream" }; return { status: "malformed", errorMessage: "incomplete stream" };
} }
yield assistantText("done"); yield assistantText("done");
yield tokenUsage(emptyTokenCounts(), { yield tokenUsage(emptyTokenCounts(), {
@@ -1799,7 +1799,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
async *streamGenerate(params) { async *streamGenerate(params) {
calls += 1; calls += 1;
inputs.push(params.newMessages); inputs.push(params.newMessages);
return { status: "failed", message: "400 unknown parameter: max_output_tokens" }; return { status: "failed", errorMessage: "400 unknown parameter: max_output_tokens" };
}, },
}; };
const environment = new Environment({ const environment = new Environment({
@@ -1884,7 +1884,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
// eslint-disable-next-line require-yield // eslint-disable-next-line require-yield
async *streamGenerate() { async *streamGenerate() {
calls += 1; calls += 1;
return { status: "auth", message: "401 invalid x-api-key" }; return { status: "auth", errorMessage: "401 invalid x-api-key" };
}, },
}; };
const environment = new Environment({ const environment = new Environment({
@@ -1898,7 +1898,9 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
// The request's own terminal status is the host signal (streams to the web). // The request's own terminal status is the host signal (streams to the web).
const end = all.find((m) => (m.payload as { type?: string }).type === "request_end"); const end = all.find((m) => (m.payload as { type?: string }).type === "request_end");
expect((end!.payload as { status?: string }).status).toBe("auth"); expect((end!.payload as { status?: string }).status).toBe("auth");
expect((end!.payload as { message?: string }).message).toBe("401 invalid x-api-key"); expect((end!.payload as { error_message?: string }).error_message).toBe(
"401 invalid x-api-key",
);
// No planned retry is announced for a terminal failure. // No planned retry is announced for a terminal failure.
expect((end!.payload as { retry_in_ms?: number }).retry_in_ms).toBeUndefined(); expect((end!.payload as { retry_in_ms?: number }).retry_in_ms).toBeUndefined();
const abort = all.find((m) => (m.payload as { type?: string }).type === "abort"); const abort = all.find((m) => (m.payload as { type?: string }).type === "abort");
@@ -2037,7 +2039,10 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
if (calls === 1) { if (calls === 1) {
// A retryable provider rejection: the detail must reach observability via the // A retryable provider rejection: the detail must reach observability via the
// event — a retried request never produces an abort to carry it. // event — a retried request never produces an abort to carry it.
return { status: "timeout", message: "403 quota exceeded (insufficient_user_quota)" }; return {
status: "timeout",
errorMessage: "403 quota exceeded (insufficient_user_quota)",
};
} }
yield assistantText("ok"); yield assistantText("ok");
yield tokenUsage(emptyTokenCounts(), { yield tokenUsage(emptyTokenCounts(), {
@@ -2057,16 +2062,21 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
const all = await collectRun(engine, [userText("go")], allowAll); const all = await collectRun(engine, [userText("go")], allowAll);
const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end") as { const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end") as {
payload: { status?: string; message?: string; retry_in_ms?: number }; payload: { status?: string; error_message?: string; attempt?: number; retry_in_ms?: number };
}[]; }[];
expect(ends).toHaveLength(2); expect(ends).toHaveLength(2);
expect(ends[0]!.payload.status).toBe("timeout"); expect(ends[0]!.payload.status).toBe("timeout");
expect(ends[0]!.payload.message).toBe("403 quota exceeded (insufficient_user_quota)"); expect(ends[0]!.payload.error_message).toBe("403 quota exceeded (insufficient_user_quota)");
// The engine will retry: the planned backoff is announced (base 0 here -> 0ms). // The engine will retry: the planned backoff is announced (base 0 here -> 0ms), and the
// authoritative attempt ordinal is stamped (this was the run's 1st request).
expect(ends[0]!.payload.retry_in_ms).toBe(0); expect(ends[0]!.payload.retry_in_ms).toBe(0);
expect(ends[0]!.payload.attempt).toBe(1);
expect(ends[1]!.payload.status).toBe("completed"); expect(ends[1]!.payload.status).toBe("completed");
expect(ends[1]!.payload.message).toBeUndefined(); expect(ends[1]!.payload.error_message).toBeUndefined();
expect(ends[1]!.payload.retry_in_ms).toBeUndefined(); expect(ends[1]!.payload.retry_in_ms).toBeUndefined();
// A completion that needed retries still carries its ordinal ("recovered on attempt 2");
// only a clean first-try completion stays unstamped.
expect(ends[1]!.payload.attempt).toBe(2);
}); });
it("request_end announces the planned backoff (retry_in_ms) from the shared ladder; absent once the cap is reached", async () => { it("request_end announces the planned backoff (retry_in_ms) from the shared ladder; absent once the cap is reached", async () => {
@@ -2178,7 +2188,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
// exhausted-retries test below). // exhausted-retries test below).
yield thinkingMessage("half-thought", "failed"); yield thinkingMessage("half-thought", "failed");
yield assistantText("half-text", "failed"); yield assistantText("half-text", "failed");
return { status: "auth", message: "boom" }; return { status: "auth", errorMessage: "boom" };
} }
yield assistantText("ok"); yield assistantText("ok");
yield tokenUsage(emptyTokenCounts(), { yield tokenUsage(emptyTokenCounts(), {
+4 -4
View File
@@ -1573,7 +1573,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
model.streamGenerate({ newMessages: [userText("go")] }), model.streamGenerate({ newMessages: [userText("go")] }),
); );
expect(outcome.status).toBe("malformed"); expect(outcome.status).toBe("malformed");
expect(outcome.message).toContain("Unexpected token"); expect(outcome.errorMessage).toContain("Unexpected token");
const complete = messages.find((m) => typeOf(m) === "text"); const complete = messages.find((m) => typeOf(m) === "text");
expect((complete!.payload as TextPayload).text).toBe("hi"); expect((complete!.payload as TextPayload).text).toBe("hi");
expect((complete!.payload as TextPayload).stop_reason).toBe("malformed"); expect((complete!.payload as TextPayload).stop_reason).toBe("malformed");
@@ -1634,7 +1634,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
// 401 = credentials failure: its own stop reason so hosts can gate input until the // 401 = credentials failure: its own stop reason so hosts can gate input until the
// model's API key is updated (same engine behavior as failed). // model's API key is updated (same engine behavior as failed).
expect(outcome.status).toBe("auth"); expect(outcome.status).toBe("auth");
expect(outcome.message).toContain("invalid api key"); expect(outcome.errorMessage).toContain("invalid api key");
expect(messages.map(typeOf)).not.toContain("token_usage"); expect(messages.map(typeOf)).not.toContain("token_usage");
// A genuinely non-retryable parameter error stays a plain failure. // A genuinely non-retryable parameter error stays a plain failure.
@@ -1663,7 +1663,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
model.streamGenerate({ newMessages: [userText("go")] }), model.streamGenerate({ newMessages: [userText("go")] }),
); );
expect(outcome.status).toBe("auth"); expect(outcome.status).toBe("auth");
expect(outcome.message).toContain("subscription key invalid"); expect(outcome.errorMessage).toContain("subscription key invalid");
expect(messages.map(typeOf)).not.toContain("token_usage"); expect(messages.map(typeOf)).not.toContain("token_usage");
}); });
@@ -1681,7 +1681,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
expect(outcome.status).toBe("timeout"); expect(outcome.status).toBe("timeout");
// The real reason rides on the outcome (request_end -> the Cost center's errors panel // The real reason rides on the outcome (request_end -> the Cost center's errors panel
// shows it): a retried quota rejection must not surface as a bare "timeout". // shows it): a retried quota rejection must not surface as a bare "timeout".
expect(outcome.message).toContain("insufficient_user_quota"); expect(outcome.errorMessage).toContain("insufficient_user_quota");
expect(messages.map(typeOf)).not.toContain("token_usage"); expect(messages.map(typeOf)).not.toContain("token_usage");
}); });
+17
View File
@@ -132,6 +132,23 @@ describe("engine blocks ([turn_aborted] / [turn_retried] / [context_summary] / [
expect(extractSummary("[summary]a[/summary] <summary>b</summary>")).toBe("a"); expect(extractSummary("[summary]a[/summary] <summary>b</summary>")).toBe("a");
}); });
it("extractSummary rescues content the model wrote outside the tags (issue #170)", () => {
// deepseek-v4-flash writes the tags as a "title" and the body after the closing tag.
expect(extractSummary("[summary]\n[/summary]\nThe actual summary body.")).toBe(
"The actual summary body.",
);
expect(extractSummary("[summary]\n[summary][/summary]\nBody after a doubled open tag.")).toBe(
"Body after a doubled open tag.",
);
// A stray opening tag inside the pair doesn't hide the text that follows it.
expect(extractSummary("[summary]\n[summary]\nBody inside.[/summary]")).toBe("Body inside.");
// An empty current-form pair doesn't shadow a non-empty legacy pair.
expect(extractSummary("[summary][/summary]<summary>legacy body</summary>")).toBe("legacy body");
// Genuinely empty output still extracts to "" — the engine treats that as a failed attempt.
expect(extractSummary("[summary][/summary]")).toBe("");
expect(extractSummary("[summary]\n[/summary]")).toBe("");
});
it("turn blocks round-trip through unwrapSyntheticBlock, both forms, single-level", () => { it("turn blocks round-trip through unwrapSyntheticBlock, both forms, single-level", () => {
const lines = [transcribeUserInput("go"), transcribeToolCall("read_file", "t1", "{}")]; const lines = [transcribeUserInput("go"), transcribeToolCall("read_file", "t1", "{}")];
const aborted = buildTurnAbortedBlock(lines); const aborted = buildTurnAbortedBlock(lines);
+1 -1
View File
@@ -97,7 +97,7 @@ describe("session-title", () => {
assistantText("partial"), assistantText("partial"),
tokenUsage(emptyTokenCounts(), { cache_read: 0, cache_write: 0, output: 1, total: 1 }), tokenUsage(emptyTokenCounts(), { cache_read: 0, cache_write: 0, output: 1, total: 1 }),
], ],
{ status: "failed", message: "401" }, { status: "failed", errorMessage: "401" },
), ),
{ userText: "u", assistantText: "a" }, { userText: "u", assistantText: "a" },
); );
+3 -3
View File
@@ -103,7 +103,7 @@ Goal mode is the exception because its objective is re-injected as text every ro
## Automatic reconnect ## Automatic reconnect
Every LLM-side failure except `auth` triggers an in-run reconnect — `timeout` (network timeouts, transport disconnects, rate limits, 5xx, transient provider quota errors), `malformed` (truncated streams, JSON parse failures), and **`failed` as well**. `failed` retries even though the classifier judged it non-transient: that judgement is an allowlist of known codes, statuses and message vocabulary, so a gateway phrasing a transient fault its own way (`Upstream HTTP/2 stream failed`, say) lands there and used to kill the turn. Retrying a genuinely permanent error costs the ladder and ends the same way; aborting a transient one destroys the turn. Note this changes the *policy*, not the *taxonomy*: a `failed` request is still recorded as `failed` on its `request_end` and in the Cost center, rather than being relabelled a timeout. On a reconnect the engine re-sends the original input plus a `[turn_retried]` block carrying the previous partial output, so tools are never re-executed. Default limit is 5 reconnects with exponential backoff under a ceiling (base 250ms, cap 30s: 250ms, 500ms, 1s, 2s, 4s ≈ 7.75s of total patience — one shared schedule growing toward the slower retryable classes, with the first steps as fast as a transport blip needs); beyond that the turn settles as `failed`. Each failure's `request_end` announces the planned wait as `retry_in_ms` (same formula as the sleep), which the Web App renders as a live countdown with "retry now" (skips the remaining wait via `Session.skipReconnectWait` — the attempt counter is unchanged) and "give up" (the ordinary abort; the engine's abort-during-backoff path ends the turn) controls; the CLI prints its own `[retry]` line. All three retryable statuses render identically — a retry the user cannot see is a stalled session with no explanation and no way out. Compaction requests retry the same statuses under their own tighter cap (3 retries): a compaction that gives up keeps the original context and tries again at the next trigger, so a short ladder there beats stalling the session behind the full one. Authentication errors are classified before any retry heuristic and never retry: the request ends with its own terminal status `auth` (only the model reference is fixed at Session creation — credentials are read from the current Project config when the Session loads), and the Web App disables that Session's composer until the model's credential is updated (which auto-unlocks it) or the notice is dismissed for a retry. Tool errors are never retried — they are fed back to the model as `tool_call_output` and the model decides what to do next. Every LLM-side failure except `auth` triggers an in-run reconnect — `timeout` (network timeouts, transport disconnects, rate limits, 5xx, transient provider quota errors), `malformed` (truncated streams, JSON parse failures), and **`failed` as well**. `failed` retries even though the classifier judged it non-transient: that judgement is an allowlist of known codes, statuses and message vocabulary, so a gateway phrasing a transient fault its own way (`Upstream HTTP/2 stream failed`, say) lands there and used to kill the turn. Retrying a genuinely permanent error costs the ladder and ends the same way; aborting a transient one destroys the turn. Note this changes the *policy*, not the *taxonomy*: a `failed` request is still recorded as `failed` on its `request_end` and in the Cost center, rather than being relabelled a timeout. On a reconnect the engine re-sends the original input plus a `[turn_retried]` block carrying the previous partial output, so tools are never re-executed. Default limit is 5 reconnects with exponential backoff under a ceiling (base 250ms, cap 30s: 250ms, 500ms, 1s, 2s, 4s ≈ 7.75s of total patience — one shared schedule growing toward the slower retryable classes, with the first steps as fast as a transport blip needs); beyond that the turn settles as `failed`. Each failure's `request_end` announces the planned wait as `retry_in_ms` (same formula as the sleep) and stamps `attempt`, the authoritative 1-based ordinal of the request within its retry run (the CLI and Web App display it verbatim); the Web App renders the wait as a live countdown with "retry now" (skips the remaining wait via `Session.skipReconnectWait` — the attempt counter is unchanged) and "give up" (the ordinary abort; the engine's abort-during-backoff path ends the turn) controls; the CLI prints its own `[retry]` line. All three retryable statuses render identically — a retry the user cannot see is a stalled session with no explanation and no way out. A compaction request is an ordinary LLM request and by default retries on the same cap and ladder (an unusable summary draws on the same budget — see "Context compaction"); a compaction that gives up keeps the original context and tries again at the next trigger. Authentication errors are classified before any retry heuristic and never retry: the request ends with its own terminal status `auth` (only the model reference is fixed at Session creation — credentials are read from the current Project config when the Session loads), and the Web App disables that Session's composer until the model's credential is updated (which auto-unlocks it) or the notice is dismissed for a retry. Tool errors are never retried — they are fed back to the model as `tool_call_output` and the model decides what to do next.
## Compaction ## Compaction
@@ -126,9 +126,9 @@ Three triggers (`compaction_begin.reason`):
| `turns` | Session turn count ≥ `maxSessionTurns` (default -1 = unlimited) | | `turns` | Session turn count ≥ `maxSessionTurns` (default -1 = unlimited) |
| `manual` | the user runs `/compact` or calls `session.compact()` | | `manual` | the user runs `/compact` or calls `session.compact()` |
Two modes: `summarize` (default) appends the compaction Prompt to the old context, extracts the `[summary]`, wraps it as a `[context_summary]` user text and continues in a **fresh model context**; `discard` simply drops the old context. System markers are written as `[tag]…[/tag]`; the earlier angle-bracket form (`<summary>`, `<context_summary>`, …) is still recognized when reading old Traces and old persisted compaction prompts. Compaction rotates the [Trace file](/sessions-and-traces) (`_002`, `_003`, …) — one Trace file always equals one complete model context. `compactability()` probes feasibility before `session.compact()` (`ok | unsupported | empty | just_compacted`). Two modes: `summarize` (default) appends the compaction Prompt to the old context, extracts the `[summary]`, wraps it as a `[context_summary]` user text and continues in a **fresh model context**; `discard` simply drops the old context. System markers are written as `[tag]…[/tag]`; the earlier angle-bracket form (`<summary>`, `<context_summary>`, …) is still recognized when reading old Traces and old persisted compaction prompts. Summary extraction applies a tolerance ladder: the first non-empty `[summary]` tag pair wins; when every pair is empty, the text left after stripping the tags is used instead (rescuing models that write the body after the closing tag); with no tags at all, the whole output is used verbatim. Compaction rotates the [Trace file](/sessions-and-traces) (`_002`, `_003`, …) — one Trace file always equals one complete model context. `compactability()` probes feasibility before `session.compact()` (`ok | unsupported | empty | just_compacted`).
The compaction request keeps the session's toolset **unchanged** — the request prefix (tool list included) stays byte-identical to ordinary turns, so the provider's prompt cache remains valid at the moment the context is largest. Compaction still succeeds only with a valid summary: a response that calls a tool or whose extracted summary is empty is rejected — any tool calls are answered with synthesized failed outputs (keeping `tool_use`/`tool_result` pairing intact) and the repaired request is resent immediately (a rejection is model behavior, not a transport failure, so no backoff applies), up to 5 rejected attempts; then the compaction ends `failed`, keeping the original context and Trace file until the next trigger. Retryable attempts (`failed`/`timeout`/`malformed`, the same set the turn loop retries) follow the compaction-specific reconnect cap and backoff ladder described under "Automatic reconnect" above; only `auth` stops the compaction at once. The first **committed** attempt — adopted or rejected — also absorbs whatever turn input was folded into the compaction request (mid-task tool results, or the carry-over a manual `/compact` folds in) into the old context's history: retries resend only the repairs and the Prompt, and the continuation after an abandoned compaction never resends the absorbed input. The compaction request keeps the session's toolset **unchanged** — the request prefix (tool list included) stays byte-identical to ordinary turns, so the provider's prompt cache remains valid at the moment the context is largest. Compaction still succeeds only with a valid summary: a response that calls a tool or whose extracted summary is empty counts as one more failed attempt — any tool calls are answered with synthesized failed outputs (keeping `tool_use`/`tool_result` pairing intact), and the resent request carries a corrective note ahead of the compaction Prompt (committed history can only be appended to; rewriting it would invalidate the prompt cache). Every failure — unusable summaries and the `failed`/`timeout`/`malformed` transport statuses alike — draws on the one reconnect budget and backoff ladder described under "Automatic reconnect" above; only `auth` stops the compaction at once. Once the budget is exhausted the compaction ends `failed`, keeping the original context and Trace file until the next trigger; `compaction_end` reuses the unified retry detail block — `attempt` (the final attempt's ordinal, failed attempts included) and, on failure, the last `error_message` detail (shown on the chat banner and the CLI line) — and a failed compaction also lands in the cost center as a `compaction_failed` error record. The first **committed** attempt — adopted or rejected — also absorbs whatever turn input was folded into the compaction request (mid-task tool results, or the carry-over a manual `/compact` folds in) into the old context's history: retries resend only the repairs and the Prompt, and the continuation after an abandoned compaction never resends the absorbed input.
## Concurrency model ## Concurrency model
+3 -3
View File
@@ -100,7 +100,7 @@ Task 运行期间,宿主可通过 `session.steer(input)` 排队一条用户消
## 自动重连 ## 自动重连
除 `auth` 外,LLM 侧的所有失败都会触发引擎内自动重连——`timeout`(网络超时、传输层断连、限流、5xx、瞬时的供应商额度错误)、`malformed`(流截断、JSON 解析失败),**以及 `failed`**。`failed` 也重试,尽管分类器判定它不是瞬时错误:那个判定本质是一张允许清单(已知错误码、状态码与消息措辞),所以用自己说法描述瞬时故障的网关(例如 `Upstream HTTP/2 stream failed`)会落到这一档,此前会直接终止本轮。重试一个真正的永久错误,代价是走完退避梯度后以同样的方式收场;而把瞬时错误直接中断,则毁掉这一轮。注意改的是**策略**而非**分类**:`failed` 请求在 `request_end` 与成本中心里仍然记为 `failed`,不会被改标成超时。重连时同一次 `run` 内重发原始输入,并附加 `[turn_retried]` 块携带上一次的部分输出,避免工具重复执行。默认最多重连 5 次,指数退避并设上限(基数 250ms、上限 30s:250ms、500ms、1s、2s、4s,总耐心约 7.75s——所有可重试类别共用一张时间表,向较慢的类别递增,头几步仍与传输层抖动所需的一样快);超限后该轮以 `failed` 收场。每次失败的 `request_end` 会以 `retry_in_ms` 宣告计划中的等待(与实际休眠同一公式),Web App 据此实时倒计时,并提供「立即重试」(经 `Session.skipReconnectWait` 跳过剩余等待——重试计数不变)与「放弃」(普通中断;引擎的退避中中断路径结束本轮)两个内联按钮,CLI 则打印自己的 `[重试]` 行。三种可重试终态的渲染完全一致——用户看不见的重试,等于一次没有任何解释、也无从退出的卡顿。压缩请求重试同样的终态,只是用独立的更小上限(重试 3 次):压缩放弃后会保留原上下文、等下一次触发再试,所以那里用一段短梯度好过让会话干等完整的一段。鉴权错误在任何重试启发式之前判定、从不重试:请求以专属终态 `auth` 收场(Session 锁定的只是模型引用,凭据在会话装载时取自当前 Project 配置),Web App 据此禁用该 Session 的输入框,直到该模型的凭据被更新(更新后自动解锁)或用户点击「重试」。工具错误从不重试——它们作为 `tool_call_output` 反馈给模型,由模型决定下一步。 除 `auth` 外,LLM 侧的所有失败都会触发引擎内自动重连——`timeout`(网络超时、传输层断连、限流、5xx、瞬时的供应商额度错误)、`malformed`(流截断、JSON 解析失败),**以及 `failed`**。`failed` 也重试,尽管分类器判定它不是瞬时错误:那个判定本质是一张允许清单(已知错误码、状态码与消息措辞),所以用自己说法描述瞬时故障的网关(例如 `Upstream HTTP/2 stream failed`)会落到这一档,此前会直接终止本轮。重试一个真正的永久错误,代价是走完退避梯度后以同样的方式收场;而把瞬时错误直接中断,则毁掉这一轮。注意改的是**策略**而非**分类**:`failed` 请求在 `request_end` 与成本中心里仍然记为 `failed`,不会被改标成超时。重连时同一次 `run` 内重发原始输入,并附加 `[turn_retried]` 块携带上一次的部分输出,避免工具重复执行。默认最多重连 5 次,指数退避并设上限(基数 250ms、上限 30s:250ms、500ms、1s、2s、4s,总耐心约 7.75s——所有可重试类别共用一张时间表,向较慢的类别递增,头几步仍与传输层抖动所需的一样快);超限后该轮以 `failed` 收场。每次失败的 `request_end` 会以 `retry_in_ms` 宣告计划中的等待(与实际休眠同一公式)、以 `attempt` 标注这是本轮第几次尝试(权威序号,CLI 与 Web 的重试行直接显示它),Web App 据此实时倒计时,并提供「立即重试」(经 `Session.skipReconnectWait` 跳过剩余等待——重试计数不变)与「放弃」(普通中断;引擎的退避中中断路径结束本轮)两个内联按钮,CLI 则打印自己的 `[重试]` 行。三种可重试终态的渲染完全一致——用户看不见的重试,等于一次没有任何解释、也无从退出的卡顿。压缩请求是一次普通的 LLM 请求,默认沿用同一重连上限与退避阶梯(无效摘要也计入同一预算,见「上下文压缩」一节);压缩放弃后保留原上下文、等下一次触发再试。鉴权错误在任何重试启发式之前判定、从不重试:请求以专属终态 `auth` 收场(Session 锁定的只是模型引用,凭据在会话装载时取自当前 Project 配置),Web App 据此禁用该 Session 的输入框,直到该模型的凭据被更新(更新后自动解锁)或用户点击「重试」。工具错误从不重试——它们作为 `tool_call_output` 反馈给模型,由模型决定下一步。
## 上下文压缩(Compaction) ## 上下文压缩(Compaction)
@@ -123,9 +123,9 @@ interface CompactionSettings {
| `turns` | Session 轮数 ≥ `maxSessionTurns`(默认 -1,即不限) | | `turns` | Session 轮数 ≥ `maxSessionTurns`(默认 -1,即不限) |
| `manual` | 用户执行 `/compact` 或调用 `session.compact()` | | `manual` | 用户执行 `/compact` 或调用 `session.compact()` |
两种模式:`summarize`(默认)向旧上下文追加压缩 Prompt,提取 `[summary]` 后包装为 `[context_summary]` 用户文本,在**全新的模型上下文**中继续;`discard` 直接丢弃旧上下文。系统标记统一写作 `[tag]…[/tag]`;读取旧 Trace 与旧压缩 Prompt 时仍识别早期的尖括号形式(`<summary>`、`<context_summary>` 等)。压缩时 [Trace 文件随之轮转](/sessions-and-traces)(`_002`、`_003`……),一个 Trace 文件恒等于一个完整模型上下文。`session.compact()` 前可用 `compactability()` 探询可行性(`ok | unsupported | empty | just_compacted`)。 两种模式:`summarize`(默认)向旧上下文追加压缩 Prompt,提取 `[summary]` 后包装为 `[context_summary]` 用户文本,在**全新的模型上下文**中继续;`discard` 直接丢弃旧上下文。系统标记统一写作 `[tag]…[/tag]`;读取旧 Trace 与旧压缩 Prompt 时仍识别早期的尖括号形式(`<summary>`、`<context_summary>` 等)。摘要提取自带容忍阶梯:优先取第一个非空的 `[summary]` 标签对;标签对全为空时,改取剥离标签后剩余的全部文本(兼容把正文写在闭合标签之后的模型);完全没有标签则整段输出照用。压缩时 [Trace 文件随之轮转](/sessions-and-traces)(`_002`、`_003`……),一个 Trace 文件恒等于一个完整模型上下文。`session.compact()` 前可用 `compactability()` 探询可行性(`ok | unsupported | empty | just_compacted`)。
压缩请求**保持会话工具集不变**——请求前缀(含工具列表)与普通轮次逐字节一致,确保上下文最大的时刻提供商的提示词缓存依然有效。只有得到有效摘要,压缩才算成功:若响应中出现工具调用、或提取出的摘要为空,则判为无效并重试——工具调用会先以合成的失败输出逐一应答(保持 `tool_use`/`tool_result` 配对完整),修复后的请求**立即重发**(无效摘要是模型行为而非传输故障,不做退避),最多允许 5 次无效尝试,之后压缩以 `failed` 结束,保留原上下文与 Trace 文件,等待下次触发。可重试的结束态(`failed`/`timeout`/`malformed`,与轮次循环同一套)走上文「自动重连」一节所述的压缩专用重连上限与退避阶梯;只有 `auth` 会让压缩当场停止。首个**已提交**的尝试(无论被采纳还是被判无效)同时会把折叠进压缩请求的本轮输入(任务中途的工具结果、手动 `/compact` 折叠的补发内容)吸收进旧上下文:重试只重发修复输出与压缩 Prompt,压缩放弃后的续跑也不再重发已吸收的输入。 压缩请求**保持会话工具集不变**——请求前缀(含工具列表)与普通轮次逐字节一致,确保上下文最大的时刻提供商的提示词缓存依然有效。只有得到有效摘要,压缩才算成功:若响应中出现工具调用、或提取出的摘要为空,视同一次普通的失败尝试——工具调用先以合成的失败输出逐一应答(保持 `tool_use`/`tool_result` 配对完整),重发的请求在压缩 Prompt 前附加一条纠正说明(已提交的历史只能追加、不能改写,改写会使提示词缓存失效)。所有失败——无效摘要与 `failed`/`timeout`/`malformed` 传输失败——共用上文「自动重连」一节的同一重连预算与退避阶梯;只有 `auth` 会让压缩当场停止。预算耗尽后压缩以 `failed` 结束,保留原上下文与 Trace 文件,等待下次触发;`compaction_end` 复用统一的重试详情块——`attempt`(最终尝试序号,失败尝试也计入)与失败时的 `error_message` 详情(聊天横幅与 CLI 直接展示),压缩失败还会作为一条 `compaction_failed` 错误记录进入成本中心。首个**已提交**的尝试(无论被采纳还是被判无效)同时会把折叠进压缩请求的本轮输入(任务中途的工具结果、手动 `/compact` 折叠的补发内容)吸收进旧上下文:重试只重发修复输出与压缩 Prompt,压缩放弃后的续跑也不再重发已吸收的输入。
## 并发模型 ## 并发模型
+12 -4
View File
@@ -190,10 +190,18 @@ interface RequestBeginPayload {
interface RequestEndPayload { interface RequestEndPayload {
type: "request_end"; type: "request_end";
status: StopReason; // "completed" is the mechanical commit criterion for replay status: StopReason; // "completed" is the mechanical commit criterion for replay
message?: string; // failure detail (LLMOutcome.message), non-completed only: // The unified RetryDetail block below is stamped in one place by the builders; every
// the real reason behind a retried/failed Request (e.g. a // field is additive — old Traces replay unchanged. compaction_end reuses the same
// provider quota code) — read by the Cost center's errors // block (attempt = final attempt ordinal, error_message = the last failure's detail).
// panel; additive, old Traces replay unchanged error_message?: string; // error detail (LLMOutcome.errorMessage internally — one
// name across the stack), non-completed only: the real
// reason behind a retried/failed Request (e.g. a provider
// quota code) — read by the Cost center's errors panel
attempt?: number; // 1-based ordinal of this request within its retry run (the
// authoritative retry count): stamped on failures and on a
// completion that needed retries; absent on a clean first try
retry_in_ms?: number; // planned reconnect wait (ms), present only when the engine
// will retry in-run — the Web App renders it as a countdown
} }
interface ApprovalDecisionPayload { interface ApprovalDecisionPayload {
+9 -3
View File
@@ -189,9 +189,15 @@ interface RequestBeginPayload {
interface RequestEndPayload { interface RequestEndPayload {
type: "request_end"; type: "request_end";
status: StopReason; // completed 是回放判定「该轮已提交」的机械标准 status: StopReason; // completed 是回放判定「该轮已提交」的机械标准
message?: string; // 失败详情(LLMOutcome.message),仅非 completed 携带: // 以下为统一的重试详情块 RetryDetail(由 builder 一处盖章;均为增量字段,旧 Trace 回放
// 被重试/失败的 Request 背后的真实原因(如供应商额度码), // 不受影响;compaction_end 复用同一块——attempt 为最终尝试序号,error_message 为失败详情)
// 供成本中心错误面板读取;增量字段,旧 Trace 回放不受影响 error_message?: string; // 错误详情(内部 LLMOutcome.errorMessage,内外同名),仅非
// completed 携带:被重试/失败的 Request 背后的真实原因
// (如供应商额度码),供成本中心错误面板读取
attempt?: number; // 本次重试序列内的第几次请求(1 起,权威计数):失败请求
// 与经重试后成功的请求携带;首发即成功不携带
retry_in_ms?: number; // 计划中的重连等待(毫秒),仅当引擎将在本轮内重试时携带,
// Web App 据此渲染倒计时
} }
interface ApprovalDecisionPayload { interface ApprovalDecisionPayload {
+1 -1
View File
@@ -82,7 +82,7 @@ CREATE TABLE IF NOT EXISTS error_records ( -- server-side error capture (the
project_id TEXT, -- nullable: sign-in/registration and process-level errors have no Project context project_id TEXT, -- nullable: sign-in/registration and process-level errors have no Project context
agent_id TEXT, agent_id TEXT,
session_id TEXT, session_id TEXT,
source TEXT NOT NULL, -- http | session | usage | title | subagent | process | llm | environment | schedule source TEXT NOT NULL, -- http | session | usage | title | subagent | process | llm | environment | compaction | schedule
kind TEXT NOT NULL, -- expected (HttpError, business 4xx) | unexpected (500/runtime) kind TEXT NOT NULL, -- expected (HttpError, business 4xx) | unexpected (500/runtime)
code TEXT NOT NULL, -- HttpError.code / internal / session_run_failed / ... code TEXT NOT NULL, -- HttpError.code / internal / session_run_failed / ...
status INTEGER, -- HTTP status code; NULL for non-HTTP sources status INTEGER, -- HTTP status code; NULL for non-HTTP sources
@@ -61,6 +61,7 @@ export type ErrorSource =
| "session" | "session"
| "llm" | "llm"
| "environment" | "environment"
| "compaction"
| "usage" | "usage"
| "title" | "title"
| "subagent" | "subagent"
@@ -4,7 +4,8 @@
* converges them into the message stream (LLM and Environment handle errors * converges them into the message stream (LLM and Environment handle errors
* internally and never throw), so a try/catch can't catch a single one. This watcher * internally and never throw), so a try/catch can't catch a single one. This watcher
* hooks onto SessionManager's drive, inspects messages one by one, and fishes them out * hooks onto SessionManager's drive, inspects messages one by one, and fishes them out
* into error_records (source = `llm` / `environment`), matching usage-recorder's shape: * into error_records (source = `llm` / `environment` / `compaction`), matching
* usage-recorder's shape:
* recognizes only a few payload types, no-op on the rest. **One instance per run/compact** * recognizes only a few payload types, no-op on the rest. **One instance per run/compact**
* (its state wraps up accordingly, see close). * (its state wraps up accordingly, see close).
* *
@@ -27,14 +28,14 @@
* - `aborted` / `completed` are not recorded (the former is a user-initiated interrupt, * - `aborted` / `completed` are not recorded (the former is a user-initiated interrupt,
* not an error). * not an error).
* *
* The message uses the real reason: `request_end` carries the failure detail on * The message uses the real reason: `request_end` carries the error detail on
* non-completed statuses (`message`, from LLMOutcome), and the `abort` event's reason is * non-completed statuses (`error_message`, from LLMOutcome), and the `abort` event's reason is
* core's failure-reason prose (e.g. `llm request error: 401 …` / `malformed response * core's failure-reason prose (e.g. `llm request error: 401 …` / `malformed response
* failed after N retries`). A `request_end` failure is first held pending (status + its * failed after N retries`). A `request_end` failure is first held pending (status + its
* own detail), not persisted immediately, and is resolved at the next request boundary: * own detail), not persisted immediately, and is resolved at the next request boundary:
* - Immediately followed by `abort` → use its reason as the message (the real reason); * - Immediately followed by `abort` → use its reason as the message (the real reason);
* - Immediately followed by `request_begin` (the engine is retrying — no abort will ever * - Immediately followed by `request_begin` (the engine is retrying — no abort will ever
* arrive) → use the staged request_end's own `message` (the real detail, e.g. a quota * arrive) → use the staged request_end's own `error_message` detail (e.g. a quota
* code), falling back to the generic status text when it carried none. This boundary is * code), falling back to the generic status text when it carried none. This boundary is
* also what tells a survived `failed` from an exhausted one (see the classification above); * also what tells a survived `failed` from an exhausted one (see the classification above);
* - Still unresolved when the run ends → close persists it the same way. * - Still unresolved when the run ends → close persists it the same way.
@@ -295,17 +296,23 @@ export class StreamErrorWatcher {
type?: string; type?: string;
status?: StopReason; status?: StopReason;
reason?: string | null; reason?: string | null;
message?: string; error_message?: string;
}; };
const key = originKey(msg); const key = originKey(msg);
if (p.type === "compaction_end") {
this.observeCompactionEnd(msg, key);
return;
}
if (p.type === "request_end") { if (p.type === "request_end") {
this.flush(key); // Defensive: if a previous failure is still pending (normally resolved by request_begin), persist it first this.flush(key); // Defensive: if a previous failure is still pending (normally resolved by request_begin), persist it first
if (isLlmFailure(p.status)) { if (isLlmFailure(p.status)) {
this.pending.set(key, { this.pending.set(key, {
status: p.status, status: p.status,
// The event's own failure detail (LLMOutcome.message): the message of record when // The event's own error detail (request_end.error_message): the message of record
// no abort follows (the retry path). // when no abort follows (the retry path).
...(typeof p.message === "string" && p.message.trim() ? { message: p.message } : {}), ...(typeof p.error_message === "string" && p.error_message.trim()
? { message: p.error_message }
: {}),
}); });
} }
return; return;
@@ -356,6 +363,43 @@ export class StreamErrorWatcher {
}); });
} }
/**
* A failed compaction becomes its own error record (source = `compaction`) — a compaction
* is an ordinary LLM request whose failures (unusable summary and transport alike) core
* retries under the standard budget, so a `failed` end means the retries ran out
* (issue #170). This is the single record a failed compaction produces: the compaction
* request's own request_begin/request_end pair is written to Trace only and never reaches
* this stream, so the llm-source records above cannot double-count it. `completed` is not
* an error and `aborted` is a user interrupt — neither is recorded. kind is unexpected for
* the same reason `llm_failed` is: the budget is exhausted, and while the original context
* is kept, the trigger still holds — the session keeps re-entering compaction until a
* human changes something. The message carries the attempts and the last error detail
* (compaction_end's share of the RetryDetail block) so the cost center shows what failed.
*/
private observeCompactionEnd(msg: OmniMessage, key: string): void {
const p = msg.payload as {
mode?: string;
reason?: string;
status?: StopReason;
attempt?: number;
error_message?: string;
};
if (p.status !== "failed") return;
const attempts =
typeof p.attempt === "number" && p.attempt > 0
? ` after ${p.attempt} attempt${p.attempt === 1 ? "" : "s"}`
: "";
const detail =
typeof p.error_message === "string" && p.error_message ? `: ${p.error_message}` : "";
this.errors.record({
source: "compaction",
err: `${p.mode ?? "summarize"} compaction failed${attempts}${detail}; trigger ${p.reason ?? "unknown"}, original context kept.`,
ctx: this.ctxFor(key),
code: "compaction_failed",
kind: "unexpected",
});
}
// —— Environment (tool execution) —— // —— Environment (tool execution) ——
private observeTool(msg: OmniMessage): void { private observeTool(msg: OmniMessage): void {
@@ -608,7 +608,8 @@ export function probeVerdict(
): { ok: true } | { ok: false; message: string } { ): { ok: true } | { ok: false; message: string } {
if (outcome.status === "completed") return { ok: true }; if (outcome.status === "completed") return { ok: true };
if (outcome.status === "malformed" && sawContent) return { ok: true }; if (outcome.status === "malformed" && sawContent) return { ok: true };
const detail = "message" in outcome && outcome.message ? outcome.message : outcome.status; const detail =
"errorMessage" in outcome && outcome.errorMessage ? outcome.errorMessage : outcome.status;
return { ok: false, message: String(detail).slice(0, 300) }; return { ok: false, message: String(detail).slice(0, 300) };
} }
+70 -7
View File
@@ -15,6 +15,7 @@ import type { DatabaseSync } from "node:sqlite";
import { import {
abortEvent, abortEvent,
assistantText, assistantText,
compactionEnd,
partialToolCallOutput, partialToolCallOutput,
requestBegin, requestBegin,
requestEnd, requestEnd,
@@ -414,7 +415,7 @@ describe("stream-error-watcher (LLM / Environment errors)", () => {
// a bucket would let a real credential failure be dropped as a duplicate. // a bucket would let a real credential failure be dropped as a duplicate.
const got = feed([ const got = feed([
requestBegin(), requestBegin(),
requestEnd("auth", "401 invalid x-api-key (invalid_api_key)"), requestEnd("auth", { errorMessage: "401 invalid x-api-key (invalid_api_key)" }),
abortEvent("llm request error: 401 invalid x-api-key (invalid_api_key)"), abortEvent("llm request error: 401 invalid x-api-key (invalid_api_key)"),
]); ]);
expect(got).toHaveLength(1); expect(got).toHaveLength(1);
@@ -432,7 +433,9 @@ describe("stream-error-watcher (LLM / Environment errors)", () => {
// request_begin proves another attempt happened — that one is expected, like a timeout. // request_begin proves another attempt happened — that one is expected, like a timeout.
const got = feed([ const got = feed([
requestBegin(), requestBegin(),
requestEnd("failed", "Upstream HTTP/2 stream failed (upstream_http2_stream_error)"), requestEnd("failed", {
errorMessage: "Upstream HTTP/2 stream failed (upstream_http2_stream_error)",
}),
requestBegin(), requestBegin(),
requestEnd("completed"), requestEnd("completed"),
]); ]);
@@ -453,9 +456,9 @@ describe("stream-error-watcher (LLM / Environment errors)", () => {
// the one that never does. // the one that never does.
const got = feed([ const got = feed([
requestBegin(), requestBegin(),
requestEnd("failed", "Upstream HTTP/2 stream failed"), requestEnd("failed", { errorMessage: "Upstream HTTP/2 stream failed" }),
requestBegin(), // The retry: resolves the failure above as recovered. requestBegin(), // The retry: resolves the failure above as recovered.
requestEnd("auth", "401 invalid x-api-key"), requestEnd("auth", { errorMessage: "401 invalid x-api-key" }),
abortEvent("llm request error: 401 invalid x-api-key"), abortEvent("llm request error: 401 invalid x-api-key"),
]); ]);
expect(got.map((r) => [r.code, r.kind])).toEqual([ expect(got.map((r) => [r.code, r.kind])).toEqual([
@@ -471,7 +474,9 @@ describe("stream-error-watcher (LLM / Environment errors)", () => {
// "timed out" status text. This is what the Cost center shows for a retried quota 403. // "timed out" status text. This is what the Cost center shows for a retried quota 403.
const got = feed([ const got = feed([
requestBegin(), requestBegin(),
requestEnd("timeout", "403 no active subscription (insufficient_user_quota)"), requestEnd("timeout", {
errorMessage: "403 no active subscription (insufficient_user_quota)",
}),
requestBegin(), requestBegin(),
requestEnd("completed"), requestEnd("completed"),
]); ]);
@@ -487,7 +492,7 @@ describe("stream-error-watcher (LLM / Environment errors)", () => {
it("an abort reason still outranks the staged request_end detail (failed exit path)", () => { it("an abort reason still outranks the staged request_end detail (failed exit path)", () => {
const got = feed([ const got = feed([
requestBegin(), requestBegin(),
requestEnd("failed", "401 invalid x-api-key (invalid_api_key)"), requestEnd("failed", { errorMessage: "401 invalid x-api-key (invalid_api_key)" }),
abortEvent("llm request error: 401 invalid x-api-key (invalid_api_key)"), abortEvent("llm request error: 401 invalid x-api-key (invalid_api_key)"),
]); ]);
expect(got).toHaveLength(1); expect(got).toHaveLength(1);
@@ -500,7 +505,7 @@ describe("stream-error-watcher (LLM / Environment errors)", () => {
// request_end detail IS the failure's reason — prefer it over the generic status text. // request_end detail IS the failure's reason — prefer it over the generic status text.
const got = feed([ const got = feed([
requestBegin(), requestBegin(),
requestEnd("timeout", "403 quota exceeded (insufficient_user_quota)"), requestEnd("timeout", { errorMessage: "403 quota exceeded (insufficient_user_quota)" }),
abortEvent("aborted during reconnect backoff"), abortEvent("aborted during reconnect backoff"),
]); ]);
expect(got).toHaveLength(1); expect(got).toHaveLength(1);
@@ -843,6 +848,64 @@ describe("stream-error-watcher (LLM / Environment errors)", () => {
expect(got).toHaveLength(1); expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({ code: "llm_failed", agent_id: "a1", session_id: "s1" }); expect(got[0]).toMatchObject({ code: "llm_failed", agent_id: "a1", session_id: "s1" });
}); });
// —— Compaction ——
it("a failed compaction records one error row; the message carries attempts and the error", () => {
// Issue #170: a compaction is an ordinary LLM request whose failures core retries under
// the standard budget — a failed end means the retries ran out, and the cost center's
// row shows how many attempts were spent and what the last failure was.
const got = feed([
compactionEnd({
reason: "context",
mode: "summarize",
status: "failed",
attempt: 5,
errorMessage: "the response contained no usable summary",
}),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
source: "compaction",
kind: "unexpected",
code: "compaction_failed",
message:
"summarize compaction failed after 5 attempts: the response contained no usable summary; trigger context, original context kept.",
project_id: "p1",
agent_id: "a1",
session_id: "s1",
});
});
it("compaction completed / aborted are not errors; an old-core failed still records", () => {
const got = feed([
compactionEnd({ reason: "context", mode: "summarize", status: "completed", attempt: 1 }),
compactionEnd({ reason: "manual", mode: "summarize", status: "aborted", attempt: 2 }),
// An old core's compaction_end has no attempt/error fields at all.
compactionEnd({ reason: "turns", mode: "summarize", status: "failed" }),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
code: "compaction_failed",
message: "summarize compaction failed; trigger turns, original context kept.",
});
});
it("a child session's failed compaction attributes to the child Agent/Session", () => {
const got = feed([
childMeta("session-child", "/data/agents/agent-child/agent_state"),
withOrigin(
compactionEnd({ reason: "context", mode: "summarize", status: "failed", attempt: 6 }),
"session-child",
),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
code: "compaction_failed",
agent_id: "agent-child",
session_id: "session-child",
});
});
}); });
describe("HTTP onError persistence (integration)", () => { describe("HTTP onError persistence (integration)", () => {
+3 -3
View File
@@ -48,13 +48,13 @@ describe("probeVerdict", () => {
}); });
it("fails a malformed ending with nothing received (broken response, not a working endpoint)", () => { it("fails a malformed ending with nothing received (broken response, not a working endpoint)", () => {
const verdict = probeVerdict({ status: "malformed", message: "unexpected EOF" }, false); const verdict = probeVerdict({ status: "malformed", errorMessage: "unexpected EOF" }, false);
expect(verdict).toEqual({ ok: false, message: "unexpected EOF" }); expect(verdict).toEqual({ ok: false, message: "unexpected EOF" });
}); });
it("fails timeouts and errors even when content was streamed (flaky is not ok)", () => { it("fails timeouts and errors even when content was streamed (flaky is not ok)", () => {
expect(probeVerdict({ status: "timeout" }, true)).toEqual({ ok: false, message: "timeout" }); expect(probeVerdict({ status: "timeout" }, true)).toEqual({ ok: false, message: "timeout" });
expect(probeVerdict({ status: "failed", message: "401 unauthorized" }, true)).toEqual({ expect(probeVerdict({ status: "failed", errorMessage: "401 unauthorized" }, true)).toEqual({
ok: false, ok: false,
message: "401 unauthorized", message: "401 unauthorized",
}); });
@@ -62,7 +62,7 @@ describe("probeVerdict", () => {
}); });
it("truncates long failure messages to 300 characters", () => { it("truncates long failure messages to 300 characters", () => {
const verdict = probeVerdict({ status: "failed", message: "x".repeat(500) }, false); const verdict = probeVerdict({ status: "failed", errorMessage: "x".repeat(500) }, false);
expect(verdict.ok).toBe(false); expect(verdict.ok).toBe(false);
if (!verdict.ok) expect(verdict.message).toHaveLength(300); if (!verdict.ok) expect(verdict.message).toHaveLength(300);
}); });
@@ -23,6 +23,6 @@ export function CompactionBanner({ item }: { item: CompactionItem }) {
const text = const text =
item.status === "completed" item.status === "completed"
? S.chat.compactionDone(item.mode) ? S.chat.compactionDone(item.mode)
: S.chat.compactionFailed(item.status ?? "failed"); : S.chat.compactionFailed(item.status ?? "failed", item.errorMessage);
return <p className="my-1 font-mono text-xs text-gray-500 dark:text-gray-400">{text}</p>; return <p className="my-1 font-mono text-xs text-gray-500 dark:text-gray-400">{text}</p>;
} }
+10 -13
View File
@@ -220,7 +220,7 @@ export interface ReconnectItem {
id: number; id: number;
/** Trigger reason: timeout (timed out / disconnected), malformed (an incomplete or unparseable response), or failed (the provider returned an error). */ /** Trigger reason: timeout (timed out / disconnected), malformed (an incomplete or unparseable response), or failed (the provider returned an error). */
status: ReconnectStatus; status: ReconnectStatus;
/** Which retry attempt this is (increments on consecutive failures within the same round; resets to 1 after a request finishes normally). */ /** Which retry attempt this is — request_end.attempt, the core's authoritative 1-based ordinal (1 for Traces written before the field existed). */
attempt: number; attempt: number;
/** The retry request has been sent (set true by the next request_begin). */ /** The retry request has been sent (set true by the next request_begin). */
retrying: boolean; retrying: boolean;
@@ -250,6 +250,8 @@ export interface CompactionItem {
/** True between begin and end (renders a "compaction in progress" banner). */ /** True between begin and end (renders a "compaction in progress" banner). */
running: boolean; running: boolean;
status?: StopReason; status?: StopReason;
/** Last failure detail from compaction_end.error_message (its share of the RetryDetail block; present on failed ends from new cores). */
errorMessage?: string;
} }
export interface TaskStatsItem { export interface TaskStatsItem {
@@ -375,8 +377,6 @@ export interface StreamModel {
* dispatches it via `void executeOne`, which doesn't block the streaming loop — execution happens between two Requests). * dispatches it via `void executeOne`, which doesn't block the streaming loop — execution happens between two Requests).
*/ */
openApprovalWaitMs: number; openApprovalWaitMs: number;
/** Consecutive reconnect-failure count (incremented when request_end carries a status the engine reconnects on, reset to zero on any other terminal status). */
reconnectRun: number;
/** /**
* Timestamp (ms) of the most recent auth failure: a main-session `request_end` with * Timestamp (ms) of the most recent auth failure: a main-session `request_end` with
* status "auth" arrived on THIS model (a subagent's request events route to the nested * status "auth" arrived on THIS model (a subagent's request events route to the nested
@@ -459,7 +459,6 @@ function newModel(nested: boolean, localDecisions: Set<string>): StreamModel {
lastTsMs: 0, lastTsMs: 0,
openRequestBeginMs: null, openRequestBeginMs: null,
openApprovalWaitMs: 0, openApprovalWaitMs: 0,
reconnectRun: 0,
lastAuthFailureMs: null, lastAuthFailureMs: null,
taskOpen: false, taskOpen: false,
taskStartLocalMs: 0, taskStartLocalMs: 0,
@@ -723,10 +722,9 @@ function startTask(model: StreamModel, timestamp: string, nowMs: number): void {
// server dies during a backoff window, the Trace's tail is // server dies during a backoff window, the Trace's tail is
// request_end(timeout) with no abort, and history rebuild would leave a // request_end(timeout) with no abort, and history rebuild would leave a
// dangling "retrying…" — the new Task's first request_begin isn't its // dangling "retrying…" — the new Task's first request_begin isn't its
// retry, so mark it gaveUp and reset the consecutive-failure count (the new Task's failures count from 1 again). // retry, so mark it gaveUp.
const waiting = findLastWaitingReconnect(model); const waiting = findLastWaitingReconnect(model);
if (waiting) waiting.gaveUp = true; if (waiting) waiting.gaveUp = true;
model.reconnectRun = 0;
// Any unclosed Request start / approval wait left over from the previous Task isn't carried into this Task's LLM timing. // Any unclosed Request start / approval wait left over from the previous Task isn't carried into this Task's LLM timing.
model.openRequestBeginMs = null; model.openRequestBeginMs = null;
model.openApprovalWaitMs = 0; model.openApprovalWaitMs = 0;
@@ -1302,11 +1300,9 @@ function handleEvent(model: StreamModel, p: EventPayload, tsMs?: number, nowMs?:
// placeholder resend goes only to the model, never written to Trace): finalize the executing cards. // placeholder resend goes only to the model, never written to Trace): finalize the executing cards.
closeExecutingToolCards(model); closeExecutingToolCards(model);
// A reconnect hint waiting to retry: an interruption means retries are // A reconnect hint waiting to retry: an interruption means retries are
// exhausted/abandoned, so mark it gaveUp (this interruption marker item gives the reason); // exhausted/abandoned, so mark it gaveUp (this interruption marker item gives the reason).
// the run has ended, so reset the consecutive-failure count.
const waiting = findLastWaitingReconnect(model); const waiting = findLastWaitingReconnect(model);
if (waiting) waiting.gaveUp = true; if (waiting) waiting.gaveUp = true;
model.reconnectRun = 0;
const item: AbortItem = { kind: "abort", id: nextId(model) }; const item: AbortItem = { kind: "abort", id: nextId(model) };
if (p.reason != null) item.reason = p.reason; if (p.reason != null) item.reason = p.reason;
model.items.push(item); model.items.push(item);
@@ -1333,6 +1329,7 @@ function handleEvent(model: StreamModel, p: EventPayload, tsMs?: number, nowMs?:
if (item) { if (item) {
item.running = false; item.running = false;
item.status = p.status; item.status = p.status;
if (p.error_message !== undefined) item.errorMessage = p.error_message;
} else { } else {
// Mid-stream join (missed the begin): append a completed banner directly. // Mid-stream join (missed the begin): append a completed banner directly.
const created: CompactionItem = { const created: CompactionItem = {
@@ -1342,6 +1339,7 @@ function handleEvent(model: StreamModel, p: EventPayload, tsMs?: number, nowMs?:
mode: p.mode, mode: p.mode,
running: false, running: false,
status: p.status, status: p.status,
...(p.error_message !== undefined ? { errorMessage: p.error_message } : {}),
}; };
model.items.push(created); model.items.push(created);
} }
@@ -1404,12 +1402,13 @@ function handleEvent(model: StreamModel, p: EventPayload, tsMs?: number, nowMs?:
// ladder with nothing on screen and no give-up control. It would also reset the counter // ladder with nothing on screen and no give-up control. It would also reset the counter
// mid-ladder, renumbering a mixed timeout → failed → timeout run back to retry #1. // mid-ladder, renumbering a mixed timeout → failed → timeout run back to retry #1.
if (isReconnectStatus(p.status)) { if (isReconnectStatus(p.status)) {
model.reconnectRun += 1;
const item: ReconnectItem = { const item: ReconnectItem = {
kind: "reconnect", kind: "reconnect",
id: nextId(model), id: nextId(model),
status: p.status, status: p.status,
attempt: model.reconnectRun, // The core stamps the authoritative ordinal on every retryable request_end; only
// Traces written before the field existed lack it (rendered as attempt 1).
attempt: p.attempt ?? 1,
retrying: false, retrying: false,
}; };
// The engine announced its planned backoff: keep it with the CLIENT arrival time // The engine announced its planned backoff: keep it with the CLIENT arrival time
@@ -1419,8 +1418,6 @@ function handleEvent(model: StreamModel, p: EventPayload, tsMs?: number, nowMs?:
item.arrivedAtMs = nowMs ?? Date.now(); item.arrivedAtMs = nowMs ?? Date.now();
} }
model.items.push(item); model.items.push(item);
} else {
model.reconnectRun = 0;
} }
return; return;
} }
+6 -2
View File
@@ -852,8 +852,12 @@ Scenarios:
mode === "discard" mode === "discard"
? "[Compaction] done, old context discarded" ? "[Compaction] done, old context discarded"
: "[Compaction] done, switched to the summarized context", : "[Compaction] done, switched to the summarized context",
compactionFailed: (status: string) => compactionFailed: (status: string, errorMessage?: string): string => {
`[Compaction] ${status === "aborted" ? "aborted" : "failed"}, keeping current context`, if (status === "aborted") return "[Compaction] aborted, keeping current context";
return errorMessage !== undefined
? `[Compaction] failed (${errorMessage}), keeping current context`
: "[Compaction] failed, keeping current context";
},
unknownTool: "(unknown tool)", unknownTool: "(unknown tool)",
workRunning: "Running", workRunning: "Running",
workDone: "Done", workDone: "Done",
+6 -2
View File
@@ -831,8 +831,12 @@ Benchmark:
compactionRunning: (mode: string) => `压缩进行中(${mode})…`, compactionRunning: (mode: string) => `压缩进行中(${mode})…`,
compactionDone: (mode: string): string => compactionDone: (mode: string): string =>
mode === "discard" ? "[压缩] 完成,旧上下文已丢弃" : "[压缩] 完成,已切换到摘要后的新上下文", mode === "discard" ? "[压缩] 完成,旧上下文已丢弃" : "[压缩] 完成,已切换到摘要后的新上下文",
compactionFailed: (status: string) => compactionFailed: (status: string, errorMessage?: string): string => {
`[压缩] ${status === "aborted" ? "已中断" : "失败"},保留当前上下文`, if (status === "aborted") return "[压缩] 已中断,保留当前上下文";
return errorMessage !== undefined
? `[压缩] 失败(${errorMessage}),保留当前上下文`
: "[压缩] 失败,保留当前上下文";
},
unknownTool: "(未知工具)", unknownTool: "(未知工具)",
workRunning: "运行中", workRunning: "运行中",
workDone: "运行完毕", workDone: "运行完毕",
+64 -28
View File
@@ -355,7 +355,7 @@ describe("approvals and events", () => {
const m = createStreamModel(); const m = createStreamModel();
pushMessage(m, abortEvent("aborted by user")); pushMessage(m, abortEvent("aborted by user"));
expect(m.lastAuthFailureMs).toBeNull(); expect(m.lastAuthFailureMs).toBeNull();
const end = requestEnd("auth", "401 invalid x-api-key"); const end = requestEnd("auth", { errorMessage: "401 invalid x-api-key" });
pushMessage(m, end); pushMessage(m, end);
// The recorded time is the event's envelope timestamp (so a reload can compare it // The recorded time is the event's envelope timestamp (so a reload can compare it
// against the Project's credentials-updated time). // against the Project's credentials-updated time).
@@ -375,7 +375,7 @@ describe("approvals and events", () => {
it("a later completed request clears the auth-dead state; a new auth failure re-arms it", () => { it("a later completed request clears the auth-dead state; a new auth failure re-arms it", () => {
const m = createStreamModel(); const m = createStreamModel();
pushMessage(m, requestEnd("auth", "401")); pushMessage(m, requestEnd("auth", { errorMessage: "401" }));
expect(m.lastAuthFailureMs).not.toBeNull(); expect(m.lastAuthFailureMs).not.toBeNull();
// The key was fixed and a request succeeded: the state must not outlive the success. // The key was fixed and a request succeeded: the state must not outlive the success.
pushMessage(m, userText("again")); pushMessage(m, userText("again"));
@@ -383,7 +383,7 @@ describe("approvals and events", () => {
pushMessage(m, requestEnd("completed")); pushMessage(m, requestEnd("completed"));
expect(m.lastAuthFailureMs).toBeNull(); expect(m.lastAuthFailureMs).toBeNull();
// A fresh auth failure re-arms (e.g. the replacement key is wrong too). // A fresh auth failure re-arms (e.g. the replacement key is wrong too).
pushMessage(m, requestEnd("auth", "401")); pushMessage(m, requestEnd("auth", { errorMessage: "401" }));
expect(m.lastAuthFailureMs).not.toBeNull(); expect(m.lastAuthFailureMs).not.toBeNull();
}); });
@@ -394,7 +394,7 @@ describe("approvals and events", () => {
pushMessages(recovered, [ pushMessages(recovered, [
userText("go"), userText("go"),
requestBegin(), requestBegin(),
requestEnd("auth", "401 invalid x-api-key"), requestEnd("auth", { errorMessage: "401 invalid x-api-key" }),
abortEvent("llm request error: 401 invalid x-api-key"), abortEvent("llm request error: 401 invalid x-api-key"),
userText("after fix"), userText("after fix"),
requestBegin(), requestBegin(),
@@ -411,7 +411,7 @@ describe("approvals and events", () => {
requestEnd("completed"), requestEnd("completed"),
userText("later"), userText("later"),
requestBegin(), requestBegin(),
requestEnd("auth", "401 invalid x-api-key"), requestEnd("auth", { errorMessage: "401 invalid x-api-key" }),
abortEvent("llm request error: 401 invalid x-api-key"), abortEvent("llm request error: 401 invalid x-api-key"),
]); ]);
finalizeHistory(dead); finalizeHistory(dead);
@@ -427,7 +427,7 @@ describe("approvals and events", () => {
it("a subagent-origin auth failure does NOT kill the parent session's input", () => { it("a subagent-origin auth failure does NOT kill the parent session's input", () => {
const m = createStreamModel(); const m = createStreamModel();
pushMessage(m, withOrigin(requestEnd("auth", "401"), "child1")); pushMessage(m, withOrigin(requestEnd("auth", { errorMessage: "401" }), "child1"));
// The failure belongs to the child session: its nested model carries the state, the // The failure belongs to the child session: its nested model carries the state, the
// parent composer stays usable (the subagent simply surfaces as failed). // parent composer stays usable (the subagent simply surfaces as failed).
expect(m.lastAuthFailureMs).toBeNull(); expect(m.lastAuthFailureMs).toBeNull();
@@ -460,6 +460,28 @@ describe("approvals and events", () => {
expect(banner).not.toHaveProperty("tokens"); expect(banner).not.toHaveProperty("tokens");
}); });
it("compaction_end carries its RetryDetail share onto the banner (error shown on failure)", () => {
const m = createStreamModel();
pushMessage(
m,
compactionBegin({ reason: "context", mode: "summarize", context: 1000, turns: 3 }),
);
const banner = items(m)[0] as CompactionItem;
pushMessage(
m,
compactionEnd({
reason: "context",
mode: "summarize",
status: "failed",
attempt: 5,
errorMessage: "the response contained no usable summary",
}),
);
expect(banner.running).toBe(false);
expect(banner.status).toBe("failed");
expect(banner.errorMessage).toBe("the response contained no usable summary");
});
it("request_begin/end (normal final state) and main-session session_meta do not render", () => { it("request_begin/end (normal final state) and main-session session_meta do not render", () => {
const m = createStreamModel(); const m = createStreamModel();
pushMessage(m, meta("session-x")); pushMessage(m, meta("session-x"));
@@ -468,10 +490,11 @@ describe("approvals and events", () => {
expect(items(m)).toHaveLength(0); expect(items(m)).toHaveLength(0);
}); });
it("request_end final state timeout/malformed produces a retry notice item (with attempt number); request_begin marks it as resent", () => { it("request_end final state timeout/malformed produces a retry notice item (with the stamped attempt); request_begin marks it as resent", () => {
const m = createStreamModel(); const m = createStreamModel();
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("malformed")); // The core stamps the authoritative ordinal on every retryable request_end.
pushMessage(m, requestEnd("malformed", { attempt: 1 }));
const retry = items(m)[0] as ReconnectItem; const retry = items(m)[0] as ReconnectItem;
expect(retry).toMatchObject({ expect(retry).toMatchObject({
kind: "reconnect", kind: "reconnect",
@@ -482,17 +505,21 @@ describe("approvals and events", () => {
// Retry request sent: the notice is marked as retrying. // Retry request sent: the notice is marked as retrying.
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
expect(retry.retrying).toBe(true); expect(retry.retrying).toBe(true);
// Retry fails again: a second notice, attempt count increments. // Retry fails again: a second notice, carrying the next ordinal.
pushMessage(m, requestEnd("timeout")); pushMessage(m, requestEnd("timeout", { attempt: 2 }));
const retry2 = items(m)[1] as ReconnectItem; const retry2 = items(m)[1] as ReconnectItem;
expect(retry2).toMatchObject({ kind: "reconnect", status: "timeout", attempt: 2 }); expect(retry2).toMatchObject({ kind: "reconnect", status: "timeout", attempt: 2 });
// Retry succeeds: no new entry, the consecutive-failure count resets to 0 — the next round's failure starts back at 1. // Retry succeeds: no new entry; the next round's failure carries attempt 1 again.
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("completed")); pushMessage(m, requestEnd("completed"));
expect(items(m)).toHaveLength(2); expect(items(m)).toHaveLength(2);
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("timeout")); pushMessage(m, requestEnd("timeout", { attempt: 1 }));
expect((items(m)[2] as ReconnectItem).attempt).toBe(1); expect((items(m)[2] as ReconnectItem).attempt).toBe(1);
// A Trace written before the field existed lacks it: rendered as attempt 1.
pushMessage(m, requestBegin());
pushMessage(m, requestEnd("timeout"));
expect((items(m)[3] as ReconnectItem).attempt).toBe(1);
}); });
it("request_end(failed) renders a retry notice too, with its countdown inputs and give-up target", () => { it("request_end(failed) renders a retry notice too, with its countdown inputs and give-up target", () => {
@@ -503,7 +530,10 @@ describe("approvals and events", () => {
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage( pushMessage(
m, m,
requestEnd("failed", "Upstream HTTP/2 stream failed (upstream_http2_stream_error)", 4000), requestEnd("failed", {
errorMessage: "Upstream HTTP/2 stream failed (upstream_http2_stream_error)",
retryInMs: 4000,
}),
111_000, 111_000,
); );
const retry = items(m)[0] as ReconnectItem; const retry = items(m)[0] as ReconnectItem;
@@ -521,39 +551,38 @@ describe("approvals and events", () => {
expect(retry.retrying).toBe(true); expect(retry.retrying).toBe(true);
}); });
it("a mixed ladder keeps counting: failed no longer resets the attempt number mid-run", () => { it("a mixed ladder keeps counting: the stamped ordinal carries across failure kinds", () => {
// `failed` used to fall through to the reset branch, so timeout → failed → timeout // The engine numbers timeout → failed → timeout as one run (all three draw on the same
// renumbered the third attempt back to #1 while the engine was on its third. // budget), and each notice shows the event's own ordinal.
const m = createStreamModel(); const m = createStreamModel();
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("timeout")); pushMessage(m, requestEnd("timeout", { attempt: 1 }));
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("failed", "502 bad gateway")); pushMessage(m, requestEnd("failed", { errorMessage: "502 bad gateway", attempt: 2 }));
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("timeout")); pushMessage(m, requestEnd("timeout", { attempt: 3 }));
expect((items(m) as ReconnectItem[]).map((i) => [i.status, i.attempt])).toEqual([ expect((items(m) as ReconnectItem[]).map((i) => [i.status, i.attempt])).toEqual([
["timeout", 1], ["timeout", 1],
["failed", 2], ["failed", 2],
["timeout", 3], ["timeout", 3],
]); ]);
// A normal finish still resets it: the next run's first failure is #1 again. // After a normal finish the next run's first failure is stamped #1 again.
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("completed")); pushMessage(m, requestEnd("completed"));
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("failed")); pushMessage(m, requestEnd("failed", { attempt: 1 }));
expect((items(m)[3] as ReconnectItem).attempt).toBe(1); expect((items(m)[3] as ReconnectItem).attempt).toBe(1);
}); });
it("request_end(auth) stays out of the ladder: terminal, so no retry notice and the count resets", () => { it("request_end(auth) stays out of the ladder: terminal, so no retry notice", () => {
// The one status the engine does not retry — an item would promise a countdown and a // The one status the engine does not retry — an item would promise a countdown and a
// "Retry now" that will never happen, on top of the composer already being gated. // "Retry now" that will never happen, on top of the composer already being gated.
const m = createStreamModel(); const m = createStreamModel();
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("timeout")); pushMessage(m, requestEnd("timeout", { attempt: 1 }));
pushMessage(m, requestBegin()); pushMessage(m, requestBegin());
pushMessage(m, requestEnd("auth", "401 invalid x-api-key")); pushMessage(m, requestEnd("auth", { errorMessage: "401 invalid x-api-key", attempt: 2 }));
expect((items(m) as ReconnectItem[]).filter((i) => i.kind === "reconnect")).toHaveLength(1); expect((items(m) as ReconnectItem[]).filter((i) => i.kind === "reconnect")).toHaveLength(1);
expect(m.reconnectRun).toBe(0);
}); });
it("retries exhausted: an arriving abort marks the waiting retry notice gaveUp and resets the consecutive-failure count", () => { it("retries exhausted: an arriving abort marks the waiting retry notice gaveUp and resets the consecutive-failure count", () => {
@@ -582,7 +611,14 @@ describe("approvals and events", () => {
// The engine announced a 4s backoff before retry #1; nowMs (the injected client clock) // The engine announced a 4s backoff before retry #1; nowMs (the injected client clock)
// is the countdown anchor — NOT the envelope timestamp, so server clock skew cannot // is the countdown anchor — NOT the envelope timestamp, so server clock skew cannot
// bend the ticker. // bend the ticker.
pushMessage(m, requestEnd("timeout", "403 quota (insufficient_user_quota)", 4000), 111_000); pushMessage(
m,
requestEnd("timeout", {
errorMessage: "403 quota (insufficient_user_quota)",
retryInMs: 4000,
}),
111_000,
);
const item = items(m)[0] as ReconnectItem; const item = items(m)[0] as ReconnectItem;
expect(item).toMatchObject({ expect(item).toMatchObject({
kind: "reconnect", kind: "reconnect",
@@ -608,7 +644,7 @@ describe("approvals and events", () => {
pushMessages(retried, [ pushMessages(retried, [
userText("go"), userText("go"),
requestBegin(), requestBegin(),
requestEnd("timeout", "quota", 30_000), requestEnd("timeout", { errorMessage: "quota", retryInMs: 30_000 }),
requestBegin(), // the engine's retry — replayed immediately after requestBegin(), // the engine's retry — replayed immediately after
requestEnd("completed"), requestEnd("completed"),
]); ]);
@@ -620,7 +656,7 @@ describe("approvals and events", () => {
pushMessages(aborted, [ pushMessages(aborted, [
userText("go"), userText("go"),
requestBegin(), requestBegin(),
requestEnd("timeout", "quota", 30_000), requestEnd("timeout", { errorMessage: "quota", retryInMs: 30_000 }),
abortEvent("aborted during reconnect backoff"), // the user gave up mid-wait abortEvent("aborted during reconnect backoff"), // the user gave up mid-wait
]); ]);
finalizeHistory(aborted); finalizeHistory(aborted);