fix(core,server,web,cli): compaction tolerance, retry parity, and failure classification (#174)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yaowei ZhengandClaude Fable 5 authored and GitHub committed 2026-08-04 17:27:45 +08:00
1 parent 7015fb0153
commit 6f60b242bd
32 files changed
+741 -326

No files matched your search

+152 -101
View File
@@ -183,10 +183,14 @@ export interface ContextEngineDeps {
/** Ceiling (ms) for a single reconnect backoff wait. Defaults to 30000. */
reconnectBackoffMaxMs?: number;
/**
* Maximum retries for a failing compaction request (instead of the shared `maxReconnects`;
* same statuses, shorter budget — see RETRY_STATUSES). Compaction failure is already
* graceful — the original context is kept and compaction retries at the next trigger — so a
* broken request fails fast (~1.75s of backoff). Defaults to 3.
* Maximum retries for a failing compaction request — one budget for every failure kind:
* the transport statuses (see RETRY_STATUSES) and a committed response that isn't a usable
* summary (empty, or tool calls) all draw from it; only `auth` stops without retrying.
* Defaults to the shared `maxReconnects`: a compaction request is an ordinary LLM request
* and deserves the same patience (issue #170 — the earlier tighter budget made a
* struggling provider fail compaction fast, and a session whose every turn re-triggers
* compaction is stuck). Failure stays graceful either way: the original context is kept
* and compaction retries at the next trigger.
*/
compactionMaxReconnects?: number;
/**
@@ -225,17 +229,19 @@ const carriesSteering = (m: OmniMessage): boolean => {
export type CompactAvailability = "ok" | "unsupported" | "empty" | "just_compacted";
/**
* Maximum summarize attempts when the compaction response is rejected as an invalid summary
* (empty extracted text, or the model answered with tool calls — issue #83/#84). Deliberately
* separate from (and larger than) `compactionMaxReconnects`: that cap governs the retryable
* attempts that were never committed (failed/timeout/malformed) and back off exponentially, while a
* rejection is a well-formed committed response — the request itself works, the model just
* didn't produce a summary, so the repaired input is resent immediately with no backoff. And
* since the compaction request keeps the session's toolset (the prefix cache must stay valid,
* see summarizeContext), a model insisting on tools deserves several chances.
* Beyond this many rejected attempts the compaction fails (original context kept).
* Corrective note prepended to the re-sent compaction Prompt after a committed-but-unusable
* response (empty summary, or tool calls — issues #83/#170). The unusable
* response is committed on the live LLM object and can only be *appended* to (rewriting the
* prefix would invalidate the provider's prompt cache at the moment the context is largest —
* the same invariant that pins the toolset, issue #84); without an explicit correction the
* model sees its own bad output as the freshest example and copies it verbatim on every
* retry (issue #170: deepseek-v4-flash kept writing the body after `[/summary]`).
* Exported for unit tests.
*/
const MAX_SUMMARY_REJECTIONS = 5;
export const SUMMARY_RETRY_GUIDANCE =
"Your previous reply was not a usable summary. Reply again with text only, no tool " +
"calls, in exactly this format and nothing after it:\n\n" +
"[summary]put the summary text here...[/summary]";
/** Result of executing one LLM turn (the return value of runTurn). */
interface TurnResult {
@@ -342,9 +348,11 @@ function downgradeCarriedGoalInput(msg: OmniMessage): OmniMessage {
* own way lands here; retrying a genuinely permanent error costs the ladder and ends the same
* way, while aborting a transient one destroys the turn.
*
* Both loops retry the same set. What differs is the budget: compaction runs on
* `compactionMaxReconnects` (shorter than the turn loop's), because a compaction that gives
* up keeps the original context and tries again at the next trigger.
* Both loops retry the same set on the same backoff ladder, and by default with the same
* budget: compaction runs on `compactionMaxReconnects`, which follows `maxReconnects` unless
* set explicitly (issue #170 — a compaction request is an ordinary LLM request and gets the
* turn loop's patience; compaction additionally routes a committed-but-unusable summary
* through the same budget, see summarizeContext).
*/
const RETRY_STATUSES: readonly StopReason[] = ["failed", "timeout", "malformed"];
@@ -406,7 +414,7 @@ export class ContextEngine {
this.maxReconnects = deps.maxReconnects ?? 5;
this.reconnectBackoffMs = deps.reconnectBackoffMs ?? 250;
this.reconnectBackoffMaxMs = deps.reconnectBackoffMaxMs ?? 30_000;
this.compactionMaxReconnects = deps.compactionMaxReconnects ?? 3;
this.compactionMaxReconnects = deps.compactionMaxReconnects ?? this.maxReconnects;
this.llm = deps.llm;
// Session resumption: apply the initial state derived from replay.
const init = deps.initialState;
@@ -619,7 +627,7 @@ export class ContextEngine {
// else — `failed` included — goes to the reconnect loop below.
if (turn.outcome.status === "auth") {
this.pendingCarryOver = this.buildCarryOver(attemptInput, turn);
yield* this.emitAbort(`llm request error: ${turn.outcome.message ?? "unknown"}`);
yield* this.emitAbort(`llm request error: ${turn.outcome.errorMessage ?? "unknown"}`);
return;
}
// Completed normally.
@@ -653,7 +661,7 @@ export class ContextEngine {
turn.outcome.status === "malformed"
? `malformed response failed after ${this.maxReconnects} retries`
: turn.outcome.status === "failed"
? `llm request failed after ${this.maxReconnects} retries: ${turn.outcome.message ?? "unknown"}`
? `llm request failed after ${this.maxReconnects} retries: ${turn.outcome.errorMessage ?? "unknown"}`
: `reconnect failed after ${this.maxReconnects} retries`;
yield* this.emitAbort(reason);
return;
@@ -967,16 +975,21 @@ export class ContextEngine {
// (the errors panel) can learn the real reason (e.g. a quota code). When the
// engine will retry in-run, the planned backoff rides along as retry_in_ms
// (the frontend's live countdown); absent on final failures and completions.
const stopEvt = requestEnd(
outcome.status,
outcome.message,
this.plannedRetryDelayMs(
outcome,
reconnectsSoFar,
this.maxReconnects,
RETRY_STATUSES,
),
const retryInMs = this.plannedRetryDelayMs(
outcome,
reconnectsSoFar,
this.maxReconnects,
RETRY_STATUSES,
);
const stopEvt = requestEnd(outcome.status, {
...(outcome.errorMessage !== undefined ? { errorMessage: outcome.errorMessage } : {}),
// The authoritative attempt ordinal (1-based, within this retry run); a clean
// first-try completion stays unstamped so the common case adds no noise.
...(outcome.status !== "completed" || reconnectsSoFar > 0
? { attempt: reconnectsSoFar + 1 }
: {}),
...(retryInMs !== undefined ? { retryInMs } : {}),
});
queue.push(stopEvt);
await this.write(stopEvt);
break;
@@ -1230,15 +1243,17 @@ export class ContextEngine {
* transcript uncached costs tens of times more (issue #84 — this is why tools are *not*
* omitted and no `tool_choice` override is used). The
* compaction request's streamed output is not pushed to the Human output stream (it emits
* paired compaction events, plus the compaction request's `token_usage` — positioned between
* the two events, so the frontend can count compaction cost into its stats), but it is written
* paired compaction events, plus every attempt's `token_usage` — positioned between the two
* events, so the frontend stats and the server's usage records count the compaction's true
* spend, rejected attempts included), but it is written
* to the old Trace. Compaction succeeds only with a **valid summary** — non-empty extracted
* text and no tool calls in the response. An invalid summary is rejected: any tool calls the
* model issued are answered with synthesized failed outputs (pairing repair, see the loop
* body) and the repaired input is resent immediately, up to MAX_SUMMARY_REJECTIONS attempts,
* then the compaction fails. failed/timeout/malformed reconnect under the compaction-specific
* cap (`compactionMaxReconnects` — a shorter budget than the turn loop's, not a narrower
* set), collapsing to failed once retries are exhausted; on failure/abort, the original
* text and no tool calls in the response. Everything short of that is one kind of failure,
* handled exactly like an ordinary LLM request's (issue #170): an unusable committed
* response (empty summary, or tool calls — answered with synthesized failed outputs and
* retried behind a corrective note, see the loop body) and the transport failures
* (failed/timeout/malformed) all reconnect under the one `compactionMaxReconnects` budget
* (defaulting to the turn loop's budget and ladder — see RETRY_STATUSES); only `auth` stops
* without retrying. Once the budget is exhausted the compaction fails; on failure/abort, the original
* context and Trace index are kept — it does not fall back to discard. The first **committed**
* attempt absorbs `pendingToolOutputs` into the old context's history (issue #85): later
* resends carry only the repairs and the Prompt, and the result's `committed` flag tells
@@ -1271,24 +1286,40 @@ export class ContextEngine {
// clean end — none of those reach the stateful commit). Returned as `committed`: the
// callers' two-case carry rule branches on it.
let committed = false;
// Synthesized outputs answering the latest rejected attempt's tool calls, not yet carried
// Synthesized outputs answering the latest unusable attempt's tool calls, not yet carried
// by a committed request: prepended to the retry input, and stashed as carry-over should
// the compaction be abandoned first (see stashRepairs).
let pendingRepairs: OmniMessage[] = [];
// Two independent retry budgets: retryable attempts (failed/timeout/malformed, never
// committed) follow the compaction-specific reconnect cap with the exponential backoff
// ladder; invalid-summary rejections (committed, well-formed responses that just aren't
// summaries) get the larger dedicated cap and resend immediately — see
// MAX_SUMMARY_REJECTIONS and the rejection branch below.
// One retry budget for every failure: an unusable committed response (empty summary /
// tool calls) counts exactly like a transport failure (issue #170) — same counter, same
// exponential ladder — and only `auth` stops without retrying.
let reconnects = 0;
let rejections = 0;
// The compaction_end event's share of the RetryDetail block (also what the server's
// error record carries): the final attempt ordinal, and the last failure's detail.
let attempts = 0;
let lastError: string | undefined;
for (;;) {
if (signal?.aborted) {
this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "aborted");
yield* this.emitCompactionEnd(
reason,
"summarize",
"aborted",
attempts > 0 ? { attempt: attempts } : undefined,
);
return { status: "aborted", committed };
}
const attempt = await this.runCompactionRequest(input, signal, reconnects);
attempts += 1;
// Every attempt's token_usage is pushed to the Human output stream (already written to
// Trace in runCompactionRequest, so it's only yielded here, never rewritten): the frontend
// stats and the server's usage records then carry the compaction's true spend — failed
// attempts burn real tokens (issue #170), and surfacing only the adopted attempt's usage
// understated the cost center.
if (attempt.usage) yield attempt.usage;
// A committed-but-unusable response (empty summary or tool calls): its retry input must
// be rebuilt below — repairs + corrective note + Prompt — instead of resent unchanged.
let unusable = false;
if (attempt.status === "completed") {
// The attempt was committed by AgentHub, so whatever its input carried — including
// repairs synthesized for a previous rejection — is now in history and must not be
@@ -1306,30 +1337,26 @@ export class ContextEngine {
// live possibility, not just a hallucination (issue #83).
const summaryText = extractSummary(attempt.text);
if (summaryText !== "" && attempt.toolCalls.length === 0) {
// The compaction request's token_usage is pushed to the Human output stream (already
// written to Trace in runCompactionRequest, so here it's only yielded, not rewritten);
// the frontend uses this to count compaction cost into stats and display it on the
// compaction-complete line. Only the adopted attempt's usage is surfaced: rejected
// attempts still feed observeTokenUsage (Session cumulative cost and context
// tracking stay correct), so the displayed compaction cost deliberately understates
// the true spend when retries happened — chosen so the line reflects the attempt
// that produced the summary.
if (attempt.usage) yield attempt.usage;
const summary = userText(buildContextSummaryText(summaryText));
yield* this.emitCompactionEnd(reason, "summarize", "completed");
yield* this.emitCompactionEnd(reason, "summarize", "completed", { attempt: attempts });
await this.startNewContext();
return { status: "completed", summary, committed };
}
// Rejected. Tool calls were never dispatched, yet the assistant turn holding them IS
// Not a summary — one more failed attempt, sharing the reconnect budget below. Tool
// calls were never dispatched, yet the assistant turn holding them IS
// committed on the live LLM object — leaving them unanswered would get every
// subsequent request rejected by the provider (unanswered tool_use, issue #33): the
// exact state this file's other safety nets exist to prevent. Answer each call with a
// synthesized failed output (the same shape executeOne uses), written to Trace so
// resume replays the identical pairing, and prepended to the retry input so the
// provider sees tool_use/tool_result paired. The empty-text rejection needs no repair:
// provider sees tool_use/tool_result paired. The empty-text case needs no repair:
// that committed turn is plain assistant text/thinking, and re-sending the compaction
// Prompt on top of it is structurally sound.
rejections += 1;
unusable = true;
lastError =
attempt.toolCalls.length > 0
? "the response called tools instead of writing a summary"
: "the response contained no usable summary";
pendingRepairs = attempt.toolCalls.map((tc) =>
toolCallOutput({
output: "[tool error] the compaction request expects a summary, not tool calls",
@@ -1338,53 +1365,61 @@ export class ContextEngine {
}),
);
for (const repair of pendingRepairs) await this.write(repair);
// Rebuild from the (shrunken) base rather than appending: everything the rejected
// attempt's input carried is committed, so only the fresh repairs and the Prompt go
// out again.
input = pendingRepairs.length > 0 ? [...pendingRepairs, ...base] : base;
if (rejections >= MAX_SUMMARY_REJECTIONS) {
this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "failed");
return { status: "failed", committed };
}
// A rejection is model behavior, not a transport failure: the request pipeline is
// healthy, so the repaired input is resent immediately — no backoff and no
// retry_in_ms announcement (the rejected attempt's request_end carries status
// completed, for which plannedRetryDelayMs yields nothing). The exponential ladder
// below belongs to transport failures only.
continue;
}
if (attempt.status === "aborted") {
} else if (attempt.status === "aborted") {
this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "aborted");
yield* this.emitCompactionEnd(reason, "summarize", "aborted", { attempt: attempts });
return { status: "aborted", committed };
}
if (attempt.status === "auth") {
// `auth` folds into `failed` here: the compaction event pair keeps its
} else if (attempt.status === "auth") {
// `auth` is the one status that never retries: credentials don't heal on a ladder.
// It folds into `failed` here: the compaction event pair keeps its
// completed/failed/aborted set, the original context is kept, and the host learns
// about the credential problem from the request's own terminal status (a turn-loop
// request will surface it; the compaction request_end is Trace-only).
this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "failed");
yield* this.emitCompactionEnd(reason, "summarize", "failed", {
attempt: attempts,
...(attempt.errorMessage !== undefined ? { errorMessage: attempt.errorMessage } : {}),
});
return { status: "failed", committed };
} else {
// Transport failure: keep its detail as the last error of record.
lastError = attempt.errorMessage;
}
// failed / timeout / malformed: retried via reconnect — none of them is committed by
// AgentHub (case B), so the input (any pending repairs included) is resent unchanged.
// Compaction uses its own, tighter cap (not the shared maxReconnects): a compaction
// that ends up failing keeps the original context and tries again on the next trigger,
// so a short ladder here beats holding the session through the full one.
// One failure path for everything else — unusable summaries and the transport statuses
// (failed / timeout / malformed, never committed by AgentHub) — treated like an
// ordinary LLM request's failures: the same budget (defaulting to the shared
// maxReconnects, issue #170) and the same exponential ladder. An unusable attempt's
// request_end carries status completed, for which no retry_in_ms is announced — the
// backoff wait still happens.
if (reconnects >= this.compactionMaxReconnects) {
this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "failed");
yield* this.emitCompactionEnd(reason, "summarize", "failed", {
attempt: attempts,
...(lastError !== undefined ? { errorMessage: lastError } : {}),
});
return { status: "failed", committed };
}
reconnects += 1;
const ok = await this.backoff(reconnects, signal);
if (!ok) {
this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "aborted");
yield* this.emitCompactionEnd(reason, "summarize", "aborted", { attempt: attempts });
return { status: "aborted", committed };
}
if (unusable) {
// Rebuild from the (shrunken) base rather than appending: everything the unusable
// attempt's input carried is committed — the live object's history can only grow, so
// the retry appends the fresh repairs, a corrective note, and the Prompt. The note
// (written to Trace like the Prompt, and only when a retry actually follows) is what
// breaks the copy-my-own-mistake loop: the model's freshest example is its committed
// bad output, and rewriting history to hide it would invalidate the provider's
// prompt cache (issue #84) — correcting forward is the one cache-safe option
// (issue #170). Transport failures skip this: nothing was committed, so their input
// is resent unchanged (any pending repairs included).
const guidance = userText(SUMMARY_RETRY_GUIDANCE);
await this.write(guidance);
input = [...pendingRepairs, guidance, ...base];
}
}
}
@@ -1426,6 +1461,8 @@ export class ContextEngine {
text: string;
toolCalls: OmniMessage<ToolCallPayload>[];
usage: OmniMessage | null;
/** Error detail (LLMOutcome.errorMessage) on non-completed statuses — becomes compaction_end.error_message when this failure ends the compaction. */
errorMessage?: string;
}> {
// The compaction request is itself an ordinary Request, emitting paired request events —
// written to the (old) Trace only, not pushed to the stream, keeping the compaction process
@@ -1448,19 +1485,32 @@ export class ContextEngine {
// compaction event pair. A rejected summary ends `completed`, for which
// plannedRetryDelayMs yields nothing — rejection resends are immediate (see
// summarizeContext), so no wait is ever announced for them.
await this.write(
requestEnd(
res.value.status,
res.value.message,
this.plannedRetryDelayMs(
res.value,
reconnectsSoFar,
this.compactionMaxReconnects,
RETRY_STATUSES,
),
),
const retryInMs = this.plannedRetryDelayMs(
res.value,
reconnectsSoFar,
this.compactionMaxReconnects,
RETRY_STATUSES,
);
return { status: res.value.status, text, toolCalls, usage };
await this.write(
requestEnd(res.value.status, {
...(res.value.errorMessage !== undefined
? { errorMessage: res.value.errorMessage }
: {}),
// Same stamping rule as the turn loop; for compaction the ordinal counts every
// retry kind (transport and unusable-summary alike share one budget).
...(res.value.status !== "completed" || reconnectsSoFar > 0
? { attempt: reconnectsSoFar + 1 }
: {}),
...(retryInMs !== undefined ? { retryInMs } : {}),
}),
);
return {
status: res.value.status,
text,
toolCalls,
usage,
...(res.value.errorMessage !== undefined ? { errorMessage: res.value.errorMessage } : {}),
};
}
const msg = res.value;
await this.write(msg);
@@ -1512,13 +1562,14 @@ export class ContextEngine {
await this.write(msg);
}
/** Yields and records a compaction stop event (carrying the result status; non-completed means compaction was abandoned). */
/** Yields and records a compaction stop event (carrying the result status — non-completed means compaction was abandoned — plus its share of the RetryDetail block: final attempt ordinal, and the last error detail on failures). */
private async *emitCompactionEnd(
reason: CompactionReason,
mode: CompactionMode,
status: StopReason,
detail?: { attempt?: number; errorMessage?: string },
): AsyncGenerator<OmniMessage> {
const msg = compactionEnd({ reason, mode, status });
const msg = compactionEnd({ reason, mode, status, ...detail });
yield msg;
await this.write(msg);
}
@@ -103,7 +103,9 @@ export function createDescribeImageTool(
if (signal?.aborted) return { stopReason: "aborted" };
if (!outcome || outcome.status !== "completed") {
const detail =
outcome && "message" in outcome && outcome.message ? `: ${outcome.message}` : "";
outcome && "errorMessage" in outcome && outcome.errorMessage
? `: ${outcome.errorMessage}`
: "";
yield delta(
`${streamedAny ? "\n" : ""}Vision model (${describer.modelId}) request ${outcome?.status ?? "failed"}${detail}`,
);
+6 -5
View File
@@ -148,7 +148,7 @@ export interface GenerativeModelParameters {
* `context_engine`;
* - `aborted`: user-initiated interruption — stop and hand back to the user;
* - `failed`: an error the retry classifier did not judge transient (params, etc.) — still
* retried by `context_engine` within the same run (`message` provides the display text).
* retried by `context_engine` within the same run (`errorMessage` provides the display text).
* The classification stays honest — this is reported as `failed`, not relabelled a
* timeout — while the *policy* retries it, because that classifier is an allowlist and a
* gateway phrasing a transient fault its own way lands here;
@@ -162,12 +162,13 @@ export interface GenerativeModelParameters {
export interface LLMOutcome {
status: StopReason;
/**
* Failure detail (`describeError` text): present on `failed` / `auth`, and on `timeout` /
* Error detail (`describeError` text): present on `failed` / `auth`, and on `timeout` /
* `malformed` when a concrete transport/provider error was caught (a plain idle timeout
* has none). Carried onto the `request_end` event so observability (the Cost center's
* errors panel) can show the real reason behind a retried request.
* has none). Carried onto the `request_end` event as `error_message` — one name across
* the internal outcome and the wire — so observability (the Cost center's errors panel)
* can show the real reason behind a retried request.
*/
message?: string;
errorMessage?: string;
}
/**
+5 -5
View File
@@ -1107,7 +1107,7 @@ export class GenerativeModel implements LLMInterface {
try {
uniMessage = mergeOmniToUniMessage(params.newMessages);
} catch (err) {
return { status: "failed", message: describeError(err) };
return { status: "failed", errorMessage: describeError(err) };
}
const translator = new EventTranslator(this.toolCallIds);
@@ -1214,7 +1214,7 @@ export class GenerativeModel implements LLMInterface {
// malformed to reconnect and retry — must not be classified as failed.
outcome = {
status: "malformed",
message: describeError(error),
errorMessage: describeError(error),
};
} else if (isAuthenticationError(error)) {
// Credentials failure: its own terminal status so hosts can tell "update this
@@ -1222,17 +1222,17 @@ export class GenerativeModel implements LLMInterface {
// fixed at Session creation; the credential is read from the current Project
// config on load) apart from a one-off failure. Checked before the retryable
// branch as a belt — isRetryableError itself already refuses auth signals.
outcome = { status: "auth", message: describeError(error) };
outcome = { status: "auth", errorMessage: describeError(error) };
} else if (isRetryableError(error)) {
// Network drop / transient provider rejection -> needs reconnection. The detail
// (e.g. "403 … (insufficient_user_quota)") rides on the outcome so observability
// (request_end -> the Cost center's errors panel) shows the real reason behind a
// retried request, not just "timeout".
outcome = { status: "timeout", message: describeError(error) };
outcome = { status: "timeout", errorMessage: describeError(error) };
} else if ((error as { name?: string })?.name === "AbortError") {
outcome = { status: "aborted" }; // Fallback: an unexpected abort (neither timeout nor user)
} else {
outcome = { status: "failed", message: describeError(error) };
outcome = { status: "failed", errorMessage: describeError(error) };
}
} finally {
clearTimer();
+13 -8
View File
@@ -273,20 +273,21 @@ export function requestBegin(): OmniMessage<RequestBeginPayload> {
/**
* request end event: carries the terminal state (`completed` means this turn was already
* committed to AgentHub), plus — on non-completed statuses — the failure detail (from
* LLMOutcome.message) and, when the engine will retry in-run, the planned backoff wait
* (`retry_in_ms`, rendered by the Web App as a live countdown).
* committed to AgentHub), plus the unified retry detail block (see RequestRetryDetail):
* the error detail (from LLMOutcome.errorMessage), the 1-based attempt ordinal, and — when the
* engine will retry in-run — the planned backoff wait (`retry_in_ms`, rendered by the Web
* App as a live countdown). This builder is the one place the block is stamped.
*/
export function requestEnd(
status: StopReason,
message?: string,
retryInMs?: number,
retry: { errorMessage?: string; attempt?: number; retryInMs?: number } = {},
): OmniMessage<RequestEndPayload> {
return event({
type: "request_end",
status,
...(message !== undefined ? { message } : {}),
...(retryInMs !== undefined ? { retry_in_ms: retryInMs } : {}),
...(retry.errorMessage !== undefined ? { error_message: retry.errorMessage } : {}),
...(retry.attempt !== undefined ? { attempt: retry.attempt } : {}),
...(retry.retryInMs !== undefined ? { retry_in_ms: retry.retryInMs } : {}),
});
}
@@ -306,17 +307,21 @@ export function compactionBegin(args: {
});
}
/** compaction end event: carries the compaction result (non-`completed` means compaction was abandoned and the original context is kept). */
/** compaction end event: carries the compaction result (non-`completed` means compaction was abandoned and the original context is kept), plus its share of the RetryDetail block (final attempt ordinal; last error_message detail on failures). */
export function compactionEnd(args: {
reason: CompactionReason;
mode: CompactionMode;
status: StopReason;
attempt?: number;
errorMessage?: string;
}): OmniMessage<CompactionEndPayload> {
return event({
type: "compaction_end",
reason: args.reason,
mode: args.mode,
status: args.status,
...(args.attempt !== undefined ? { attempt: args.attempt } : {}),
...(args.errorMessage !== undefined ? { error_message: args.errorMessage } : {}),
});
}
@@ -4,7 +4,7 @@
* host — but their spelling belongs here with the rest of the markers so the engine and the
* resume path cannot drift apart (they used to duplicate the `[context_summary]` template).
*/
import { dualFormPatterns, markerBlock, matchDualForm } from "./block.js";
import { dualFormPatterns, markerBlock, matchDualForm, stripMarkerBlocks } from "./block.js";
import { MARKER_TAGS, TRANSCRIPT_TAGS } from "./tags.js";
/** Wraps a compaction summary as the new context's first input: `[context_summary]…[/context_summary]`. */
@@ -12,19 +12,33 @@ export function buildContextSummaryText(summary: string): string {
return markerBlock(MARKER_TAGS.contextSummary, summary);
}
const SUMMARY_PATTERNS = dualFormPatterns(MARKER_TAGS.summary, "([\\s\\S]*?)");
const SUMMARY_PATTERNS = dualFormPatterns(MARKER_TAGS.summary, "([\\s\\S]*?)", "g");
/**
* Extracts the summary the model wrote inside `[summary]…[/summary]` during compaction; when
* the tag is missing, leniently uses the entire output as-is (not treated as a failure).
* Extracts the summary the model wrote during compaction, with a tolerance ladder (issue #170:
* some models — deepseek-v4-flash observed — treat the tags as a "title" and write the body
* outside them):
* 1. the first `[summary]…[/summary]` pair whose content is non-empty once stray summary
* tags are stripped (so a nested `[summary]` line doesn't hide the text after it);
* 2. otherwise, everything left after stripping the (empty) summary blocks and stray tags —
* rescues output shaped `[summary]\n[/summary]\n<the actual summary>`;
* 3. tagless output stays used verbatim. An empty return means the output was genuinely
* empty — the caller treats that as a failed attempt.
* Step 1 preserves the historical result for every healthy output, so resume re-extracting
* from an old Trace reconstructs the same summary the live session used.
*
* The legacy `<summary></summary>` form must stay accepted **indefinitely**:
* `compaction.prompt` is persisted in every existing agent's `system_config.yaml`, so old
* agents keep instructing the model to use the angle-bracket tags.
*/
export function extractSummary(raw: string): string {
const match = matchDualForm(SUMMARY_PATTERNS, raw);
return (match ? match[1]! : raw).trim();
for (const pattern of SUMMARY_PATTERNS) {
for (const match of raw.matchAll(pattern)) {
const inside = stripMarkerBlocks(match[1]!, MARKER_TAGS.summary).trim();
if (inside !== "") return inside;
}
}
return stripMarkerBlocks(raw, MARKER_TAGS.summary).trim();
}
/** One transcribed line inside a turn block: an inner tag, optional attributes, and the body. */
+37 -12
View File
@@ -280,29 +280,47 @@ export interface RequestBeginPayload {
type: "request_begin";
}
export interface RequestEndPayload {
type: "request_end";
/** Terminal state of this Request (reuses the six StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome; `auth` is the credentials-failure signal hosts key on). */
status: StopReason;
/**
* The unified retry/failure detail block shared by `request_end` and `compaction_end` —
* one standard group of fields, stamped by the builders (the way `withOrigin` stamps
* `origin`) rather than accreting as scattered ad-hoc parameters. Every field is optional
* and additive: old Traces replay unchanged.
*/
export interface RetryDetail {
/**
* Failure detail from `LLMOutcome.message`, present only on non-completed statuses: the
* real reason behind a retried/failed Request (e.g. `403 … (insufficient_user_quota)`),
* for observability — the server's error records / Cost center read it here because a
* retried request never produces an abort event. Additive: old Traces without it replay
* unchanged.
* Error detail, one name across the stack (`LLMOutcome.errorMessage` internally): present
* only on non-completed statuses — the real reason behind a retried/failed Request (e.g.
* `403 … (insufficient_user_quota)`), for observability — the server's error records /
* Cost center read it here because a retried request never produces an abort event. Not
* plain `message`: in this protocol "message" means an OmniMessage / model output, and
* this field is neither. (Traces written before the rename carry it as `message`; nothing
* reads that field semantically after the fact, so no dual-read is kept.)
*/
message?: string;
error_message?: string;
/**
* 1-based ordinal of this Request within its retry run — the authoritative retry count
* the CLI/Web display verbatim. Stamped on every non-completed request_end and on a
* completed one that needed retries; absent on a clean first-try completion (the common
* case stays noise-free) and in old Traces.
*/
attempt?: number;
/**
* Planned in-run retry wait (ms) — present ONLY when the engine will retry this failure
* within the same run (status `timeout`/`malformed` with attempts remaining under the
* applicable cap). Computed by the same formula as the actual backoff sleep
* (`reconnectDelayMs`), so the announced wait and the real one cannot drift; the Web App
* renders it as a live countdown to the next attempt. Absent on final failures (an abort
* follows instead) and on completed requests. Additive: old Traces replay unchanged.
* follows instead) and on completed requests.
*/
retry_in_ms?: number;
}
export interface RequestEndPayload extends RetryDetail {
type: "request_end";
/** Terminal state of this Request (reuses the six StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome; `auth` is the credentials-failure signal hosts key on). */
status: StopReason;
}
/** Compaction trigger reason: context threshold / turn-count threshold / user-initiated request. */
export type CompactionReason = "context" | "turns" | "manual";
@@ -328,7 +346,14 @@ export interface CompactionBeginPayload {
turns: number;
}
export interface CompactionEndPayload {
/**
* Inherits the shared RetryDetail block: `attempt` is the final attempt's 1-based ordinal
* (failed attempts and retries included — stamped by summarize mode), `error_message` is
* the last failure's detail (present only when `status` is `failed`), and `retry_in_ms` is
* never stamped here (compaction retries are announced on the compaction request's own
* request_end, which is Trace-only).
*/
export interface CompactionEndPayload extends RetryDetail {
type: "compaction_end";
reason: CompactionReason;
mode: CompactionMode;
+12 -10
View File
@@ -156,18 +156,20 @@ Skills are reusable instruction packages at <app_data_dir>/agents/<agent_id>/age
- Session ID: {{SESSION_ID}}`;
/**
* Built-in default compaction Prompt (summarize mode): tells the model that after
* compaction the raw transcript is no longer visible and the
* summary is the only record, so it must include everything needed to continue the task,
* and no tools may be called while writing the summary.
* Built-in default compaction Prompt (summarize mode): tells the model the summary will
* replace the transcript as its only record (so it must include everything needed to
* continue the task) and that no tools may be called. The format is shown as a concrete
* example rather than described in prose — some models treat the tags as a "title" and
* write the body after the closing tag (issue #170); extraction salvages that shape, but
* generating it right beats repairing it. Persisted per-agent in system_config.yaml —
* existing agents keep their stored prompt.
*/
export const DEFAULT_COMPACTION_PROMPT =
"You have a partial transcript of the task above. Write a summary of it wrapped in " +
"`[summary][/summary]` tags. This summary will replace the transcript: in the next " +
"context window the raw transcript above will no longer be visible and this summary " +
"will be its only record, so include everything needed to continue the task — the " +
"original request, current state, next steps, and any learnings. Do not call any " +
"tools while writing the summary; respond with text only.";
"Summarize the task transcript above. The summary will replace the transcript as its " +
"only record, so include everything needed to continue the task — the original request, " +
"current state, next steps, and any learnings. Do not call any tools; reply with text " +
"only, in exactly this format and nothing after it:\n\n" +
"[summary]put the summary text here...[/summary]";
/**
* Default built-in system tools: file reading/editing/writing first, then bash execution
+170 -58
View File
@@ -49,7 +49,7 @@ import type {
LLMInterface,
LLMOutcome,
} from "../src/interfaces.js";
import { ContextEngine } from "../src/engine/context-engine.js";
import { ContextEngine, SUMMARY_RETRY_GUIDANCE } from "../src/engine/context-engine.js";
import type { CompactionSettings } from "../src/engine/context-engine.js";
import { GenerativeModel } from "../src/llm/index.js";
import type { UniConfig, UniEvent, UniMessage } from "@prismshadow/agenthub";
@@ -78,7 +78,7 @@ class ScriptedLLM implements LLMInterface {
this.calls.push(params.newMessages);
const next = this.responses.shift();
if (!next) {
return { status: "failed", message: `${this.label}: no scripted response` };
return { status: "failed", errorMessage: `${this.label}: no scripted response` };
}
for (const msg of next.messages) yield msg;
return next.outcome ?? { status: "completed" };
@@ -308,7 +308,7 @@ describe("context compaction", () => {
messages: [toolCall({ name: "t", arguments: "{}", toolCallId: "c1" }), usage(150, 150)],
},
// Compaction request fails on the one status no ladder can fix (a rejected credential).
{ messages: [], outcome: { status: "auth", message: "auth error" } },
{ messages: [], outcome: { status: "auth", errorMessage: "auth error" } },
// Original context is kept: the task continues, tool outputs feed back into the old instance as usual (context usage keeps growing).
{ messages: [assistantText("finished on old context"), usage(190, 340)] },
// Second trigger (context still over the limit) -> retries compaction at the boundary, this time succeeding.
@@ -415,12 +415,55 @@ describe("context compaction", () => {
const out = await collect(engine.run([userText("go")], { approve: allowAll }));
const events = compactionEvents(out);
expect(events[1]).toMatchObject({ type: "compaction_end", status: "failed" });
// The end event reports the attempts spent (issue #170's cost-center row message).
expect(events[1]).toMatchObject({
type: "compaction_end",
status: "failed",
attempt: 2,
});
// The retry resends the original input (tool results + prompt; here there are no tool results, just the prompt).
expect(llm1.calls).toHaveLength(3);
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
});
it("compaction transport retries default to the shared maxReconnects budget", async () => {
// Issue #170: without an explicit compactionMaxReconnects, the compaction request gets
// the same retry budget as the turn loop (here 4) — under the old tighter default of 3
// this script would have failed before reaching the 5th, succeeding attempt.
const failing = (n: number): ScriptedResponse => ({
messages: [],
outcome: { status: "failed", errorMessage: `blip ${n}` },
});
const llm1 = new ScriptedLLM(
[
{ messages: [assistantText("answer"), usage(150, 150)] },
failing(1),
failing(2),
failing(3),
failing(4),
{ messages: [assistantText("[summary]after the shared ladder[/summary]"), usage(30, 300)] },
],
"llm1",
);
const llm2 = new ScriptedLLM([], "llm2");
const engine = new ContextEngine({
llm: llm1,
environment: fakeEnvironment,
compaction: settings(),
createLLM: () => llm2,
maxReconnects: 4,
reconnectBackoffMs: 1,
});
const out = await collect(engine.run([userText("go")], { approve: allowAll }));
expect(compactionEvents(out)[1]).toMatchObject({
type: "compaction_end",
status: "completed",
attempt: 5,
});
expect(llm1.calls).toHaveLength(6);
});
it("a failed compaction request takes the ladder and can recover on a later attempt", async () => {
// The compaction loop retries the same statuses the turn loop does: `failed` is where a
// transient fault lands whenever the classifier doesn't recognize the gateway's wording,
@@ -429,7 +472,7 @@ describe("context compaction", () => {
const llm1 = new ScriptedLLM(
[
{ messages: [assistantText("answer"), usage(150, 150)] },
{ messages: [], outcome: { status: "failed", message: "502 upstream" } },
{ messages: [], outcome: { status: "failed", errorMessage: "502 upstream" } },
{ messages: [assistantText("[summary]recovered[/summary]")] },
],
"llm1",
@@ -455,11 +498,11 @@ describe("context compaction", () => {
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
});
it("the compaction loop uses its own cap, not the shared maxReconnects ladder", async () => {
// Compaction failure has graceful semantics (the original context is kept, compaction
// retries on the next trigger), so it fails fast: with compactionMaxReconnects=2, the
// compaction request runs 1+2 attempts and gives up — even though maxReconnects is far
// larger and would have kept the session stalled through the full exponential ladder.
it("an explicit compactionMaxReconnects overrides the shared maxReconnects budget", async () => {
// The dep still allows a caller to bound compaction retries independently: with
// compactionMaxReconnects=2, the compaction request runs 1+2 attempts and gives up —
// even though maxReconnects is far larger (the default merely *follows* maxReconnects,
// it does not override an explicit value).
const llm1 = new ScriptedLLM(
[
{ messages: [assistantText("answer"), usage(150, 150)] },
@@ -495,26 +538,31 @@ describe("context compaction", () => {
// The Trace-written compaction request_ends announce the planned backoff under the
// COMPACTION cap (base 1ms: 1, then 2), with none on the final failure — these events
// are never streamed, so the value lands in the Trace record only.
const retryPlans = recorded
.filter((m) => (m.payload as { type?: string }).type === "request_end")
.map((m) => (m.payload as { retry_in_ms?: number }).retry_in_ms);
const compactionEnds = recorded.filter(
(m) => (m.payload as { type?: string }).type === "request_end",
);
const retryPlans = compactionEnds.map(
(m) => (m.payload as { retry_in_ms?: number }).retry_in_ms,
);
expect(retryPlans).toEqual([undefined, 1, 2, undefined]);
// The attempt ordinal rides the same events: the turn's clean completion stays
// unstamped, the three compaction failures count 1..3.
const attempts = compactionEnds.map((m) => (m.payload as { attempt?: number }).attempt);
expect(attempts).toEqual([undefined, 1, 2, 3]);
});
it("an empty compaction response (thinking only, no text) is rejected: 5 attempts, then failed with the context kept", async () => {
// Issue #83: the compaction request completes but yields no text. Committing the empty
// summary would discard the whole context and lose the task state — the response is
// rejected and retried under the dedicated rejection cap (5 attempts, #84), then the
// compaction fails while the original context and Trace file stay current.
it("an empty compaction response (thinking only, no text) is retried like any failure: 5 attempts, then failed with the context kept", async () => {
// Issue #83/#170: the compaction request completes but yields no text. Committing the
// empty summary would discard the whole context and lose the task state — the response
// counts as one more failed attempt on the unified reconnect budget, and once the budget
// is exhausted the compaction fails while the original context and Trace file stay current.
const empty = (n: number): ScriptedResponse => ({
messages: [thinkingMessage(`pondering, attempt ${n}, no text`)],
});
const llm1 = new ScriptedLLM(
[
{ messages: [assistantText("answer one"), usage(150, 150)] },
// Five completed-but-empty compaction attempts: the dedicated cap allows exactly 5
// rejections. compactionMaxReconnects is 1 here on purpose — rejections must NOT
// consume the transport reconnect budget, or the loop would stop after 2 attempts.
// Five completed-but-empty compaction attempt: 4 retries of budget allow exactly 5.
empty(1),
empty(2),
empty(3),
@@ -538,7 +586,7 @@ describe("context compaction", () => {
created += 1;
return new ScriptedLLM([], "llm2");
},
compactionMaxReconnects: 1,
compactionMaxReconnects: 4,
reconnectBackoffMs: 1,
});
const oldPath = trace.currentPath();
@@ -546,12 +594,17 @@ describe("context compaction", () => {
const out1 = await collect(engine.run([userText("task one")], { approve: allowAll }));
// Exactly one event pair, ending failed — an empty summary is never a completed compaction.
// The end event reports the attempts spent (issue #170).
const events = compactionEvents(out1);
expect(
events.map((e) => `${e.type}:${(e as Partial<CompactionEndPayload>).status ?? ""}`),
).toEqual(["compaction_begin:", "compaction_end:failed"]);
// No rejected attempt's token_usage is surfaced (only a successful compaction yields
// its usage between the paired events).
expect(events[1]).toMatchObject({
attempt: 5,
error_message: "the response contained no usable summary",
});
// These scripted attempts carry no token_usage of their own, so none appears between the
// paired events (attempts that do carry usage surface it — see the 5th-attempt test).
const types1 = payloadTypes(out1);
const between = out1.slice(
types1.indexOf("compaction_begin") + 1,
@@ -560,14 +613,16 @@ describe("context compaction", () => {
expect(between.filter((m) => (m.payload as { type?: string }).type === "token_usage")).toEqual(
[],
);
// Turn + exactly five compaction attempts (the 5th rejection exhausts the cap, no 6th
// request), each resending the prompt unchanged — an empty rejection needs no repair.
// Turn + exactly five compaction attempts (the 5th exhausts the budget, no 6th
// request); each retry leads with the corrective note, then the prompt — an empty
// response needs no repair.
expect(llm1.calls).toHaveLength(6);
for (let i = 1; i <= 5; i += 1) {
expect(llm1.calls[i]!.map(textOf)).toEqual(["COMPACT NOW"]);
expect(llm1.calls[1]!.map(textOf)).toEqual(["COMPACT NOW"]);
for (let i = 2; i <= 5; i += 1) {
expect(llm1.calls[i]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
}
// Rejection resends are immediate, never announced: no compaction request_end carries a
// retry_in_ms (they all end `completed`, unlike the transport ladder's timeout ends).
// An unusable attempt's request_end carries status `completed`, for which no retry_in_ms
// is ever announced (the backoff wait still ran between attempts).
const rejectionPlans = (await readTrace(oldPath))
.filter((m) => (m.payload as { type?: string }).type === "request_end")
.map((m) => (m.payload as { retry_in_ms?: number }).retry_in_ms);
@@ -585,9 +640,9 @@ describe("context compaction", () => {
expect(await readdir(dirname(oldPath))).toEqual(["sess_empty_001.jsonl"]);
});
it("tool-calling rejections exhaust the 5-attempt cap: every call is paired, and the next ordinary turn stays clean", async () => {
it("tool-calling responses exhaust the retry budget: every call is paired, and the next ordinary turn stays clean", async () => {
// A tool-calling response is not a summary — even when it also carries plausible summary
// text — but its assistant turn IS committed on the live LLM object. Each rejection's
// text — but its assistant turn IS committed on the live LLM object. Each such attempt's
// calls are answered with synthesized failed outputs (written to Trace, prepended to the
// retried input), so no tool_use ever dangles: after the compaction fails, the same
// object must still serve ordinary turns with a well-formed history (#84 review).
@@ -621,7 +676,7 @@ describe("context compaction", () => {
created += 1;
return new ScriptedLLM([], "llm2");
},
compactionMaxReconnects: 1,
compactionMaxReconnects: 4,
reconnectBackoffMs: 1,
});
const oldPath = trace.currentPath();
@@ -635,12 +690,18 @@ describe("context compaction", () => {
expect(payloadTypes(out)).not.toContain("tool_call_output");
expect(created).toBe(0);
// The end event reports the attempts spent and the last failure (issue #170's
// cost-center row message and the frontend banner detail).
expect(events[1]).toMatchObject({
attempt: 5,
error_message: "the response called tools instead of writing a summary",
});
// Five attempts; from the second on, the input leads with the repair answering the
// previous rejection's call, then re-issues the prompt.
// previous attempt's call, then the corrective note, then the prompt.
expect(llm1.calls).toHaveLength(6);
for (let attempt = 2; attempt <= 5; attempt += 1) {
const retry = llm1.calls[attempt]!;
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text"]);
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text", "text"]);
const repair = retry[0]!.payload as {
tool_call_id: string;
output: string;
@@ -651,7 +712,8 @@ describe("context compaction", () => {
"[tool error] the compaction request expects a summary, not tool calls",
);
expect(repair.stop_reason).toBe("failed");
expect(textOf(retry[1]!)).toBe("COMPACT NOW");
expect(textOf(retry[1]!)).toBe(SUMMARY_RETRY_GUIDANCE);
expect(textOf(retry[2]!)).toBe("COMPACT NOW");
}
// All five synthesized repairs are written to the (old) Trace for replay to mirror.
@@ -681,8 +743,9 @@ describe("context compaction", () => {
});
it("a valid summary on the 5th and final allowed attempt completes the compaction", async () => {
// Counting pin for the rejection cap: four rejected attempts spend the budget but the 5th
// attempt still gets its chance — a valid summary there succeeds (5 rejections would fail).
// Counting pin for the unified budget: four failed attempts spend the 4 retries, but the
// 5th (last allowed) attempt still gets its chance — a valid summary there succeeds
// (a 5th failure would exhaust the budget instead).
const llm1 = new ScriptedLLM(
[
{ messages: [assistantText("answer"), usage(150, 150)] },
@@ -706,7 +769,7 @@ describe("context compaction", () => {
factoryTokens = tokens;
return llm2;
},
maxReconnects: 1,
compactionMaxReconnects: 4,
reconnectBackoffMs: 1,
});
@@ -714,8 +777,10 @@ describe("context compaction", () => {
const events = compactionEvents(out);
expect(events).toHaveLength(2);
expect(events[1]).toMatchObject({ type: "compaction_end", status: "completed" });
// Only the adopted attempt's token_usage is surfaced between the paired events; rejected
// attempts' usage still feeds the Session cumulative totals (see below) but is not shown.
// Every attempt's token_usage is surfaced between the paired events — rejected attempts
// burn real tokens, and hiding them understated the cost center (issue #170). Here the
// 1st (rejected) and 5th (adopted) attempts carry usage.
expect(events[1]).toMatchObject({ attempt: 5 });
const types = payloadTypes(out);
const between = out.slice(
types.indexOf("compaction_begin") + 1,
@@ -724,8 +789,9 @@ describe("context compaction", () => {
const usageBetween = between.filter(
(m) => (m.payload as { type?: string }).type === "token_usage",
);
expect(usageBetween).toHaveLength(1);
expect((usageBetween[0]!.payload as TokenUsagePayload).request.total).toBe(170);
expect(usageBetween.map((m) => (m.payload as TokenUsagePayload).request.total)).toEqual([
160, 170,
]);
// Session cumulative tokens carried into the new instance include the rejected attempts' usage.
expect(factoryTokens).toMatchObject({ total: 480 });
@@ -778,10 +844,11 @@ describe("context compaction", () => {
status: "completed",
});
// The retried input answers the rejected attempt's call first, then re-issues the prompt.
// The retried input answers the rejected attempt's call first, then carries the
// corrective note and re-issues the prompt.
expect(llm1.calls).toHaveLength(3);
const retry = llm1.calls[2]!;
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text"]);
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text", "text"]);
const repair = retry[0]!.payload as {
tool_call_id: string;
output: string;
@@ -792,7 +859,8 @@ describe("context compaction", () => {
"[tool error] the compaction request expects a summary, not tool calls",
);
expect(repair.stop_reason).toBe("failed");
expect(textOf(retry[1]!)).toBe("COMPACT NOW");
expect(textOf(retry[1]!)).toBe(SUMMARY_RETRY_GUIDANCE);
expect(textOf(retry[2]!)).toBe("COMPACT NOW");
// The repair belongs to the compaction dialogue: written to the old Trace (so replay
// mirrors the pairing), never pushed to the output stream.
@@ -897,6 +965,7 @@ describe("context compaction", () => {
created += 1;
return new ScriptedLLM([], "llm2");
},
compactionMaxReconnects: 4,
reconnectBackoffMs: 1,
});
const oldPath = trace.currentPath();
@@ -908,15 +977,16 @@ describe("context compaction", () => {
// is the whole story.
expect(payloadTypes(out1)).not.toContain("abort");
expect(llm1.calls).toHaveLength(6);
// Attempt 1 folds the turn's tool output in; attempt 2 carries the repair + Prompt but
// NOT the absorbed output; attempts 3-5 are Prompt-only.
// Attempt 1 folds the turn's tool output in; attempt 2 carries the repair + note + Prompt
// but NOT the absorbed output; attempts 3-5 are note + Prompt.
expect(payloadTypes(llm1.calls[1]!)).toEqual(["tool_call_output", "text"]);
expect((llm1.calls[1]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("ct");
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text"]);
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text", "text"]);
expect((llm1.calls[2]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("c1");
expect(textOf(llm1.calls[2]![1]!)).toBe("COMPACT NOW");
expect(textOf(llm1.calls[2]![1]!)).toBe(SUMMARY_RETRY_GUIDANCE);
expect(textOf(llm1.calls[2]![2]!)).toBe("COMPACT NOW");
for (let attempt = 3; attempt <= 5; attempt += 1) {
expect(llm1.calls[attempt]!.map(textOf)).toEqual(["COMPACT NOW"]);
expect(llm1.calls[attempt]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
}
// Original context kept: no LLM swap, no Trace rotation.
expect(created).toBe(0);
@@ -955,16 +1025,17 @@ describe("context compaction", () => {
environment: fakeEnvironment,
compaction: settings(),
createLLM: () => new ScriptedLLM([], "llm2"),
compactionMaxReconnects: 4,
reconnectBackoffMs: 1,
});
const out = await collect(engine.run([userText("go")], { approve: allowAll }));
expect(compactionEvents(out)[1]).toMatchObject({ type: "compaction_end", status: "failed" });
expect(llm1.calls).toHaveLength(7);
// Attempt 1 folds the outputs; attempts 2-5 are Prompt-only (absorbed by the first commit).
// Attempt 1 folds the outputs; attempts 2-5 are note + Prompt (absorbed by the first commit).
expect(payloadTypes(llm1.calls[1]!)).toEqual(["tool_call_output", "text"]);
for (let attempt = 2; attempt <= 5; attempt += 1) {
expect(llm1.calls[attempt]!.map(textOf)).toEqual(["COMPACT NOW"]);
expect(llm1.calls[attempt]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
}
// The continuation input is the repair answering c5 — nothing else: no absorbed outputs,
// no prompt.
@@ -1044,8 +1115,8 @@ describe("context compaction", () => {
expect(events[1]).toMatchObject({ type: "compaction_end", status: "aborted" });
expect(payloadTypes(out)).toContain("abort");
expect(llm1.calls).toHaveLength(3);
// Attempt 2 carried the repair + Prompt (not the absorbed outputs).
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text"]);
// Attempt 2 carried the repair + note + Prompt (not the absorbed outputs).
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text", "text"]);
expect((llm1.calls[2]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("c1");
// The next run leads with the still-unanswered repair; the absorbed turn outputs are
@@ -1255,6 +1326,46 @@ describe("context compaction", () => {
);
});
it("rescues a summary written outside the tags (issue #170) on the first attempt", async () => {
// deepseek-v4-flash writes `[summary]\n[/summary]` as a "title" and the body after the
// closing tag. Extraction salvages the outside text, so the very first attempt succeeds —
// no rejection loop, no burned retries.
const llm1 = new ScriptedLLM(
[
{ messages: [assistantText("answer"), usage(150, 150)] },
{
messages: [
assistantText("[summary]\n[/summary]\nRescued body outside the tags."),
usage(40, 200),
],
},
],
"llm1",
);
const llm2 = new ScriptedLLM([{ messages: [assistantText("fresh"), usage(10, 210)] }], "llm2");
const engine = new ContextEngine({
llm: llm1,
environment: fakeEnvironment,
compaction: settings(),
createLLM: () => llm2,
reconnectBackoffMs: 1,
});
const out = await collect(engine.run([userText("task one")], { approve: allowAll }));
expect(compactionEvents(out)[1]).toMatchObject({
type: "compaction_end",
status: "completed",
attempt: 1,
});
// No rejection retry: the turn request plus a single compaction attempt.
expect(llm1.calls).toHaveLength(2);
await collect(engine.run([userText("task two")], { approve: allowAll }));
expect(llm2.calls[0]!.map(textOf)).toEqual([
"[context_summary]\nRescued body outside the tags.\n[/context_summary]",
"task two",
]);
});
it("manual compaction skips threshold checks and reuses the same flow", async () => {
const llm1 = new ScriptedLLM(
[
@@ -1356,6 +1467,7 @@ describe("context compaction", () => {
environment: fakeEnvironment,
compaction: settings(),
createLLM: () => new ScriptedLLM([], "llm2"),
compactionMaxReconnects: 4,
reconnectBackoffMs: 1,
});
await collect(engine.run([userText("start")], { approve: allowAll }));
@@ -1368,11 +1480,11 @@ describe("context compaction", () => {
const out = await collect(engine.compact());
expect(compactionEvents(out)[1]).toMatchObject({ type: "compaction_end", status: "failed" });
expect(llm1.calls).toHaveLength(6);
// Attempt 1 folded the carry-over in (and committed it); later attempts resend the Prompt
// alone — the absorbed carry-over never again.
// Attempt 1 folded the carry-over in (and committed it); later attempts resend the note +
// Prompt — the absorbed carry-over never again.
expect(llm1.calls[1]!.map(textOf)).toEqual(["pending question", "COMPACT NOW"]);
for (let attempt = 2; attempt <= 5; attempt += 1) {
expect(llm1.calls[attempt]!.map(textOf)).toEqual(["COMPACT NOW"]);
expect(llm1.calls[attempt]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
}
// The carry-over is consumed at the seam — the next run leads with the stashed repair
+1 -1
View File
@@ -177,7 +177,7 @@ describe("describe_image (the text-only-model variant of read_image)", () => {
it("vision model request failure: failed with the status and message", async () => {
await writeFile(path.join(tmp, "a.png"), PNG_1X1);
const { llm } = fakeLLM("", { status: "failed", message: "401 unauthorized" });
const { llm } = fakeLLM("", { status: "failed", errorMessage: "401 unauthorized" });
const { result, text } = await run({ source: "a.png" }, tmp, {
modelId: "vis-1",
createLLM: () => llm,
+21 -11
View File
@@ -684,7 +684,7 @@ describe("ContextEngine ReAct loop (mock LLM, approve callback)", () => {
yield partialText("start", "");
yield partialText("delta", "half a thought");
// `auth`: the one LLM status that still exits straight to the flatten path.
return { status: "auth", message: "boom" };
return { status: "auth", errorMessage: "boom" };
}
yield assistantText("ok");
yield tokenUsage(emptyTokenCounts(), {
@@ -1646,7 +1646,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
toolCallId: "tc-broken",
stopReason: "malformed",
});
return { status: "malformed", message: "incomplete stream" };
return { status: "malformed", errorMessage: "incomplete stream" };
}
yield assistantText("done");
yield tokenUsage(emptyTokenCounts(), {
@@ -1799,7 +1799,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
async *streamGenerate(params) {
calls += 1;
inputs.push(params.newMessages);
return { status: "failed", message: "400 unknown parameter: max_output_tokens" };
return { status: "failed", errorMessage: "400 unknown parameter: max_output_tokens" };
},
};
const environment = new Environment({
@@ -1884,7 +1884,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
// eslint-disable-next-line require-yield
async *streamGenerate() {
calls += 1;
return { status: "auth", message: "401 invalid x-api-key" };
return { status: "auth", errorMessage: "401 invalid x-api-key" };
},
};
const environment = new Environment({
@@ -1898,7 +1898,9 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
// The request's own terminal status is the host signal (streams to the web).
const end = all.find((m) => (m.payload as { type?: string }).type === "request_end");
expect((end!.payload as { status?: string }).status).toBe("auth");
expect((end!.payload as { message?: string }).message).toBe("401 invalid x-api-key");
expect((end!.payload as { error_message?: string }).error_message).toBe(
"401 invalid x-api-key",
);
// No planned retry is announced for a terminal failure.
expect((end!.payload as { retry_in_ms?: number }).retry_in_ms).toBeUndefined();
const abort = all.find((m) => (m.payload as { type?: string }).type === "abort");
@@ -2037,7 +2039,10 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
if (calls === 1) {
// A retryable provider rejection: the detail must reach observability via the
// event — a retried request never produces an abort to carry it.
return { status: "timeout", message: "403 quota exceeded (insufficient_user_quota)" };
return {
status: "timeout",
errorMessage: "403 quota exceeded (insufficient_user_quota)",
};
}
yield assistantText("ok");
yield tokenUsage(emptyTokenCounts(), {
@@ -2057,16 +2062,21 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
const all = await collectRun(engine, [userText("go")], allowAll);
const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end") as {
payload: { status?: string; message?: string; retry_in_ms?: number };
payload: { status?: string; error_message?: string; attempt?: number; retry_in_ms?: number };
}[];
expect(ends).toHaveLength(2);
expect(ends[0]!.payload.status).toBe("timeout");
expect(ends[0]!.payload.message).toBe("403 quota exceeded (insufficient_user_quota)");
// The engine will retry: the planned backoff is announced (base 0 here -> 0ms).
expect(ends[0]!.payload.error_message).toBe("403 quota exceeded (insufficient_user_quota)");
// The engine will retry: the planned backoff is announced (base 0 here -> 0ms), and the
// authoritative attempt ordinal is stamped (this was the run's 1st request).
expect(ends[0]!.payload.retry_in_ms).toBe(0);
expect(ends[0]!.payload.attempt).toBe(1);
expect(ends[1]!.payload.status).toBe("completed");
expect(ends[1]!.payload.message).toBeUndefined();
expect(ends[1]!.payload.error_message).toBeUndefined();
expect(ends[1]!.payload.retry_in_ms).toBeUndefined();
// A completion that needed retries still carries its ordinal ("recovered on attempt 2");
// only a clean first-try completion stays unstamped.
expect(ends[1]!.payload.attempt).toBe(2);
});
it("request_end announces the planned backoff (retry_in_ms) from the shared ladder; absent once the cap is reached", async () => {
@@ -2178,7 +2188,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
// exhausted-retries test below).
yield thinkingMessage("half-thought", "failed");
yield assistantText("half-text", "failed");
return { status: "auth", message: "boom" };
return { status: "auth", errorMessage: "boom" };
}
yield assistantText("ok");
yield tokenUsage(emptyTokenCounts(), {
+4 -4
View File
@@ -1573,7 +1573,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
model.streamGenerate({ newMessages: [userText("go")] }),
);
expect(outcome.status).toBe("malformed");
expect(outcome.message).toContain("Unexpected token");
expect(outcome.errorMessage).toContain("Unexpected token");
const complete = messages.find((m) => typeOf(m) === "text");
expect((complete!.payload as TextPayload).text).toBe("hi");
expect((complete!.payload as TextPayload).stop_reason).toBe("malformed");
@@ -1634,7 +1634,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
// 401 = credentials failure: its own stop reason so hosts can gate input until the
// model's API key is updated (same engine behavior as failed).
expect(outcome.status).toBe("auth");
expect(outcome.message).toContain("invalid api key");
expect(outcome.errorMessage).toContain("invalid api key");
expect(messages.map(typeOf)).not.toContain("token_usage");
// A genuinely non-retryable parameter error stays a plain failure.
@@ -1663,7 +1663,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
model.streamGenerate({ newMessages: [userText("go")] }),
);
expect(outcome.status).toBe("auth");
expect(outcome.message).toContain("subscription key invalid");
expect(outcome.errorMessage).toContain("subscription key invalid");
expect(messages.map(typeOf)).not.toContain("token_usage");
});
@@ -1681,7 +1681,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
expect(outcome.status).toBe("timeout");
// The real reason rides on the outcome (request_end -> the Cost center's errors panel
// shows it): a retried quota rejection must not surface as a bare "timeout".
expect(outcome.message).toContain("insufficient_user_quota");
expect(outcome.errorMessage).toContain("insufficient_user_quota");
expect(messages.map(typeOf)).not.toContain("token_usage");
});
+17
View File
@@ -132,6 +132,23 @@ describe("engine blocks ([turn_aborted] / [turn_retried] / [context_summary] / [
expect(extractSummary("[summary]a[/summary] <summary>b</summary>")).toBe("a");
});
it("extractSummary rescues content the model wrote outside the tags (issue #170)", () => {
// deepseek-v4-flash writes the tags as a "title" and the body after the closing tag.
expect(extractSummary("[summary]\n[/summary]\nThe actual summary body.")).toBe(
"The actual summary body.",
);
expect(extractSummary("[summary]\n[summary][/summary]\nBody after a doubled open tag.")).toBe(
"Body after a doubled open tag.",
);
// A stray opening tag inside the pair doesn't hide the text that follows it.
expect(extractSummary("[summary]\n[summary]\nBody inside.[/summary]")).toBe("Body inside.");
// An empty current-form pair doesn't shadow a non-empty legacy pair.
expect(extractSummary("[summary][/summary]<summary>legacy body</summary>")).toBe("legacy body");
// Genuinely empty output still extracts to "" — the engine treats that as a failed attempt.
expect(extractSummary("[summary][/summary]")).toBe("");
expect(extractSummary("[summary]\n[/summary]")).toBe("");
});
it("turn blocks round-trip through unwrapSyntheticBlock, both forms, single-level", () => {
const lines = [transcribeUserInput("go"), transcribeToolCall("read_file", "t1", "{}")];
const aborted = buildTurnAbortedBlock(lines);
+1 -1
View File
@@ -97,7 +97,7 @@ describe("session-title", () => {
assistantText("partial"),
tokenUsage(emptyTokenCounts(), { cache_read: 0, cache_write: 0, output: 1, total: 1 }),
],
{ status: "failed", message: "401" },
{ status: "failed", errorMessage: "401" },
),
{ userText: "u", assistantText: "a" },
);