fix(core,server,web,cli): compaction tolerance, retry parity, and failure classification (#174)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7015fb0153
commit
6f60b242bd
32 files changed
+741
-326
No files matched your search
@@ -183,10 +183,14 @@ export interface ContextEngineDeps {
|
||||
/** Ceiling (ms) for a single reconnect backoff wait. Defaults to 30000. */
|
||||
reconnectBackoffMaxMs?: number;
|
||||
/**
|
||||
* Maximum retries for a failing compaction request (instead of the shared `maxReconnects`;
|
||||
* same statuses, shorter budget — see RETRY_STATUSES). Compaction failure is already
|
||||
* graceful — the original context is kept and compaction retries at the next trigger — so a
|
||||
* broken request fails fast (~1.75s of backoff). Defaults to 3.
|
||||
* Maximum retries for a failing compaction request — one budget for every failure kind:
|
||||
* the transport statuses (see RETRY_STATUSES) and a committed response that isn't a usable
|
||||
* summary (empty, or tool calls) all draw from it; only `auth` stops without retrying.
|
||||
* Defaults to the shared `maxReconnects`: a compaction request is an ordinary LLM request
|
||||
* and deserves the same patience (issue #170 — the earlier tighter budget made a
|
||||
* struggling provider fail compaction fast, and a session whose every turn re-triggers
|
||||
* compaction is stuck). Failure stays graceful either way: the original context is kept
|
||||
* and compaction retries at the next trigger.
|
||||
*/
|
||||
compactionMaxReconnects?: number;
|
||||
/**
|
||||
@@ -225,17 +229,19 @@ const carriesSteering = (m: OmniMessage): boolean => {
|
||||
export type CompactAvailability = "ok" | "unsupported" | "empty" | "just_compacted";
|
||||
|
||||
/**
|
||||
* Maximum summarize attempts when the compaction response is rejected as an invalid summary
|
||||
* (empty extracted text, or the model answered with tool calls — issue #83/#84). Deliberately
|
||||
* separate from (and larger than) `compactionMaxReconnects`: that cap governs the retryable
|
||||
* attempts that were never committed (failed/timeout/malformed) and back off exponentially, while a
|
||||
* rejection is a well-formed committed response — the request itself works, the model just
|
||||
* didn't produce a summary, so the repaired input is resent immediately with no backoff. And
|
||||
* since the compaction request keeps the session's toolset (the prefix cache must stay valid,
|
||||
* see summarizeContext), a model insisting on tools deserves several chances.
|
||||
* Beyond this many rejected attempts the compaction fails (original context kept).
|
||||
* Corrective note prepended to the re-sent compaction Prompt after a committed-but-unusable
|
||||
* response (empty summary, or tool calls — issues #83/#170). The unusable
|
||||
* response is committed on the live LLM object and can only be *appended* to (rewriting the
|
||||
* prefix would invalidate the provider's prompt cache at the moment the context is largest —
|
||||
* the same invariant that pins the toolset, issue #84); without an explicit correction the
|
||||
* model sees its own bad output as the freshest example and copies it verbatim on every
|
||||
* retry (issue #170: deepseek-v4-flash kept writing the body after `[/summary]`).
|
||||
* Exported for unit tests.
|
||||
*/
|
||||
const MAX_SUMMARY_REJECTIONS = 5;
|
||||
export const SUMMARY_RETRY_GUIDANCE =
|
||||
"Your previous reply was not a usable summary. Reply again with text only, no tool " +
|
||||
"calls, in exactly this format and nothing after it:\n\n" +
|
||||
"[summary]put the summary text here...[/summary]";
|
||||
|
||||
/** Result of executing one LLM turn (the return value of runTurn). */
|
||||
interface TurnResult {
|
||||
@@ -342,9 +348,11 @@ function downgradeCarriedGoalInput(msg: OmniMessage): OmniMessage {
|
||||
* own way lands here; retrying a genuinely permanent error costs the ladder and ends the same
|
||||
* way, while aborting a transient one destroys the turn.
|
||||
*
|
||||
* Both loops retry the same set. What differs is the budget: compaction runs on
|
||||
* `compactionMaxReconnects` (shorter than the turn loop's), because a compaction that gives
|
||||
* up keeps the original context and tries again at the next trigger.
|
||||
* Both loops retry the same set on the same backoff ladder, and by default with the same
|
||||
* budget: compaction runs on `compactionMaxReconnects`, which follows `maxReconnects` unless
|
||||
* set explicitly (issue #170 — a compaction request is an ordinary LLM request and gets the
|
||||
* turn loop's patience; compaction additionally routes a committed-but-unusable summary
|
||||
* through the same budget, see summarizeContext).
|
||||
*/
|
||||
const RETRY_STATUSES: readonly StopReason[] = ["failed", "timeout", "malformed"];
|
||||
|
||||
@@ -406,7 +414,7 @@ export class ContextEngine {
|
||||
this.maxReconnects = deps.maxReconnects ?? 5;
|
||||
this.reconnectBackoffMs = deps.reconnectBackoffMs ?? 250;
|
||||
this.reconnectBackoffMaxMs = deps.reconnectBackoffMaxMs ?? 30_000;
|
||||
this.compactionMaxReconnects = deps.compactionMaxReconnects ?? 3;
|
||||
this.compactionMaxReconnects = deps.compactionMaxReconnects ?? this.maxReconnects;
|
||||
this.llm = deps.llm;
|
||||
// Session resumption: apply the initial state derived from replay.
|
||||
const init = deps.initialState;
|
||||
@@ -619,7 +627,7 @@ export class ContextEngine {
|
||||
// else — `failed` included — goes to the reconnect loop below.
|
||||
if (turn.outcome.status === "auth") {
|
||||
this.pendingCarryOver = this.buildCarryOver(attemptInput, turn);
|
||||
yield* this.emitAbort(`llm request error: ${turn.outcome.message ?? "unknown"}`);
|
||||
yield* this.emitAbort(`llm request error: ${turn.outcome.errorMessage ?? "unknown"}`);
|
||||
return;
|
||||
}
|
||||
// Completed normally.
|
||||
@@ -653,7 +661,7 @@ export class ContextEngine {
|
||||
turn.outcome.status === "malformed"
|
||||
? `malformed response failed after ${this.maxReconnects} retries`
|
||||
: turn.outcome.status === "failed"
|
||||
? `llm request failed after ${this.maxReconnects} retries: ${turn.outcome.message ?? "unknown"}`
|
||||
? `llm request failed after ${this.maxReconnects} retries: ${turn.outcome.errorMessage ?? "unknown"}`
|
||||
: `reconnect failed after ${this.maxReconnects} retries`;
|
||||
yield* this.emitAbort(reason);
|
||||
return;
|
||||
@@ -967,16 +975,21 @@ export class ContextEngine {
|
||||
// (the errors panel) can learn the real reason (e.g. a quota code). When the
|
||||
// engine will retry in-run, the planned backoff rides along as retry_in_ms
|
||||
// (the frontend's live countdown); absent on final failures and completions.
|
||||
const stopEvt = requestEnd(
|
||||
outcome.status,
|
||||
outcome.message,
|
||||
this.plannedRetryDelayMs(
|
||||
outcome,
|
||||
reconnectsSoFar,
|
||||
this.maxReconnects,
|
||||
RETRY_STATUSES,
|
||||
),
|
||||
const retryInMs = this.plannedRetryDelayMs(
|
||||
outcome,
|
||||
reconnectsSoFar,
|
||||
this.maxReconnects,
|
||||
RETRY_STATUSES,
|
||||
);
|
||||
const stopEvt = requestEnd(outcome.status, {
|
||||
...(outcome.errorMessage !== undefined ? { errorMessage: outcome.errorMessage } : {}),
|
||||
// The authoritative attempt ordinal (1-based, within this retry run); a clean
|
||||
// first-try completion stays unstamped so the common case adds no noise.
|
||||
...(outcome.status !== "completed" || reconnectsSoFar > 0
|
||||
? { attempt: reconnectsSoFar + 1 }
|
||||
: {}),
|
||||
...(retryInMs !== undefined ? { retryInMs } : {}),
|
||||
});
|
||||
queue.push(stopEvt);
|
||||
await this.write(stopEvt);
|
||||
break;
|
||||
@@ -1230,15 +1243,17 @@ export class ContextEngine {
|
||||
* transcript uncached costs tens of times more (issue #84 — this is why tools are *not*
|
||||
* omitted and no `tool_choice` override is used). The
|
||||
* compaction request's streamed output is not pushed to the Human output stream (it emits
|
||||
* paired compaction events, plus the compaction request's `token_usage` — positioned between
|
||||
* the two events, so the frontend can count compaction cost into its stats), but it is written
|
||||
* paired compaction events, plus every attempt's `token_usage` — positioned between the two
|
||||
* events, so the frontend stats and the server's usage records count the compaction's true
|
||||
* spend, rejected attempts included), but it is written
|
||||
* to the old Trace. Compaction succeeds only with a **valid summary** — non-empty extracted
|
||||
* text and no tool calls in the response. An invalid summary is rejected: any tool calls the
|
||||
* model issued are answered with synthesized failed outputs (pairing repair, see the loop
|
||||
* body) and the repaired input is resent immediately, up to MAX_SUMMARY_REJECTIONS attempts,
|
||||
* then the compaction fails. failed/timeout/malformed reconnect under the compaction-specific
|
||||
* cap (`compactionMaxReconnects` — a shorter budget than the turn loop's, not a narrower
|
||||
* set), collapsing to failed once retries are exhausted; on failure/abort, the original
|
||||
* text and no tool calls in the response. Everything short of that is one kind of failure,
|
||||
* handled exactly like an ordinary LLM request's (issue #170): an unusable committed
|
||||
* response (empty summary, or tool calls — answered with synthesized failed outputs and
|
||||
* retried behind a corrective note, see the loop body) and the transport failures
|
||||
* (failed/timeout/malformed) all reconnect under the one `compactionMaxReconnects` budget
|
||||
* (defaulting to the turn loop's budget and ladder — see RETRY_STATUSES); only `auth` stops
|
||||
* without retrying. Once the budget is exhausted the compaction fails; on failure/abort, the original
|
||||
* context and Trace index are kept — it does not fall back to discard. The first **committed**
|
||||
* attempt absorbs `pendingToolOutputs` into the old context's history (issue #85): later
|
||||
* resends carry only the repairs and the Prompt, and the result's `committed` flag tells
|
||||
@@ -1271,24 +1286,40 @@ export class ContextEngine {
|
||||
// clean end — none of those reach the stateful commit). Returned as `committed`: the
|
||||
// callers' two-case carry rule branches on it.
|
||||
let committed = false;
|
||||
// Synthesized outputs answering the latest rejected attempt's tool calls, not yet carried
|
||||
// Synthesized outputs answering the latest unusable attempt's tool calls, not yet carried
|
||||
// by a committed request: prepended to the retry input, and stashed as carry-over should
|
||||
// the compaction be abandoned first (see stashRepairs).
|
||||
let pendingRepairs: OmniMessage[] = [];
|
||||
// Two independent retry budgets: retryable attempts (failed/timeout/malformed, never
|
||||
// committed) follow the compaction-specific reconnect cap with the exponential backoff
|
||||
// ladder; invalid-summary rejections (committed, well-formed responses that just aren't
|
||||
// summaries) get the larger dedicated cap and resend immediately — see
|
||||
// MAX_SUMMARY_REJECTIONS and the rejection branch below.
|
||||
// One retry budget for every failure: an unusable committed response (empty summary /
|
||||
// tool calls) counts exactly like a transport failure (issue #170) — same counter, same
|
||||
// exponential ladder — and only `auth` stops without retrying.
|
||||
let reconnects = 0;
|
||||
let rejections = 0;
|
||||
// The compaction_end event's share of the RetryDetail block (also what the server's
|
||||
// error record carries): the final attempt ordinal, and the last failure's detail.
|
||||
let attempts = 0;
|
||||
let lastError: string | undefined;
|
||||
for (;;) {
|
||||
if (signal?.aborted) {
|
||||
this.stashRepairs(pendingRepairs);
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "aborted");
|
||||
yield* this.emitCompactionEnd(
|
||||
reason,
|
||||
"summarize",
|
||||
"aborted",
|
||||
attempts > 0 ? { attempt: attempts } : undefined,
|
||||
);
|
||||
return { status: "aborted", committed };
|
||||
}
|
||||
const attempt = await this.runCompactionRequest(input, signal, reconnects);
|
||||
attempts += 1;
|
||||
// Every attempt's token_usage is pushed to the Human output stream (already written to
|
||||
// Trace in runCompactionRequest, so it's only yielded here, never rewritten): the frontend
|
||||
// stats and the server's usage records then carry the compaction's true spend — failed
|
||||
// attempts burn real tokens (issue #170), and surfacing only the adopted attempt's usage
|
||||
// understated the cost center.
|
||||
if (attempt.usage) yield attempt.usage;
|
||||
// A committed-but-unusable response (empty summary or tool calls): its retry input must
|
||||
// be rebuilt below — repairs + corrective note + Prompt — instead of resent unchanged.
|
||||
let unusable = false;
|
||||
if (attempt.status === "completed") {
|
||||
// The attempt was committed by AgentHub, so whatever its input carried — including
|
||||
// repairs synthesized for a previous rejection — is now in history and must not be
|
||||
@@ -1306,30 +1337,26 @@ export class ContextEngine {
|
||||
// live possibility, not just a hallucination (issue #83).
|
||||
const summaryText = extractSummary(attempt.text);
|
||||
if (summaryText !== "" && attempt.toolCalls.length === 0) {
|
||||
// The compaction request's token_usage is pushed to the Human output stream (already
|
||||
// written to Trace in runCompactionRequest, so here it's only yielded, not rewritten);
|
||||
// the frontend uses this to count compaction cost into stats and display it on the
|
||||
// compaction-complete line. Only the adopted attempt's usage is surfaced: rejected
|
||||
// attempts still feed observeTokenUsage (Session cumulative cost and context
|
||||
// tracking stay correct), so the displayed compaction cost deliberately understates
|
||||
// the true spend when retries happened — chosen so the line reflects the attempt
|
||||
// that produced the summary.
|
||||
if (attempt.usage) yield attempt.usage;
|
||||
const summary = userText(buildContextSummaryText(summaryText));
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "completed");
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "completed", { attempt: attempts });
|
||||
await this.startNewContext();
|
||||
return { status: "completed", summary, committed };
|
||||
}
|
||||
// Rejected. Tool calls were never dispatched, yet the assistant turn holding them IS
|
||||
// Not a summary — one more failed attempt, sharing the reconnect budget below. Tool
|
||||
// calls were never dispatched, yet the assistant turn holding them IS
|
||||
// committed on the live LLM object — leaving them unanswered would get every
|
||||
// subsequent request rejected by the provider (unanswered tool_use, issue #33): the
|
||||
// exact state this file's other safety nets exist to prevent. Answer each call with a
|
||||
// synthesized failed output (the same shape executeOne uses), written to Trace so
|
||||
// resume replays the identical pairing, and prepended to the retry input so the
|
||||
// provider sees tool_use/tool_result paired. The empty-text rejection needs no repair:
|
||||
// provider sees tool_use/tool_result paired. The empty-text case needs no repair:
|
||||
// that committed turn is plain assistant text/thinking, and re-sending the compaction
|
||||
// Prompt on top of it is structurally sound.
|
||||
rejections += 1;
|
||||
unusable = true;
|
||||
lastError =
|
||||
attempt.toolCalls.length > 0
|
||||
? "the response called tools instead of writing a summary"
|
||||
: "the response contained no usable summary";
|
||||
pendingRepairs = attempt.toolCalls.map((tc) =>
|
||||
toolCallOutput({
|
||||
output: "[tool error] the compaction request expects a summary, not tool calls",
|
||||
@@ -1338,53 +1365,61 @@ export class ContextEngine {
|
||||
}),
|
||||
);
|
||||
for (const repair of pendingRepairs) await this.write(repair);
|
||||
// Rebuild from the (shrunken) base rather than appending: everything the rejected
|
||||
// attempt's input carried is committed, so only the fresh repairs and the Prompt go
|
||||
// out again.
|
||||
input = pendingRepairs.length > 0 ? [...pendingRepairs, ...base] : base;
|
||||
if (rejections >= MAX_SUMMARY_REJECTIONS) {
|
||||
this.stashRepairs(pendingRepairs);
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "failed");
|
||||
return { status: "failed", committed };
|
||||
}
|
||||
// A rejection is model behavior, not a transport failure: the request pipeline is
|
||||
// healthy, so the repaired input is resent immediately — no backoff and no
|
||||
// retry_in_ms announcement (the rejected attempt's request_end carries status
|
||||
// completed, for which plannedRetryDelayMs yields nothing). The exponential ladder
|
||||
// below belongs to transport failures only.
|
||||
continue;
|
||||
}
|
||||
if (attempt.status === "aborted") {
|
||||
} else if (attempt.status === "aborted") {
|
||||
this.stashRepairs(pendingRepairs);
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "aborted");
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "aborted", { attempt: attempts });
|
||||
return { status: "aborted", committed };
|
||||
}
|
||||
if (attempt.status === "auth") {
|
||||
// `auth` folds into `failed` here: the compaction event pair keeps its
|
||||
} else if (attempt.status === "auth") {
|
||||
// `auth` is the one status that never retries: credentials don't heal on a ladder.
|
||||
// It folds into `failed` here: the compaction event pair keeps its
|
||||
// completed/failed/aborted set, the original context is kept, and the host learns
|
||||
// about the credential problem from the request's own terminal status (a turn-loop
|
||||
// request will surface it; the compaction request_end is Trace-only).
|
||||
this.stashRepairs(pendingRepairs);
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "failed");
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "failed", {
|
||||
attempt: attempts,
|
||||
...(attempt.errorMessage !== undefined ? { errorMessage: attempt.errorMessage } : {}),
|
||||
});
|
||||
return { status: "failed", committed };
|
||||
} else {
|
||||
// Transport failure: keep its detail as the last error of record.
|
||||
lastError = attempt.errorMessage;
|
||||
}
|
||||
// failed / timeout / malformed: retried via reconnect — none of them is committed by
|
||||
// AgentHub (case B), so the input (any pending repairs included) is resent unchanged.
|
||||
// Compaction uses its own, tighter cap (not the shared maxReconnects): a compaction
|
||||
// that ends up failing keeps the original context and tries again on the next trigger,
|
||||
// so a short ladder here beats holding the session through the full one.
|
||||
// One failure path for everything else — unusable summaries and the transport statuses
|
||||
// (failed / timeout / malformed, never committed by AgentHub) — treated like an
|
||||
// ordinary LLM request's failures: the same budget (defaulting to the shared
|
||||
// maxReconnects, issue #170) and the same exponential ladder. An unusable attempt's
|
||||
// request_end carries status completed, for which no retry_in_ms is announced — the
|
||||
// backoff wait still happens.
|
||||
if (reconnects >= this.compactionMaxReconnects) {
|
||||
this.stashRepairs(pendingRepairs);
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "failed");
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "failed", {
|
||||
attempt: attempts,
|
||||
...(lastError !== undefined ? { errorMessage: lastError } : {}),
|
||||
});
|
||||
return { status: "failed", committed };
|
||||
}
|
||||
reconnects += 1;
|
||||
const ok = await this.backoff(reconnects, signal);
|
||||
if (!ok) {
|
||||
this.stashRepairs(pendingRepairs);
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "aborted");
|
||||
yield* this.emitCompactionEnd(reason, "summarize", "aborted", { attempt: attempts });
|
||||
return { status: "aborted", committed };
|
||||
}
|
||||
if (unusable) {
|
||||
// Rebuild from the (shrunken) base rather than appending: everything the unusable
|
||||
// attempt's input carried is committed — the live object's history can only grow, so
|
||||
// the retry appends the fresh repairs, a corrective note, and the Prompt. The note
|
||||
// (written to Trace like the Prompt, and only when a retry actually follows) is what
|
||||
// breaks the copy-my-own-mistake loop: the model's freshest example is its committed
|
||||
// bad output, and rewriting history to hide it would invalidate the provider's
|
||||
// prompt cache (issue #84) — correcting forward is the one cache-safe option
|
||||
// (issue #170). Transport failures skip this: nothing was committed, so their input
|
||||
// is resent unchanged (any pending repairs included).
|
||||
const guidance = userText(SUMMARY_RETRY_GUIDANCE);
|
||||
await this.write(guidance);
|
||||
input = [...pendingRepairs, guidance, ...base];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1426,6 +1461,8 @@ export class ContextEngine {
|
||||
text: string;
|
||||
toolCalls: OmniMessage<ToolCallPayload>[];
|
||||
usage: OmniMessage | null;
|
||||
/** Error detail (LLMOutcome.errorMessage) on non-completed statuses — becomes compaction_end.error_message when this failure ends the compaction. */
|
||||
errorMessage?: string;
|
||||
}> {
|
||||
// The compaction request is itself an ordinary Request, emitting paired request events —
|
||||
// written to the (old) Trace only, not pushed to the stream, keeping the compaction process
|
||||
@@ -1448,19 +1485,32 @@ export class ContextEngine {
|
||||
// compaction event pair. A rejected summary ends `completed`, for which
|
||||
// plannedRetryDelayMs yields nothing — rejection resends are immediate (see
|
||||
// summarizeContext), so no wait is ever announced for them.
|
||||
await this.write(
|
||||
requestEnd(
|
||||
res.value.status,
|
||||
res.value.message,
|
||||
this.plannedRetryDelayMs(
|
||||
res.value,
|
||||
reconnectsSoFar,
|
||||
this.compactionMaxReconnects,
|
||||
RETRY_STATUSES,
|
||||
),
|
||||
),
|
||||
const retryInMs = this.plannedRetryDelayMs(
|
||||
res.value,
|
||||
reconnectsSoFar,
|
||||
this.compactionMaxReconnects,
|
||||
RETRY_STATUSES,
|
||||
);
|
||||
return { status: res.value.status, text, toolCalls, usage };
|
||||
await this.write(
|
||||
requestEnd(res.value.status, {
|
||||
...(res.value.errorMessage !== undefined
|
||||
? { errorMessage: res.value.errorMessage }
|
||||
: {}),
|
||||
// Same stamping rule as the turn loop; for compaction the ordinal counts every
|
||||
// retry kind (transport and unusable-summary alike share one budget).
|
||||
...(res.value.status !== "completed" || reconnectsSoFar > 0
|
||||
? { attempt: reconnectsSoFar + 1 }
|
||||
: {}),
|
||||
...(retryInMs !== undefined ? { retryInMs } : {}),
|
||||
}),
|
||||
);
|
||||
return {
|
||||
status: res.value.status,
|
||||
text,
|
||||
toolCalls,
|
||||
usage,
|
||||
...(res.value.errorMessage !== undefined ? { errorMessage: res.value.errorMessage } : {}),
|
||||
};
|
||||
}
|
||||
const msg = res.value;
|
||||
await this.write(msg);
|
||||
@@ -1512,13 +1562,14 @@ export class ContextEngine {
|
||||
await this.write(msg);
|
||||
}
|
||||
|
||||
/** Yields and records a compaction stop event (carrying the result status; non-completed means compaction was abandoned). */
|
||||
/** Yields and records a compaction stop event (carrying the result status — non-completed means compaction was abandoned — plus its share of the RetryDetail block: final attempt ordinal, and the last error detail on failures). */
|
||||
private async *emitCompactionEnd(
|
||||
reason: CompactionReason,
|
||||
mode: CompactionMode,
|
||||
status: StopReason,
|
||||
detail?: { attempt?: number; errorMessage?: string },
|
||||
): AsyncGenerator<OmniMessage> {
|
||||
const msg = compactionEnd({ reason, mode, status });
|
||||
const msg = compactionEnd({ reason, mode, status, ...detail });
|
||||
yield msg;
|
||||
await this.write(msg);
|
||||
}
|
||||
|
||||
@@ -103,7 +103,9 @@ export function createDescribeImageTool(
|
||||
if (signal?.aborted) return { stopReason: "aborted" };
|
||||
if (!outcome || outcome.status !== "completed") {
|
||||
const detail =
|
||||
outcome && "message" in outcome && outcome.message ? `: ${outcome.message}` : "";
|
||||
outcome && "errorMessage" in outcome && outcome.errorMessage
|
||||
? `: ${outcome.errorMessage}`
|
||||
: "";
|
||||
yield delta(
|
||||
`${streamedAny ? "\n" : ""}Vision model (${describer.modelId}) request ${outcome?.status ?? "failed"}${detail}`,
|
||||
);
|
||||
|
||||
@@ -148,7 +148,7 @@ export interface GenerativeModelParameters {
|
||||
* `context_engine`;
|
||||
* - `aborted`: user-initiated interruption — stop and hand back to the user;
|
||||
* - `failed`: an error the retry classifier did not judge transient (params, etc.) — still
|
||||
* retried by `context_engine` within the same run (`message` provides the display text).
|
||||
* retried by `context_engine` within the same run (`errorMessage` provides the display text).
|
||||
* The classification stays honest — this is reported as `failed`, not relabelled a
|
||||
* timeout — while the *policy* retries it, because that classifier is an allowlist and a
|
||||
* gateway phrasing a transient fault its own way lands here;
|
||||
@@ -162,12 +162,13 @@ export interface GenerativeModelParameters {
|
||||
export interface LLMOutcome {
|
||||
status: StopReason;
|
||||
/**
|
||||
* Failure detail (`describeError` text): present on `failed` / `auth`, and on `timeout` /
|
||||
* Error detail (`describeError` text): present on `failed` / `auth`, and on `timeout` /
|
||||
* `malformed` when a concrete transport/provider error was caught (a plain idle timeout
|
||||
* has none). Carried onto the `request_end` event so observability (the Cost center's
|
||||
* errors panel) can show the real reason behind a retried request.
|
||||
* has none). Carried onto the `request_end` event as `error_message` — one name across
|
||||
* the internal outcome and the wire — so observability (the Cost center's errors panel)
|
||||
* can show the real reason behind a retried request.
|
||||
*/
|
||||
message?: string;
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1107,7 +1107,7 @@ export class GenerativeModel implements LLMInterface {
|
||||
try {
|
||||
uniMessage = mergeOmniToUniMessage(params.newMessages);
|
||||
} catch (err) {
|
||||
return { status: "failed", message: describeError(err) };
|
||||
return { status: "failed", errorMessage: describeError(err) };
|
||||
}
|
||||
|
||||
const translator = new EventTranslator(this.toolCallIds);
|
||||
@@ -1214,7 +1214,7 @@ export class GenerativeModel implements LLMInterface {
|
||||
// malformed to reconnect and retry — must not be classified as failed.
|
||||
outcome = {
|
||||
status: "malformed",
|
||||
message: describeError(error),
|
||||
errorMessage: describeError(error),
|
||||
};
|
||||
} else if (isAuthenticationError(error)) {
|
||||
// Credentials failure: its own terminal status so hosts can tell "update this
|
||||
@@ -1222,17 +1222,17 @@ export class GenerativeModel implements LLMInterface {
|
||||
// fixed at Session creation; the credential is read from the current Project
|
||||
// config on load) apart from a one-off failure. Checked before the retryable
|
||||
// branch as a belt — isRetryableError itself already refuses auth signals.
|
||||
outcome = { status: "auth", message: describeError(error) };
|
||||
outcome = { status: "auth", errorMessage: describeError(error) };
|
||||
} else if (isRetryableError(error)) {
|
||||
// Network drop / transient provider rejection -> needs reconnection. The detail
|
||||
// (e.g. "403 … (insufficient_user_quota)") rides on the outcome so observability
|
||||
// (request_end -> the Cost center's errors panel) shows the real reason behind a
|
||||
// retried request, not just "timeout".
|
||||
outcome = { status: "timeout", message: describeError(error) };
|
||||
outcome = { status: "timeout", errorMessage: describeError(error) };
|
||||
} else if ((error as { name?: string })?.name === "AbortError") {
|
||||
outcome = { status: "aborted" }; // Fallback: an unexpected abort (neither timeout nor user)
|
||||
} else {
|
||||
outcome = { status: "failed", message: describeError(error) };
|
||||
outcome = { status: "failed", errorMessage: describeError(error) };
|
||||
}
|
||||
} finally {
|
||||
clearTimer();
|
||||
|
||||
@@ -273,20 +273,21 @@ export function requestBegin(): OmniMessage<RequestBeginPayload> {
|
||||
|
||||
/**
|
||||
* request end event: carries the terminal state (`completed` means this turn was already
|
||||
* committed to AgentHub), plus — on non-completed statuses — the failure detail (from
|
||||
* LLMOutcome.message) and, when the engine will retry in-run, the planned backoff wait
|
||||
* (`retry_in_ms`, rendered by the Web App as a live countdown).
|
||||
* committed to AgentHub), plus the unified retry detail block (see RequestRetryDetail):
|
||||
* the error detail (from LLMOutcome.errorMessage), the 1-based attempt ordinal, and — when the
|
||||
* engine will retry in-run — the planned backoff wait (`retry_in_ms`, rendered by the Web
|
||||
* App as a live countdown). This builder is the one place the block is stamped.
|
||||
*/
|
||||
export function requestEnd(
|
||||
status: StopReason,
|
||||
message?: string,
|
||||
retryInMs?: number,
|
||||
retry: { errorMessage?: string; attempt?: number; retryInMs?: number } = {},
|
||||
): OmniMessage<RequestEndPayload> {
|
||||
return event({
|
||||
type: "request_end",
|
||||
status,
|
||||
...(message !== undefined ? { message } : {}),
|
||||
...(retryInMs !== undefined ? { retry_in_ms: retryInMs } : {}),
|
||||
...(retry.errorMessage !== undefined ? { error_message: retry.errorMessage } : {}),
|
||||
...(retry.attempt !== undefined ? { attempt: retry.attempt } : {}),
|
||||
...(retry.retryInMs !== undefined ? { retry_in_ms: retry.retryInMs } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -306,17 +307,21 @@ export function compactionBegin(args: {
|
||||
});
|
||||
}
|
||||
|
||||
/** compaction end event: carries the compaction result (non-`completed` means compaction was abandoned and the original context is kept). */
|
||||
/** compaction end event: carries the compaction result (non-`completed` means compaction was abandoned and the original context is kept), plus its share of the RetryDetail block (final attempt ordinal; last error_message detail on failures). */
|
||||
export function compactionEnd(args: {
|
||||
reason: CompactionReason;
|
||||
mode: CompactionMode;
|
||||
status: StopReason;
|
||||
attempt?: number;
|
||||
errorMessage?: string;
|
||||
}): OmniMessage<CompactionEndPayload> {
|
||||
return event({
|
||||
type: "compaction_end",
|
||||
reason: args.reason,
|
||||
mode: args.mode,
|
||||
status: args.status,
|
||||
...(args.attempt !== undefined ? { attempt: args.attempt } : {}),
|
||||
...(args.errorMessage !== undefined ? { error_message: args.errorMessage } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* host — but their spelling belongs here with the rest of the markers so the engine and the
|
||||
* resume path cannot drift apart (they used to duplicate the `[context_summary]` template).
|
||||
*/
|
||||
import { dualFormPatterns, markerBlock, matchDualForm } from "./block.js";
|
||||
import { dualFormPatterns, markerBlock, matchDualForm, stripMarkerBlocks } from "./block.js";
|
||||
import { MARKER_TAGS, TRANSCRIPT_TAGS } from "./tags.js";
|
||||
|
||||
/** Wraps a compaction summary as the new context's first input: `[context_summary]…[/context_summary]`. */
|
||||
@@ -12,19 +12,33 @@ export function buildContextSummaryText(summary: string): string {
|
||||
return markerBlock(MARKER_TAGS.contextSummary, summary);
|
||||
}
|
||||
|
||||
const SUMMARY_PATTERNS = dualFormPatterns(MARKER_TAGS.summary, "([\\s\\S]*?)");
|
||||
const SUMMARY_PATTERNS = dualFormPatterns(MARKER_TAGS.summary, "([\\s\\S]*?)", "g");
|
||||
|
||||
/**
|
||||
* Extracts the summary the model wrote inside `[summary]…[/summary]` during compaction; when
|
||||
* the tag is missing, leniently uses the entire output as-is (not treated as a failure).
|
||||
* Extracts the summary the model wrote during compaction, with a tolerance ladder (issue #170:
|
||||
* some models — deepseek-v4-flash observed — treat the tags as a "title" and write the body
|
||||
* outside them):
|
||||
* 1. the first `[summary]…[/summary]` pair whose content is non-empty once stray summary
|
||||
* tags are stripped (so a nested `[summary]` line doesn't hide the text after it);
|
||||
* 2. otherwise, everything left after stripping the (empty) summary blocks and stray tags —
|
||||
* rescues output shaped `[summary]\n[/summary]\n<the actual summary>`;
|
||||
* 3. tagless output stays used verbatim. An empty return means the output was genuinely
|
||||
* empty — the caller treats that as a failed attempt.
|
||||
* Step 1 preserves the historical result for every healthy output, so resume re-extracting
|
||||
* from an old Trace reconstructs the same summary the live session used.
|
||||
*
|
||||
* The legacy `<summary></summary>` form must stay accepted **indefinitely**:
|
||||
* `compaction.prompt` is persisted in every existing agent's `system_config.yaml`, so old
|
||||
* agents keep instructing the model to use the angle-bracket tags.
|
||||
*/
|
||||
export function extractSummary(raw: string): string {
|
||||
const match = matchDualForm(SUMMARY_PATTERNS, raw);
|
||||
return (match ? match[1]! : raw).trim();
|
||||
for (const pattern of SUMMARY_PATTERNS) {
|
||||
for (const match of raw.matchAll(pattern)) {
|
||||
const inside = stripMarkerBlocks(match[1]!, MARKER_TAGS.summary).trim();
|
||||
if (inside !== "") return inside;
|
||||
}
|
||||
}
|
||||
return stripMarkerBlocks(raw, MARKER_TAGS.summary).trim();
|
||||
}
|
||||
|
||||
/** One transcribed line inside a turn block: an inner tag, optional attributes, and the body. */
|
||||
|
||||
@@ -280,29 +280,47 @@ export interface RequestBeginPayload {
|
||||
type: "request_begin";
|
||||
}
|
||||
|
||||
export interface RequestEndPayload {
|
||||
type: "request_end";
|
||||
/** Terminal state of this Request (reuses the six StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome; `auth` is the credentials-failure signal hosts key on). */
|
||||
status: StopReason;
|
||||
/**
|
||||
* The unified retry/failure detail block shared by `request_end` and `compaction_end` —
|
||||
* one standard group of fields, stamped by the builders (the way `withOrigin` stamps
|
||||
* `origin`) rather than accreting as scattered ad-hoc parameters. Every field is optional
|
||||
* and additive: old Traces replay unchanged.
|
||||
*/
|
||||
export interface RetryDetail {
|
||||
/**
|
||||
* Failure detail from `LLMOutcome.message`, present only on non-completed statuses: the
|
||||
* real reason behind a retried/failed Request (e.g. `403 … (insufficient_user_quota)`),
|
||||
* for observability — the server's error records / Cost center read it here because a
|
||||
* retried request never produces an abort event. Additive: old Traces without it replay
|
||||
* unchanged.
|
||||
* Error detail, one name across the stack (`LLMOutcome.errorMessage` internally): present
|
||||
* only on non-completed statuses — the real reason behind a retried/failed Request (e.g.
|
||||
* `403 … (insufficient_user_quota)`), for observability — the server's error records /
|
||||
* Cost center read it here because a retried request never produces an abort event. Not
|
||||
* plain `message`: in this protocol "message" means an OmniMessage / model output, and
|
||||
* this field is neither. (Traces written before the rename carry it as `message`; nothing
|
||||
* reads that field semantically after the fact, so no dual-read is kept.)
|
||||
*/
|
||||
message?: string;
|
||||
error_message?: string;
|
||||
/**
|
||||
* 1-based ordinal of this Request within its retry run — the authoritative retry count
|
||||
* the CLI/Web display verbatim. Stamped on every non-completed request_end and on a
|
||||
* completed one that needed retries; absent on a clean first-try completion (the common
|
||||
* case stays noise-free) and in old Traces.
|
||||
*/
|
||||
attempt?: number;
|
||||
/**
|
||||
* Planned in-run retry wait (ms) — present ONLY when the engine will retry this failure
|
||||
* within the same run (status `timeout`/`malformed` with attempts remaining under the
|
||||
* applicable cap). Computed by the same formula as the actual backoff sleep
|
||||
* (`reconnectDelayMs`), so the announced wait and the real one cannot drift; the Web App
|
||||
* renders it as a live countdown to the next attempt. Absent on final failures (an abort
|
||||
* follows instead) and on completed requests. Additive: old Traces replay unchanged.
|
||||
* follows instead) and on completed requests.
|
||||
*/
|
||||
retry_in_ms?: number;
|
||||
}
|
||||
|
||||
export interface RequestEndPayload extends RetryDetail {
|
||||
type: "request_end";
|
||||
/** Terminal state of this Request (reuses the six StopReason values, sharing its source with this turn's complete message's stop_reason / LLMOutcome; `auth` is the credentials-failure signal hosts key on). */
|
||||
status: StopReason;
|
||||
}
|
||||
|
||||
/** Compaction trigger reason: context threshold / turn-count threshold / user-initiated request. */
|
||||
export type CompactionReason = "context" | "turns" | "manual";
|
||||
|
||||
@@ -328,7 +346,14 @@ export interface CompactionBeginPayload {
|
||||
turns: number;
|
||||
}
|
||||
|
||||
export interface CompactionEndPayload {
|
||||
/**
|
||||
* Inherits the shared RetryDetail block: `attempt` is the final attempt's 1-based ordinal
|
||||
* (failed attempts and retries included — stamped by summarize mode), `error_message` is
|
||||
* the last failure's detail (present only when `status` is `failed`), and `retry_in_ms` is
|
||||
* never stamped here (compaction retries are announced on the compaction request's own
|
||||
* request_end, which is Trace-only).
|
||||
*/
|
||||
export interface CompactionEndPayload extends RetryDetail {
|
||||
type: "compaction_end";
|
||||
reason: CompactionReason;
|
||||
mode: CompactionMode;
|
||||
|
||||
@@ -156,18 +156,20 @@ Skills are reusable instruction packages at <app_data_dir>/agents/<agent_id>/age
|
||||
- Session ID: {{SESSION_ID}}`;
|
||||
|
||||
/**
|
||||
* Built-in default compaction Prompt (summarize mode): tells the model that after
|
||||
* compaction the raw transcript is no longer visible and the
|
||||
* summary is the only record, so it must include everything needed to continue the task,
|
||||
* and no tools may be called while writing the summary.
|
||||
* Built-in default compaction Prompt (summarize mode): tells the model the summary will
|
||||
* replace the transcript as its only record (so it must include everything needed to
|
||||
* continue the task) and that no tools may be called. The format is shown as a concrete
|
||||
* example rather than described in prose — some models treat the tags as a "title" and
|
||||
* write the body after the closing tag (issue #170); extraction salvages that shape, but
|
||||
* generating it right beats repairing it. Persisted per-agent in system_config.yaml —
|
||||
* existing agents keep their stored prompt.
|
||||
*/
|
||||
export const DEFAULT_COMPACTION_PROMPT =
|
||||
"You have a partial transcript of the task above. Write a summary of it wrapped in " +
|
||||
"`[summary][/summary]` tags. This summary will replace the transcript: in the next " +
|
||||
"context window the raw transcript above will no longer be visible and this summary " +
|
||||
"will be its only record, so include everything needed to continue the task — the " +
|
||||
"original request, current state, next steps, and any learnings. Do not call any " +
|
||||
"tools while writing the summary; respond with text only.";
|
||||
"Summarize the task transcript above. The summary will replace the transcript as its " +
|
||||
"only record, so include everything needed to continue the task — the original request, " +
|
||||
"current state, next steps, and any learnings. Do not call any tools; reply with text " +
|
||||
"only, in exactly this format and nothing after it:\n\n" +
|
||||
"[summary]put the summary text here...[/summary]";
|
||||
|
||||
/**
|
||||
* Default built-in system tools: file reading/editing/writing first, then bash execution
|
||||
|
||||
@@ -49,7 +49,7 @@ import type {
|
||||
LLMInterface,
|
||||
LLMOutcome,
|
||||
} from "../src/interfaces.js";
|
||||
import { ContextEngine } from "../src/engine/context-engine.js";
|
||||
import { ContextEngine, SUMMARY_RETRY_GUIDANCE } from "../src/engine/context-engine.js";
|
||||
import type { CompactionSettings } from "../src/engine/context-engine.js";
|
||||
import { GenerativeModel } from "../src/llm/index.js";
|
||||
import type { UniConfig, UniEvent, UniMessage } from "@prismshadow/agenthub";
|
||||
@@ -78,7 +78,7 @@ class ScriptedLLM implements LLMInterface {
|
||||
this.calls.push(params.newMessages);
|
||||
const next = this.responses.shift();
|
||||
if (!next) {
|
||||
return { status: "failed", message: `${this.label}: no scripted response` };
|
||||
return { status: "failed", errorMessage: `${this.label}: no scripted response` };
|
||||
}
|
||||
for (const msg of next.messages) yield msg;
|
||||
return next.outcome ?? { status: "completed" };
|
||||
@@ -308,7 +308,7 @@ describe("context compaction", () => {
|
||||
messages: [toolCall({ name: "t", arguments: "{}", toolCallId: "c1" }), usage(150, 150)],
|
||||
},
|
||||
// Compaction request fails on the one status no ladder can fix (a rejected credential).
|
||||
{ messages: [], outcome: { status: "auth", message: "auth error" } },
|
||||
{ messages: [], outcome: { status: "auth", errorMessage: "auth error" } },
|
||||
// Original context is kept: the task continues, tool outputs feed back into the old instance as usual (context usage keeps growing).
|
||||
{ messages: [assistantText("finished on old context"), usage(190, 340)] },
|
||||
// Second trigger (context still over the limit) -> retries compaction at the boundary, this time succeeding.
|
||||
@@ -415,12 +415,55 @@ describe("context compaction", () => {
|
||||
|
||||
const out = await collect(engine.run([userText("go")], { approve: allowAll }));
|
||||
const events = compactionEvents(out);
|
||||
expect(events[1]).toMatchObject({ type: "compaction_end", status: "failed" });
|
||||
// The end event reports the attempts spent (issue #170's cost-center row message).
|
||||
expect(events[1]).toMatchObject({
|
||||
type: "compaction_end",
|
||||
status: "failed",
|
||||
attempt: 2,
|
||||
});
|
||||
// The retry resends the original input (tool results + prompt; here there are no tool results, just the prompt).
|
||||
expect(llm1.calls).toHaveLength(3);
|
||||
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
|
||||
});
|
||||
|
||||
it("compaction transport retries default to the shared maxReconnects budget", async () => {
|
||||
// Issue #170: without an explicit compactionMaxReconnects, the compaction request gets
|
||||
// the same retry budget as the turn loop (here 4) — under the old tighter default of 3
|
||||
// this script would have failed before reaching the 5th, succeeding attempt.
|
||||
const failing = (n: number): ScriptedResponse => ({
|
||||
messages: [],
|
||||
outcome: { status: "failed", errorMessage: `blip ${n}` },
|
||||
});
|
||||
const llm1 = new ScriptedLLM(
|
||||
[
|
||||
{ messages: [assistantText("answer"), usage(150, 150)] },
|
||||
failing(1),
|
||||
failing(2),
|
||||
failing(3),
|
||||
failing(4),
|
||||
{ messages: [assistantText("[summary]after the shared ladder[/summary]"), usage(30, 300)] },
|
||||
],
|
||||
"llm1",
|
||||
);
|
||||
const llm2 = new ScriptedLLM([], "llm2");
|
||||
const engine = new ContextEngine({
|
||||
llm: llm1,
|
||||
environment: fakeEnvironment,
|
||||
compaction: settings(),
|
||||
createLLM: () => llm2,
|
||||
maxReconnects: 4,
|
||||
reconnectBackoffMs: 1,
|
||||
});
|
||||
|
||||
const out = await collect(engine.run([userText("go")], { approve: allowAll }));
|
||||
expect(compactionEvents(out)[1]).toMatchObject({
|
||||
type: "compaction_end",
|
||||
status: "completed",
|
||||
attempt: 5,
|
||||
});
|
||||
expect(llm1.calls).toHaveLength(6);
|
||||
});
|
||||
|
||||
it("a failed compaction request takes the ladder and can recover on a later attempt", async () => {
|
||||
// The compaction loop retries the same statuses the turn loop does: `failed` is where a
|
||||
// transient fault lands whenever the classifier doesn't recognize the gateway's wording,
|
||||
@@ -429,7 +472,7 @@ describe("context compaction", () => {
|
||||
const llm1 = new ScriptedLLM(
|
||||
[
|
||||
{ messages: [assistantText("answer"), usage(150, 150)] },
|
||||
{ messages: [], outcome: { status: "failed", message: "502 upstream" } },
|
||||
{ messages: [], outcome: { status: "failed", errorMessage: "502 upstream" } },
|
||||
{ messages: [assistantText("[summary]recovered[/summary]")] },
|
||||
],
|
||||
"llm1",
|
||||
@@ -455,11 +498,11 @@ describe("context compaction", () => {
|
||||
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
|
||||
});
|
||||
|
||||
it("the compaction loop uses its own cap, not the shared maxReconnects ladder", async () => {
|
||||
// Compaction failure has graceful semantics (the original context is kept, compaction
|
||||
// retries on the next trigger), so it fails fast: with compactionMaxReconnects=2, the
|
||||
// compaction request runs 1+2 attempts and gives up — even though maxReconnects is far
|
||||
// larger and would have kept the session stalled through the full exponential ladder.
|
||||
it("an explicit compactionMaxReconnects overrides the shared maxReconnects budget", async () => {
|
||||
// The dep still allows a caller to bound compaction retries independently: with
|
||||
// compactionMaxReconnects=2, the compaction request runs 1+2 attempts and gives up —
|
||||
// even though maxReconnects is far larger (the default merely *follows* maxReconnects,
|
||||
// it does not override an explicit value).
|
||||
const llm1 = new ScriptedLLM(
|
||||
[
|
||||
{ messages: [assistantText("answer"), usage(150, 150)] },
|
||||
@@ -495,26 +538,31 @@ describe("context compaction", () => {
|
||||
// The Trace-written compaction request_ends announce the planned backoff under the
|
||||
// COMPACTION cap (base 1ms: 1, then 2), with none on the final failure — these events
|
||||
// are never streamed, so the value lands in the Trace record only.
|
||||
const retryPlans = recorded
|
||||
.filter((m) => (m.payload as { type?: string }).type === "request_end")
|
||||
.map((m) => (m.payload as { retry_in_ms?: number }).retry_in_ms);
|
||||
const compactionEnds = recorded.filter(
|
||||
(m) => (m.payload as { type?: string }).type === "request_end",
|
||||
);
|
||||
const retryPlans = compactionEnds.map(
|
||||
(m) => (m.payload as { retry_in_ms?: number }).retry_in_ms,
|
||||
);
|
||||
expect(retryPlans).toEqual([undefined, 1, 2, undefined]);
|
||||
// The attempt ordinal rides the same events: the turn's clean completion stays
|
||||
// unstamped, the three compaction failures count 1..3.
|
||||
const attempts = compactionEnds.map((m) => (m.payload as { attempt?: number }).attempt);
|
||||
expect(attempts).toEqual([undefined, 1, 2, 3]);
|
||||
});
|
||||
|
||||
it("an empty compaction response (thinking only, no text) is rejected: 5 attempts, then failed with the context kept", async () => {
|
||||
// Issue #83: the compaction request completes but yields no text. Committing the empty
|
||||
// summary would discard the whole context and lose the task state — the response is
|
||||
// rejected and retried under the dedicated rejection cap (5 attempts, #84), then the
|
||||
// compaction fails while the original context and Trace file stay current.
|
||||
it("an empty compaction response (thinking only, no text) is retried like any failure: 5 attempts, then failed with the context kept", async () => {
|
||||
// Issue #83/#170: the compaction request completes but yields no text. Committing the
|
||||
// empty summary would discard the whole context and lose the task state — the response
|
||||
// counts as one more failed attempt on the unified reconnect budget, and once the budget
|
||||
// is exhausted the compaction fails while the original context and Trace file stay current.
|
||||
const empty = (n: number): ScriptedResponse => ({
|
||||
messages: [thinkingMessage(`pondering, attempt ${n}, no text`)],
|
||||
});
|
||||
const llm1 = new ScriptedLLM(
|
||||
[
|
||||
{ messages: [assistantText("answer one"), usage(150, 150)] },
|
||||
// Five completed-but-empty compaction attempts: the dedicated cap allows exactly 5
|
||||
// rejections. compactionMaxReconnects is 1 here on purpose — rejections must NOT
|
||||
// consume the transport reconnect budget, or the loop would stop after 2 attempts.
|
||||
// Five completed-but-empty compaction attempt: 4 retries of budget allow exactly 5.
|
||||
empty(1),
|
||||
empty(2),
|
||||
empty(3),
|
||||
@@ -538,7 +586,7 @@ describe("context compaction", () => {
|
||||
created += 1;
|
||||
return new ScriptedLLM([], "llm2");
|
||||
},
|
||||
compactionMaxReconnects: 1,
|
||||
compactionMaxReconnects: 4,
|
||||
reconnectBackoffMs: 1,
|
||||
});
|
||||
const oldPath = trace.currentPath();
|
||||
@@ -546,12 +594,17 @@ describe("context compaction", () => {
|
||||
const out1 = await collect(engine.run([userText("task one")], { approve: allowAll }));
|
||||
|
||||
// Exactly one event pair, ending failed — an empty summary is never a completed compaction.
|
||||
// The end event reports the attempts spent (issue #170).
|
||||
const events = compactionEvents(out1);
|
||||
expect(
|
||||
events.map((e) => `${e.type}:${(e as Partial<CompactionEndPayload>).status ?? ""}`),
|
||||
).toEqual(["compaction_begin:", "compaction_end:failed"]);
|
||||
// No rejected attempt's token_usage is surfaced (only a successful compaction yields
|
||||
// its usage between the paired events).
|
||||
expect(events[1]).toMatchObject({
|
||||
attempt: 5,
|
||||
error_message: "the response contained no usable summary",
|
||||
});
|
||||
// These scripted attempts carry no token_usage of their own, so none appears between the
|
||||
// paired events (attempts that do carry usage surface it — see the 5th-attempt test).
|
||||
const types1 = payloadTypes(out1);
|
||||
const between = out1.slice(
|
||||
types1.indexOf("compaction_begin") + 1,
|
||||
@@ -560,14 +613,16 @@ describe("context compaction", () => {
|
||||
expect(between.filter((m) => (m.payload as { type?: string }).type === "token_usage")).toEqual(
|
||||
[],
|
||||
);
|
||||
// Turn + exactly five compaction attempts (the 5th rejection exhausts the cap, no 6th
|
||||
// request), each resending the prompt unchanged — an empty rejection needs no repair.
|
||||
// Turn + exactly five compaction attempts (the 5th exhausts the budget, no 6th
|
||||
// request); each retry leads with the corrective note, then the prompt — an empty
|
||||
// response needs no repair.
|
||||
expect(llm1.calls).toHaveLength(6);
|
||||
for (let i = 1; i <= 5; i += 1) {
|
||||
expect(llm1.calls[i]!.map(textOf)).toEqual(["COMPACT NOW"]);
|
||||
expect(llm1.calls[1]!.map(textOf)).toEqual(["COMPACT NOW"]);
|
||||
for (let i = 2; i <= 5; i += 1) {
|
||||
expect(llm1.calls[i]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
|
||||
}
|
||||
// Rejection resends are immediate, never announced: no compaction request_end carries a
|
||||
// retry_in_ms (they all end `completed`, unlike the transport ladder's timeout ends).
|
||||
// An unusable attempt's request_end carries status `completed`, for which no retry_in_ms
|
||||
// is ever announced (the backoff wait still ran between attempts).
|
||||
const rejectionPlans = (await readTrace(oldPath))
|
||||
.filter((m) => (m.payload as { type?: string }).type === "request_end")
|
||||
.map((m) => (m.payload as { retry_in_ms?: number }).retry_in_ms);
|
||||
@@ -585,9 +640,9 @@ describe("context compaction", () => {
|
||||
expect(await readdir(dirname(oldPath))).toEqual(["sess_empty_001.jsonl"]);
|
||||
});
|
||||
|
||||
it("tool-calling rejections exhaust the 5-attempt cap: every call is paired, and the next ordinary turn stays clean", async () => {
|
||||
it("tool-calling responses exhaust the retry budget: every call is paired, and the next ordinary turn stays clean", async () => {
|
||||
// A tool-calling response is not a summary — even when it also carries plausible summary
|
||||
// text — but its assistant turn IS committed on the live LLM object. Each rejection's
|
||||
// text — but its assistant turn IS committed on the live LLM object. Each such attempt's
|
||||
// calls are answered with synthesized failed outputs (written to Trace, prepended to the
|
||||
// retried input), so no tool_use ever dangles: after the compaction fails, the same
|
||||
// object must still serve ordinary turns with a well-formed history (#84 review).
|
||||
@@ -621,7 +676,7 @@ describe("context compaction", () => {
|
||||
created += 1;
|
||||
return new ScriptedLLM([], "llm2");
|
||||
},
|
||||
compactionMaxReconnects: 1,
|
||||
compactionMaxReconnects: 4,
|
||||
reconnectBackoffMs: 1,
|
||||
});
|
||||
const oldPath = trace.currentPath();
|
||||
@@ -635,12 +690,18 @@ describe("context compaction", () => {
|
||||
expect(payloadTypes(out)).not.toContain("tool_call_output");
|
||||
expect(created).toBe(0);
|
||||
|
||||
// The end event reports the attempts spent and the last failure (issue #170's
|
||||
// cost-center row message and the frontend banner detail).
|
||||
expect(events[1]).toMatchObject({
|
||||
attempt: 5,
|
||||
error_message: "the response called tools instead of writing a summary",
|
||||
});
|
||||
// Five attempts; from the second on, the input leads with the repair answering the
|
||||
// previous rejection's call, then re-issues the prompt.
|
||||
// previous attempt's call, then the corrective note, then the prompt.
|
||||
expect(llm1.calls).toHaveLength(6);
|
||||
for (let attempt = 2; attempt <= 5; attempt += 1) {
|
||||
const retry = llm1.calls[attempt]!;
|
||||
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text"]);
|
||||
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text", "text"]);
|
||||
const repair = retry[0]!.payload as {
|
||||
tool_call_id: string;
|
||||
output: string;
|
||||
@@ -651,7 +712,8 @@ describe("context compaction", () => {
|
||||
"[tool error] the compaction request expects a summary, not tool calls",
|
||||
);
|
||||
expect(repair.stop_reason).toBe("failed");
|
||||
expect(textOf(retry[1]!)).toBe("COMPACT NOW");
|
||||
expect(textOf(retry[1]!)).toBe(SUMMARY_RETRY_GUIDANCE);
|
||||
expect(textOf(retry[2]!)).toBe("COMPACT NOW");
|
||||
}
|
||||
|
||||
// All five synthesized repairs are written to the (old) Trace for replay to mirror.
|
||||
@@ -681,8 +743,9 @@ describe("context compaction", () => {
|
||||
});
|
||||
|
||||
it("a valid summary on the 5th and final allowed attempt completes the compaction", async () => {
|
||||
// Counting pin for the rejection cap: four rejected attempts spend the budget but the 5th
|
||||
// attempt still gets its chance — a valid summary there succeeds (5 rejections would fail).
|
||||
// Counting pin for the unified budget: four failed attempts spend the 4 retries, but the
|
||||
// 5th (last allowed) attempt still gets its chance — a valid summary there succeeds
|
||||
// (a 5th failure would exhaust the budget instead).
|
||||
const llm1 = new ScriptedLLM(
|
||||
[
|
||||
{ messages: [assistantText("answer"), usage(150, 150)] },
|
||||
@@ -706,7 +769,7 @@ describe("context compaction", () => {
|
||||
factoryTokens = tokens;
|
||||
return llm2;
|
||||
},
|
||||
maxReconnects: 1,
|
||||
compactionMaxReconnects: 4,
|
||||
reconnectBackoffMs: 1,
|
||||
});
|
||||
|
||||
@@ -714,8 +777,10 @@ describe("context compaction", () => {
|
||||
const events = compactionEvents(out);
|
||||
expect(events).toHaveLength(2);
|
||||
expect(events[1]).toMatchObject({ type: "compaction_end", status: "completed" });
|
||||
// Only the adopted attempt's token_usage is surfaced between the paired events; rejected
|
||||
// attempts' usage still feeds the Session cumulative totals (see below) but is not shown.
|
||||
// Every attempt's token_usage is surfaced between the paired events — rejected attempts
|
||||
// burn real tokens, and hiding them understated the cost center (issue #170). Here the
|
||||
// 1st (rejected) and 5th (adopted) attempts carry usage.
|
||||
expect(events[1]).toMatchObject({ attempt: 5 });
|
||||
const types = payloadTypes(out);
|
||||
const between = out.slice(
|
||||
types.indexOf("compaction_begin") + 1,
|
||||
@@ -724,8 +789,9 @@ describe("context compaction", () => {
|
||||
const usageBetween = between.filter(
|
||||
(m) => (m.payload as { type?: string }).type === "token_usage",
|
||||
);
|
||||
expect(usageBetween).toHaveLength(1);
|
||||
expect((usageBetween[0]!.payload as TokenUsagePayload).request.total).toBe(170);
|
||||
expect(usageBetween.map((m) => (m.payload as TokenUsagePayload).request.total)).toEqual([
|
||||
160, 170,
|
||||
]);
|
||||
// Session cumulative tokens carried into the new instance include the rejected attempts' usage.
|
||||
expect(factoryTokens).toMatchObject({ total: 480 });
|
||||
|
||||
@@ -778,10 +844,11 @@ describe("context compaction", () => {
|
||||
status: "completed",
|
||||
});
|
||||
|
||||
// The retried input answers the rejected attempt's call first, then re-issues the prompt.
|
||||
// The retried input answers the rejected attempt's call first, then carries the
|
||||
// corrective note and re-issues the prompt.
|
||||
expect(llm1.calls).toHaveLength(3);
|
||||
const retry = llm1.calls[2]!;
|
||||
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text"]);
|
||||
expect(payloadTypes(retry)).toEqual(["tool_call_output", "text", "text"]);
|
||||
const repair = retry[0]!.payload as {
|
||||
tool_call_id: string;
|
||||
output: string;
|
||||
@@ -792,7 +859,8 @@ describe("context compaction", () => {
|
||||
"[tool error] the compaction request expects a summary, not tool calls",
|
||||
);
|
||||
expect(repair.stop_reason).toBe("failed");
|
||||
expect(textOf(retry[1]!)).toBe("COMPACT NOW");
|
||||
expect(textOf(retry[1]!)).toBe(SUMMARY_RETRY_GUIDANCE);
|
||||
expect(textOf(retry[2]!)).toBe("COMPACT NOW");
|
||||
|
||||
// The repair belongs to the compaction dialogue: written to the old Trace (so replay
|
||||
// mirrors the pairing), never pushed to the output stream.
|
||||
@@ -897,6 +965,7 @@ describe("context compaction", () => {
|
||||
created += 1;
|
||||
return new ScriptedLLM([], "llm2");
|
||||
},
|
||||
compactionMaxReconnects: 4,
|
||||
reconnectBackoffMs: 1,
|
||||
});
|
||||
const oldPath = trace.currentPath();
|
||||
@@ -908,15 +977,16 @@ describe("context compaction", () => {
|
||||
// is the whole story.
|
||||
expect(payloadTypes(out1)).not.toContain("abort");
|
||||
expect(llm1.calls).toHaveLength(6);
|
||||
// Attempt 1 folds the turn's tool output in; attempt 2 carries the repair + Prompt but
|
||||
// NOT the absorbed output; attempts 3-5 are Prompt-only.
|
||||
// Attempt 1 folds the turn's tool output in; attempt 2 carries the repair + note + Prompt
|
||||
// but NOT the absorbed output; attempts 3-5 are note + Prompt.
|
||||
expect(payloadTypes(llm1.calls[1]!)).toEqual(["tool_call_output", "text"]);
|
||||
expect((llm1.calls[1]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("ct");
|
||||
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text"]);
|
||||
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text", "text"]);
|
||||
expect((llm1.calls[2]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("c1");
|
||||
expect(textOf(llm1.calls[2]![1]!)).toBe("COMPACT NOW");
|
||||
expect(textOf(llm1.calls[2]![1]!)).toBe(SUMMARY_RETRY_GUIDANCE);
|
||||
expect(textOf(llm1.calls[2]![2]!)).toBe("COMPACT NOW");
|
||||
for (let attempt = 3; attempt <= 5; attempt += 1) {
|
||||
expect(llm1.calls[attempt]!.map(textOf)).toEqual(["COMPACT NOW"]);
|
||||
expect(llm1.calls[attempt]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
|
||||
}
|
||||
// Original context kept: no LLM swap, no Trace rotation.
|
||||
expect(created).toBe(0);
|
||||
@@ -955,16 +1025,17 @@ describe("context compaction", () => {
|
||||
environment: fakeEnvironment,
|
||||
compaction: settings(),
|
||||
createLLM: () => new ScriptedLLM([], "llm2"),
|
||||
compactionMaxReconnects: 4,
|
||||
reconnectBackoffMs: 1,
|
||||
});
|
||||
|
||||
const out = await collect(engine.run([userText("go")], { approve: allowAll }));
|
||||
expect(compactionEvents(out)[1]).toMatchObject({ type: "compaction_end", status: "failed" });
|
||||
expect(llm1.calls).toHaveLength(7);
|
||||
// Attempt 1 folds the outputs; attempts 2-5 are Prompt-only (absorbed by the first commit).
|
||||
// Attempt 1 folds the outputs; attempts 2-5 are note + Prompt (absorbed by the first commit).
|
||||
expect(payloadTypes(llm1.calls[1]!)).toEqual(["tool_call_output", "text"]);
|
||||
for (let attempt = 2; attempt <= 5; attempt += 1) {
|
||||
expect(llm1.calls[attempt]!.map(textOf)).toEqual(["COMPACT NOW"]);
|
||||
expect(llm1.calls[attempt]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
|
||||
}
|
||||
// The continuation input is the repair answering c5 — nothing else: no absorbed outputs,
|
||||
// no prompt.
|
||||
@@ -1044,8 +1115,8 @@ describe("context compaction", () => {
|
||||
expect(events[1]).toMatchObject({ type: "compaction_end", status: "aborted" });
|
||||
expect(payloadTypes(out)).toContain("abort");
|
||||
expect(llm1.calls).toHaveLength(3);
|
||||
// Attempt 2 carried the repair + Prompt (not the absorbed outputs).
|
||||
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text"]);
|
||||
// Attempt 2 carried the repair + note + Prompt (not the absorbed outputs).
|
||||
expect(payloadTypes(llm1.calls[2]!)).toEqual(["tool_call_output", "text", "text"]);
|
||||
expect((llm1.calls[2]![0]!.payload as { tool_call_id?: string }).tool_call_id).toBe("c1");
|
||||
|
||||
// The next run leads with the still-unanswered repair; the absorbed turn outputs are
|
||||
@@ -1255,6 +1326,46 @@ describe("context compaction", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("rescues a summary written outside the tags (issue #170) on the first attempt", async () => {
|
||||
// deepseek-v4-flash writes `[summary]\n[/summary]` as a "title" and the body after the
|
||||
// closing tag. Extraction salvages the outside text, so the very first attempt succeeds —
|
||||
// no rejection loop, no burned retries.
|
||||
const llm1 = new ScriptedLLM(
|
||||
[
|
||||
{ messages: [assistantText("answer"), usage(150, 150)] },
|
||||
{
|
||||
messages: [
|
||||
assistantText("[summary]\n[/summary]\nRescued body outside the tags."),
|
||||
usage(40, 200),
|
||||
],
|
||||
},
|
||||
],
|
||||
"llm1",
|
||||
);
|
||||
const llm2 = new ScriptedLLM([{ messages: [assistantText("fresh"), usage(10, 210)] }], "llm2");
|
||||
const engine = new ContextEngine({
|
||||
llm: llm1,
|
||||
environment: fakeEnvironment,
|
||||
compaction: settings(),
|
||||
createLLM: () => llm2,
|
||||
reconnectBackoffMs: 1,
|
||||
});
|
||||
|
||||
const out = await collect(engine.run([userText("task one")], { approve: allowAll }));
|
||||
expect(compactionEvents(out)[1]).toMatchObject({
|
||||
type: "compaction_end",
|
||||
status: "completed",
|
||||
attempt: 1,
|
||||
});
|
||||
// No rejection retry: the turn request plus a single compaction attempt.
|
||||
expect(llm1.calls).toHaveLength(2);
|
||||
await collect(engine.run([userText("task two")], { approve: allowAll }));
|
||||
expect(llm2.calls[0]!.map(textOf)).toEqual([
|
||||
"[context_summary]\nRescued body outside the tags.\n[/context_summary]",
|
||||
"task two",
|
||||
]);
|
||||
});
|
||||
|
||||
it("manual compaction skips threshold checks and reuses the same flow", async () => {
|
||||
const llm1 = new ScriptedLLM(
|
||||
[
|
||||
@@ -1356,6 +1467,7 @@ describe("context compaction", () => {
|
||||
environment: fakeEnvironment,
|
||||
compaction: settings(),
|
||||
createLLM: () => new ScriptedLLM([], "llm2"),
|
||||
compactionMaxReconnects: 4,
|
||||
reconnectBackoffMs: 1,
|
||||
});
|
||||
await collect(engine.run([userText("start")], { approve: allowAll }));
|
||||
@@ -1368,11 +1480,11 @@ describe("context compaction", () => {
|
||||
const out = await collect(engine.compact());
|
||||
expect(compactionEvents(out)[1]).toMatchObject({ type: "compaction_end", status: "failed" });
|
||||
expect(llm1.calls).toHaveLength(6);
|
||||
// Attempt 1 folded the carry-over in (and committed it); later attempts resend the Prompt
|
||||
// alone — the absorbed carry-over never again.
|
||||
// Attempt 1 folded the carry-over in (and committed it); later attempts resend the note +
|
||||
// Prompt — the absorbed carry-over never again.
|
||||
expect(llm1.calls[1]!.map(textOf)).toEqual(["pending question", "COMPACT NOW"]);
|
||||
for (let attempt = 2; attempt <= 5; attempt += 1) {
|
||||
expect(llm1.calls[attempt]!.map(textOf)).toEqual(["COMPACT NOW"]);
|
||||
expect(llm1.calls[attempt]!.map(textOf)).toEqual([SUMMARY_RETRY_GUIDANCE, "COMPACT NOW"]);
|
||||
}
|
||||
|
||||
// The carry-over is consumed at the seam — the next run leads with the stashed repair
|
||||
|
||||
@@ -177,7 +177,7 @@ describe("describe_image (the text-only-model variant of read_image)", () => {
|
||||
|
||||
it("vision model request failure: failed with the status and message", async () => {
|
||||
await writeFile(path.join(tmp, "a.png"), PNG_1X1);
|
||||
const { llm } = fakeLLM("", { status: "failed", message: "401 unauthorized" });
|
||||
const { llm } = fakeLLM("", { status: "failed", errorMessage: "401 unauthorized" });
|
||||
const { result, text } = await run({ source: "a.png" }, tmp, {
|
||||
modelId: "vis-1",
|
||||
createLLM: () => llm,
|
||||
|
||||
@@ -684,7 +684,7 @@ describe("ContextEngine ReAct loop (mock LLM, approve callback)", () => {
|
||||
yield partialText("start", "");
|
||||
yield partialText("delta", "half a thought");
|
||||
// `auth`: the one LLM status that still exits straight to the flatten path.
|
||||
return { status: "auth", message: "boom" };
|
||||
return { status: "auth", errorMessage: "boom" };
|
||||
}
|
||||
yield assistantText("ok");
|
||||
yield tokenUsage(emptyTokenCounts(), {
|
||||
@@ -1646,7 +1646,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
toolCallId: "tc-broken",
|
||||
stopReason: "malformed",
|
||||
});
|
||||
return { status: "malformed", message: "incomplete stream" };
|
||||
return { status: "malformed", errorMessage: "incomplete stream" };
|
||||
}
|
||||
yield assistantText("done");
|
||||
yield tokenUsage(emptyTokenCounts(), {
|
||||
@@ -1799,7 +1799,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
async *streamGenerate(params) {
|
||||
calls += 1;
|
||||
inputs.push(params.newMessages);
|
||||
return { status: "failed", message: "400 unknown parameter: max_output_tokens" };
|
||||
return { status: "failed", errorMessage: "400 unknown parameter: max_output_tokens" };
|
||||
},
|
||||
};
|
||||
const environment = new Environment({
|
||||
@@ -1884,7 +1884,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
// eslint-disable-next-line require-yield
|
||||
async *streamGenerate() {
|
||||
calls += 1;
|
||||
return { status: "auth", message: "401 invalid x-api-key" };
|
||||
return { status: "auth", errorMessage: "401 invalid x-api-key" };
|
||||
},
|
||||
};
|
||||
const environment = new Environment({
|
||||
@@ -1898,7 +1898,9 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
// The request's own terminal status is the host signal (streams to the web).
|
||||
const end = all.find((m) => (m.payload as { type?: string }).type === "request_end");
|
||||
expect((end!.payload as { status?: string }).status).toBe("auth");
|
||||
expect((end!.payload as { message?: string }).message).toBe("401 invalid x-api-key");
|
||||
expect((end!.payload as { error_message?: string }).error_message).toBe(
|
||||
"401 invalid x-api-key",
|
||||
);
|
||||
// No planned retry is announced for a terminal failure.
|
||||
expect((end!.payload as { retry_in_ms?: number }).retry_in_ms).toBeUndefined();
|
||||
const abort = all.find((m) => (m.payload as { type?: string }).type === "abort");
|
||||
@@ -2037,7 +2039,10 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
if (calls === 1) {
|
||||
// A retryable provider rejection: the detail must reach observability via the
|
||||
// event — a retried request never produces an abort to carry it.
|
||||
return { status: "timeout", message: "403 quota exceeded (insufficient_user_quota)" };
|
||||
return {
|
||||
status: "timeout",
|
||||
errorMessage: "403 quota exceeded (insufficient_user_quota)",
|
||||
};
|
||||
}
|
||||
yield assistantText("ok");
|
||||
yield tokenUsage(emptyTokenCounts(), {
|
||||
@@ -2057,16 +2062,21 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
|
||||
const all = await collectRun(engine, [userText("go")], allowAll);
|
||||
const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end") as {
|
||||
payload: { status?: string; message?: string; retry_in_ms?: number };
|
||||
payload: { status?: string; error_message?: string; attempt?: number; retry_in_ms?: number };
|
||||
}[];
|
||||
expect(ends).toHaveLength(2);
|
||||
expect(ends[0]!.payload.status).toBe("timeout");
|
||||
expect(ends[0]!.payload.message).toBe("403 quota exceeded (insufficient_user_quota)");
|
||||
// The engine will retry: the planned backoff is announced (base 0 here -> 0ms).
|
||||
expect(ends[0]!.payload.error_message).toBe("403 quota exceeded (insufficient_user_quota)");
|
||||
// The engine will retry: the planned backoff is announced (base 0 here -> 0ms), and the
|
||||
// authoritative attempt ordinal is stamped (this was the run's 1st request).
|
||||
expect(ends[0]!.payload.retry_in_ms).toBe(0);
|
||||
expect(ends[0]!.payload.attempt).toBe(1);
|
||||
expect(ends[1]!.payload.status).toBe("completed");
|
||||
expect(ends[1]!.payload.message).toBeUndefined();
|
||||
expect(ends[1]!.payload.error_message).toBeUndefined();
|
||||
expect(ends[1]!.payload.retry_in_ms).toBeUndefined();
|
||||
// A completion that needed retries still carries its ordinal ("recovered on attempt 2");
|
||||
// only a clean first-try completion stays unstamped.
|
||||
expect(ends[1]!.payload.attempt).toBe(2);
|
||||
});
|
||||
|
||||
it("request_end announces the planned backoff (retry_in_ms) from the shared ladder; absent once the cap is reached", async () => {
|
||||
@@ -2178,7 +2188,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
|
||||
// exhausted-retries test below).
|
||||
yield thinkingMessage("half-thought", "failed");
|
||||
yield assistantText("half-text", "failed");
|
||||
return { status: "auth", message: "boom" };
|
||||
return { status: "auth", errorMessage: "boom" };
|
||||
}
|
||||
yield assistantText("ok");
|
||||
yield tokenUsage(emptyTokenCounts(), {
|
||||
|
||||
@@ -1573,7 +1573,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
|
||||
model.streamGenerate({ newMessages: [userText("go")] }),
|
||||
);
|
||||
expect(outcome.status).toBe("malformed");
|
||||
expect(outcome.message).toContain("Unexpected token");
|
||||
expect(outcome.errorMessage).toContain("Unexpected token");
|
||||
const complete = messages.find((m) => typeOf(m) === "text");
|
||||
expect((complete!.payload as TextPayload).text).toBe("hi");
|
||||
expect((complete!.payload as TextPayload).stop_reason).toBe("malformed");
|
||||
@@ -1634,7 +1634,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
|
||||
// 401 = credentials failure: its own stop reason so hosts can gate input until the
|
||||
// model's API key is updated (same engine behavior as failed).
|
||||
expect(outcome.status).toBe("auth");
|
||||
expect(outcome.message).toContain("invalid api key");
|
||||
expect(outcome.errorMessage).toContain("invalid api key");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
|
||||
// A genuinely non-retryable parameter error stays a plain failure.
|
||||
@@ -1663,7 +1663,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
|
||||
model.streamGenerate({ newMessages: [userText("go")] }),
|
||||
);
|
||||
expect(outcome.status).toBe("auth");
|
||||
expect(outcome.message).toContain("subscription key invalid");
|
||||
expect(outcome.errorMessage).toContain("subscription key invalid");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
});
|
||||
|
||||
@@ -1681,7 +1681,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
|
||||
expect(outcome.status).toBe("timeout");
|
||||
// The real reason rides on the outcome (request_end -> the Cost center's errors panel
|
||||
// shows it): a retried quota rejection must not surface as a bare "timeout".
|
||||
expect(outcome.message).toContain("insufficient_user_quota");
|
||||
expect(outcome.errorMessage).toContain("insufficient_user_quota");
|
||||
expect(messages.map(typeOf)).not.toContain("token_usage");
|
||||
});
|
||||
|
||||
|
||||
@@ -132,6 +132,23 @@ describe("engine blocks ([turn_aborted] / [turn_retried] / [context_summary] / [
|
||||
expect(extractSummary("[summary]a[/summary] <summary>b</summary>")).toBe("a");
|
||||
});
|
||||
|
||||
it("extractSummary rescues content the model wrote outside the tags (issue #170)", () => {
|
||||
// deepseek-v4-flash writes the tags as a "title" and the body after the closing tag.
|
||||
expect(extractSummary("[summary]\n[/summary]\nThe actual summary body.")).toBe(
|
||||
"The actual summary body.",
|
||||
);
|
||||
expect(extractSummary("[summary]\n[summary][/summary]\nBody after a doubled open tag.")).toBe(
|
||||
"Body after a doubled open tag.",
|
||||
);
|
||||
// A stray opening tag inside the pair doesn't hide the text that follows it.
|
||||
expect(extractSummary("[summary]\n[summary]\nBody inside.[/summary]")).toBe("Body inside.");
|
||||
// An empty current-form pair doesn't shadow a non-empty legacy pair.
|
||||
expect(extractSummary("[summary][/summary]<summary>legacy body</summary>")).toBe("legacy body");
|
||||
// Genuinely empty output still extracts to "" — the engine treats that as a failed attempt.
|
||||
expect(extractSummary("[summary][/summary]")).toBe("");
|
||||
expect(extractSummary("[summary]\n[/summary]")).toBe("");
|
||||
});
|
||||
|
||||
it("turn blocks round-trip through unwrapSyntheticBlock, both forms, single-level", () => {
|
||||
const lines = [transcribeUserInput("go"), transcribeToolCall("read_file", "t1", "{}")];
|
||||
const aborted = buildTurnAbortedBlock(lines);
|
||||
|
||||
@@ -97,7 +97,7 @@ describe("session-title", () => {
|
||||
assistantText("partial"),
|
||||
tokenUsage(emptyTokenCounts(), { cache_read: 0, cache_write: 0, output: 1, total: 1 }),
|
||||
],
|
||||
{ status: "failed", message: "401" },
|
||||
{ status: "failed", errorMessage: "401" },
|
||||
),
|
||||
{ userText: "u", assistantText: "a" },
|
||||
);
|
||||
|
||||
Reference in new issue
Block a user