3dtours/penguin-harness
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(core): retry on failed as well as timeout; only auth stops the run (#104)

Browse Source 
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yaowei ZhengandClaude Opus 5 authored and GitHub committed 2026-07-29 16:57:05 +08:00
1 parent 8dc560eedd
commit 5b22ad06da
30 files changed
+603 -174
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/core/src/engine/context-engine.ts
+92 -45
View File
@@ -183,11 +183,10 @@ export interface ContextEngineDeps {
/** Ceiling (ms) for a single reconnect backoff wait. Defaults to 30000. */
reconnectBackoffMaxMs?: number;
/**
* Maximum retries for a failing compaction request (instead of the shared
* `maxReconnects`). Compaction failure already has graceful semantics — the original
* context is kept and compaction retries on the next trigger — so it fails fast
* (~1.75s of backoff) rather than stalling the session through the full reconnect
* ladder. Defaults to 3.
* Maximum retries for a failing compaction request (instead of the shared `maxReconnects`;
* same statuses, shorter budget — see RETRY_STATUSES). Compaction failure is already
* graceful — the original context is kept and compaction retries at the next trigger — so a
* broken request fails fast (~1.75s of backoff). Defaults to 3.
*/
compactionMaxReconnects?: number;
/**
@@ -209,8 +208,8 @@ export type CompactAvailability = "ok" | "unsupported" | "empty" | "just_compact
/**
* Maximum summarize attempts when the compaction response is rejected as an invalid summary
* (empty extracted text, or the model answered with tool calls — issue #83/#84). Deliberately
* separate from (and larger than) `compactionMaxReconnects`: that cap governs transport-level
* timeout/malformed attempts that were never committed and back off exponentially, while a
* separate from (and larger than) `compactionMaxReconnects`: that cap governs the retryable
* attempts that were never committed (failed/timeout/malformed) and back off exponentially, while a
* rejection is a well-formed committed response — the request itself works, the model just
* didn't produce a summary, so the repaired input is resent immediately with no backoff. And
* since the compaction request keeps the session's toolset (the prefix cache must stay valid,
@@ -317,6 +316,19 @@ function downgradeCarriedGoalInput(msg: OmniMessage): OmniMessage {
return { ...msg, payload: { ...msg.payload, text: downgraded } as OmniMessage["payload"] };
}
/**
* LLM outcomes that reconnect in-run — everything but `auth`, which cannot be retried into
* working. `failed` is included on purpose: the classifier producing it is an allowlist of
* known transport codes and message vocabulary, so a gateway phrasing a transient fault its
* own way lands here; retrying a genuinely permanent error costs the ladder and ends the same
* way, while aborting a transient one destroys the turn.
*
* Both loops retry the same set. What differs is the budget: compaction runs on
* `compactionMaxReconnects` (shorter than the turn loop's), because a compaction that gives
* up keeps the original context and tries again at the next trigger.
*/
const RETRY_STATUSES: readonly StopReason[] = ["failed", "timeout", "malformed"];
/**
* Delay before reconnect attempt N (1-based): exponential growth from `base` with a hard
* ceiling `max` — `min(base × 2^(N−1), max)`. With the defaults (250ms base, 30s ceiling,
@@ -507,11 +519,18 @@ export class ContextEngine {
}
turnCount += 1;
// This turn's input. A timeout/malformed attempt is never committed to history by
// AgentHub (an abnormally interrupted stream doesn't land in history), so reconnect
// resends this turn's input unchanged, appending a `[turn_retried]` block carrying what
// the failed attempt already produced — the model continues from there instead of
// re-running tools; the tag is distinct from the user-interruption `[turn_aborted]`.
// This turn's input. The safety invariant behind resending it: **no retryable attempt
// is ever committed to AgentHub's history**. AgentHub appends a turn to `_history` only
// after its stream has been consumed to the end and validated, so every abnormal exit —
// `timeout`, `malformed` and `failed` alike, whether the stream was cut, the payload
// failed to parse, or the request was rejected outright — leaves history untouched.
// Nothing can therefore be duplicated or left as an unanswered tool_use by a reconnect;
// that is what makes it safe to resend this turn's input unchanged, appending a
// `[turn_retried]` block carrying what the failed attempt already produced — the model
// continues from there instead of re-running tools; the tag is distinct from the
// user-interruption `[turn_aborted]`. (The one attempt that IS committed — a fully
// delivered response whose finish_reason arrived — is forced to `completed` in
// GenerativeModel precisely so it never reaches this loop.)
const failedTurns: TurnResult[] = [];
let attemptInput = nextInput;
let reconnects = 0;
@@ -532,12 +551,10 @@ export class ContextEngine {
yield* this.emitAbort("aborted by user");
return;
}
// Non-retryable error: stop and hand control back to the user; the failure reason
// is written to the abort event / Trace. `auth` (credentials rejected) behaves
// exactly like `failed` here — direct stop, never the retry loop below (the
// classifier already refuses to mark auth retryable; this branch is the second
// belt) — hosts read the status from this turn's request_end to gate input.
if (turn.outcome.status === "failed" || turn.outcome.status === "auth") {
// `auth` is the one status that stops the run: a rejected credential cannot be retried
// into working, and hosts read it off the turn's request_end to gate input. Everything
// else — `failed` included — goes to the reconnect loop below.
if (turn.outcome.status === "auth") {
this.pendingCarryOver = this.buildCarryOver(attemptInput, turn);
yield* this.emitAbort(`llm request error: ${turn.outcome.message ?? "unknown"}`);
return;
@@ -545,17 +562,37 @@ export class ContextEngine {
// Completed normally.
if (turn.outcome.status === "completed") break;
// Only timeout / malformed remain: reconnect automatically within the same run. When
// retries are exhausted or the backoff is interrupted, the retry input is held as-is as
// carry-over (the original input is already written to Trace, so it isn't rewritten).
// The frontend surfaces the retry process and count via request_end(timeout|malformed)
// followed by the next request_begin.
// failed / timeout / malformed remain: reconnect automatically within the same run.
//
// `failed` retries too, even though the classifier judged it non-transient. That
// judgement is a hint, not a verdict: it is an allowlist of known network codes,
// statuses and message vocabulary, so every gateway that phrases a transient failure
// its own way falls through it — `Upstream HTTP/2 stream failed
// (upstream_http2_stream_error)` is plainly a transport fault and matched nothing.
// Retrying a genuinely permanent error costs the ladder and then ends the same way;
// aborting a transient one destroys the turn. So the *classification* stays honest
// (`failed` is still recorded as a real failure, not relabelled a timeout) while the
// *policy* retries it. Only `auth` is terminal, above.
//
// When retries are exhausted or the backoff is interrupted, the retry input is held
// as-is as carry-over (the original input is already written to Trace, so it isn't
// rewritten). The frontend surfaces the retry process and count via
// request_end(failed|timeout|malformed) followed by the next request_begin.
failedTurns.push(turn);
attemptInput = this.withRetriedTurns(nextInput, failedTurns);
if (reconnects >= this.maxReconnects) {
this.pendingCarryOver = attemptInput;
const reason = turn.outcome.status === "malformed" ? "malformed response" : "reconnect";
yield* this.emitAbort(`${reason} failed after ${this.maxReconnects} retries`);
// This reason is user-visible (the Web App's error panel, the CLI's abort line) and
// is what observability persists as the error message, so it has to read as a
// sentence: what gave out, then how many attempts it took, then — for the one class
// that carries provider words worth showing — the detail, last.
const reason =
turn.outcome.status === "malformed"
? `malformed response failed after ${this.maxReconnects} retries`
: turn.outcome.status === "failed"
? `llm request failed after ${this.maxReconnects} retries: ${turn.outcome.message ?? "unknown"}`
: `reconnect failed after ${this.maxReconnects} retries`;
yield* this.emitAbort(reason);
return;
}
reconnects += 1;
@@ -740,14 +777,16 @@ export class ContextEngine {
* follows instead). Announced on the failure's `request_end` as `retry_in_ms` so the
* frontend can render a live countdown; shares `reconnectDelayMs` with `backoff`, so the
* announced wait and the actual sleep cannot drift. The caps differ by loop: the turn
* loop passes `maxReconnects`, the compaction loop `compactionMaxReconnects`.
* loop passes `maxReconnects`, the compaction loop `compactionMaxReconnects`; `retries` must
* match what the calling loop actually does, or the announced countdown is a lie.
*/
private plannedRetryDelayMs(
status: StopReason,
outcome: LLMOutcome,
reconnectsSoFar: number,
cap: number,
retries: readonly StopReason[],
): number | undefined {
if (status !== "timeout" && status !== "malformed") return undefined;
if (!retries.includes(outcome.status)) return undefined;
if (reconnectsSoFar >= cap) return undefined;
return reconnectDelayMs(
this.reconnectBackoffMs,
@@ -868,7 +907,12 @@ export class ContextEngine {
const stopEvt = requestEnd(
outcome.status,
outcome.message,
this.plannedRetryDelayMs(outcome.status, reconnectsSoFar, this.maxReconnects),
this.plannedRetryDelayMs(
outcome,
reconnectsSoFar,
this.maxReconnects,
RETRY_STATUSES,
),
);
queue.push(stopEvt);
await this.write(stopEvt);
@@ -896,8 +940,9 @@ export class ContextEngine {
// added to this turn's ledger, and gets no paired output backfilled: such a tool_call
// was never committed to history by AgentHub, so there's nothing to pair. This turn
// must then end with a non-completed outcome (only interruption closure produces such
// a tool_call): timeout/malformed is cleaned up by reconnect resending the flatten
// carry-over, failed/auth/aborted exits directly.
// a tool_call): a retryable outcome (failed/timeout/malformed) is cleaned up by
// reconnect resending the flatten carry-over, while the run-ending ones
// (aborted/auth) exit directly.
if (isCompleteModelMessage(msg) && msg.payload.type === "tool_call") {
const tc = msg as OmniMessage<ToolCallPayload>;
if (tc.payload.stop_reason !== "completed") continue;
@@ -1128,9 +1173,9 @@ export class ContextEngine {
* text and no tool calls in the response. An invalid summary is rejected: any tool calls the
* model issued are answered with synthesized failed outputs (pairing repair, see the loop
* body) and the repaired input is resent immediately, up to MAX_SUMMARY_REJECTIONS attempts,
* then the compaction fails. timeout/malformed reconnect via the existing retry mechanism
* under the compaction-specific cap (`compactionMaxReconnects`, tighter than the turn loop's
* ladder), collapsing to failed once retries are exhausted; on failure/abort, the original
* then the compaction fails. failed/timeout/malformed reconnect under the compaction-specific
* cap (`compactionMaxReconnects` — a shorter budget than the turn loop's, not a narrower
* set), collapsing to failed once retries are exhausted; on failure/abort, the original
* context and Trace index are kept — it does not fall back to discard. The first **committed**
* attempt absorbs `pendingToolOutputs` into the old context's history (issue #85): later
* resends carry only the repairs and the Prompt, and the result's `committed` flag tells
@@ -1167,7 +1212,7 @@ export class ContextEngine {
// by a committed request: prepended to the retry input, and stashed as carry-over should
// the compaction be abandoned first (see stashRepairs).
let pendingRepairs: OmniMessage[] = [];
// Two independent retry budgets: transport-level timeout/malformed attempts (never
// Two independent retry budgets: retryable attempts (failed/timeout/malformed, never
// committed) follow the compaction-specific reconnect cap with the exponential backoff
// ladder; invalid-summary rejections (committed, well-formed responses that just aren't
// summaries) get the larger dedicated cap and resend immediately — see
@@ -1251,7 +1296,7 @@ export class ContextEngine {
yield* this.emitCompactionEnd(reason, "summarize", "aborted");
return { status: "aborted", committed };
}
if (attempt.status === "failed" || attempt.status === "auth") {
if (attempt.status === "auth") {
// `auth` folds into `failed` here: the compaction event pair keeps its
// completed/failed/aborted set, the original context is kept, and the host learns
// about the credential problem from the request's own terminal status (a turn-loop
@@ -1260,11 +1305,11 @@ export class ContextEngine {
yield* this.emitCompactionEnd(reason, "summarize", "failed");
return { status: "failed", committed };
}
// timeout / malformed: retried via reconnect — transport-level, never committed by
// failed / timeout / malformed: retried via reconnect — none of them is committed by
// AgentHub (case B), so the input (any pending repairs included) is resent unchanged.
// Compaction uses its own, tighter cap (not the shared maxReconnects): a failed
// compaction keeps the original context and retries on the next trigger, so failing
// fast beats holding the session through the full exponential ladder.
// Compaction uses its own, tighter cap (not the shared maxReconnects): a compaction
// that ends up failing keeps the original context and tries again on the next trigger,
// so a short ladder here beats holding the session through the full one.
if (reconnects >= this.compactionMaxReconnects) {
this.stashRepairs(pendingRepairs);
yield* this.emitCompactionEnd(reason, "summarize", "failed");
@@ -1345,9 +1390,10 @@ export class ContextEngine {
res.value.status,
res.value.message,
this.plannedRetryDelayMs(
res.value.status,
res.value,
reconnectsSoFar,
this.compactionMaxReconnects,
RETRY_STATUSES,
),
),
);
@@ -1423,10 +1469,11 @@ export class ContextEngine {
/**
* Builds the interruption resend content (carry-over, interruption cleanup)
* based on the LLM's terminal state. Used only for the **exit** cleanup of
* aborted / failed / auth (reconnect retry doesn't go through here — retry input is assembled by
* withRetriedTurns, appending `[turn_retried]` with the failed attempt's output, distinct
* from the user-interruption `[turn_aborted]`):
* based on the LLM's terminal state. Used only for the **exit** cleanup of the statuses that
* end the run: `aborted` and `auth` (a retried `failed` does not reach here, and neither
* does reconnect retry: retry input is assembled by withRetriedTurns, appending
* `[turn_retried]` with the failed attempt's output, distinct from the user-interruption
* `[turn_aborted]`):
* - Model output completed (case A, outcome=completed): AgentHub already committed an
* assistant turn containing `tool_call`, so it can only be resent as a structured
* `tool_call_output` to pair with it (cannot flatten, or the already-committed tool_call
packages/core/src/interfaces.ts
+10 -7
View File
@@ -147,13 +147,16 @@ export interface GenerativeModelParameters {
* - `malformed`: AgentHub response failed JSON parsing, needs reconnect — also retried by
* `context_engine`;
* - `aborted`: user-initiated interruption — stop and hand back to the user;
* - `failed`: other non-retryable errors (params, etc.) — stop and hand back to the user
* (`message` provides the display text);
* - `auth`: the provider rejected the credentials (see `isAuthenticationError`) — behaves
* like `failed` in the engine (direct stop, never retried), but hosts key on the status
* to disable input until the model's API key is updated (only the model reference is
* fixed at Session creation; credentials come from the current Project config, so a key
* update lets the Session continue).
* - `failed`: an error the retry classifier did not judge transient (params, etc.) — still
* retried by `context_engine` within the same run (`message` provides the display text).
* The classification stays honest — this is reported as `failed`, not relabelled a
* timeout — while the *policy* retries it, because that classifier is an allowlist and a
* gateway phrasing a transient fault its own way lands here;
* - `auth`: the provider rejected the credentials (see `isAuthenticationError`) — the one
* status that stops the run outright, since no retry can turn a rejected credential into
* a working one; hosts also key on it to disable input until the model's API key is
* updated (only the model reference is fixed at Session creation; credentials come from
* the current Project config, so a key update lets the Session continue).
* Docs: /docs/interfaces § "LLMOutcome semantics".
*/
export interface LLMOutcome {
packages/core/src/llm/generative-model.ts
+16 -15
View File
@@ -12,13 +12,16 @@
* (observability/Token).
* 3. Interruption/error handling: `finishInterrupted` first closes any open
* streaming segments and backfills the complete message, then the output ends — never
* leaking a malformed structure. This interface **never retries internally** — retryable
* errors (network/transport drops, timeouts, 429/5xx, provider quota exhaustion, see
* leaking a malformed structure. This interface **never retries internally** — it only
* classifies, and `context_engine` owns the retry policy: errors recognised as retryable
* (network/transport drops, timeouts, 429/5xx, provider quota exhaustion, see
* `isRetryableError`) end with `timeout`; AgentHub JSON parse errors end with `malformed`;
* both are handed to `context_engine` to reconnect within the same run. User interruption
* ends with `aborted`; non-retryable errors (parameters etc.) end with `failed`;
* credentials failures end with their own terminal status `auth` (see
* `isAuthenticationError`) — same engine behavior as `failed`, but hosts key on it.
* everything else the classifier does not recognise ends with `failed` — which the engine
* reconnects on all the same, because this classifier is an allowlist and a gateway
* wording a transient fault its own way falls through it. User interruption ends with
* `aborted`; credentials failures end with their own terminal status `auth` (see
* `isAuthenticationError`) — the one status the engine refuses to retry, and the one
* hosts key on to gate input.
*
* `context_engine` only consumes OmniMessage; all Uni* protocol details are encapsulated here.
* Docs: /docs/interfaces § "The built-in implementation: GenerativeModel".
@@ -1078,11 +1081,12 @@ export class GenerativeModel implements LLMInterface {
* - **User interruption**: `finishInterrupted("aborted")` closes out, produces no usage →
* `aborted`;
* - **Credentials failure**: `finishInterrupted("auth")` closes out, produces no usage →
* `auth` (carrying `message`) — the engine stops exactly like `failed`, hosts key on
* the status to gate input until the model's API key is updated;
* - **Other non-retryable errors** (parameters etc.): `finishInterrupted("failed")`
* closes out, produces no usage → `failed` (carrying `message`), handed to
* `context_engine` to stop and return control to the user.
* `auth` (carrying `message`) — the one status the engine stops the run on, and the one
* hosts key on to gate input until the model's API key is updated;
* - **Every other error** (parameters etc., and input that never assembled into a
* request): `finishInterrupted("failed")` closes out, produces no usage → `failed`
* (carrying `message`), which `context_engine` reconnects on as well — the
* classification stays honest, the retry decision is the engine's.
*
* Timeout detection: the idle timer resets on every event received; once idle exceeds
* `requestTimeoutMs`, the underlying stream is aborted and handled as needing reconnection
@@ -1103,10 +1107,7 @@ export class GenerativeModel implements LLMInterface {
try {
uniMessage = mergeOmniToUniMessage(params.newMessages);
} catch (err) {
return {
status: "failed",
message: describeError(err),
};
return { status: "failed", message: describeError(err) };
}
const translator = new EventTranslator(this.toolCallIds);
packages/core/src/state/default-config.ts
+1 -1
View File
@@ -112,7 +112,7 @@ Communicate with the user precisely and concisely, yet with warmth. Do not repea
# System markers
Some messages contain system-synthesized blocks written as \`[tag]...[/tag]\`, not user text to answer directly:
- \`[turn_aborted]\`: the previous round was interrupted. Inside are the original request, your partial thinking/text, and the tool calls already issued with their results. Continue from where it left off; do not re-run tools whose results are already included.
- \`[turn_retried]\`: the previous attempt of this round failed on a transient error (timeout, disconnect, malformed response, or a temporary provider error) — the user did NOT interrupt — and this request is the automatic retry. Inside are your partial thinking/text and the tool calls already executed with their results. Continue from them; do not re-run tools whose results are already included.
- \`[turn_retried]\`: the previous attempt of this round did not get through (a timeout, a disconnect, a malformed response, or an error the provider returned) — the user did NOT interrupt — and this request is the automatic retry. Inside are your partial thinking/text and the tool calls already executed with their results. Continue from them; do not re-run tools whose results are already included.
- \`[context_summary]\`: earlier conversation was compacted. This summary replaced the raw transcript and is its only record; treat it as established context and continue the task from it.
- \`[user_steering]\`: a user message sent while you were still working, delivered between turns alongside tool results. It is not a new task: incorporate it immediately and adjust course within the current task.
packages/core/test/compaction.test.ts
+39 -4
View File
@@ -12,8 +12,9 @@
* stay byte-identical, #84), and a completed response only counts as success with a valid
* summary — non-empty extracted text and no tool calls (issue #83). A rejected response has its
* tool calls answered by synthesized failed outputs (pairing repair) and is retried under a
* dedicated cap of 5 rejections; then the compaction fails. Transport timeout/malformed
* attempts keep the shared reconnect cap.
* dedicated cap of 5 rejections; then the compaction fails. Retryable attempts
* (failed/timeout/malformed) take the compaction-specific reconnect cap — the same set the
* turn loop retries, on a shorter budget; only `auth` stops at once.
* - discard: deferred until task end if mid-task; sends no compaction request, just swaps in a new LLM instance directly.
* - Process visibility: the compaction request's streamed output is never surfaced to the human,
* only the paired compaction events are emitted; the dialogue is written to the old trace, and
@@ -306,8 +307,8 @@ describe("context compaction", () => {
{
messages: [toolCall({ name: "t", arguments: "{}", toolCallId: "c1" }), usage(150, 150)],
},
// Compaction request fails (not retryable).
{ messages: [], outcome: { status: "failed", message: "auth error" } },
// Compaction request fails on the one status no ladder can fix (a rejected credential).
{ messages: [], outcome: { status: "auth", message: "auth error" } },
// Original context is kept: the task continues, tool outputs feed back into the old instance as usual (context usage keeps growing).
{ messages: [assistantText("finished on old context"), usage(190, 340)] },
// Second trigger (context still over the limit) -> retries compaction at the boundary, this time succeeding.
@@ -420,6 +421,40 @@ describe("context compaction", () => {
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
});
it("a failed compaction request takes the ladder and can recover on a later attempt", async () => {
// The compaction loop retries the same statuses the turn loop does: `failed` is where a
// transient fault lands whenever the classifier doesn't recognize the gateway's wording,
// and giving up here keeps the full context, so the next request re-triggers compaction
// against the same wall with less headroom.
const llm1 = new ScriptedLLM(
[
{ messages: [assistantText("answer"), usage(150, 150)] },
{ messages: [], outcome: { status: "failed", message: "502 upstream" } },
{ messages: [assistantText("[summary]recovered[/summary]")] },
],
"llm1",
);
const llm2 = new ScriptedLLM([], "llm2");
const engine = new ContextEngine({
llm: llm1,
environment: fakeEnvironment,
compaction: settings(),
createLLM: () => llm2,
compactionMaxReconnects: 2,
reconnectBackoffMs: 1,
});
const out = await collect(engine.run([userText("go")], { approve: allowAll }));
expect(compactionEvents(out)[1]).toMatchObject({
type: "compaction_end",
status: "completed",
});
// 1 turn request + 2 compaction attempts (the failed one, then the retry that succeeds).
expect(llm1.calls).toHaveLength(3);
// The retry resends the same input (tool results + prompt; only the prompt here).
expect(payloadTypes(llm1.calls[2]!)).toEqual(["text"]);
});
it("the compaction loop uses its own cap, not the shared maxReconnects ladder", async () => {
// Compaction failure has graceful semantics (the original context is kept, compaction
// retries on the next trigger), so it fails fast: with compactionMaxReconnects=2, the
packages/core/test/engine.test.ts
+78 -24
View File
@@ -521,7 +521,7 @@ describe("ContextEngine ReAct loop (mock LLM, approve callback)", () => {
expect(texts.join("\n")).not.toContain("[turn_aborted]");
});
it("downgrades a goal round's protocol in the [turn_aborted] transcript (LLM failure path)", async () => {
it("downgrades a goal round's protocol in the [turn_aborted] transcript (auth exit path)", async () => {
// An aborted/failed goal round's input rides into the next task via flatten carry-over;
// its [goal] protocol ("the system sends the next round automatically", the file rules)
// is stale the moment the goal ends and must not re-enter the model as live instructions.
@@ -541,7 +541,8 @@ describe("ContextEngine ReAct loop (mock LLM, approve callback)", () => {
if (++calls === 1) {
yield partialText("start", "");
yield partialText("delta", "half a thought");
return { status: "failed", message: "boom" };
// `auth`: the one LLM status that still exits straight to the flatten path.
return { status: "auth", message: "boom" };
}
yield assistantText("ok");
yield tokenUsage(emptyTokenCounts(), {
@@ -1644,14 +1645,14 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
expect(nextRunTexts.join("\n")).not.toContain("[turn_aborted]");
});
it("surfaces a non-retryable LLM failure (outcome=failed) as a graceful abort (run does not throw)", async () => {
it("a failed outcome retries like a timeout, then converges to a graceful abort (run does not throw)", async () => {
let calls = 0;
const inputs: OmniMessage[][] = [];
const llm: LLMInterface = {
// The LLM must never throw an exception at the engine: a non-retryable error resolves
// by returning a failed outcome after closing the structure. A genuinely non-retryable
// failure nowadays is a parameter error (quota 403s retry as timeout, 401s carry
// code "auth" — both covered by their own tests below).
// The LLM must never throw an exception at the engine: an error resolves by returning
// a failed outcome after closing the structure. `failed` is still the honest
// classification for a parameter error — it is simply retried anyway, because the
// classifier cannot reliably tell a permanent 4xx from a gateway's transient one.
// eslint-disable-next-line require-yield
async *streamGenerate(params) {
calls += 1;
@@ -1663,31 +1664,81 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
workspaceDir: workspace,
toolConfig: execCommandToolConfig(),
});
const engine = new ContextEngine({ llm, environment, reconnectBackoffMs: 0 });
const engine = new ContextEngine({
llm,
environment,
maxReconnects: 2,
reconnectBackoffMs: 0,
});
// Must not throw; should gracefully converge to an abort.
// Must not throw; should gracefully converge to an abort once the ladder is spent.
const all = await collectRun(engine, [userText("go")], allowAll);
expect(calls).toBe(1); // failed -> no retry.
expect(calls).toBe(3); // initial attempt + maxReconnects(2): `failed` takes the ladder now.
const abort = all.find((m) => (m.payload as { type?: string }).type === "abort");
expect(abort).toBeDefined();
// Asserted whole, not by fragments: this string is shown verbatim in the error panel and
// the CLI and is persisted as the error message, so its grammar is part of the contract.
const reason = (abort!.payload as { reason?: string }).reason ?? "";
expect(reason).toContain("llm request error");
expect(reason).toContain("unknown parameter");
expect(reason).toBe(
"llm request failed after 2 retries: 400 unknown parameter: max_output_tokens",
);
// The failed turn's input is flattened and stashed; the next run resends it merged with
// the new input.
// The spent turn's input is stashed as carry-over; the next run (attempt index 3, after
// this run's three) resends it merged with the new input.
await collectRun(engine, [userText("next")], allowAll);
const text = inputs[1]!.map((m) => (m.payload as { text?: string }).text ?? "").join("\n");
const text = inputs[3]!.map((m) => (m.payload as { text?: string }).text ?? "").join("\n");
expect(text).toContain("go");
expect(text).toContain("next");
});
it("an auth outcome stops immediately like failed: no retry, request_end carries status auth", async () => {
it("a failed request that succeeds on retry never reaches the user as an error", async () => {
// The point of retrying `failed`: the classifier is an allowlist, so a transient gateway
// fault phrased its own way ("Upstream HTTP/2 stream failed") lands here. It used to kill
// the turn; now the turn simply completes.
let calls = 0;
const llm: LLMInterface = {
async *streamGenerate() {
calls += 1;
if (calls === 1) {
return {
status: "failed" as const,
message: "Upstream HTTP/2 stream failed (upstream_http2_stream_error)",
};
}
yield assistantText("recovered");
return { status: "completed" as const };
},
};
const engine = new ContextEngine({
llm,
environment: new Environment({
workspaceDir: workspace,
toolConfig: execCommandToolConfig(),
}),
reconnectBackoffMs: 0,
});
const all = await collectRun(engine, [userText("go")], allowAll);
expect(calls).toBe(2);
expect(all.find((m) => (m.payload as { type?: string }).type === "abort")).toBeUndefined();
// The failure is still classified `failed` on the wire — the retry is a policy decision,
// not a relabelling, so observability still sees a real failure rather than a "timeout".
const ends = all.filter((m) => (m.payload as { type?: string }).type === "request_end");
expect(ends.map((m) => (m.payload as { status?: string }).status)).toEqual([
"failed",
"completed",
]);
// ...and it announces its retry wait like any other retryable failure, so the frontend
// countdown works for it too.
expect((ends[0]!.payload as { retry_in_ms?: number }).retry_in_ms).toBeGreaterThanOrEqual(0);
});
it("auth is the only LLM status that stops the run: no retry, request_end carries status auth", async () => {
let calls = 0;
const llm: LLMInterface = {
// GenerativeModel classifies a 401/invalid_api_key as status "auth" (see
// llm.test.ts); the engine must stop directly — the auth/failed branch is the second
// belt keeping a dead credential out of the retry loop (the classifier is the first).
// llm.test.ts); the engine must stop directly — the auth branch is the second belt
// keeping a dead credential out of the retry loop (the classifier is the first).
// Every other failure, `failed` included, takes the ladder instead.
// eslint-disable-next-line require-yield
async *streamGenerate() {
calls += 1;
@@ -1701,7 +1752,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
const engine = new ContextEngine({ llm, environment, reconnectBackoffMs: 1 });
const all = await collectRun(engine, [userText("go")], allowAll);
expect(calls).toBe(1); // Auth behaves like failed: never enters the reconnect loop.
expect(calls).toBe(1); // A rejected credential cannot be retried into working.
// The request's own terminal status is the host signal (streams to the web).
const end = all.find((m) => (m.payload as { type?: string }).type === "request_end");
expect((end!.payload as { status?: string }).status).toBe("auth");
@@ -1970,7 +2021,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
expect(all.map((m) => (m.payload as { type?: string }).type)).not.toContain("abort");
});
it("flatten carry-over (failed exit) includes the model's partial thinking and text (PRN-014)", async () => {
it("flatten carry-over (auth exit) includes the model's partial thinking and text (PRN-014)", async () => {
let calls = 0;
const inputs: OmniMessage[][] = [];
const llm: LLMInterface = {
@@ -1978,11 +2029,14 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
calls += 1;
inputs.push(params.newMessages);
if (calls === 1) {
// Before the non-retryable error, partial thinking and text were already produced
// (the LLM finishes them as complete messages, stop_reason failed).
// Before the terminal error, partial thinking and text were already produced (the
// LLM finishes them as complete messages). `auth` is the trigger because it is the
// one LLM status that still exits straight to the flatten path — `failed` now takes
// the reconnect ladder, whose carry-over is [turn_retried] instead (covered by the
// exhausted-retries test below).
yield thinkingMessage("half-thought", "failed");
yield assistantText("half-text", "failed");
return { status: "failed", message: "boom" };
return { status: "auth", message: "boom" };
}
yield assistantText("ok");
yield tokenUsage(emptyTokenCounts(), {
@@ -2001,7 +2055,7 @@ describe("ContextEngine LLM timeout / network interruption (PRN-012)", () => {
const engine = new ContextEngine({ llm, environment, reconnectBackoffMs: 0 });
await collectRun(engine, [userText("go")], allowAll);
expect(calls).toBe(1); // failed -> no retry, exits immediately.
expect(calls).toBe(1); // auth -> no retry, exits immediately.
// Next run: the flattened carry-over contains the original input plus partial thinking/text
// (both completed and incomplete messages are carried over).
packages/core/test/llm.test.ts
+1 -1
View File
@@ -1482,7 +1482,7 @@ describe("GenerativeModel.streamGenerate outcome classification (PRN-013)", () =
return { messages, outcome: res.value as LLMOutcome };
}
it("returns failed (never throws) on a build failure such as empty input", async () => {
it("returns failed on a build failure such as empty input (never throws)", async () => {
const model = new SeamModel((sig) => hang(sig));
const { messages, outcome } = await drain(model.streamGenerate({ newMessages: [] }));
expect(outcome.status).toBe("failed"); // A mergeOmniToUniMessage failure converges to failed, never throws