fix(core): serialize Trace appends; tolerate malformed middle lines on read (#234)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yaowei ZhengandClaude Fable 5 authored and GitHub committed 2026-08-07 17:49:40 +08:00
1 parent 9745773cb9
commit 4a899a265f
9 files changed
+344 -33

No files matched your search

+1 -1
View File
@@ -7,4 +7,4 @@ export {
readTraceTolerant, readTraceTolerant,
resumeTrace, resumeTrace,
} from "./resume.js"; } from "./resume.js";
export type { LocatedTraceFile, ResumeResult } from "./resume.js"; export type { LocatedTraceFile, ParseTraceLinesOptions, ResumeResult } from "./resume.js";
+58 -8
View File
@@ -77,13 +77,44 @@ export interface ResumeResult {
/** Content of the pairing-backfill placeholder output (the tool hadn't finished and no output was persisted before the process exited). */ /** Content of the pairing-backfill placeholder output (the tool hadn't finished and no output was persisted before the process exited). */
const PROCESS_EXIT_PLACEHOLDER = "[interrupted: process exited before the tool finished]"; const PROCESS_EXIT_PLACEHOLDER = "[interrupted: process exited before the tool finished]";
/** Options for `parseTraceLines`. */
export interface ParseTraceLinesOptions {
/**
* Handling of a malformed line **in the middle** of the content (the trailing truncated
* line is always tolerated and ignored, see `parseTraceLines`):
* - "skip" (default): drop that line and keep every parseable record. Reading a Trace back
* is best-effort — the corruption already happened, and throwing away the whole file
* helps no one. Replay is built to stay structurally valid with records missing (crash
* tolerance, pairing backfill), so surviving records still resume.
* - "throw": rethrow the JSON parse error. For validating untrusted input where damage
* should be reported instead of silently repaired (server-side Trace import).
*/
onMalformed?: "skip" | "throw";
/** Called for each skipped malformed middle line (1-based line number). Not called for the tolerated trailing truncated line. */
onSkip?: (lineNumber: number, error: unknown) => void;
}
/** /**
* Parse Trace JSONL content. Tolerates a **truncated last line** left behind by an abnormal * Parse Trace JSONL content. Tolerates a **truncated last line** left behind by an abnormal
* process exit (that line is ignored); corruption in the middle is outside the crash window * process exit (that line is silently ignored). A malformed line in the **middle** — e.g. a
* (append-only, single writer), so it throws loudly. * record torn by the pre-fix concurrent-append interleaving (#215) — is skipped by default
* (reported via `onSkip`), keeping every parseable record; pass `onMalformed: "throw"` to
* make middle corruption a hard error instead.
*/ */
export function parseTraceLines(content: string): OmniMessage[] { export function parseTraceLines(content: string, options?: ParseTraceLinesOptions): OmniMessage[] {
const onMalformed = options?.onMalformed ?? "skip";
const lines = content.split("\n"); const lines = content.split("\n");
// Index of the last non-empty line, found with one backward scan up front. The malformed
// branch below runs once per skipped line in skip mode, so rescanning the remainder there
// (slice + every) would make widely damaged content O(n^2) — seconds of synchronous CPU on
// the server history path for a large non-JSONL file.
let lastNonEmptyIndex = -1;
for (let i = lines.length - 1; i >= 0; i--) {
if (lines[i]!.trim().length > 0) {
lastNonEmptyIndex = i;
break;
}
}
const out: OmniMessage[] = []; const out: OmniMessage[] = [];
for (let i = 0; i < lines.length; i++) { for (let i = 0; i < lines.length; i++) {
const line = lines[i]!.trim(); const line = lines[i]!.trim();
@@ -91,17 +122,36 @@ export function parseTraceLines(content: string): OmniMessage[] {
try { try {
out.push(JSON.parse(line) as OmniMessage); out.push(JSON.parse(line) as OmniMessage);
} catch (err) { } catch (err) {
const isLastNonEmpty = lines.slice(i + 1).every((l) => l.trim().length === 0); if (i === lastNonEmptyIndex) break; // truncated tail: expected crash artifact, ignore silently
if (isLastNonEmpty) break; if (onMalformed === "throw") throw err;
throw err; options?.onSkip?.(i + 1, err);
} }
} }
return out; return out;
} }
/** Read and parse a Trace file (tolerates a truncated last line). */ /** Maximum count of skipped line numbers spelled out in the readTraceTolerant diagnostic. */
const SKIPPED_LINES_SHOWN = 20;
/**
* Read and parse a Trace file, best-effort: tolerates a truncated last line, and skips
* malformed middle lines (files damaged by the pre-fix concurrent-append bug, #215) while
* keeping every parseable record. Skipped lines are diagnosed once per read on stderr, with
* the spelled-out line-number list capped (a heavily damaged file can skip thousands).
*/
export async function readTraceTolerant(path: string): Promise<OmniMessage[]> { export async function readTraceTolerant(path: string): Promise<OmniMessage[]> {
return parseTraceLines(await readFile(path, "utf8")); const skipped: number[] = [];
const messages = parseTraceLines(await readFile(path, "utf8"), {
onSkip: (lineNumber) => skipped.push(lineNumber),
});
if (skipped.length > 0) {
const shown = skipped.slice(0, SKIPPED_LINES_SHOWN);
const ellipsis = skipped.length > shown.length ? ", …" : "";
process.stderr.write(
`[trace] skipped ${skipped.length} malformed line(s) [${shown.join(", ")}${ellipsis}] in ${path}\n`,
);
}
return messages;
} }
/** A located Trace file: its path, containing date-directory name, and index. */ /** A located Trace file: its path, containing date-directory name, and index. */
+59 -10
View File
@@ -68,9 +68,30 @@ function isRecordable(msg: OmniMessage): boolean {
/** /**
* append-only JSONL Trace writer. * append-only JSONL Trace writer.
* *
* Single-writer scenario (MVP): concurrency safety isn't required, but every write uses * Every write uses `appendFile` (O_APPEND) rather than caching a file handle and seeking to
* `appendFile` (O_APPEND) rather than caching a file handle and seeking to write, avoiding * write, avoiding overwriting existing content; this also removes the need for an explicit
* overwriting existing content; this also removes the need for an explicit close. * close.
*
* Concurrency: one live Session has exactly one Writer instance, but that instance receives
* appends from **multiple async producers in the same process** — the engine's LLM stream
* driver and each parallel tool execution write independently. `appendFile` splits large
* payloads (multi-MB records such as base64 image Data URLs) into multiple underlying writes,
* so two overlapping appends can interleave mid-record and corrupt the JSONL (#215). All
* mutating operations (`write`, `rotate`) are therefore serialized through one per-instance
* promise chain: each record lands as one uninterrupted append, and a rotation cannot land in
* the middle of an append. Cross-instance/file concurrency does not occur by design **within a
* single server/CLI process**: a Session allows one active run at a time, child sessions write
* their own files, and server-side Trace import only ever creates brand-new files (`wx`). Two
* processes pointed at the same agent data directory are outside this contract (and outside
* supported usage) — the chain cannot cover them.
*
* Error semantics: a failed operation rejects **that caller's** returned promise only; the
* chain itself absorbs the failure so subsequent operations still run (a transient disk error
* must not wedge Trace recording for the rest of the session). A **hung** append is different:
* it blocks this instance's chain until it settles — deliberate head-of-line blocking, because
* ordering cannot be preserved around an append whose outcome is still unknown. Callers
* (context_engine, Session) already treat Trace writes as best-effort and log the surfaced
* error.
*/ */
export class Writer { export class Writer {
private readonly tracesDir: string; private readonly tracesDir: string;
@@ -80,6 +101,8 @@ export class Writer {
private index = 1; private index = 1;
/** Set true once the date directory has been created for the current file, to avoid a redundant mkdir. */ /** Set true once the date directory has been created for the current file, to avoid a redundant mkdir. */
private ensuredDirForIndex = -1; private ensuredDirForIndex = -1;
/** Serialization chain: mutating operations run strictly in submission order (see class docs). */
private chain: Promise<void> = Promise.resolve();
constructor(opts: WriterOptions) { constructor(opts: WriterOptions) {
this.tracesDir = opts.tracesDir; this.tracesDir = opts.tracesDir;
@@ -94,18 +117,40 @@ export class Writer {
return join(this.tracesDir, this.dateDir, fileName); return join(this.tracesDir, this.dateDir, fileName);
} }
/**
* Enqueues one mutating operation on the serialization chain. The returned promise settles
* with that operation's own outcome (so a failure surfaces to its caller), while the chain
* swallows the failure and proceeds with whatever was enqueued next.
*/
private enqueue<T>(op: () => Promise<T>): Promise<T> {
const result = this.chain.then(op);
this.chain = result.then(
() => undefined,
() => undefined, // a failed operation must not wedge the chain
);
return result;
}
/** /**
* Appends one message. Only written if it's a recordable message; streaming `partial_*` is * Appends one message. Only written if it's a recordable message; streaming `partial_*` is
* skipped. `mkdir -p`s the date directory on the first write to the current file. * skipped. `mkdir -p`s the date directory on the first write to the current file.
*
* The append is serialized on the instance chain, so the record lands as one uninterrupted
* JSONL line even when other writes (or a rotation) are submitted concurrently; the target
* path is resolved when the operation runs, so a write submitted after `rotate()` goes to
* the new file. The returned promise resolves only after this record has been appended
* (callers may rely on write-then-send ordering), and rejects if this append failed.
*/ */
async write(msg: OmniMessage): Promise<void> { async write(msg: OmniMessage): Promise<void> {
if (!isRecordable(msg)) return; if (!isRecordable(msg)) return;
const path = this.currentPath(); return this.enqueue(async () => {
if (this.ensuredDirForIndex !== this.index) { const path = this.currentPath();
await mkdir(dirname(path), { recursive: true }); if (this.ensuredDirForIndex !== this.index) {
this.ensuredDirForIndex = this.index; await mkdir(dirname(path), { recursive: true });
} this.ensuredDirForIndex = this.index;
await appendFile(path, `${JSON.stringify(msg)}\n`, "utf8"); }
await appendFile(path, `${JSON.stringify(msg)}\n`, "utf8");
});
} }
/** Writes multiple messages in sequence. */ /** Writes multiple messages in sequence. */
@@ -132,10 +177,14 @@ export class Writer {
/** /**
* Starts a new Trace file: increments the index, so the next `write` goes to the new file. * Starts a new Trace file: increments the index, so the next `write` goes to the new file.
* Used to split into a separate file when the context is compacted and a new context segment is produced. * Used to split into a separate file when the context is compacted and a new context segment is produced.
* Serialized on the instance chain: appends submitted before the rotation land in the old
* file, appends submitted after it land in the new one — a rotation can never split a record.
* Docs: /docs/sessions-and-traces § "Trace design". * Docs: /docs/sessions-and-traces § "Trace design".
*/ */
async rotate(): Promise<void> { async rotate(): Promise<void> {
this.index += 1; return this.enqueue(async () => {
this.index += 1;
});
} }
} }
+122 -6
View File
@@ -12,15 +12,16 @@
* - Tolerates a truncated trailing line left by an abnormal process exit. * - Tolerates a truncated trailing line left by an abnormal process exit.
* - Round-trip: a Trace written out by the engine, once replayed, matches the history the model actually received. * - Round-trip: a Trace written out by the engine, once replayed, matches the history the model actually received.
*/ */
import { mkdtemp, rm } from "node:fs/promises"; import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { import {
assistantText, assistantText,
compactionBegin, compactionBegin,
compactionEnd, compactionEnd,
emptyTokenCounts, emptyTokenCounts,
imageUrlMessage,
requestBegin, requestBegin,
requestEnd, requestEnd,
sessionMeta, sessionMeta,
@@ -31,7 +32,7 @@ import {
userText, userText,
} from "../src/omnimessage/index.js"; } from "../src/omnimessage/index.js";
import type { OmniMessage, TokenCounts } from "../src/omnimessage/index.js"; import type { OmniMessage, TokenCounts } from "../src/omnimessage/index.js";
import { parseTraceLines, resumeTrace } from "../src/trace/resume.js"; import { parseTraceLines, readTraceTolerant, resumeTrace } from "../src/trace/resume.js";
import { ContextEngine } from "../src/engine/context-engine.js"; import { ContextEngine } from "../src/engine/context-engine.js";
import { Environment } from "../src/environment/index.js"; import { Environment } from "../src/environment/index.js";
import { Writer, readTrace } from "../src/trace/index.js"; import { Writer, readTrace } from "../src/trace/index.js";
@@ -324,15 +325,130 @@ describe("resumeTrace", () => {
}); });
describe("parseTraceLines", () => { describe("parseTraceLines", () => {
/**
* Content shaped like the #215 corruption: a large record torn in half by a concurrent
* append, with a complete small record and a newline landing in the middle. Line 2
* (big prefix + small record) and line 3 (big suffix) are both unparseable.
*/
function tornContent(): { content: string; survivors: OmniMessage[] } {
const before = userText("before");
const after = userText("after");
const big = JSON.stringify(imageUrlMessage(`data:image/png;base64,${"A".repeat(2048)}`));
const interleaved = JSON.stringify(tokenUsage(usage(1), usage(1)));
const torn = `${big.slice(0, 100)}${interleaved}\n${big.slice(100)}`;
const content = `${JSON.stringify(before)}\n${torn}\n${JSON.stringify(after)}\n`;
return { content, survivors: [before, after] };
}
it("tolerates a torn trailing line (process crash mid-write)", () => { it("tolerates a torn trailing line (process crash mid-write)", () => {
const content = `${JSON.stringify(userText("a"))}\n{"timestamp":"2026-07-06T`; const content = `${JSON.stringify(userText("a"))}\n{"timestamp":"2026-07-06T`;
const msgs = parseTraceLines(content); const skipped: number[] = [];
const msgs = parseTraceLines(content, { onSkip: (line) => skipped.push(line) });
expect(msgs).toHaveLength(1); expect(msgs).toHaveLength(1);
// The trailing truncated line is the expected crash artifact — ignored, not diagnosed.
expect(skipped).toEqual([]);
}); });
it("throws on mid-file corruption", () => { it("skips a malformed middle line by default, keeping every parseable record (#215)", () => {
const { content, survivors } = tornContent();
const skipped: number[] = [];
const msgs = parseTraceLines(content, { onSkip: (line) => skipped.push(line) });
expect(msgs).toEqual(survivors);
expect(skipped).toEqual([2, 3]);
});
it('throws on mid-file corruption when onMalformed is "throw" (import validation)', () => {
const content = `not-json\n${JSON.stringify(userText("a"))}\n`; const content = `not-json\n${JSON.stringify(userText("a"))}\n`;
expect(() => parseTraceLines(content)).toThrow(); expect(() => parseTraceLines(content, { onMalformed: "throw" })).toThrow();
// The trailing truncated line stays tolerated even in strict mode.
const truncatedTail = `${JSON.stringify(userText("a"))}\n{"timestamp":"2026-07-06T`;
expect(parseTraceLines(truncatedTail, { onMalformed: "throw" })).toHaveLength(1);
});
it("handles thousands of malformed lines without quadratic rescans (wide corruption)", () => {
// A large mostly-unparseable file (e.g. a non-JSONL file dropped into the traces dir).
// The tail check must not rescan the remainder per skipped line — with the old
// slice-per-hit check, this content took seconds of synchronous CPU; now it's O(n).
const early = userText("early survivor");
const late = userText("late survivor");
const lines: string[] = [];
for (let i = 0; i < 5000; i++) lines.push(`{torn ${i}`);
lines.push(JSON.stringify(early));
for (let i = 0; i < 5000; i++) lines.push(`{torn ${5000 + i}`);
lines.push(JSON.stringify(late));
const content = `${lines.join("\n")}\n`;
const skipped: number[] = [];
const msgs = parseTraceLines(content, { onSkip: (line) => skipped.push(line) });
expect(msgs).toEqual([early, late]);
// Every malformed line is reported exactly once, in order, none treated as the tail
// (the last non-empty line is the valid "late survivor").
expect(skipped).toHaveLength(10000);
expect(skipped[0]).toBe(1);
expect(skipped[4999]).toBe(5000);
expect(skipped[5000]).toBe(5002);
expect(skipped[9999]).toBe(10001);
});
});
describe("readTraceTolerant", () => {
let dir: string;
beforeEach(async () => {
dir = await mkdtemp(join(tmpdir(), "penguin-tolerant-"));
});
afterEach(async () => {
await rm(dir, { recursive: true, force: true });
vi.restoreAllMocks();
});
it("returns the surviving records of a corrupted file without throwing (#215 recovery)", async () => {
// A file damaged by the pre-fix interleaving: a large record torn in half around a
// complete small record. No migration exists — the tolerant read IS the recovery.
const before = userText("before");
const after = userText("after");
const big = JSON.stringify(imageUrlMessage(`data:image/png;base64,${"B".repeat(4096)}`));
const interleaved = JSON.stringify(assistantText("landed mid-record"));
const file = join(dir, "sess_abc_001.jsonl");
await mkdir(dir, { recursive: true });
await writeFile(
file,
`${JSON.stringify(before)}\n${big.slice(0, 64)}${interleaved}\n${big.slice(64)}\n${JSON.stringify(after)}\n`,
"utf8",
);
const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
const msgs = await readTraceTolerant(file);
expect(msgs).toEqual([before, after]);
// The skipped lines are diagnosed once, with the file path.
const diagnostics = stderr.mock.calls
.map((c) => String(c[0]))
.filter((s) => s.includes("[trace]"));
expect(diagnostics).toHaveLength(1);
expect(diagnostics[0]).toContain("skipped 2 malformed line(s)");
expect(diagnostics[0]).toContain(file);
});
it("caps the spelled-out line numbers in the diagnostic on heavily damaged files", async () => {
const survivor = userText("survivor");
const badLines = Array.from({ length: 30 }, (_, i) => `{torn ${i}`);
const file = join(dir, "sess_abc_001.jsonl");
await mkdir(dir, { recursive: true });
await writeFile(file, `${badLines.join("\n")}\n${JSON.stringify(survivor)}\n`, "utf8");
const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
const msgs = await readTraceTolerant(file);
expect(msgs).toEqual([survivor]);
const diagnostic = stderr.mock.calls
.map((c) => String(c[0]))
.find((s) => s.includes("[trace]"))!;
// The total is exact, but only the first 20 line numbers are spelled out.
expect(diagnostic).toContain("skipped 30 malformed line(s)");
const list = /malformed line\(s\) \[([^\]]*)\]/.exec(diagnostic);
expect(list).not.toBeNull();
const firstTwenty = Array.from({ length: 20 }, (_, i) => i + 1).join(", ");
expect(list![1]).toBe(`${firstTwenty}, …`);
}); });
}); });
+89 -1
View File
@@ -1,4 +1,4 @@
import { mkdtemp, rm, stat } from "node:fs/promises"; import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join, sep } from "node:path"; import { join, sep } from "node:path";
@@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { import {
assistantText, assistantText,
imageUrlMessage,
partialText, partialText,
sessionMeta, sessionMeta,
subagentEvent, subagentEvent,
@@ -13,6 +14,7 @@ import {
emptyTokenCounts, emptyTokenCounts,
withOrigin, withOrigin,
} from "../src/omnimessage/index.js"; } from "../src/omnimessage/index.js";
import type { OmniMessage } from "../src/omnimessage/index.js";
import { Writer, readTrace } from "../src/trace/index.js"; import { Writer, readTrace } from "../src/trace/index.js";
const SESSION_ID = "sess_abc"; const SESSION_ID = "sess_abc";
@@ -155,4 +157,90 @@ describe("Writer", () => {
// append-only verification: the old file's row count does not increase. // append-only verification: the old file's row count does not increase.
expect((await readTrace(firstPath)).length).toBe(2); expect((await readTrace(firstPath)).length).toBe(2);
}); });
// #215 regression: appends come from multiple async producers in one process (the LLM stream
// driver plus each parallel tool), and fs.appendFile splits multi-MB payloads into multiple
// underlying writes — without serialization a concurrent append can land mid-record.
describe("concurrency (#215)", () => {
it("keeps a multi-MB record intact under concurrent small appends", async () => {
const writer = new Writer({
tracesDir,
sessionId: SESSION_ID,
date: new Date(2026, 0, 9),
});
// Several MB, like a base64 image Data URL — large enough that fs.appendFile splits it
// into multiple underlying writes (Node chunks the data at 512 KiB), small enough to
// keep the test fast.
const big = imageUrlMessage(`data:image/png;base64,${"A".repeat(3 * 1024 * 1024)}`);
const submitted: OmniMessage[] = [
meta(),
big,
...[1, 2, 3, 4, 5].map((i) => assistantText(`small ${i}`)),
];
// Submit every write before awaiting any: concurrent producers, submission order known.
await Promise.all(submitted.map((msg) => writer.write(msg)));
// Every non-empty line parses independently, and count and order match submission.
const raw = await readFile(writer.currentPath(), "utf8");
const lines = raw.split("\n").filter((line) => line.trim().length > 0);
expect(lines).toEqual(submitted.map((msg) => JSON.stringify(msg)));
for (const line of lines) expect(() => JSON.parse(line)).not.toThrow();
});
it("serializes rotate() against in-flight writes: shard boundaries follow submission order", async () => {
const writer = new Writer({
tracesDir,
sessionId: SESSION_ID,
date: new Date(2026, 0, 9),
});
const firstPath = writer.currentPath();
// A large record right before the rotation: the rotation must wait for the whole
// append, and writes submitted after it must land in the next shard.
const big = imageUrlMessage(`data:image/png;base64,${"B".repeat(2 * 1024 * 1024)}`);
const shardOne: OmniMessage[] = [meta(), big];
const shardTwo: OmniMessage[] = [meta(), assistantText("new context")];
await Promise.all([
writer.write(shardOne[0]!),
writer.write(shardOne[1]!),
writer.rotate(),
writer.write(shardTwo[0]!),
writer.write(shardTwo[1]!),
]);
const secondPath = writer.currentPath();
expect(secondPath.endsWith(`${SESSION_ID}_002.jsonl`)).toBe(true);
const firstLines = (await readFile(firstPath, "utf8"))
.split("\n")
.filter((line) => line.trim().length > 0);
const secondLines = (await readFile(secondPath, "utf8"))
.split("\n")
.filter((line) => line.trim().length > 0);
expect(firstLines).toEqual(shardOne.map((msg) => JSON.stringify(msg)));
expect(secondLines).toEqual(shardTwo.map((msg) => JSON.stringify(msg)));
});
it("a failed write rejects its own caller without wedging subsequent writes", async () => {
const writer = new Writer({
tracesDir,
sessionId: SESSION_ID,
date: new Date(2026, 0, 9),
});
// Occupy the date-dir path with a regular file so mkdir (and the append) must fail.
const dateDirPath = join(tracesDir, "2026-01-09");
await writeFile(dateDirPath, "blocker", "utf8");
const results = await Promise.allSettled([
writer.write(meta()),
writer.write(assistantText("also blocked")),
]);
// Each failure surfaces to its own caller (callers log it as best-effort)...
expect(results.map((r) => r.status)).toEqual(["rejected", "rejected"]);
// ...and the chain is not wedged: once the cause clears, later writes land normally.
await rm(dateDirPath);
const recovered = assistantText("after recovery");
await writer.write(recovered);
const rows = await readTrace(writer.currentPath());
expect(rows).toEqual([recovered]);
});
});
}); });
@@ -54,6 +54,7 @@ A Trace is an append-only JSON Lines file; each line is one OmniMessage envelope
- Recorded: `session_meta`, complete `model_msg`, and all `event_msg`. - Recorded: `session_meta`, complete `model_msg`, and all `event_msg`.
- Not recorded: streaming `partial_*` fragments (the producer appends the complete message once the segment ends), and nested messages tagged with `origin` — a subagent's messages go to the child Session's own Trace, while the parent Trace keeps a single `subagent` pointer event at the spawn site recording the child Session id. - Not recorded: streaming `partial_*` fragments (the producer appends the complete message once the segment ends), and nested messages tagged with `origin` — a subagent's messages go to the child Session's own Trace, while the parent Trace keeps a single `subagent` pointer event at the spawn site recording the child Session id.
- `request_begin` and `request_end(status)` come in pairs delimiting one Request; replay uses `request_end.status === "completed"` as the commit criterion for that turn. - `request_begin` and `request_end(status)` come in pairs delimiting one Request; replay uses `request_end.status === "completed"` as the commit criterion for that turn.
- Appends are serialized inside the writer: records from concurrent producers (the model stream, parallel tool executions) land strictly one after another, each as a single uninterrupted line — a multi-megabyte record such as a base64 image Data URL can never be torn apart by a concurrent append, and file rotation never splits a record.
See `packages/core/src/trace/writer.ts` for the implementation. See `packages/core/src/trace/writer.ts` for the implementation.
@@ -82,7 +83,7 @@ The Trace is the single source of truth for recovery — there is no separate se
4. Reconstruct the carry-over (undelivered tool outputs, interruption markers) plus turn and Token counters; 4. Reconstruct the carry-over (undelivered tool outputs, interruption markers) plus turn and Token counters;
5. Continue appending to the same Trace file. 5. Continue appending to the same Trace file.
Recovery requires that the Workspace and the model still exist. What recovery guarantees is structural legality: only committed turns are replayed, with `tool_call` / `tool_call_output` pairing intact; incomplete model output (thinking, text) is allowed to be lost. A truncated last line left by an abnormal process exit is tolerated and ignored. See `packages/core/src/trace/resume.ts`. Recovery requires that the Workspace and the model still exist. What recovery guarantees is structural legality: only committed turns are replayed, with `tool_call` / `tool_call_output` pairing intact; incomplete model output (thinking, text) is allowed to be lost. A truncated last line left by an abnormal process exit is tolerated and ignored; a malformed line in the middle of a file (e.g. in a file damaged before writer appends were serialized) is skipped with a diagnostic on stderr, and every parseable record is kept. See `packages/core/src/trace/resume.ts`.
Special case: if the latest Trace file ends with a completed compaction, that context is closed as a whole — resume starts from an empty context; in summarize mode the `[context_summary]` is reconstructed and prepended to the first input after resume (old Traces using the earlier angle-bracket `<summary>` form are still understood). Special case: if the latest Trace file ends with a completed compaction, that context is closed as a whole — resume starts from an empty context; in summarize mode the `[context_summary]` is reconstructed and prepended to the first input after resume (old Traces using the earlier angle-bracket `<summary>` form are still understood).
@@ -52,6 +52,7 @@ Trace 是 append-only 的 JSON Lines 文件,每行一个 OmniMessage 信封(
- 记录的消息:`session_meta`、完整的 `model_msg`、全部 `event_msg`。 - 记录的消息:`session_meta`、完整的 `model_msg`、全部 `event_msg`。
- 不记录的消息:流式 `partial_*` 分片(片段结束后由生产方补写完整消息);带 `origin` 标记的嵌套消息——子 Agent 的消息写入子 Session 自己的 Trace,父 Trace 只在派生位置保留一个 `subagent` 指针事件,记录子 Session id。 - 不记录的消息:流式 `partial_*` 分片(片段结束后由生产方补写完整消息);带 `origin` 标记的嵌套消息——子 Agent 的消息写入子 Session 自己的 Trace,父 Trace 只在派生位置保留一个 `subagent` 指针事件,记录子 Session id。
- `request_begin` 与 `request_end(status)` 成对出现,界定一轮 Request;回放以 `request_end.status === "completed"` 作为该轮已提交的判据。 - `request_begin` 与 `request_end(status)` 成对出现,界定一轮 Request;回放以 `request_end.status === "completed"` 作为该轮已提交的判据。
- 追加在写入器内部串行执行:并发生产者(模型流、并行工具执行)的记录严格逐条落盘,每条都是一行完整内容——base64 图片 Data URL 之类的多 MB 大记录不会被并发追加撕裂,文件轮转也不会切断任何记录。
实现见 `packages/core/src/trace/writer.ts`。 实现见 `packages/core/src/trace/writer.ts`。
@@ -80,7 +81,7 @@ Trace 是恢复的唯一事实来源,没有独立的会话数据库需要与
4. 重建 carry-over(未送达的工具输出、中断标记)与轮数、Token 计数器; 4. 重建 carry-over(未送达的工具输出、中断标记)与轮数、Token 计数器;
5. 继续追加写入同一个 Trace 文件。 5. 继续追加写入同一个 Trace 文件。
恢复的前提是 Workspace 与模型仍然存在。恢复保证的是结构合法性:只回放已提交的轮次,`tool_call` 与 `tool_call_output` 配对完整;未完成的模型输出(thinking、文本)允许丢失。异常退出留下的截断末行会被容忍并忽略。实现见 `packages/core/src/trace/resume.ts`。 恢复的前提是 Workspace 与模型仍然存在。恢复保证的是结构合法性:只回放已提交的轮次,`tool_call` 与 `tool_call_output` 配对完整;未完成的模型输出(thinking、文本)允许丢失。异常退出留下的截断末行会被容忍并忽略;文件中间的损坏行(如写入器追加串行化之前受损的存量文件)会被跳过并在 stderr 给出诊断,其余可解析的记录全部保留。实现见 `packages/core/src/trace/resume.ts`。
特殊情形:若最新 Trace 文件以一次完成的压缩收尾,则该上下文已整体关闭——恢复从空上下文开始;summarize 模式下会重建 `[context_summary]` 摘要,前置到恢复后第一轮输入中(旧 Trace 中早期的尖括号 `<summary>` 形式仍可识别)。 特殊情形:若最新 Trace 文件以一次完成的压缩收尾,则该上下文已整体关闭——恢复从空上下文开始;summarize 模式下会重建 `[context_summary]` 摘要,前置到恢复后第一轮输入中(旧 Trace 中早期的尖括号 `<summary>` 形式仍可识别)。
@@ -2,9 +2,10 @@
* Trace service. * Trace service.
* *
* History messages: all of the Session's index files concatenated in order * History messages: all of the Session's index files concatenated in order
* (readTraceTolerant, tolerating a truncated last line), containing only the * (readTraceTolerant, tolerating a truncated last line and skipping malformed
* complete messages and events that were actually written to Trace (naturally * middle lines), containing only the complete messages and events that were
* excluding partial_*); in-flight increments are continued by SSE. * actually written to Trace (naturally excluding partial_*); in-flight
* increments are continued by SSE.
* Performance analysis is derived from a single Trace file: nearest-neighbor * Performance analysis is derived from a single Trace file: nearest-neighbor
* pairing of request_begin/end, tool call duration pairing, reconnect / compaction * pairing of request_begin/end, tool call duration pairing, reconnect / compaction
* counts, and Token trend. * counts, and Token trend.
@@ -1206,7 +1207,9 @@ export class TraceService {
const invalid = (message: string) => new HttpError(400, "invalid_trace", message); const invalid = (message: string) => new HttpError(400, "invalid_trace", message);
let records: OmniMessage[]; let records: OmniMessage[];
try { try {
records = parseTraceLines(content); // Strict parse: import is the gate for user-supplied files, so a malformed middle
// line is reported as 400 rather than silently dropped (read paths skip it instead).
records = parseTraceLines(content, { onMalformed: "throw" });
} catch { } catch {
throw invalid("The file is not valid Trace JSONL."); throw invalid("The file is not valid Trace JSONL.");
} }
@@ -189,7 +189,10 @@ describe("trace-import-export", () => {
it("import: malformed middle line → 400 invalid_trace", async () => { it("import: malformed middle line → 400 invalid_trace", async () => {
const lines = toContent(sampleTrace(SID)).split("\n"); const lines = toContent(sampleTrace(SID)).split("\n");
lines[1] = "{not json"; // corrupt a middle line (non-final: parseTraceLines throws loudly) // Corrupt a middle (non-final) line. Read paths skip malformed middle lines (#215
// recovery), but import validates strictly (onMalformed: "throw") — damage in an
// uploaded file is reported, not silently dropped.
lines[1] = "{not json";
const res = await owner.post(`${base()}/import`, { dataBase64: b64(lines.join("\n")) }); const res = await owner.post(`${base()}/import`, { dataBase64: b64(lines.join("\n")) });
expect(res.status).toBe(400); expect(res.status).toBe(400);
expect(await errorCode(res)).toBe("invalid_trace"); expect(await errorCode(res)).toBe("invalid_trace");