/** * Trace service. * * History messages: all of the Session's index files concatenated in order * (readTraceTolerant, tolerating a truncated last line), containing only the * complete messages and events that were actually written to Trace (naturally * excluding partial_*); in-flight increments are continued by SSE. * Performance analysis is derived from a single Trace file: nearest-neighbor * pairing of request_begin/end, tool call duration pairing, reconnect / compaction * counts, and Token trend. */ import fs from "node:fs/promises"; import path from "node:path"; import { isSessionMeta, parseTraceLines, parseUserSteeringText, readTraceTolerant, tracesDir, } from "@prismshadow/penguin-core"; import type { OmniMessage } from "@prismshadow/penguin-core"; import type { AgentTraceSessionEntry, AgentTracesResponse, RequestSpan, SessionCategory, SessionCategoryCounts, ToolCallSpan, TraceAnalysisResponse, TraceEventsResponse, TraceFileInfo, TraceImportResponse, TraceModelSegment, TraceTaskStats, TraceToolSpan, UsageTrendPointInTrace, } from "../api/types.js"; import type { SessionRow } from "../db/repos/sessions.js"; import type { TraceFileRow, TraceSessionRow } from "../db/repos/trace-index.js"; import { HttpError } from "../http/errors.js"; import { formatLocalDate } from "../internal/dates.js"; import type { SessionSources } from "../runtime/session-sources.js"; import { cloneScanState, deserializePrefix, encodeCursor, finalizeScan, initialScanState, scanMessages, serializePrefix, } from "./message-window.js"; import type { ChildAggregate, MessageCursor, ScanState, WindowPriorStats, } from "./message-window.js"; import { TraceIndexService, traceFilePath } from "./trace-index.js"; const TRACE_FILE_RE = /^(.+)_(\d{3})\.jsonl$/; /** * session_id an imported Trace file may declare: same alphabet as resource ids plus a length * cap. The value becomes part of the target **filename**, so this is a path-traversal defense, * checked right next to the path construction — never trust the caller to have validated it. */ const IMPORT_SESSION_ID_RE = /^[A-Za-z0-9_-]{1,128}$/; /** Recursion depth cap for sub-session expansion (run_subagent depth is already constrained by the SDK; this is just a defensive backstop against cycles). */ const MAX_SUBAGENT_DEPTH = 4; interface LocatedFile { path: string; date: string; index: number; /** Last observed size from the index row (the page-stats cache write guard). */ sizeBytes: number; } /** A windowed history read's request shape (see readMessagesPage). */ export type MessagesPageRequest = { kind: "tail"; limit: number } | { kind: "before"; cursor: MessageCursor; limit: number }; /** A windowed history read's result (route maps it onto MessagesResponse.page). */ export interface MessagesPageResult { messages: OmniMessage[]; /** Cursor of the window's first unit; absent = the window reaches the beginning. */ before?: string; /** Cumulative stats before the window (earlierTurns = prior.turns). */ prior: WindowPriorStats; } /** * Per-request expansion context: `projectScanned` caps the miss-path force reconcile at * one per request; `ancestry` guards cycles; `raw` memoizes each child session's * concatenated shard messages so the window expansion and the subagent-token * aggregation never read the same child twice within one request. */ interface ExpandCtx { projectScanned: boolean; ancestry: Set; depth: number; raw: Map; } /** * A **direct sub-session pointer** (the `subagent` event in the parent Trace) -> * the sub-session's Session id. The pointer only * records the Session id; the sub-session's Agent is located within the Project by * its Trace file. */ function subagentPointer(msg: OmniMessage): string | null { if (msg.type !== "event_msg") return null; const p = msg.payload as { type?: string; session_id?: unknown }; if (p.type !== "subagent" || typeof p.session_id !== "string" || p.session_id === "") { return null; } return p.session_id; } /** * Read-only slice of the Session index the paginated listing consults (SessionsRepo * structurally satisfies it; injected so service tests need no full app): one indexed * query answers title / archived / workspace / client for every DB-tracked Session of * the Agent. */ export interface TraceSessionIndex { listByAgent(projectId: string, agentId: string): SessionRow[]; } /** TraceService wiring (the trace-file index is required: every listing/locating path serves from it — no directory walks). */ export interface TraceServiceDeps { index: TraceIndexService; /** Optional (narrow tests may omit): DB rows supplying titles / archived / workspace / client. */ sessions?: TraceSessionIndex; /** Optional (narrow tests may omit): the shared in-process Session-origin registry (single source of truth for `source`). */ sources?: SessionSources; /** * Test observability: called with the path of every Trace shard this service reads * from disk (windowed-read tests assert an old-window request never touches the * newest shard and vice versa). Production wiring omits it. */ observeShardRead?: (path: string) => void; } /** One Session's classification result (see classify): its sidebar category + Workspace path ("" = unknown). */ interface TraceSessionFacts { category: SessionCategory; workspace: string; } export class TraceService { constructor( private readonly root: string, private readonly deps: TraceServiceDeps, ) {} /** * All of this Session's Trace files (sorted by index ascending), served from the * index. An empty answer force-reconciles once and retries before being believed: * disk is the source of truth, and a gate blind spot (an external write into an old * date dir) must cost one extra scan, never a false 404. */ private async locateAll( projectId: string, agentId: string, sessionId: string, ): Promise { await this.deps.index.reconcileAgent(projectId, agentId); let rows = this.deps.index.repo.listFilesBySession(projectId, agentId, sessionId); if (rows.length === 0) { await this.deps.index.reconcileAgent(projectId, agentId, { force: true }); rows = this.deps.index.repo.listFilesBySession(projectId, agentId, sessionId); } return rows.map((r) => ({ path: traceFilePath(this.root, r), date: r.date, index: r.fileIndex, sizeBytes: r.sizeBytes, })); } /** All shard reads funnel through here (deps.observeShardRead is the windowed-read tests' proof of which files were touched). */ private async readShard(path: string): Promise { this.deps.observeShardRead?.(path); return readTraceTolerant(path); } /** Deletes all of this Session's Trace files (called when the Session is deleted); the index rows go with them. */ async deleteSessionTraces(projectId: string, agentId: string, sessionId: string): Promise { const files = await this.locateAll(projectId, agentId, sessionId); for (const file of files) { await fs.rm(file.path, { force: true }); } this.deps.index.removeSession(projectId, agentId, sessionId); } /** * History messages: all index files concatenated in order, with sub-sessions * **expanded in place**. * * The parent Trace only records a `subagent` pointer event at the spawn point * (recording just the child Session id; the content lives in the child * Session's own Trace). Here the pointer is used to locate the child Trace * within the Project, read it recursively, and splice the child messages — * tagged with an origin chain — back in at the pointer's position, so that when * the session is reopened, the frontend can re-attach the sub-session to the * run_subagent tool card via origin (the child Trace's first `session_meta`, * once given an origin, takes the same shape as what's forwarded over the live * stream). When expansion succeeds, the pointer event itself is no longer * emitted; when the child Trace is missing (deleted), the pointer event is kept * so API consumers can still know it existed. */ async readMessages( projectId: string, agentId: string, sessionId: string, ): Promise { const ctx: ExpandCtx = { projectScanned: false, ancestry: new Set([sessionId]), depth: 0, raw: new Map(), }; const files = await this.locateAll(projectId, agentId, sessionId); const out: OmniMessage[] = []; for (const file of files) { out.push(...(await this.expandMessages(projectId, await this.readShard(file.path), ctx))); } return out; } /** * Locates a subagent pointer's owning Agent within the Project: a single index * lookup (the old implementation walked EVERY Agent's whole traces tree to build a * sessionId → agentId map per readMessages call). On a miss, the whole Project is * force-reconciled ONCE per readMessages call and the lookup retried — a child * Trace written by an external process is one scan away, never invisible. */ private async resolveChildAgent( projectId: string, childSid: string, ctx: { projectScanned: boolean }, ): Promise { const hit = this.deps.index.repo.findAgentBySession(projectId, childSid); if (hit !== null || ctx.projectScanned) return hit; ctx.projectScanned = true; await this.deps.index.reconcileProject(projectId, { force: true }); return this.deps.index.repo.findAgentBySession(projectId, childSid); } /** A child session's concatenated raw shard messages, memoized per request; null = no Trace located. */ private async readChildRaw( projectId: string, childSid: string, ctx: ExpandCtx, ): Promise { const cached = ctx.raw.get(childSid); if (cached !== undefined) return cached; const childAgent = await this.resolveChildAgent(projectId, childSid, ctx); let raw: OmniMessage[] | null = null; if (childAgent) { const files = await this.locateAll(projectId, childAgent, childSid); if (files.length > 0) { raw = []; for (const file of files) raw.push(...(await this.readShard(file.path))); } } ctx.raw.set(childSid, raw); return raw; } /** * Expand subagent pointers within one message span (the whole transcript on the full * path, one window on the paged path — same recursive shape either way). */ private async expandMessages( projectId: string, messages: readonly OmniMessage[], ctx: ExpandCtx, ): Promise { const out: OmniMessage[] = []; for (const msg of messages) { // The depth cap guards against runaway recursion; ancestry guards against a // cyclic pointer (a tampered Trace pointing to itself/an ancestor is not expanded). const childSid = ctx.depth < MAX_SUBAGENT_DEPTH ? subagentPointer(msg) : null; if (!childSid || ctx.ancestry.has(childSid)) { out.push(msg); continue; } const raw = await this.readChildRaw(projectId, childSid, ctx); let nested: OmniMessage[] = []; if (raw !== null) { ctx.ancestry.add(childSid); nested = await this.expandMessages(projectId, raw, { ...ctx, depth: ctx.depth + 1 }); ctx.ancestry.delete(childSid); } // Child Trace missing (deleted): keep the pointer event, since the sub-session's content can't be recovered. if (nested.length === 0) { out.push(msg); continue; } for (const m of nested) out.push({ ...m, origin: [childSid, ...(m.origin ?? [])] }); } return out; } /** * A subagent pointer's subtree aggregate for the window scanner (message-window.ts): * descendant token_usage request totals plus the subtree's max timestamp — the same * contributions the expanded messages make to the Web's parent-level stats tracker. * The recursion mirrors expandMessages' depth/ancestry rules exactly, so an * unexpandable pointer contributes nothing on both paths. */ private async aggregateChild( projectId: string, childSid: string, ctx: ExpandCtx, ): Promise { if (ctx.depth >= MAX_SUBAGENT_DEPTH || ctx.ancestry.has(childSid)) return null; const raw = await this.readChildRaw(projectId, childSid, ctx); if (raw === null || raw.length === 0) return null; let requestTokens = 0; let maxTsMs: number | null = null; ctx.ancestry.add(childSid); for (const msg of raw) { const ms = Date.parse(msg.timestamp); if (Number.isFinite(ms) && (maxTsMs === null || ms > maxTsMs)) maxTsMs = ms; const p = msg.payload as { type?: string; request?: { total?: number } }; if (msg.type === "event_msg" && p.type === "token_usage") { requestTokens += typeof p.request?.total === "number" ? p.request.total : 0; continue; } const grandchild = subagentPointer(msg); if (grandchild !== null) { const agg = await this.aggregateChild(projectId, grandchild, { ...ctx, depth: ctx.depth + 1, }); if (agg !== null) { requestTokens += agg.requestTokens; if (agg.maxTsMs !== null && (maxTsMs === null || agg.maxTsMs > maxTsMs)) { maxTsMs = agg.maxTsMs; } } } } ctx.ancestry.delete(childSid); return { requestTokens, maxTsMs }; } /** * End-of-shard scan states for files[0..upto] (cumulative from the transcript's * beginning), served from the trace_files.page_stats cache. A missing/stale record * costs one read of THAT shard — once ever: every shard here is immutable (rotation * opens a new shard; the newest shard never appears in a prefix, because windows are * suffixes and their first shard is always read anyway). The write-back is guarded on * the indexed size, so an externally rewritten shard can only invalidate, never * poison, the cache. */ private async prefixStates( projectId: string, agentId: string, sessionId: string, files: LocatedFile[], upto: number, ctx: ExpandCtx, ): Promise { const states: ScanState[] = []; for (let j = 0; j <= upto; j++) { const file = files[j]!; const cached = deserializePrefix( this.deps.index.repo.getPageStats(projectId, agentId, sessionId, file.index), ); if (cached !== null) { states.push(cached); continue; } const state = cloneScanState(j === 0 ? initialScanState() : states[j - 1]!); const messages = await this.readShard(file.path); await scanMessages( state, messages, () => {}, (sid) => this.aggregateChild(projectId, sid, ctx), ); this.deps.index.repo.setPageStats( projectId, agentId, sessionId, file.index, file.sizeBytes, serializePrefix(state), ); states.push(state); } return states; } /** * Windowed history read (the `tailLimit` / `before` forms of GET /messages). The * window is a run of whole units — cut points and unit semantics live in * message-window.ts — assembled by reading ONLY the shards the window overlaps * (plus, once ever per old shard, the prefix-cache backfill above). Subagent * pointers are expanded exactly as the full path expands them, but only within the * window: children referenced by older windows load when those windows do. */ async readMessagesPage( projectId: string, agentId: string, sessionId: string, req: MessagesPageRequest, ): Promise { const files = await this.locateAll(projectId, agentId, sessionId); const empty = (): MessagesPageResult => ({ messages: [], prior: initialScanState().totals, }); if (files.length === 0) return empty(); const ctx: ExpandCtx = { projectScanned: false, ancestry: new Set([sessionId]), depth: 0, raw: new Map(), }; // The window's exclusive end: the newest shard's end (tail), or the cursor (before). let endPos: number; let endOrdinal: number | null = null; // null = to the shard's end if (req.kind === "tail") { endPos = files.length - 1; } else { endPos = files.findIndex((f) => f.index === req.cursor.fileIndex); // Cursor shard no longer on disk (external deletion — locateAll already // force-retried): the history it pointed into is gone; report end-of-history // rather than a guess at what used to precede it. if (endPos < 0) return empty(); endOrdinal = req.cursor.ordinal; } const prefixes = await this.prefixStates(projectId, agentId, sessionId, files, endPos - 1, ctx); // Walk backward from the end, scanning whole shards (each from its cached carry-in) // until the window has more units than requested or the beginning is reached. const shardMessages = new Map(); let boundaries: Array<{ pos: number; ordinal: number; stats: WindowPriorStats }> = []; let startPos = endPos + 1; while (boundaries.length <= req.limit && startPos > 0) { startPos -= 1; const messages = await this.readShard(files[startPos]!.path); shardMessages.set(startPos, messages); const state = cloneScanState(startPos === 0 ? initialScanState() : prefixes[startPos - 1]!); const shardBoundaries: Array<{ pos: number; ordinal: number; stats: WindowPriorStats }> = []; const to = startPos === endPos && endOrdinal !== null ? endOrdinal : messages.length; await scanMessages( state, messages, (ordinal, stats) => shardBoundaries.push({ pos: startPos, ordinal, stats }), (sid) => this.aggregateChild(projectId, sid, ctx), 0, Math.min(to, messages.length), ); boundaries = [...shardBoundaries, ...boundaries]; } // Window start: the last `limit` units, or the very beginning (preamble included) // when the whole remaining history fits — then there is no `before` cursor. let start: { pos: number; ordinal: number }; let before: string | undefined; let prior: WindowPriorStats; if (boundaries.length > req.limit) { const wb = boundaries[boundaries.length - req.limit]!; start = { pos: wb.pos, ordinal: wb.ordinal }; before = encodeCursor({ fileIndex: files[wb.pos]!.index, ordinal: wb.ordinal }); prior = wb.stats; } else { start = { pos: 0, ordinal: 0 }; prior = initialScanState().totals; } const windowRaw: OmniMessage[] = []; for (let pos = start.pos; pos <= endPos; pos++) { const messages = shardMessages.get(pos) ?? (await this.readShard(files[pos]!.path)); const from = pos === start.pos ? start.ordinal : 0; const to = pos === endPos && endOrdinal !== null ? Math.min(endOrdinal, messages.length) : messages.length; for (let i = from; i < to; i++) windowRaw.push(messages[i]!); } const expanded = await this.expandMessages(projectId, windowRaw, ctx); return { messages: expanded, ...(before !== undefined ? { before } : {}), prior }; } /** List of Trace files (index / date / size / mtime). */ async listTraceFiles( projectId: string, agentId: string, sessionId: string, ): Promise { const files = await this.locateAll(projectId, agentId, sessionId); const out: TraceFileInfo[] = []; for (const file of files) { const stat = await fs.stat(file.path); out.push({ index: file.index, date: file.date, sizeBytes: stat.size, mtime: stat.mtime.toISOString(), }); } return out; } /** Reads events from the Trace file at the given index, paginated by line (for loading large files in pages). */ async readEvents( projectId: string, agentId: string, sessionId: string, index: number, offset: number, limit: number, ): Promise { const messages = await this.readFileByIndex(projectId, agentId, sessionId, index); return { events: messages.slice(offset, offset + limit), offset, limit, total: messages.length, }; } /** Performance analysis: derived from a single Trace file. */ async analyze( projectId: string, agentId: string, sessionId: string, index: number, ): Promise { const messages = await this.readFileByIndex(projectId, agentId, sessionId, index); const requests: RequestSpan[] = []; let openRequest: RequestSpan | null = null; const toolCalls: ToolCallSpan[] = []; const openToolCalls = new Map(); let reconnectCount = 0; let compactionCount = 0; const usageTrend: UsageTrendPointInTrace[] = []; // Timeline (serial-duration estimation): Trace records completion times; model // messages are produced // serially (autoregressive decoding), so each segment's start = the previous // event's time (the request's first segment = request_begin); a tool's // approval/execution runs in parallel with model decoding, on its own lane; // prevSerialTs is cleared after request_end, and the next request_begin // restarts the count (which presumes all of the previous round's // tool_call_output have already come back). const modelSegments: TraceModelSegment[] = []; const toolSpans: TraceToolSpan[] = []; const openSpansById = new Map(); let prevSerialTs: string | null = null; // Task grouping: one user turn contains multiple Request rounds (the Agent // loop sends another round each time it calls a tool); the turn ends once the // model produces only text with no further tool call. Consecutive Requests are // merged into one Task on this basis, and each Task gets its own independent // timeline — different Tasks can be far apart in time (the user is thinking or // has stepped away), and sharing one timeline would leave large gaps. // Compaction forms its own turn: both compaction_begin/compaction_end break a // continuation, so the compaction request becomes its own Task, and the // request that resumes after compaction starts yet another Task. let taskIndex = -1; let continuation = false; // The previous round's Request called a tool -> the next request_begin continues the same Task /** Inside the image run that follows a `[user_steering]` text (see the turn-start rule below). */ let steeringImages = false; let sawToolCallThisRequest = false; // Compaction interval (compaction_begin..compaction_end): the compaction // request's request_begin/request_end and token_usage all fall inside it (see // core context-engine's summarize flow), which is used to exclude the // compaction request entirely from TPS — matching the same convention as // compactionActive in the Chat page's task-stats. let compactionActive = false; // Token / duration totals per Task (computed server-side over the whole file: // frontend events are fetched in pages, so summing them there would be // mismatched). const taskStats = new Map(); const ensureTask = (ti: number): TraceTaskStats => { let t = taskStats.get(ti); if (t === undefined) { t = { taskIndex: ti, messageFrom: -1, messageTo: -1, startTs: "", endTs: "", tokens: { cacheRead: 0, cacheWrite: 0, output: 0 }, llmMs: 0, }; taskStats.set(ti, t); } return t; }; /** * Which turn each message belongs to: **decided definitively in one * sequential pass**, not left for the frontend to guess by timestamp. * * Timestamp boundaries can't be pulled apart — the same millisecond can * contain "the previous turn's last reply, compaction_begin, the compaction * prompt, and the next turn's request_begin" all at once, so assigning by * time would inevitably misfile this turn's reply into the next turn. * * Rule (a turn = one user turn; `request_end` is the end of some Request * within a turn): * - The **starting marker** of a new turn: the main session's user Prompt * (outside compaction), or compaction_begin (compaction forms its own * turn). Messages after the marker and before that turn's first * `request_begin` (subsequent images from a multi-image send, the * compaction prompt) are always held pending, waiting for * `request_begin` to settle the new taskIndex before the whole span is * assigned at once — they belong to the **new** turn, not the tail of * the previous one. * - Other messages belong to the current taskIndex: tool output and * approval decisions arriving after request_end still belong to this * turn (they're the results of tools this turn's Request initiated). */ const msgTask: number[] = new Array(messages.length).fill(-1); /** The pending new turn's starting point (message index); settled once request_begin determines the taskIndex. */ let pendingFrom: number | null = null; for (let mi = 0; mi < messages.length; mi++) { const msg = messages[mi]!; const p = msg.payload as Record & { type?: string }; // The timeline only looks at the main session (a Trace itself never contains origin messages; this is a defensive skip). const hasOrigin = msg.origin !== undefined && msg.origin.length > 0; // Starting marker of a new turn: the main session's user Prompt (outside // compaction) -> a new user turn; compaction_begin -> a compaction turn // (compaction forms its own turn). A single send can be "text + multiple // images" = multiple messages; only the **first** one counts (once // pendingFrom is set, it's not changed again), otherwise the turn's start // would shift to the last image. // Mid-run steering (`[user_steering]`-wrapped user text, delivered between turns of a // running Task): never a turn starter — same exclusion idea as the compaction summary — // and it forces the next request to be a continuation (a steering-only continuation // turn has no preceding tool call, but it is still the same Task). const isSteeringText = !hasOrigin && msg.type === "model_msg" && p.type === "text" && p.role === "user" && typeof p.text === "string" && parseUserSteeringText(p.text) !== null; if (isSteeringText) continuation = true; // Images sent with a steering message ride immediately behind its text, exactly as a // Prompt's images ride behind theirs — and they inherit its exclusion: still the same // Task, so `steeringImages` keeps the window open across the whole run of them and // anything else on the main session closes it (an images-only Prompt after a steering // message is a genuine new turn). A subagent's messages pass through without closing it: // they belong to another session's stream and say nothing about this one's grouping. // The Web answers the same "what is one Task" question over the live stream — see // `openSteering` in web/src/lib/omni/stream-model.ts; the two need to stay in step. const isImage = !hasOrigin && msg.type === "model_msg" && p.type === "image_url"; if (!hasOrigin && !isSteeringText && !(isImage && steeringImages)) steeringImages = false; if (isSteeringText) steeringImages = true; const startsUserTurn = !hasOrigin && !compactionActive && !isSteeringText && !steeringImages && msg.type === "model_msg" && ((p.type === "text" && p.role === "user") || p.type === "image_url"); const startsCompactionTurn = !hasOrigin && msg.type === "event_msg" && p.type === "compaction_begin"; if (startsUserTurn || startsCompactionTurn) { if (pendingFrom === null) pendingFrom = mi; // A user Prompt **always starts a new turn**: judging continuation solely // by "did the previous turn call a tool" isn't enough — if the previous // turn ended in a retryable status (given up after exhausting retries), // retryable would leave continuation at true, and this new message would // get merged into that failed turn, smearing the two turns' messages / // Tokens / TPS / duration together. if (startsUserTurn) continuation = false; } // A main-session message that isn't pending belongs to the current turn immediately (taskIndex < 0 = before the first request_begin, e.g. session_meta). if (!hasOrigin && pendingFrom === null) msgTask[mi] = taskIndex; if (msg.type === "event_msg") { if (p.type === "request_begin") { if (!hasOrigin) { prevSerialTs = msg.timestamp; if (!continuation) taskIndex++; // Not a continuation -> a new Task sawToolCallThisRequest = false; } // Settle taskIndex before opening the span: the span belongs directly to // the current Task. Nearest-neighbor pairing: if the previous begin was // never closed (process exited mid-run), the span is left open. openRequest = { beginTs: msg.timestamp, taskIndex }; if (compactionActive) openRequest.compaction = true; requests.push(openRequest); if (!hasOrigin) { const t = ensureTask(taskIndex); if (compactionActive) t.compaction = true; // This turn is a compaction turn // This turn's duration starts at the first request_begin. It doesn't // use the timestamp of the user Prompt / compaction summary or other // user text: `[context_summary]` is created during compaction but only // written to disk on the next run, so resuming the next day would // stretch the first turn out by a whole day for no reason; the Prompt // to request-dispatch gap is only ever milliseconds anyway. if (t.startTs === "") t.startTs = msg.timestamp; // The new turn's taskIndex is only settled here: the pending span // (user Prompt / multiple images / compaction prompt) is assigned in // full to **this** turn — they're the start of the new turn, not the // tail of the previous one. if (pendingFrom !== null) { for (let k = pendingFrom; k < mi; k++) { if (messages[k]!.origin === undefined) msgTask[k] = taskIndex; } pendingFrom = null; } msgTask[mi] = taskIndex; } } else if (p.type === "approval_decision") { if (!hasOrigin && typeof p.tool_call_id === "string") { const span = openSpansById.get(p.tool_call_id); if (span && span.approvalTs === undefined) { span.approvalTs = msg.timestamp; if (typeof p.decision === "string") span.decision = p.decision; // Approval wait time is subtracted out of the LLM generation // duration: core does `await approve(tc)` inside the streaming loop, // so the entire manual wait sits between request_begin and // request_end (see RequestSpan.approvalWaitMs). Without subtracting // it, "5s generation + 55s approval wait" would show 100 tok/s as 8 tok/s. if (openRequest) { const wait = Date.parse(msg.timestamp) - Date.parse(span.callTs); if (Number.isFinite(wait) && wait > 0) { openRequest.approvalWaitMs = (openRequest.approvalWaitMs ?? 0) + wait; } } } } } else if (p.type === "request_end") { const status = typeof p.status === "string" ? p.status : undefined; // A status core reconnects on within the same run (context-engine's retry // loop) leaves the resent Request in **the same user turn**: it must // continue the turn, otherwise a single blip would split that turn's // Tokens/duration/TPS across two Tasks and inflate the Task count. // // This list must track the engine's, and the engine's differs by loop: // the turn loop retries `failed` too, while compaction deliberately fails // fast and stops on it (core's TURN_RETRY_STATUSES vs // COMPACTION_RETRY_STATUSES). Hence the compactionActive guard — a failed // compaction request really is the end of that request, and counting it // as a reconnect would invent an attempt that never happened. const retryable = status === "timeout" || status === "malformed" || (status === "failed" && !compactionActive); if (!hasOrigin) { prevSerialTs = null; continuation = sawToolCallThisRequest || retryable; } if (retryable) reconnectCount++; if (openRequest) { openRequest.endTs = msg.timestamp; const dur = Date.parse(msg.timestamp) - Date.parse(openRequest.beginTs); if (Number.isFinite(dur)) { openRequest.durationMs = dur; openRequest.activeMs = Math.max(0, dur - (openRequest.approvalWaitMs ?? 0)); } if (status !== undefined) openRequest.status = status; // TPS denominator: accumulated per the turn a Request belongs to. A // compaction request counts too — it belongs to **its own compaction // turn** (compaction forms its own turn), so it neither pollutes a // user turn's TPS, nor does the compaction turn fail to report its own // generation speed accurately. A failed retry's duration is counted as // well — it belongs to the same turn as the retry that eventually // succeeded, and "how long this turn took to produce these tokens" // should include the retries by definition. if (!hasOrigin && openRequest.activeMs !== undefined) { ensureTask(openRequest.taskIndex).llmMs += openRequest.activeMs; } openRequest = null; } } else if (p.type === "compaction_begin") { compactionCount++; // Compaction forms its own turn: otherwise, if the previous turn called // a tool, continuation would still be true and the compaction request // would get merged into the previous Task. if (!hasOrigin) { continuation = false; compactionActive = true; } } else if (p.type === "compaction_end") { // Both ends of compaction break a continuation. This closing one can't // be skipped: if the compaction request itself exhausts its retries and // ends in timeout, the retryable check above would mark it as "continued", // and without clearing it here, the next user turn after compaction // would get merged into this compaction Task. if (!hasOrigin) { continuation = false; compactionActive = false; } } else if (p.type === "token_usage") { const request = p.request as | { total?: number; cache_read?: number; cache_write?: number; output?: number } | undefined; const session = p.session as { total?: number } | undefined; usageTrend.push({ ts: msg.timestamp, requestTotal: request?.total ?? 0, sessionTotal: session?.total ?? 0, }); if (!hasOrigin) { const t = ensureTask(taskIndex); // Cumulative usage for this turn (a running total): those tokens were // actually paid for, so the cost can't be dropped. `tokens.output` also // doubles as the numerator for output TPS — compaction's output // belongs to **its own compaction turn** (compaction forms its own // turn), so a user turn's TPS isn't polluted by it, while the // compaction turn can still accurately report "how fast the summary // was generated". t.tokens.cacheRead += request?.cache_read ?? 0; t.tokens.cacheWrite += request?.cache_write ?? 0; t.tokens.output += request?.output ?? 0; if (!compactionActive) { // The context snapshot only takes non-compaction Requests: tokens // consumed by compaction aren't the post-compaction context // footprint. A later write overwrites an earlier one -> this // naturally leaves behind the snapshot of the Task's **last** // non-compaction Request = the context footprint at the end of this // turn. Accumulating would be wrong: each Request's input carries // the entire history afresh (see TraceTaskStats). t.context = { cacheRead: request?.cache_read ?? 0, cacheWrite: request?.cache_write ?? 0, output: request?.output ?? 0, }; } } } continue; } if (msg.type !== "model_msg") continue; // Model serial segments: assistant-side thinking/text/tool_call (a user input sent instantaneously occupies no segment). if ( !hasOrigin && prevSerialTs !== null && (p.type === "thinking" || p.type === "tool_call" || (p.type === "text" && p.role === "assistant")) ) { const segment: TraceModelSegment = { kind: p.type === "thinking" ? "thinking" : p.type === "tool_call" ? "tool_call" : "text", startTs: prevSerialTs, endTs: msg.timestamp, taskIndex, }; if (p.type === "tool_call" && typeof p.tool_call_id === "string") { segment.toolCallId = p.tool_call_id; if (typeof p.name === "string") segment.name = p.name; } modelSegments.push(segment); prevSerialTs = msg.timestamp; } if (p.type === "tool_call" && typeof p.tool_call_id === "string") { if (!hasOrigin) sawToolCallThisRequest = true; // This turn called a tool -> the next turn continues the same Task const callStop = typeof p.stop_reason === "string" ? p.stop_reason : undefined; const span: ToolCallSpan = { toolCallId: p.tool_call_id, name: typeof p.name === "string" ? p.name : "", startTs: msg.timestamp, }; if (callStop !== undefined) span.stopReason = callStop; openToolCalls.set(p.tool_call_id, span); toolCalls.push(span); // The timeline lane only accepts calls that "will actually be executed": // an interrupt-compensation tool_call (stop_reason other than completed) // never gets an approval/output, and putting it on a lane would render as // a phantom "executing" state spanning the whole timeline — so it's // skipped outright. if (!hasOrigin && (callStop === undefined || callStop === "completed")) { const timeline: TraceToolSpan = { toolCallId: p.tool_call_id, name: typeof p.name === "string" ? p.name : "", callTs: msg.timestamp, taskIndex, }; openSpansById.set(p.tool_call_id, timeline); toolSpans.push(timeline); } } else if (p.type === "tool_call_output" && typeof p.tool_call_id === "string") { const span = openToolCalls.get(p.tool_call_id); if (span && span.endTs === undefined) { span.endTs = msg.timestamp; const dur = Date.parse(msg.timestamp) - Date.parse(span.startTs); if (Number.isFinite(dur)) span.durationMs = dur; if (typeof p.stop_reason === "string") span.stopReason = p.stop_reason; } const timeline = openSpansById.get(p.tool_call_id); if (timeline && timeline.outputTs === undefined) { timeline.outputTs = msg.timestamp; if (typeof p.stop_reason === "string") timeline.stopReason = p.stop_reason; } } } // A pending span that never got a request_begin (interrupted right after the // user sent it / the process exited): it's a turn that never got to run, // and forms its own turn — reattaching it to the previous turn would smear // two separate user sends together. if (pendingFrom !== null) { taskIndex++; for (let k = pendingFrom; k < messages.length; k++) { if (messages[k]!.origin === undefined) msgTask[k] = taskIndex; } ensureTask(taskIndex); } // Each turn's message index range and end-of-turn time are always derived from // the per-message assignment done above (same source, so they never disagree // with each other). Messages before the first request_begin (session_meta) // have taskIndex -1 and are assigned to the first turn, otherwise they'd have // nowhere to sit on the page. The turn duration's **starting point** isn't // decided here — it was already settled at request_begin (duration only looks // at LLM requests; timestamps of the user Prompt / compaction summary or other // user text don't participate, see TraceTaskStats.startTs). const firstTask = [...taskStats.keys()].sort((a, b) => a - b)[0]; for (let k = 0; k < messages.length; k++) { let ti = msgTask[k]!; if (ti < 0) { if (firstTask === undefined) continue; ti = firstTask; msgTask[k] = ti; } const t = ensureTask(ti); if (t.messageFrom < 0 || k < t.messageFrom) t.messageFrom = k; if (k > t.messageTo) t.messageTo = k; // session_meta is only **listed** in the first turn, and doesn't count // toward the end-of-turn time: it's metadata written when the session was // created, and its timestamp has nothing to do with this turn (it also gets // rewritten verbatim at the start of a new file after compaction splits the file). if (messages[k]!.type === "session_meta") continue; const ts = messages[k]!.timestamp; if (t.endTs === "" || ts > t.endTs) t.endTs = ts; } const tasks = [...taskStats.values()].sort((a, b) => a.taskIndex - b.taskIndex); // Total elapsed time = **the sum of each turn's duration**, matching exactly // the scope shown per-turn below (**including compaction turns** — their wall // clock time genuinely elapsed, each turn's card has its own duration, and the // overall total is their sum, so the numbers must add up). It is not "last // message timestamp minus first message timestamp": that would be the whole // file's wall-clock span, counting in the gaps **between** turns (the user // thinking, stepping out for coffee, coming back the next day) — none of which // is time the Agent spent working. A degenerate turn with no Request has an // empty startTs and counts as 0. // Note this uses a different convention from the Session's cumulative elapsed // time on the Chat page: that one only accumulates user turns (compaction // after a turn ends doesn't count toward the turn). const elapsedMs = tasks.reduce((sum, t) => { const span = Date.parse(t.endTs) - Date.parse(t.startTs); return sum + (Number.isFinite(span) ? Math.max(0, span) : 0); }, 0); return { elapsedMs, requests, tasks, toolCalls, modelSegments, toolSpans, reconnectCount, compactionCount, usageTrend, }; } /** * Agent-level browsing, served from the trace-file index (one mtime-gated reconcile, * then pure DB — no directory walks, no head-reads). Without `paging`: the legacy full * drill-down (newest first: Agent -> date -> Session -> Trace files; sizes are the * index's last observed values). With `paging`: session-group-centric — Sessions * ordered by id descending (ids embed a timestamp, so that is reverse chronological), * optionally filtered to one sidebar `category`, CLI-origin Sessions excluded unless * `includeCli` (the "show CLI sessions" preference; the legacy shape is never * filtered — back-compat), and only the returned slice gets per-file `fs.stat` for * fresh sizes (written back to the index). */ async agentTraces( projectId: string, agentId: string, paging?: { offset: number; limit: number } | null, opts: { category?: SessionCategory; includeCli?: boolean } = {}, ): Promise { await this.deps.index.reconcileAgent(projectId, agentId); const files = this.deps.index.repo.listFilesByAgent(projectId, agentId); if (paging) return this.agentTracesPage(projectId, agentId, files, paging, opts); const byDate = new Map>(); for (const f of files) { const sessions = byDate.get(f.date) ?? new Map(); const list = sessions.get(f.sessionId) ?? []; list.push({ index: f.fileIndex, sizeBytes: f.sizeBytes }); sessions.set(f.sessionId, list); byDate.set(f.date, sessions); } return { dates: [...byDate.keys()] .sort() .reverse() .map((date) => ({ date, // session_id embeds a timestamp, so reverse lexicographic order is reverse chronological order. sessions: [...byDate.get(date)!.entries()] .sort((a, b) => b[0].localeCompare(a[0])) .map(([sessionId, list]) => ({ sessionId, files: list.sort((a, b) => a.index - b.index), })), })), }; } /** * Classification (no IO): `archived` comes exactly from the DB row; the origin comes * from the shared sources registry, else from the Session's registration-time facts * (trace_sessions — the reconciler head-read its earliest shard once when the file * first appeared, so by listing time every indexed Session is classified exactly). */ private classify( sessionId: string, row: SessionRow | undefined, facts: TraceSessionRow | undefined, ): TraceSessionFacts { const known = this.deps.sources?.get(sessionId); // Registry answer (including null = known user-created) wins — it can be fresher // (subagent registration happens at spawn, before any reconcile); else the stored facts. const source = known !== undefined ? known : (facts?.source ?? undefined); const category: SessionCategory = (row?.archivedAt ?? null) !== null ? "archived" : source === "subagent" || source === "schedule" ? source : "active"; return { category, workspace: row?.workspace ?? facts?.workspace ?? "" }; } /** The paginated listing behind agentTraces: pure index reads; per-file stat only for the returned slice (sizes written back). */ private async agentTracesPage( projectId: string, agentId: string, files: TraceFileRow[], paging: { offset: number; limit: number }, opts: { category?: SessionCategory; includeCli?: boolean }, ): Promise { const bySession = new Map(); for (const f of files) { const list = bySession.get(f.sessionId) ?? []; list.push(f); bySession.set(f.sessionId, list); } // One indexed query per source: the Agent's DB rows (title / archived / workspace / // client — CLI rows included) and the registration-time facts. const rows = new Map( (this.deps.sessions?.listByAgent(projectId, agentId) ?? []).map((r) => [r.sessionId, r]), ); const factsBySession = new Map( this.deps.index.repo.listSessionsByAgent(projectId, agentId).map((r) => [r.sessionId, r]), ); /** * CLI-origin = no web-created DB row AND not a subagent/schedule Session. Server-created * Sessions always have a row (creation / subagent registration insert one), so rowless = * external CLI; adopted CLI rows carry client='cli'. The default listing excludes these — * the same "show CLI sessions" preference the sidebar applies — while subagent/schedule * Sessions stay in their folders regardless of which process ran them. */ const cliOrigin = (sessionId: string, facts: TraceSessionFacts): boolean => { if (facts.category === "subagent" || facts.category === "schedule") return false; const row = rows.get(sessionId); return !( row !== undefined && (row.client === null || row.client === undefined || row.client === "web") ); }; const ids = [...bySession.keys()].sort((a, b) => b.localeCompare(a)); // Classify every group once; the same result drives the CLI/category filters, the // counts AND the returned fields, so a row can never appear in a bucket its own // `category` denies. Hidden CLI-origin groups are excluded from counts too. const counts: SessionCategoryCounts = { active: 0, subagent: 0, schedule: 0, archived: 0 }; const workspaceCounts: Record = {}; const factsById = new Map(); const visible: string[] = []; for (const id of ids) { const facts = this.classify(id, rows.get(id), factsBySession.get(id)); if (opts.includeCli !== true && cliOrigin(id, facts)) continue; factsById.set(id, facts); visible.push(id); counts[facts.category] += 1; const ws = (workspaceCounts[facts.workspace] ??= { active: 0, subagent: 0, schedule: 0, archived: 0, }); ws[facts.category] += 1; } const filtered = opts.category === undefined ? visible : visible.filter((id) => factsById.get(id)!.category === opts.category); const page = filtered.slice(paging.offset, paging.offset + paging.limit); const sessions: AgentTraceSessionEntry[] = []; for (const sessionId of page) { const shard = bySession.get(sessionId)!.sort((a, b) => a.fileIndex - b.fileIndex); // Fresh sizes for the returned page only (an actively-appended shard grows without // moving any directory mtime): bounded metadata stats, written back to the index. const withSize = await Promise.all( shard.map(async (f) => { let sizeBytes = f.sizeBytes; try { sizeBytes = (await fs.stat(traceFilePath(this.root, f))).size; if (sizeBytes !== f.sizeBytes) { this.deps.index.repo.updateFileSize( projectId, agentId, sessionId, f.fileIndex, sizeBytes, ); } } catch { /* Vanished mid-request: serve the last observed size; the next reconcile settles the rows. */ } return { index: f.fileIndex, date: f.date, sizeBytes }; }), ); const facts = factsById.get(sessionId)!; // Title: DB row first (?? sends both "no row" and "row with NULL title" onward); // else the registration-time first-prompt fallback stored in trace_sessions. const title = rows.get(sessionId)?.title ?? factsBySession.get(sessionId)?.title ?? undefined; sessions.push({ sessionId, ...(title !== undefined ? { title } : {}), category: facts.category, workspace: facts.workspace, files: withSize, }); } return { dates: [], sessions, totalSessions: filtered.length, counts, workspaceCounts }; } private async readFileByIndex( projectId: string, agentId: string, sessionId: string, index: number, ): Promise { const file = await this.locateByIndex(projectId, agentId, sessionId, index); return readTraceTolerant(file.path); } private async locateByIndex( projectId: string, agentId: string, sessionId: string, index: number, ): Promise { const files = await this.locateAll(projectId, agentId, sessionId); const file = files.find((f) => f.index === index); if (!file) { throw new HttpError( 404, "trace_not_found", `This Session has no Trace file with index ${index}.`, ); } return file; } /** Raw bytes of the Trace file at the given index (export/download: the file is served verbatim). */ async readFileRaw( projectId: string, agentId: string, sessionId: string, index: number, ): Promise { const file = await this.locateByIndex(projectId, agentId, sessionId, index); return fs.readFile(file.path); } /** * Imports an uploaded Trace file (raw JSONL content). Validates the content itself — * parseable JSONL whose first record is a `session_meta` carrying a filename-safe * `session_id` (400 invalid_trace otherwise) — then writes it under the Agent's traces * directory as a **new Session**: a session id the Agent already has is rejected with * 409 `trace_session_exists` (splicing a further index into an existing Session would * corrupt its concatenated transcript, silently become its resume point, and could * collide with a live Writer's rotation), so the imported file is always index 1. The * date dir comes from the first record's timestamp (falling back to now), formatted as * a **local** date — the same convention as core's Trace Writer — so an export → * import round-trip lands in the same date dir on non-UTC servers. */ async importTraceFile( projectId: string, agentId: string, content: string, ): Promise { const invalid = (message: string) => new HttpError(400, "invalid_trace", message); let records: OmniMessage[]; try { records = parseTraceLines(content); } catch { throw invalid("The file is not valid Trace JSONL."); } if (records.length === 0) throw invalid("The file contains no Trace records."); const first = records[0]!; if (!isSessionMeta(first)) throw invalid("The first record of a Trace file must be session_meta."); // The payload type declares session_id: string, but the value came from user-supplied JSON — // re-check the runtime shape before it becomes part of a filename. const sessionId: unknown = (first.payload as { session_id?: unknown }).session_id; if (typeof sessionId !== "string" || !IMPORT_SESSION_ID_RE.test(sessionId)) { throw invalid("session_meta carries a missing or invalid session_id."); } const duplicate = () => new HttpError( 409, "trace_session_exists", `This Agent already has a Session with id ${sessionId}; a duplicate Trace cannot be imported.`, ); if ((await this.locateAll(projectId, agentId, sessionId)).length > 0) throw duplicate(); const ts = Date.parse(first.timestamp); const date = formatLocalDate(Number.isNaN(ts) ? new Date() : new Date(ts)); const index = 1; const dir = path.join(tracesDir(this.root, projectId, agentId), date); await fs.mkdir(dir, { recursive: true }); const file = path.join(dir, `${sessionId}_${String(index).padStart(3, "0")}.jsonl`); const body = content.replace(/\n+$/, "") + "\n"; try { // Normalize to exactly one trailing newline (the JSONL convention the writer // follows). `wx` closes the check-then-write race: two concurrent imports of the // same new session id both pass the locateAll check, but only one can create the // file — the loser's EEXIST maps to the same 409 as the pre-check. await fs.writeFile(file, body, { encoding: "utf8", flag: "wx" }); } catch (err) { if ((err as NodeJS.ErrnoException).code === "EEXIST") throw duplicate(); throw err; } // Write-time registration: this path knows the file's identity and already holds the // parsed records, so the index row + session facts land synchronously (no re-read). this.deps.index.registerImportedFile({ projectId, agentId, sessionId, fileIndex: index, date, sizeBytes: Buffer.byteLength(body, "utf8"), records, }); return { sessionId, index, date }; } }