feat(traces): mirror sidebar grouping (workspace mode, category folders, shared components), paginate sessions, title fallback to first prompt (#183)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/server/src/api/types.ts

@@ -1093,9 +1093,58 @@ export interface AgentTraceDateGroup {
   sessions: AgentTraceSessionGroup[];
 }
 
-/** Agent → date → Session → Trace file drill-down browsing structure (reverse chronological). */
+/** One Trace file in the session-centric listing (`date` carried per file: one Session's shards can span date directories). */
+export interface AgentTraceSessionFile {
+  index: number;
+  date: string;
+  sizeBytes: number;
+}
+
+/** One Session's Trace files merged across date directories (the paginated listing's unit). */
+export interface AgentTraceSessionEntry {
+  sessionId: string;
+  /**
+   * Display title, resolved only for the returned page: the sessions DB title when one
+   * exists, else derived from the Session's first user prompt (bounded head-read of the
+   * earliest shard); absent when neither yields one (the client falls back to its
+   * default title — raw session ids are never rendered).
+   */
+  title?: string;
+  /**
+   * Sidebar category of this Session, from the same bounded classification the listing
+   * filters and counts with: archived exactly from the DB row; origin from the shared
+   * in-process registry / previously observed session_meta; a DB-untracked Session this
+   * process has not yet head-read falls into `active` until a page surfaces it (its
+   * head-read then registers the true origin for subsequent requests).
+   */
+  category: SessionCategory;
+  /** Workspace path locked at creation (DB row or observed session_meta); "" when unknown — the client's merged temp-group fallback. */
+  workspace: string;
+  /** Sorted by index ascending (a higher index is newer). */
+  files: AgentTraceSessionFile[];
+}
+
+/**
+ * Agent-level Trace browsing structure. Without `limit` the response is the legacy full
+ * drill-down (`dates`: Agent → date → Session → Trace file, reverse chronological) and the
+ * paging fields are absent. With `offset`/`limit` the response is session-group-centric:
+ * `sessions` carries the requested slice (newest first by sessionId desc — ids embed a
+ * timestamp, so that is reverse chronological) with titles and classification,
+ * `totalSessions` the session-group count (within `category` when one is given, so paging
+ * and the count agree), `counts` / `workspaceCounts` the per-category totals over ALL of
+ * the Agent's session groups (folder labels / workspace-mode group headers), and `dates`
+ * stays empty (per-file stats are only taken for the returned page).
+ */
 export interface AgentTracesResponse {
   dates: AgentTraceDateGroup[];
+  /** Present only when the request paginates: the requested slice of Session groups, newest first. */
+  sessions?: AgentTraceSessionEntry[];
+  /** Present only when the request paginates: session-group count of the paged (category-filtered) set. */
+  totalSessions?: number;
+  /** Present only when the request paginates: per-category totals over all of the Agent's session groups. */
+  counts?: SessionCategoryCounts;
+  /** Present only when the request paginates: `counts` broken down by Workspace path ("" = unknown). */
+  workspaceCounts?: Record<string, SessionCategoryCounts>;
 }
 
 export interface TraceImportRequest {

packages/server/src/app.ts

@@ -23,6 +23,7 @@ import { ProjectsRepo } from "./db/repos/projects.js";
 import { GoalsRepo } from "./db/repos/goals.js";
 import { SchedulesRepo } from "./db/repos/schedules.js";
 import { SessionsRepo } from "./db/repos/sessions.js";
+import { TraceIndexRepo } from "./db/repos/trace-index.js";
 import { UiPrefsRepo } from "./db/repos/ui-prefs.js";
 import { UsageRepo } from "./db/repos/usage.js";
 import { UsersRepo } from "./db/repos/users.js";
@@ -70,6 +71,7 @@ import { SnapshotService } from "./services/snapshot-service.js";
 import { ProjectConfigService } from "./services/project-config-service.js";
 import { ProjectService } from "./services/project-service.js";
 import { SessionService } from "./services/session-service.js";
+import { TraceIndexService } from "./services/trace-index.js";
 import { TraceService } from "./services/trace-service.js";
 import { UpdateCheckService } from "./services/update-check-service.js";
 import { UsageService } from "./services/usage-service.js";
@@ -99,6 +101,8 @@ export interface AppDeps {
   agentConfigService: AgentConfigService;
   sessionService: SessionService;
   traceService: TraceService;
+  /** Trace-file index (derived cache + reconciler); routes use it for delete-time coherence. */
+  traceIndex: TraceIndexService;
   usageService: UsageService;
   /** GitHub latest-release lookup for the web UI's update reminder (cached, fail-soft). */
   updateCheck: UpdateCheckService;
@@ -154,7 +158,20 @@ export function buildAppDeps(config: ServerConfig, overrides: BuildDepsOverrides
   const projectConfigService = new ProjectConfigService(config.root);
   const agentConfigService = new AgentConfigService(config.root);
   const agentService = new AgentService(config.root, agentsRepo, agentConfigService);
-  const traceService = new TraceService(config.root);
+  // Session-origin registry: session_meta is the single source of truth (no DB column);
+  // shared by the manager (subagent registration), the loader (self-heal rebuild),
+  // SessionService (creation / adoption / lazy list resolution), and the Trace index /
+  // listing classification.
+  const sessionSources = new SessionSources();
+  // Trace-file index: the derived cache every trace listing/locating path serves from
+  // (mtime-gated reconciler keeps it in step with the on-disk tree; see trace-index.ts).
+  const traceIndexRepo = new TraceIndexRepo(db);
+  const traceIndex = new TraceIndexService(config.root, traceIndexRepo, sessionSources);
+  const traceService = new TraceService(config.root, {
+    index: traceIndex,
+    sessions: sessionsRepo,
+    sources: sessionSources,
+  });
   const workspaceFiles = new WorkspaceFilesService();
   // Per-process secret: preview tokens are short-lived, so losing them on restart is
   // harmless and there is nothing to persist or rotate.
@@ -183,10 +200,6 @@ export function buildAppDeps(config: ServerConfig, overrides: BuildDepsOverrides
   const titles =
     overrides.titles ??
     new TitleGenerator({ sessions: sessionsRepo, channels, recorder, errors, log });
-  // Session-origin registry: session_meta is the single source of truth (no DB column);
-  // shared by the manager (subagent registration), the loader (self-heal rebuild) and
-  // SessionService (creation / adoption / lazy list resolution).
-  const sessionSources = new SessionSources();
   const manager = new SessionManager({
     sessions: sessionsRepo,
     channels,
@@ -213,6 +226,7 @@ export function buildAppDeps(config: ServerConfig, overrides: BuildDepsOverrides
     goals: goalsRepo,
     projectConfig: projectConfigService,
     manager,
+    traceIndex,
   });
   const authService = new AuthService({
     users: usersRepo,
@@ -237,6 +251,7 @@ export function buildAppDeps(config: ServerConfig, overrides: BuildDepsOverrides
     manager,
     projectConfig: projectConfigService,
     sources: sessionSources,
+    traceIndex,
   });
   // Schedule scheduler: active only while the server is running. Only
   // assembled here; start() is called in index.ts (tests drive it via tickOnce, no real timer).
@@ -267,6 +282,7 @@ export function buildAppDeps(config: ServerConfig, overrides: BuildDepsOverrides
     agentConfigService,
     sessionService,
     traceService,
+    traceIndex,
     usageService,
     updateCheck,
     workspaceFiles,

packages/server/src/db/repos/trace-index.ts

@@ -0,0 +1,187 @@
+/**
+ * trace_files / trace_sessions repo: the Trace-tree derived cache (see schema.ts and
+ * services/trace-index.ts for the cache rules). Pure row access — reconciliation
+ * policy (mtime gates, head-reads) lives in the service.
+ */
+import type { DatabaseSync } from "node:sqlite";
+import type { SessionSource } from "../../api/types.js";
+
+/** One indexed Trace shard. */
+export interface TraceFileRow {
+  projectId: string;
+  agentId: string;
+  sessionId: string;
+  fileIndex: number;
+  date: string;
+  sizeBytes: number;
+}
+
+/** One Session's registration-time facts (earliest shard's head). */
+export interface TraceSessionRow {
+  sessionId: string;
+  projectId: string;
+  agentId: string;
+  source: SessionSource | null;
+  workspace: string;
+  title: string | null;
+  provider: string | null;
+  modelId: string | null;
+  firstTs: string | null;
+  metaRead: boolean;
+}
+
+function mapFile(r: Record<string, unknown>): TraceFileRow {
+  return {
+    projectId: r.project_id as string,
+    agentId: r.agent_id as string,
+    sessionId: r.session_id as string,
+    fileIndex: r.file_index as number,
+    date: r.date as string,
+    sizeBytes: r.size_bytes as number,
+  };
+}
+
+function mapSession(r: Record<string, unknown>): TraceSessionRow {
+  const source = r.source as string | null;
+  return {
+    sessionId: r.session_id as string,
+    projectId: r.project_id as string,
+    agentId: r.agent_id as string,
+    // Narrowed on read as well as write: junk in a hand-edited DB must not leak out as a source.
+    source: source === "subagent" || source === "schedule" ? source : null,
+    workspace: (r.workspace as string | null) ?? "",
+    title: (r.title as string | null) ?? null,
+    provider: (r.provider as string | null) ?? null,
+    modelId: (r.model_id as string | null) ?? null,
+    firstTs: (r.first_ts as string | null) ?? null,
+    metaRead: (r.meta_read as number) === 1,
+  };
+}
+
+export class TraceIndexRepo {
+  constructor(private readonly db: DatabaseSync) {}
+
+  upsertFile(row: TraceFileRow): void {
+    this.db
+      .prepare(
+        `INSERT INTO trace_files (project_id, agent_id, session_id, file_index, date, size_bytes)
+         VALUES (?, ?, ?, ?, ?, ?)
+         ON CONFLICT(project_id, agent_id, session_id, file_index)
+         DO UPDATE SET date = excluded.date, size_bytes = excluded.size_bytes`,
+      )
+      .run(row.projectId, row.agentId, row.sessionId, row.fileIndex, row.date, row.sizeBytes);
+  }
+
+  /** Refreshes one shard's last-observed size (listings write back their page stats). */
+  updateFileSize(
+    projectId: string,
+    agentId: string,
+    sessionId: string,
+    fileIndex: number,
+    sizeBytes: number,
+  ): void {
+    this.db
+      .prepare(
+        `UPDATE trace_files SET size_bytes = ?
+         WHERE project_id = ? AND agent_id = ? AND session_id = ? AND file_index = ?`,
+      )
+      .run(sizeBytes, projectId, agentId, sessionId, fileIndex);
+  }
+
+  deleteFile(projectId: string, agentId: string, sessionId: string, fileIndex: number): void {
+    this.db
+      .prepare(
+        `DELETE FROM trace_files
+         WHERE project_id = ? AND agent_id = ? AND session_id = ? AND file_index = ?`,
+      )
+      .run(projectId, agentId, sessionId, fileIndex);
+  }
+
+  /** All indexed shards of one Agent (date desc, session desc, index asc — the listing's canonical order). */
+  listFilesByAgent(projectId: string, agentId: string): TraceFileRow[] {
+    const rows = this.db
+      .prepare(
+        `SELECT * FROM trace_files WHERE project_id = ? AND agent_id = ?
+         ORDER BY session_id DESC, file_index ASC`,
+      )
+      .all(projectId, agentId);
+    return rows.map(mapFile);
+  }
+
+  /** One Session's shards, index ascending. */
+  listFilesBySession(projectId: string, agentId: string, sessionId: string): TraceFileRow[] {
+    const rows = this.db
+      .prepare(
+        `SELECT * FROM trace_files
+         WHERE project_id = ? AND agent_id = ? AND session_id = ?
+         ORDER BY file_index ASC`,
+      )
+      .all(projectId, agentId, sessionId);
+    return rows.map(mapFile);
+  }
+
+  /** Owning Agent of a Session's Trace within a Project (the subagent-pointer resolver); null when unindexed. */
+  findAgentBySession(projectId: string, sessionId: string): string | null {
+    const r = this.db
+      .prepare(`SELECT agent_id FROM trace_files WHERE project_id = ? AND session_id = ? LIMIT 1`)
+      .get(projectId, sessionId);
+    return r ? (r.agent_id as string) : null;
+  }
+
+  upsertSession(row: TraceSessionRow): void {
+    this.db
+      .prepare(
+        `INSERT INTO trace_sessions (session_id, project_id, agent_id, source, workspace, title, provider, model_id, first_ts, meta_read)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(session_id) DO UPDATE SET
+           project_id = excluded.project_id, agent_id = excluded.agent_id,
+           source = excluded.source, workspace = excluded.workspace, title = excluded.title,
+           provider = excluded.provider, model_id = excluded.model_id,
+           first_ts = excluded.first_ts, meta_read = excluded.meta_read`,
+      )
+      .run(
+        row.sessionId,
+        row.projectId,
+        row.agentId,
+        row.source,
+        row.workspace,
+        row.title,
+        row.provider,
+        row.modelId,
+        row.firstTs,
+        row.metaRead ? 1 : 0,
+      );
+  }
+
+  getSession(sessionId: string): TraceSessionRow | null {
+    const r = this.db.prepare("SELECT * FROM trace_sessions WHERE session_id = ?").get(sessionId);
+    return r ? mapSession(r) : null;
+  }
+
+  listSessionsByAgent(projectId: string, agentId: string): TraceSessionRow[] {
+    const rows = this.db
+      .prepare("SELECT * FROM trace_sessions WHERE project_id = ? AND agent_id = ?")
+      .all(projectId, agentId);
+    return rows.map(mapSession);
+  }
+
+  /** Removes one Session's rows from both tables (Session delete keeps the cache coherent). */
+  deleteBySession(sessionId: string): void {
+    this.db.prepare("DELETE FROM trace_files WHERE session_id = ?").run(sessionId);
+    this.db.prepare("DELETE FROM trace_sessions WHERE session_id = ?").run(sessionId);
+  }
+
+  deleteByAgent(projectId: string, agentId: string): void {
+    this.db
+      .prepare("DELETE FROM trace_files WHERE project_id = ? AND agent_id = ?")
+      .run(projectId, agentId);
+    this.db
+      .prepare("DELETE FROM trace_sessions WHERE project_id = ? AND agent_id = ?")
+      .run(projectId, agentId);
+  }
+
+  deleteByProject(projectId: string): void {
+    this.db.prepare("DELETE FROM trace_files WHERE project_id = ?").run(projectId);
+    this.db.prepare("DELETE FROM trace_sessions WHERE project_id = ?").run(projectId);
+  }
+}

packages/server/src/db/schema.ts

@@ -121,4 +121,28 @@ CREATE TABLE IF NOT EXISTS ui_prefs (
   user_id    TEXT PRIMARY KEY REFERENCES users(user_id) ON DELETE CASCADE,
   prefs_json TEXT NOT NULL                    -- {theme?, lastProjectId?, ...} free-form JSON
 );
+CREATE TABLE IF NOT EXISTS trace_files (       -- DERIVED CACHE of the on-disk Trace tree (services/trace-index.ts): the directories stay the single source of truth, every row is rebuildable from disk, and a row is never authority for absence — consumers reconcile + retry on a miss, so a stale index costs one extra scan, never a false 404
+  project_id TEXT NOT NULL,
+  agent_id   TEXT NOT NULL,
+  session_id TEXT NOT NULL,
+  file_index INTEGER NOT NULL,               -- shard index NNN of <session_id>_NNN.jsonl (name/path are reconstructed from the row, never stored: the data root may move)
+  date       TEXT NOT NULL,                  -- date directory name (local yyyy-mm-dd)
+  size_bytes INTEGER NOT NULL,               -- last observed size (listings stat-refresh the returned page and write back; an actively-appended shard may lag in between)
+  PRIMARY KEY (project_id, agent_id, session_id, file_index)
+);
+CREATE INDEX IF NOT EXISTS idx_trace_files_agent_date ON trace_files(project_id, agent_id, date);
+CREATE INDEX IF NOT EXISTS idx_trace_files_session ON trace_files(session_id);
+CREATE TABLE IF NOT EXISTS trace_sessions (    -- per-session facts read ONCE at registration from the earliest shard's head (same derived-cache rules as trace_files; listing needs zero head-reads)
+  session_id TEXT PRIMARY KEY,
+  project_id TEXT NOT NULL,
+  agent_id   TEXT NOT NULL,
+  source     TEXT,                           -- session_meta origin: 'subagent' | 'schedule' | NULL = user-created (or head not yet readable)
+  workspace  TEXT NOT NULL DEFAULT '',
+  title      TEXT,                           -- first-prompt fallback title (sessions.title always wins when present)
+  provider   TEXT,                           -- model reference from session_meta (CLI adoption reads it from here; NULL = meta unreadable / legacy without provider)
+  model_id   TEXT,
+  first_ts   TEXT,                           -- first record's timestamp (adoption's createdAt fallback when the id embeds no time)
+  meta_read  INTEGER NOT NULL DEFAULT 0      -- 1 once the head parsed; 0 = facts unknown, retried by the next reconcile that touches the session
+);
+CREATE INDEX IF NOT EXISTS idx_trace_sessions_agent ON trace_sessions(project_id, agent_id);
 `;

packages/server/src/http/routes/agent-traces.ts

@@ -1,6 +1,18 @@
 /**
  * Agent-level Trace browsing routes:
- *   - GET /api/projects/:p/agents/:a/traces — drills down Agent -> date -> Session -> index (reverse order);
+ *   - GET /api/projects/:p/agents/:a/traces — Agent-level listing, served from the
+ *     trace-file index (mtime-gated reconcile, then pure DB — see services/trace-index.ts).
+ *     Without `limit`: the legacy full drill-down (Agent -> date -> Session -> index,
+ *     reverse order), never filtered. With optional `offset`/`limit` (+ optional
+ *     `category`, `cli`): pages Session groups newest-first (within the category when
+ *     given), stat-ing only the returned page for fresh sizes, and resolves per group a
+ *     display title (sessions DB title, else the registration-time first-prompt
+ *     fallback) plus its sidebar category / Workspace and per-category totals.
+ *     CLI-origin Sessions (no web sessions-table row, not subagent/schedule) are
+ *     excluded unless `cli=1` — the same "show CLI sessions" preference the sessions
+ *     list honors, applied server-side to rows, counts and workspace groups alike.
+ *     The listing consults the sessions table read-only (titles, archived, workspace,
+ *     client); discovery itself still comes from the Trace directory tree via the index.
  *   - GET /api/projects/:p/agents/:a/traces/:sessionId/:index (including /analysis, /download) —
  *     read-only Trace detail endpoints (FD-3): locate the Trace file directly by
  *     (projectId, agentId, sessionId), without depending on the sessions table for
@@ -13,9 +25,10 @@
  */
 import { Hono } from "hono";
 import type { AppEnv } from "../../auth/middleware.js";
-import type { TraceImportResponse } from "../../api/types.js";
+import type { SessionCategory, TraceImportResponse } from "../../api/types.js";
 import {
   badRequest,
+  optionalPagingQuery,
   paginationQuery,
   positiveIntParam,
   readJson,
@@ -27,6 +40,14 @@ import type { AppDeps } from "../../app.js";
 /** Import file size cap: aligned with the snapshot import (stays within the 20MB body limit after base64). */
 const MAX_TRACE_BYTES = 14 * 1024 * 1024;
 
+/** Accepted `category` query values of the paginated listing (SessionCategory, spelled out for validation — same as the sessions list route). */
+const SESSION_CATEGORIES: readonly SessionCategory[] = [
+  "active",
+  "subagent",
+  "schedule",
+  "archived",
+];
+
 export function agentTracesRoutes(deps: AppDeps): Hono<AppEnv> {
   const app = new Hono<AppEnv>();
 
@@ -35,7 +56,30 @@ export function agentTracesRoutes(deps: AppDeps): Hono<AppEnv> {
     const projectId = requireValidId(c, "projectId");
     const agentId = requireValidId(c, "agentId");
     deps.projectService.requireProjectAccess(c.var.user.userId, projectId);
-    return c.json(await deps.traceService.agentTraces(projectId, agentId));
+    // Both params absent -> null -> the legacy full response, byte-for-byte as before.
+    const paging = optionalPagingQuery(c);
+    // Optional category filter (paging then applies within the category): only meaningful
+    // on the paginated session-centric shape — the legacy full drill-down has no category
+    // notion, so a filtered-but-unpaged request is a client error, not a silent no-op.
+    const rawCategory = c.req.query("category");
+    if (
+      rawCategory !== undefined &&
+      !(SESSION_CATEGORIES as readonly string[]).includes(rawCategory)
+    ) {
+      throw badRequest(`category must be one of ${SESSION_CATEGORIES.join(" / ")}.`);
+    }
+    if (rawCategory !== undefined && paging === null) throw badRequest("category requires limit.");
+    // `cli=1` widens the paginated listing to CLI-origin Sessions (mirroring the sessions
+    // list's parameter): the default follows the "show CLI sessions" preference's OFF
+    // state. The legacy unpaged shape is never filtered (back-compat).
+    const rawCli = c.req.query("cli");
+    if (rawCli !== undefined && rawCli !== "1") throw badRequest("cli only accepts 1.");
+    return c.json(
+      await deps.traceService.agentTraces(projectId, agentId, paging, {
+        ...(rawCategory !== undefined ? { category: rawCategory as SessionCategory } : {}),
+        ...(rawCli !== undefined ? { includeCli: true } : {}),
+      }),
+    );
   });
 
   app.get("/:sessionId/:index", async (c) => {

packages/server/src/http/routes/agents.ts

@@ -76,6 +76,8 @@ export function agentsRoutes(deps: AppDeps): Hono<AppEnv> {
       await settleWithin(runnings, 5000);
       await deps.agentService.deleteAgent(projectId, agentId);
       deps.sessionsRepo.deleteByAgent(projectId, agentId);
+      // Trace-index rows describe files the rm above just removed: drop them with the Agent.
+      deps.traceIndex.removeAgent(projectId, agentId);
       // Per-Agent runtime state keyed on the now-removed sessions/agent: drop it so nothing is
       // orphaned (session ids are never reused, so a leftover row is dead weight). Usage records
       // are deliberately kept — historical stats survive Agent deletion (see deleteAgent).

packages/server/src/internal/trace-head.ts

@@ -0,0 +1,41 @@
+/**
+ * Bounded Trace-head reads (shared by the Session index's session_meta backfill and the
+ * Trace listing's first-prompt title fallback): parse only the head window of a shard
+ * instead of pulling a whole multi-MB file into memory.
+ */
+import { open } from "node:fs/promises";
+import { parseTraceLines, readTraceTolerant } from "@prismshadow/penguin-core";
+import type { OmniMessage } from "@prismshadow/penguin-core";
+
+/** Head window for Trace head reads: generous for a long system prompt, far below a whole multi-MB shard. */
+export const TRACE_HEAD_BYTES = 256 * 1024;
+
+/**
+ * Parse a Trace file's head window only. session_meta is the first line core writes to
+ * every shard (and the first user prompt follows right behind it in the earliest shard),
+ * so a bounded read finds them without pulling the whole file into memory.
+ * The window is cut at its last newline (the tail fragment is incomplete); a first line
+ * larger than the whole window falls back to the full tolerant read.
+ */
+export async function readTraceHead(filePath: string): Promise<OmniMessage[]> {
+  const fh = await open(filePath, "r");
+  let text: string;
+  let truncated: boolean;
+  try {
+    // allocUnsafe: only subarray(0, bytesRead) is ever read, so the uninitialized tail never leaks.
+    const { buffer, bytesRead } = await fh.read(
+      Buffer.allocUnsafe(TRACE_HEAD_BYTES),
+      0,
+      TRACE_HEAD_BYTES,
+      0,
+    );
+    text = buffer.subarray(0, bytesRead).toString("utf8");
+    truncated = bytesRead === TRACE_HEAD_BYTES;
+  } finally {
+    await fh.close();
+  }
+  if (!truncated) return parseTraceLines(text);
+  const nl = text.lastIndexOf("\n");
+  if (nl === -1) return readTraceTolerant(filePath);
+  return parseTraceLines(text.slice(0, nl + 1));
+}

packages/server/src/runtime/title-generator.ts

@@ -135,8 +135,14 @@ export class TitleGenerator implements TitleNotifier {
   }
 }
 
-/** Fallback title: take the material's first non-empty line, sanitize and truncate; if sanitizing empties it out (pure punctuation, etc.) fall back to the truncated original text; returns null if all-whitespace. */
-function fallbackTitle(text: string): string | null {
+/**
+ * Fallback title: take the material's first non-empty line, sanitize and truncate; if
+ * sanitizing empties it out (pure punctuation, etc.) fall back to the truncated original
+ * text; returns null if all-whitespace. Exported for the Trace listing's title fallback
+ * (trace-service derives a title from the first user prompt of Sessions the DB has no
+ * title for) so both fallbacks stay one algorithm.
+ */
+export function fallbackTitle(text: string): string | null {
   // Strip machine markers first: a skill invocation prepends a `[use_skills]` block, so the
   // raw first non-empty line would otherwise be that marker rather than the user's request.
   const firstLine = stripConversationMarkers(text)

packages/server/src/services/project-service.ts

@@ -31,6 +31,7 @@ import {
   SEMANTIC_ID_RULE,
 } from "./ids.js";
 import type { ProjectConfigService } from "./project-config-service.js";
+import type { TraceIndexService } from "./trace-index.js";
 
 /** Fallback timeout for waiting on runs to settle before deleting a Project. */
 const ABORT_SETTLE_TIMEOUT_MS = 5000;
@@ -57,6 +58,8 @@ export interface ProjectServiceDeps {
   goals: GoalsRepo;
   projectConfig: ProjectConfigService;
   manager: SessionManager;
+  /** Trace-index cleanup on Project destruction (rows describe files removed with the Project dir). */
+  traceIndex: TraceIndexService;
 }
 
 export class ProjectService {
@@ -312,6 +315,7 @@ export class ProjectService {
     this.deps.errors.deleteByProject(projectId);
     this.deps.schedules.deleteByProject(projectId);
     this.deps.goals.deleteByProject(projectId);
+    this.deps.traceIndex.removeProject(projectId);
     await fs.rm(projectDir(this.deps.root, projectId), { recursive: true, force: true });
   }
 

packages/server/src/services/session-service.ts

@@ -14,17 +14,7 @@
  * Project's default reference, 400 if there is none); the new Session is
  * added to session-manager's active table (state idle).
  */
-import path from "node:path";
-import { open, readdir } from "node:fs/promises";
-import {
-  createAgent,
-  findLatestTraceFile,
-  isSessionMeta,
-  parseTraceLines,
-  readTraceTolerant,
-  tracesDir,
-} from "@prismshadow/penguin-core";
-import type { SessionMetaMessage } from "@prismshadow/penguin-core";
+import { createAgent, isSessionMeta } from "@prismshadow/penguin-core";
 import type {
   ApprovalMode,
   SessionCategory,
@@ -38,44 +28,11 @@ import type { SessionRow, SessionsRepo } from "../db/repos/sessions.js";
 import type { SessionManager } from "../runtime/session-manager.js";
 import { asSessionSource } from "../runtime/session-sources.js";
 import type { SessionSources } from "../runtime/session-sources.js";
+import { TraceIndexService, traceFilePath } from "./trace-index.js";
 import type { ProjectConfigService } from "./project-config-service.js";
 
-const TRACE_FILE_RE = /^(.+)_(\d{3})\.jsonl$/;
 const SESSION_ID_TS_RE = /^session-(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-[0-9a-f]{8}$/;
 
-/** Head window for session_meta reads: generous for a long system prompt, far below a whole multi-MB shard. */
-const TRACE_HEAD_BYTES = 256 * 1024;
-
-/**
- * Parse a Trace file's head window only. session_meta is the first line core writes to
- * every shard, so a bounded read finds it without pulling the whole file into memory —
- * category filtering / counts may need every Session's source in a single request.
- * The window is cut at its last newline (the tail fragment is incomplete); a first line
- * larger than the whole window falls back to the full tolerant read.
- */
-async function readTraceHead(filePath: string) {
-  const fh = await open(filePath, "r");
-  let text: string;
-  let truncated: boolean;
-  try {
-    // allocUnsafe: only subarray(0, bytesRead) is ever read, so the uninitialized tail never leaks.
-    const { buffer, bytesRead } = await fh.read(
-      Buffer.allocUnsafe(TRACE_HEAD_BYTES),
-      0,
-      TRACE_HEAD_BYTES,
-      0,
-    );
-    text = buffer.subarray(0, bytesRead).toString("utf8");
-    truncated = bytesRead === TRACE_HEAD_BYTES;
-  } finally {
-    await fh.close();
-  }
-  if (!truncated) return parseTraceLines(text);
-  const nl = text.lastIndexOf("\n");
-  if (nl === -1) return readTraceTolerant(filePath);
-  return parseTraceLines(text.slice(0, nl + 1));
-}
-
 /** Derives creation time from the local timestamp embedded in session_id; returns null if it doesn't match. */
 export function sessionIdCreatedAt(sessionId: string): string | null {
   const m = SESSION_ID_TS_RE.exec(sessionId);
@@ -92,6 +49,8 @@ export interface SessionServiceDeps {
   projectConfig: ProjectConfigService;
   /** In-process origin registry derived from session_meta (the DB stores no source column). */
   sources: SessionSources;
+  /** Trace-file index: discovery / adoption / stats serve from it (mtime-gated reconciler; no per-request walks). */
+  traceIndex: TraceIndexService;
 }
 
 export class SessionService {
@@ -103,12 +62,8 @@ export class SessionService {
    * this process) falls back to reading the Trace head once (see sourceOf). `traces` is the
    * list flow's one-walk discovery result; without it a miss locates the shard itself.
    */
-  async toInfo(
-    row: SessionRow,
-    hasTrace: boolean,
-    traces?: ReadonlyMap<string, TraceLocation>,
-  ): Promise<SessionInfo> {
-    const source = await this.sourceOf(row, hasTrace, traces);
+  async toInfo(row: SessionRow, hasTrace: boolean): Promise<SessionInfo> {
+    const source = await this.sourceOf(row, hasTrace);
     return {
       sessionId: row.sessionId,
       projectId: row.projectId,
@@ -130,33 +85,24 @@ export class SessionService {
 
   /**
    * A Session's origin, with session_meta as the single source of truth: the in-process
-   * registry answers first (populated at creation / subagent registration / adoption);
-   * on a miss (a Session created before this process started) the earliest Trace shard's
-   * session_meta is read once and cached. A Session with no Trace yet stays unknown and
-   * is NOT cached negatively — its meta may appear with the first run.
+   * registry answers first (populated at creation / subagent registration / adoption /
+   * index registration); on a miss (a Session created before this process started) the
+   * trace index's registration-time facts answer — the reconciler head-read the earliest
+   * shard once when the file first appeared, so no file is touched here. A Session with
+   * no Trace yet stays unknown and is NOT cached negatively — its meta may appear with
+   * the first run.
    */
-  private async sourceOf(
-    row: SessionRow,
-    hasTrace: boolean,
-    traces?: ReadonlyMap<string, TraceLocation>,
-  ): Promise<SessionSource | undefined> {
+  private async sourceOf(row: SessionRow, hasTrace: boolean): Promise<SessionSource | undefined> {
     const known = this.deps.sources.get(row.sessionId);
     if (known !== undefined) return known ?? undefined;
     if (!hasTrace) return undefined;
-    // Single-session paths carry no discovery map: locate this Session's earliest shard on demand.
-    const location =
-      traces?.get(row.sessionId) ??
-      (await this.discoverTraces(row.projectId, row.agentId)).get(row.sessionId);
-    if (!location) return undefined;
-    const meta = await this.readTraceMeta(location);
-    if (!meta) return undefined; // Unreadable/corrupt Trace: stay unknown, retry on the next list.
-    // On-disk values are untrusted: only the exact known origins pass, junk = user-created.
-    const source = asSessionSource(meta.payload.source) ?? null;
-    this.deps.sources.set(row.sessionId, source);
-    return source ?? undefined;
+    const facts = this.deps.traceIndex.repo.getSession(row.sessionId);
+    if (!facts?.metaRead) return undefined; // Unreadable/unregistered: stay unknown, retry on the next list.
+    this.deps.sources.set(row.sessionId, facts.source);
+    return facts.source ?? undefined;
   }
 
-  /** Whether this Session already has a Trace record (a Task has been run). */
+  /** Whether this Session already has a Trace record (a Task has been run): answered by the index (reconciled first). */
   async hasTrace(row: SessionRow): Promise<boolean> {
     return (await this.discoverTraces(row.projectId, row.agentId)).has(row.sessionId);
   }
@@ -167,13 +113,9 @@ export class SessionService {
    * sidebar's partition applies to loaded rows, so server filtering and client
    * rendering can never disagree.
    */
-  private async categoryOf(
-    row: SessionRow,
-    hasTrace: boolean,
-    traces?: ReadonlyMap<string, TraceLocation>,
-  ): Promise<SessionCategory> {
+  private async categoryOf(row: SessionRow, hasTrace: boolean): Promise<SessionCategory> {
     if ((row.archivedAt ?? null) !== null) return "archived";
-    return (await this.sourceOf(row, hasTrace, traces)) ?? "active";
+    return (await this.sourceOf(row, hasTrace)) ?? "active";
   }
 
   /**
@@ -220,20 +162,22 @@ export class SessionService {
         .map((r) => [r.sessionId, r]),
     );
 
-    let traces: ReadonlyMap<string, TraceLocation> | undefined;
+    let traces: ReadonlySet<string> | undefined;
     if (includeCli) {
       traces = await this.discoverTraces(projectId, agentId);
-      // Unmanaged Trace Sessions (the CLI's): backfill an index row by reading the first
-      // line's session_meta, marked client "cli" so the default list can exclude them.
-      for (const [sessionId, location] of traces) {
+      // Unmanaged Trace Sessions (the CLI's): backfill an index row from the trace
+      // index's registration-time facts, marked client "cli" so the default list can
+      // exclude them.
+      for (const sessionId of traces) {
         if (rows.has(sessionId)) continue;
-        const discovered = await this.adoptTraceSession(projectId, agentId, sessionId, location);
+        const discovered = this.adoptTraceSession(projectId, agentId, sessionId);
         if (discovered) rows.set(sessionId, discovered);
       }
     } else if ([...rows.values()].some((r) => this.deps.sources.get(r.sessionId) === undefined)) {
-      // Hydration walk: some rows predate this process and are unclassified — locate their
-      // Traces once so sourceOf's head reads (cached afterwards) and the has_trace cache
-      // don't have to walk per row. Steady state (everything classified) skips this.
+      // Hydration pass: some rows predate this process and are unclassified — one
+      // reconciled index read supplies discovery so sourceOf's facts lookups and the
+      // has_trace cache need no per-row work. Steady state (everything classified)
+      // skips this.
       traces = await this.discoverTraces(projectId, agentId);
       for (const row of rows.values()) {
         if (!row.hasTrace && traces.has(row.sessionId)) {
@@ -249,7 +193,7 @@ export class SessionService {
     const rowHasTrace = (row: SessionRow): boolean =>
       traces ? traces.has(row.sessionId) : row.hasTrace === true;
     const toPage = (page: SessionRow[]) =>
-      Promise.all(page.map((row) => this.toInfo(row, rowHasTrace(row), traces)));
+      Promise.all(page.map((row) => this.toInfo(row, rowHasTrace(row))));
 
     // No classification asked for: slice straight away (the pre-category behavior).
     if (category === undefined && !withCounts) {
@@ -266,7 +210,7 @@ export class SessionService {
     const matched: SessionRow[] = [];
     for (const row of sorted) {
       if (!withCounts && matched.length >= want) break;
-      const cat = await this.categoryOf(row, rowHasTrace(row), traces);
+      const cat = await this.categoryOf(row, rowHasTrace(row));
       counts[cat] += 1;
       if (withCounts) {
         const ws = (workspaceCounts[row.workspace] ??= {
@@ -304,13 +248,12 @@ export class SessionService {
       byDate.set(date, set);
     };
 
-    // Trace directory: the date directory name is the local date (yyyy-mm-dd) that core uses when writing to disk.
-    const dir = tracesDir(this.deps.root, projectId, agentId);
-    for (const dateDir of await listDirsSafe(dir)) {
-      for (const file of await listFilesSafe(path.join(dir, dateDir))) {
-        const match = TRACE_FILE_RE.exec(file);
-        if (match) mark(dateDir, match[1]!);
-      }
+    // Trace activity from the index (one mtime-gated reconcile, then a pure DB read —
+    // this used to walk the Agent's ENTIRE trace history on every agents-list request):
+    // the date is the shard's date directory (local yyyy-mm-dd, core's writing convention).
+    await this.deps.traceIndex.reconcileAgent(projectId, agentId);
+    for (const f of this.deps.traceIndex.repo.listFilesByAgent(projectId, agentId)) {
+      mark(f.date, f.sessionId);
     }
     // DB index: the creation day also counts as active (a Session that hasn't run a Task yet produces no Trace).
     for (const row of this.deps.sessions.listByAgent(projectId, agentId)) {
@@ -432,11 +375,24 @@ export class SessionService {
    * source history itself when it needs it.
    */
   async latestTracePath(row: SessionRow): Promise<string | undefined> {
-    const located = await findLatestTraceFile(
-      tracesDir(this.deps.root, row.projectId, row.agentId),
+    await this.deps.traceIndex.reconcileAgent(row.projectId, row.agentId);
+    let files = this.deps.traceIndex.repo.listFilesBySession(
+      row.projectId,
+      row.agentId,
       row.sessionId,
     );
-    return located?.path;
+    if (files.length === 0) {
+      // Index miss with disk possibly ahead: one forced diff, then retry (the consumers'
+      // rule — a stale index costs one extra scan, never a missing resume shard).
+      await this.deps.traceIndex.reconcileAgent(row.projectId, row.agentId, { force: true });
+      files = this.deps.traceIndex.repo.listFilesBySession(
+        row.projectId,
+        row.agentId,
+        row.sessionId,
+      );
+    }
+    const latest = files.at(-1);
+    return latest === undefined ? undefined : traceFilePath(this.deps.root, latest);
   }
 
   /**
@@ -445,65 +401,41 @@ export class SessionService {
    * records) and the meta-read location come out of a single pass, so classifying every
    * row (`counts=1`) costs one directory walk total instead of one per Session.
    */
-  private async discoverTraces(
-    projectId: string,
-    agentId: string,
-  ): Promise<Map<string, TraceLocation>> {
-    const dir = tracesDir(this.deps.root, projectId, agentId);
-    const out = new Map<string, TraceLocation>();
-    for (const dateDir of await listDirsSafe(dir)) {
-      for (const file of await listFilesSafe(path.join(dir, dateDir))) {
-        const match = TRACE_FILE_RE.exec(file);
-        if (!match) continue;
-        const sessionId = match[1]!;
-        const index = Number(match[2]);
-        const cur = out.get(sessionId);
-        if (!cur || index < cur.index) {
-          out.set(sessionId, { path: path.join(dir, dateDir, file), index });
-        }
-      }
+  private async discoverTraces(projectId: string, agentId: string): Promise<Set<string>> {
+    await this.deps.traceIndex.reconcileAgent(projectId, agentId);
+    const out = new Set<string>();
+    for (const f of this.deps.traceIndex.repo.listFilesByAgent(projectId, agentId)) {
+      out.add(f.sessionId);
     }
     return out;
   }
 
   /**
-   * session_meta from a located Trace shard head; null when unreadable or it has no
-   * meta. Shared by adoption backfill and lazy `source` resolution.
+   * Adopts a Session that exists only in the Trace directory, from the index's
+   * registration-time facts (the reconciler head-read the earliest shard's session_meta
+   * once when the file first appeared — adoption itself reads no file).
    */
-  private async readTraceMeta(location: TraceLocation): Promise<SessionMetaMessage | null> {
-    let messages;
-    try {
-      messages = await readTraceHead(location.path);
-    } catch {
-      return null; // Corrupt file: skip (does not block the list)
-    }
-    return messages.find(isSessionMeta) ?? null;
-  }
-
-  /** Adopts a Session that exists only in the Trace directory: reads session_meta from the first line of the earliest index file. */
-  private async adoptTraceSession(
+  private adoptTraceSession(
     projectId: string,
     agentId: string,
     sessionId: string,
-    location: TraceLocation,
-  ): Promise<SessionRow | null> {
-    const meta = await this.readTraceMeta(location);
-    if (!meta) return null;
+  ): SessionRow | null {
+    const facts = this.deps.traceIndex.repo.getSession(sessionId);
+    if (!facts?.metaRead) return null; // Corrupt/unreadable head: skip (does not block the list; retried by a later reconcile)
     // An older Trace version's session_meta lacks provider (the model reference
     // wasn't split into separate fields yet): no backward compat, skip adoption
     // (core will give a clear error on resume; the product hasn't launched yet, so
     // old data can simply be deleted and recreated).
-    if (typeof meta.payload.provider !== "string") return null;
-    // The adoption read already has the meta in hand: record the origin (single source of
-    // truth); on-disk values are narrowed — junk counts as user-created.
-    this.deps.sources.set(sessionId, asSessionSource(meta.payload.source) ?? null);
+    if (facts.provider === null || facts.modelId === null) return null;
+    // Registration already narrowed the origin; record it in the registry (single source of truth).
+    this.deps.sources.set(sessionId, facts.source);
     const row: SessionRow = {
       sessionId,
       projectId,
       agentId,
-      provider: meta.payload.provider,
-      modelId: meta.payload.model_id,
-      workspace: meta.payload.workspace,
+      provider: facts.provider,
+      modelId: facts.modelId,
+      workspace: facts.workspace,
       // The approval mode for an unmanaged Session (started via the CLI) isn't in the Trace, so it's backfilled with the default value.
       approvalMode: "allow-all",
       title: null,
@@ -511,7 +443,7 @@ export class SessionService {
       // (web-only) excludes these rows; the "show CLI sessions" preference includes them.
       client: "cli",
       hasTrace: true,
-      createdAt: sessionIdCreatedAt(sessionId) ?? meta.timestamp,
+      createdAt: sessionIdCreatedAt(sessionId) ?? facts.firstTs ?? new Date().toISOString(),
     };
     // Idempotent backfill: concurrent list calls may discover the same Session for the first time simultaneously (consistent with AgentsRepo's convention).
     this.deps.sessions.insertOrIgnore(row);
@@ -519,32 +451,8 @@ export class SessionService {
   }
 }
 
-/** A located Trace shard of one Session: absolute path plus its shard index. */
-interface TraceLocation {
-  path: string;
-  index: number;
-}
-
 /** Local date as yyyy-mm-dd (matches the Trace date directory convention: core's internal formatLocalDate, not publicly exported). */
 function localDate(d: Date): string {
   const pad = (n: number) => (n < 10 ? `0${n}` : `${n}`);
   return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
 }
-
-async function listDirsSafe(dir: string): Promise<string[]> {
-  try {
-    const entries = await readdir(dir, { withFileTypes: true });
-    return entries.filter((e) => e.isDirectory()).map((e) => e.name);
-  } catch {
-    return [];
-  }
-}
-
-async function listFilesSafe(dir: string): Promise<string[]> {
-  try {
-    const entries = await readdir(dir, { withFileTypes: true });
-    return entries.filter((e) => e.isFile()).map((e) => e.name);
-  } catch {
-    return [];
-  }
-}

packages/server/src/services/trace-index.ts

@@ -0,0 +1,329 @@
+/**
+ * Trace-file index: keeps the trace_files / trace_sessions derived cache in step with
+ * the on-disk Trace tree so hot request paths never walk directories or head-read
+ * files. The DISK remains the single source of truth — every row is rebuildable, and
+ * the cache is never authority for absence (consumers force-reconcile and retry on a
+ * miss before erroring).
+ *
+ * Keeping in step happens on three paths:
+ *   - Write-time registration: server paths that create/delete shards with known
+ *     identities register synchronously (trace import via registerImportedFile;
+ *     Session delete via removeSession). Server task runs write through core's Trace
+ *     Writer without telling the server which shard rotated — those land via the
+ *     reconciler like external writes (the run bumps the date-dir mtime).
+ *   - mtime-gated reconciliation (reconcileAgent): the hot path stats the Agent's
+ *     traces root plus its newest known date dir (TWO stat calls, no readdir; a new
+ *     date dir bumps the root's mtime, a new file in the current date dir bumps that
+ *     dir's mtime). Only on a change does it readdir — and only the date dirs whose
+ *     mtime moved — registering new shards and classifying each new Session ONCE
+ *     (bounded head-read of the earliest shard for origin/workspace/title). Restart
+ *     forgets the in-memory gate, so the first request per Agent after an upgrade or
+ *     restart runs one full diff (this is also how the index first populates: no
+ *     migration step needed).
+ *   - Forced reconciliation (`force`): ignores the gate and diffs every date dir —
+ *     the miss-retry path for gate blind spots (e.g. an external write into an OLD
+ *     date dir moves neither gated mtime). A stale index therefore degrades to one
+ *     extra scan, never to a false 404.
+ *
+ * Single-flight per Agent: concurrent requests share one in-flight reconcile instead
+ * of stampeding the directory.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { agentsDir, isSessionMeta, tracesDir } from "@prismshadow/penguin-core";
+import type { OmniMessage } from "@prismshadow/penguin-core";
+import type { TraceFileRow, TraceSessionRow } from "../db/repos/trace-index.js";
+import { TraceIndexRepo } from "../db/repos/trace-index.js";
+import { readTraceHead } from "../internal/trace-head.js";
+import { asSessionSource } from "../runtime/session-sources.js";
+import type { SessionSources } from "../runtime/session-sources.js";
+import { fallbackTitle } from "../runtime/title-generator.js";
+
+const TRACE_FILE_RE = /^(.+)_(\d{3})\.jsonl$/;
+
+/**
+ * mtimes younger than this are treated as UNSTABLE and never cached as clean:
+ * filesystem timestamps are coarse (a write and a later change can land on the same
+ * tick), so a fresh directory keeps being re-diffed until it has been quiet for this
+ * long — an active Agent costs one small readdir per request while writing, and the
+ * gate can never wedge on a same-tick change.
+ */
+const FRESH_MS = 2000;
+
+/** A gate-cacheable mtime: the real value once stable, else a sentinel that never matches. */
+function cacheable(mtimeMs: number): number {
+  return Date.now() - mtimeMs > FRESH_MS ? mtimeMs : -1;
+}
+
+/** Absolute path of an indexed shard (reconstructed — rows never store paths; the data root may move). */
+export function traceFilePath(root: string, row: TraceFileRow): string {
+  const name = `${row.sessionId}_${String(row.fileIndex).padStart(3, "0")}.jsonl`;
+  return path.join(tracesDir(root, row.projectId, row.agentId), row.date, name);
+}
+
+/** mtimeMs of a path; null when it does not exist. */
+async function statMtime(p: string): Promise<number | null> {
+  try {
+    return (await fs.stat(p)).mtimeMs;
+  } catch {
+    return null;
+  }
+}
+
+async function listDirs(dir: string): Promise<string[]> {
+  try {
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    return entries.filter((e) => e.isDirectory()).map((e) => e.name);
+  } catch {
+    return [];
+  }
+}
+
+async function listFiles(dir: string): Promise<string[]> {
+  try {
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    return entries.filter((e) => e.isFile()).map((e) => e.name);
+  } catch {
+    return [];
+  }
+}
+
+/** Session facts extracted from a shard's records (registration-time classification). */
+function factsFromRecords(
+  projectId: string,
+  agentId: string,
+  sessionId: string,
+  records: OmniMessage[],
+): TraceSessionRow {
+  const meta = records.find(isSessionMeta);
+  let firstPrompt: string | null = null;
+  for (const msg of records) {
+    if (msg.type !== "model_msg") continue;
+    const p = msg.payload as { type?: string; role?: string; text?: string };
+    if (p.type === "text" && p.role === "user" && typeof p.text === "string") {
+      firstPrompt = p.text;
+      break;
+    }
+  }
+  return {
+    sessionId,
+    projectId,
+    agentId,
+    source: meta ? (asSessionSource(meta.payload.source) ?? null) : null,
+    workspace: meta && typeof meta.payload.workspace === "string" ? meta.payload.workspace : "",
+    title: firstPrompt !== null ? fallbackTitle(firstPrompt) : null,
+    provider: meta && typeof meta.payload.provider === "string" ? meta.payload.provider : null,
+    modelId: meta && typeof meta.payload.model_id === "string" ? meta.payload.model_id : null,
+    firstTs: records[0]?.timestamp ?? null,
+    metaRead: meta !== undefined,
+  };
+}
+
+/** In-memory reconcile gate of one Agent: last seen mtimes (traces root + per date dir). */
+interface SeenDirs {
+  root: number;
+  dates: Map<string, number>;
+}
+
+export class TraceIndexService {
+  private readonly seen = new Map<string, SeenDirs>();
+  private readonly inflight = new Map<string, Promise<void>>();
+  /**
+   * Test observability (asserting the hot path stays readdir/head-read free):
+   * gateStats = gate stat calls, dirScans = date-dir readdir passes, headReads =
+   * registration-time classification reads.
+   */
+  readonly counters = { gateStats: 0, dirScans: 0, headReads: 0 };
+
+  constructor(
+    private readonly root: string,
+    readonly repo: TraceIndexRepo,
+    /** Shared origin registry: registration-time classification publishes into it (single source of truth for `source`). */
+    private readonly sources?: SessionSources,
+  ) {}
+
+  /**
+   * Brings one Agent's index in step with disk. Hot path (nothing changed): two stat
+   * calls, zero readdir. `force` ignores the mtime gate and diffs every date dir (the
+   * consumers' miss-retry path).
+   */
+  reconcileAgent(
+    projectId: string,
+    agentId: string,
+    opts: { force?: boolean } = {},
+  ): Promise<void> {
+    const key = `${projectId}\0${agentId}`;
+    const existing = this.inflight.get(key);
+    if (existing) {
+      // A forced request must observe disk AFTER the point it was issued: chain a fresh
+      // pass behind the in-flight one instead of piggybacking on possibly-gated work.
+      return opts.force === true
+        ? existing.then(() => this.reconcileAgent(projectId, agentId, opts))
+        : existing;
+    }
+    const run = this.doReconcile(projectId, agentId, opts.force === true).finally(() => {
+      this.inflight.delete(key);
+    });
+    this.inflight.set(key, run);
+    return run;
+  }
+
+  /** Reconciles every Agent of a Project (the subagent-pointer resolver's miss path). */
+  async reconcileProject(projectId: string, opts: { force?: boolean } = {}): Promise<void> {
+    for (const agentId of await listDirs(agentsDir(this.root, projectId))) {
+      await this.reconcileAgent(projectId, agentId, opts);
+    }
+  }
+
+  private async doReconcile(projectId: string, agentId: string, force: boolean): Promise<void> {
+    const key = `${projectId}\0${agentId}`;
+    const dir = tracesDir(this.root, projectId, agentId);
+    const cached = this.seen.get(key);
+    this.counters.gateStats += 1;
+    const rootMtime = await statMtime(dir);
+    if (rootMtime === null) {
+      // No traces directory: whatever the index still holds for this Agent is stale.
+      this.repo.deleteByAgent(projectId, agentId);
+      this.seen.set(key, { root: -1, dates: new Map() });
+      return;
+    }
+    if (!force && cached && cached.root === rootMtime) {
+      // Root unchanged (no date dir created/removed). The one blind spot on this level is
+      // a new file inside an EXISTING date dir — in practice always the newest one (the
+      // Writer names dirs by current local date) — so gate on that single dir's mtime too.
+      // Cached sentinels (-1: the dir was fresh when last seen) never match, forcing a
+      // re-diff until the tree has been quiet (see FRESH_MS).
+      const newest = [...cached.dates.keys()].sort().at(-1);
+      if (newest === undefined) return;
+      this.counters.gateStats += 1;
+      const m = await statMtime(path.join(dir, newest));
+      if (m !== null && m === cached.dates.get(newest)) return;
+    }
+
+    // Change detected / first look / force: diff date dirs (readdir only the changed ones).
+    this.counters.dirScans += 1;
+    const next: SeenDirs = { root: cacheable(rootMtime), dates: new Map() };
+    const knownByDate = new Map<string, TraceFileRow[]>();
+    for (const row of this.repo.listFilesByAgent(projectId, agentId)) {
+      const list = knownByDate.get(row.date) ?? [];
+      list.push(row);
+      knownByDate.set(row.date, list);
+    }
+    const newSessions = new Set<string>();
+    const dateDirs = await listDirs(dir);
+    for (const date of dateDirs) {
+      const m = await statMtime(path.join(dir, date));
+      if (m === null) continue;
+      next.dates.set(date, cacheable(m));
+      // Unchanged dir already reflected in the gate cache: its rows are current. A fresh
+      // mtime is never treated as unchanged (and was cached as a non-matching sentinel).
+      if (!force && cached?.dates.get(date) === m) continue;
+      const known = new Map(
+        (knownByDate.get(date) ?? []).map((r) => [`${r.sessionId}\0${r.fileIndex}`, r]),
+      );
+      for (const file of await listFiles(path.join(dir, date))) {
+        const match = TRACE_FILE_RE.exec(file);
+        if (!match) continue;
+        const sessionId = match[1]!;
+        const fileIndex = Number(match[2]);
+        const fileKey = `${sessionId}\0${fileIndex}`;
+        let size = 0;
+        try {
+          size = (await fs.stat(path.join(dir, date, file))).size;
+        } catch {
+          continue; // Vanished between readdir and stat: skip; a later pass settles it.
+        }
+        this.repo.upsertFile({ projectId, agentId, sessionId, fileIndex, date, sizeBytes: size });
+        known.delete(fileKey);
+        if (this.repo.getSession(sessionId)?.metaRead !== true) newSessions.add(sessionId);
+      }
+      // Rows whose files vanished from this dir (external delete / rename).
+      for (const row of known.values()) {
+        this.repo.deleteFile(projectId, agentId, row.sessionId, row.fileIndex);
+      }
+    }
+    // Whole date dirs deleted from disk.
+    for (const [date, rows] of knownByDate) {
+      if (next.dates.has(date)) continue;
+      for (const row of rows)
+        this.repo.deleteFile(projectId, agentId, row.sessionId, row.fileIndex);
+    }
+    // Registration-time classification: ONCE per newly seen Session (bounded head-read
+    // of its earliest shard) — listings afterwards never touch file contents.
+    for (const sessionId of newSessions) {
+      await this.classifySession(projectId, agentId, sessionId);
+    }
+    this.seen.set(key, next);
+  }
+
+  /** Head-reads the Session's earliest indexed shard and stores its facts (origin/workspace/title/model ref). */
+  private async classifySession(
+    projectId: string,
+    agentId: string,
+    sessionId: string,
+  ): Promise<void> {
+    const files = this.repo.listFilesBySession(projectId, agentId, sessionId);
+    const earliest = files[0];
+    if (earliest === undefined) return;
+    this.counters.headReads += 1;
+    let records: OmniMessage[];
+    try {
+      records = await readTraceHead(traceFilePath(this.root, earliest));
+    } catch {
+      records = []; // Unreadable head: facts stay unknown (meta_read=0 → retried by a later reconcile pass).
+    }
+    const facts = factsFromRecords(projectId, agentId, sessionId, records);
+    this.repo.upsertSession(facts);
+    if (facts.metaRead) this.sources?.set(sessionId, facts.source);
+  }
+
+  /**
+   * Write-time registration for the Trace import route: the caller has the file's
+   * identity AND parsed records in hand, so the row and facts are stored synchronously
+   * with zero additional IO.
+   */
+  registerImportedFile(args: {
+    projectId: string;
+    agentId: string;
+    sessionId: string;
+    fileIndex: number;
+    date: string;
+    sizeBytes: number;
+    records: OmniMessage[];
+  }): void {
+    this.repo.upsertFile({
+      projectId: args.projectId,
+      agentId: args.agentId,
+      sessionId: args.sessionId,
+      fileIndex: args.fileIndex,
+      date: args.date,
+      sizeBytes: args.sizeBytes,
+    });
+    const facts = factsFromRecords(args.projectId, args.agentId, args.sessionId, args.records);
+    this.repo.upsertSession(facts);
+    if (facts.metaRead) this.sources?.set(args.sessionId, facts.source);
+    // The write moved the directory mtimes: drop the gate so the next reconcile re-syncs
+    // (cheap — the import's own rows are already upserted; the pass just confirms).
+    this.seen.delete(`${args.projectId}\0${args.agentId}`);
+  }
+
+  /** Write-time removal for Session deletion (files are being rm'ed by the caller). */
+  removeSession(projectId: string, agentId: string, sessionId: string): void {
+    this.repo.deleteBySession(sessionId);
+    this.seen.delete(`${projectId}\0${agentId}`);
+  }
+
+  /** Agent deletion: its whole tree is going away with it. */
+  removeAgent(projectId: string, agentId: string): void {
+    this.repo.deleteByAgent(projectId, agentId);
+    this.seen.delete(`${projectId}\0${agentId}`);
+  }
+
+  /** Project deletion. */
+  removeProject(projectId: string): void {
+    this.repo.deleteByProject(projectId);
+    for (const key of this.seen.keys()) {
+      if (key.startsWith(`${projectId}\0`)) this.seen.delete(key);
+    }
+  }
+}

packages/server/src/services/trace-service.ts

@@ -12,7 +12,6 @@
 import fs from "node:fs/promises";
 import path from "node:path";
 import {
-  agentsDir,
   isSessionMeta,
   parseTraceLines,
   parseUserSteeringText,
@@ -21,8 +20,11 @@ import {
 } from "@prismshadow/penguin-core";
 import type { OmniMessage } from "@prismshadow/penguin-core";
 import type {
+  AgentTraceSessionEntry,
   AgentTracesResponse,
   RequestSpan,
+  SessionCategory,
+  SessionCategoryCounts,
   ToolCallSpan,
   TraceAnalysisResponse,
   TraceEventsResponse,
@@ -33,8 +35,12 @@ import type {
   TraceToolSpan,
   UsageTrendPointInTrace,
 } from "../api/types.js";
+import type { SessionRow } from "../db/repos/sessions.js";
+import type { TraceFileRow, TraceSessionRow } from "../db/repos/trace-index.js";
 import { HttpError } from "../http/errors.js";
 import { formatLocalDate } from "../internal/dates.js";
+import type { SessionSources } from "../runtime/session-sources.js";
+import { TraceIndexService, traceFilePath } from "./trace-index.js";
 
 const TRACE_FILE_RE = /^(.+)_(\d{3})\.jsonl$/;
 
@@ -69,51 +75,68 @@ function subagentPointer(msg: OmniMessage): string | null {
   return p.session_id;
 }
 
-async function listDirs(dir: string): Promise<string[]> {
-  try {
-    const entries = await fs.readdir(dir, { withFileTypes: true });
-    return entries.filter((e) => e.isDirectory()).map((e) => e.name);
-  } catch {
-    return [];
-  }
+/**
+ * Read-only slice of the Session index the paginated listing consults (SessionsRepo
+ * structurally satisfies it; injected so service tests need no full app): one indexed
+ * query answers title / archived / workspace / client for every DB-tracked Session of
+ * the Agent.
+ */
+export interface TraceSessionIndex {
+  listByAgent(projectId: string, agentId: string): SessionRow[];
 }
 
-async function listFiles(dir: string): Promise<string[]> {
-  try {
-    const entries = await fs.readdir(dir, { withFileTypes: true });
-    return entries.filter((e) => e.isFile()).map((e) => e.name);
-  } catch {
-    return [];
-  }
+/** TraceService wiring (the trace-file index is required: every listing/locating path serves from it — no directory walks). */
+export interface TraceServiceDeps {
+  index: TraceIndexService;
+  /** Optional (narrow tests may omit): DB rows supplying titles / archived / workspace / client. */
+  sessions?: TraceSessionIndex;
+  /** Optional (narrow tests may omit): the shared in-process Session-origin registry (single source of truth for `source`). */
+  sources?: SessionSources;
+}
+
+/** One Session's classification result (see classify): its sidebar category + Workspace path ("" = unknown). */
+interface TraceSessionFacts {
+  category: SessionCategory;
+  workspace: string;
 }
 
 export class TraceService {
-  constructor(private readonly root: string) {}
+  constructor(
+    private readonly root: string,
+    private readonly deps: TraceServiceDeps,
+  ) {}
 
-  /** All of this Session's Trace files (sorted by index ascending). */
+  /**
+   * All of this Session's Trace files (sorted by index ascending), served from the
+   * index. An empty answer force-reconciles once and retries before being believed:
+   * disk is the source of truth, and a gate blind spot (an external write into an old
+   * date dir) must cost one extra scan, never a false 404.
+   */
   private async locateAll(
     projectId: string,
     agentId: string,
     sessionId: string,
   ): Promise<LocatedFile[]> {
-    const dir = tracesDir(this.root, projectId, agentId);
-    const out: LocatedFile[] = [];
-    for (const dateDir of await listDirs(dir)) {
-      for (const file of await listFiles(path.join(dir, dateDir))) {
-        const match = TRACE_FILE_RE.exec(file);
-        if (!match || match[1] !== sessionId) continue;
-        out.push({ path: path.join(dir, dateDir, file), date: dateDir, index: Number(match[2]) });
-      }
+    await this.deps.index.reconcileAgent(projectId, agentId);
+    let rows = this.deps.index.repo.listFilesBySession(projectId, agentId, sessionId);
+    if (rows.length === 0) {
+      await this.deps.index.reconcileAgent(projectId, agentId, { force: true });
+      rows = this.deps.index.repo.listFilesBySession(projectId, agentId, sessionId);
     }
-    return out.sort((a, b) => a.index - b.index);
+    return rows.map((r) => ({
+      path: traceFilePath(this.root, r),
+      date: r.date,
+      index: r.fileIndex,
+    }));
   }
 
-  /** Deletes all of this Session's Trace files (called when the Session is deleted). */
+  /** Deletes all of this Session's Trace files (called when the Session is deleted); the index rows go with them. */
   async deleteSessionTraces(projectId: string, agentId: string, sessionId: string): Promise<void> {
     const files = await this.locateAll(projectId, agentId, sessionId);
     for (const file of files) {
       await fs.rm(file.path, { force: true });
     }
+    this.deps.index.removeSession(projectId, agentId, sessionId);
   }
 
   /**
@@ -138,38 +161,36 @@ export class TraceService {
     sessionId: string,
   ): Promise<OmniMessage[]> {
     return this.readMessagesExpanded(projectId, agentId, sessionId, {
-      index: null,
+      projectScanned: false,
       ancestry: new Set([sessionId]),
       depth: 0,
     });
   }
 
   /**
-   * A Project-wide session location index (sessionId -> agentId): built by
-   * scanning every Agent's traces directory. Built lazily the first time a
-   * subagent pointer is encountered, then reused across the whole readMessages
-   * call — rescanning per pointer would blow up into tens of thousands of readdir
-   * calls under multiple sub-sessions plus recursive expansion.
+   * Locates a subagent pointer's owning Agent within the Project: a single index
+   * lookup (the old implementation walked EVERY Agent's whole traces tree to build a
+   * sessionId → agentId map per readMessages call). On a miss, the whole Project is
+   * force-reconciled ONCE per readMessages call and the lookup retried — a child
+   * Trace written by an external process is one scan away, never invisible.
    */
-  private async buildSessionIndex(projectId: string): Promise<Map<string, string>> {
-    const index = new Map<string, string>();
-    for (const agentId of await listDirs(agentsDir(this.root, projectId))) {
-      const dir = tracesDir(this.root, projectId, agentId);
-      for (const dateDir of await listDirs(dir)) {
-        for (const file of await listFiles(path.join(dir, dateDir))) {
-          const match = TRACE_FILE_RE.exec(file);
-          if (match && !index.has(match[1]!)) index.set(match[1]!, agentId);
-        }
-      }
-    }
-    return index;
+  private async resolveChildAgent(
+    projectId: string,
+    childSid: string,
+    ctx: { projectScanned: boolean },
+  ): Promise<string | null> {
+    const hit = this.deps.index.repo.findAgentBySession(projectId, childSid);
+    if (hit !== null || ctx.projectScanned) return hit;
+    ctx.projectScanned = true;
+    await this.deps.index.reconcileProject(projectId, { force: true });
+    return this.deps.index.repo.findAgentBySession(projectId, childSid);
   }
 
   private async readMessagesExpanded(
     projectId: string,
     agentId: string,
     sessionId: string,
-    ctx: { index: Map<string, string> | null; ancestry: Set<string>; depth: number },
+    ctx: { projectScanned: boolean; ancestry: Set<string>; depth: number },
   ): Promise<OmniMessage[]> {
     const files = await this.locateAll(projectId, agentId, sessionId);
     const out: OmniMessage[] = [];
@@ -182,8 +203,7 @@ export class TraceService {
           out.push(msg);
           continue;
         }
-        ctx.index ??= await this.buildSessionIndex(projectId);
-        const childAgent = ctx.index.get(childSid);
+        const childAgent = await this.resolveChildAgent(projectId, childSid, ctx);
         let nested: OmniMessage[] = [];
         if (childAgent) {
           ctx.ancestry.add(childSid);
@@ -685,35 +705,178 @@ export class TraceService {
     };
   }
 
-  /** Level-by-level browsing (newest first): Agent -> date -> Session -> Trace files. */
-  async agentTraces(projectId: string, agentId: string): Promise<AgentTracesResponse> {
-    const dir = tracesDir(this.root, projectId, agentId);
-    const dates = (await listDirs(dir)).sort().reverse();
-    const out: AgentTracesResponse = { dates: [] };
-    for (const date of dates) {
-      const bySession = new Map<string, { index: number; sizeBytes: number }[]>();
-      for (const file of await listFiles(path.join(dir, date))) {
-        const match = TRACE_FILE_RE.exec(file);
-        if (!match) continue;
-        const sessionId = match[1]!;
-        const stat = await fs.stat(path.join(dir, date, file));
-        const files = bySession.get(sessionId) ?? [];
-        files.push({ index: Number(match[2]), sizeBytes: stat.size });
-        bySession.set(sessionId, files);
-      }
-      if (bySession.size === 0) continue;
-      out.dates.push({
-        date,
-        // session_id embeds a timestamp, so reverse lexicographic order is reverse chronological order.
-        sessions: [...bySession.entries()]
-          .sort((a, b) => b[0].localeCompare(a[0]))
-          .map(([sessionId, files]) => ({
-            sessionId,
-            files: files.sort((a, b) => a.index - b.index),
-          })),
+  /**
+   * Agent-level browsing, served from the trace-file index (one mtime-gated reconcile,
+   * then pure DB — no directory walks, no head-reads). Without `paging`: the legacy full
+   * drill-down (newest first: Agent -> date -> Session -> Trace files; sizes are the
+   * index's last observed values). With `paging`: session-group-centric — Sessions
+   * ordered by id descending (ids embed a timestamp, so that is reverse chronological),
+   * optionally filtered to one sidebar `category`, CLI-origin Sessions excluded unless
+   * `includeCli` (the "show CLI sessions" preference; the legacy shape is never
+   * filtered — back-compat), and only the returned slice gets per-file `fs.stat` for
+   * fresh sizes (written back to the index).
+   */
+  async agentTraces(
+    projectId: string,
+    agentId: string,
+    paging?: { offset: number; limit: number } | null,
+    opts: { category?: SessionCategory; includeCli?: boolean } = {},
+  ): Promise<AgentTracesResponse> {
+    await this.deps.index.reconcileAgent(projectId, agentId);
+    const files = this.deps.index.repo.listFilesByAgent(projectId, agentId);
+    if (paging) return this.agentTracesPage(projectId, agentId, files, paging, opts);
+    const byDate = new Map<string, Map<string, { index: number; sizeBytes: number }[]>>();
+    for (const f of files) {
+      const sessions =
+        byDate.get(f.date) ?? new Map<string, { index: number; sizeBytes: number }[]>();
+      const list = sessions.get(f.sessionId) ?? [];
+      list.push({ index: f.fileIndex, sizeBytes: f.sizeBytes });
+      sessions.set(f.sessionId, list);
+      byDate.set(f.date, sessions);
+    }
+    return {
+      dates: [...byDate.keys()]
+        .sort()
+        .reverse()
+        .map((date) => ({
+          date,
+          // session_id embeds a timestamp, so reverse lexicographic order is reverse chronological order.
+          sessions: [...byDate.get(date)!.entries()]
+            .sort((a, b) => b[0].localeCompare(a[0]))
+            .map(([sessionId, list]) => ({
+              sessionId,
+              files: list.sort((a, b) => a.index - b.index),
+            })),
+        })),
+    };
+  }
+
+  /**
+   * Classification (no IO): `archived` comes exactly from the DB row; the origin comes
+   * from the shared sources registry, else from the Session's registration-time facts
+   * (trace_sessions — the reconciler head-read its earliest shard once when the file
+   * first appeared, so by listing time every indexed Session is classified exactly).
+   */
+  private classify(
+    sessionId: string,
+    row: SessionRow | undefined,
+    facts: TraceSessionRow | undefined,
+  ): TraceSessionFacts {
+    const known = this.deps.sources?.get(sessionId);
+    // Registry answer (including null = known user-created) wins — it can be fresher
+    // (subagent registration happens at spawn, before any reconcile); else the stored facts.
+    const source = known !== undefined ? known : (facts?.source ?? undefined);
+    const category: SessionCategory =
+      (row?.archivedAt ?? null) !== null
+        ? "archived"
+        : source === "subagent" || source === "schedule"
+          ? source
+          : "active";
+    return { category, workspace: row?.workspace ?? facts?.workspace ?? "" };
+  }
+
+  /** The paginated listing behind agentTraces: pure index reads; per-file stat only for the returned slice (sizes written back). */
+  private async agentTracesPage(
+    projectId: string,
+    agentId: string,
+    files: TraceFileRow[],
+    paging: { offset: number; limit: number },
+    opts: { category?: SessionCategory; includeCli?: boolean },
+  ): Promise<AgentTracesResponse> {
+    const bySession = new Map<string, TraceFileRow[]>();
+    for (const f of files) {
+      const list = bySession.get(f.sessionId) ?? [];
+      list.push(f);
+      bySession.set(f.sessionId, list);
+    }
+    // One indexed query per source: the Agent's DB rows (title / archived / workspace /
+    // client — CLI rows included) and the registration-time facts.
+    const rows = new Map(
+      (this.deps.sessions?.listByAgent(projectId, agentId) ?? []).map((r) => [r.sessionId, r]),
+    );
+    const factsBySession = new Map(
+      this.deps.index.repo.listSessionsByAgent(projectId, agentId).map((r) => [r.sessionId, r]),
+    );
+    /**
+     * CLI-origin = no web-created DB row AND not a subagent/schedule Session. Server-created
+     * Sessions always have a row (creation / subagent registration insert one), so rowless =
+     * external CLI; adopted CLI rows carry client='cli'. The default listing excludes these —
+     * the same "show CLI sessions" preference the sidebar applies — while subagent/schedule
+     * Sessions stay in their folders regardless of which process ran them.
+     */
+    const cliOrigin = (sessionId: string, facts: TraceSessionFacts): boolean => {
+      if (facts.category === "subagent" || facts.category === "schedule") return false;
+      const row = rows.get(sessionId);
+      return !(
+        row !== undefined &&
+        (row.client === null || row.client === undefined || row.client === "web")
+      );
+    };
+    const ids = [...bySession.keys()].sort((a, b) => b.localeCompare(a));
+    // Classify every group once; the same result drives the CLI/category filters, the
+    // counts AND the returned fields, so a row can never appear in a bucket its own
+    // `category` denies. Hidden CLI-origin groups are excluded from counts too.
+    const counts: SessionCategoryCounts = { active: 0, subagent: 0, schedule: 0, archived: 0 };
+    const workspaceCounts: Record<string, SessionCategoryCounts> = {};
+    const factsById = new Map<string, TraceSessionFacts>();
+    const visible: string[] = [];
+    for (const id of ids) {
+      const facts = this.classify(id, rows.get(id), factsBySession.get(id));
+      if (opts.includeCli !== true && cliOrigin(id, facts)) continue;
+      factsById.set(id, facts);
+      visible.push(id);
+      counts[facts.category] += 1;
+      const ws = (workspaceCounts[facts.workspace] ??= {
+        active: 0,
+        subagent: 0,
+        schedule: 0,
+        archived: 0,
+      });
+      ws[facts.category] += 1;
+    }
+    const filtered =
+      opts.category === undefined
+        ? visible
+        : visible.filter((id) => factsById.get(id)!.category === opts.category);
+    const page = filtered.slice(paging.offset, paging.offset + paging.limit);
+    const sessions: AgentTraceSessionEntry[] = [];
+    for (const sessionId of page) {
+      const shard = bySession.get(sessionId)!.sort((a, b) => a.fileIndex - b.fileIndex);
+      // Fresh sizes for the returned page only (an actively-appended shard grows without
+      // moving any directory mtime): bounded metadata stats, written back to the index.
+      const withSize = await Promise.all(
+        shard.map(async (f) => {
+          let sizeBytes = f.sizeBytes;
+          try {
+            sizeBytes = (await fs.stat(traceFilePath(this.root, f))).size;
+            if (sizeBytes !== f.sizeBytes) {
+              this.deps.index.repo.updateFileSize(
+                projectId,
+                agentId,
+                sessionId,
+                f.fileIndex,
+                sizeBytes,
+              );
+            }
+          } catch {
+            /* Vanished mid-request: serve the last observed size; the next reconcile settles the rows. */
+          }
+          return { index: f.fileIndex, date: f.date, sizeBytes };
+        }),
+      );
+      const facts = factsById.get(sessionId)!;
+      // Title: DB row first (?? sends both "no row" and "row with NULL title" onward);
+      // else the registration-time first-prompt fallback stored in trace_sessions.
+      const title = rows.get(sessionId)?.title ?? factsBySession.get(sessionId)?.title ?? undefined;
+      sessions.push({
+        sessionId,
+        ...(title !== undefined ? { title } : {}),
+        category: facts.category,
+        workspace: facts.workspace,
+        files: withSize,
       });
     }
-    return out;
+    return { dates: [], sessions, totalSessions: filtered.length, counts, workspaceCounts };
   }
 
   private async readFileByIndex(
@@ -802,19 +965,28 @@ export class TraceService {
     const dir = path.join(tracesDir(this.root, projectId, agentId), date);
     await fs.mkdir(dir, { recursive: true });
     const file = path.join(dir, `${sessionId}_${String(index).padStart(3, "0")}.jsonl`);
+    const body = content.replace(/\n+$/, "") + "\n";
     try {
       // Normalize to exactly one trailing newline (the JSONL convention the writer
       // follows). `wx` closes the check-then-write race: two concurrent imports of the
       // same new session id both pass the locateAll check, but only one can create the
       // file — the loser's EEXIST maps to the same 409 as the pre-check.
-      await fs.writeFile(file, content.replace(/\n+$/, "") + "\n", {
-        encoding: "utf8",
-        flag: "wx",
-      });
+      await fs.writeFile(file, body, { encoding: "utf8", flag: "wx" });
     } catch (err) {
       if ((err as NodeJS.ErrnoException).code === "EEXIST") throw duplicate();
       throw err;
     }
+    // Write-time registration: this path knows the file's identity and already holds the
+    // parsed records, so the index row + session facts land synchronously (no re-read).
+    this.deps.index.registerImportedFile({
+      projectId,
+      agentId,
+      sessionId,
+      fileIndex: index,
+      date,
+      sizeBytes: Buffer.byteLength(body, "utf8"),
+      records,
+    });
     return { sessionId, index, date };
   }
 }