feat(core,server,web,cli): mid-run steering messages and square-bracket markers (#63)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yaowei ZhengandClaude Fable 5 authored and GitHub committed 2026-07-26 23:01:18 +08:00
1 parent abd5b13d52
commit 7584171482
81 files changed
+3139 -930

No files matched your search

+3
View File
@@ -1,3 +1,6 @@
export * from "./types.js";
export * from "./builders.js";
export * from "./aggregate.js";
// Special message markers ([user_steering], [use_skills], [handoff_from], …): one module,
// also reachable directly as `@prismshadow/penguin-core/markers`.
export * from "./markers/index.js";
@@ -0,0 +1,73 @@
/**
* Marker block primitives — the one place that knows how a special message marker is
* *spelled*.
*
* Canonical form is `[tag]…[/tag]`. The earlier angle-bracket form `<tag>…</tag>` is still
* **recognized by every parser** and must stay recognized indefinitely: markers are persisted
* in Traces (re-rendered later) and in each Agent's `system_config.yaml` compaction prompt
* (old agents keep instructing the model to use `<summary>`). Producers emit the square form
* only — that asymmetry is the whole compatibility policy, and it lives here rather than being
* re-derived at each call site.
*/
/** Escapes a tag name for embedding in a RegExp (tags are `[a-z_]`, but keep this honest). */
function escapeTag(tag: string): string {
return tag.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
/** Wraps `body` in the canonical paired form: `[tag]\n…\n[/tag]` (newlines around the body). */
export function markerBlock(tag: string, body: string): string {
return `[${tag}]\n${body}\n[/${tag}]`;
}
/** Opening/closing delimiters of the canonical form (for prefix checks and inline tags). */
export function markerOpen(tag: string): string {
return `[${tag}]`;
}
export function markerClose(tag: string): string {
return `[/${tag}]`;
}
/**
* A pair of regexes for one tag — the current square form and the legacy angle form — built
* from a single source pattern. `body` is the sub-pattern placed between the tags (capture
* groups inside it keep their order), `flags` are applied to both.
*
* Callers match square first and fall back to angle (`matchDualForm`), so a message mixing
* both resolves to the current form.
*/
export function dualFormPatterns(tag: string, body: string, flags = ""): [RegExp, RegExp] {
const t = escapeTag(tag);
return [
new RegExp(`\\[${t}\\]${body}\\[/${t}\\]`, flags),
new RegExp(`<${t}>${body}</${t}>`, flags),
];
}
/** Runs the square-form pattern first, then the legacy angle form; returns the first match (or null). */
export function matchDualForm(
patterns: readonly [RegExp, RegExp],
text: string,
): RegExpExecArray | null {
return patterns[0].exec(text) ?? patterns[1].exec(text);
}
/**
* Strips every occurrence of one tag's block from `text` (both forms, across lines), plus any
* stray unpaired opening/closing tag left behind. Used by title generation, where a leaked
* marker must never become the title.
*/
export function stripMarkerBlocks(text: string, tag: string): string {
const t = escapeTag(tag);
return text
.replace(new RegExp(`\\[${t}\\][\\s\\S]*?\\[/${t}\\]`, "g"), "")
.replace(new RegExp(`\\[/?${t}\\]`, "g"), "")
.replace(new RegExp(`<${t}>[\\s\\S]*?</${t}>`, "g"), "")
.replace(new RegExp(`</?${t}>`, "g"), "");
}
/** True when `text` starts with this marker's opening tag in either form (prefix detection, e.g. context_summary). */
export function startsWithMarker(text: string, tag: string): boolean {
return text.startsWith(`[${tag}]`) || text.startsWith(`<${tag}>`);
}
@@ -0,0 +1,102 @@
/**
* Engine-synthesized marker blocks: the interruption/retry transcripts and the compaction
* summary. These are produced by the ReAct loop (and rebuilt by Trace replay), never by a
* host — but their spelling belongs here with the rest of the markers so the engine and the
* resume path cannot drift apart (they used to duplicate the `[context_summary]` template).
*/
import { dualFormPatterns, markerBlock, matchDualForm } from "./block.js";
import { MARKER_TAGS, TRANSCRIPT_TAGS } from "./tags.js";
/** Wraps a compaction summary as the new context's first input: `[context_summary]…[/context_summary]`. */
export function buildContextSummaryText(summary: string): string {
return markerBlock(MARKER_TAGS.contextSummary, summary);
}
const SUMMARY_PATTERNS = dualFormPatterns(MARKER_TAGS.summary, "([\\s\\S]*?)");
/**
* Extracts the summary the model wrote inside `[summary]…[/summary]` during compaction; when
* the tag is missing, leniently uses the entire output as-is (not treated as a failure).
*
* The legacy `<summary></summary>` form must stay accepted **indefinitely**:
* `compaction.prompt` is persisted in every existing agent's `system_config.yaml`, so old
* agents keep instructing the model to use the angle-bracket tags.
*/
export function extractSummary(raw: string): string {
const match = matchDualForm(SUMMARY_PATTERNS, raw);
return (match ? match[1]! : raw).trim();
}
/** One transcribed line inside a turn block: an inner tag, optional attributes, and the body. */
function transcriptLine(tag: string, body: string, attrs = ""): string {
return ` [${tag}${attrs}]${body}[/${tag}]`;
}
/** `[user_input]…[/user_input]` — a text input of the interrupted turn. */
export function transcribeUserInput(text: string): string {
return transcriptLine(TRANSCRIPT_TAGS.userInput, text);
}
/** `[thinking]…[/thinking]` — a thinking segment the model produced before the interruption. */
export function transcribeThinking(thinking: string): string {
return transcriptLine(TRANSCRIPT_TAGS.thinking, thinking);
}
/** `[text]…[/text]` — a text segment the model produced before the interruption. */
export function transcribeText(text: string): string {
return transcriptLine(TRANSCRIPT_TAGS.text, text);
}
/** `[tool_call name=… id=…]arguments[/tool_call]` — a tool call already issued. */
export function transcribeToolCall(name: string, toolCallId: string, args: string): string {
return transcriptLine(TRANSCRIPT_TAGS.toolCall, args, ` name="${name}" id="${toolCallId}"`);
}
/** `[tool_call_output id=… status=…]output[/tool_call_output]` — a tool result already produced. */
export function transcribeToolCallOutput(
toolCallId: string,
status: string,
output: string,
): string {
return transcriptLine(
TRANSCRIPT_TAGS.toolCallOutput,
output,
` id="${toolCallId}" status="${status}"`,
);
}
/** Wraps transcribed lines as the interruption block resent as carry-over. */
export function buildTurnAbortedBlock(lines: readonly string[]): string {
return [`[${MARKER_TAGS.turnAborted}]`, ...lines, `[/${MARKER_TAGS.turnAborted}]`].join("\n");
}
/** Wraps transcribed lines as the reconnect-retry block appended to the resent input. */
export function buildTurnRetriedBlock(lines: readonly string[]): string {
return [`[${MARKER_TAGS.turnRetried}]`, ...lines, `[/${MARKER_TAGS.turnRetried}]`].join("\n");
}
/**
* Both synthetic turn blocks share one matcher (a backreference keeps the tags paired): the
* current square form first, then the legacy angle form — resume replays a discarded turn's
* original input as-is, so text written to a Trace before the marker format changed can still
* flow back in.
*/
const SYNTHETIC_BLOCK_PATTERNS: readonly [RegExp, RegExp] = [
new RegExp(
`^\\[(${MARKER_TAGS.turnAborted}|${MARKER_TAGS.turnRetried})\\]\\n?([\\s\\S]*?)\\n?\\[/\\1\\]\\s*$`,
),
new RegExp(
`^<(${MARKER_TAGS.turnAborted}|${MARKER_TAGS.turnRetried})>\\n?([\\s\\S]*?)\\n?</\\1>\\s*$`,
),
];
/**
* If the text is an engine-synthesized whole block (`[turn_aborted]` or `[turn_retried]`),
* strips the outer tags and returns the inner lines (possibly an empty string); otherwise
* null. Both kinds use identical inner markup, so the content can be merged straight into a
* new block, keeping a single-level structure.
*/
export function unwrapSyntheticBlock(text: string): string | null {
const m = matchDualForm(SYNTHETIC_BLOCK_PATTERNS, text);
return m ? m[2]! : null;
}
@@ -0,0 +1,27 @@
/**
* Special message markers — one module for every marker the product writes into message text.
*
* A marker is a block of the form `[tag]…[/tag]` embedded in an OmniMessage's text: it tells
* the model (and the render layers) that this part of the conversation is machine-inserted
* rather than something the user typed. They come in three families:
*
* - **engine blocks** (`engine-blocks.ts`): `[turn_aborted]` / `[turn_retried]` transcripts,
* the `[context_summary]` injection and the `[summary]` the compaction prompt asks for —
* produced by the ReAct loop and Trace replay;
* - **origin blocks** (`origin-blocks.ts`): `[use_skills]`, `[handoff_from]`,
* `[scheduled_task]`, `[model_switch_from]` — prefixed to a user message by the hosts
* (Web composer, server scheduler) and collapsed into a banner when rendered;
* - **steering** (`steering.ts`): `[user_steering]`, a mid-run user message.
*
* `block.ts` owns the spelling itself — the canonical square form for producers and the
* dual-form (square + legacy angle) matching every parser applies, because markers persist in
* Traces and in each agent's stored compaction prompt. `tags.ts` owns the tag list.
*
* Everything here is pure string work: no OmniMessage envelopes, no I/O — the callers wrap
* the result in `userText(...)` (or match against a payload's text) themselves.
*/
export * from "./block.js";
export * from "./tags.js";
export * from "./engine-blocks.js";
export * from "./origin-blocks.js";
export * from "./steering.js";
@@ -0,0 +1,256 @@
/**
* Origin blocks — marker blocks that **prefix a user message** to say where it came from or
* what it should use, produced by the hosts (Web composer, server scheduler) and collapsed
* into a banner by the render layers.
*
* Each marker gets a `build*` producer (canonical square form) and a `parse*` inverse that
* also accepts the legacy angle form, so messages persisted in older Traces still render as
* banners rather than raw markers. The block's field lines are `key: value`; free-text
* explanation lines are ignored by the parsers.
*/
import { dualFormPatterns, markerBlock, matchDualForm } from "./block.js";
import { MARKER_TAGS } from "./tags.js";
// ---------------------------------------------------------------------------
// [use_skills] — skill invocation prefixed to the user's message
// ---------------------------------------------------------------------------
/** Parsed `[use_skills]` block: the selected skill names plus the message body after it. */
export interface SkillsMessage {
skills: string[];
rest: string;
}
/** Builds a message body with a `[use_skills]` block: an empty list omits the block; with no body text only the block is returned (no trailing blank line). */
export function buildSkillsMessage(names: string[], text: string): string {
if (names.length === 0) return text;
const block = markerBlock(MARKER_TAGS.useSkills, `skills: ${names.join(", ")}`);
return text ? `${block}\n\n${text}` : block;
}
const SKILLS_PATTERNS = dualFormPatterns(MARKER_TAGS.useSkills, "\\nskills: ([^\\n]+)\\n");
/**
* Inverse of `buildSkillsMessage`: recognizes a block only at **the start of the message**,
* returning the skill names and the remaining body (a block appearing mid-body is plain
* text). The `skills:` line is split on commas and trimmed; an empty list is not a block.
*/
export function parseSkillsMessage(text: string): SkillsMessage | null {
const m = matchDualForm(SKILLS_PATTERNS, text);
if (!m || m.index !== 0) return null;
const skills = m[1]!
.split(",")
.map((s) => s.trim())
.filter((s) => s.length > 0);
if (skills.length === 0) return null;
return { skills, rest: text.slice(m[0].length).replace(/^\n+/, "") };
}
// ---------------------------------------------------------------------------
// Field-line helpers shared by the origin blocks below
// ---------------------------------------------------------------------------
/** Splits a `value` of the shape `id (label)` into its parts (label undefined when absent). */
function splitLabeled(value: string): { id: string; label?: string } {
const m = /^([\w-]+) \((.*)\)$/.exec(value);
return m ? { id: m[1]!, label: m[2]! } : { id: value };
}
/** Iterates the `key: value` field lines of a block body, ignoring prose lines. */
function fieldLines(body: string, keys: readonly string[]): Array<[string, string]> {
const pattern = new RegExp(`^(${keys.join("|")}): (.+)$`);
const out: Array<[string, string]> = [];
for (const line of body.split("\n")) {
const kv = pattern.exec(line);
if (kv) out.push([kv[1]!, kv[2]!]);
}
return out;
}
// ---------------------------------------------------------------------------
// [handoff_from] — @-mention handoff into a new conversation
// ---------------------------------------------------------------------------
/** Origin info for an @-handoff new conversation: source agent is always present; the source Session is omitted while it's still a draft. */
export interface HandoffOrigin {
agentId: string;
agentName?: string;
sessionId?: string;
sessionTitle?: string;
workspace?: string;
}
/**
* First message of an @-handoff new conversation (English): the `[handoff_from]` block states
* that this conversation was opened by an @ mention and carries the source agent / Session /
* Workspace, so the @-mentioned agent knows its origin (e.g. defaulting to the source agent as
* its working target, or reaching source files via the Workspace path). The parenthetical
* label is omitted when the display name/title equals the id or is absent.
*/
export function buildHandoffMessage(origin: HandoffOrigin): string {
const name =
origin.agentName && origin.agentName !== origin.agentId ? ` (${origin.agentName})` : "";
const lines = [`agent: ${origin.agentId}${name}`];
if (origin.sessionId) {
const title = origin.sessionTitle ? ` (${origin.sessionTitle})` : "";
lines.push(`session: ${origin.sessionId}${title}`);
}
if (origin.workspace) lines.push(`workspace: ${origin.workspace}`);
return markerBlock(
MARKER_TAGS.handoffFrom,
[
"This conversation was opened by @-mentioning you from another conversation; its origin is listed below and the user's message, if any, follows. When the request refers to an agent, session, or files without naming them, it means this origin.",
...lines,
].join("\n"),
);
}
const HANDOFF_PATTERNS = dualFormPatterns(MARKER_TAGS.handoffFrom, "\\n([\\s\\S]*)\\n", "");
/**
* Inverse of `buildHandoffMessage`: returns origin info when the **whole** message is one
* `[handoff_from]` block, otherwise null (a normal user message renders as-is).
*/
export function parseHandoffMessage(text: string): HandoffOrigin | null {
const trimmed = text.trim();
const block = matchDualForm(HANDOFF_PATTERNS, trimmed);
if (!block || block.index !== 0 || block[0].length !== trimmed.length) return null;
const origin: HandoffOrigin = { agentId: "" };
for (const [key, value] of fieldLines(block[1]!, ["agent", "session", "workspace"])) {
if (key === "workspace") {
origin.workspace = value;
continue;
}
const { id, label } = splitLabeled(value);
if (key === "agent") {
origin.agentId = id;
if (label !== undefined) origin.agentName = label;
} else {
origin.sessionId = id;
if (label !== undefined) origin.sessionTitle = label;
}
}
return origin.agentId ? origin : null;
}
// ---------------------------------------------------------------------------
// [scheduled_task] — scheduled trigger, followed by the task's own Prompt body
// ---------------------------------------------------------------------------
/** Origin info for a scheduled-task trigger. */
export interface ScheduledOrigin {
/** Task name (filename minus .toml). */
name: string;
/** Trigger timestamp (ISO 8601); empty string when absent from the block. */
firedAt: string;
}
/**
* Trigger input = a `[scheduled_task]` origin block (task name and fire time) + the prompt
* body: tells the model this was fired by a schedule; the frontend collapses the origin block
* into a one-line schedule hint (Trace shows it verbatim).
*/
export function buildScheduledMessage(name: string, firedAt: string, prompt: string): string {
const block = markerBlock(
MARKER_TAGS.scheduledTask,
[
"This message was sent automatically by a scheduled task; its origin is listed below and the task prompt follows.",
`schedule: ${name}`,
`fired_at: ${firedAt}`,
].join("\n"),
);
return `${block}\n\n${prompt}`;
}
const SCHEDULED_PATTERNS = dualFormPatterns(MARKER_TAGS.scheduledTask, "\\n([\\s\\S]*?)\\n");
/**
* Inverse of `buildScheduledMessage`: returns origin info and the remaining text when the
* message **starts with** a `[scheduled_task]` block, otherwise null. Unlike handoff, the
* block is followed by the task's Prompt body, which is returned alongside it for normal
* rendering (the raw block isn't shown; the Trace page shows it as-is).
*/
export function parseScheduledMessage(
text: string,
): { origin: ScheduledOrigin; rest: string } | null {
const m = matchDualForm(SCHEDULED_PATTERNS, text);
if (!m || m.index !== 0) return null;
const origin: ScheduledOrigin = { name: "", firedAt: "" };
for (const [key, value] of fieldLines(m[1]!, ["schedule", "fired_at"])) {
if (key === "schedule") origin.name = value;
else origin.firedAt = value;
}
if (!origin.name) return null;
return { origin, rest: text.slice(m[0].length).replace(/^\n+/, "") };
}
// ---------------------------------------------------------------------------
// [model_switch_from] — the /model command's handoff-style conversation switch
// ---------------------------------------------------------------------------
/**
* Origin info for a `/model` switch new conversation (modeled on HandoffOrigin): the source
* Session, the absolute path of its latest Trace file (the model reads it for the earlier
* history — nothing is injected into the new context), the shared Workspace, and the previous
* model's paired reference.
*/
export interface ModelSwitchOrigin {
sessionId: string;
sessionTitle?: string;
/** Absolute path of the source session's latest Trace file (JSONL of OmniMessage envelopes). */
tracePath?: string;
workspace?: string;
/** The source session's model reference pair (never concatenated — two separate fields). */
prevProvider?: string;
prevModelId?: string;
}
/**
* First message of a `/model` switch new conversation (English, mirroring the handoff block):
* the `[model_switch_from]` block states that this conversation continues an earlier one on a
* different model and carries the source Session / Trace path / Workspace / previous model
* pair. The earlier history is deliberately NOT injected — some models require thinking
* payloads and provider `fidelity` byte-for-byte when history is replayed, which cannot cross
* models — so the model reads the Trace file itself when it needs the context.
*/
export function buildModelSwitchMessage(origin: ModelSwitchOrigin): string {
const title = origin.sessionTitle ? ` (${origin.sessionTitle})` : "";
const lines = [`session: ${origin.sessionId}${title}`];
if (origin.tracePath) lines.push(`trace: ${origin.tracePath}`);
if (origin.workspace) lines.push(`workspace: ${origin.workspace}`);
if (origin.prevModelId) lines.push(`previous_model: ${origin.prevModelId}`);
if (origin.prevProvider) lines.push(`previous_provider: ${origin.prevProvider}`);
return markerBlock(
MARKER_TAGS.modelSwitchFrom,
[
"The user switched models: this conversation continues an earlier conversation, now on a different model. Its origin is listed below and the user's message, if any, follows. The earlier history is NOT in your context — when you need it, read the trace file at the path below (JSONL, one message envelope per line: user/assistant text, tool calls and results).",
...lines,
].join("\n"),
);
}
const MODEL_SWITCH_PATTERNS = dualFormPatterns(MARKER_TAGS.modelSwitchFrom, "\\n([\\s\\S]*)\\n");
/**
* Inverse of `buildModelSwitchMessage`: returns origin info when the **whole** message is one
* `[model_switch_from]` block, otherwise null. The session line's `(title)` label is split
* off; prose lines are ignored.
*/
export function parseModelSwitchMessage(text: string): ModelSwitchOrigin | null {
const trimmed = text.trim();
const block = matchDualForm(MODEL_SWITCH_PATTERNS, trimmed);
if (!block || block.index !== 0 || block[0].length !== trimmed.length) return null;
const origin: ModelSwitchOrigin = { sessionId: "" };
const keys = ["session", "trace", "workspace", "previous_model", "previous_provider"];
for (const [key, value] of fieldLines(block[1]!, keys)) {
if (key === "session") {
const { id, label } = splitLabeled(value);
origin.sessionId = id;
if (label !== undefined) origin.sessionTitle = label;
} else if (key === "trace") origin.tracePath = value;
else if (key === "workspace") origin.workspace = value;
else if (key === "previous_model") origin.prevModelId = value;
else origin.prevProvider = value;
}
return origin.sessionId ? origin : null;
}
@@ -0,0 +1,38 @@
/**
* `[user_steering]` — mid-run user messages delivered between turns.
*
* While a Task is running, the user can send a steering message (`Session.steer`): the engine
* queues it and delivers it as a **standalone user text message** wrapped in a
* `[user_steering]…[/user_steering]` block, sent with the next request input alongside that
* turn's tool outputs (or as the continuation input when the turn produced no tool calls) —
* the model sees it without the agent loop being interrupted. The message is real user input:
* written to Trace like any Prompt and yielded to the output stream. The marker exists so the
* model and the render layers can tell it apart from a task-starting Prompt: UIs keep it inside
* the running Task (no new Task segment) and render it as user speech instead of raw markers.
*
* Unlike the other markers this one has **no legacy angle form** to accept: it was introduced
* after the square-bracket convention, so no Trace can contain `<user_steering>`.
*/
import { markerBlock } from "./block.js";
import { MARKER_TAGS } from "./tags.js";
/** Wraps one queued steering message as the standalone user-text body sent to the model. */
export function userSteeringText(text: string): string {
return markerBlock(MARKER_TAGS.userSteering, text);
}
/** Matches a whole user text that is exactly one `[user_steering]` block (square form only, see the module note). */
const STEERING_TEXT_RE = new RegExp(
`^\\[${MARKER_TAGS.userSteering}\\]\\n([\\s\\S]*?)\\n\\[/${MARKER_TAGS.userSteering}\\]\\s*$`,
);
/**
* Inverse of `userSteeringText`: when the whole user text is one `[user_steering]` block,
* returns the inner message; otherwise null (a normal user message — including one merely
* mentioning the marker mid-body — is left alone). Used by task segmentation (a steering
* message must not start a new Task) and by the Web/CLI steering rendering.
*/
export function parseUserSteeringText(text: string): string | null {
const m = STEERING_TEXT_RE.exec(text);
return m ? m[1]! : null;
}
@@ -0,0 +1,52 @@
/**
* The canonical marker tag list — every special marker the product writes into message text,
* in one place so producers, parsers and the title stripper cannot drift apart.
*
* All are spelled `[tag]…[/tag]`; see ./block.ts for the form and the legacy-compat policy.
*/
/** Marker tags that wrap a whole message or a leading block of one. */
export const MARKER_TAGS = {
/** Interrupted turn, transcribed for resend (engine carry-over). */
turnAborted: "turn_aborted",
/** Automatic reconnect retry, carrying the failed attempt's partial output (engine). */
turnRetried: "turn_retried",
/** Compaction summary injected as the new context's first input (engine / resume). */
contextSummary: "context_summary",
/** The summary the model is asked to write during compaction (compaction prompt / parser). */
summary: "summary",
/** Mid-run user message delivered between turns (Session.steer). */
userSteering: "user_steering",
/** Skill invocation block prefixed to a user message (Web composer). */
useSkills: "use_skills",
/** @-handoff origin block, first message of the delegated conversation (Web). */
handoffFrom: "handoff_from",
/** Scheduled-task trigger origin block (server scheduler). */
scheduledTask: "scheduled_task",
/** `/model` switch origin block, first message of the continued conversation (Web). */
modelSwitchFrom: "model_switch_from",
/** AGENTS.md wrapper inside the system prompt (static template text). */
developerInstructions: "developer_instructions",
} as const;
/** Inner tags used inside a transcribed turn block (`[turn_aborted]` / `[turn_retried]`). */
export const TRANSCRIPT_TAGS = {
userInput: "user_input",
thinking: "thinking",
text: "text",
toolCall: "tool_call",
toolCallOutput: "tool_call_output",
} as const;
/**
* Machine-inserted marker blocks that must never leak into a generated Session title: the
* blocks that **prefix a user message** (a skill invocation, a handoff / scheduled-task /
* model-switch origin note). Engine-synthesized blocks are never title material, so they are
* deliberately not in this list.
*/
export const TITLE_NOISE_TAGS: readonly string[] = [
MARKER_TAGS.useSkills,
MARKER_TAGS.handoffFrom,
MARKER_TAGS.scheduledTask,
MARKER_TAGS.modelSwitchFrom,
];