feat(core): recover truncated tool output via the Session scratchpad (#145)
Co-authored-by: Yaowei Zheng <hiyouga@buaa.edu.cn> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -23,7 +23,7 @@ import {
|
||||
projectDir,
|
||||
goalFilePath,
|
||||
resolveModelRef,
|
||||
scratchpadDir,
|
||||
sessionScratchpadDir,
|
||||
systemConfigPath,
|
||||
tracesDir,
|
||||
type AgentState,
|
||||
@@ -256,6 +256,7 @@ export class Agent {
|
||||
);
|
||||
|
||||
const rt = await this.buildRuntime({
|
||||
sessionId,
|
||||
workspaceDir,
|
||||
modelEntry,
|
||||
apiKey,
|
||||
@@ -292,8 +293,10 @@ export class Agent {
|
||||
createBareLLM: rt.createBareLLM,
|
||||
compaction: rt.compaction,
|
||||
// Where an input image lands when it becomes a path line (see SessionConfig.imagesDir).
|
||||
imagesDir: path.join(
|
||||
scratchpadDir(this.state.root, this.state.projectId, this.state.agentId),
|
||||
imagesDir: sessionScratchpadDir(
|
||||
this.state.root,
|
||||
this.state.projectId,
|
||||
this.state.agentId,
|
||||
sessionId,
|
||||
),
|
||||
modelHasVision: modelEntry.vision !== false,
|
||||
@@ -388,6 +391,7 @@ export class Agent {
|
||||
const thinkingLevel = this.state.systemConfig.model?.thinking_level;
|
||||
|
||||
const rt = await this.buildRuntime({
|
||||
sessionId,
|
||||
workspaceDir,
|
||||
modelEntry,
|
||||
apiKey,
|
||||
@@ -449,8 +453,10 @@ export class Agent {
|
||||
createBareLLM: rt.createBareLLM,
|
||||
compaction: rt.compaction,
|
||||
// Where an input image lands when it becomes a path line (see SessionConfig.imagesDir).
|
||||
imagesDir: path.join(
|
||||
scratchpadDir(this.state.root, this.state.projectId, this.state.agentId),
|
||||
imagesDir: sessionScratchpadDir(
|
||||
this.state.root,
|
||||
this.state.projectId,
|
||||
this.state.agentId,
|
||||
sessionId,
|
||||
),
|
||||
modelHasVision: modelEntry.vision !== false,
|
||||
@@ -492,6 +498,7 @@ export class Agent {
|
||||
* and its post-compaction rebuild factory, and the compaction config.
|
||||
*/
|
||||
private async buildRuntime(args: {
|
||||
sessionId: string;
|
||||
workspaceDir: string;
|
||||
/** This Session's Model entry: the caller (createSession / resumeSession) has already validated it exists in the config. */
|
||||
modelEntry: ModelEntry;
|
||||
@@ -511,6 +518,7 @@ export class Agent {
|
||||
compaction: CompactionSettings;
|
||||
}> {
|
||||
const {
|
||||
sessionId,
|
||||
workspaceDir,
|
||||
modelEntry,
|
||||
apiKey,
|
||||
@@ -685,6 +693,14 @@ export class Agent {
|
||||
const environment = new Environment({
|
||||
workspaceDir,
|
||||
toolConfig,
|
||||
// The Session's generic scratchpad root; Environment derives its truncated-tool-output
|
||||
// recovery directory from it.
|
||||
sessionScratchpadDir: sessionScratchpadDir(
|
||||
this.state.root,
|
||||
this.state.projectId,
|
||||
this.state.agentId,
|
||||
sessionId,
|
||||
),
|
||||
services: { subagentRunner, ...(visionDescriber ? { visionDescriber } : {}) },
|
||||
...(Object.keys(vault).length > 0 ? { vault } : {}),
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
* never empty under any circumstance**.
|
||||
* Docs: /docs/tools § "Execution contract".
|
||||
*/
|
||||
import path from "node:path";
|
||||
import { partialToolCallOutput, toolCallOutput } from "../omnimessage/index.js";
|
||||
import type { OmniMessage, StopReason } from "../omnimessage/index.js";
|
||||
import type {
|
||||
@@ -37,6 +38,13 @@ import type { BuiltinTool, ToolResult } from "./tools/types.js";
|
||||
import { BUILTIN_TOOL_FACTORIES } from "./tools/registry.js";
|
||||
import { CommandSessionManager } from "./tools/command/index.js";
|
||||
import { SubagentSessionManager } from "./tools/subagent/index.js";
|
||||
import {
|
||||
TRUNCATED_TOOL_OUTPUT_FILE_LIMIT_BYTES,
|
||||
TruncatedToolOutputArchive,
|
||||
type TruncatedToolOutputArchiveSaveResult,
|
||||
type TruncatedToolOutputCapture,
|
||||
} from "./truncated-tool-output-archive.js";
|
||||
import { modelVisiblePath } from "../internal/model-visible-path.js";
|
||||
|
||||
/** Default cap on tool output truncation (characters). */
|
||||
const DEFAULT_MAX_OUTPUT_LENGTH = 16000;
|
||||
@@ -78,6 +86,11 @@ function noteSuffix(base: string, note: string): string {
|
||||
export class Environment implements EnvironmentInterface {
|
||||
private readonly workspaceDir: string;
|
||||
private readonly toolConfig: ToolConfig;
|
||||
/**
|
||||
* Truncated-output recovery, derived from the generic `sessionScratchpadDir` config; null for
|
||||
* standalone embedders without a Session directory (legacy truncation-only behavior).
|
||||
*/
|
||||
private readonly truncatedToolOutputArchive: TruncatedToolOutputArchive | null;
|
||||
/** Assembled built-in tools: tool name -> BuiltinTool. Only tools supported by the registry and present in config. */
|
||||
private readonly tools: Map<string, BuiltinTool>;
|
||||
/** Long-running command session registry: constructed within this Environment and shared between exec_command / input_command. */
|
||||
@@ -88,6 +101,11 @@ export class Environment implements EnvironmentInterface {
|
||||
constructor(config: EnvironmentConfig) {
|
||||
this.workspaceDir = config.workspaceDir;
|
||||
this.toolConfig = config.toolConfig;
|
||||
this.truncatedToolOutputArchive = config.sessionScratchpadDir
|
||||
? new TruncatedToolOutputArchive({
|
||||
rootDir: path.join(config.sessionScratchpadDir, "truncated-tool-output"),
|
||||
})
|
||||
: null;
|
||||
this.tools = new Map();
|
||||
// The background session registry is created alongside Environment (one per Session) and
|
||||
// injected into whichever tools need it; all sessions are finalized together on dispose.
|
||||
@@ -216,6 +234,10 @@ export class Environment implements EnvironmentInterface {
|
||||
let selfNote: string | null = null; // Tool's self-reported end marker (e.g. exit code), appended outside truncation
|
||||
let selfImages: string[] | undefined; // Tool's self-reported images (data URL), carried via a single streamed delta and the full message
|
||||
let thrown: unknown = null;
|
||||
// Created lazily on the first over-limit text delta when this Environment has a Session
|
||||
// scratchpad. It captures the tool's complete text before Environment drops the overflow,
|
||||
// but does not alter the model/frontend stream.
|
||||
let archiveCapture: TruncatedToolOutputCapture | null = null;
|
||||
const gen = tool.execute(args, {
|
||||
workspaceDir: this.workspaceDir,
|
||||
toolCallId,
|
||||
@@ -249,6 +271,17 @@ export class Environment implements EnvironmentInterface {
|
||||
// Only takes delta content; start/stop are ignored (framing is uniformly handled by Environment).
|
||||
if (p.event_type !== "delta" || !p.output) continue;
|
||||
contentLen += p.output.length;
|
||||
const exceedsVisibleLimit = maxOutputLength > 0 && contentLen > maxOutputLength;
|
||||
if (exceedsVisibleLimit && this.truncatedToolOutputArchive) {
|
||||
if (!archiveCapture) {
|
||||
archiveCapture = this.truncatedToolOutputArchive.startCapture();
|
||||
// `streamed` is the exact prefix already accepted before this delta. Appending it
|
||||
// once, then every complete current/future delta, reconstructs the pre-truncation
|
||||
// tool text without changing what is forwarded.
|
||||
archiveCapture.append(streamed);
|
||||
}
|
||||
archiveCapture.append(p.output);
|
||||
}
|
||||
// maxOutputLength <= 0 means truncation is disabled (same semantics as timeoutMs).
|
||||
const room =
|
||||
maxOutputLength > 0 ? maxOutputLength - streamed.length : Number.POSITIVE_INFINITY;
|
||||
@@ -265,6 +298,18 @@ export class Environment implements EnvironmentInterface {
|
||||
} else if (p.type === "tool_call_output") {
|
||||
// Fallback: if the tool still produces a full message, use it as the basis for content and stop reason (not needed under the new contract).
|
||||
toolOutput = p.output ?? "";
|
||||
if (
|
||||
maxOutputLength > 0 &&
|
||||
toolOutput.length > maxOutputLength &&
|
||||
this.truncatedToolOutputArchive
|
||||
) {
|
||||
if (!archiveCapture) {
|
||||
archiveCapture = this.truncatedToolOutputArchive.startCapture();
|
||||
}
|
||||
// A compatibility tool's complete message is Environment's content basis, so it
|
||||
// also becomes the recovery basis instead of any deltas it happened to emit.
|
||||
archiveCapture.replace(toolOutput);
|
||||
}
|
||||
if (selfReported === undefined && p.stop_reason) {
|
||||
selfReported = p.stop_reason as StopReason;
|
||||
}
|
||||
@@ -293,16 +338,40 @@ export class Environment implements EnvironmentInterface {
|
||||
? contentBase.slice(0, maxOutputLength)
|
||||
: contentBase;
|
||||
const truncated = capped.length < contentBase.length || contentLen > streamed.length;
|
||||
|
||||
// Freeze the tool's terminal facts before auxiliary archive I/O. A user abort arriving
|
||||
// while the file is being written must not reclassify an already-finished tool.
|
||||
const aborted =
|
||||
signal?.aborted === true ||
|
||||
(!timedOut &&
|
||||
(selfReported === "aborted" ||
|
||||
(thrown as { name?: string } | null)?.name === "AbortError"));
|
||||
let archiveResult: TruncatedToolOutputArchiveSaveResult | null = null;
|
||||
if (truncated && archiveCapture) {
|
||||
// Both truncation paths initialize this capture at the exact point they first exceed the
|
||||
// visible cap, so a truncated call with a Session scratchpad always has one to save. A
|
||||
// standalone Environment has no capture and retains truncation-only behavior.
|
||||
archiveResult = await archiveCapture.save(name, toolCallId);
|
||||
} else {
|
||||
archiveCapture?.cancel();
|
||||
}
|
||||
|
||||
let stopReason: StopReason;
|
||||
const notes: string[] = [];
|
||||
if (truncated) {
|
||||
notes.push(`[output truncated: exceeded ${maxOutputLength} chars]`);
|
||||
if (archiveResult?.status === "saved") {
|
||||
const archivePath = modelVisiblePath(archiveResult.path);
|
||||
if (archiveResult.archiveTruncated) {
|
||||
const limitMiB = Math.ceil(TRUNCATED_TOOL_OUTPUT_FILE_LIMIT_BYTES / (1024 * 1024));
|
||||
notes.push(
|
||||
`[output archived (${limitMiB} MiB limit; head and tail kept): ${archivePath}]`,
|
||||
);
|
||||
} else {
|
||||
notes.push(`[output archived: ${archivePath}]`);
|
||||
}
|
||||
} else if (archiveResult?.status === "failed") {
|
||||
notes.push(`[output archive failed: ${archiveResult.code}]`);
|
||||
}
|
||||
}
|
||||
// The tool's self-reported end marker (e.g. exit code): appended outside the truncation —
|
||||
// if treated as a content delta it would get cut off once long output hits the cap, and the
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
* only reports `aborted` — the interruption note is appended by Environment.
|
||||
* Docs: /docs/tools § "File tools".
|
||||
*/
|
||||
import { modelVisiblePath } from "../../internal/model-visible-path.js";
|
||||
import path from "node:path";
|
||||
import { open, realpath, stat } from "node:fs/promises";
|
||||
import { partialToolCallOutput } from "../../omnimessage/index.js";
|
||||
@@ -300,7 +301,7 @@ export function createReadFileTool(definition: ToolDefinitionConfig): BuiltinToo
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "ENOENT") {
|
||||
yield delta(
|
||||
`File not found: "${filePath}". Check the path — relative paths resolve against the workspace (${ctx.workspaceDir}).`,
|
||||
`File not found: "${filePath}". Check the path — relative paths resolve against the workspace (${modelVisiblePath(ctx.workspaceDir)}).`,
|
||||
);
|
||||
} else {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
/**
|
||||
* TruncatedToolOutputArchive — bounded, Session-scoped recovery for text that Environment cannot
|
||||
* place in the model-visible tool result because of maxOutputLength.
|
||||
*
|
||||
* The archive is deliberately not a second tool protocol, and this module is internal:
|
||||
* Environment constructs it from the generic `EnvironmentConfig.sessionScratchpadDir` rather
|
||||
* than taking a manager object through the public config surface. Environment returns the file
|
||||
* path in the same truncated tool result seen by the frontend and the model; the model can then
|
||||
* use the existing file tools to inspect it. Files live in the Session scratchpad and are removed
|
||||
* by the host's existing Session-deletion path together with the rest of that scratchpad.
|
||||
*
|
||||
* Files are written only after a call actually exceeds maxOutputLength. A capture retains at
|
||||
* most one file's budget while the tool is streaming, then writes one UTF-8 .log file with mode
|
||||
* 0600. Small archives are exact. If a single call exceeds the per-file budget, the file keeps
|
||||
* bounded head/tail windows with an explicit gap marker.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { READ_FILE_SCAN_CAP_BYTES } from "./tools/read-file.js";
|
||||
|
||||
/**
|
||||
* Maximum stored bytes for one truncated tool call. One byte of headroom below read_file's
|
||||
* 8 MiB scan cap lets that tool perform its final zero-byte read and confirm EOF.
|
||||
*/
|
||||
export const TRUNCATED_TOOL_OUTPUT_FILE_LIMIT_BYTES = READ_FILE_SCAN_CAP_BYTES - 1;
|
||||
|
||||
const ARCHIVE_GAP_MARKER = "\n[archive middle truncated]\n";
|
||||
const ARCHIVE_GAP_MARKER_BYTES = Buffer.byteLength(ARCHIVE_GAP_MARKER);
|
||||
|
||||
export type TruncatedToolOutputArchiveSaveResult =
|
||||
| {
|
||||
status: "saved";
|
||||
path: string;
|
||||
archiveTruncated: boolean;
|
||||
}
|
||||
| { status: "failed"; code: string };
|
||||
|
||||
interface TruncatedToolOutputArchiveOptions {
|
||||
rootDir: string;
|
||||
/** Test-only override; production and public SDK composition use the fixed default. */
|
||||
fileLimitBytes?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies one byte range into a dedicated Buffer. Using Buffer.subarray directly would retain
|
||||
* the source's entire backing ArrayBuffer, defeating the capture's memory bound.
|
||||
*/
|
||||
function copyBufferRange(buffer: Buffer, start: number, end: number): Buffer {
|
||||
const result = Buffer.alloc(Math.max(0, end - start));
|
||||
buffer.copy(result, 0, start, end);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies a UTF-8-safe Buffer prefix. Moving a cut inside a multi-byte code point back to its
|
||||
* leading byte excludes that partial character rather than writing U+FFFD.
|
||||
*/
|
||||
function utf8BufferPrefix(buffer: Buffer, maxBytes: number): Buffer {
|
||||
if (maxBytes <= 0 || buffer.length === 0) return Buffer.alloc(0);
|
||||
if (buffer.length <= maxBytes) return copyBufferRange(buffer, 0, buffer.length);
|
||||
let cut = maxBytes;
|
||||
while (cut > 0 && (buffer[cut]! & 0xc0) === 0x80) cut -= 1;
|
||||
return copyBufferRange(buffer, 0, cut);
|
||||
}
|
||||
|
||||
/** Copies a UTF-8-safe Buffer suffix whose encoded size does not exceed maxBytes. */
|
||||
function utf8BufferSuffix(buffer: Buffer, maxBytes: number): Buffer {
|
||||
if (maxBytes <= 0 || buffer.length === 0) return Buffer.alloc(0);
|
||||
if (buffer.length <= maxBytes) return copyBufferRange(buffer, 0, buffer.length);
|
||||
let start = buffer.length - maxBytes;
|
||||
while (start < buffer.length && (buffer[start]! & 0xc0) === 0x80) start += 1;
|
||||
return copyBufferRange(buffer, start, buffer.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encodes only a bounded string prefix before applying the byte cap. One UTF-16 code unit
|
||||
* contributes at least one UTF-8 byte, so maxBytes (+ one paired surrogate) is sufficient to
|
||||
* find the complete prefix without ever encoding an unbounded input delta.
|
||||
*/
|
||||
function utf8Prefix(text: string, maxBytes: number): Buffer {
|
||||
if (maxBytes <= 0 || text.length === 0) return Buffer.alloc(0);
|
||||
let end = Math.min(text.length, maxBytes);
|
||||
if (
|
||||
end < text.length &&
|
||||
end > 0 &&
|
||||
text.charCodeAt(end - 1) >= 0xd800 &&
|
||||
text.charCodeAt(end - 1) <= 0xdbff &&
|
||||
text.charCodeAt(end) >= 0xdc00 &&
|
||||
text.charCodeAt(end) <= 0xdfff
|
||||
) {
|
||||
end += 1;
|
||||
}
|
||||
return utf8BufferPrefix(Buffer.from(text.slice(0, end), "utf8"), maxBytes);
|
||||
}
|
||||
|
||||
/** Encodes only a bounded string suffix, preserving a surrogate pair at the slice boundary. */
|
||||
function utf8Suffix(text: string, maxBytes: number): Buffer {
|
||||
if (maxBytes <= 0 || text.length === 0) return Buffer.alloc(0);
|
||||
let start = Math.max(0, text.length - maxBytes);
|
||||
if (
|
||||
start > 0 &&
|
||||
text.charCodeAt(start) >= 0xdc00 &&
|
||||
text.charCodeAt(start) <= 0xdfff &&
|
||||
text.charCodeAt(start - 1) >= 0xd800 &&
|
||||
text.charCodeAt(start - 1) <= 0xdbff
|
||||
) {
|
||||
start -= 1;
|
||||
}
|
||||
return utf8BufferSuffix(Buffer.from(text.slice(start), "utf8"), maxBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* One bounded capture. Each capture independently enforces the per-file memory and disk limit.
|
||||
*/
|
||||
export class TruncatedToolOutputCapture {
|
||||
private exactChunks: Buffer[] = [];
|
||||
private exactBytes = 0;
|
||||
private head: Buffer = Buffer.alloc(0);
|
||||
/** Fixed-capacity ring storage for the rolling UTF-8 tail after promotion. */
|
||||
private tail: Buffer = Buffer.alloc(0);
|
||||
private tailStart = 0;
|
||||
private tailLength = 0;
|
||||
private archiveTruncated = false;
|
||||
private settled = false;
|
||||
/** A streamed JS string may split one UTF-16 surrogate pair across deltas. */
|
||||
private pendingHighSurrogate = "";
|
||||
|
||||
constructor(
|
||||
private readonly owner: TruncatedToolOutputArchive,
|
||||
private readonly fileLimitBytes: number,
|
||||
) {}
|
||||
|
||||
/** Appends the exact text delta produced by the tool before Environment truncates it. */
|
||||
append(text: string): void {
|
||||
if (this.settled || text.length === 0) return;
|
||||
if (this.pendingHighSurrogate) {
|
||||
const pending = this.pendingHighSurrogate;
|
||||
this.pendingHighSurrogate = "";
|
||||
const first = text.charCodeAt(0);
|
||||
if (first >= 0xdc00 && first <= 0xdfff) {
|
||||
// Join only the actual pair, not the whole new delta: concatenating a one-character
|
||||
// pending surrogate with an arbitrarily large delta would create an unbounded copy.
|
||||
this.appendStable(pending + text.slice(0, 1));
|
||||
text = text.slice(1);
|
||||
if (text.length === 0) return;
|
||||
} else {
|
||||
// The pending high surrogate is now known to be lone. Keep the current delta untouched:
|
||||
// its own final high surrogate may still pair with the following delta.
|
||||
this.appendStable(pending);
|
||||
}
|
||||
}
|
||||
const last = text.charCodeAt(text.length - 1);
|
||||
if (last >= 0xd800 && last <= 0xdbff) {
|
||||
this.pendingHighSurrogate = text.slice(-1);
|
||||
text = text.slice(0, -1);
|
||||
}
|
||||
if (text.length === 0) return;
|
||||
this.appendStable(text);
|
||||
}
|
||||
|
||||
private appendStable(text: string): void {
|
||||
const chunkBytes = Buffer.byteLength(text, "utf8");
|
||||
|
||||
if (!this.archiveTruncated) {
|
||||
if (this.exactBytes + chunkBytes <= this.fileLimitBytes) {
|
||||
// Keep this encoding inside the accepted branch. Moving Buffer.from above the size
|
||||
// guard would allocate an unbounded Buffer for one huge delta before rejecting it.
|
||||
// append() also guarantees no chunk boundary can split a still-pairable surrogate:
|
||||
// paired halves are joined first, while confirmed lone surrogates intentionally encode
|
||||
// as U+FFFD (guarded by the cross-delta Unicode test).
|
||||
this.exactChunks.push(Buffer.from(text, "utf8"));
|
||||
this.exactBytes += chunkBytes;
|
||||
return;
|
||||
}
|
||||
this.archiveTruncated = true;
|
||||
this.promoteToHeadTail(text, chunkBytes);
|
||||
return;
|
||||
}
|
||||
|
||||
this.appendTail(text, chunkBytes);
|
||||
}
|
||||
|
||||
/** Replaces the capture basis (used only by the compatibility full-message tool path). */
|
||||
replace(text: string): void {
|
||||
if (this.settled) return;
|
||||
this.exactChunks = [];
|
||||
this.exactBytes = 0;
|
||||
this.head = Buffer.alloc(0);
|
||||
this.tail = Buffer.alloc(0);
|
||||
this.tailStart = 0;
|
||||
this.tailLength = 0;
|
||||
this.archiveTruncated = false;
|
||||
this.pendingHighSurrogate = "";
|
||||
this.append(text);
|
||||
}
|
||||
|
||||
/** Writes this single-use capture to the Session archive directory. */
|
||||
async save(toolName: string, toolCallId: string): Promise<TruncatedToolOutputArchiveSaveResult> {
|
||||
if (this.settled) return { status: "failed", code: "ALREADY_SAVED" };
|
||||
if (this.pendingHighSurrogate) {
|
||||
const pending = this.pendingHighSurrogate;
|
||||
this.pendingHighSurrogate = "";
|
||||
// A truly lone high surrogate has no direct UTF-8 representation; Node's UTF-8 encoder
|
||||
// serializes it as U+FFFD, which is the same behavior writeFile(text, "utf8") would use.
|
||||
this.appendStable(pending);
|
||||
}
|
||||
this.settled = true;
|
||||
const data = this.serialized();
|
||||
return this.owner.commit(toolName, toolCallId, data, this.archiveTruncated);
|
||||
}
|
||||
|
||||
/** Discards an unfinished in-memory capture without writing a file. */
|
||||
cancel(): void {
|
||||
if (this.settled) return;
|
||||
this.settled = true;
|
||||
}
|
||||
|
||||
private promoteToHeadTail(text: string, chunkBytes: number): void {
|
||||
const contentBudget = Math.max(0, this.fileLimitBytes - ARCHIVE_GAP_MARKER_BYTES);
|
||||
const headBudget = Math.floor(contentBudget / 2);
|
||||
const tailBudget = contentBudget - headBudget;
|
||||
const exact = Buffer.concat(this.exactChunks);
|
||||
|
||||
this.head =
|
||||
this.exactBytes >= headBudget
|
||||
? utf8BufferPrefix(exact, headBudget)
|
||||
: Buffer.concat([exact, utf8Prefix(text, headBudget - this.exactBytes)]);
|
||||
const initialTail =
|
||||
chunkBytes >= tailBudget
|
||||
? utf8Suffix(text, tailBudget)
|
||||
: Buffer.concat([
|
||||
utf8BufferSuffix(exact, tailBudget - chunkBytes),
|
||||
Buffer.from(text, "utf8"),
|
||||
]);
|
||||
this.resetTail(initialTail, tailBudget);
|
||||
this.exactChunks = [];
|
||||
this.exactBytes = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends one stable delta to the rolling tail without rebuilding the retained window.
|
||||
* Encoding remains below the tail budget: a delta at least that large takes the bounded
|
||||
* string-suffix path instead of allocating a Buffer for the whole delta.
|
||||
*/
|
||||
private appendTail(text: string, chunkBytes: number): void {
|
||||
const tailBudget = this.tailBudget();
|
||||
if (tailBudget <= 0) {
|
||||
this.resetTail(Buffer.alloc(0), 0);
|
||||
return;
|
||||
}
|
||||
if (chunkBytes >= tailBudget) {
|
||||
this.resetTail(utf8Suffix(text, tailBudget), tailBudget);
|
||||
return;
|
||||
}
|
||||
|
||||
const chunk = Buffer.from(text, "utf8");
|
||||
const overflow = Math.max(0, this.tailLength + chunk.length - tailBudget);
|
||||
this.tailStart = (this.tailStart + overflow) % tailBudget;
|
||||
const nextLength = Math.min(tailBudget, this.tailLength + chunk.length);
|
||||
const writeStart = (this.tailStart + nextLength - chunk.length) % tailBudget;
|
||||
const firstLength = Math.min(chunk.length, tailBudget - writeStart);
|
||||
chunk.copy(this.tail, writeStart, 0, firstLength);
|
||||
if (firstLength < chunk.length) {
|
||||
chunk.copy(this.tail, 0, firstLength);
|
||||
}
|
||||
this.tailLength = nextLength;
|
||||
}
|
||||
|
||||
/** Reinitializes the rolling tail from one already-bounded, code-point-aligned suffix. */
|
||||
private resetTail(buffer: Buffer, tailBudget: number): void {
|
||||
if (tailBudget <= 0) {
|
||||
this.tail = Buffer.alloc(0);
|
||||
this.tailStart = 0;
|
||||
this.tailLength = 0;
|
||||
return;
|
||||
}
|
||||
this.tail = Buffer.allocUnsafe(tailBudget);
|
||||
buffer.copy(this.tail);
|
||||
this.tailStart = 0;
|
||||
this.tailLength = buffer.length;
|
||||
}
|
||||
|
||||
private tailBudget(): number {
|
||||
const contentBudget = Math.max(0, this.fileLimitBytes - ARCHIVE_GAP_MARKER_BYTES);
|
||||
return contentBudget - Math.floor(contentBudget / 2);
|
||||
}
|
||||
|
||||
/** Copies the logical ring suffix once, dropping a leading partial UTF-8 code point. */
|
||||
private serializedTail(): Buffer {
|
||||
if (this.tailLength === 0) return Buffer.alloc(0);
|
||||
const capacity = this.tail.length;
|
||||
let skip = 0;
|
||||
while (
|
||||
skip < this.tailLength &&
|
||||
(this.tail[(this.tailStart + skip) % capacity]! & 0xc0) === 0x80
|
||||
) {
|
||||
skip += 1;
|
||||
}
|
||||
const length = this.tailLength - skip;
|
||||
const result = Buffer.allocUnsafe(length);
|
||||
const readStart = (this.tailStart + skip) % capacity;
|
||||
const firstLength = Math.min(length, capacity - readStart);
|
||||
this.tail.copy(result, 0, readStart, readStart + firstLength);
|
||||
if (firstLength < length) {
|
||||
this.tail.copy(result, firstLength, 0, length - firstLength);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private serialized(): Buffer {
|
||||
if (!this.archiveTruncated) return Buffer.concat(this.exactChunks);
|
||||
return Buffer.concat([
|
||||
this.head,
|
||||
Buffer.from(ARCHIVE_GAP_MARKER, "utf8"),
|
||||
this.serializedTail(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
export class TruncatedToolOutputArchive {
|
||||
private readonly rootDir: string;
|
||||
private readonly fileLimitBytes: number;
|
||||
|
||||
constructor(opts: TruncatedToolOutputArchiveOptions) {
|
||||
// The explicit gap marker is part of every bounded head/tail archive, so even internal
|
||||
// test overrides must leave enough room for it; production stays just below 8 MiB.
|
||||
this.fileLimitBytes = Math.max(
|
||||
ARCHIVE_GAP_MARKER_BYTES,
|
||||
opts.fileLimitBytes ?? TRUNCATED_TOOL_OUTPUT_FILE_LIMIT_BYTES,
|
||||
);
|
||||
this.rootDir = opts.rootDir;
|
||||
}
|
||||
|
||||
/** Starts one independently bounded capture; the directory remains lazy until save(). */
|
||||
startCapture(): TruncatedToolOutputCapture {
|
||||
return new TruncatedToolOutputCapture(this, this.fileLimitBytes);
|
||||
}
|
||||
|
||||
/** Internal commit path used by TruncatedToolOutputCapture. */
|
||||
async commit(
|
||||
toolName: string,
|
||||
toolCallId: string,
|
||||
data: Buffer,
|
||||
archiveTruncated: boolean,
|
||||
): Promise<TruncatedToolOutputArchiveSaveResult> {
|
||||
const safeToolName = toolName.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 48) || "tool";
|
||||
const idHash = createHash("sha256").update(toolCallId).digest("hex").slice(0, 16);
|
||||
const filePath = path.join(this.rootDir, `${safeToolName}-${idHash}.log`);
|
||||
try {
|
||||
// Create shared Session ancestors with their existing/default policy, then apply the
|
||||
// archive's private directory mode only to the archive directory itself.
|
||||
await mkdir(path.dirname(this.rootDir), { recursive: true });
|
||||
await mkdir(this.rootDir, { recursive: true, mode: 0o700 });
|
||||
await writeFile(filePath, data, { flag: "wx", mode: 0o600 });
|
||||
return {
|
||||
status: "saved",
|
||||
path: filePath,
|
||||
archiveTruncated,
|
||||
};
|
||||
} catch (err) {
|
||||
const rawCode = (err as { code?: unknown }).code;
|
||||
const code = typeof rawCode === "string" ? rawCode : "UNKNOWN";
|
||||
process.stderr.write(
|
||||
`[penguin] tool "${toolName}" truncated output archive write failed (${code}).\n`,
|
||||
);
|
||||
return { status: "failed", code };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@
|
||||
* budget numbers. The embedded `objective` value is user data, which is why the closing tag
|
||||
* is matched line-anchored (see markers/goal-block.ts).
|
||||
*/
|
||||
import { modelVisiblePath } from "../internal/model-visible-path.js";
|
||||
import { markerBlock, MARKER_TAGS } from "../omnimessage/markers/index.js";
|
||||
import { serializeGoalFile, UNLIMITED_BUDGET } from "./goal-file.js";
|
||||
|
||||
@@ -43,7 +44,7 @@ export interface GoalPromptArgs {
|
||||
/** The goal-file paragraph shared by both blocks: path, the status protocol, and the file's content. */
|
||||
function goalFileLines(args: GoalPromptArgs): string[] {
|
||||
return [
|
||||
`Goal file: ${args.goalFilePath}`,
|
||||
`Goal file: ${modelVisiblePath(args.goalFilePath)}`,
|
||||
"You may modify ONLY the `status` field of this file, and only to `complete` or",
|
||||
"`blocked`; the system reads it after every round. Its content:",
|
||||
"",
|
||||
|
||||
@@ -52,6 +52,9 @@ export type { SessionTitleResult } from "./internal/session-title.js";
|
||||
// re-exported, because the server appends `[attached file: …]` lines for the composer's
|
||||
// uploads and both producers must place them identically (see the markers module).
|
||||
export { appendAttachmentLines } from "./internal/session-support.js";
|
||||
// Model-visible path spelling (forward slashes on Windows); the server uses it for its
|
||||
// [attached file: ...] lines so every path the model reads has one spelling per platform.
|
||||
export { modelVisiblePath } from "./internal/model-visible-path.js";
|
||||
export { Agent, createAgent } from "./agent.js";
|
||||
export type { CreateAgentOptions, CreateSessionOptions, ResumeSessionOptions } from "./agent.js";
|
||||
|
||||
|
||||
@@ -275,6 +275,14 @@ export interface EnvironmentServices {
|
||||
export interface EnvironmentConfig {
|
||||
workspaceDir: string;
|
||||
toolConfig: ToolConfig;
|
||||
/**
|
||||
* This Session's private scratchpad directory (`scratchpad/<sessionId>`), the generic
|
||||
* Session-scoped storage root for Environment by-products. Currently it backs
|
||||
* truncated-tool-output recovery: output beyond an entry's `maxOutputLength` is saved under
|
||||
* `<sessionScratchpadDir>/truncated-tool-output/`. Agent Sessions always pass it; standalone
|
||||
* embedders without a stable Session directory omit it and keep truncation-only behavior.
|
||||
*/
|
||||
sessionScratchpadDir?: string;
|
||||
/** Runtime services (optional); Environment forwards these to each tool factory to use as needed. */
|
||||
services?: EnvironmentServices;
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
/**
|
||||
* Model-visible spelling of an absolute path.
|
||||
*
|
||||
* Every path core composes for the model to read — the system prompt's App Data Dir and CWD
|
||||
* lines, `[attached image/file: …]` lines, the goal-file line, truncated-output recovery notes —
|
||||
* goes through this helper, because the model re-emits those spellings into JSON tool arguments
|
||||
* and shell commands. On Windows that spelling uses forward slashes: Node's fs APIs accept them,
|
||||
* `exec_command` runs through (Git) Bash, and the form has no JSON backslash-escaping ambiguity.
|
||||
* Harness-composed paths are ordinary absolute paths (never `\\?\`-prefixed), so the swap is
|
||||
* lossless. POSIX paths pass through untouched — a backslash is a valid filename character there.
|
||||
*/
|
||||
export function modelVisiblePath(filePath: string): string {
|
||||
return process.platform === "win32" ? filePath.replaceAll("\\", "/") : filePath;
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { formatLocalDate } from "./dates.js";
|
||||
import { sessionShell } from "../environment/tools/command/shell.js";
|
||||
import type { SessionEnvironmentValues } from "../state/agent-state.js";
|
||||
import { modelVisiblePath } from "./model-visible-path.js";
|
||||
import { workspacesDir } from "../state/index.js";
|
||||
import { attachedImageLine, isWholeOriginBlock, userText } from "../omnimessage/index.js";
|
||||
import type { OmniMessage } from "../omnimessage/index.js";
|
||||
@@ -41,9 +42,11 @@ export function sessionEnvironment(
|
||||
): SessionEnvironment {
|
||||
return {
|
||||
sessionId,
|
||||
cwd: workspaceDir,
|
||||
// Model-visible spelling (forward slashes on Windows): the model composes tool arguments
|
||||
// and shell commands from these two lines, so they must be safe in both contexts.
|
||||
cwd: modelVisiblePath(workspaceDir),
|
||||
agentId: ids.agentId,
|
||||
projectDir: ids.projectDir,
|
||||
projectDir: modelVisiblePath(ids.projectDir),
|
||||
provider: ids.provider,
|
||||
modelId: ids.modelId,
|
||||
platform: process.platform,
|
||||
@@ -167,7 +170,7 @@ export async function imagesToScratchpadPaths(
|
||||
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
|
||||
}
|
||||
}
|
||||
lines.push(attachedImageLine(file));
|
||||
lines.push(attachedImageLine(modelVisiblePath(file)));
|
||||
}
|
||||
|
||||
return appendAttachmentLines(
|
||||
|
||||
@@ -67,6 +67,21 @@ export function workspacesDir(root: string, projectId: string, agentId: string):
|
||||
return path.join(agentDir(root, projectId, agentId), "workspaces");
|
||||
}
|
||||
|
||||
/**
|
||||
* `<agentDir>/scratchpad/<sessionId>`, one Session's private scratchpad directory. The single
|
||||
* Session-scoped storage root shared by every by-product bound to that Session: input images
|
||||
* saved as path lines, the goal-mode control file, and Environment's truncated-tool-output
|
||||
* recovery files. Deleted together with the Session by the existing scratchpad cleanup path.
|
||||
*/
|
||||
export function sessionScratchpadDir(
|
||||
root: string,
|
||||
projectId: string,
|
||||
agentId: string,
|
||||
sessionId: string,
|
||||
): string {
|
||||
return path.join(scratchpadDir(root, projectId, agentId), sessionId);
|
||||
}
|
||||
|
||||
/**
|
||||
* `<agentDir>/scratchpad/<sessionId>/GOAL.yaml`, the goal-mode control file of one Session
|
||||
* (sibling of the model's PLAN.md convention; see goal/goal-file.ts for field ownership).
|
||||
@@ -77,7 +92,7 @@ export function goalFilePath(
|
||||
agentId: string,
|
||||
sessionId: string,
|
||||
): string {
|
||||
return path.join(scratchpadDir(root, projectId, agentId), sessionId, "GOAL.yaml");
|
||||
return path.join(sessionScratchpadDir(root, projectId, agentId, sessionId), "GOAL.yaml");
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user