feat(core,server,web): images reach every input — steering, goal objectives, one fold (#71)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/core/src/agent.ts

@@ -291,15 +291,12 @@ export class Agent {
       createLLM: rt.createLLM,
       createBareLLM: rt.createBareLLM,
       compaction: rt.compaction,
-      // Model doesn't support images: input images are written to the session scratchpad and their paths appended to the text (viewed via describe_image).
-      ...(modelEntry.vision === false
-        ? {
-            inputImagesDir: path.join(
-              scratchpadDir(this.state.root, this.state.projectId, this.state.agentId),
-              sessionId,
-            ),
-          }
-        : {}),
+      // Where an input image lands when it becomes a path line (see SessionConfig.imagesDir).
+      imagesDir: path.join(
+        scratchpadDir(this.state.root, this.state.projectId, this.state.agentId),
+        sessionId,
+      ),
+      modelHasVision: modelEntry.vision !== false,
       // Goal mode's control file lives in the session scratchpad; the path is fixed per
       // Session, so it is wired here rather than passed per-run.
       goalFilePath: goalFilePath(
@@ -451,15 +448,12 @@ export class Agent {
       createLLM: rt.createLLM,
       createBareLLM: rt.createBareLLM,
       compaction: rt.compaction,
-      // Model doesn't support images: input images are written to the session scratchpad and their paths appended to the text (viewed via describe_image).
-      ...(modelEntry.vision === false
-        ? {
-            inputImagesDir: path.join(
-              scratchpadDir(this.state.root, this.state.projectId, this.state.agentId),
-              sessionId,
-            ),
-          }
-        : {}),
+      // Where an input image lands when it becomes a path line (see SessionConfig.imagesDir).
+      imagesDir: path.join(
+        scratchpadDir(this.state.root, this.state.projectId, this.state.agentId),
+        sessionId,
+      ),
+      modelHasVision: modelEntry.vision !== false,
       // Goal mode's control file lives in the session scratchpad; the path is fixed per
       // Session, so it is wired here rather than passed per-run.
       goalFilePath: goalFilePath(

packages/core/src/engine/context-engine.ts

@@ -200,8 +200,27 @@ export interface ContextEngineDeps {
   compaction?: CompactionSettings;
   /** This Session's session_meta message; written at the start of the new Trace file after compaction splits it. */
   sessionMeta?: OmniMessage;
+  /**
+   * Input adapter for a session whose model has no vision: folds image messages into text
+   * lines appended to the input's user text. Absent = the model takes images directly. `run`'s
+   * Prompt is folded by the caller before it reaches the engine; this hook exists for the one
+   * input the engine assembles itself — steering (see `steeringMessages`).
+   *
+   * Expected to settle rather than reject: it runs mid-Task, and Session's binding already
+   * degrades a failure into text saying the images were dropped.
+   */
+  foldInputImages?: (messages: OmniMessage[]) => Promise<OmniMessage[]>;
 }
 
+const isImageMessage = (m: OmniMessage): boolean =>
+  (m.payload as { type?: string }).type === "image_url";
+
+/** Whether a message carries steering of its own — an image, or text that isn't blank. */
+const carriesSteering = (m: OmniMessage): boolean => {
+  const p = m.payload as { type?: string; text?: string };
+  return p.type === "image_url" || (p.type === "text" && (p.text ?? "").trim().length > 0);
+};
+
 /** Whether compaction is possible; when not `ok`, `compact()` is a no-op and yields no messages (see ContextEngine.compactability). */
 export type CompactAvailability = "ok" | "unsupported" | "empty" | "just_compacted";
 
@@ -378,7 +397,7 @@ export class ContextEngine {
    * later Task would be more surprising than losing it; hosts get `steer() === false`
    * after that point and fall back to a normal task).
    */
-  private steeringQueue: string[] = [];
+  private steeringQueue: OmniMessage[][] = [];
   /** Whether a `run` is currently in flight (gates `steer`; compaction does not count). */
   private taskRunning = false;
 
@@ -424,28 +443,39 @@ export class ContextEngine {
    * Queues a steering message for the running Task: it is delivered with the next request
    * input as a standalone `[user_steering]` user message — alongside that turn's tool
    * outputs, or alone as the continuation input when the turn produced no tool calls.
-   * Returns false when no Task is running (the host should then submit the text as a
-   * normal task instead).
+   * `input` is an OmniMessage list, the shape `run` takes a Prompt in: its user text becomes
+   * the block's body and its images ride behind that text, exactly as a Prompt carries them;
+   * on a model without vision they are folded into path lines at delivery (see
+   * deliverSteering). Returns false when no Task is running (the host should then submit the
+   * message as a normal task instead).
+   *
+   * An input with neither text nor images queues nothing and still returns true: `false` is
+   * specifically "send this as a normal task", which would be the wrong advice for an empty
+   * one. Every host guards against this already; the check is here so an empty
+   * `[user_steering]` block can't reach the model through a host that forgets.
    */
-  steer(text: string): boolean {
+  steer(input: OmniMessage[]): boolean {
     if (!this.taskRunning) return false;
-    this.steeringQueue.push(text);
+    if (!input.some(carriesSteering)) return true;
+    this.steeringQueue.push(input);
     return true;
   }
 
   /**
    * Drains the steering queue into standalone `[user_steering]` user messages (one per
-   * queued text, in arrival order), yielding each to the output stream and writing it to
-   * Trace — steering is real user input: unlike a normal Prompt (which the render layer
-   * already holds locally) this text never reached the consumer, and replay attributes it
-   * positionally to the next turn's input like any other user message. Returns the messages
-   * for the caller to append to the next request input; an empty queue is a no-op.
+   * queued entry, in arrival order, each followed by its images), yielding every message to
+   * the output stream and writing it to Trace — steering is real user input: unlike a normal
+   * Prompt (which the render layer already holds locally) this text never reached the
+   * consumer, and replay attributes it positionally to the next turn's input like any other
+   * user message. Returns the messages for the caller to append to the next request input;
+   * an empty queue is a no-op.
    */
   private async *deliverSteering(): AsyncGenerator<OmniMessage, OmniMessage[]> {
     if (this.steeringQueue.length === 0) return [];
     const drained = this.steeringQueue;
     this.steeringQueue = [];
-    const messages = drained.map((text) => userText(userSteeringText(text)));
+    const messages: OmniMessage[] = [];
+    for (const input of drained) messages.push(...(await this.steeringMessages(input)));
     for (const msg of messages) {
       yield msg;
       await this.write(msg);
@@ -453,6 +483,39 @@ export class ContextEngine {
     return messages;
   }
 
+  /**
+   * One queued steering input -> the messages carrying it: its user text collected into the
+   * `[user_steering]`-wrapped message, followed by everything else it held — the images, on a
+   * vision model. That is the shape a Prompt uses, so every consumer down the line — LLM
+   * client, Trace, replay — already knows it.
+   *
+   * When `deps.foldInputImages` is given, the input goes through it **before** the wrapping so
+   * the images land inside the block: `parseUserSteeringText` only recognizes a text that is
+   * exactly one block, and anything appended after the closing tag would cost the message its
+   * steering identity — every render layer would read it as a new Task.
+   */
+  private async steeringMessages(input: OmniMessage[]): Promise<OmniMessage[]> {
+    // No images, no fold: an image-free steering message is the same message either way.
+    const fold = input.some(isImageMessage) ? this.deps.foldInputImages : undefined;
+    const messages = fold ? await fold(input) : input;
+    const texts: string[] = [];
+    const rest: OmniMessage[] = [];
+    for (const msg of messages) {
+      const p = msg.payload as { type?: string; role?: string; text?: string };
+      if (p.type === "text" && p.role === "user") texts.push(p.text ?? "");
+      else rest.push(msg);
+    }
+    // `foldInputImages` is public API, so a third-party adapter can return something else, and
+    // both ways it can break lose the picture: an image that survived the fold goes to the one
+    // model known to refuse it, and no text at all means the images were dropped rather than
+    // written down as paths. Name the contract instead of delivering a steering message that
+    // lost what it was sent to carry.
+    if (fold && (rest.some(isImageMessage) || texts.length === 0)) {
+      throw new Error("foldInputImages must return the input's images folded into a user text.");
+    }
+    return [userText(userSteeringText(texts.join("\n\n"))), ...rest];
+  }
+
   /** The actual Task loop behind `run` (split out so run's finally can close the steering window on every exit path). */
   private async *runToCompletion(
     newMessages: OmniMessage[],

packages/core/src/internal/session-support.ts

@@ -103,6 +103,14 @@ export async function createTempWorkspace(
   );
 }
 
+/**
+ * Stands in for a single image that could not be turned into a path line — a data URL that
+ * doesn't parse. Shown to the model and the user instead of silently dropping the attachment.
+ * A scratchpad that can't be written to is a different matter: the fold throws and the run
+ * ends, rather than carrying on without what the sender attached.
+ */
+const IMAGE_DROPPED_NOTE = "[an attached image could not be saved and was dropped]";
+
 /** Maps a data URL's mime type to a file extension on disk; unknown mimes use bin (the image-reading tool sniffs the magic bytes and doesn't rely on the extension). */
 const MIME_TO_EXT: Record<string, string> = {
   "image/png": "png",
@@ -111,6 +119,9 @@ const MIME_TO_EXT: Record<string, string> = {
   "image/webp": "webp",
 };
 
+/** An input image message — what both conversions below pull out of the input. */
+const isImage = (m: OmniMessage): boolean => (m.payload as { type?: string }).type === "image_url";
+
 /**
  * Input conversion for when the session model doesn't support images: image messages
  * in the `run` input are written to disk as files (base64 data URLs are saved to the
@@ -126,8 +137,6 @@ export async function imagesToScratchpadPaths(
   input: OmniMessage[],
   dir: string,
 ): Promise<OmniMessage[]> {
-  const isImage = (m: OmniMessage): boolean =>
-    (m.payload as { type?: string }).type === "image_url";
   if (!input.some(isImage)) return input;
 
   const lines: string[] = [];
@@ -140,7 +149,7 @@ export async function imagesToScratchpadPaths(
     }
     const match = /^data:([^;,]+);base64,(.+)$/s.exec(url);
     if (!match) {
-      lines.push("[an attached image could not be saved and was dropped]");
+      lines.push(IMAGE_DROPPED_NOTE);
       continue;
     }
     await fs.mkdir(dir, { recursive: true });

packages/core/src/omnimessage/markers/steering.ts

@@ -5,7 +5,11 @@
  * queues it and delivers it as a **standalone user text message** wrapped in a
  * `[user_steering]…[/user_steering]` block, sent with the next request input alongside that
  * turn's tool outputs (or as the continuation input when the turn produced no tool calls) —
- * the model sees it without the agent loop being interrupted. The message is real user input:
+ * the model sees it without the agent loop being interrupted. Images sent with the message
+ * follow it as ordinary user image messages (a model without vision gets `[attached image: …]`
+ * lines inside the block instead), so consumers group them the way they already group a
+ * Prompt's images: everything up to the next non-image message belongs to the steering
+ * message, and none of it starts a new Task. The message is real user input:
  * written to Trace like any Prompt and yielded to the output stream. The marker exists so the
  * model and the render layers can tell it apart from a task-starting Prompt: UIs keep it inside
  * the running Task (no new Task segment) and render it as user speech instead of raw markers.

packages/core/src/session.ts

@@ -59,13 +59,18 @@ export interface SessionConfig {
   /** Session resume: the full historical messages of the current context (for rendering, including interrupted turns and their markers), for frontend display. */
   resumedHistory?: OmniMessage[];
   /**
-   * Set when the session's model doesn't support images (the composition layer decides this via
-   * ModelEntry.vision): images in `run` input are saved to this directory (the session's
-   * scratchpad), and the path is appended to the user text instead — the model views the image
-   * via describe_image, and images never enter the session history directly (some providers
-   * return a 400 outright on image input).
+   * This Session's scratchpad directory: where an input image is saved when it becomes an
+   * `[attached image: <path>]` line instead of riding the request as an image. The model
+   * reads it back with describe_image / read_image, and the Web turns the path into a
+   * thumbnail again. Always set — each input path decides on its own whether to use it.
    */
-  inputImagesDir?: string;
+  imagesDir: string;
+  /**
+   * Whether the session's model accepts image input (from ModelEntry.vision). Prompts and
+   * steering messages fold their images only when this is false; goal objectives fold either
+   * way, since they are re-injected as text every round (see `runGoal`).
+   */
+  modelHasVision: boolean;
   /**
    * Absolute path of this Session's GOAL.yaml (the composition layer derives it from the
    * agent scratchpad — see `goalFilePath` in state/paths.ts). Goal mode
@@ -126,9 +131,18 @@ export class Session {
   private readonly trace?: TraceSink;
   private readonly meta: OmniMessage;
   private readonly createBareLLM?: () => LLMInterface;
-  private readonly inputImagesDir?: string;
+  private readonly imagesDir: string;
+  private readonly modelHasVision: boolean;
   private readonly goalFile?: string;
   private metaWritten = false;
+  /**
+   * The image fold, bound to this Session's scratchpad — Session is the layer that knows both
+   * the directory and the model's capability, so it binds the conversion once and each input
+   * path calls it under its own rule (see `modelHasVision`). The body reads `imagesDir` at call
+   * time, so field ordering in the constructor doesn't matter.
+   */
+  private readonly foldImages = (messages: OmniMessage[]): Promise<OmniMessage[]> =>
+    imagesToScratchpadPaths(messages, this.imagesDir);
   /** Title material (used by `generateTitle` as the default): the user input and model body text of the first Task that contains user text. */
   private titleUserText = "";
   private titleAssistantText = "";
@@ -146,7 +160,8 @@ export class Session {
     this.metaWritten = config.metaAlreadyWritten ?? false;
     if (config.resumedHistory) this.resumedHistory = config.resumedHistory;
     if (config.createBareLLM) this.createBareLLM = config.createBareLLM;
-    if (config.inputImagesDir) this.inputImagesDir = config.inputImagesDir;
+    this.imagesDir = config.imagesDir;
+    this.modelHasVision = config.modelHasVision;
     if (config.goalFilePath) this.goalFile = config.goalFilePath;
     this.engine = new ContextEngine({
       llm: config.llm,
@@ -157,6 +172,13 @@ export class Session {
       ...(config.createLLM ? { createLLM: config.createLLM } : {}),
       ...(config.compaction ? { compaction: config.compaction } : {}),
       ...(config.initialEngineState ? { initialState: config.initialEngineState } : {}),
+      // The engine assembles one input of its own — a steering message with images — and folds
+      // it through the same converter `runTask` uses, failures included: a scratchpad that
+      // can't be written to ends the run rather than dropping the attachment and carrying on.
+      // The picture usually arrives BECAUSE the run is going the wrong way, so continuing
+      // without it spends the rest of the Task heading further that way. A vision model simply
+      // isn't given the function, which is all the engine needs to know about the subject.
+      ...(config.modelHasVision ? {} : { foldInputImages: this.foldImages }),
       sessionMeta: this.meta,
     });
   }
@@ -173,9 +195,10 @@ export class Session {
    * Docs: /docs/agent-loop § "The loop at a glance".
    *
    * With `opts.goal` present, the same call runs **goal mode**: the input's text becomes the
-   * objective, and the Session loops Tasks — each round's input is the `[goal]` protocol
-   * block followed by the text (round 1 verbatim, later rounds the objective) — until the
-   * goal file says stop, the budget runs out, or a round is cut off. Round inputs are
+   * objective (attached images fold into `[attached image: …]` lines within it, whatever the
+   * model's vision — see `runGoal`), and the Session loops Tasks — each round's input is the
+   * `[goal]` protocol block followed by the text (round 1 verbatim, later rounds the objective)
+   * — until the goal file says stop, the budget runs out, or a round is cut off. Round inputs are
    * yielded onto the stream before each round (a plain run never yields its own input), and
    * the final message is exactly one `goal_finished` event carrying the outcome.
    * Docs: /docs/goal-mode.
@@ -195,11 +218,8 @@ export class Session {
     newMessages: OmniMessage[],
     opts?: RunOptions,
   ): AsyncGenerator<OmniMessage> {
-    // Model doesn't support images: input images are saved to disk first (session scratchpad),
-    // then the path is appended to the text before it reaches the engine/Trace.
-    if (this.inputImagesDir) {
-      newMessages = await imagesToScratchpadPaths(newMessages, this.inputImagesDir);
-    }
+    // Folded before Trace and title material, so the path lines are what gets recorded.
+    if (!this.modelHasVision) newMessages = await this.foldImages(newMessages);
     await this.ensureMetaWritten();
     // Self-captures title material (the title is derived from the first-turn
     // conversation text): while material isn't frozen yet, collect this call's user text and
@@ -231,12 +251,20 @@ export class Session {
   }
 
   /**
-   * The goal-mode branch of `run`: validates the input (text-only — the objective is
-   * re-injected every round, and images have no place in the protocol block), then drives
-   * the goal loop, running each round through the single-Task path with the same per-call
-   * options (approval, signal, thinking level). The loop's terminal `goal_finished` event is
-   * additionally written to the Trace (best-effort, like session_meta) so the goal's end
-   * survives with its conversation.
+   * The goal-mode branch of `run`: validates the input, folds any attached images into the
+   * objective, then drives the goal loop, running each round through the single-Task path with
+   * the same per-call options (approval, signal, thinking level). The loop's terminal
+   * `goal_finished` event is additionally written to the Trace (best-effort, like session_meta)
+   * so the goal's end survives with its conversation.
+   *
+   * Images fold here whether or not the model has vision, because the objective is re-injected
+   * as the text of every round's `[goal]` block — an image message has nowhere to sit in that.
+   * Sending it in round 1 alone would leave later rounds pointing at something compaction has
+   * since dropped, while the objective still reads correct; a path line survives every round
+   * and every compaction, and the model pays for the picture only when it looks. The lines go
+   * at the end of the text, where goal-loop's `stripLeadingMarkerBlocks` leaves them alone.
+   *
+   * Text is required and checked before the fold: a picture alone doesn't state a goal.
    */
   private async *runGoal(
     newMessages: OmniMessage[],
@@ -246,16 +274,27 @@ export class Session {
     if (!this.goalFile) {
       throw new Error("Goal mode is unavailable: this Session has no goal file path configured.");
     }
-    const texts: string[] = [];
-    for (const m of newMessages) {
+    const isUserText = (m: OmniMessage): boolean => {
       const p = m.payload as { type?: string; role?: string; text?: string };
+      return m.type === "model_msg" && p.type === "text" && p.role === "user" && !!p.text?.trim();
+    };
+    // The objective itself. Checked before the fold, which only ever appends to a user text —
+    // so once this passes, the joined text below cannot come out empty.
+    if (!newMessages.some(isUserText)) {
+      throw new Error("Goal mode requires a non-empty text objective.");
+    }
+    const folded = await this.foldImages(newMessages);
+    const texts: string[] = [];
+    for (const m of folded) {
+      const p = m.payload as { type?: string; role?: string; text?: string };
+      // The fold has already turned the images into text, so anything left that isn't user
+      // text was never one of the two kinds goal mode accepts.
       if (m.type !== "model_msg" || p.type !== "text" || p.role !== "user" || !p.text) {
-        throw new Error("Goal mode requires text-only user input (the objective).");
+        throw new Error("Goal mode accepts user text and images only (the objective).");
       }
       texts.push(p.text);
     }
     const text = texts.join("\n").trim();
-    if (!text) throw new Error("Goal mode requires a non-empty objective.");
     const loop = runGoalLoop(
       { run: (msgs) => this.runTask(msgs, opts) },
       {
@@ -283,13 +322,16 @@ export class Session {
    * Queues a steering message for the running Task: the engine delivers it between turns as
    * a standalone `[user_steering]` user message — sent with the next request input alongside
    * that turn's tool outputs, or alone as the continuation input when the turn produced no
-   * tool calls — so the model sees it without the loop being interrupted. Returns false when
-   * no Task is running — the host should then submit the text as a normal task instead.
-   * Delivery is independent of approval mode; anything still queued when the run exits
-   * (abort included) is discarded.
+   * tool calls — so the model sees it without the loop being interrupted. `input` is an
+   * OmniMessage list, the same shape `run` takes a Prompt in: its user text becomes the
+   * block's body, and its images ride behind that message just as a Prompt's do; on a model
+   * without vision they become `[attached image: …]` path lines inside the block instead (see
+   * the engine's steeringMessages). Returns false when no Task is running — the host should
+   * then submit the message as a normal task instead. Delivery is independent of approval
+   * mode; anything still queued when the run exits (abort included) is discarded.
    */
-  steer(text: string): boolean {
-    return this.engine.steer(text);
+  steer(input: OmniMessage[]): boolean {
+    return this.engine.steer(input);
   }
 
   /**