feat(core,server,web): images reach every input — steering, goal objectives, one fold (#71)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/server/src/api/types.ts

@@ -674,8 +674,16 @@ export interface TaskCreateResponse {
  * task POST).
  */
 export interface SteerRequest {
-  /** Non-empty message text (trimmed server-side). */
+  /** Message text (trimmed server-side); may be empty when `images` carries the message. */
   text: string;
+  /**
+   * Images sent with the steering message (`data:` or http(s) URLs, same rule as
+   * `TaskInputPart.image_url`): delivered as user image messages right behind the
+   * `[user_steering]` text. A model without vision receives them as scratchpad path lines
+   * instead, exactly as it would a Prompt's images. At least one of `text` / `images` must
+   * be non-empty.
+   */
+  images?: string[];
 }
 
 export interface ApprovalDecisionRequest {

packages/server/src/http/routes/sessions.ts

@@ -79,6 +79,36 @@ const SESSION_CATEGORIES: readonly SessionCategory[] = [
   "archived",
 ];
 
+/**
+ * A base64 `data:` URL of an image, in the exact shape core parses it back out of
+ * (`imagesToScratchpadPaths`): one mime type, the `;base64,` marker, a non-empty base64 body.
+ * The mime is deliberately unconstrained — core maps the ones it knows to a file extension and
+ * falls back to `.bin`, and the image tools sniff the magic bytes rather than trusting either.
+ *
+ * Checking the body, not just the `data:` prefix, is what keeps the failure here instead of
+ * three layers down: core turns a data URL it cannot parse into an "[an attached image could
+ * not be saved and was dropped]" line, which for an HTTP caller means a 202 followed by a
+ * message quietly missing its picture. The file-attachment field has always validated its own
+ * payload this way (parseAttachmentPart); this is the same rule for images.
+ */
+const IMAGE_DATA_URL = /^data:[^;,]+;base64,[A-Za-z0-9+/=\s]+$/;
+
+/**
+ * The image-URL rule every image-carrying request field obeys: a `data:` URL the session
+ * keeps (inline, or written to the scratchpad without vision) or an http(s) URL it references.
+ * `field` names the offending value in the error, so each caller reads as if it validated
+ * inline.
+ */
+function requireImageUrl(url: unknown, field: string): string {
+  if (typeof url === "string") {
+    if (url.startsWith("http://") || url.startsWith("https://")) return url;
+    if (IMAGE_DATA_URL.test(url)) return url;
+  }
+  throw badRequest(
+    `${field} must be an http(s) URL or a base64 data: URL (data:<mime>;base64,<bytes>).`,
+  );
+}
+
 /**
  * Resolve a scratchpad file name to an absolute path inside `dir`, or null when it could point
  * anywhere else (the caller turns that into the same 404 a missing file gets, so a probe learns
@@ -133,14 +163,7 @@ function parseTaskInput(body: Record<string, unknown>): ParsedTaskInput {
       return;
     }
     if (part.type === "image_url") {
-      const url = part.imageUrl;
-      if (
-        typeof url !== "string" ||
-        !(url.startsWith("data:") || url.startsWith("http://") || url.startsWith("https://"))
-      ) {
-        throw badRequest(`input[${i}].imageUrl only supports data: or http(s) URLs.`);
-      }
-      messages.push(imageUrlMessage(url));
+      messages.push(imageUrlMessage(requireImageUrl(part.imageUrl, `input[${i}].imageUrl`)));
       return;
     }
     if (part.type === "file") {
@@ -157,6 +180,17 @@ function parseTaskInput(body: Record<string, unknown>): ParsedTaskInput {
   return { messages, attachments };
 }
 
+/**
+ * Validate the optional `images` field of a steer request: a list of `data:` / http(s) URLs
+ * (same rule as a task input's `imageUrl`), absent or empty = a text-only steering message.
+ */
+function parseSteerImages(body: Record<string, unknown>): string[] {
+  const images = body.images;
+  if (images === undefined) return [];
+  if (!Array.isArray(images)) throw badRequest("images must be an array.");
+  return images.map((url, i) => requireImageUrl(url, `images[${i}]`));
+}
+
 /**
  * Validate the optional `goal` field of a task request: absent = a regular task (null);
  * present = goal mode with a token budget (a positive integer, or -1/omitted = unlimited).
@@ -491,21 +525,23 @@ export function sessionsRoutes(deps: AppDeps): Hono<AppEnv> {
     // follow-up keeps its level for its auto-start.
     const thinkingLevel = optionalEnum(body, "thinkingLevel", THINKING_LEVELS);
     if (goal) {
-      // Goal mode: the input must be plain non-empty text (its marker-stripped text becomes
-      // the objective, re-injected every round — images and file attachments have no place in
-      // the protocol; rejected before any upload is written to disk).
+      // Goal mode: the input needs non-empty text, since its marker-stripped text becomes the
+      // objective that every round re-injects and an image on its own doesn't say what the
+      // goal is. Images can come along — core folds them into `[attached image: <path>]` lines
+      // inside the objective (whatever the model's vision) so they survive the rounds. File
+      // attachments cannot: nothing folds them into the objective, so they are turned away
+      // here, before any upload is written to disk.
       const { messages, attachments } = parseTaskInput(body);
       const text = messages
         .filter((m) => (m.payload as { type?: string }).type === "text")
         .map((m) => (m.payload as { text: string }).text)
         .join("\n")
         .trim();
-      if (
-        !text ||
-        attachments.length > 0 ||
-        messages.some((m) => (m.payload as { type?: string }).type !== "text")
-      ) {
-        throw badRequest("goal mode requires text-only input (the objective).");
+      if (!text) {
+        throw badRequest("goal mode requires a non-empty text objective.");
+      }
+      if (attachments.length > 0) {
+        throw badRequest("goal mode accepts text and images only (no file attachments).");
       }
       const { sessionId } = await deps.manager.startGoal(row.sessionId, {
         input: messages,
@@ -552,15 +588,27 @@ export function sessionsRoutes(deps: AppDeps): Hono<AppEnv> {
   });
 
   // Mid-run steering: queue a user message for the running Task; core delivers it between
-  // turns as a standalone `[user_steering]` user message (the model sees it without the loop
-  // being interrupted). 409 not_running when no Task is in progress — the frontend then falls
-  // back to a normal task POST.
+  // turns as a standalone `[user_steering]` user message, with any images following it as
+  // user image messages (the model sees the whole thing without the loop being interrupted).
+  // 409 not_running when no Task is in progress — the frontend then falls back to a normal
+  // task POST.
   app.post("/:sessionId/steer", async (c) => {
     const row = resolveSession(c);
     const body = await readJson(c);
     const text = typeof body.text === "string" ? body.text.trim() : "";
-    if (!text) throw badRequest("text must be a non-empty string.");
-    deps.manager.steer(row.sessionId, text);
+    const images = parseSteerImages(body);
+    // Either half can carry the message on its own: an image with no caption is a complete
+    // steering message, and so is plain text.
+    if (!text && images.length === 0) {
+      throw badRequest("text or images must carry the steering message.");
+    }
+    // The wire shape becomes core's: a user text message (omitted when the images are the
+    // whole message, so the fold's path lines aren't preceded by a blank one) plus one image
+    // message each — the same input a normal task would carry.
+    deps.manager.steer(row.sessionId, [
+      ...(text ? [userText(text)] : []),
+      ...images.map((url) => imageUrlMessage(url)),
+    ]);
     return c.body(null, 202);
   });
 

packages/server/src/runtime/session-manager.ts

@@ -88,8 +88,8 @@ export interface RuntimeSession {
   compact(opts: { signal: AbortSignal }): AsyncGenerator<OmniMessage>;
   /** Whether compaction is possible and why; when not ok, compact() yields no messages (see core ContextEngine.compactability). */
   compactability(): CompactAvailability;
-  /** Queues a mid-run steering message (core `Session.steer`); false when no Task is running. */
-  steer(text: string): boolean;
+  /** Queues a mid-run steering input (core `Session.steer`); false when no Task is running. */
+  steer(input: OmniMessage[]): boolean;
   /** Skips the in-progress reconnect backoff, firing the next retry immediately (core `Session.skipReconnectWait`); false when no wait is in progress. */
   skipReconnectWait(): boolean;
   toolPermission(name: string): "r" | "rw" | undefined;
@@ -510,7 +510,7 @@ export class SessionManager {
   async startGoal(
     sessionId: string,
     args: {
-      /** Round-1 input (text-only, route-validated); its marker-stripped text is the objective. */
+      /** Round-1 input (route-validated to carry text; images may ride along); its marker-stripped text is the objective. */
       input: OmniMessage[];
       budget: number;
       /** Optional per-goal thinking level: rides every round's Task (route-validated). */
@@ -526,6 +526,10 @@ export class SessionManager {
       // The objective is the user's own text (leading skill-invocation blocks stripped) —
       // the same derivation core records in GOAL.yaml; used for the run-state row, the
       // goal_started event, and as title material.
+      // `isPlainText` leaves attached images out, so this copy carries no
+      // `[attached image: <path>]` lines — which suits its readers, since a status card and a
+      // generated title read better without absolute scratchpad paths. Core keeps its own
+      // folded copy (Session.runGoal) for what the rounds actually re-inject.
       const text = args.input
         .filter(isPlainText("user"))
         .map((m) => m.payload.text)
@@ -773,15 +777,16 @@ export class SessionManager {
   }
 
   /**
-   * Mid-run steering: forward the text to the running Session (core delivers it between
-   * turns as a standalone `[user_steering]` user message — no SSE event of its own; the
-   * message arrives through the stream the drive loop already publishes). 409 when the
-   * Session isn't running a Task (idle / compacting / not loaded) or the run finished in the
-   * race window — the caller falls back to submitting a normal task.
+   * Mid-run steering: forward the message to the running Session (core delivers it between
+   * turns as a standalone `[user_steering]` user message followed by its images — no SSE
+   * event of its own; the messages arrive through the stream the drive loop already
+   * publishes). 409 when the Session isn't running a Task (idle / compacting / not loaded)
+   * or the run finished in the race window — the caller falls back to submitting a normal
+   * task, which carries the same text and images.
    */
-  steer(sessionId: string, text: string): void {
+  steer(sessionId: string, input: OmniMessage[]): void {
     const entry = this.entries.get(sessionId);
-    if (!entry || entry.status !== "running" || !entry.session.steer(text)) {
+    if (!entry || entry.status !== "running" || !entry.session.steer(input)) {
       throw new HttpError(
         409,
         "not_running",

packages/server/src/services/trace-service.ts

@@ -281,6 +281,8 @@ export class TraceService {
     // request that resumes after compaction starts yet another Task.
     let taskIndex = -1;
     let continuation = false; // The previous round's Request called a tool -> the next request_begin continues the same Task
+    /** Inside the image run that follows a `[user_steering]` text (see the turn-start rule below). */
+    let steeringImages = false;
     let sawToolCallThisRequest = false;
     // Compaction interval (compaction_begin..compaction_end): the compaction
     // request's request_begin/request_end and token_usage all fall inside it (see
@@ -360,10 +362,22 @@ export class TraceService {
         typeof p.text === "string" &&
         parseUserSteeringText(p.text) !== null;
       if (isSteeringText) continuation = true;
+      // Images sent with a steering message ride immediately behind its text, exactly as a
+      // Prompt's images ride behind theirs — and they inherit its exclusion: still the same
+      // Task, so `steeringImages` keeps the window open across the whole run of them and
+      // anything else on the main session closes it (an images-only Prompt after a steering
+      // message is a genuine new turn). A subagent's messages pass through without closing it:
+      // they belong to another session's stream and say nothing about this one's grouping.
+      // The Web answers the same "what is one Task" question over the live stream — see
+      // `openSteering` in web/src/lib/omni/stream-model.ts; the two need to stay in step.
+      const isImage = !hasOrigin && msg.type === "model_msg" && p.type === "image_url";
+      if (!hasOrigin && !isSteeringText && !(isImage && steeringImages)) steeringImages = false;
+      if (isSteeringText) steeringImages = true;
       const startsUserTurn =
         !hasOrigin &&
         !compactionActive &&
         !isSteeringText &&
+        !steeringImages &&
         msg.type === "model_msg" &&
         ((p.type === "text" && p.role === "user") || p.type === "image_url");
       const startsCompactionTurn =