feat(core,server,web): images reach every input — steering, goal objectives, one fold (#71)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/core/test/goal.test.ts

@@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";
 import { parse as parseYaml } from "yaml";
 import {
   UNLIMITED_BUDGET,
+  Session,
   abortEvent,
   assistantText,
   buildSkillsMessage,
@@ -12,6 +13,7 @@ import {
   emptyTokenCounts,
   goalFilePath,
   goalFinishedOf,
+  imageUrlMessage,
   isGoalRoundInput,
   parseGoalMessage,
   stripConversationMarkers,
@@ -19,7 +21,15 @@ import {
   userText,
   withOrigin,
 } from "../src/index.js";
-import type { GoalOutcome, OmniMessage, TokenCounts } from "../src/index.js";
+import type {
+  EnvironmentInterface,
+  GoalOutcome,
+  LLMInterface,
+  LLMOutcome,
+  OmniMessage,
+  SessionMetaPayload,
+  TokenCounts,
+} from "../src/index.js";
 // The file protocol, prompt composition and the loop are internal to `session.run` (not part
 // of the SDK barrel); tests reach them through their modules directly.
 import { readGoalStatus, serializeGoalFile, writeGoalFile } from "../src/goal/goal-file.js";
@@ -389,3 +399,99 @@ describe("runGoalLoop", () => {
     expect(emptyTokenCounts().total).toBe(0);
   });
 });
+
+/**
+ * The Session-level goal entry (`run(input, { goal })`): input validation and the image fold.
+ * A goal objective folds its images to `[attached image: <path>]` lines on any model, unlike a
+ * Prompt — it is re-injected as the text of each round's `[goal]` block, which leaves an image
+ * message nowhere to sit. See Session.runGoal.
+ */
+describe("Session.runGoal input", () => {
+  const PNG_DATA_URL =
+    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==";
+
+  const fakeEnvironment: EnvironmentInterface = {
+    listTools: async () => [],
+    // eslint-disable-next-line require-yield
+    executeTool: async function* () {
+      throw new Error("not used");
+    },
+    toolPermission: () => undefined,
+  };
+
+  /** A model that answers each round with one final text, marking the goal complete on `completeOn`. */
+  function fakeLLM(completeOn: number): LLMInterface {
+    let round = 0;
+    return {
+      async *streamGenerate() {
+        round++;
+        if (round >= completeOn) await setStatus("complete");
+        yield assistantText(`round ${round} done`);
+        return { status: "completed" } satisfies LLMOutcome;
+      },
+    };
+  }
+
+  // `modelHasVision: true` throughout: the fold runs regardless, and a vision model is the case
+  // that would break if runGoal ever grew the `if (!this.modelHasVision)` the other paths have.
+  function makeSession(completeOn = 1): Session {
+    const meta: SessionMetaPayload = {
+      session_id: "session-1",
+      provider: "custom",
+      model_id: "m1",
+      model_context_window: 1000,
+      system_prompt: "sp",
+      tools: [],
+      agent_state: dir,
+      workspace: dir,
+    };
+    return new Session({
+      meta,
+      llm: fakeLLM(completeOn),
+      environment: fakeEnvironment,
+      imagesDir: path.join(dir, "scratchpad", "session-1"),
+      modelHasVision: true,
+      goalFilePath: file,
+    });
+  }
+
+  /** Drives the goal and returns the text of each round's injected `[goal]` input. */
+  async function roundInputs(session: Session, input: OmniMessage[]): Promise<string[]> {
+    const texts: string[] = [];
+    for await (const msg of session.run(input, { goal: {} })) {
+      if (isGoalRoundInput(msg)) texts.push((msg.payload as { text: string }).text);
+    }
+    return texts;
+  }
+
+  it("folds an attached image into the objective and re-injects it every round — vision model included", async () => {
+    const session = makeSession(2);
+    const rounds = await roundInputs(session, [
+      userText("Match this mockup"),
+      imageUrlMessage(PNG_DATA_URL),
+    ]);
+    expect(rounds).toHaveLength(2);
+    // The picture is on disk, and both rounds point at that same file.
+    const saved = await fs.readdir(path.join(dir, "scratchpad", "session-1"));
+    expect(saved).toHaveLength(1);
+    const line = `[attached image: ${path.join(dir, "scratchpad", "session-1", saved[0]!)}]`;
+    for (const text of rounds) expect(text).toContain(line);
+    // Round 2 re-injects the objective alone, which is where the line matters most: it
+    // survives because stripLeadingMarkerBlocks only removes leading blocks, and the fold
+    // appends at the end.
+    expect(parseGoalMessage(rounds[1]!)?.rest).toBe(`Match this mockup\n\n${line}`);
+    // No image message ever reaches the round input.
+    expect(rounds.every((t) => !t.includes("data:image"))).toBe(true);
+  });
+
+  it("rejects an objective with no text: an image alone states no goal", async () => {
+    const session = makeSession();
+    await expect(roundInputs(session, [imageUrlMessage(PNG_DATA_URL)])).rejects.toThrow(
+      /non-empty text objective/,
+    );
+    // A blank text is no better.
+    await expect(
+      roundInputs(session, [userText("   "), imageUrlMessage(PNG_DATA_URL)]),
+    ).rejects.toThrow(/non-empty text objective/);
+  });
+});