feat(core,server,web): images reach every input — steering, goal objectives, one fold (#71)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -104,6 +104,7 @@ import {
|
||||
skillSlashItems,
|
||||
} from "./skill-use";
|
||||
import { GOAL_ICON, UNLIMITED_BUDGET, parseBudgetInput } from "./goal-use";
|
||||
import { midRunAction } from "./composer-send";
|
||||
import { PAPERCLIP_ICON } from "./attached-files-banner";
|
||||
|
||||
const APPROVAL_MODES: ApprovalMode[] = ["always-ask", "read-only", "allow-all", "deny-all"];
|
||||
@@ -1178,13 +1179,13 @@ export function ChatInput({
|
||||
onSend: (input: TaskInputPart[], goal: { budget: number } | null) => Promise<boolean>;
|
||||
/**
|
||||
* Mid-run steering (session state only): while a Task is running, Enter/send queues the
|
||||
* trimmed text for the running agent — it is delivered between turns as a standalone
|
||||
* `[user_steering]` user message. `"queued"` clears the text and shows the queued hint;
|
||||
* `"not_running"` (409 race with completion) makes the input fall back to its full normal
|
||||
* send path; `"failed"` keeps the draft. When absent (draft state), the input stays
|
||||
* send-disabled while running, as before.
|
||||
* trimmed text **and any attached images** for the running agent — delivered between turns
|
||||
* as a standalone `[user_steering]` user message followed by its images. `"queued"` clears
|
||||
* the text and images and shows the queued hint; `"not_running"` (409 race with completion)
|
||||
* makes the input fall back to its full normal send path; `"failed"` keeps the draft. When
|
||||
* absent (draft state), the input stays send-disabled while running, as before.
|
||||
*/
|
||||
onSteer?: (text: string) => Promise<"queued" | "not_running" | "failed">;
|
||||
onSteer?: (text: string, images: string[]) => Promise<"queued" | "not_running" | "failed">;
|
||||
/**
|
||||
* Count of steering messages already visible in the message stream: the queued hint stays
|
||||
* up until this increases past its value at queue time (i.e. the message was delivered).
|
||||
@@ -1371,9 +1372,18 @@ export function ChatInput({
|
||||
|
||||
const running = status === "running";
|
||||
const compacting = status === "compacting";
|
||||
// The draft's "anything sendable at all" rule, shared by canSend / canFollowUp / the
|
||||
// steer-mode queue fallback and (negated) by the Stop face of the action button.
|
||||
const draftHasContent =
|
||||
text.trim().length > 0 ||
|
||||
images.length > 0 ||
|
||||
attachments.length > 0 ||
|
||||
target !== null ||
|
||||
pendingModel !== null ||
|
||||
selectedSkills.length > 0;
|
||||
// Goal mode (engaged via the "+" menu or /goal): the text body becomes the objective. It is
|
||||
// exclusive with a staged /agent or /model switch (engaging either clears the other) and with
|
||||
// images (the objective is re-injected every round as plain text); selected skills ride the
|
||||
// exclusive with a staged /agent or /model switch (engaging either clears the other); attached
|
||||
// images ride along (core folds them into the objective as path lines) and selected skills ride
|
||||
// round-1 message as a [use_skills] block, exactly like a normal send.
|
||||
const [goalOn, setGoalOn] = useState(false);
|
||||
const [goalBudgetText, setGoalBudgetText] = useState("");
|
||||
@@ -1407,6 +1417,9 @@ export function ChatInput({
|
||||
// parseable budget — and an open editor showing an invalid draft disables Send outright:
|
||||
// combined with the editor refusing to close over an invalid draft (below), no click sequence
|
||||
// can fire a goal with a stale committed budget.
|
||||
// Images may come along with a goal objective (core folds them into `[attached image: …]`
|
||||
// lines so they survive the rounds), but they don't substitute for the text; file
|
||||
// attachments cannot — nothing folds those into a re-injected objective.
|
||||
const canSend =
|
||||
!running &&
|
||||
!compacting &&
|
||||
@@ -1414,16 +1427,10 @@ export function ChatInput({
|
||||
!modelAuthDead &&
|
||||
(goalOn
|
||||
? text.trim().length > 0 &&
|
||||
images.length === 0 &&
|
||||
attachments.length === 0 &&
|
||||
goalBudget !== null &&
|
||||
!(goalBudgetOpen && goalBudgetDraftInvalid)
|
||||
: text.trim().length > 0 ||
|
||||
images.length > 0 ||
|
||||
attachments.length > 0 ||
|
||||
target !== null ||
|
||||
pendingModel !== null ||
|
||||
selectedSkills.length > 0);
|
||||
: draftHasContent);
|
||||
|
||||
/**
|
||||
* The budget editor is a fixed upward popover. Opening copies the committed value; closing
|
||||
@@ -1482,74 +1489,59 @@ export function ChatInput({
|
||||
onHandoffTargetChange?.(null);
|
||||
setPendingModel(null);
|
||||
onPendingModelChange?.(null);
|
||||
// Attachments can't ride a goal (the server rejects non-text goal input): clear any
|
||||
// already attached, or canSend would stay silently false with the objective looking ready.
|
||||
setImages([]);
|
||||
// Images ride a goal (folded into the objective as path lines), file attachments do not
|
||||
// — the server refuses those, so clear them or canSend would stay silently false with
|
||||
// the objective looking ready.
|
||||
setAttachments([]);
|
||||
}
|
||||
},
|
||||
[onHandoffTargetChange, onPendingModelChange],
|
||||
);
|
||||
|
||||
// Mid-run steering: while running, Enter/send queues plain text for the running agent
|
||||
// (delivered between turns as a [user_steering] user message). Text only — attachments /
|
||||
// skills / a staged switch stay in the draft for a later normal send (a staged /agent or
|
||||
// /model chip also blocks steering: the text belongs to the conversation the switch is about
|
||||
// to open, not to the agent running here).
|
||||
// Mid-run steering: while running, Enter/send queues the text **and the attached images**
|
||||
// for the running agent (delivered between turns as a [user_steering] user message followed
|
||||
// by its images) — so an image with no caption is a complete steering message on its own.
|
||||
// File attachments and selected skills stay in the draft for a later normal send: a
|
||||
// [use_skills] block is task-level setup, not something to hand a turn already under way. A
|
||||
// staged /agent or /model chip also blocks steering: the text belongs to the conversation
|
||||
// that switch is about to open, not to the agent running here.
|
||||
// `!goalOn`: with the goal chip engaged the text is an OBJECTIVE — steering it into a run
|
||||
// that happens to be active (e.g. a schedule fired) would silently repurpose it.
|
||||
const canSteer =
|
||||
running &&
|
||||
!busy &&
|
||||
!goalOn &&
|
||||
!modelAuthDead &&
|
||||
onSteer !== undefined &&
|
||||
target === null &&
|
||||
pendingModel === null &&
|
||||
text.trim().length > 0;
|
||||
// Mid-run send mode (owner directive): the user chooses between "steer" (delivered
|
||||
// mid-run as a [user_steering] input) and "follow-up" (held server-side and auto-sent as
|
||||
// an ordinary next task once this run finishes). Set from the "+" menu's settings row —
|
||||
// available in draft state and active sessions alike — and **remembered** across
|
||||
// sessions/reloads (localStorage, see STEER_MODE_KEY); the running-state send simply
|
||||
// follows the remembered mode.
|
||||
//
|
||||
// Mid-run send mode (owner directive): the user chooses between "steer" (delivered mid-run
|
||||
// as a [user_steering] input) and "follow-up" (held server-side and auto-sent as an ordinary
|
||||
// next task once this run finishes). Set from the "+" menu's settings row — available in
|
||||
// draft state and active sessions alike — and **remembered** across sessions/reloads
|
||||
// (localStorage, see STEER_MODE_KEY).
|
||||
const [steerMode, setSteerModeState] = useState<SteerMode>(initialSteerMode);
|
||||
const setSteerMode = (mode: SteerMode): void => {
|
||||
setSteerModeState(mode);
|
||||
localStorage.setItem(STEER_MODE_KEY, mode);
|
||||
};
|
||||
const followUpMode = steerMode === "followup" && onQueueFollowUp !== undefined;
|
||||
// A follow-up is a full normal message: the whole draft (text / attachments / skills / a
|
||||
// staged switch) is eligible, same content rule as canSend.
|
||||
// `stagedRoute !== "blocked"`: a staged model fork is never eligible mid-run — the follow-up
|
||||
// path composes the whole draft and then hands it to onSwitchModel rather than to the queue,
|
||||
// so without this gate Enter would fork off a Trace that is still being written.
|
||||
const canFollowUp =
|
||||
running &&
|
||||
!busy &&
|
||||
!goalOn &&
|
||||
!modelAuthDead &&
|
||||
followUpMode &&
|
||||
stagedRoute !== "blocked" &&
|
||||
(text.trim().length > 0 ||
|
||||
images.length > 0 ||
|
||||
attachments.length > 0 ||
|
||||
target !== null ||
|
||||
pendingModel !== null ||
|
||||
selectedSkills.length > 0);
|
||||
// The single action button's mode: while running, an **empty** composer means Stop
|
||||
// (abort); as soon as there is something to send it becomes the send button (steer or
|
||||
// follow-up per the remembered mode). Idle/compacting is always send.
|
||||
const canMidRunSend = followUpMode ? canFollowUp : canSteer;
|
||||
const midRunSendLabel = followUpMode ? S.chat.followUpSend : S.chat.steerSend;
|
||||
const stopAction =
|
||||
running &&
|
||||
text.trim().length === 0 &&
|
||||
images.length === 0 &&
|
||||
attachments.length === 0 &&
|
||||
target === null &&
|
||||
pendingModel === null &&
|
||||
selectedSkills.length === 0;
|
||||
// Which of the two channels this draft can use, or Stop when neither will take it — the whole
|
||||
// decision lives in midRunAction so it can be reasoned about and tested on its own, and so
|
||||
// that Stop stays the fallthrough rather than a case somebody has to remember to widen. Only
|
||||
// meaningful while running; idle/compacting is always Send, gated by canSend above.
|
||||
const midRun = midRunAction({
|
||||
sending: busy,
|
||||
goalOn,
|
||||
modelAuthDead,
|
||||
canSteerChannel: onSteer !== undefined,
|
||||
canQueueChannel: onQueueFollowUp !== undefined,
|
||||
followUpMode,
|
||||
stagedRoute,
|
||||
hasHandoffTarget: target !== null,
|
||||
hasPendingModel: pendingModel !== null,
|
||||
hasText: text.trim().length > 0,
|
||||
hasImages: images.length > 0,
|
||||
hasContent: draftHasContent,
|
||||
});
|
||||
const steerAction = running && midRun === "steer";
|
||||
const queueAction = running && midRun === "queue";
|
||||
const canMidRunSend = steerAction || queueAction;
|
||||
const midRunSendLabel = midRun === "queue" ? S.chat.followUpSend : S.chat.steerSend;
|
||||
const stopAction = running && midRun === "stop";
|
||||
// Queued hint: shown after a successful steer until the message shows up in the stream
|
||||
// (steeringDeliveredCount increases past the baseline captured at queue time) or the run
|
||||
// stops being observable (task no longer running).
|
||||
@@ -1902,11 +1894,15 @@ export function ChatInput({
|
||||
// from being added since), so there is nothing to carry here.
|
||||
setBusy(true);
|
||||
try {
|
||||
const ok = await onSend([{ type: "text", text: buildSkillsMessage(selectedSkills, t) }], {
|
||||
budget: goalBudget!,
|
||||
});
|
||||
// Attached images go with the objective (see the goalOn declaration above).
|
||||
const goalInput: TaskInputPart[] = [
|
||||
{ type: "text", text: buildSkillsMessage(selectedSkills, t) },
|
||||
];
|
||||
for (const url of images) goalInput.push({ type: "image_url", imageUrl: url });
|
||||
const ok = await onSend(goalInput, { budget: goalBudget! });
|
||||
if (ok) {
|
||||
setText("");
|
||||
setImages([]);
|
||||
setSelectedSkills([]);
|
||||
toggleGoal(false);
|
||||
}
|
||||
@@ -1970,37 +1966,41 @@ export function ChatInput({
|
||||
|
||||
const send = async () => {
|
||||
if (running) {
|
||||
// Follow-up branch: the whole draft goes out through the normal composition path,
|
||||
// but posted with queueIfBusy — the server holds it and auto-sends once this run
|
||||
// finishes (a staged switch still opens its new chat directly: neither the handoff
|
||||
// target nor the model fork is the session that is running).
|
||||
if (followUpMode) {
|
||||
if (!canFollowUp) return;
|
||||
// Queue branch: the whole draft goes out through the normal composition path, posted
|
||||
// with queueIfBusy — the server holds it and auto-sends once this run finishes (a staged
|
||||
// switch still opens its new chat directly: neither the handoff target nor the model fork
|
||||
// is the session that is running). One branch for both ways of getting here — follow-up
|
||||
// mode, and steer mode meeting a draft steering cannot carry — since the message sent is
|
||||
// the same either way.
|
||||
if (queueAction) {
|
||||
await sendNormal(onQueueFollowUp!);
|
||||
return;
|
||||
}
|
||||
// Steering branch: queue the trimmed text for the running agent; only the text is
|
||||
// sent and cleared — attached images / selected skills stay for a normal send (a
|
||||
// staged switch chip blocks this branch outright, see canSteer).
|
||||
if (!canSteer) return;
|
||||
// Steering branch: queue the trimmed text and the attached images for the running agent;
|
||||
// both are sent and cleared together — file attachments and selected skills stay for a
|
||||
// normal send (a staged switch chip blocks this branch outright, see midRunAction).
|
||||
if (!steerAction) return;
|
||||
const steerText = text.trim();
|
||||
const steerImages = images;
|
||||
setBusy(true);
|
||||
let res: "queued" | "not_running" | "failed" = "failed";
|
||||
try {
|
||||
res = await onSteer!(steerText);
|
||||
res = await onSteer!(steerText, steerImages);
|
||||
if (res === "queued") {
|
||||
// Show the "queued" hint until the steering message shows up in the stream
|
||||
// (steeringDeliveredCount increases) — see the effect below.
|
||||
steerBaseline.current = steeringDeliveredCount ?? 0;
|
||||
setSteerPending(true);
|
||||
setText("");
|
||||
setImages([]);
|
||||
}
|
||||
} finally {
|
||||
setBusy(false);
|
||||
textareaRef.current?.focus();
|
||||
}
|
||||
// Completion race (server: no Task running anymore): deliver the whole draft — images,
|
||||
// skills and all — through the full normal send path instead of a text-only task.
|
||||
// Completion race (server: no Task running anymore): deliver the whole draft — skills
|
||||
// and all — through the full normal send path. The draft is untouched in this branch
|
||||
// (nothing was cleared), so the images go out with it.
|
||||
if (res === "not_running") await sendNormal();
|
||||
return;
|
||||
}
|
||||
@@ -2051,9 +2051,6 @@ export function ChatInput({
|
||||
};
|
||||
|
||||
const addFiles = (files: Iterable<File>) => {
|
||||
// Goal mode is text-only (the objective is re-injected each round): drop image attachments
|
||||
// outright — including pastes — so send never lands in a silently-disabled state.
|
||||
if (goalOn) return;
|
||||
for (const file of files) {
|
||||
if (!file.type.startsWith("image/")) continue;
|
||||
const reader = new FileReader();
|
||||
@@ -2607,7 +2604,6 @@ export function ChatInput({
|
||||
type="file"
|
||||
accept="image/*"
|
||||
multiple
|
||||
disabled={goalOn}
|
||||
className="hidden"
|
||||
onChange={onPickFiles}
|
||||
/>
|
||||
@@ -2634,11 +2630,11 @@ export function ChatInput({
|
||||
icon: IMAGE_ICON,
|
||||
label: S.chat.uploadImage,
|
||||
// Without vision the images still send — as scratchpad file paths — so the
|
||||
// entry stays usable and the hint explains what will happen instead.
|
||||
desc: vision ? S.chat.uploadImageDesc : S.chat.imagesAsPathHint,
|
||||
// entry stays usable and the hint says what will happen instead. Goal mode
|
||||
// sends them that way on any model, since the objective is re-injected as
|
||||
// text every round.
|
||||
desc: vision && !goalOn ? S.chat.uploadImageDesc : S.chat.imagesAsPathHint,
|
||||
active: images.length > 0,
|
||||
// Goal mode is text-only (the objective is re-injected each round).
|
||||
disabled: goalOn,
|
||||
onSelect: () => imageInputRef.current?.click(),
|
||||
},
|
||||
{
|
||||
@@ -2650,7 +2646,8 @@ export function ChatInput({
|
||||
// inlined into the conversation.
|
||||
desc: S.chat.uploadFileDesc,
|
||||
active: attachments.length > 0,
|
||||
// Same rule as images: goal input is text-only.
|
||||
// Unlike images, a file cannot ride a goal: nothing folds it into the
|
||||
// objective that every round re-injects, so the server refuses it.
|
||||
disabled: goalOn,
|
||||
onSelect: () => attachmentInputRef.current?.click(),
|
||||
},
|
||||
|
||||
@@ -707,16 +707,16 @@ export function ChatPage() {
|
||||
[selected, discardSessionDraft, syncHealedSessionId],
|
||||
);
|
||||
|
||||
// Mid-run steering: the text is queued on the server and delivered between turns as a
|
||||
// standalone `[user_steering]` user message (visible once it arrives over SSE / from the
|
||||
// Trace). "not_running" (409) means no Task is in progress anymore (race with completion):
|
||||
// the input area then falls back to its **full** normal send path — images / skills / the
|
||||
// whole draft included — rather than a text-only task.
|
||||
// Mid-run steering: the message is queued on the server and delivered between turns as a
|
||||
// standalone `[user_steering]` user message followed by its images (visible once they
|
||||
// arrive over SSE / from the Trace). "not_running" (409) means no Task is in progress
|
||||
// anymore (race with completion): the input area then falls back to its **full** normal
|
||||
// send path — skills and the whole draft included — rather than a text+images task.
|
||||
const onSteer = useCallback(
|
||||
async (text: string): Promise<"queued" | "not_running" | "failed"> => {
|
||||
async (text: string, images: string[] = []): Promise<"queued" | "not_running" | "failed"> => {
|
||||
if (!selected) return "failed";
|
||||
try {
|
||||
await api.postSteer(selected.sessionId, { text });
|
||||
await api.postSteer(selected.sessionId, { text, ...(images.length > 0 ? { images } : {}) });
|
||||
return "queued";
|
||||
} catch (e) {
|
||||
if (e instanceof ApiError && e.status === 409) return "not_running";
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* What the composer's single action button does while a Task is running.
|
||||
*
|
||||
* Pulled out of ChatInput for the same reason as `stagedSendRoute`: the decision has more
|
||||
* inputs than it looks (two send channels, a mode toggle, a staged switch, a dead model key,
|
||||
* a goal draft) and every one of them can take a channel away. Inline, that made a specific
|
||||
* mistake easy to repeat — deriving Stop from "is the composer empty" instead of "does this
|
||||
* send have anywhere to go". Those agree only while every non-empty draft is sendable, and
|
||||
* several are not: a goal objective is an objective rather than a message for the turn under
|
||||
* way, a rejected model key refuses everything, and a staged `/model` fork waits for idle.
|
||||
* Where they disagreed the button showed a permanently disabled Send *in place of* Stop, so
|
||||
* the run could not be sent to or stopped without emptying the composer first.
|
||||
*
|
||||
* Stop is therefore the fallthrough here, not a case: whatever cannot be sent leaves Stop.
|
||||
*/
|
||||
export type MidRunAction =
|
||||
/** Abort the running Task — the button's face whenever this draft has no send channel. */
|
||||
| "stop"
|
||||
/** Deliver text and images to the running agent as a `[user_steering]` message. */
|
||||
| "steer"
|
||||
/** Hold the whole draft server-side and auto-send it as the next ordinary message. */
|
||||
| "queue"
|
||||
/** A send of this draft is already in flight; the button is inert until it settles. */
|
||||
| "disabled";
|
||||
|
||||
export interface MidRunComposerState {
|
||||
/** A send started from this composer is in flight (ChatInput's `busy`). */
|
||||
sending: boolean;
|
||||
/** The goal chip is engaged: the body is an objective, which no mid-run channel carries. */
|
||||
goalOn: boolean;
|
||||
/** The model API rejected this Session's credentials — nothing can be sent at all. */
|
||||
modelAuthDead: boolean;
|
||||
/** The host wired a steer channel (`onSteer`); the draft page does not. */
|
||||
canSteerChannel: boolean;
|
||||
/** The host wired a follow-up queue (`onQueueFollowUp`); the draft page does not. */
|
||||
canQueueChannel: boolean;
|
||||
/** Mid-run send mode is "follow-up" rather than "steer" (remembered per user). */
|
||||
followUpMode: boolean;
|
||||
/** Where a staged `/agent` / `/model` chip would send this (see stagedSendRoute). */
|
||||
stagedRoute: "post" | "handoff" | "model" | "blocked";
|
||||
/** An `/agent` handoff target is staged. */
|
||||
hasHandoffTarget: boolean;
|
||||
/** A `/model` fork target is staged. */
|
||||
hasPendingModel: boolean;
|
||||
/** The body has non-whitespace text. */
|
||||
hasText: boolean;
|
||||
/** At least one image is attached. */
|
||||
hasImages: boolean;
|
||||
/**
|
||||
* The draft carries anything at all — text, images, file attachments, a staged switch chip
|
||||
* or selected skills. The queue takes a whole message, so this is its content rule.
|
||||
*/
|
||||
hasContent: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* The button's mode while a Task runs. Idle and compacting are not this function's business:
|
||||
* the button is always Send there, gated by the ordinary `canSend`.
|
||||
*
|
||||
* Steering is preferred over the queue when both could carry the draft, since it reaches the
|
||||
* turn already under way; the queue picks up what steering cannot — a skills-only draft, file
|
||||
* attachments, a staged chip — so that "unsendable here" never costs the user Stop.
|
||||
*/
|
||||
export function midRunAction(s: MidRunComposerState): MidRunAction {
|
||||
if (s.sending) return "disabled";
|
||||
// A goal draft and a dead key close both channels; a blocked `/model` fork closes the queue
|
||||
// (see stagedSendRoute) and cannot reach steering anyway, since a staged chip rules it out.
|
||||
const open = !s.goalOn && !s.modelAuthDead;
|
||||
const canSteer =
|
||||
open &&
|
||||
s.canSteerChannel &&
|
||||
!s.hasHandoffTarget &&
|
||||
!s.hasPendingModel &&
|
||||
(s.hasText || s.hasImages);
|
||||
if (canSteer && !s.followUpMode) return "steer";
|
||||
const canQueue = open && s.canQueueChannel && s.stagedRoute !== "blocked" && s.hasContent;
|
||||
if (canQueue) return "queue";
|
||||
// Follow-up mode with a draft the queue refuses: steering is not a silent substitute for it,
|
||||
// because the two put the message in different places. Stop, as with anything unsendable.
|
||||
return "stop";
|
||||
}
|
||||
@@ -10,13 +10,69 @@
|
||||
* count, token usage against the budget, and the terminal state once the run ends. The
|
||||
* stop control is the regular abort (one signal spans the whole goal loop server-side).
|
||||
*/
|
||||
import { useState } from "react";
|
||||
import { S } from "../../lib/strings";
|
||||
import { humanizeTokens } from "../../lib/format";
|
||||
import { GlyphIcon } from "../../components/ui/glyph-icon";
|
||||
import { ZoomableImage } from "../../components/ui/image-zoom";
|
||||
import { GOAL_ICON, UNLIMITED_BUDGET } from "./goal-use";
|
||||
import type { GoalBannerState } from "./goal-use";
|
||||
|
||||
export function GoalRoundBanner({ round, objective }: { round: number; objective?: string }) {
|
||||
/** Image glyph (24×24 line path) for the collapsed attachment chip on later goal rounds. */
|
||||
const ATTACHMENT_ICON = "M3 5h18v14H3zM3 16l5-5 4 4 3-3 6 6M15.5 8.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0";
|
||||
|
||||
/** The objective's attachments under a round bubble: thumbnails when `showFull`, a chip otherwise. */
|
||||
function GoalRoundImages({
|
||||
images,
|
||||
showFull,
|
||||
onExpand,
|
||||
}: {
|
||||
images: string[];
|
||||
showFull: boolean;
|
||||
onExpand: () => void;
|
||||
}) {
|
||||
if (images.length === 0) return null;
|
||||
if (showFull) {
|
||||
return (
|
||||
<div className="mt-1.5 flex max-w-[88%] flex-wrap justify-end gap-1.5 md:max-w-[75%]">
|
||||
{images.map((src, i) => (
|
||||
<ZoomableImage
|
||||
key={i}
|
||||
src={src}
|
||||
alt={S.chat.imageAlt}
|
||||
className="max-h-40 max-w-full rounded-md"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onExpand}
|
||||
className="mt-1.5 flex items-center gap-1.5 rounded-md border border-gray-200 bg-gray-50 px-2 py-1 text-xs text-gray-500 hover:bg-gray-100 dark:border-gray-700 dark:bg-gray-900 dark:text-gray-400 dark:hover:bg-gray-800"
|
||||
>
|
||||
<GlyphIcon d={ATTACHMENT_ICON} size={12} className="shrink-0" />
|
||||
{S.chat.goalRoundImages(images.length)}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
export function GoalRoundBanner({
|
||||
round,
|
||||
objective,
|
||||
images = [],
|
||||
}: {
|
||||
round: number;
|
||||
objective?: string;
|
||||
/** Images attached to the objective, restored from its `[attached image: …]` path lines. */
|
||||
images?: string[];
|
||||
}) {
|
||||
// The path lines ride the re-injected text, so the images really are in every round's input
|
||||
// — dropping them after round 1 would misreport what was sent. Showing them full-size every
|
||||
// time would bury a long goal under the same picture, so later rounds collapse to a chip.
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const showFull = images.length > 0 && (round === 1 || expanded);
|
||||
// A regular right-aligned user bubble (same classes as message-item's user_text
|
||||
// rendering), with the round notice under the bubble.
|
||||
if (objective !== undefined && objective !== "") {
|
||||
@@ -27,6 +83,7 @@ export function GoalRoundBanner({ round, objective }: { round: number; objective
|
||||
{objective}
|
||||
</p>
|
||||
</div>
|
||||
<GoalRoundImages images={images} showFull={showFull} onExpand={() => setExpanded(true)} />
|
||||
<p className="mt-1 flex items-center gap-1.5 px-0.5 text-xs text-gray-400 dark:text-gray-500">
|
||||
<GlyphIcon d={GOAL_ICON} size={12} className="shrink-0" />
|
||||
{S.chat.goalRoundBanner(round)}
|
||||
|
||||
@@ -203,12 +203,13 @@ export function MessageItem({ item, ctx }: { item: ChatItem; ctx: StreamRenderCo
|
||||
// files come out of the same pass and collapse into one banner naming them.
|
||||
const { text, images, files } = splitAttachments(skills ? skills.rest : afterScheduled);
|
||||
// Every goal round reads like a normal user message: the body in a user bubble with
|
||||
// the round notice beneath (the system IS re-sending the user's request each round).
|
||||
// the round notice beneath (the system IS re-sending the user's request each round) —
|
||||
// the objective's images included, since they ride it as path lines.
|
||||
if (goalRound) {
|
||||
return (
|
||||
<>
|
||||
{skills && <SkillsBanner names={skills.skills} />}
|
||||
<GoalRoundBanner round={goalRound.round} objective={text} />
|
||||
<GoalRoundBanner round={goalRound.round} objective={text} images={images} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -218,7 +219,7 @@ export function MessageItem({ item, ctx }: { item: ChatItem; ctx: StreamRenderCo
|
||||
{skills && <SkillsBanner names={skills.skills} />}
|
||||
{/* Files uploaded with this message: named above the bubble, like the other
|
||||
message-level notices — the bytes live in the session scratchpad, the model
|
||||
opens them by path (goal mode never gets here: it rejects non-text input). */}
|
||||
opens them by path (goal mode never gets here: it takes text and images only). */}
|
||||
{files.length > 0 && <AttachedFilesBanner files={files} />}
|
||||
{text && (
|
||||
<div className="anim-msg group my-4 flex flex-col items-end">
|
||||
@@ -253,32 +254,53 @@ export function MessageItem({ item, ctx }: { item: ChatItem; ctx: StreamRenderCo
|
||||
</>
|
||||
);
|
||||
}
|
||||
case "user_steering":
|
||||
case "user_steering": {
|
||||
// Mid-run steering ([user_steering]-wrapped user text delivered between turns): a
|
||||
// compact right-aligned user-styled chip inside the running Task's flow — visually
|
||||
// lighter than a full prompt bubble, since it doesn't start a new Task (the Trace page
|
||||
// still shows the raw marker text as-is).
|
||||
// Images sent with the message live inside the same chip: `item.images` on a vision
|
||||
// model (delivered as image messages right behind the text), or restored from the
|
||||
// [attached image: …] path lines without one — the same two shapes a user_text bubble
|
||||
// handles, so both render identically here.
|
||||
const { text: steerText, images: steerImages } = splitAttachments(item.text);
|
||||
const shown = [...steerImages, ...(item.images ?? [])];
|
||||
return (
|
||||
<div className="anim-msg group my-2 flex flex-col items-end">
|
||||
<div className="flex max-w-[88%] items-start gap-1.5 rounded-md border border-gray-200 bg-gray-100 px-3 py-1.5 md:max-w-[75%] dark:border-gray-700 dark:bg-gray-800">
|
||||
<GlyphIcon
|
||||
d={USER_STEERING_ICON}
|
||||
className="mt-1 shrink-0 text-gray-400 dark:text-gray-500"
|
||||
/>
|
||||
<p className="wrap-anywhere whitespace-pre-wrap text-sm leading-relaxed text-gray-800 dark:text-gray-100">
|
||||
<span className="mr-1.5 text-xs font-semibold text-gray-500 dark:text-gray-400">
|
||||
{S.chat.userSteering}
|
||||
</span>
|
||||
{item.text}
|
||||
</p>
|
||||
<div className="flex max-w-[88%] flex-col gap-1.5 rounded-md border border-gray-200 bg-gray-100 px-3 py-1.5 md:max-w-[75%] dark:border-gray-700 dark:bg-gray-800">
|
||||
<div className="flex items-start gap-1.5">
|
||||
<GlyphIcon
|
||||
d={USER_STEERING_ICON}
|
||||
className="mt-1 shrink-0 text-gray-400 dark:text-gray-500"
|
||||
/>
|
||||
<p className="wrap-anywhere whitespace-pre-wrap text-sm leading-relaxed text-gray-800 dark:text-gray-100">
|
||||
<span className="mr-1.5 text-xs font-semibold text-gray-500 dark:text-gray-400">
|
||||
{S.chat.userSteering}
|
||||
</span>
|
||||
{steerText}
|
||||
</p>
|
||||
</div>
|
||||
{shown.length > 0 && (
|
||||
<div className="flex flex-wrap justify-end gap-1.5">
|
||||
{shown.map((src, i) => (
|
||||
<ZoomableImage
|
||||
key={i}
|
||||
src={src}
|
||||
alt={S.chat.imageAlt}
|
||||
className="max-h-28 max-w-full rounded-md"
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<MessageMeta
|
||||
{...(item.atMs !== undefined ? { atMs: item.atMs } : {})}
|
||||
text={item.text}
|
||||
text={steerText}
|
||||
align="right"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
case "user_image":
|
||||
return (
|
||||
<div className="anim-msg group my-4 flex flex-col items-end">
|
||||
|
||||
@@ -101,6 +101,14 @@ export interface UserSteeringItem {
|
||||
kind: "user_steering";
|
||||
id: number;
|
||||
text: string;
|
||||
/**
|
||||
* Images sent with this steering message: core delivers them as ordinary user image
|
||||
* messages right behind the text, and they are folded in here rather than rendered as
|
||||
* standalone bubbles — they are part of the same message and must not start a Task.
|
||||
* (Without vision the images arrive as `[attached image: …]` lines inside `text` instead,
|
||||
* which the chip restores at render time like any user message.)
|
||||
*/
|
||||
images?: string[];
|
||||
/** Message timestamp (milliseconds): shown on footer hover. */
|
||||
atMs?: number;
|
||||
}
|
||||
@@ -328,6 +336,13 @@ export interface StreamModel {
|
||||
/** A fragment that has stopped and is waiting to be replaced by the complete message. */
|
||||
pendingText: AssistantTextItem | null;
|
||||
pendingThinking: ThinkingItem | null;
|
||||
/**
|
||||
* The steering chip still collecting its images: core delivers a steering message's images
|
||||
* as user image messages immediately behind its text, so an image arriving while this is
|
||||
* set belongs to that chip. Any other message closes the window (see pushMessage) — an
|
||||
* images-only Prompt sent after a steering message is a genuine new Task.
|
||||
*/
|
||||
openSteering: UserSteeringItem | null;
|
||||
/** tool_call_id → tool card (shared by both fragment attribution and complete-message replacement). */
|
||||
toolCards: Map<string, ToolCallItem>;
|
||||
/** Direct child Session id → nested model. */
|
||||
@@ -435,6 +450,7 @@ function newModel(nested: boolean, localDecisions: Set<string>): StreamModel {
|
||||
openThinking: null,
|
||||
pendingText: null,
|
||||
pendingThinking: null,
|
||||
openSteering: null,
|
||||
toolCards: new Map(),
|
||||
subagents: new Map(),
|
||||
localDecisions,
|
||||
@@ -494,6 +510,13 @@ export function pushMessage(
|
||||
routeNested(model, msg, nowMs);
|
||||
return;
|
||||
}
|
||||
// A steering message's images arrive as user image messages directly behind its text, with
|
||||
// nothing interleaved (core delivers the batch in one go) — so anything else on this session
|
||||
// closes the collection window opened by the chip (see openSteering). Subagent messages
|
||||
// returned above never reach here, so they leave the window alone.
|
||||
// The server answers the same "what is one Task" question over the Trace — see
|
||||
// `steeringImages` in server/src/services/trace-service.ts; the two need to stay in step.
|
||||
if (!isCompleteUserImage(msg)) model.openSteering = null;
|
||||
if (msg.type === "model_msg") {
|
||||
// Internal messages within a compaction range (between begin and end)
|
||||
// (the compaction prompt, summary output): never rendered, never
|
||||
@@ -539,6 +562,11 @@ export function pushMessage(
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the message is a complete user image — the only kind that can join an open steering chip. */
|
||||
function isCompleteUserImage(msg: OmniMessage): boolean {
|
||||
return msg.type === "model_msg" && (msg.payload as { type?: string }).type === "image_url";
|
||||
}
|
||||
|
||||
/**
|
||||
* Agent id from a session_meta `agent_state` path: the path is
|
||||
* `<root>/<projectId>/agents/<agentId>/agent_state`, so the agent id is the parent directory
|
||||
@@ -931,12 +959,15 @@ function handleComplete(
|
||||
if (steering !== null) {
|
||||
touchTask(model, timestamp);
|
||||
const steerMs = tsOf(timestamp);
|
||||
model.items.push({
|
||||
const item: UserSteeringItem = {
|
||||
kind: "user_steering",
|
||||
id: nextId(model),
|
||||
text: steering,
|
||||
...(steerMs !== undefined ? { atMs: steerMs } : {}),
|
||||
});
|
||||
};
|
||||
model.items.push(item);
|
||||
// Open the window for the images core delivers right behind this text.
|
||||
model.openSteering = item;
|
||||
return;
|
||||
}
|
||||
// A complete text message on the main session's user side: starts a new Task.
|
||||
@@ -1000,6 +1031,13 @@ function handleComplete(
|
||||
return;
|
||||
}
|
||||
case "image_url": {
|
||||
// An image belonging to the steering message just rendered: it joins that chip and
|
||||
// leaves the running Task alone — unlike a Prompt's image, it starts nothing.
|
||||
if (model.openSteering) {
|
||||
touchTask(model, timestamp);
|
||||
model.openSteering.images = [...(model.openSteering.images ?? []), p.image_url];
|
||||
return;
|
||||
}
|
||||
startTask(model, timestamp, nowMs);
|
||||
const imgMs = tsOf(timestamp);
|
||||
model.items.push({
|
||||
|
||||
@@ -848,6 +848,9 @@ Scenarios:
|
||||
goalBudgetSave: "Save budget",
|
||||
goalRemove: "Exit goal mode",
|
||||
goalRoundBanner: (round: number): string => `Goal · round ${round}`,
|
||||
/** Later rounds collapse the objective's images into this chip (round 1 shows them in full). */
|
||||
goalRoundImages: (count: number): string =>
|
||||
count === 1 ? "1 attached image" : `${count} attached images`,
|
||||
goalProgress: (rounds: number, tokens: string): string => `round ${rounds} · tokens ${tokens}`,
|
||||
goalStatus: {
|
||||
active: "running",
|
||||
|
||||
@@ -827,6 +827,8 @@ Benchmark:
|
||||
goalBudgetSave: "保存预算",
|
||||
goalRemove: "退出目标模式",
|
||||
goalRoundBanner: (round: number): string => `目标 · 第 ${round} 轮`,
|
||||
/** Later rounds collapse the objective's images into this chip (round 1 shows them in full). */
|
||||
goalRoundImages: (count: number): string => `${count} 张附图`,
|
||||
goalProgress: (rounds: number, tokens: string): string => `第 ${rounds} 轮 · tokens ${tokens}`,
|
||||
goalStatus: {
|
||||
active: "进行中",
|
||||
|
||||
Reference in New Issue
Block a user