3dtours/penguin-harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(server,web): steering survives reloads with visible content, and file attachments steer too (#157)

Browse Source 
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yaowei Zheng
2026-08-03 15:35:32 +08:00
committed by GitHub
parent 835dfb5c5a
commit a0968496b7
17 changed files with 554 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/web/src/features/chat/chat-input.tsx
+63 -23
View File
@@ -69,6 +69,7 @@ import type {
ApprovalMode,
ModelInfo,
ModelRefDto,
PendingSteeringInfo,
SessionStatus,
SkillMetadataItem,
TaskInputPart,
@@ -1116,6 +1117,16 @@ function readDataUrl(file: File): Promise<string | null> {
* One place, because every send path submits the same draft: the normal send, the follow-up
* queue, the @ handoff and the `/model` switch.
*/
/** One-line summary of a queued steering message: its text, then image/file counts for what the text cannot show. */
function steeringSummary(p: PendingSteeringInfo): string {
const parts: string[] = [];
const line = p.text.replace(/\s+/g, " ").trim();
if (line) parts.push(line);
if (p.images > 0) parts.push(S.chat.imagesInMessage(p.images));
if (p.files > 0) parts.push(S.chat.filesInMessage(p.files));
return parts.join(" \u00b7 ");
}
function appendAttachmentParts(
input: TaskInputPart[],
images: string[],
@@ -1132,6 +1143,7 @@ export function ChatInput({
onSend,
onSteer,
steeringDeliveredCount,
pendingSteering = [],
onQueueFollowUp,
queuedFollowUps = 0,
onStop,
@@ -1179,18 +1191,30 @@ export function ChatInput({
onSend: (input: TaskInputPart[], goal: { budget: number } | null) => Promise<boolean>;
/**
* Mid-run steering (session state only): while a Task is running, Enter/send queues the
* trimmed text **and any attached images** for the running agent — delivered between turns
* as a standalone `[user_steering]` user message followed by its images. `"queued"` clears
* the text and images and shows the queued hint; `"not_running"` (409 race with completion)
* makes the input fall back to its full normal send path; `"failed"` keeps the draft. When
* absent (draft state), the input stays send-disabled while running, as before.
* trimmed text **and any attached images and files** for the running agent — delivered
* between turns as a standalone `[user_steering]` user message followed by its images,
* with the files riding the text as `[attached file: <path>]` lines (#140: a file-only
* draft steers exactly like an image-only one). `"queued"` clears the text, images and
* files and shows the queued hint; `"not_running"` (409 race with completion) makes the
* input fall back to its full normal send path; `"failed"` keeps the draft. When absent
* (draft state), the input stays send-disabled while running, as before.
*/
onSteer?: (text: string, images: string[]) => Promise<"queued" | "not_running" | "failed">;
onSteer?: (
text: string,
images: string[],
files: { fileName: string; dataUrl: string }[],
) => Promise<"queued" | "not_running" | "failed">;
/**
* Count of steering messages already visible in the message stream: the queued hint stays
* up until this increases past its value at queue time (i.e. the message was delivered).
*/
steeringDeliveredCount?: number;
/**
* The server's undelivered-steering mirror (from task_state events): each entry renders as
* a "steering queued" line with its content, so the hint — and what was sent — survives
* reloads (#136). The local post-202 flag only bridges until the first event arrives.
*/
pendingSteering?: PendingSteeringInfo[];
/**
* Follow-up queue (session state only): posts the full input with `queueIfBusy` — a busy
* session holds it server-side and auto-sends it as an ordinary next task once the current
@@ -1498,13 +1522,14 @@ export function ChatInput({
[onHandoffTargetChange, onPendingModelChange],
);
// Mid-run steering: while running, Enter/send queues the text **and the attached images**
// for the running agent (delivered between turns as a [user_steering] user message followed
// by its images) — so an image with no caption is a complete steering message on its own.
// File attachments and selected skills stay in the draft for a later normal send: a
// [use_skills] block is task-level setup, not something to hand a turn already under way. A
// staged /agent or /model chip also blocks steering: the text belongs to the conversation
// that switch is about to open, not to the agent running here.
// Mid-run steering: while running, Enter/send queues the text **and the attached images
// and files** for the running agent (delivered between turns as a [user_steering] user
// message followed by its images; files ride the text as [attached file: <path>] lines) —
// so an image or a file with no caption is a complete steering message on its own (#140).
// Selected skills stay in the draft for a later normal send: a [use_skills] block is
// task-level setup, not something to hand a turn already under way. A staged /agent or
// /model chip also blocks steering: the text belongs to the conversation that switch is
// about to open, not to the agent running here.
// `!goalOn`: with the goal chip engaged the text is an OBJECTIVE — steering it into a run
// that happens to be active (e.g. a schedule fired) would silently repurpose it.
//
@@ -1535,6 +1560,7 @@ export function ChatInput({
hasPendingModel: pendingModel !== null,
hasText: text.trim().length > 0,
hasImages: images.length > 0,
hasFiles: attachments.length > 0,
hasContent: draftHasContent,
});
const steerAction = running && midRun === "steer";
@@ -1976,16 +2002,17 @@ export function ChatInput({
await sendNormal(onQueueFollowUp!);
return;
}
// Steering branch: queue the trimmed text and the attached images for the running agent;
// both are sent and cleared together — file attachments and selected skills stay for a
// normal send (a staged switch chip blocks this branch outright, see midRunAction).
// Steering branch: queue the trimmed text, the attached images and the attached files
// for the running agent; all are sent and cleared together — selected skills stay for
// a normal send (a staged switch chip blocks this branch outright, see midRunAction).
if (!steerAction) return;
const steerText = text.trim();
const steerImages = images;
const steerFiles = attachments.map((f) => ({ fileName: f.name, dataUrl: f.dataUrl }));
setBusy(true);
let res: "queued" | "not_running" | "failed" = "failed";
try {
res = await onSteer!(steerText, steerImages);
res = await onSteer!(steerText, steerImages, steerFiles);
if (res === "queued") {
// Show the "queued" hint until the steering message shows up in the stream
// (steeringDeliveredCount increases) — see the effect below.
@@ -1993,6 +2020,7 @@ export function ChatInput({
setSteerPending(true);
setText("");
setImages([]);
setAttachments([]);
}
} finally {
setBusy(false);
@@ -2275,12 +2303,24 @@ export function ChatInput({
</p>
)}
{/* Mid-run steering queued: lightweight hint until the steering message appears in the
stream (or the run ends). */}
{steerPending && (
<p className="anim-fade mb-1 text-xs text-gray-400 dark:text-gray-500">
{S.chat.steerQueuedIndicator}
</p>
{/* Mid-run steering queued: the server's undelivered-steering mirror, one line per
queued message with its content — task_state-fed, so it survives reloads (#136,
#140). The local steerPending flag only bridges the gap between the 202 and the
first task_state that carries the mirror. */}
{pendingSteering.length > 0 ? (
<div className="anim-fade mb-1">
{pendingSteering.map((p, i) => (
<p key={i} className="truncate text-xs text-gray-400 dark:text-gray-500">
{S.chat.steerQueuedItem(steeringSummary(p))}
</p>
))}
</div>
) : (
steerPending && (
<p className="anim-fade mb-1 text-xs text-gray-400 dark:text-gray-500">
{S.chat.steerQueuedIndicator}
</p>
)
)}
{/* Queued follow-ups (server-side, auto-sent once this run finishes): count from
packages/web/src/features/chat/chat-page.tsx
+20 -6
View File
@@ -721,14 +721,27 @@ export function ChatPage() {
// Mid-run steering: the message is queued on the server and delivered between turns as a
// standalone `[user_steering]` user message followed by its images (visible once they
// arrive over SSE / from the Trace). "not_running" (409) means no Task is in progress
// anymore (race with completion): the input area then falls back to its **full** normal
// send path — skills and the whole draft included — rather than a text+images task.
// arrive over SSE / from the Trace); file attachments land in the Session scratchpad and
// ride the steering text as `[attached file: <path>]` lines, exactly as a task's do. On
// "queued" the localStorage draft is discarded, like a successful send — without this a
// reload resurrects the already-sent text as a draft, and re-sending it duplicates the
// steering message (#136). "not_running" (409) means no Task is in progress anymore (race
// with completion): the input area then falls back to its **full** normal send path —
// skills and the whole draft included — rather than a text+images task.
const onSteer = useCallback(
async (text: string, images: string[] = []): Promise<"queued" | "not_running" | "failed"> => {
async (
text: string,
images: string[] = [],
files: { fileName: string; dataUrl: string }[] = [],
): Promise<"queued" | "not_running" | "failed"> => {
if (!selected) return "failed";
try {
await api.postSteer(selected.sessionId, { text, ...(images.length > 0 ? { images } : {}) });
await api.postSteer(selected.sessionId, {
text,
...(images.length > 0 ? { images } : {}),
...(files.length > 0 ? { files } : {}),
});
discardSessionDraft();
return "queued";
} catch (e) {
if (e instanceof ApiError && e.status === 409) return "not_running";
@@ -736,7 +749,7 @@ export function ChatPage() {
return "failed";
}
},
[selected],
[selected, discardSessionDraft],
);
const onApprove = useCallback(
@@ -881,6 +894,7 @@ export function ChatPage() {
// Count of steering messages already visible in the stream: the input area keeps its
// "queued" indicator up until this count increases (i.e. the steering message arrived).
steeringDeliveredCount={stream.model.items.filter((i) => i.kind === "user_steering").length}
pendingSteering={stream.pendingSteering}
onQueueFollowUp={onQueueFollowUp}
queuedFollowUps={stream.queuedFollowUps}
onStop={onStop}
packages/web/src/features/chat/composer-send.ts
+6 -4
View File
@@ -16,7 +16,7 @@
export type MidRunAction =
/** Abort the running Task — the button's face whenever this draft has no send channel. */
| "stop"
/** Deliver text and images to the running agent as a `[user_steering]` message. */
/** Deliver text, images and file attachments to the running agent as a `[user_steering]` message. */
| "steer"
/** Hold the whole draft server-side and auto-send it as the next ordinary message. */
| "queue"
@@ -46,6 +46,8 @@ export interface MidRunComposerState {
hasText: boolean;
/** At least one image is attached. */
hasImages: boolean;
/** At least one file attachment is staged (steers as `[attached file: <path>]` lines, like a task's). */
hasFiles: boolean;
/**
* The draft carries anything at all — text, images, file attachments, a staged switch chip
* or selected skills. The queue takes a whole message, so this is its content rule.
@@ -58,8 +60,8 @@ export interface MidRunComposerState {
* the button is always Send there, gated by the ordinary `canSend`.
*
* Steering is preferred over the queue when both could carry the draft, since it reaches the
* turn already under way; the queue picks up what steering cannot — a skills-only draft, file
* attachments, a staged chip — so that "unsendable here" never costs the user Stop.
* turn already under way; the queue picks up what steering cannot — a skills-only draft, a
* staged chip — so that "unsendable here" never costs the user Stop.
*/
export function midRunAction(s: MidRunComposerState): MidRunAction {
if (s.sending) return "disabled";
@@ -71,7 +73,7 @@ export function midRunAction(s: MidRunComposerState): MidRunAction {
s.canSteerChannel &&
!s.hasHandoffTarget &&
!s.hasPendingModel &&
(s.hasText || s.hasImages);
(s.hasText || s.hasImages || s.hasFiles);
if (canSteer && !s.followUpMode) return "steer";
const canQueue = open && s.canQueueChannel && s.stagedRoute !== "blocked" && s.hasContent;
if (canQueue) return "queue";
packages/web/src/features/chat/use-session-draft.ts
+5 -3
View File
@@ -132,9 +132,11 @@ export function useSessionDraft(sessionId: string | null): {
const discard = useCallback(() => {
cancelPending();
// Also clear the selected skills and both staged switch chips: ChatInput's clear after a
// successful send doesn't fire a callback (same convention as onTextChange); without this,
// a later text flush would resurrect the already-sent selection.
// Also clear the text, the selected skills and both staged switch chips: ChatInput's
// clear after a successful send doesn't fire a callback (same convention as
// onTextChange); without this, a later flush triggered by any discrete action (a skill
// toggle, a staged chip) would resurrect the already-sent text or selection.
textRef.current = "";
skillsRef.current = [];
handoffRef.current = null;
switchModelRef.current = null;
packages/web/src/features/chat/use-session-stream.ts
+12 -1
View File
@@ -16,7 +16,11 @@
* server re-sends still-pending requests.
*/
import { useCallback, useEffect, useRef, useState } from "react";
import type { GoalServerEvent, SessionStatus } from "@prismshadow/penguin-server/api";
import type {
GoalServerEvent,
PendingSteeringInfo,
SessionStatus,
} from "@prismshadow/penguin-server/api";
import { getGoal, getMe, getMessages } from "../../api/endpoints";
import { openSessionStream } from "../../api/sse";
import { createStreamController } from "../../lib/omni/stream-controller";
@@ -39,6 +43,8 @@ export interface SessionStreamState {
taskState: SessionStatus;
/** Queued follow-up count from the stream's task_state events (auto-sent once the session is idle). */
queuedFollowUps: number;
/** Steering messages queued on the server but not yet delivered (from task_state events): keeps the composer's hint and its content across reloads. */
pendingSteering: PendingSteeringInfo[];
/**
* Timestamp (ms) of the last main-session auth failure (request_end with status "auth"),
* or null. Derived from the model, so it survives history replay and resets when
@@ -86,6 +92,7 @@ export function useSessionStream(
const [loading, setLoading] = useState(true);
const [taskState, setTaskState] = useState<SessionStatus>(initialStatus);
const [queuedFollowUps, setQueuedFollowUps] = useState(0);
const [pendingSteering, setPendingSteering] = useState<PendingSteeringInfo[]>([]);
const [error, setError] = useState<string | null>(null);
const [pendingTick, setPendingTick] = useState(0);
const [goal, setGoal] = useState<GoalBannerState | null>(null);
@@ -163,6 +170,7 @@ export function useSessionStream(
controllerRef.current = null;
setTaskState("idle");
setQueuedFollowUps(0);
setPendingSteering([]);
setLoading(false);
setError(null);
setGoal(null);
@@ -178,6 +186,7 @@ export function useSessionStream(
setTaskState(initialStatus);
setGoal(null);
setQueuedFollowUps(0);
setPendingSteering([]);
setPendingTick((t) => t + 1);
// Restore an in-flight goal's banner (only when still active — a long-finished goal
@@ -203,6 +212,7 @@ export function useSessionStream(
loadMessages: () => getMessages(sessionId),
onTaskState: setTaskState,
onQueuedFollowUps: setQueuedFollowUps,
onPendingSteering: setPendingSteering,
onLoading: setLoading,
onError: setError,
onModelChange: bump,
@@ -276,6 +286,7 @@ export function useSessionStream(
loading,
taskState,
queuedFollowUps,
pendingSteering,
lastAuthFailureMs: (controllerRef.current?.model ?? placeholderRef.current).lastAuthFailureMs,
dismissModelAuthDead,
pendingApprovals: controllerRef.current?.pendingApprovals ?? EMPTY_PENDING,