Yaowei Zheng
240 lines
12 KiB
JavaScript
240 lines
12 KiB
JavaScript
/**
|
|
* LLM request-failure recovery:
|
|
*
|
|
* 1. Provider quota exhaustion (403 insufficient_user_quota) is retryable: the mock rejects
|
|
* the first five requests, GenerativeModel classifies them as timeout, the engine
|
|
* reconnects with exponential backoff (250/500/1000/2000/4000ms — engine deps are not
|
|
* env-configurable, so the mock's failure count is chosen to open a ≥2s countdown
|
|
* window instead of injecting knobs; five failures + five retries sits exactly at the
|
|
* default cap of 5, so attempt 6 is the last allowed and succeeds). The 4s wait before
|
|
* retry #5 shows a live countdown whose seconds tick DOWN; clicking "retry now"
|
|
* (立即重试) skips the rest of the wait and the turn completes normally — no abort.
|
|
* 2. Give-up: a conversation whose quota rejections never stop — clicking 放弃 on the
|
|
* countdown fires the ordinary abort; the engine's abort-during-backoff path ends the
|
|
* turn and the composer is immediately usable again.
|
|
* 3. An authentication failure (401 invalid_api_key) marks the Session auth-dead but
|
|
* RECOVERABLE: only the model reference is fixed at creation — credentials come from the
|
|
* current Project config — so the notice points at the Models page, updating the key
|
|
* auto-unlocks the composer (live via the credentials_updated event; across reloads via
|
|
* the credentials-updated-vs-abort time gate), Retry is the manual escape hatch (and the
|
|
* dead state re-arms if the key is still bad), and New Session stays as the way out.
|
|
*/
|
|
import { test, expect } from "@playwright/test";
|
|
import { provisionAndLogin } from "./auth.mjs";
|
|
|
|
const BASE = process.env.BASE_URL;
|
|
const MOCK = process.env.MOCK_URL;
|
|
|
|
/** Provision a user with a configured mock model and one session; returns ids for later config updates. */
|
|
async function makeSession(page, userId, apiKey = "sk-mock") {
|
|
await provisionAndLogin(page.request, userId, "password123");
|
|
const projects = await (await page.request.get(`${BASE}/api/projects`)).json();
|
|
const projectId = projects.projects[0].projectId;
|
|
await page.request.put(`${BASE}/api/projects/${projectId}/models`, {
|
|
data: {
|
|
defaultModel: { provider: "custom", modelId: "claude-4-8" },
|
|
models: [
|
|
{
|
|
provider: "custom",
|
|
modelId: "claude-4-8",
|
|
apiKey,
|
|
baseUrl: MOCK,
|
|
contextWindow: 200000,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
const sess = await (
|
|
await page.request.post(`${BASE}/api/projects/${projectId}/agents/default_agent/sessions`, {
|
|
data: { provider: "custom", modelId: "claude-4-8" },
|
|
})
|
|
).json();
|
|
return { sessionId: sess.session.sessionId, projectId };
|
|
}
|
|
|
|
test("a quota-403 retries with a live countdown; 'retry now' skips the wait and the turn completes", async ({
|
|
page,
|
|
}) => {
|
|
const { sessionId } = await makeSession(page, "quotauser");
|
|
|
|
await page.goto(`${BASE}/chat/${sessionId}`);
|
|
await page.getByPlaceholder(/输入消息/).fill("quota retry test");
|
|
await page.getByRole("button", { name: "发送" }).click();
|
|
|
|
// Early retries flip fast (250/500ms waits — below the 2s countdown floor they keep the
|
|
// plain waiting/retried text).
|
|
await expect(page.locator("p.text-amber-600", { hasText: "已发起第 1 次重试" })).toBeVisible({
|
|
timeout: 20000,
|
|
});
|
|
await expect(page.locator("p.text-amber-600", { hasText: "已发起第 2 次重试" })).toBeVisible();
|
|
|
|
// The 4s wait before retry #5: a live countdown (whole seconds, ticking down).
|
|
const countdown = page.locator("p.text-amber-600", { hasText: /第 5 次重试，\d+ 秒后发起/ });
|
|
await expect(countdown).toBeVisible({ timeout: 20000 });
|
|
const readSecs = async () => {
|
|
const txt = await countdown.textContent({ timeout: 500 }).catch(() => null);
|
|
const m = txt === null ? null : /第 5 次重试，(\d+) 秒后发起/.exec(txt);
|
|
return m ? parseInt(m[1], 10) : null;
|
|
};
|
|
const first = await readSecs();
|
|
expect(first).not.toBeNull();
|
|
// Poll until the displayed seconds DECREASE (a live ticker, not a static label).
|
|
let second = first;
|
|
for (let i = 0; i < 12 && second !== null && second >= first; i++) {
|
|
await page.waitForTimeout(300);
|
|
second = await readSecs();
|
|
}
|
|
expect(second).not.toBeNull();
|
|
expect(second).toBeLessThan(first);
|
|
|
|
// "Retry now" skips the remaining wait: the retry fires promptly — well before the
|
|
// scheduled 4s would have elapsed (the 1.5s expectation window is the proof: without
|
|
// the skip, the natural timer still had >2s to go).
|
|
await page.getByRole("button", { name: "立即重试" }).click();
|
|
await expect(page.locator("p.text-amber-600", { hasText: "已发起第 5 次重试" })).toBeVisible({
|
|
timeout: 1500,
|
|
});
|
|
|
|
// Attempt 6 succeeds: the final answer streams in.
|
|
await expect(page.getByText("Quota recovered; the answer is 42.")).toBeVisible({
|
|
timeout: 20000,
|
|
});
|
|
|
|
// No abort: the run recovered, the composer stays usable.
|
|
await expect(page.getByText(/已中断/)).toHaveCount(0);
|
|
await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();
|
|
|
|
// Trace: all five quota rejections recorded as request_end(timeout) — the reconnect
|
|
// path — carrying the real failure detail (the Cost center's errors panel reads it from
|
|
// here) and the announced backoff ladder; no abort event.
|
|
const msgs = await (await page.request.get(`${BASE}/api/sessions/${sessionId}/messages`)).json();
|
|
const timeouts = msgs.messages.filter(
|
|
(m) => m.payload.type === "request_end" && m.payload.status === "timeout",
|
|
);
|
|
expect(timeouts.length).toBe(5);
|
|
for (const t of timeouts) expect(t.payload.message).toContain("insufficient_user_quota");
|
|
expect(timeouts.map((t) => t.payload.retry_in_ms)).toEqual([250, 500, 1000, 2000, 4000]);
|
|
expect(msgs.messages.some((m) => m.payload.type === "abort")).toBe(false);
|
|
});
|
|
|
|
test("'give up' on the countdown aborts the backoff: the turn ends and the composer is usable again", async ({
|
|
page,
|
|
}) => {
|
|
const { sessionId } = await makeSession(page, "giveupuser");
|
|
|
|
await page.goto(`${BASE}/chat/${sessionId}`);
|
|
await page.getByPlaceholder(/输入消息/).fill("quota giveup test");
|
|
await page.getByRole("button", { name: "发送" }).click();
|
|
|
|
// The mock rejects every request: by the 2s wait before retry #4 the countdown (and its
|
|
// inline controls) are on screen.
|
|
await expect(page.getByRole("button", { name: "放弃" })).toBeVisible({ timeout: 20000 });
|
|
await page.getByRole("button", { name: "放弃" }).click();
|
|
|
|
// The ordinary abort lands mid-backoff: the engine's abort-during-backoff path ends the
|
|
// turn (abort line + the waiting notice flips to "stopped"), and the composer is
|
|
// immediately usable again.
|
|
await expect(page.getByText(/已中断/)).toBeVisible({ timeout: 10000 });
|
|
await expect(page.locator("p.text-amber-600", { hasText: "已停止重试" })).toBeVisible();
|
|
await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();
|
|
});
|
|
|
|
test("an auth-401 marks the Session dead but recoverable: Models CTA, key update auto-unlocks, Retry re-arms", async ({
|
|
page,
|
|
}) => {
|
|
// The mock rejects the provisioned key (`sk-auth-bad`) with a 401 and accepts any other.
|
|
const { sessionId, projectId } = await makeSession(page, "authuser", "sk-auth-bad");
|
|
|
|
await page.goto(`${BASE}/chat/${sessionId}`);
|
|
await page.getByPlaceholder(/输入消息/).fill("auth dead test");
|
|
await page.getByRole("button", { name: "发送" }).click();
|
|
|
|
// The existing abort line renders unchanged (the notice is additional).
|
|
await expect(page.getByText(/已中断/)).toBeVisible({ timeout: 20000 });
|
|
await expect(page.getByText(/llm request error/)).toBeVisible();
|
|
|
|
// Action-only notice (per review: tell the user what to do, no explanations) and the
|
|
// composer disabled with the matching placeholder.
|
|
await expect(page.getByText(/请在模型配置页更新该模型的 API key/)).toBeVisible();
|
|
await expect(page.getByPlaceholder(/模型认证失败/)).toBeDisabled();
|
|
|
|
// Trace: the credentials failure is the request's own terminal status (no abort code —
|
|
// "auth" is a stop reason now); the abort event only carries the prose reason.
|
|
const msgs = await (await page.request.get(`${BASE}/api/sessions/${sessionId}/messages`)).json();
|
|
const authEnd = msgs.messages.find(
|
|
(m) => m.payload.type === "request_end" && m.payload.status === "auth",
|
|
);
|
|
expect(authEnd).toBeTruthy();
|
|
expect(authEnd.payload.message).toContain("invalid x-api-key");
|
|
const abort = msgs.messages.find((m) => m.payload.type === "abort");
|
|
expect(abort.payload.code).toBeUndefined();
|
|
|
|
// Reload: the state is rebuilt from Trace replay (the abort event is persisted) and the
|
|
// key has not changed, so the session stays dead.
|
|
await page.reload();
|
|
await expect(page.getByText(/模型 API 认证失败/)).toBeVisible({ timeout: 15000 });
|
|
await expect(page.getByPlaceholder(/模型认证失败/)).toBeDisabled();
|
|
|
|
// Primary CTA targets the Models page — where the credential is actually fixed.
|
|
await page.getByRole("button", { name: "打开模型配置" }).click();
|
|
await expect(page).toHaveURL(/\/models$/);
|
|
await page.goBack();
|
|
await expect(page.getByText(/模型 API 认证失败/)).toBeVisible({ timeout: 15000 });
|
|
|
|
// Secondary escape: New Session still jumps to a usable fresh draft.
|
|
await page.getByRole("button", { name: "新建会话" }).click();
|
|
await expect(page).toHaveURL(/\/chat\/new$/);
|
|
const draftInput = page.getByPlaceholder(/输入消息/);
|
|
await expect(draftInput).toBeVisible();
|
|
await expect(draftInput).toBeEnabled();
|
|
await page.goBack();
|
|
await expect(page.getByText(/模型 API 认证失败/)).toBeVisible({ timeout: 15000 });
|
|
|
|
// Retry (escape hatch) WITHOUT fixing the key: the composer re-enables for one more
|
|
// attempt, the mock rejects again, and the dead state re-arms.
|
|
await page.getByRole("button", { name: "重试", exact: true }).click();
|
|
await expect(page.getByText(/模型 API 认证失败/)).toHaveCount(0);
|
|
const input = page.getByPlaceholder(/输入消息/);
|
|
await expect(input).toBeEnabled();
|
|
await input.fill("auth dead test again");
|
|
await page.getByRole("button", { name: "发送" }).click();
|
|
await expect(page.getByText(/模型 API 认证失败/)).toBeVisible({ timeout: 20000 });
|
|
await expect(page.getByPlaceholder(/模型认证失败/)).toBeDisabled();
|
|
|
|
// PRIMARY PATH: update the model's key (as the Models page would). The server
|
|
// invalidates the Project's cached runtimes and publishes credentials_updated to this
|
|
// open tab — the composer unlocks WITHOUT a reload and WITHOUT clicking Retry.
|
|
const put = await page.request.put(`${BASE}/api/projects/${projectId}/models`, {
|
|
data: {
|
|
defaultModel: { provider: "custom", modelId: "claude-4-8" },
|
|
models: [
|
|
{
|
|
provider: "custom",
|
|
modelId: "claude-4-8",
|
|
apiKey: "sk-auth-good",
|
|
baseUrl: MOCK,
|
|
contextWindow: 200000,
|
|
},
|
|
],
|
|
},
|
|
});
|
|
expect(put.status()).toBe(200);
|
|
await expect(page.getByText(/模型 API 认证失败/)).toHaveCount(0, { timeout: 15000 });
|
|
await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();
|
|
|
|
// The SAME conversation continues on the new key (the rebuilt runtime re-reads the
|
|
// Project config): the send completes and the notice stays gone.
|
|
await page.getByPlaceholder(/输入消息/).fill("auth dead test after fix");
|
|
await page.getByRole("button", { name: "发送" }).click();
|
|
await expect(page.getByText("Auth restored; hello again.")).toBeVisible({ timeout: 20000 });
|
|
await expect(page.getByText(/模型 API 认证失败/)).toHaveCount(0);
|
|
|
|
// Reload AFTER the success: replay still contains the auth aborts, but they are followed
|
|
// by a completed request AND predate the credential update (the time gate) — the dead
|
|
// state must not resurrect.
|
|
await page.reload();
|
|
await expect(page.getByText("Auth restored; hello again.")).toBeVisible({ timeout: 15000 });
|
|
await expect(page.getByText(/模型 API 认证失败/)).toHaveCount(0);
|
|
await expect(page.getByPlaceholder(/输入消息/)).toBeEnabled();
|
|
});
|