Jingzhe Xu
349 lines
13 KiB
TypeScript
349 lines
13 KiB
TypeScript
/**
|
|
* Benchmark scoreboard read integration tests (read-only display): benchmark_config.toml title/description and runs
|
|
* pass-through (falls back to directory name if missing), scoreboard.yaml v2's
|
|
* evaluations[] (summary pass-through, model-written Case/Evaluation averages and per-case
|
|
* runs arrays), rejection of legacy Scoreboard entries, case count, empty when
|
|
* unconfigured, permissions (members can read,
|
|
* outsiders get 404).
|
|
*
|
|
* Tested with a plain Agent (no sample Benchmark pre-installed); default_agent's sample
|
|
* Benchmark assertions live in builtin-agents.test.ts.
|
|
*/
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
import { benchmarksDir } from "@prismshadow/penguin-core";
|
|
import type {
|
|
BenchmarkCasesResponse,
|
|
BenchmarksResponse,
|
|
ProjectCreateResponse,
|
|
WorkspaceFilesResponse,
|
|
} from "../src/api/types.js";
|
|
import { apiClient, createTestApp, provisionUser } from "./helpers.js";
|
|
import type { TestApp } from "./helpers.js";
|
|
|
|
const AGENT = "bench_agent";
|
|
|
|
describe("benchmarks api", () => {
|
|
let t: TestApp;
|
|
let owner: ReturnType<typeof apiClient>;
|
|
let member: ReturnType<typeof apiClient>;
|
|
let outsider: ReturnType<typeof apiClient>;
|
|
let projectId: string;
|
|
let base: string;
|
|
|
|
beforeEach(async () => {
|
|
t = await createTestApp();
|
|
const a = await provisionUser(t.app, "owner_a");
|
|
const b = await provisionUser(t.app, "member_b");
|
|
const c = await provisionUser(t.app, "outsider_c");
|
|
owner = apiClient(t.app, a.cookie);
|
|
member = apiClient(t.app, b.cookie);
|
|
outsider = apiClient(t.app, c.cookie);
|
|
const created = (await (
|
|
await owner.post("/api/projects", { projectId: "owner_a-bench", name: "Bench project" })
|
|
).json()) as ProjectCreateResponse;
|
|
projectId = created.project.projectId;
|
|
// A plain Agent has no sample Benchmark pre-installed (only default_agent provides one).
|
|
expect((await owner.post(`/api/projects/${projectId}/agents`, { agentId: AGENT })).status).toBe(
|
|
201,
|
|
);
|
|
base = `/api/projects/${projectId}/agents/${AGENT}/benchmarks`;
|
|
expect(
|
|
(await owner.post(`/api/projects/${projectId}/members`, { userId: "member_b" })).status,
|
|
).toBe(201);
|
|
});
|
|
afterEach(async () => {
|
|
await t.cleanup();
|
|
});
|
|
|
|
it("returns an empty list when unconfigured", async () => {
|
|
expect((await (await owner.get(base)).json()) as BenchmarksResponse).toEqual({
|
|
benchmarks: [],
|
|
});
|
|
});
|
|
|
|
it("current scoreboard: model-written averages, runtime, and runs pass through", async () => {
|
|
const dir = path.join(benchmarksDir(t.root, projectId, AGENT), "swe-bench-v2");
|
|
await fs.mkdir(path.join(dir, "CASE-001-excel-task", "statement"), { recursive: true });
|
|
await fs.mkdir(path.join(dir, "CASE-001-excel-task", "statement", "assets"), {
|
|
recursive: true,
|
|
});
|
|
await fs.mkdir(path.join(dir, "CASE-001-excel-task", "rubric"), { recursive: true });
|
|
await fs.mkdir(path.join(dir, "CASE-002-web-task", "statement"), { recursive: true });
|
|
await fs.mkdir(path.join(dir, "CASE-002-web-task", "rubric"), { recursive: true });
|
|
await fs.writeFile(
|
|
path.join(dir, "CASE-001-excel-task", "statement", "README.md"),
|
|
"# Case 001: Excel cleanup\n\nClean the workbook.",
|
|
"utf8",
|
|
);
|
|
await fs.writeFile(
|
|
path.join(dir, "CASE-001-excel-task", "statement", "data.csv"),
|
|
"id,value\n1,alpha\n",
|
|
"utf8",
|
|
);
|
|
await fs.writeFile(
|
|
path.join(dir, "CASE-001-excel-task", "statement", "assets", "notes.txt"),
|
|
"public notes",
|
|
"utf8",
|
|
);
|
|
await fs.writeFile(
|
|
path.join(dir, "CASE-001-excel-task", "statement", "large.txt"),
|
|
"x".repeat(300 * 1024),
|
|
"utf8",
|
|
);
|
|
await fs.writeFile(
|
|
path.join(dir, "CASE-001-excel-task", "rubric", "README.md"),
|
|
"# Scoring rubric\n\n- Correct workbook: 100 points",
|
|
"utf8",
|
|
);
|
|
await fs.writeFile(
|
|
path.join(dir, "CASE-001-excel-task", "rubric", "expected.json"),
|
|
'{"rows": 1}\n',
|
|
"utf8",
|
|
);
|
|
await fs.writeFile(
|
|
path.join(dir, "CASE-002-web-task", "statement", "README.md"),
|
|
"# Web task\n\nBuild the page.",
|
|
"utf8",
|
|
);
|
|
await fs.writeFile(
|
|
path.join(dir, "CASE-002-web-task", "rubric", "README.md"),
|
|
"PRIVATE GOLD: never return this text",
|
|
"utf8",
|
|
);
|
|
await fs.symlink(
|
|
path.join(dir, "CASE-002-web-task", "rubric", "README.md"),
|
|
path.join(dir, "CASE-001-excel-task", "statement", "private-link.md"),
|
|
);
|
|
await fs.writeFile(
|
|
path.join(dir, "benchmark_config.toml"),
|
|
`title = "SWE Bench v2"\ndescription = "Example"\nruns = 2\n`,
|
|
"utf8",
|
|
);
|
|
await fs.writeFile(
|
|
path.join(dir, "scoreboard.yaml"),
|
|
[
|
|
"evaluations:",
|
|
' - time: "2026-07-16T10:00:00Z"',
|
|
" version: 3",
|
|
' provider: "deepseek"',
|
|
' model_id: "deepseek-v4-pro"',
|
|
' thinking_level: "medium"',
|
|
' summary_title: "Added planning steps to the system Prompt"',
|
|
' summary: "Each case run twice and averaged; added planning steps."',
|
|
" score: 72.35",
|
|
" cost: 0.04",
|
|
" duration_ms: 42500",
|
|
" cases:",
|
|
// Stored averages are authoritative even when inconsistent with the raw Runs.
|
|
' - case: "CASE-001-excel-task"',
|
|
" score: 80.2",
|
|
" cost: 0.04",
|
|
" duration_ms: 50000",
|
|
" runs:",
|
|
" - score: 80",
|
|
" cost: null",
|
|
" duration_ms: 48000",
|
|
' session_id: "session-run-1"',
|
|
" - score: 82",
|
|
" cost: 0.04",
|
|
" duration_ms: 52000",
|
|
' session_id: "session-run-2"',
|
|
// All unknown Run costs produce a model-written null Case cost; the Evaluation ignores it.
|
|
' - case: "CASE-002-web-task"',
|
|
" score: 64.5",
|
|
" cost: null",
|
|
" duration_ms: 35000",
|
|
" runs:",
|
|
" - score: 60",
|
|
" cost: null",
|
|
" duration_ms: 30000",
|
|
' session_id: "session-run-3"',
|
|
" - score: 70",
|
|
" cost: null",
|
|
" duration_ms: 40000",
|
|
' session_id: "session-run-4"',
|
|
].join("\n"),
|
|
"utf8",
|
|
);
|
|
|
|
const res = (await (await member.get(base)).json()) as BenchmarksResponse;
|
|
const bench = res.benchmarks[0]!;
|
|
expect(bench).toMatchObject({
|
|
id: "swe-bench-v2",
|
|
title: "SWE Bench v2",
|
|
description: "Example",
|
|
runs: 2,
|
|
caseCount: 2,
|
|
});
|
|
// config carries no model reference (the model lives on each evaluation).
|
|
expect("modelId" in bench).toBe(false);
|
|
expect("provider" in bench).toBe(false);
|
|
const evaluation = bench.evaluations[0]!;
|
|
// The evaluation entry carries this run's model (as a pair) and a summary title (curve series / title-body are displayed separately).
|
|
expect(evaluation.provider).toBe("deepseek");
|
|
expect(evaluation.modelId).toBe("deepseek-v4-pro");
|
|
expect(evaluation.thinkingLevel).toBe("medium");
|
|
expect(evaluation.summaryTitle).toBe("Added planning steps to the system Prompt");
|
|
expect(evaluation.summary).toBe("Each case run twice and averaged; added planning steps.");
|
|
expect(evaluation.score).toBe(72.35);
|
|
expect(evaluation.cost).toBe(0.04);
|
|
expect(evaluation.durationMs).toBe(42500);
|
|
expect("maxScore" in evaluation).toBe(false);
|
|
// Per-case metrics trust the file (80.2, not the Runs' arithmetic mean of 81).
|
|
const full = evaluation.cases.find((c) => c.case === "CASE-001-excel-task")!;
|
|
expect(full.score).toBe(80.2);
|
|
expect(full.cost).toBe(0.04);
|
|
expect(full.durationMs).toBe(50000);
|
|
expect(full.runs).toEqual([
|
|
{ score: 80, cost: null, durationMs: 48000, sessionId: "session-run-1" },
|
|
{ score: 82, cost: 0.04, durationMs: 52000, sessionId: "session-run-2" },
|
|
]);
|
|
const partialCost = evaluation.cases.find((c) => c.case === "CASE-002-web-task")!;
|
|
expect(partialCost.score).toBe(64.5);
|
|
expect(partialCost.cost).toBeNull();
|
|
expect(partialCost.durationMs).toBe(35000);
|
|
expect(partialCost.runs).toEqual([
|
|
{ score: 60, cost: null, durationMs: 30000, sessionId: "session-run-3" },
|
|
{ score: 70, cost: null, durationMs: 40000, sessionId: "session-run-4" },
|
|
]);
|
|
|
|
const caseResponse = (await (
|
|
await member.get(`${base}/swe-bench-v2/cases`)
|
|
).json()) as BenchmarkCasesResponse;
|
|
expect(caseResponse).toEqual({
|
|
cases: [
|
|
{
|
|
id: "CASE-001-excel-task",
|
|
title: "Excel cleanup",
|
|
},
|
|
{
|
|
id: "CASE-002-web-task",
|
|
title: "Web task",
|
|
},
|
|
],
|
|
});
|
|
expect(JSON.stringify(caseResponse)).not.toContain("PRIVATE GOLD");
|
|
|
|
const filesBase = `${base}/swe-bench-v2/cases/CASE-001-excel-task/files`;
|
|
const files = (await (await member.get(filesBase)).json()) as WorkspaceFilesResponse;
|
|
expect(files.path).toBe("");
|
|
expect(files.entries.map((entry) => `${entry.kind}:${entry.name}`)).toEqual([
|
|
"dir:assets",
|
|
"file:data.csv",
|
|
"file:large.txt",
|
|
"file:README.md",
|
|
]);
|
|
expect(JSON.stringify(files)).not.toContain("private-link.md");
|
|
expect(
|
|
(
|
|
await member.get(
|
|
`${filesBase}/content?path=${encodeURIComponent("private-link.md")}&preview=1`,
|
|
)
|
|
).status,
|
|
).toBe(400);
|
|
|
|
const nested = (await (
|
|
await member.get(`${filesBase}?path=${encodeURIComponent("assets")}`)
|
|
).json()) as WorkspaceFilesResponse;
|
|
expect(nested.entries.map((entry) => entry.name)).toEqual(["notes.txt"]);
|
|
|
|
const statement = await member.get(
|
|
`${filesBase}/content?path=${encodeURIComponent("README.md")}&preview=1`,
|
|
);
|
|
expect(statement.status).toBe(200);
|
|
expect(statement.headers.get("content-type")).toContain("markdown");
|
|
expect(await statement.text()).toBe("# Case 001: Excel cleanup\n\nClean the workbook.");
|
|
|
|
const rubricFilesBase = `${base}/swe-bench-v2/cases/CASE-001-excel-task/rubric/files`;
|
|
const rubricFiles = (await (
|
|
await member.get(rubricFilesBase)
|
|
).json()) as WorkspaceFilesResponse;
|
|
expect(rubricFiles.entries.map((entry) => `${entry.kind}:${entry.name}`)).toEqual([
|
|
"file:expected.json",
|
|
"file:README.md",
|
|
]);
|
|
|
|
const rubric = await member.get(
|
|
`${rubricFilesBase}/content?path=${encodeURIComponent("README.md")}&preview=1`,
|
|
);
|
|
expect(rubric.status).toBe(200);
|
|
expect(rubric.headers.get("content-type")).toContain("markdown");
|
|
expect(await rubric.text()).toContain("Correct workbook: 100 points");
|
|
|
|
const large = await member.get(
|
|
`${filesBase}/content?path=${encodeURIComponent("large.txt")}&preview=1`,
|
|
);
|
|
expect(large.status).toBe(200);
|
|
expect(large.headers.get("x-content-truncated")).toBe("1");
|
|
expect((await large.text()).length).toBe(256 * 1024);
|
|
|
|
const download = await member.get(
|
|
`${filesBase}/content?path=${encodeURIComponent("data.csv")}&download=1`,
|
|
);
|
|
expect(download.headers.get("content-disposition")).toContain("attachment");
|
|
expect(await download.text()).toBe("id,value\n1,alpha\n");
|
|
|
|
expect(
|
|
(await member.get(`${filesBase}/content?path=${encodeURIComponent("../rubric/README.md")}`))
|
|
.status,
|
|
).toBe(400);
|
|
expect(
|
|
(
|
|
await member.get(
|
|
`${rubricFilesBase}/content?path=${encodeURIComponent("../statement/README.md")}`,
|
|
)
|
|
).status,
|
|
).toBe(400);
|
|
expect((await outsider.get(`${base}/swe-bench-v2/cases`)).status).toBe(404);
|
|
expect((await outsider.get(filesBase)).status).toBe(404);
|
|
});
|
|
|
|
it("does not migrate or backfill legacy Scoreboard entries", async () => {
|
|
const dir = path.join(benchmarksDir(t.root, projectId, AGENT), "swe-bench-v1");
|
|
await fs.mkdir(path.join(dir, "CASE-001-excel-task", "statement"), { recursive: true });
|
|
await fs.writeFile(path.join(dir, "benchmark_config.toml"), `title = "SWE Bench v1"\n`, "utf8");
|
|
await fs.writeFile(
|
|
path.join(dir, "scoreboard.yaml"),
|
|
[
|
|
"evaluations:",
|
|
' - time: "2026-07-16T10:00:00Z"',
|
|
" version: 1",
|
|
" score: 62.5",
|
|
" cost: 1.25",
|
|
" duration_ms: 60000",
|
|
" cases:",
|
|
' - case: "CASE-001-excel-task"',
|
|
" score: 30",
|
|
" cost: 0.5",
|
|
" duration_ms: 20000",
|
|
' session_id: "session-abc"',
|
|
' - case: ""', // Bad entry: discarded
|
|
" score: 1",
|
|
" - time: 42", // Bad evaluation: discarded
|
|
" score: 1",
|
|
].join("\n"),
|
|
"utf8",
|
|
);
|
|
// A benchmark with no config file: title falls back to the directory name, config runs field is absent by default.
|
|
await fs.mkdir(path.join(benchmarksDir(t.root, projectId, AGENT), "empty-bench"), {
|
|
recursive: true,
|
|
});
|
|
|
|
const res = (await (await member.get(base)).json()) as BenchmarksResponse;
|
|
expect(res.benchmarks.map((b) => b.id)).toEqual(["empty-bench", "swe-bench-v1"]);
|
|
const bench = res.benchmarks[1]!;
|
|
expect(bench).toMatchObject({ title: "SWE Bench v1", caseCount: 1 });
|
|
expect("runs" in bench).toBe(false);
|
|
expect(bench.evaluations).toEqual([]);
|
|
expect(res.benchmarks[0]).toMatchObject({
|
|
title: "empty-bench",
|
|
caseCount: 0,
|
|
evaluations: [],
|
|
});
|
|
|
|
expect((await outsider.get(base)).status).toBe(404);
|
|
});
|
|
});
|