Changelog, dev startup, README, AgentHub 0.4.0, model catalog, and landing site (#7)
Branch-length batch covering tooling, the model layer, the Web App and the public surfaces. Highlights: - Changelog: a per-release `changelog/<version>/` tree, grouped by the surface each change touches, with a root CHANGELOG.md holding one line per release. - Dev startup: `scripts/dev-prebuild.mjs` serializes the skills+core prebuild behind a lock and keeps `pnpm install` current; `pnpm dev` runs server+web together. - AgentHub 0.3.3 -> 0.4.0: OmniMessage complete payloads carry one opaque `fidelity` object in place of item-level `signature`/`phase`, threaded verbatim through Trace, replay and resume; malformed classification adapted to the new error types. - Model layer: a model is always referenced by an explicit `(provider, model_id)` pair. The provider is never inferred, guessed or defaulted -- both the catalog inference and the unique-match config resolution are gone, and CLI, SDK, server routes and run_subagent all require the complete pair. Catalog gains the Qwen Token Plan, Qwen Pay-As-You-Go and Fireworks AI gateways, plus an expanded OpenRouter group. - Web App: catalog preset sync and per-group speed test on the Models page, positional slash commands, a markdown renderer, skill-library update reminders, and a vertically centred draft page whose upward menus size themselves to the room available. - Public surfaces: restructured READMEs, the penguin.ooo landing site and blog, refreshed benchmark results for both suites, and the demo videos playing on the landing page. Includes the fixes from a full review of the branch: 23 confirmed findings, among them a provider-inference bug that could send one vendor's API key to another vendor's endpoint, and an Escape handler that destroyed the composer's contents unrecoverably. Verified on the branch head: pnpm test (1127 passing, 7 packages), pnpm typecheck and pnpm format:check clean, Playwright e2e 14/14.
This commit is contained in:
Yaowei Zheng authored and GitHub committed
1 parent
abf0a2f248
commit
d4faee3a1e
214 files changed
+9406
-1688
No files matched your search
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* Renders the README / blog benchmark chart from the same data the landing page uses
|
||||
* (src/lib/benchmark-data.ts), so the static SVGs cannot drift from the site when the
|
||||
* numbers are refreshed. Emits:
|
||||
* assets/readme/benchmark-light.svg
|
||||
* assets/readme/benchmark-dark.svg
|
||||
* packages/landing/public/blog-assets/benchmark-light.svg
|
||||
*
|
||||
* Two panels per suite (accuracy, cost), horizontal bars scaled linearly from zero — the
|
||||
* cost spread is ~70x, so the PenguinHarness bar is by far the shortest. That is the
|
||||
* result, not a defect; a zoomed baseline would flatter everyone else. Bars are floored at
|
||||
* MIN_BAR, though: at true scale the cost bar comes out ~2.5px, which reads as "no bar" and
|
||||
* loses the series entirely. The floor keeps it visible and unmistakably smallest, and the
|
||||
* exact figure is printed beside every bar, so nothing is overstated.
|
||||
*
|
||||
* Run: node packages/landing/scripts/render-benchmark-svg.mjs
|
||||
*/
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const LANDING = path.resolve(HERE, "..");
|
||||
const REPO = path.resolve(LANDING, "..", "..");
|
||||
|
||||
/**
|
||||
* The data module is TypeScript; rather than add a build step, read the two exported
|
||||
* arrays out of the source. Any shape change here fails loudly instead of silently
|
||||
* rendering a stale chart.
|
||||
*/
|
||||
function loadBench() {
|
||||
const src = readFileSync(path.join(LANDING, "src/lib/benchmark-data.ts"), "utf8");
|
||||
const consts = Object.fromEntries(
|
||||
[...src.matchAll(/^const (\w+) = "([^"]+)";$/gm)].map((m) => [m[1], m[2]]),
|
||||
);
|
||||
const pick = (name) => {
|
||||
const block = src.match(
|
||||
new RegExp(`export const ${name}: BenchResult\\[\\] = \\[([\\s\\S]*?)\\n\\];`),
|
||||
);
|
||||
if (!block) throw new Error(`could not find ${name} in benchmark-data.ts`);
|
||||
const rows = [...block[1].matchAll(/\{([\s\S]*?)\n {2}\}/g)].map((m) => {
|
||||
const body = m[1];
|
||||
const field = (k) => body.match(new RegExp(`${k}:\\s*([^,\\n]+)`))?.[1]?.trim();
|
||||
const str = (k) => {
|
||||
const v = field(k);
|
||||
if (!v) throw new Error(`missing ${k}`);
|
||||
return v.startsWith('"') ? v.slice(1, -1) : (consts[v] ?? v);
|
||||
};
|
||||
return {
|
||||
framework: str("framework"),
|
||||
model: str("model"),
|
||||
accuracyPct: Number(field("accuracyPct")),
|
||||
tokensM: Number(field("tokensM")),
|
||||
costUsd: Number(field("costUsd")),
|
||||
emphasized: /emphasized:\s*true/.test(body),
|
||||
};
|
||||
});
|
||||
if (rows.length !== 3) throw new Error(`${name}: expected 3 rows, got ${rows.length}`);
|
||||
for (const r of rows) {
|
||||
if (!Number.isFinite(r.accuracyPct) || !Number.isFinite(r.costUsd)) {
|
||||
throw new Error(`${name}: ${r.framework} has a non-numeric field`);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
};
|
||||
return { DATA_BENCH: pick("DATA_BENCH"), CODE_BENCH: pick("CODE_BENCH") };
|
||||
}
|
||||
|
||||
const THEMES = {
|
||||
light: {
|
||||
strong: "#1f2328",
|
||||
mid: "#52514e",
|
||||
muted: "#898781",
|
||||
rule: "#c3c2b7",
|
||||
brand: "#2a78d6",
|
||||
bar: "#898781",
|
||||
},
|
||||
dark: {
|
||||
strong: "#f0f3f6",
|
||||
mid: "#c3c2b7",
|
||||
muted: "#898781",
|
||||
rule: "#383835",
|
||||
brand: "#3987e5",
|
||||
bar: "#6b6a64",
|
||||
},
|
||||
};
|
||||
|
||||
const FONT = "system-ui, -apple-system, 'Segoe UI', sans-serif";
|
||||
const W = 920;
|
||||
const H = 368;
|
||||
const BAR_H = 16;
|
||||
const ROW_H = 30;
|
||||
const MAX_BAR = 176;
|
||||
/** Shortest a bar may render, so a near-zero value still reads as a bar (~8% of full). */
|
||||
const MIN_BAR = 14;
|
||||
|
||||
const esc = (s) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
||||
|
||||
function text(x, y, s, { size = 12, weight = 400, fill, anchor = "start" }) {
|
||||
return `<text x="${x}" y="${y}" font-size="${size}" font-weight="${weight}" fill="${fill}" text-anchor="${anchor}" font-family="${FONT}">${esc(s)}</text>`;
|
||||
}
|
||||
|
||||
/** Rounded-end horizontal bar starting at the axis. */
|
||||
function bar(x, y, w, fill) {
|
||||
const r = Math.min(4, w / 2);
|
||||
return `<path d="M${x} ${y} h${(w - r).toFixed(1)} a${r} ${r} 0 0 1 ${r} ${r} v${BAR_H - 2 * r} a${r} ${r} 0 0 1 -${r} ${r} h-${(w - r).toFixed(1)} z" fill="${fill}"/>`;
|
||||
}
|
||||
|
||||
/** One measure for one suite: label column, axis rule, three bars, value labels. */
|
||||
function panel(t, x0, yTop, rows, value, format) {
|
||||
const axis = x0 + 124;
|
||||
const max = Math.max(...rows.map(value));
|
||||
const out = [
|
||||
`<rect x="${axis - 1}" y="${yTop}" width="1" height="${ROW_H * rows.length + 2}" fill="${t.rule}"/>`,
|
||||
];
|
||||
rows.forEach((row, i) => {
|
||||
const y = yTop + 4 + i * ROW_H;
|
||||
const w = Math.max(MIN_BAR, (value(row) / max) * MAX_BAR);
|
||||
const weight = row.emphasized ? 600 : 400;
|
||||
out.push(
|
||||
text(axis - 12, y + 12.5, row.framework, {
|
||||
fill: row.emphasized ? t.strong : t.mid,
|
||||
weight,
|
||||
anchor: "end",
|
||||
}),
|
||||
bar(axis + 1, y, w, row.emphasized ? t.brand : t.bar),
|
||||
text(axis + w + 13, y + 12.5, format(value(row)), { fill: t.strong, weight }),
|
||||
);
|
||||
});
|
||||
return out.join("\n");
|
||||
}
|
||||
|
||||
function suite(t, yTop, title, rows) {
|
||||
return [
|
||||
text(24, yTop, title, { size: 13, weight: 600, fill: t.strong }),
|
||||
panel(
|
||||
t,
|
||||
24,
|
||||
yTop + 10,
|
||||
rows,
|
||||
(r) => r.accuracyPct,
|
||||
(v) => `${v.toFixed(2)}%`,
|
||||
),
|
||||
panel(
|
||||
t,
|
||||
472,
|
||||
yTop + 10,
|
||||
rows,
|
||||
(r) => r.costUsd,
|
||||
(v) => `$${v.toFixed(2)}`,
|
||||
),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function render(theme, { DATA_BENCH, CODE_BENCH }) {
|
||||
const t = THEMES[theme];
|
||||
const alt =
|
||||
"Benchmark: PenguinHarness vs Claude Code vs OpenAI Codex on two suites — comparable accuracy at a small fraction of the cost";
|
||||
return `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${W} ${H}" width="${W}" height="${H}" role="img" aria-label="${esc(alt)}">
|
||||
${text(148, 36, "Accuracy · suite total · higher is better", { fill: t.muted })}
|
||||
${text(596, 36, "Total cost (USD) · lower is better", { fill: t.muted })}
|
||||
${suite(t, 68, "Data analysis — 15 tasks, single run", DATA_BENCH)}
|
||||
${suite(t, 210, "Coding — 40 tasks × 2 runs", CODE_BENCH)}
|
||||
${text(24, 338, "Each harness runs the model it is normally paired with: PenguinHarness on DeepSeek V4 Pro, Claude Code on Claude Opus 4.8,", { fill: t.muted })}
|
||||
${text(24, 356, "OpenAI Codex on GPT-5.5. Accuracy, Tokens and cost are suite totals at official pricing.", { fill: t.muted })}
|
||||
</svg>
|
||||
`;
|
||||
}
|
||||
|
||||
const bench = loadBench();
|
||||
const targets = [
|
||||
["light", path.join(REPO, "assets/readme/benchmark-light.svg")],
|
||||
["dark", path.join(REPO, "assets/readme/benchmark-dark.svg")],
|
||||
["light", path.join(LANDING, "public/blog-assets/benchmark-light.svg")],
|
||||
];
|
||||
for (const [theme, file] of targets) {
|
||||
writeFileSync(file, render(theme, bench));
|
||||
console.log(`wrote ${path.relative(REPO, file)}`);
|
||||
}
|
||||
Reference in new issue
Block a user