更新数据分析 Skill,优化多 Run 评估流程 (#194)
This commit is contained in:
1 parent
67adf93b47
commit
5e90da2439
10 files changed
+134
-70
No files matched your search
@@ -182,17 +182,17 @@ describe("librarySkill", () => {
|
||||
expect(librarySkill("no-such-skill")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("benchmark-design selects a valid Pilot revision before the Formal Baseline", () => {
|
||||
it("benchmark-design records the selected one-Run Pilot as the Formal Baseline", () => {
|
||||
const content = librarySkill("benchmark-design")?.content;
|
||||
expect(content).toBeDefined();
|
||||
|
||||
const pilotIndex = content!.indexOf("## Refine the Benchmark");
|
||||
const baselineIndex = content!.indexOf("## Freeze and run the Formal Baseline");
|
||||
const baselineIndex = content!.indexOf("## Freeze and record the Formal Baseline");
|
||||
const normalizedContent = content!.replace(/\s+/g, " ");
|
||||
|
||||
expect(pilotIndex).toBeGreaterThan(-1);
|
||||
expect(baselineIndex).toBeGreaterThan(pilotIndex);
|
||||
expect(normalizedContent).toContain("Keep Pilot results out of the Scoreboard");
|
||||
expect(normalizedContent).toContain("Keep unselected Pilot results out of the Scoreboard");
|
||||
expect(normalizedContent).toContain("requested valid-iteration limit");
|
||||
expect(normalizedContent).toContain("lowest-scoring valid Pilot revision");
|
||||
expect(normalizedContent).toContain("retain only one temporary restorable copy");
|
||||
@@ -245,7 +245,11 @@ describe("librarySkill", () => {
|
||||
"Do not run another difficulty refinement merely to create more score margin",
|
||||
);
|
||||
expect(normalizedContent).toContain("missing the desired score alone is not a failure");
|
||||
expect(normalizedContent).toContain("never reuse a Pilot result");
|
||||
expect(normalizedContent).toContain("`runs = 1`");
|
||||
expect(normalizedContent).toContain(
|
||||
"Do not launch a fresh Formal matrix, rerun the selected Pilot, or backfill it",
|
||||
);
|
||||
expect(normalizedContent).toContain("Accept the selected Pilot result as the Formal Baseline");
|
||||
expect(normalizedContent).toContain(
|
||||
"Record the Formal Baseline even when its score does not meet the desired baseline score",
|
||||
);
|
||||
@@ -293,7 +297,7 @@ describe("librarySkill", () => {
|
||||
"Read `thinking_level` from the Test Agent's `model.thinking_level`",
|
||||
);
|
||||
expect(benchmarkDesign).toContain(
|
||||
"Pass the resolved `(provider, model_id)` explicitly in every Pilot and Formal Evaluator request",
|
||||
"Pass the resolved `(provider, model_id)` explicitly in every Pilot Evaluator request",
|
||||
);
|
||||
expect(benchmarkDesign).toContain("Every Case Rubric has a fixed maximum of 100 points");
|
||||
expect(benchmarkDesign).toContain("average of the Case scores");
|
||||
@@ -311,6 +315,17 @@ describe("librarySkill", () => {
|
||||
);
|
||||
expect(benchmarkDesign).toContain("Ask the same Evaluator to resend only the clean YAML");
|
||||
expect(optimization).toContain("Delegate every evaluation to an `agent-evaluation` subagent");
|
||||
expect(optimization).toContain("a positive `runs` value");
|
||||
expect(optimization).toContain(
|
||||
"do not infer it from `benchmark_config.toml` or the Formal Baseline",
|
||||
);
|
||||
expect(optimization).toContain(
|
||||
"The initial Formal Baseline has one Run per Case; do not rerun or backfill it",
|
||||
);
|
||||
expect(optimization).toContain(
|
||||
"Compare each Candidate's stored top-level average directly with the current Reference score even when their Run counts differ",
|
||||
);
|
||||
expect(optimization).toContain("frozen Case set × requested `runs` matrix");
|
||||
expect(optimization).toContain("Before reading `status`, `score`, or any other protocol field");
|
||||
expect(optimization).toContain("Ask the same Evaluator to resend only the clean YAML");
|
||||
expect(optimizationRaw).toMatch(/summary_title:\s*>-\n\s+<public title>/);
|
||||
|
||||
Reference in new issue
Block a user