From 0d98e6cc5e924ed9c94112abcb19e1251a3274b4 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Wed, 12 Aug 2026 17:29:56 -0700 Subject: [PATCH] fix: clarify skill evaluation provenance --- .../nvidia-skill-evaluation.test.ts | 13 +++-- scripts/prototypes/nvidia-skill-evaluation.ts | 48 ++++++++++++----- src/components/SkillDetailPage.tsx | 17 +++---- src/components/SkillEvaluationReport.test.tsx | 34 ++++++++++--- src/components/SkillEvaluationReport.tsx | 51 ++++++++++++++++--- .../SkillEvaluationReportLoader.test.tsx | 9 ++-- .../SkillEvaluationReportLoader.tsx | 2 +- src/styles.css | 26 +++++++--- 8 files changed, 149 insertions(+), 51 deletions(-) diff --git a/scripts/prototypes/nvidia-skill-evaluation.test.ts b/scripts/prototypes/nvidia-skill-evaluation.test.ts index b8058eb0..01b1958e 100644 --- a/scripts/prototypes/nvidia-skill-evaluation.test.ts +++ b/scripts/prototypes/nvidia-skill-evaluation.test.ts @@ -131,7 +131,8 @@ describe("SkillEvaluator eval discovery", () => { await expect(discoverSkillEvals(skillPath)).resolves.toEqual({ status: "skipped", reason: "no-evals", - message: "No SkillEvaluator dataset or native Harbor tasks were found in evals/.", + message: + "No SkillEvaluator dataset or native Harbor tasks were found. Add evals/evals.json or native Harbor tasks under evals/.", }); }); @@ -263,13 +264,15 @@ describe("canonical SkillEvaluator execution", () => { ).toThrow("Unsupported SkillEvaluator environment variable: GITHUB_TOKEN"); }); - it("pins the evaluator and Codex model for a one-attempt Tier 3 smoke run", () => { + it("pins distinct judge and Codex models for a labeled Tier 3 smoke run", () => { expect( buildTier3EvaluateInvocation({ evaluatorRepoPath: "/tmp/SkillEvaluator", skillDirectory: "/tmp/skills/skills/doca-dpa", resultsDirectory: "/tmp/results", - model: "gpt-5.4-mini", + agentModel: "gpt-5.4-mini", + judgeModel: "gpt-5.4", + attempts: 1, }), ).toEqual({ command: [ @@ -277,6 +280,8 @@ describe("canonical SkillEvaluator execution", () => { "run", "--project", "/tmp/SkillEvaluator", + "--extra", + "tier3", "skillevaluator", "tier3", "evaluate", @@ -299,7 +304,7 @@ describe("canonical SkillEvaluator execution", () => { "/tmp/results", ], environment: { - SKILL_EVAL_LLM_MODEL: "gpt-5.4-mini", + SKILL_EVAL_LLM_MODEL: "gpt-5.4", SKILL_EVAL_LLM_PROVIDER: "openai", }, }); diff --git a/scripts/prototypes/nvidia-skill-evaluation.ts b/scripts/prototypes/nvidia-skill-evaluation.ts index 852fb47b..c53fe8de 100644 --- a/scripts/prototypes/nvidia-skill-evaluation.ts +++ b/scripts/prototypes/nvidia-skill-evaluation.ts @@ -130,7 +130,8 @@ export async function discoverSkillEvals(skillDirectory: string): Promise import.meta.env.DEV && evaluationSourceRepo?.toLowerCase() === "nvidia/skills" && - githubBackedFields.githubCurrentCommit && - githubBackedFields.githubPath + evaluationCurrentCommit && + evaluationSkillPath ? { repository: "nvidia/skills", - commit: evaluationDemoCommit ?? githubBackedFields.githubCurrentCommit, - path: githubBackedFields.githubPath, + commit: evaluationDemoCommit ?? evaluationCurrentCommit, + path: evaluationSkillPath, } : null, - [ - githubBackedFields?.githubCurrentCommit, - githubBackedFields?.githubPath, - evaluationSourceRepo, - evaluationDemoCommit, - ], + [evaluationCurrentCommit, evaluationDemoCommit, evaluationSkillPath, evaluationSourceRepo], ); const modInfo = result?.moderationInfo ?? null; const relatedCategory = useMemo(() => (skill ? getSkillCategoryForSkill(skill) : null), [skill]); diff --git a/src/components/SkillEvaluationReport.test.tsx b/src/components/SkillEvaluationReport.test.tsx index f907dfaf..4d1fef5b 100644 --- a/src/components/SkillEvaluationReport.test.tsx +++ b/src/components/SkillEvaluationReport.test.tsx @@ -13,7 +13,7 @@ function record( overrides: Partial = {}, ): SkillEvaluationRunRecord { return { - schemaVersion: 1, + schemaVersion: 2, state, smokeRun: true, source: { @@ -34,8 +34,9 @@ function record( commit: "b".repeat(40), version: "0.1.0", agent: "codex", - model: "gpt-5.4-mini", - provider: "openai", + agentModel: "gpt-5.4-mini", + judgeModel: "gpt-5.4", + judgeProvider: "openai", environment: "local", attempts: 1, }, @@ -106,11 +107,16 @@ describe("SkillEvaluationReport", () => { />, ); - expect(screen.getByText("95.9%")).toBeTruthy(); - expect(screen.getByText("+35.3 pts")).toBeTruthy(); + expect(screen.getByText("96%")).toBeTruthy(); + expect(screen.getByText("+35 pts")).toBeTruthy(); expect(screen.getByText("4 / 4")).toBeTruthy(); expect(screen.getByRole("rowheader", { name: "Accuracy" })).toBeTruthy(); - expect(screen.getByRole("cell", { name: "+45.0 pts" })).toBeTruthy(); + expect(screen.getByRole("cell", { name: "+45 pts" })).toBeTruthy(); + expect(screen.getByText("Single-run result")).toBeTruthy(); + expect(screen.getByText("Agent: Codex (gpt-5.4-mini)")).toBeTruthy(); + expect(screen.getByText("Judge: gpt-5.4")).toBeTruthy(); + expect(screen.getByText("SkillEvaluator 0.1.0")).toBeTruthy(); + expect(screen.getByText("Run: Aug 4, 2026")).toBeTruthy(); expect(screen.getByRole("link", { name: "SkillEvaluator" }).getAttribute("href")).toBe( "https://github.com/NVIDIA/SkillEvaluator", ); @@ -121,4 +127,20 @@ describe("SkillEvaluationReport", () => { expect(screen.queryByText("result.json")).toBeNull(); expect(screen.queryByText("run_config.json")).toBeNull(); }); + + it("gives a skipped no-evals run an actionable reason", () => { + render( + , + ); + + expect(screen.getByText("No evals attached")).toBeTruthy(); + expect(screen.getByText("Add evals/evals.json to evaluate this skill version.")).toBeTruthy(); + }); }); diff --git a/src/components/SkillEvaluationReport.tsx b/src/components/SkillEvaluationReport.tsx index 376a536c..ab27147b 100644 --- a/src/components/SkillEvaluationReport.tsx +++ b/src/components/SkillEvaluationReport.tsx @@ -1,7 +1,7 @@ import { Ban, CircleCheck, CircleX, Clock3 } from "lucide-react"; export type SkillEvaluationRunRecord = { - schemaVersion: 1; + schemaVersion: 2; state: "pending" | "skipped" | "failed" | "completed"; smokeRun: boolean; source: { @@ -22,8 +22,9 @@ export type SkillEvaluationRunRecord = { commit: string; version: string; agent: string; - model: string; - provider: string; + agentModel: string; + judgeModel: string; + judgeProvider: string; environment: string; attempts: number; }; @@ -69,13 +70,42 @@ const STATE_PRESENTATION = { } as const; function formatPercent(value: number) { - return `${(value * 100).toFixed(1)}%`; + return `${Math.round(value * 100)}%`; } function formatPoints(value: number) { - return `${value >= 0 ? "+" : ""}${(value * 100).toFixed(1)} pts`; + return `${value >= 0 ? "+" : ""}${Math.round(value * 100)} pts`; } +function formatAgent(agent: string) { + const labels: Record = { + "claude-code": "Claude Code", + codex: "Codex", + opencode: "OpenCode", + }; + return labels[agent] ?? agent; +} + +function formatRunDate(record: SkillEvaluationRunRecord) { + return new Intl.DateTimeFormat("en-US", { + day: "numeric", + month: "short", + timeZone: "UTC", + year: "numeric", + }).format(new Date(record.timing.finishedAt ?? record.timing.startedAt)); +} + +function formatAttempts(attempts: number) { + return attempts === 1 ? "Single-run result" : `${attempts} attempts per case`; +} + +const REASON_LABELS: Record = { + "ambiguous-evals-config": "Eval configuration is ambiguous", + "eval-source-config-mismatch": "Eval configuration does not match its files", + "no-evals": "No evals attached", + "unsupported-eval-layout": "Eval layout is not supported", +}; + function formatCaseDelta(withSkill: number, withoutSkill: number) { const delta = withSkill - withoutSkill; return `${delta >= 0 ? "+" : ""}${delta} ${Math.abs(delta) === 1 ? "case" : "cases"}`; @@ -113,13 +143,22 @@ export function SkillEvaluationReport({ {record.reason ? (
- {record.reason.code} + {REASON_LABELS[record.reason.code] ?? record.reason.code} {record.reason.message}
) : null} {record.state === "completed" && metrics ? (
+
+ {formatAttempts(record.evaluator.attempts)} + + Agent: {formatAgent(record.evaluator.agent)} ({record.evaluator.agentModel}) + + Judge: {record.evaluator.judgeModel} + SkillEvaluator {record.evaluator.version} + Run: {formatRunDate(record)} +
diff --git a/src/components/SkillEvaluationReportLoader.test.tsx b/src/components/SkillEvaluationReportLoader.test.tsx index a3ec59ac..c46c754b 100644 --- a/src/components/SkillEvaluationReportLoader.test.tsx +++ b/src/components/SkillEvaluationReportLoader.test.tsx @@ -17,7 +17,7 @@ const contentHash = "b".repeat(64); function pendingRecord(recordSource: typeof source, recordContentHash: string, model: string) { return { - schemaVersion: 1, + schemaVersion: 2, state: "pending", smokeRun: true, source: { @@ -36,8 +36,9 @@ function pendingRecord(recordSource: typeof source, recordContentHash: string, m commit: "d".repeat(40), version: "0.1.0", agent: "codex", - model, - provider: "openai", + agentModel: model, + judgeModel: "gpt-5.4", + judgeProvider: "openai", environment: "local", attempts: 1, }, @@ -169,7 +170,7 @@ describe("local SkillEvaluator report loading", () => { render(); await waitFor(() => { - expect(screen.getByText("95.9%")).toBeTruthy(); + expect(screen.getByText("96%")).toBeTruthy(); }); expect(screen.getByRole("rowheader", { name: "Accuracy" })).toBeTruthy(); expect(fetchImpl).toHaveBeenNthCalledWith( diff --git a/src/components/SkillEvaluationReportLoader.tsx b/src/components/SkillEvaluationReportLoader.tsx index f50f2a5b..65a2665c 100644 --- a/src/components/SkillEvaluationReportLoader.tsx +++ b/src/components/SkillEvaluationReportLoader.tsx @@ -136,7 +136,7 @@ function validateRecord( contentHash: string, manifestUrl: string, ): SkillEvaluationRunRecord { - if (!isRecord(value) || value.schemaVersion !== 1) { + if (!isRecord(value) || value.schemaVersion !== 2) { throw new Error("Unsupported evaluation record"); } if (!new Set(["pending", "skipped", "failed", "completed"]).has(String(value.state))) { diff --git a/src/styles.css b/src/styles.css index 781d251e..1a040f3a 100644 --- a/src/styles.css +++ b/src/styles.css @@ -461,12 +461,12 @@ code { } .app-shell:has( - .skill-detail-page, - .plugin-detail-page, - .security-audit-page, - .publisher-profile-route, - .dashboard-route -)::before { + .skill-detail-page, + .plugin-detail-page, + .security-audit-page, + .publisher-profile-route, + .dashboard-route + )::before { position: absolute; top: 0; right: 0; @@ -11046,9 +11046,23 @@ a.skills-sh-security-audit-row:focus-visible { } .skill-evaluation-metrics { + display: grid; + gap: var(--space-3); min-width: 0; } +.skill-evaluation-context { + display: flex; + flex-wrap: wrap; + gap: var(--space-2) var(--space-4); + color: var(--ink-soft); + font-size: var(--fs-xs); +} + +.skill-evaluation-context strong { + color: var(--ink); +} + .skill-evaluation-table-wrap { overflow-x: auto; border: 1px solid color-mix(in srgb, var(--line) 82%, transparent);