admin/clawhub
1
0
Fork 0
mirror of https://github.com/openclaw/clawhub.git synced 2026-08-14 00:47:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: clarify skill evaluation provenance

Browse Source
This commit is contained in:
Patrick Erichsen
2026-08-12 17:29:56 -07:00
parent 1f33b1a1a5
commit 0d98e6cc5e
8 changed files with 149 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/prototypes/nvidia-skill-evaluation.test.ts
+9 -4
View File
@@ -131,7 +131,8 @@ describe("SkillEvaluator eval discovery", () => {
await expect(discoverSkillEvals(skillPath)).resolves.toEqual({
status: "skipped",
reason: "no-evals",
message: "No SkillEvaluator dataset or native Harbor tasks were found in evals/.",
message:
"No SkillEvaluator dataset or native Harbor tasks were found. Add evals/evals.json or native Harbor tasks under evals/.",
});
});
@@ -263,13 +264,15 @@ describe("canonical SkillEvaluator execution", () => {
).toThrow("Unsupported SkillEvaluator environment variable: GITHUB_TOKEN");
});
it("pins the evaluator and Codex model for a one-attempt Tier 3 smoke run", () => {
it("pins distinct judge and Codex models for a labeled Tier 3 smoke run", () => {
expect(
buildTier3EvaluateInvocation({
evaluatorRepoPath: "/tmp/SkillEvaluator",
skillDirectory: "/tmp/skills/skills/doca-dpa",
resultsDirectory: "/tmp/results",
model: "gpt-5.4-mini",
agentModel: "gpt-5.4-mini",
judgeModel: "gpt-5.4",
attempts: 1,
}),
).toEqual({
command: [
@@ -277,6 +280,8 @@ describe("canonical SkillEvaluator execution", () => {
"run",
"--project",
"/tmp/SkillEvaluator",
"--extra",
"tier3",
"skillevaluator",
"tier3",
"evaluate",
@@ -299,7 +304,7 @@ describe("canonical SkillEvaluator execution", () => {
"/tmp/results",
],
environment: {
SKILL_EVAL_LLM_MODEL: "gpt-5.4-mini",
SKILL_EVAL_LLM_MODEL: "gpt-5.4",
SKILL_EVAL_LLM_PROVIDER: "openai",
},
});
scripts/prototypes/nvidia-skill-evaluation.ts
+34 -14
View File
@@ -130,7 +130,8 @@ export async function discoverSkillEvals(skillDirectory: string): Promise<SkillE
return {
status: "skipped",
reason: "no-evals",
message: "No SkillEvaluator dataset or native Harbor tasks were found in evals/.",
message:
"No SkillEvaluator dataset or native Harbor tasks were found. Add evals/evals.json or native Harbor tasks under evals/.",
};
}
@@ -205,7 +206,8 @@ export async function discoverSkillEvals(skillDirectory: string): Promise<SkillE
return {
status: "skipped",
reason: "no-evals",
message: "No SkillEvaluator dataset or native Harbor tasks were found in evals/.",
message:
"No SkillEvaluator dataset or native Harbor tasks were found. Add evals/evals.json or native Harbor tasks under evals/.",
};
}
@@ -239,12 +241,16 @@ export function buildTier3EvaluateInvocation({
evaluatorRepoPath,
skillDirectory,
resultsDirectory,
model,
agentModel,
judgeModel,
attempts,
}: {
evaluatorRepoPath: string;
skillDirectory: string;
resultsDirectory: string;
model: string;
agentModel: string;
judgeModel: string;
attempts: number;
}) {
return {
command: [
@@ -252,6 +258,8 @@ export function buildTier3EvaluateInvocation({
"run",
"--project",
evaluatorRepoPath,
"--extra",
"tier3",
"skillevaluator",
"tier3",
"evaluate",
@@ -263,9 +271,9 @@ export function buildTier3EvaluateInvocation({
"--env-mode",
"local",
"--agent-model",
`codex=${model}`,
`codex=${agentModel}`,
"--n-attempts",
"1",
String(attempts),
"--n-concurrent",
"1",
"--max-agents",
@@ -276,7 +284,7 @@ export function buildTier3EvaluateInvocation({
resultsDirectory,
],
environment: {
SKILL_EVAL_LLM_MODEL: model,
SKILL_EVAL_LLM_MODEL: judgeModel,
SKILL_EVAL_LLM_PROVIDER: "openai",
},
};
@@ -759,7 +767,9 @@ async function main() {
"skill-path": { type: "string" },
"source-repo": { type: "string", default: "nvidia/skills" },
"output-dir": { type: "string", default: ".artifacts/skill-evaluator-demo" },
model: { type: "string", default: "gpt-5.4-mini" },
"agent-model": { type: "string", default: "gpt-5.4-mini" },
"judge-model": { type: "string", default: "gpt-5.4" },
"n-attempts": { type: "string", default: "1" },
rerun: { type: "boolean", default: false },
},
strict: true,
@@ -779,7 +789,12 @@ async function main() {
}
const outputDirectory = resolve(values["output-dir"]);
const webRoot = resolve(LOCAL_EVALUATION_WEB_ROOT);
const model = values.model;
const agentModel = values["agent-model"];
const judgeModel = values["judge-model"];
const attempts = Number(values["n-attempts"]);
if (!Number.isSafeInteger(attempts) || attempts < 1) {
throw new Error("--n-attempts must be a positive integer");
}
const sourceCommit = await capture(["git", "-C", checkoutPath, "rev-parse", "HEAD"]);
const evaluatorCommit = await capture(["git", "-C", evaluatorRepoPath, "rev-parse", "HEAD"]);
await Promise.all([
@@ -839,7 +854,7 @@ async function main() {
const skillDirectory = resolve(checkoutPath, sourcePath);
const discovery = plan.action === "run" ? plan.evals : await discoverSkillEvals(skillDirectory);
const recordBase = {
schemaVersion: 1 as const,
schemaVersion: 2 as const,
smokeRun: true,
source: {
repository: sourceRepo,
@@ -864,10 +879,11 @@ async function main() {
commit: evaluatorCommit,
version: evaluatorVersion.replace(/^skillevaluator(?:,\s*version)?\s+/i, ""),
agent: "codex",
model,
provider: "openai",
agentModel,
judgeModel,
judgeProvider: "openai",
environment: "local",
attempts: 1,
attempts,
},
timing: { startedAt: now.toISOString() },
};
@@ -960,7 +976,9 @@ async function main() {
evaluatorRepoPath,
skillDirectory,
resultsDirectory,
model,
agentModel,
judgeModel,
attempts,
});
const pendingRecord: SkillEvaluationRunRecord = { ...recordBase, state: "pending" };
await Promise.all([
@@ -981,6 +999,8 @@ async function main() {
"run",
"--project",
evaluatorRepoPath,
"--extra",
"tier3",
"skillevaluator",
"tier3",
"validate",
src/components/SkillDetailPage.tsx
+7 -10
View File
@@ -273,24 +273,21 @@ export function SkillDetailPage({
const evaluationDemoCommit = localEvaluationDemoCommit(searchStr);
const evaluationSourceRepo =
githubBackedFields?.githubSourceRepo ?? githubBackedFields?.githubCurrentRepo;
const evaluationCurrentCommit = githubBackedFields?.githubCurrentCommit;
const evaluationSkillPath = githubBackedFields?.githubPath;
const localEvaluationSource = useMemo(
() =>
import.meta.env.DEV &&
evaluationSourceRepo?.toLowerCase() === "nvidia/skills" &&
githubBackedFields.githubCurrentCommit &&
githubBackedFields.githubPath
evaluationCurrentCommit &&
evaluationSkillPath
? {
repository: "nvidia/skills",
commit: evaluationDemoCommit ?? githubBackedFields.githubCurrentCommit,
path: githubBackedFields.githubPath,
commit: evaluationDemoCommit ?? evaluationCurrentCommit,
path: evaluationSkillPath,
}
: null,
[
githubBackedFields?.githubCurrentCommit,
githubBackedFields?.githubPath,
evaluationSourceRepo,
evaluationDemoCommit,
],
[evaluationCurrentCommit, evaluationDemoCommit, evaluationSkillPath, evaluationSourceRepo],
);
const modInfo = result?.moderationInfo ?? null;
const relatedCategory = useMemo(() => (skill ? getSkillCategoryForSkill(skill) : null), [skill]);
src/components/SkillEvaluationReport.test.tsx
+28 -6
View File
@@ -13,7 +13,7 @@ function record(
overrides: Partial<SkillEvaluationRunRecord> = {},
): SkillEvaluationRunRecord {
return {
schemaVersion: 1,
schemaVersion: 2,
state,
smokeRun: true,
source: {
@@ -34,8 +34,9 @@ function record(
commit: "b".repeat(40),
version: "0.1.0",
agent: "codex",
model: "gpt-5.4-mini",
provider: "openai",
agentModel: "gpt-5.4-mini",
judgeModel: "gpt-5.4",
judgeProvider: "openai",
environment: "local",
attempts: 1,
},
@@ -106,11 +107,16 @@ describe("SkillEvaluationReport", () => {
/>,
);
expect(screen.getByText("95.9%")).toBeTruthy();
expect(screen.getByText("+35.3 pts")).toBeTruthy();
expect(screen.getByText("96%")).toBeTruthy();
expect(screen.getByText("+35 pts")).toBeTruthy();
expect(screen.getByText("4 / 4")).toBeTruthy();
expect(screen.getByRole("rowheader", { name: "Accuracy" })).toBeTruthy();
expect(screen.getByRole("cell", { name: "+45.0 pts" })).toBeTruthy();
expect(screen.getByRole("cell", { name: "+45 pts" })).toBeTruthy();
expect(screen.getByText("Single-run result")).toBeTruthy();
expect(screen.getByText("Agent: Codex (gpt-5.4-mini)")).toBeTruthy();
expect(screen.getByText("Judge: gpt-5.4")).toBeTruthy();
expect(screen.getByText("SkillEvaluator 0.1.0")).toBeTruthy();
expect(screen.getByText("Run: Aug 4, 2026")).toBeTruthy();
expect(screen.getByRole("link", { name: "SkillEvaluator" }).getAttribute("href")).toBe(
"https://github.com/NVIDIA/SkillEvaluator",
);
@@ -121,4 +127,20 @@ describe("SkillEvaluationReport", () => {
expect(screen.queryByText("result.json")).toBeNull();
expect(screen.queryByText("run_config.json")).toBeNull();
});
it("gives a skipped no-evals run an actionable reason", () => {
render(
<SkillEvaluationReport
record={record("skipped", {
reason: {
code: "no-evals",
message: "Add evals/evals.json to evaluate this skill version.",
},
})}
/>,
);
expect(screen.getByText("No evals attached")).toBeTruthy();
expect(screen.getByText("Add evals/evals.json to evaluate this skill version.")).toBeTruthy();
});
});
src/components/SkillEvaluationReport.tsx
+45 -6
View File
@@ -1,7 +1,7 @@
import { Ban, CircleCheck, CircleX, Clock3 } from "lucide-react";
export type SkillEvaluationRunRecord = {
schemaVersion: 1;
schemaVersion: 2;
state: "pending" | "skipped" | "failed" | "completed";
smokeRun: boolean;
source: {
@@ -22,8 +22,9 @@ export type SkillEvaluationRunRecord = {
commit: string;
version: string;
agent: string;
model: string;
provider: string;
agentModel: string;
judgeModel: string;
judgeProvider: string;
environment: string;
attempts: number;
};
@@ -69,13 +70,42 @@ const STATE_PRESENTATION = {
} as const;
function formatPercent(value: number) {
return `${(value * 100).toFixed(1)}%`;
return `${Math.round(value * 100)}%`;
}
function formatPoints(value: number) {
return `${value >= 0 ? "+" : ""}${(value * 100).toFixed(1)} pts`;
return `${value >= 0 ? "+" : ""}${Math.round(value * 100)} pts`;
}
function formatAgent(agent: string) {
const labels: Record<string, string> = {
"claude-code": "Claude Code",
codex: "Codex",
opencode: "OpenCode",
};
return labels[agent] ?? agent;
}
function formatRunDate(record: SkillEvaluationRunRecord) {
return new Intl.DateTimeFormat("en-US", {
day: "numeric",
month: "short",
timeZone: "UTC",
year: "numeric",
}).format(new Date(record.timing.finishedAt ?? record.timing.startedAt));
}
function formatAttempts(attempts: number) {
return attempts === 1 ? "Single-run result" : `${attempts} attempts per case`;
}
const REASON_LABELS: Record<string, string> = {
"ambiguous-evals-config": "Eval configuration is ambiguous",
"eval-source-config-mismatch": "Eval configuration does not match its files",
"no-evals": "No evals attached",
"unsupported-eval-layout": "Eval layout is not supported",
};
function formatCaseDelta(withSkill: number, withoutSkill: number) {
const delta = withSkill - withoutSkill;
return `${delta >= 0 ? "+" : ""}${delta} ${Math.abs(delta) === 1 ? "case" : "cases"}`;
@@ -113,13 +143,22 @@ export function SkillEvaluationReport({
{record.reason ? (
<div className={`skill-evaluation-message is-${record.state}`}>
<strong>{record.reason.code}</strong>
<strong>{REASON_LABELS[record.reason.code] ?? record.reason.code}</strong>
<span>{record.reason.message}</span>
</div>
) : null}
{record.state === "completed" && metrics ? (
<section className="skill-evaluation-metrics" aria-label="SkillEvaluator metrics">
<div className="skill-evaluation-context" aria-label="Evaluation context">
<strong>{formatAttempts(record.evaluator.attempts)}</strong>
<span>
Agent: {formatAgent(record.evaluator.agent)} ({record.evaluator.agentModel})
</span>
<span>Judge: {record.evaluator.judgeModel}</span>
<span>SkillEvaluator {record.evaluator.version}</span>
<span>Run: {formatRunDate(record)}</span>
</div>
<div className="skill-evaluation-table-wrap">
<table className="skill-evaluation-table">
<caption>
src/components/SkillEvaluationReportLoader.test.tsx
+5 -4
View File
@@ -17,7 +17,7 @@ const contentHash = "b".repeat(64);
function pendingRecord(recordSource: typeof source, recordContentHash: string, model: string) {
return {
schemaVersion: 1,
schemaVersion: 2,
state: "pending",
smokeRun: true,
source: {
@@ -36,8 +36,9 @@ function pendingRecord(recordSource: typeof source, recordContentHash: string, m
commit: "d".repeat(40),
version: "0.1.0",
agent: "codex",
model,
provider: "openai",
agentModel: model,
judgeModel: "gpt-5.4",
judgeProvider: "openai",
environment: "local",
attempts: 1,
},
@@ -169,7 +170,7 @@ describe("local SkillEvaluator report loading", () => {
render(<SkillEvaluationReportLoader source={source} fetchImpl={fetchImpl} />);
await waitFor(() => {
expect(screen.getByText("95.9%")).toBeTruthy();
expect(screen.getByText("96%")).toBeTruthy();
});
expect(screen.getByRole("rowheader", { name: "Accuracy" })).toBeTruthy();
expect(fetchImpl).toHaveBeenNthCalledWith(
src/components/SkillEvaluationReportLoader.tsx
+1 -1
View File
@@ -136,7 +136,7 @@ function validateRecord(
contentHash: string,
manifestUrl: string,
): SkillEvaluationRunRecord {
if (!isRecord(value) || value.schemaVersion !== 1) {
if (!isRecord(value) || value.schemaVersion !== 2) {
throw new Error("Unsupported evaluation record");
}
if (!new Set(["pending", "skipped", "failed", "completed"]).has(String(value.state))) {
src/styles.css
+20 -6
View File
@@ -461,12 +461,12 @@ code {
}
.app-shell:has(
.skill-detail-page,
.plugin-detail-page,
.security-audit-page,
.publisher-profile-route,
.dashboard-route
)::before {
.skill-detail-page,
.plugin-detail-page,
.security-audit-page,
.publisher-profile-route,
.dashboard-route
)::before {
position: absolute;
top: 0;
right: 0;
@@ -11046,9 +11046,23 @@ a.skills-sh-security-audit-row:focus-visible {
}
.skill-evaluation-metrics {
display: grid;
gap: var(--space-3);
min-width: 0;
}
.skill-evaluation-context {
display: flex;
flex-wrap: wrap;
gap: var(--space-2) var(--space-4);
color: var(--ink-soft);
font-size: var(--fs-xs);
}
.skill-evaluation-context strong {
color: var(--ink);
}
.skill-evaluation-table-wrap {
overflow-x: auto;
border: 1px solid color-mix(in srgb, var(--line) 82%, transparent);