fix: reshape skill verify security signals (#2402)

This commit is contained in:
Patrick Erichsen
2026-05-25 18:49:56 -07:00
committed by GitHub
parent cc16d7fbd9
commit 07fed45f42
13 changed files with 314 additions and 113 deletions
+4
View File
@@ -2,6 +2,10 @@
## Unreleased
### Fixes
- API/CLI: report Skill Card verification with flattened skill/version metadata, ClawScan verdict fields at `security.*`, and supporting scanner evidence under `security.signals`.
## 0.18.0 - 2026-05-25
### Changes
+61 -22
View File
@@ -2928,17 +2928,36 @@ describe("httpApiV1 handlers", () => {
llmAnalysis: {
status: "clean",
verdict: "clean",
confidence: "high",
summary: "ClawScan clean.",
checkedAt: 3,
model: "gpt-test",
},
vtAnalysis: {
status: "clean",
verdict: "clean",
analysis: "VirusTotal clean.",
source: "engines",
checkedAt: 4,
},
skillSpectorAnalysis: {
status: "clean",
score: 0,
severity: "LOW",
recommendation: "INSTALL",
issueCount: 0,
issues: [],
scannerVersion: "skillspector-test",
summary: "SkillSpector clean.",
checkedAt: 5,
},
depRegistryAnalysis: {
status: "clean",
results: [],
notFoundPackages: [],
unresolvedPackages: [],
summary: "No dependency issues.",
checkedAt: 4,
checkedAt: 6,
},
capabilityTags: ["dev-tools"],
softDeletedAt: undefined,
@@ -2984,21 +3003,16 @@ describe("httpApiV1 handlers", () => {
ok: true,
decision: "pass",
reasons: [],
skill: {
slug: "demo",
displayName: "Demo",
pageUrl: "https://clawhub.ai/acme/demo",
},
publisher: {
handle: "acme",
displayName: "Acme",
profileUrl: "https://clawhub.ai/user/acme",
},
version: {
version: "1.0.0",
resolvedFrom: "tag",
tag: "stable",
},
slug: "demo",
displayName: "Demo",
pageUrl: "https://clawhub.ai/acme/demo",
publisherHandle: "acme",
publisherDisplayName: "Acme",
publisherProfileUrl: "https://clawhub.ai/user/acme",
version: "1.0.0",
resolvedFrom: "tag",
tag: "stable",
createdAt: 1,
card: {
available: true,
path: "skill-card.md",
@@ -3020,12 +3034,34 @@ describe("httpApiV1 handlers", () => {
security: {
status: "clean",
passed: true,
staticScan: { status: "clean", reasonCodes: [] },
clawScan: { status: "clean", rawStatus: "clean" },
depRegistry: { status: "clean" },
rawStatus: "clean",
verdict: "clean",
confidence: "high",
summary: "ClawScan clean.",
model: "gpt-test",
checkedAt: 3,
signals: {
staticScan: { status: "clean", rawStatus: "clean", reasonCodes: [] },
virusTotal: {
status: "clean",
rawStatus: "clean",
verdict: "clean",
source: "engines",
},
skillSpector: {
status: "clean",
rawStatus: "clean",
score: 0,
recommendation: "INSTALL",
issueCount: 0,
},
dependencyRegistry: { status: "clean" },
},
},
signature: { status: "unsigned" },
});
expect(json.skill).toBeUndefined();
expect(json.publisher).toBeUndefined();
});
it("does not let publisher-supplied skill-card.md satisfy verification", async () => {
@@ -3257,9 +3293,12 @@ describe("httpApiV1 handlers", () => {
expect(json.security).toMatchObject({
status: "clean",
passed: true,
staticScan: { status: "malicious", rawStatus: "malicious" },
clawScan: { status: "clean", verdict: "benign" },
depRegistry: { status: "malicious" },
rawStatus: "clean",
verdict: "benign",
signals: {
staticScan: { status: "malicious", rawStatus: "malicious" },
dependencyRegistry: { status: "malicious", rawStatus: "malicious" },
},
});
});
+79 -65
View File
@@ -452,6 +452,12 @@ function buildVerifySecurity(version: Doc<"skillVersions">) {
const staticStatus = normalizeVerificationStatus(version.staticScan?.status);
const clawRawStatus = version.llmAnalysis?.status ?? null;
const clawStatus = normalizeVerificationStatus(version.llmAnalysis?.verdict ?? clawRawStatus);
const vtStatus = version.vtAnalysis
? normalizeVerificationStatus(version.vtAnalysis.verdict ?? version.vtAnalysis.status)
: null;
const skillSpectorStatus = version.skillSpectorAnalysis
? normalizeVerificationStatus(version.skillSpectorAnalysis.status)
: null;
const depStatus = version.depRegistryAnalysis
? normalizeVerificationStatus(version.depRegistryAnalysis.status)
: null;
@@ -460,50 +466,67 @@ function buildVerifySecurity(version: Doc<"skillVersions">) {
return {
status,
passed: status === "clean",
staticScan: version.staticScan
? {
status: staticStatus,
rawStatus: version.staticScan.status,
reasonCodes: version.staticScan.reasonCodes ?? [],
summary: version.staticScan.summary ?? null,
engineVersion: version.staticScan.engineVersion ?? null,
checkedAt: version.staticScan.checkedAt ?? null,
}
: {
status: "pending" as const,
rawStatus: null,
reasonCodes: [],
summary: null,
engineVersion: null,
checkedAt: null,
},
clawScan: version.llmAnalysis
? {
status: clawStatus,
rawStatus: clawRawStatus,
verdict: version.llmAnalysis.verdict ?? null,
summary: version.llmAnalysis.summary ?? null,
model: version.llmAnalysis.model ?? null,
checkedAt: version.llmAnalysis.checkedAt ?? null,
}
: {
status: "pending" as const,
rawStatus: null,
verdict: null,
summary: null,
model: null,
checkedAt: null,
},
depRegistry: version.depRegistryAnalysis
? {
status: depStatus ?? "pending",
rawStatus: version.depRegistryAnalysis.status,
summary: version.depRegistryAnalysis.summary ?? null,
notFoundPackages: version.depRegistryAnalysis.notFoundPackages ?? [],
unresolvedPackages: version.depRegistryAnalysis.unresolvedPackages ?? [],
checkedAt: version.depRegistryAnalysis.checkedAt ?? null,
}
: null,
rawStatus: clawRawStatus,
verdict: version.llmAnalysis?.verdict ?? null,
confidence: version.llmAnalysis?.confidence ?? null,
summary: version.llmAnalysis?.summary ?? null,
model: version.llmAnalysis?.model ?? null,
checkedAt: version.llmAnalysis?.checkedAt ?? null,
signals: {
staticScan: version.staticScan
? {
status: staticStatus,
rawStatus: version.staticScan.status,
reasonCodes: version.staticScan.reasonCodes ?? [],
summary: version.staticScan.summary ?? null,
engineVersion: version.staticScan.engineVersion ?? null,
checkedAt: version.staticScan.checkedAt ?? null,
}
: {
status: "pending" as const,
rawStatus: null,
reasonCodes: [],
summary: null,
engineVersion: null,
checkedAt: null,
},
virusTotal: version.vtAnalysis
? {
status: vtStatus ?? "pending",
rawStatus: version.vtAnalysis.status,
verdict: version.vtAnalysis.verdict ?? null,
analysis: version.vtAnalysis.analysis ?? null,
source: version.vtAnalysis.source ?? null,
scanner: version.vtAnalysis.scanner ?? null,
engineStats: version.vtAnalysis.engineStats ?? null,
checkedAt: version.vtAnalysis.checkedAt ?? null,
}
: null,
skillSpector: version.skillSpectorAnalysis
? {
status: skillSpectorStatus ?? "pending",
rawStatus: version.skillSpectorAnalysis.status,
score: version.skillSpectorAnalysis.score ?? null,
severity: version.skillSpectorAnalysis.severity ?? null,
recommendation: version.skillSpectorAnalysis.recommendation ?? null,
issueCount: version.skillSpectorAnalysis.issueCount ?? 0,
scannerVersion: version.skillSpectorAnalysis.scannerVersion ?? null,
summary: version.skillSpectorAnalysis.summary ?? null,
error: version.skillSpectorAnalysis.error ?? null,
checkedAt: version.skillSpectorAnalysis.checkedAt ?? null,
}
: null,
dependencyRegistry: version.depRegistryAnalysis
? {
status: depStatus ?? "pending",
rawStatus: version.depRegistryAnalysis.status,
summary: version.depRegistryAnalysis.summary ?? null,
notFoundPackages: version.depRegistryAnalysis.notFoundPackages ?? [],
unresolvedPackages: version.depRegistryAnalysis.unresolvedPackages ?? [],
checkedAt: version.depRegistryAnalysis.checkedAt ?? null,
}
: null,
},
};
}
@@ -1173,27 +1196,18 @@ export async function skillsGetRouterV1Handler(ctx: ActionCtx, request: Request)
ok: reasons.length === 0,
decision: reasons.length === 0 ? "pass" : "fail",
reasons,
skill: {
slug: skillResult.skill.slug,
displayName: skillResult.skill.displayName,
pageUrl: ownerHandle
? `https://clawhub.ai/${ownerHandle}/${skillResult.skill.slug}`
: `https://clawhub.ai/api/v1/skills/${skillResult.skill.slug}`,
},
publisher:
ownerHandle || ownerDisplayName
? {
handle: ownerHandle,
displayName: ownerDisplayName,
profileUrl: ownerHandle ? `https://clawhub.ai/user/${ownerHandle}` : null,
}
: null,
version: {
version: version.version,
resolvedFrom,
tag: tagParam || null,
createdAt: version.createdAt,
},
slug: skillResult.skill.slug,
displayName: skillResult.skill.displayName,
pageUrl: ownerHandle
? `https://clawhub.ai/${ownerHandle}/${skillResult.skill.slug}`
: `https://clawhub.ai/api/v1/skills/${skillResult.skill.slug}`,
publisherHandle: ownerHandle,
publisherDisplayName: ownerDisplayName,
publisherProfileUrl: ownerHandle ? `https://clawhub.ai/user/${ownerHandle}` : null,
version: version.version,
resolvedFrom,
tag: tagParam || null,
createdAt: version.createdAt,
card: generatedCardFile
? {
available: true,
+17
View File
@@ -368,6 +368,23 @@ Notes:
- `moderation` is a current skill-level moderation snapshot derived from the latest version.
- When querying a historical version, check `moderation.matchesRequestedVersion` and `moderation.sourceVersion` before treating `moderation` and `security` as the same version context.
### `GET /api/v1/skills/{slug}/verify`
Returns the Skill Card verification envelope used by `clawhub skill verify`.
Query params:
- `version` (optional): specific version string.
- `tag` (optional): resolve a tagged version (for example `latest`).
Notes:
- `ok` is `true` only when the selected version has a generated Skill Card, is not malware-blocked by moderation, and ClawScan verification is clean.
- Skill identity, publisher identity, and selected version metadata are top-level envelope fields (`slug`, `displayName`, `publisherHandle`, `version`, `resolvedFrom`, `tag`, `createdAt`) so shell automation can read them without unpacking nested wrappers.
- `security` is the top-level ClawScan/security verdict. Automation should key off `ok`, `decision`, `reasons`, and `security.status`.
- `security.signals` contains supporting scanner evidence such as `staticScan`, `virusTotal`, `skillSpector`, and `dependencyRegistry`.
- `provenance` is `server-resolved-github-import` only when ClawHub resolved and stored a GitHub repo/ref/commit/path during publish or import; otherwise it is `unavailable`.
### `GET /api/v1/skills/{slug}/file`
Returns raw text content.
@@ -297,9 +297,16 @@ describe("cmdVerifySkill", () => {
ok: true,
decision: "pass",
reasons: [],
skill: { slug: "demo", displayName: "Demo" },
publisher: { handle: "acme" },
version: { version: "1.2.3", resolvedFrom: "tag", tag: "stable" },
slug: "demo",
displayName: "Demo",
pageUrl: "https://clawhub.ai/acme/demo",
publisherHandle: "acme",
publisherDisplayName: "Acme",
publisherProfileUrl: "https://clawhub.ai/user/acme",
version: "1.2.3",
resolvedFrom: "tag",
tag: "stable",
createdAt: 12,
card: {
available: true,
path: "skill-card.md",
@@ -319,9 +326,15 @@ describe("cmdVerifySkill", () => {
security: {
status: "clean",
passed: true,
staticScan: { status: "clean", reasonCodes: [] },
clawScan: { status: "clean", rawStatus: "clean" },
depRegistry: null,
rawStatus: "clean",
verdict: "clean",
summary: "ClawScan clean.",
signals: {
staticScan: { status: "clean", rawStatus: "clean", reasonCodes: [] },
virusTotal: { status: "clean", rawStatus: "clean", source: "engines" },
skillSpector: { status: "clean", rawStatus: "clean", issueCount: 0 },
dependencyRegistry: null,
},
},
signature: { status: "unsigned" },
};
@@ -343,9 +356,16 @@ describe("cmdVerifySkill", () => {
ok: false,
decision: "fail",
reasons: ["card.missing", "security.status_not_clean"],
skill: { slug: "demo", displayName: "Demo" },
publisher: null,
version: { version: "1.2.3", resolvedFrom: "latest", tag: null },
slug: "demo",
displayName: "Demo",
pageUrl: "https://clawhub.ai/acme/demo",
publisherHandle: "acme",
publisherDisplayName: "Acme",
publisherProfileUrl: "https://clawhub.ai/user/acme",
version: "1.2.3",
resolvedFrom: "latest",
tag: null,
createdAt: 12,
card: { available: false },
artifact: { sourceFingerprint: "source-fingerprint", bundleFingerprints: [], files: [] },
provenance: { source: "unavailable" },
@@ -366,9 +386,16 @@ describe("cmdVerifySkill", () => {
ok: true,
decision: "pass",
reasons: [],
skill: { slug: "demo", displayName: "Demo" },
publisher: { handle: "acme" },
version: { version: "1.2.3", resolvedFrom: "latest", tag: null },
slug: "demo",
displayName: "Demo",
pageUrl: "https://clawhub.ai/acme/demo",
publisherHandle: "acme",
publisherDisplayName: "Acme",
publisherProfileUrl: "https://clawhub.ai/user/acme",
version: "1.2.3",
resolvedFrom: "latest",
tag: null,
createdAt: 12,
card: {
available: true,
path: "skill-card.md",
+36 -1
View File
@@ -2,7 +2,11 @@
import { describe, expect, it } from "vitest";
import { parseArk } from "./ark";
import { ApiV1SearchResponseSchema, ClawdisSkillMetadataSchema } from "./schemas";
import {
ApiV1SearchResponseSchema,
ApiV1SkillVerifyResponseSchema,
ClawdisSkillMetadataSchema,
} from "./schemas";
describe("packages/clawhub skill metadata schema", () => {
it("preserves optional env var declarations", () => {
@@ -50,4 +54,35 @@ describe("packages/clawhub skill metadata schema", () => {
expect(parsed.results[0]?.ownerHandle).toBe("openclaw");
expect(parsed.results[0]?.owner?.displayName).toBe("OpenClaw");
});
it("parses flattened skill verification envelopes", () => {
const parsed = parseArk(
ApiV1SkillVerifyResponseSchema,
{
schema: "clawhub.skill.verify.v1",
ok: true,
decision: "pass",
reasons: [],
slug: "demo",
displayName: "Demo",
pageUrl: "https://clawhub.ai/openclaw/demo",
publisherHandle: "openclaw",
publisherDisplayName: "OpenClaw",
publisherProfileUrl: "https://clawhub.ai/user/openclaw",
version: "1.0.0",
resolvedFrom: "latest",
tag: null,
createdAt: 1,
card: { available: true },
artifact: { sourceFingerprint: "source", bundleFingerprints: [], files: [] },
provenance: { source: "unavailable" },
security: { status: "clean", passed: true },
signature: { status: "unsigned" },
},
"Verify",
);
expect(parsed.slug).toBe("demo");
expect(parsed.version).toBe("1.0.0");
});
});
+10 -3
View File
@@ -448,9 +448,16 @@ export const ApiV1SkillVerifyResponseSchema = type({
ok: "boolean",
decision: '"pass"|"fail"',
reasons: "string[]",
skill: "unknown",
publisher: "unknown",
version: "unknown",
slug: "string",
displayName: "string",
pageUrl: "string",
publisherHandle: "string|null",
publisherDisplayName: "string|null",
publisherProfileUrl: "string|null",
version: "string",
resolvedFrom: '"latest"|"version"|"tag"',
tag: "string|null",
createdAt: "number",
card: "unknown",
artifact: "unknown",
provenance: "unknown",
+10 -3
View File
@@ -419,9 +419,16 @@ export declare const ApiV1SkillVerifyResponseSchema: import("arktype/internal/va
ok: boolean;
decision: "pass" | "fail";
reasons: string[];
skill: unknown;
publisher: unknown;
version: unknown;
slug: string;
displayName: string;
pageUrl: string;
publisherHandle: string | null;
publisherDisplayName: string | null;
publisherProfileUrl: string | null;
version: string;
resolvedFrom: "version" | "tag" | "latest";
tag: string | null;
createdAt: number;
card: unknown;
artifact: unknown;
provenance: unknown;
+10 -3
View File
@@ -373,9 +373,16 @@ export const ApiV1SkillVerifyResponseSchema = type({
ok: "boolean",
decision: '"pass"|"fail"',
reasons: "string[]",
skill: "unknown",
publisher: "unknown",
version: "unknown",
slug: "string",
displayName: "string",
pageUrl: "string",
publisherHandle: "string|null",
publisherDisplayName: "string|null",
publisherProfileUrl: "string|null",
version: "string",
resolvedFrom: '"latest"|"version"|"tag"',
tag: "string|null",
createdAt: "number",
card: "unknown",
artifact: "unknown",
provenance: "unknown",
File diff suppressed because one or more lines are too long
+32
View File
@@ -8,6 +8,7 @@ import { getPackageScopeOwnerMismatch, inferPackageNameScope } from "./packages"
import {
ApiSearchResponseSchema,
ApiV1SearchResponseSchema,
ApiV1SkillVerifyResponseSchema,
CliPublishRequestSchema,
CliSkillDeleteRequestSchema,
LockfileSchema,
@@ -216,6 +217,37 @@ describe("clawhub-schema", () => {
expect(parsed.results[0]?.owner?.displayName).toBe("OpenClaw");
});
it("parses flattened skill verification envelopes", () => {
const parsed = parseArk(
ApiV1SkillVerifyResponseSchema,
{
schema: "clawhub.skill.verify.v1",
ok: true,
decision: "pass",
reasons: [],
slug: "demo",
displayName: "Demo",
pageUrl: "https://clawhub.ai/openclaw/demo",
publisherHandle: "openclaw",
publisherDisplayName: "OpenClaw",
publisherProfileUrl: "https://clawhub.ai/user/openclaw",
version: "1.0.0",
resolvedFrom: "latest",
tag: null,
createdAt: 1,
card: { available: true },
artifact: { sourceFingerprint: "source", bundleFingerprints: [], files: [] },
provenance: { source: "unavailable" },
security: { status: "clean", passed: true },
signature: { status: "unsigned" },
},
"Verify",
);
expect(parsed.slug).toBe("demo");
expect(parsed.version).toBe("1.0.0");
});
it("parses delete request payload", () => {
expect(
parseArk(CliSkillDeleteRequestSchema, { slug: "demo", reason: "legal hold" }, "Delete"),
+10 -3
View File
@@ -442,9 +442,16 @@ export const ApiV1SkillVerifyResponseSchema = type({
ok: "boolean",
decision: '"pass"|"fail"',
reasons: "string[]",
skill: "unknown",
publisher: "unknown",
version: "unknown",
slug: "string",
displayName: "string",
pageUrl: "string",
publisherHandle: "string|null",
publisherDisplayName: "string|null",
publisherProfileUrl: "string|null",
version: "string",
resolvedFrom: '"latest"|"version"|"tag"',
tag: "string|null",
createdAt: "number",
card: "unknown",
artifact: "unknown",
provenance: "unknown",
+5
View File
@@ -114,6 +114,11 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
hosted LLM call. Publishes enqueue a scan job that waits at most 10 minutes
for VirusTotal telemetry, then Codex reviews the materialized artifact
workspace with static and VT signals as context.
- The Skill Card verification envelope exposes ClawScan as the top-level
`security` verdict for install automation, with deterministic and third-party
scanner evidence grouped under `security.signals`. Clients should key install
decisions off `ok`, `decision`, `reasons`, and `security.status` instead of
re-deriving trust from individual signal payloads.
- ClawScan verdicts treat purpose-aligned notes as user guidance, not a
suspicious verdict. Medium-only material concerns are visible
`flagged.review` guidance and must not set `isSuspicious`; high or critical