admin/clawhub
1
0
Fork 0
mirror of https://github.com/openclaw/clawhub.git synced 2026-08-14 00:47:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: export ClawScan findings sidecar (#2201)

Browse Source 
* feat: export clawscan findings sidecar

* fix: canonicalize clawscan sidecar fields
This commit is contained in:
Patrick Erichsen
2026-05-13 09:45:33 -07:00
committed by GitHub
parent 5c98c7e3e1
commit a1666bb1e6
8 changed files with 244 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
convex/securityDataset.ts
+1
View File
@@ -345,6 +345,7 @@ function normalizeLlmAnalysis(analysis: StoredLlmAnalysis) {
dimensions: analysis.dimensions ?? null,
guidance: analysis.guidance ?? null,
findings: analysis.findings ?? null,
agenticRiskFindings: analysis.agenticRiskFindings ?? [],
model: analysis.model ?? null,
checkedAt: analysis.checkedAt,
};
scripts/security-dataset/convexExport.test.ts
+28
View File
@@ -46,6 +46,23 @@ describe("Convex export dataset ingestion", () => {
verdict: "suspicious",
confidence: "high",
summary: "asks for secrets",
agenticRiskFindings: [
{
categoryId: "ASI04",
categoryLabel: "Tool and permission overreach",
riskBucket: "permission_boundary",
status: "concern",
severity: "high",
confidence: "high",
evidence: {
path: "SKILL.md",
snippet: "Use token=supersecret123",
explanation: "References sensitive token handling.",
},
userImpact: "Could expose credentials.",
recommendation: "Require least-privilege credentials.",
},
],
model: "gpt-test",
checkedAt: 3,
},
@@ -119,6 +136,17 @@ describe("Convex export dataset ingestion", () => {
llmAnalysis: { model: "gpt-test" },
skillMdContentRedacted: "Use this skill safely. [REDACTED_SECRET]",
});
expect(rows[1]?.llmAnalysis?.agenticRiskFindings).toMatchObject([
{
categoryId: "ASI04",
riskBucket: "permission_boundary",
status: "concern",
severity: "high",
evidence: {
path: "SKILL.md",
},
},
]);
});
it("reads table JSONL files from a Convex export zip", async () => {
scripts/security-dataset/convexExport.ts
+44
View File
@@ -257,11 +257,55 @@ function llmAnalysisFromExport(value: unknown): LlmAnalysisInput | null {
dimensions: llmDimensionsFromExport(value.dimensions),
guidance: stringOrNull(value.guidance),
findings: stringOrNull(value.findings),
agenticRiskFindings: llmAgenticRiskFindingsFromExport(value.agenticRiskFindings),
model: stringOrNull(value.model),
checkedAt: numberValue(value.checkedAt, "llmAnalysis.checkedAt"),
};
}
function llmAgenticRiskFindingsFromExport(value: unknown): LlmAnalysisInput["agenticRiskFindings"] {
if (!Array.isArray(value)) return [];
return value.flatMap((finding) => {
if (!isRecord(finding)) return [];
const riskBucket = finding.riskBucket;
if (
riskBucket !== "abnormal_behavior_control" &&
riskBucket !== "permission_boundary" &&
riskBucket !== "sensitive_data_protection"
) {
return [];
}
const status = finding.status;
if (status !== "none" && status !== "note" && status !== "concern") return [];
const confidence = finding.confidence;
if (confidence !== "high" && confidence !== "medium" && confidence !== "low") return [];
return [
{
categoryId: stringOrNull(finding.categoryId) ?? "",
categoryLabel: stringOrNull(finding.categoryLabel) ?? "",
riskBucket,
status,
severity: stringOrNull(finding.severity) ?? "none",
confidence,
evidence: llmRiskEvidenceFromExport(finding.evidence),
userImpact: stringOrNull(finding.userImpact) ?? "",
recommendation: stringOrNull(finding.recommendation) ?? "",
},
];
});
}
function llmRiskEvidenceFromExport(
value: unknown,
): LlmAnalysisInput["agenticRiskFindings"][number]["evidence"] {
if (!isRecord(value)) return null;
return {
path: stringOrNull(value.path) ?? "",
snippet: stringOrNull(value.snippet) ?? "",
explanation: stringOrNull(value.explanation) ?? "",
};
}
function llmDimensionsFromExport(value: unknown): LlmAnalysisInput["dimensions"] {
if (!Array.isArray(value)) return null;
return value.flatMap((dimension) => {
scripts/security-dataset/export-snapshot.ts
+10 -1
View File
@@ -73,6 +73,7 @@ type SnapshotState = {
artifacts: number;
scanResults: number;
staticFindings: number;
clawScanFindings: number;
labels: number;
splits: number;
};
@@ -84,6 +85,7 @@ type SnapshotWriters = {
artifacts: WriteStream;
scanResults: WriteStream;
staticFindings: WriteStream;
clawScanFindings: WriteStream;
labels: WriteStream;
splits: WriteStream;
};
@@ -368,12 +370,13 @@ function buildManifest(input: {
artifacts: state.rowCounts.artifacts,
scanResults: state.rowCounts.scanResults,
staticFindings: state.rowCounts.staticFindings,
clawScanFindings: state.rowCounts.clawScanFindings,
labels: state.rowCounts.labels,
splits: state.rowCounts.splits,
},
scannerVersions: Array.from(state.scannerVersions).sort(),
modelNames: Array.from(state.modelNames).sort(),
redactionPolicyVersion: "public-signals-v1",
redactionPolicyVersion: "public-signals-v2",
sourceTables: ["skillVersions", "packageReleases"],
timeWindow: options.timeWindow,
});
@@ -396,6 +399,7 @@ function createSnapshotState(): SnapshotState {
artifacts: 0,
scanResults: 0,
staticFindings: 0,
clawScanFindings: 0,
labels: 0,
splits: 0,
},
@@ -414,6 +418,7 @@ async function processArtifactInputs(input: {
state.rowCounts.artifacts += rows.artifacts.length;
state.rowCounts.scanResults += rows.scanResults.length;
state.rowCounts.staticFindings += rows.staticFindings.length;
state.rowCounts.clawScanFindings += rows.clawScanFindings.length;
state.rowCounts.labels += rows.labels.length;
state.rowCounts.splits += rows.splits.length;
for (const row of rows.scanResults) {
@@ -433,6 +438,9 @@ async function openSnapshotWriters(snapshotDir: string): Promise<SnapshotWriters
staticFindings: createWriteStream(join(snapshotDir, "static_findings.jsonl"), {
encoding: "utf8",
}),
clawScanFindings: createWriteStream(join(snapshotDir, "clawscan_findings.jsonl"), {
encoding: "utf8",
}),
labels: createWriteStream(join(snapshotDir, "labels.jsonl"), { encoding: "utf8" }),
splits: createWriteStream(join(snapshotDir, "splits.jsonl"), { encoding: "utf8" }),
};
@@ -451,6 +459,7 @@ async function writeNormalizedRows(writers: SnapshotWriters, rows: NormalizedDat
await writeJsonlRows(writers.artifacts, rows.artifacts);
await writeJsonlRows(writers.scanResults, rows.scanResults);
await writeJsonlRows(writers.staticFindings, rows.staticFindings);
await writeJsonlRows(writers.clawScanFindings, rows.clawScanFindings);
await writeJsonlRows(writers.labels, rows.labels);
await writeJsonlRows(writers.splits, rows.splits);
}
scripts/security-dataset/manifest.test.ts
+4
View File
@@ -18,6 +18,7 @@ describe("security dataset manifest", () => {
artifacts: 1,
scanResults: 2,
staticFindings: 3,
clawScanFindings: 5,
labels: 4,
splits: 1,
},
@@ -37,6 +38,9 @@ describe("security dataset manifest", () => {
created_at_gte: 1777507200000,
created_at_lt: 1780185600000,
},
row_counts: {
clawscan_findings: 5,
},
});
});
scripts/security-dataset/manifest.ts
+2
View File
@@ -13,6 +13,7 @@ export type SnapshotManifestInput = {
artifacts: number;
scanResults: number;
staticFindings: number;
clawScanFindings: number;
labels: number;
splits: number;
};
@@ -43,6 +44,7 @@ export function buildSecurityDatasetManifest(input: SnapshotManifestInput) {
artifacts: input.rowCounts.artifacts,
scan_results: input.rowCounts.scanResults,
static_findings: input.rowCounts.staticFindings,
clawscan_findings: input.rowCounts.clawScanFindings,
labels: input.rowCounts.labels,
splits: input.rowCounts.splits,
},
scripts/security-dataset/normalize.test.ts
+55
View File
@@ -72,6 +72,49 @@ const baseArtifact: ArtifactExportInput = {
dimensions: null,
guidance: null,
findings: null,
agenticRiskFindings: [
{
categoryId: "ASI04",
categoryLabel: "Ignore this label token=supersecret123",
riskBucket: "permission_boundary",
status: "note",
severity: "MEDIUM",
confidence: "high",
evidence: {
path: "SKILL.md",
snippet: "Use TOKEN=ghp_abcdefghijklmnopqrstuvwxyz1234567890 for setup",
explanation: "The skill documents token use.",
},
userImpact: "Users should understand the token scope before install.",
recommendation: "Use a narrowly scoped token.",
},
{
categoryId: "UNKNOWN token=supersecret123",
categoryLabel: "Unknown category token=supersecret123",
riskBucket: "permission_boundary",
status: "note",
severity: "critical",
confidence: "high",
evidence: {
path: "SKILL.md",
snippet: "Unknown category should not export.",
explanation: "Unknown categories are not part of the public sidecar contract.",
},
userImpact: "Should not export.",
recommendation: "Should not export.",
},
{
categoryId: "ASI05",
categoryLabel: "Sensitive data protection",
riskBucket: "sensitive_data_protection",
status: "none",
severity: "none",
confidence: "high",
evidence: null,
userImpact: "",
recommendation: "",
},
],
model: "test-model",
checkedAt: Date.UTC(2026, 3, 29),
},
@@ -108,6 +151,18 @@ describe("security dataset normalizer", () => {
line_bucket: "21-50",
});
expect(rows.staticFindings[0]?.evidence_redacted).toContain("[REDACTED_SECRET]");
expect(rows.clawScanFindings).toHaveLength(1);
expect(rows.clawScanFindings[0]).toMatchObject({
category_id: "ASI04",
category_label: "Agentic Supply Chain Vulnerabilities",
risk_bucket: "permission_boundary",
status: "note",
severity: "medium",
confidence: "high",
evidence_path_hash: hashString("SKILL.md"),
evidence_file_ext: ".md",
});
expect(rows.clawScanFindings[0]?.evidence_snippet_redacted).toContain("[REDACTED_SECRET]");
expect(rows.labels.find((row) => row.label_source === "moderation_consensus")).toMatchObject({
label: "malicious",
label_confidence: "derived_consensus",
scripts/security-dataset/normalize.ts
+100 -1
View File
@@ -1,9 +1,16 @@
import { createHash } from "node:crypto";
import { AGENTIC_RISK_CATEGORIES } from "../../convex/lib/securityPrompt.ts";
export type SourceKind = "skill" | "package";
export type DatasetLabel = "clean" | "suspicious" | "malicious" | "unknown";
export type DatasetSplit = "train" | "validation" | "test" | "eval_holdout";
export type ScannerName = "static" | "virustotal" | "llm" | "moderation_consensus";
export type ClawScanRiskBucket =
| "abnormal_behavior_control"
| "permission_boundary"
| "sensitive_data_protection";
export type ClawScanFindingStatus = "none" | "note" | "concern";
export type ClawScanSeverity = "none" | "info" | "low" | "medium" | "high" | "critical";
export type ExportFileInput = {
path: string;
@@ -56,6 +63,21 @@ export type LlmAnalysisInput = {
}> | null;
guidance: string | null;
findings: string | null;
agenticRiskFindings: Array<{
categoryId: string;
categoryLabel: string;
riskBucket: ClawScanRiskBucket;
status: ClawScanFindingStatus;
severity: string;
confidence: "high" | "medium" | "low";
evidence: {
path: string;
snippet: string;
explanation: string;
} | null;
userImpact: string;
recommendation: string;
}>;
model: string | null;
checkedAt: number;
};
@@ -146,6 +168,23 @@ export type StaticFindingRow = {
evidence_redacted: string;
};
export type ClawScanFindingRow = {
artifact_id: string;
finding_id: string;
category_id: string;
category_label: string;
risk_bucket: ClawScanRiskBucket;
status: "note" | "concern";
severity: ClawScanSeverity;
confidence: "high" | "medium" | "low";
evidence_path_hash: string | null;
evidence_file_ext: string | null;
evidence_snippet_redacted: string | null;
evidence_explanation_redacted: string | null;
user_impact_redacted: string;
recommendation_redacted: string;
};
export type LabelRow = {
artifact_id: string;
label: DatasetLabel;
@@ -167,6 +206,7 @@ export type NormalizedDatasetRows = {
artifacts: ArtifactRow[];
scanResults: ScanResultRow[];
staticFindings: StaticFindingRow[];
clawScanFindings: ClawScanFindingRow[];
labels: LabelRow[];
splits: SplitRow[];
};
@@ -174,6 +214,17 @@ export type NormalizedDatasetRows = {
const SPLIT_VERSION = "sha256-v1";
const MAX_REDACTED_TEXT_LENGTH = 240;
const MAX_REDACTED_SKILL_CONTENT_LENGTH = 120_000;
const CLAWSCAN_SEVERITIES = new Set<ClawScanSeverity>([
"none",
"info",
"low",
"medium",
"high",
"critical",
]);
const AGENTIC_RISK_CATEGORY_LABEL_BY_ID: ReadonlyMap<string, string> = new Map(
AGENTIC_RISK_CATEGORIES.map((category) => [category.id, category.label] as const),
);
const SECRET_PATTERNS: RegExp[] = [
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
@@ -214,6 +265,7 @@ export function normalizeArtifactExport(inputs: ArtifactExportInput[]): Normaliz
const artifacts: ArtifactRow[] = [];
const scanResults: ScanResultRow[] = [];
const staticFindings: StaticFindingRow[] = [];
const clawScanFindings: ClawScanFindingRow[] = [];
const labels: LabelRow[] = [];
const splits: SplitRow[] = [];
@@ -223,11 +275,12 @@ export function normalizeArtifactExport(inputs: ArtifactExportInput[]): Normaliz
artifacts.push(artifact);
scanResults.push(...buildScanResultRows(input, artifactId));
staticFindings.push(...buildStaticFindingRows(input, artifactId));
clawScanFindings.push(...buildClawScanFindingRows(input, artifactId));
labels.push(...buildLabelRows(input, artifactId));
splits.push(buildSplitRow(input, artifactId));
}
return { artifacts, scanResults, staticFindings, labels, splits };
return { artifacts, scanResults, staticFindings, clawScanFindings, labels, splits };
}
export function buildArtifactId(input: ArtifactExportInput) {
@@ -367,6 +420,45 @@ function buildStaticFindingRows(
}));
}
function buildClawScanFindingRows(
input: ArtifactExportInput,
artifactId: string,
): ClawScanFindingRow[] {
return (input.llmAnalysis?.agenticRiskFindings ?? [])
.filter(
(finding): finding is typeof finding & { status: "note" | "concern" } =>
finding.status === "note" || finding.status === "concern",
)
.flatMap((finding, index) => {
const evidence = finding.evidence;
const categoryLabel = AGENTIC_RISK_CATEGORY_LABEL_BY_ID.get(finding.categoryId);
if (!categoryLabel) return [];
const severity = normalizeClawScanSeverity(finding.severity);
return [
{
artifact_id: artifactId,
finding_id: `${artifactId}:clawscan:${index}:${hashString(
`${finding.categoryId}:${finding.riskBucket}:${finding.status}:${severity}:${
evidence?.path ?? ""
}:${evidence?.snippet ?? ""}:${finding.userImpact}`,
).slice(0, 12)}`,
category_id: finding.categoryId,
category_label: categoryLabel,
risk_bucket: finding.riskBucket,
status: finding.status,
severity,
confidence: finding.confidence,
evidence_path_hash: evidence ? hashString(evidence.path) : null,
evidence_file_ext: evidence ? fileExtension(evidence.path) : null,
evidence_snippet_redacted: redactText(evidence?.snippet),
evidence_explanation_redacted: redactText(evidence?.explanation),
user_impact_redacted: redactText(finding.userImpact) ?? "",
recommendation_redacted: redactText(finding.recommendation) ?? "",
},
];
});
}
function buildLabelRows(input: ArtifactExportInput, artifactId: string): LabelRow[] {
const scannerLabels: DatasetLabel[] = [];
const rows: LabelRow[] = [];
@@ -488,6 +580,13 @@ function normalizeLabel(value: string): DatasetLabel {
return labelFromText(value);
}
function normalizeClawScanSeverity(value: string): ClawScanSeverity {
const normalized = value.trim().toLowerCase();
return CLAWSCAN_SEVERITIES.has(normalized as ClawScanSeverity)
? (normalized as ClawScanSeverity)
: "none";
}
function countFileExtensions(files: ExportFileInput[]) {
const counts: Record<string, number> = {};
for (const file of files) {