admin/clawhub
1
0
Fork 0
mirror of https://github.com/openclaw/clawhub.git synced 2026-08-14 00:47:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Publish security dataset from live Convex export (#2800)

Browse Source 
* feat(cli): restore skill sync command

* feat: publish security dataset from live export
This commit is contained in:
Patrick Erichsen
2026-06-22 22:40:09 -07:00
committed by GitHub
parent 4a78ca3a06
commit e82f0704c8
8 changed files with 1215 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/security-dataset-snapshot.yml
+220
View File
@@ -0,0 +1,220 @@
name: Security Dataset Snapshot
on:
workflow_dispatch:
inputs:
upload:
description: "Upload sanitized dataset files to Hugging Face"
required: true
default: "false"
type: choice
options:
- "false"
- "true"
limit:
description: "Optional source artifact cap for validation runs"
required: false
default: ""
hf-revision:
description: "Hugging Face branch/revision to upload to"
required: true
default: "main"
schedule:
- cron: "17 9 * * *"
permissions:
contents: read
id-token: write
concurrency:
group: clawhub-security-dataset-snapshot
cancel-in-progress: false
jobs:
publish-security-dataset:
name: Publish sanitized security dataset
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 360
environment: Production
env:
CONVEX_URL: ${{ vars.CONVEX_URL || vars.VITE_CONVEX_URL || 'https://wry-manatee-359.convex.cloud' }}
SECURITY_SCAN_WORKER_TOKEN: ${{ secrets.SECURITY_SCAN_WORKER_TOKEN }}
HF_DATASET_REPO: OpenClaw/clawhub-security-signals
HF_OIDC_RESOURCE: datasets/OpenClaw/clawhub-security-signals
HF_REVISION: ${{ inputs.hf-revision || 'main' }}
HF_UPLOAD: ${{ github.event_name == 'schedule' || inputs.upload == 'true' }}
SNAPSHOT_LIMIT: ${{ inputs.limit || '' }}
SANITIZED_OUT_DIR: ${{ runner.temp }}/clawhub-security-dataset/sanitized
WORK_DIR: ${{ runner.temp }}/clawhub-security-dataset
steps:
- uses: actions/checkout@v6
- uses: ./.github/actions/setup-bun
- uses: actions/setup-python@v6
if: ${{ env.HF_UPLOAD == 'true' }}
with:
python-version: "3.12"
- name: Check configuration
run: |
set -euo pipefail
if [[ -z "$SECURITY_SCAN_WORKER_TOKEN" ]]; then
echo "::error::SECURITY_SCAN_WORKER_TOKEN is required"
exit 1
fi
echo "Upload enabled: $HF_UPLOAD"
echo "Convex URL: $CONVEX_URL"
echo "Hugging Face repo: $HF_DATASET_REPO"
echo "Hugging Face OIDC resource: $HF_OIDC_RESOURCE"
echo "Hugging Face revision: $HF_REVISION"
- name: Export sanitized live dataset
run: |
set -euo pipefail
mkdir -p "$SANITIZED_OUT_DIR"
source_snapshot_id="live-convex-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
args=(
--convex-url "$CONVEX_URL"
--worker-token "$SECURITY_SCAN_WORKER_TOKEN"
--source-snapshot-id "$source_snapshot_id"
--out-dir "$SANITIZED_OUT_DIR"
--hf-dataset
--hf-repo "$HF_DATASET_REPO"
--hf-revision "$HF_REVISION"
)
if [[ -n "$SNAPSHOT_LIMIT" ]]; then
args+=(--limit "$SNAPSHOT_LIMIT")
fi
bun scripts/security-dataset/export-snapshot.ts "${args[@]}" | tee "$SANITIZED_OUT_DIR/summary.json"
snapshot_dir="$(jq -r '.snapshotDir' "$SANITIZED_OUT_DIR/summary.json")"
if [[ -z "$snapshot_dir" || "$snapshot_dir" == "null" ]]; then
echo "::error::export summary did not include snapshotDir"
exit 1
fi
echo "SNAPSHOT_DIR=$snapshot_dir" >> "$GITHUB_ENV"
jq '.manifest.row_counts, .manifest.huggingface_dataset' "$SANITIZED_OUT_DIR/summary.json"
- name: Validate sanitized output guardrails
run: |
set -euo pipefail
test -d "$SNAPSHOT_DIR/hf-dataset/data"
test -f "$SNAPSHOT_DIR/manifest.json"
for split in train validation test eval_holdout; do
test -f "$SNAPSHOT_DIR/hf-dataset/data/$split.jsonl"
done
if grep -R -E 'storageId|skillVersions:|packageReleases:|_storage/' "$SNAPSHOT_DIR/hf-dataset" "$SNAPSHOT_DIR/manifest.json"; then
echo "::error::sanitized output contains raw storage or internal document identifiers"
exit 1
fi
if grep -R -E 'gh[pousr]_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9_-]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN [A-Z0-9 ]*(PRIVATE KEY|CERTIFICATE)-----' "$SNAPSHOT_DIR/hf-dataset"; then
echo "::error::sanitized output contains obvious secret-like values"
exit 1
fi
- name: Install Hugging Face uploader
if: ${{ env.HF_UPLOAD == 'true' }}
run: |
set -euo pipefail
python -m pip install --upgrade pip
python -m pip install 'huggingface_hub[hf_xet]'
- name: Upload sanitized dataset to Hugging Face
if: ${{ env.HF_UPLOAD == 'true' }}
run: |
set -euo pipefail
python - <<'PY'
import json
import os
import urllib.request
from pathlib import Path
from huggingface_hub import HfApi
def get_github_oidc_token() -> str:
request_url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"]
separator = "&" if "?" in request_url else "?"
request = urllib.request.Request(
f"{request_url}{separator}audience=https://huggingface.co",
headers={
"Authorization": f"bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}",
"Accept": "application/json",
},
)
with urllib.request.urlopen(request) as response:
payload = json.loads(response.read().decode("utf-8"))
return payload["value"]
def exchange_hugging_face_token(oidc_token: str, resource: str) -> str:
body = json.dumps({
"grant_type": "urn:ietf:params:oauth:grant-type:token-exchange",
"subject_token_type": "urn:ietf:params:oauth:token-type:id_token",
"subject_token": oidc_token,
"resource": resource,
}).encode("utf-8")
request = urllib.request.Request(
"https://huggingface.co/oauth/token",
data=body,
headers={
"Content-Type": "application/json",
"Accept": "application/json",
},
method="POST",
)
with urllib.request.urlopen(request) as response:
payload = json.loads(response.read().decode("utf-8"))
return payload["access_token"]
snapshot_dir = Path(os.environ["SNAPSHOT_DIR"])
repo_id = os.environ["HF_DATASET_REPO"]
revision = os.environ["HF_REVISION"]
token = exchange_hugging_face_token(
get_github_oidc_token(),
os.environ["HF_OIDC_RESOURCE"],
)
api = HfApi(token=token)
data_commit = api.upload_folder(
folder_path=str(snapshot_dir / "hf-dataset" / "data"),
path_in_repo="data",
repo_id=repo_id,
repo_type="dataset",
revision=revision,
delete_patterns="data/*.jsonl",
commit_message="Update nightly ClawHub security dataset splits",
)
manifest_path = snapshot_dir / "manifest.json"
manifest = json.loads(manifest_path.read_text())
manifest["huggingface_dataset"]["commit"] = data_commit.oid
manifest["huggingface_dataset"]["revision"] = revision
manifest_path.write_text(json.dumps(manifest, indent=2) + "\n")
manifest_commit = api.upload_file(
path_or_fileobj=str(manifest_path),
path_in_repo="metadata/latest-manifest.json",
repo_id=repo_id,
repo_type="dataset",
revision=revision,
commit_message="Update nightly ClawHub security dataset manifest",
)
print(json.dumps({
"data_commit": data_commit.oid,
"manifest_commit": manifest_commit.oid,
"repo": repo_id,
"revision": revision,
}, indent=2))
PY
- name: Upload sanitized summary artifact
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v7
with:
name: security-dataset-summary-${{ github.run_id }}
path: |
${{ env.SANITIZED_OUT_DIR }}/summary.json
${{ env.SNAPSHOT_DIR }}/manifest.json
if-no-files-found: ignore
- name: Cleanup transient dataset files
if: ${{ always() }}
run: rm -rf "$WORK_DIR"
convex/securityDatasetNode.test.ts
+130
View File
@@ -0,0 +1,130 @@
/* @vitest-environment node */
import { gunzipSync } from "node:zlib";
import { describe, expect, it, vi } from "vitest";
import { listArtifactExportBatchCompressed } from "./securityDatasetNode";
type WrappedHandler<TArgs, TResult = unknown> = {
_handler: (ctx: unknown, args: TArgs) => Promise<TResult>;
};
const listArtifactExportBatchCompressedHandler = (
listArtifactExportBatchCompressed as unknown as WrappedHandler<
{
token: string;
sourceKind: "skill";
paginationOpts: { cursor: string | null; numItems: number };
pageCount: number;
},
{ encoding: "gzip-base64-json"; payload: string }
>
)._handler;
describe("security dataset worker export", () => {
it("requires the shared worker token", async () => {
vi.stubEnv("SECURITY_SCAN_WORKER_TOKEN", "worker-secret");
await expect(
listArtifactExportBatchCompressedHandler(makeCtx(), {
token: "wrong",
sourceKind: "skill",
paginationOpts: { cursor: null, numItems: 1 },
pageCount: 1,
}),
).rejects.toThrow("Unauthorized");
});
it("returns redacted storage-backed content without storage ids", async () => {
vi.stubEnv("SECURITY_SCAN_WORKER_TOKEN", "worker-secret");
const result = await listArtifactExportBatchCompressedHandler(makeCtx(), {
token: "worker-secret",
sourceKind: "skill",
paginationOpts: { cursor: null, numItems: 1 },
pageCount: 1,
});
const decoded = JSON.parse(
gunzipSync(Buffer.from(result.payload, "base64")).toString("utf8"),
) as {
page: Array<{
skillMdContentRedacted: string;
bundleFilesRedacted: Array<{ path: string; content: string }>;
files: Array<Record<string, unknown>>;
}>;
};
expect(decoded.page).toEqual([
expect.objectContaining({
skillMdContentRedacted: "Use this skill. [REDACTED_SECRET]",
bundleFilesRedacted: [
{
path: "scripts/run.sh",
content: "echo [REDACTED_SECRET]\n",
},
],
files: [
expect.not.objectContaining({ storageId: expect.anything() }),
expect.not.objectContaining({ storageId: expect.anything() }),
],
}),
]);
});
});
function makeCtx() {
return {
runQuery: vi.fn(async () => ({
page: [
{
sourceKind: "skill",
sourceDocId: "skillVersions:1",
parentDocId: "skills:1",
publicName: "Demo",
publicOwnerHandle: "owner",
publicSlug: "demo",
version: "1.0.0",
artifactSha256: "a".repeat(64),
createdAt: 1,
softDeletedAt: null,
files: [
{
path: "SKILL.md",
size: 42,
sha256: "skill-sha",
storageId: "storage:skill",
contentType: "text/markdown",
},
{
path: "scripts/run.sh",
size: 32,
sha256: "script-sha",
storageId: "storage:script",
contentType: "text/x-shellscript",
},
],
packageFamily: null,
packageChannel: null,
sourceRepoHost: null,
vtAnalysis: null,
skillSpectorAnalysis: null,
staticScan: null,
llmAnalysis: null,
moderationConsensus: null,
},
],
isDone: true,
continueCursor: "",
exportMode: "public",
})),
storage: {
get: vi.fn(async (id: string) => {
if (id === "storage:skill") {
return new Blob(["Use this skill. token=supersecret123"]);
}
if (id === "storage:script") {
return new Blob(["echo password=scriptsecret123\n"]);
}
return null;
}),
},
};
}
convex/securityDatasetNode.ts
+102 -35
View File
@@ -2,10 +2,10 @@
import { gzipSync } from "node:zlib";
import { paginationOptsValidator } from "convex/server";
import { v } from "convex/values";
import { ConvexError, v } from "convex/values";
import { internal } from "./_generated/api";
import type { ActionCtx } from "./_generated/server";
import { internalAction } from "./functions";
import { action, internalAction } from "./functions";
const MAX_EXPORT_BATCH_PAGES = 20;
const MAX_REDACTED_BUNDLE_FILE_BYTES = 192 * 1024;
@@ -19,6 +19,21 @@ type ArtifactExportPage = {
exportMode: "public";
};
type DatasetLineage = {
exportMode: "public";
generatedAt: number;
maxExportPageSize: number;
maxExportBatchPages: number;
redactionPolicyVersion: string;
sourceTables: readonly string[];
scannerSources: readonly string[];
sourceBounds: Array<{
sourceKind: "skill" | "package";
minCreatedAt: number | null;
maxCreatedAt: number | null;
}>;
};
const SECRET_PATTERNS: RegExp[] = [
/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
/\bgh[pousr]_[A-Za-z0-9_]{20,}\b/g,
@@ -31,6 +46,24 @@ const SECRET_PATTERNS: RegExp[] = [
/(["'`])(?=[A-Za-z0-9+/=_-]{32,}\1)(?=.*[A-Z])(?=.*[a-z])(?=.*\d)[A-Za-z0-9+/=_-]+\1/g,
];
type ArtifactExportBatchArgs = {
sourceKind: "skill" | "package";
mode?: "public";
createdAtGte?: number;
createdAtLt?: number;
paginationOpts: {
cursor: string | null;
numItems: number;
};
pageCount: number;
};
function assertDatasetExportWorkerToken(token: string) {
// Shared worker credential already used by the security and Skill Card GitHub workers.
const expected = process.env.SECURITY_SCAN_WORKER_TOKEN;
if (!expected || token !== expected) throw new ConvexError("Unauthorized");
}
export const listArtifactExportBatchCompressedInternal = internalAction({
args: {
sourceKind: v.union(v.literal("skill"), v.literal("package")),
@@ -40,43 +73,77 @@ export const listArtifactExportBatchCompressedInternal = internalAction({
paginationOpts: paginationOptsValidator,
pageCount: v.number(),
},
handler: async (ctx, args) => listArtifactExportBatchCompressedForWorker(ctx, args),
});
export const listArtifactExportBatchCompressed = action({
args: {
token: v.string(),
sourceKind: v.union(v.literal("skill"), v.literal("package")),
mode: v.optional(v.literal("public")),
createdAtGte: v.optional(v.number()),
createdAtLt: v.optional(v.number()),
paginationOpts: paginationOptsValidator,
pageCount: v.number(),
},
handler: async (ctx, args) => {
const pageCount = Math.min(Math.max(1, Math.floor(args.pageCount)), MAX_EXPORT_BATCH_PAGES);
let cursor = args.paginationOpts.cursor;
const page: ArtifactExportPage["page"] = [];
let isDone = false;
for (let pageIndex = 0; pageIndex < pageCount; pageIndex += 1) {
const result: ArtifactExportPage = await ctx.runQuery(
internal.securityDataset.listArtifactExportPageInternal,
{
sourceKind: args.sourceKind,
mode: args.mode,
createdAtGte: args.createdAtGte,
createdAtLt: args.createdAtLt,
paginationOpts: {
cursor,
numItems: args.paginationOpts.numItems,
},
},
);
page.push(...result.page);
cursor = result.continueCursor;
isDone = result.isDone;
if (isDone) break;
}
const json = JSON.stringify({
page: await enrichAndSanitizeArtifactRows(ctx, page),
isDone,
continueCursor: cursor,
exportMode: args.mode ?? "public",
});
return {
encoding: "gzip-base64-json" as const,
payload: gzipSync(json).toString("base64"),
};
assertDatasetExportWorkerToken(args.token);
return await listArtifactExportBatchCompressedForWorker(ctx, args);
},
});
export const getDatasetLineage = action({
args: {
token: v.string(),
mode: v.optional(v.literal("public")),
},
handler: async (ctx, args): Promise<DatasetLineage> => {
assertDatasetExportWorkerToken(args.token);
return await ctx.runQuery(internal.securityDataset.getDatasetLineageInternal, {
mode: args.mode,
});
},
});
async function listArtifactExportBatchCompressedForWorker(
ctx: ActionCtx,
args: ArtifactExportBatchArgs,
) {
const pageCount = Math.min(Math.max(1, Math.floor(args.pageCount)), MAX_EXPORT_BATCH_PAGES);
let cursor = args.paginationOpts.cursor;
const page: ArtifactExportPage["page"] = [];
let isDone = false;
for (let pageIndex = 0; pageIndex < pageCount; pageIndex += 1) {
const result: ArtifactExportPage = await ctx.runQuery(
internal.securityDataset.listArtifactExportPageInternal,
{
sourceKind: args.sourceKind,
mode: args.mode,
createdAtGte: args.createdAtGte,
createdAtLt: args.createdAtLt,
paginationOpts: {
cursor,
numItems: args.paginationOpts.numItems,
},
},
);
page.push(...result.page);
cursor = result.continueCursor;
isDone = result.isDone;
if (isDone) break;
}
const json = JSON.stringify({
page: await enrichAndSanitizeArtifactRows(ctx, page),
isDone,
continueCursor: cursor,
exportMode: args.mode ?? "public",
});
return {
encoding: "gzip-base64-json" as const,
payload: gzipSync(json).toString("base64"),
};
}
async function enrichAndSanitizeArtifactRows(ctx: ActionCtx, rows: unknown[]) {
const enrichedRows = [];
let remainingBundleBytes = MAX_REDACTED_BUNDLE_BYTES_PER_RESPONSE;
scripts/security-dataset/export-snapshot.ts
+224 -15
View File
@@ -1,17 +1,24 @@
import { execFile, spawnSync } from "node:child_process";
import { once } from "node:events";
import { createWriteStream, type WriteStream } from "node:fs";
import { mkdir, writeFile } from "node:fs/promises";
import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { promisify } from "node:util";
import { gunzipSync } from "node:zlib";
import { ConvexHttpClient } from "convex/browser";
import { api } from "../../convex/_generated/api";
import { artifactInputsFromConvexExportZip } from "./convexExport";
import { parseConvexJsonMatching } from "./convexOutput";
import { reserveExportInputs } from "./exportLimit";
import {
buildHuggingFaceSecuritySignalRows,
type HuggingFaceSecuritySignalRow,
} from "./huggingFaceExport";
import { buildSecurityDatasetManifest } from "./manifest";
import {
normalizeArtifactExport,
type ArtifactExportInput,
type DatasetSplit,
type NormalizedDatasetRows,
type SourceKind,
} from "./normalize";
@@ -38,6 +45,14 @@ type ConvexBounds = {
maxCreatedAt: number | null;
};
type DatasetLineage = {
exportMode: "public";
generatedAt: number;
redactionPolicyVersion: string;
sourceTables: string[];
sourceBounds: ConvexBounds[];
};
type CompressedConvexPage = {
encoding: "gzip-base64-json";
payload: string;
@@ -45,6 +60,8 @@ type CompressedConvexPage = {
type Options = {
deployment: string | null;
convexUrl: string | null;
workerToken: string | null;
prod: boolean;
push: boolean;
dryRun: boolean;
@@ -58,6 +75,10 @@ type Options = {
sourceKind: SourceKind | "all";
timeWindow: CreatedTimeWindow;
convexExportZip: string | null;
sourceSnapshotId: string | null;
huggingFaceDataset: boolean;
huggingFaceRepo: string;
huggingFaceRevision: string;
};
type ExportShard = {
@@ -76,7 +97,9 @@ type SnapshotState = {
clawScanFindings: number;
labels: number;
splits: number;
huggingFaceRows: number;
};
huggingFaceRowCountsBySplit: Record<DatasetSplit, number>;
scannerVersions: Set<string>;
modelNames: Set<string>;
};
@@ -88,6 +111,7 @@ type SnapshotWriters = {
clawScanFindings: WriteStream;
labels: WriteStream;
splits: WriteStream;
huggingFaceSplits?: Record<DatasetSplit, WriteStream>;
};
const DEFAULT_PAGE_SIZE = 50;
@@ -98,12 +122,13 @@ const DEFAULT_MAX_CONVEX_ATTEMPTS = 6;
const DEFAULT_OUT_DIR = ".data/security-dataset/snapshots";
const CONVEX_RUN_MAX_BUFFER_BYTES = 128 * 1024 * 1024;
const SOURCE_KINDS: SourceKind[] = ["skill", "package"];
const HF_SPLITS: DatasetSplit[] = ["train", "validation", "test", "eval_holdout"];
async function main() {
const options = parseArgs(process.argv.slice(2));
const snapshotId = buildSnapshotId(options);
const snapshotDir = resolve(options.outDir, snapshotId);
const writers = options.dryRun ? null : await openSnapshotWriters(snapshotDir);
const writers = options.dryRun ? null : await openSnapshotWriters(snapshotDir, options);
let writersClosed = false;
const state = createSnapshotState();
@@ -111,9 +136,15 @@ async function main() {
const shardCount = options.convexExportZip
? await exportConvexExportZip({ options, state, writers })
: await exportRemoteShards({ options, state, writers });
const manifest = buildManifest({ options, snapshotId, state, shardCount });
if (options.dryRun) {
const manifest = buildManifest({
options,
snapshotId,
state,
shardCount,
outputSizes: {},
});
console.log(JSON.stringify({ snapshotId, dryRun: true, manifest }, null, 2));
return;
}
@@ -121,6 +152,8 @@ async function main() {
if (!writers) throw new Error("Snapshot writers were not opened.");
await closeSnapshotWriters(writers);
writersClosed = true;
const outputSizes = await collectOutputSizes(snapshotDir);
const manifest = buildManifest({ options, snapshotId, state, shardCount, outputSizes });
await writeFile(join(snapshotDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
console.log(JSON.stringify({ snapshotId, snapshotDir, manifest }, null, 2));
@@ -231,6 +264,7 @@ async function runConvexPage(
batchPages: number,
): Promise<{ page: ConvexPage; batchPages: number }> {
const functionName = "securityDatasetNode:listArtifactExportBatchCompressedInternal";
const workerFunctionName = "securityDatasetNode:listArtifactExportBatchCompressed";
let pageCount = batchPages;
while (true) {
@@ -246,12 +280,19 @@ async function runConvexPage(
let lastError: unknown = null;
for (let attempt = 1; attempt <= DEFAULT_MAX_CONVEX_ATTEMPTS; attempt += 1) {
try {
const compressed = await runConvexJsonOnce<CompressedConvexPage>(
options,
functionName,
args,
isCompressedConvexPage,
);
const compressed = options.workerToken
? await runWorkerAction<CompressedConvexPage>(
options,
workerFunctionName,
{ ...args, token: options.workerToken },
isCompressedConvexPage,
)
: await runConvexJsonOnce<CompressedConvexPage>(
options,
functionName,
args,
isCompressedConvexPage,
);
return { page: decodeCompressedConvexPage(compressed), batchPages: pageCount };
} catch (error) {
lastError = error;
@@ -279,6 +320,19 @@ async function runConvexPage(
}
async function runConvexBounds(options: Options, sourceKind: SourceKind): Promise<ConvexBounds> {
if (options.workerToken) {
const lineage = await runWorkerAction<DatasetLineage>(
options,
"securityDatasetNode:getDatasetLineage",
{ token: options.workerToken, mode: options.mode },
isDatasetLineage,
);
const bounds = lineage.sourceBounds.find((candidate) => candidate.sourceKind === sourceKind);
if (!bounds) {
return { sourceKind, minCreatedAt: null, maxCreatedAt: null };
}
return bounds;
}
return runConvexJson<ConvexBounds>(
options,
"securityDataset:getArtifactExportBoundsInternal",
@@ -287,6 +341,31 @@ async function runConvexBounds(options: Options, sourceKind: SourceKind): Promis
);
}
async function runWorkerAction<T>(
options: Options,
functionName: string,
args: unknown,
validate: (value: unknown) => value is T,
): Promise<T> {
if (!options.convexUrl) {
throw new Error("--convex-url or CONVEX_URL is required when using --worker-token.");
}
const client = new ConvexHttpClient(options.convexUrl);
const result = await client.action(resolveWorkerAction(functionName), args as never);
if (validate(result)) return result;
throw new Error(`Invalid ${functionName} response.`);
}
function resolveWorkerAction(functionName: string) {
if (functionName === "securityDatasetNode:listArtifactExportBatchCompressed") {
return api.securityDatasetNode.listArtifactExportBatchCompressed;
}
if (functionName === "securityDatasetNode:getDatasetLineage") {
return api.securityDatasetNode.getDatasetLineage;
}
throw new Error(`Unsupported worker action: ${functionName}`);
}
async function runConvexJson<T>(
options: Options,
functionName: string,
@@ -352,14 +431,19 @@ function buildManifest(input: {
snapshotId: string;
state: SnapshotState;
shardCount: number;
outputSizes: Record<string, number>;
}) {
const { options, snapshotId, state, shardCount } = input;
const { options, snapshotId, state, shardCount, outputSizes } = input;
const repoGitSha = gitSha();
return buildSecurityDatasetManifest({
snapshotId,
sourceSnapshotId: options.sourceSnapshotId ?? snapshotId,
createdAt: new Date().toISOString(),
repoGitSha,
convexDeployment: options.deployment ?? (options.prod ? "prod" : "configured-dev"),
convexDeployment:
options.deployment ??
inferDeploymentFromConvexUrl(options.convexUrl) ??
(options.prod ? "prod" : "configured-dev"),
exportMode: options.mode,
pageSize: options.pageSize,
concurrency: options.concurrency,
@@ -373,22 +457,45 @@ function buildManifest(input: {
clawScanFindings: state.rowCounts.clawScanFindings,
labels: state.rowCounts.labels,
splits: state.rowCounts.splits,
huggingFaceRows: state.rowCounts.huggingFaceRows,
},
outputSizes,
scannerVersions: Array.from(state.scannerVersions).sort(),
modelNames: Array.from(state.modelNames).sort(),
redactionPolicyVersion: "public-signals-v2-bundle-files",
sourceTables: ["skillVersions", "packageReleases"],
timeWindow: options.timeWindow,
huggingFaceDataset: options.huggingFaceDataset
? {
repo: options.huggingFaceRepo,
revision: options.huggingFaceRevision,
commit: null,
configNames: ["default"],
splitNames: HF_SPLITS,
rowCountsBySplit: state.huggingFaceRowCountsBySplit,
}
: undefined,
});
}
function inferDeploymentFromConvexUrl(convexUrl: string | null) {
if (!convexUrl) return null;
try {
return new URL(convexUrl).hostname.split(".")[0] || null;
} catch {
return null;
}
}
function buildSnapshotId(options: Options) {
const timestamp = new Date()
.toISOString()
.replace(/[-:]/g, "")
.replace(/\.\d{3}Z$/, "Z");
const deployment =
options.deployment?.replace(/[^a-zA-Z0-9]+/g, "-") ?? (options.prod ? "prod" : "dev");
options.deployment?.replace(/[^a-zA-Z0-9]+/g, "-") ??
inferDeploymentFromConvexUrl(options.convexUrl)?.replace(/[^a-zA-Z0-9]+/g, "-") ??
(options.prod ? "prod" : "dev");
return `clawhub-${deployment}-${timestamp}-${gitSha().slice(0, 8)}`;
}
@@ -402,6 +509,13 @@ function createSnapshotState(): SnapshotState {
clawScanFindings: 0,
labels: 0,
splits: 0,
huggingFaceRows: 0,
},
huggingFaceRowCountsBySplit: {
train: 0,
validation: 0,
test: 0,
eval_holdout: 0,
},
scannerVersions: new Set(),
modelNames: new Set(),
@@ -415,12 +529,17 @@ async function processArtifactInputs(input: {
}) {
const { inputs, state, writers } = input;
const rows = normalizeArtifactExport(inputs);
const hfRows = buildHuggingFaceSecuritySignalRows(rows);
state.rowCounts.artifacts += rows.artifacts.length;
state.rowCounts.scanResults += rows.scanResults.length;
state.rowCounts.staticFindings += rows.staticFindings.length;
state.rowCounts.clawScanFindings += rows.clawScanFindings.length;
state.rowCounts.labels += rows.labels.length;
state.rowCounts.splits += rows.splits.length;
state.rowCounts.huggingFaceRows += hfRows.length;
for (const row of hfRows) {
state.huggingFaceRowCountsBySplit[row.split] += 1;
}
for (const row of rows.scanResults) {
if (row.scanner_version) state.scannerVersions.add(row.scanner_version);
if (row.model) state.modelNames.add(row.model);
@@ -428,11 +547,17 @@ async function processArtifactInputs(input: {
if (!writers) return;
await writeNormalizedRows(writers, rows);
if (writers.huggingFaceSplits) {
await writeHuggingFaceRows(writers.huggingFaceSplits, hfRows);
}
}
async function openSnapshotWriters(snapshotDir: string): Promise<SnapshotWriters> {
async function openSnapshotWriters(
snapshotDir: string,
options: Options,
): Promise<SnapshotWriters> {
await mkdir(snapshotDir, { recursive: true });
return {
const writers: SnapshotWriters = {
artifacts: createWriteStream(join(snapshotDir, "artifacts.jsonl"), { encoding: "utf8" }),
scanResults: createWriteStream(join(snapshotDir, "scan_results.jsonl"), { encoding: "utf8" }),
staticFindings: createWriteStream(join(snapshotDir, "static_findings.jsonl"), {
@@ -444,10 +569,30 @@ async function openSnapshotWriters(snapshotDir: string): Promise<SnapshotWriters
labels: createWriteStream(join(snapshotDir, "labels.jsonl"), { encoding: "utf8" }),
splits: createWriteStream(join(snapshotDir, "splits.jsonl"), { encoding: "utf8" }),
};
if (options.huggingFaceDataset) {
const dataDir = join(snapshotDir, "hf-dataset", "data");
await mkdir(dataDir, { recursive: true });
writers.huggingFaceSplits = {
train: createWriteStream(join(dataDir, "train.jsonl"), { encoding: "utf8" }),
validation: createWriteStream(join(dataDir, "validation.jsonl"), { encoding: "utf8" }),
test: createWriteStream(join(dataDir, "test.jsonl"), { encoding: "utf8" }),
eval_holdout: createWriteStream(join(dataDir, "eval_holdout.jsonl"), { encoding: "utf8" }),
};
}
return writers;
}
async function closeSnapshotWriters(writers: SnapshotWriters) {
await Promise.all(Object.values(writers).map((stream) => endStream(stream)));
const streams = [
writers.artifacts,
writers.scanResults,
writers.staticFindings,
writers.clawScanFindings,
writers.labels,
writers.splits,
...Object.values(writers.huggingFaceSplits ?? {}),
];
await Promise.all(streams.map((stream) => endStream(stream)));
}
async function endStream(stream: WriteStream) {
@@ -470,6 +615,37 @@ async function writeJsonlRows(stream: WriteStream, rows: unknown[]) {
if (!stream.write(chunk)) await once(stream, "drain");
}
async function writeHuggingFaceRows(
streams: Record<DatasetSplit, WriteStream>,
rows: HuggingFaceSecuritySignalRow[],
) {
for (const split of HF_SPLITS) {
await writeJsonlRows(
streams[split],
rows.filter((row) => row.split === split),
);
}
}
async function collectOutputSizes(root: string) {
const sizes: Record<string, number> = {};
await collectOutputSizesInto(root, root, sizes);
return sizes;
}
async function collectOutputSizesInto(root: string, dir: string, sizes: Record<string, number>) {
const entries = await readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const path = join(dir, entry.name);
if (entry.isDirectory()) {
await collectOutputSizesInto(root, path, sizes);
} else if (entry.isFile()) {
const relativePath = path.slice(root.length + 1);
sizes[relativePath] = (await stat(path)).size;
}
}
}
function boundsToShards(bounds: ConvexBounds, shardCount: number): ExportShard[] {
if (bounds.minCreatedAt === null || bounds.maxCreatedAt === null) return [];
const start = bounds.minCreatedAt;
@@ -547,6 +723,18 @@ function isCompressedConvexPage(value: unknown): value is CompressedConvexPage {
);
}
function isDatasetLineage(value: unknown): value is DatasetLineage {
return (
isRecord(value) &&
value.exportMode === "public" &&
typeof value.generatedAt === "number" &&
typeof value.redactionPolicyVersion === "string" &&
Array.isArray(value.sourceTables) &&
Array.isArray(value.sourceBounds) &&
value.sourceBounds.every(isConvexBounds)
);
}
function decodeCompressedConvexPage(value: CompressedConvexPage) {
const json = gunzipSync(Buffer.from(value.payload, "base64")).toString("utf8");
const parsed: unknown = JSON.parse(json);
@@ -588,6 +776,8 @@ function gitSha() {
function parseArgs(args: string[]): Options {
const options: Options = {
deployment: null,
convexUrl: null,
workerToken: null,
prod: false,
push: false,
dryRun: false,
@@ -601,6 +791,10 @@ function parseArgs(args: string[]): Options {
sourceKind: "all",
timeWindow: emptyCreatedTimeWindow(),
convexExportZip: null,
sourceSnapshotId: null,
huggingFaceDataset: false,
huggingFaceRepo: process.env.HF_DATASET_REPO ?? "OpenClaw/clawhub-security-signals",
huggingFaceRevision: process.env.HF_REVISION ?? "main",
};
for (let index = 0; index < args.length; index += 1) {
@@ -613,6 +807,10 @@ function parseArgs(args: string[]): Options {
options.dryRun = true;
} else if (arg === "--deployment") {
options.deployment = readValue(args, ++index, arg);
} else if (arg === "--convex-url") {
options.convexUrl = readValue(args, ++index, arg);
} else if (arg === "--worker-token") {
options.workerToken = readValue(args, ++index, arg);
} else if (arg === "--limit") {
options.limit = readPositiveInt(readValue(args, ++index, arg), arg);
} else if (arg === "--page-size") {
@@ -633,6 +831,14 @@ function parseArgs(args: string[]): Options {
options.timeWindow.createdAtLt = parseCreatedTimestamp(readValue(args, ++index, arg), arg);
} else if (arg === "--convex-export-zip" || arg === "--from-convex-export") {
options.convexExportZip = readValue(args, ++index, arg);
} else if (arg === "--source-snapshot-id") {
options.sourceSnapshotId = readValue(args, ++index, arg);
} else if (arg === "--hf-dataset") {
options.huggingFaceDataset = true;
} else if (arg === "--hf-repo") {
options.huggingFaceRepo = readValue(args, ++index, arg);
} else if (arg === "--hf-revision") {
options.huggingFaceRevision = readValue(args, ++index, arg);
} else if (arg === "--mode") {
const mode = readValue(args, ++index, arg);
if (mode !== "public") throw new Error(`Unsupported mode: ${mode}`);
@@ -645,6 +851,9 @@ function parseArgs(args: string[]): Options {
if (options.prod && options.deployment) {
throw new Error("Use either --prod or --deployment, not both.");
}
if (options.workerToken && (options.prod || options.deployment || options.push)) {
throw new Error("Use --worker-token with --convex-url instead of --prod/--deployment/--push.");
}
assertCreatedTimeWindow(options.timeWindow);
return options;
}
scripts/security-dataset/exportSnapshotCli.test.ts
+151
View File
@@ -0,0 +1,151 @@
/* @vitest-environment node */
import { execFile } from "node:child_process";
import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { strToU8, zipSync } from "fflate";
import { describe, expect, it } from "vitest";
const execFileAsync = promisify(execFile);
describe("security dataset snapshot CLI", () => {
it("writes flat Hugging Face split files from a tiny sanitized Convex export", async () => {
const directory = await mkdtemp(join(tmpdir(), "clawhub-security-dataset-cli-"));
try {
const snapshotZip = join(directory, "snapshot.zip");
const outDir = join(directory, "out");
await writeFile(snapshotZip, Buffer.from(buildTinyConvexSnapshotZip()));
const result = await execFileAsync(
"bun",
[
"scripts/security-dataset/export-snapshot.ts",
"--convex-export-zip",
snapshotZip,
"--source-snapshot-id",
"live-export-prod-123-1",
"--out-dir",
outDir,
"--hf-dataset",
],
{
cwd: process.cwd(),
encoding: "utf8",
maxBuffer: 16 * 1024 * 1024,
},
);
const summary: {
snapshotDir: string;
manifest: {
source_snapshot_id: string;
row_counts: { huggingface_rows: number };
huggingface_dataset: { repo: string; rowCountsBySplit: Record<string, number> };
};
} = JSON.parse(result.stdout);
expect(summary.manifest.source_snapshot_id).toBe("live-export-prod-123-1");
expect(summary.manifest.row_counts.huggingface_rows).toBe(1);
expect(summary.manifest.huggingface_dataset.repo).toBe("OpenClaw/clawhub-security-signals");
const dataDir = join(summary.snapshotDir, "hf-dataset", "data");
const files = await readdir(dataDir);
expect(files.sort()).toEqual([
"eval_holdout.jsonl",
"test.jsonl",
"train.jsonl",
"validation.jsonl",
]);
const splitContents = await Promise.all(
files.map(async (file) => readFile(join(dataDir, file), "utf8")),
);
const flatRows = splitContents
.join("")
.split(/\r?\n/)
.filter((line) => line.length > 0)
.map((line) => JSON.parse(line) as Record<string, unknown>);
expect(flatRows).toEqual([
expect.objectContaining({
id: "a".repeat(64),
skill_slug: "owner/stored-skill",
skill_version: "1.0.0",
skill_md_content: "Stored SKILL.md [REDACTED_SECRET]",
skill_bundle_content: [
expect.objectContaining({
path: "scripts/run.sh",
content: "echo [REDACTED_SECRET]\n",
}),
],
}),
]);
const serializedRows = JSON.stringify(flatRows);
expect(serializedRows).not.toContain("storageId");
expect(serializedRows).not.toContain("skillVersions:1");
expect(serializedRows).not.toContain("supersecret123");
expect(serializedRows).not.toContain("scriptsecret123");
} finally {
await rm(directory, { recursive: true, force: true });
}
});
});
function buildTinyConvexSnapshotZip() {
return zipSync({
"skills/documents.jsonl": strToU8(
`${JSON.stringify({
_id: "skills:1",
displayName: "Stored Skill",
slug: "stored-skill",
ownerUserId: "users:owner",
})}\n`,
),
"skillVersions/documents.jsonl": strToU8(
`${JSON.stringify({
_id: "skillVersions:1",
skillId: "skills:1",
version: "1.0.0",
createdAt: Date.UTC(2026, 5, 23),
sha256hash: "a".repeat(64),
files: [
{
path: "SKILL.md",
size: 31,
sha256: "skill-md-sha",
content: "Stored SKILL.md token=supersecret123",
contentType: "text/markdown",
},
{
path: "scripts/run.sh",
size: 35,
sha256: "script-sha",
content: "echo password=scriptsecret123\n",
contentType: "text/x-shellscript",
},
],
staticScan: {
status: "clean",
reasonCodes: [],
findings: [],
summary: "No suspicious patterns detected.",
engineVersion: "static-v1",
checkedAt: Date.UTC(2026, 5, 23),
},
llmAnalysis: {
status: "completed",
verdict: "suspicious",
confidence: "medium",
summary: "Review before trusting.",
agenticRiskFindings: [],
model: "gpt-test",
checkedAt: Date.UTC(2026, 5, 23),
},
})}\n`,
),
"packages/documents.jsonl": strToU8(""),
"packageReleases/documents.jsonl": strToU8(""),
"users/documents.jsonl": strToU8(
`${JSON.stringify({ _id: "users:owner", handle: "owner" })}\n`,
),
});
}
scripts/security-dataset/huggingFaceExport.test.ts
+176
View File
@@ -0,0 +1,176 @@
import { describe, expect, it } from "vitest";
import { buildHuggingFaceSecuritySignalRows } from "./huggingFaceExport";
import { hashString, type NormalizedDatasetRows } from "./normalize";
describe("Hugging Face security dataset export", () => {
it("builds flat researcher rows from normalized security dataset sidecars", () => {
const artifactId = `skill:${"a".repeat(64)}`;
const normalized: NormalizedDatasetRows = {
artifacts: [
{
artifact_id: artifactId,
source_kind: "skill",
source_table: "skillVersions",
source_doc_id_hash: hashString("skillVersions:1"),
parent_doc_id_hash: hashString("skills:1"),
public_name: "Demo Skill",
public_owner_handle: "openclaw",
public_slug: "demo-skill",
public_qualified_slug: "openclaw/demo-skill",
version: "1.0.0",
artifact_sha256: "a".repeat(64),
skill_md_content_redacted: "Use this skill with [REDACTED_SECRET]",
bundle_files_redacted: [
{
path: "scripts/run.sh",
content: "echo [REDACTED_SECRET]\n",
sha256: "bundle-content-sha",
size_bytes: 23,
},
],
created_at: Date.UTC(2026, 5, 23),
created_month: "2026-06",
soft_deleted: false,
is_public: true,
file_count: 2,
total_bytes: 42,
file_ext_counts: { ".md": 1, ".sh": 1 },
package_family: null,
package_channel: null,
source_repo_host: null,
has_vt_scan: true,
has_skillspector_scan: true,
has_static_scan: true,
has_llm_scan: true,
},
],
scanResults: [
{
artifact_id: artifactId,
scanner: "static",
scanner_version: "static-v1",
model: null,
status: "clean",
verdict: "clean",
confidence: null,
checked_at: Date.UTC(2026, 5, 23),
reason_codes: [],
engine_stats: null,
summary_redacted: "No suspicious patterns detected.",
raw_status_family: "clean",
},
{
artifact_id: artifactId,
scanner: "virustotal",
scanner_version: "vt-v3",
model: null,
status: "clean",
verdict: "clean",
confidence: null,
checked_at: Date.UTC(2026, 5, 23),
reason_codes: [],
engine_stats: { malicious: 0, suspicious: 0, harmless: 1, undetected: 65 },
summary_redacted: null,
raw_status_family: "clean",
},
{
artifact_id: artifactId,
scanner: "skillspector",
scanner_version: "2.0.0",
model: null,
status: "suspicious",
verdict: "CAUTION",
confidence: null,
checked_at: Date.UTC(2026, 5, 23),
score: 35,
severity: "MEDIUM",
reason_codes: ["SQP-1"],
issues: [
{
code: "SQP-1",
category: "Skill Quality",
severity: "MEDIUM",
confidence: 0.9,
explanation_redacted: "Broad trigger scope.",
},
],
engine_stats: null,
summary_redacted: "Found broad trigger scope.",
raw_status_family: "suspicious",
},
{
artifact_id: artifactId,
scanner: "llm",
scanner_version: null,
model: "gpt-test",
status: "completed",
verdict: "suspicious",
confidence: "medium",
checked_at: Date.UTC(2026, 5, 23),
reason_codes: [],
engine_stats: null,
summary_redacted: "Review before trusting.",
raw_status_family: "suspicious",
},
],
staticFindings: [
{
artifact_id: artifactId,
finding_id: "finding-1",
code: "suspicious.env_credential_access",
severity: "warn",
file_path_hash: hashString("scripts/run.sh"),
file_ext: ".sh",
line_bucket: "1-20",
message: "Reads credentials",
evidence_redacted: "[REDACTED_SECRET]",
},
],
clawScanFindings: [],
labels: [
{
artifact_id: artifactId,
label: "suspicious",
label_source: "moderation_consensus",
label_confidence: "derived_consensus",
reason_codes: ["suspicious.env_credential_access"],
scanner_agreement: 1,
notes_redacted: null,
},
],
splits: [
{
artifact_id: artifactId,
split: "train",
split_version: "sha256-v1",
split_key: hashString("a".repeat(64)),
},
],
};
expect(buildHuggingFaceSecuritySignalRows(normalized)).toEqual([
expect.objectContaining({
id: "a".repeat(64),
skill_slug: "openclaw/demo-skill",
skill_version: "1.0.0",
skill_md_content: "Use this skill with [REDACTED_SECRET]",
skill_bundle_content: [
{
path: "scripts/run.sh",
content: "echo [REDACTED_SECRET]\n",
sha256: "bundle-content-sha",
sizeBytes: 23,
},
],
clawscan_verdict: "suspicious",
clawscan_confidence: "medium",
clawscan_model: "gpt-test",
static_status: "clean",
static_finding_count: 1,
virustotal_malicious_count: 0,
skillspector_issue_categories: ["Skill Quality"],
split: "train",
}),
]);
});
});
scripts/security-dataset/huggingFaceExport.ts
+197
View File
@@ -0,0 +1,197 @@
import {
hashString,
type ArtifactRow,
type DatasetSplit,
type LabelRow,
type NormalizedDatasetRows,
type ScanResultRow,
type StaticFindingRow,
} from "./normalize";
export type HuggingFaceSecuritySignalRow = {
id: string;
skill_slug: string | null;
skill_version: string;
skill_md_content: string | null;
skill_bundle_content: Array<{
path: string;
content: string;
sha256: string;
sizeBytes: number;
}>;
clawscan_verdict: string;
clawscan_confidence: string | null;
clawscan_model: string | null;
clawscan_summary: string | null;
static_status: string | null;
static_finding_count: number;
static_reason_codes: string[];
virustotal_status: string | null;
virustotal_malicious_count: number | null;
virustotal_suspicious_count: number | null;
virustotal_harmless_count: number | null;
virustotal_undetected_count: number | null;
skillspector_status: string | null;
skillspector_score: number | null;
skillspector_severity: string | null;
skillspector_issue_count: number;
skillspector_issue_codes: string[];
skillspector_issue_categories: string[];
clawscan_context: Record<string, unknown>;
split: DatasetSplit;
};
export function buildHuggingFaceSecuritySignalRows(
rows: NormalizedDatasetRows,
): HuggingFaceSecuritySignalRow[] {
const scansByArtifact = groupBy(rows.scanResults, (row) => row.artifact_id);
const labelsByArtifact = groupBy(rows.labels, (row) => row.artifact_id);
const staticFindingsByArtifact = groupBy(rows.staticFindings, (row) => row.artifact_id);
const splitByArtifact = new Map(rows.splits.map((row) => [row.artifact_id, row.split]));
return rows.artifacts.flatMap((artifact) => {
if (artifact.source_kind !== "skill" || !artifact.is_public || artifact.soft_deleted) {
return [];
}
const scans = scansByArtifact.get(artifact.artifact_id) ?? [];
const labels = labelsByArtifact.get(artifact.artifact_id) ?? [];
const staticFindings = staticFindingsByArtifact.get(artifact.artifact_id) ?? [];
const split = splitByArtifact.get(artifact.artifact_id);
if (!split) return [];
const staticScan = scanByName(scans, "static");
const vtScan = scanByName(scans, "virustotal");
const skillSpectorScan = scanByName(scans, "skillspector");
const llmScan = scanByName(scans, "llm");
const verdict = labelBySource(labels, "moderation_consensus")?.label ?? "unknown";
return [
{
id: flatArtifactId(artifact),
skill_slug: artifact.public_qualified_slug ?? artifact.public_slug,
skill_version: artifact.version,
skill_md_content: artifact.skill_md_content_redacted ?? null,
skill_bundle_content: (artifact.bundle_files_redacted ?? []).map((file) => ({
path: file.path,
content: file.content,
sha256: file.sha256,
sizeBytes: file.size_bytes,
})),
clawscan_verdict: verdict,
clawscan_confidence: llmScan?.confidence ?? null,
clawscan_model: llmScan?.model ?? null,
clawscan_summary: llmScan?.summary_redacted ?? null,
static_status: staticScan?.status ?? null,
static_finding_count: staticFindings.length,
static_reason_codes: staticScan?.reason_codes ?? [],
virustotal_status: vtScan?.status ?? null,
virustotal_malicious_count: vtScan?.engine_stats?.malicious ?? null,
virustotal_suspicious_count: vtScan?.engine_stats?.suspicious ?? null,
virustotal_harmless_count: vtScan?.engine_stats?.harmless ?? null,
virustotal_undetected_count: vtScan?.engine_stats?.undetected ?? null,
skillspector_status: skillSpectorScan?.status ?? null,
skillspector_score: skillSpectorScan?.score ?? null,
skillspector_severity: skillSpectorScan?.severity ?? null,
skillspector_issue_count: skillSpectorScan?.issues?.length ?? 0,
skillspector_issue_codes: skillSpectorScan?.reason_codes ?? [],
skillspector_issue_categories: uniqueSorted(
(skillSpectorScan?.issues ?? []).flatMap((issue) =>
issue.category ? [issue.category] : [],
),
),
clawscan_context: buildClawScanContext({
staticScan,
vtScan,
skillSpectorScan,
staticFindings,
}),
split,
},
];
});
}
function buildClawScanContext(input: {
staticScan: ScanResultRow | undefined;
vtScan: ScanResultRow | undefined;
skillSpectorScan: ScanResultRow | undefined;
staticFindings: StaticFindingRow[];
}) {
const context: Record<string, unknown> = {};
const { staticScan, vtScan, skillSpectorScan, staticFindings } = input;
if (staticScan) {
context.static = {
status: staticScan.status,
verdict: staticScan.verdict,
reason_codes: staticScan.reason_codes,
summary: staticScan.summary_redacted,
checked_at: isoTime(staticScan.checked_at),
scanner_version: staticScan.scanner_version,
finding_count: staticFindings.length,
};
}
if (vtScan) {
context.virustotal = {
status: vtScan.status,
verdict: vtScan.verdict,
checked_at: isoTime(vtScan.checked_at),
scanner_version: vtScan.scanner_version,
engine_stats: vtScan.engine_stats,
};
}
if (skillSpectorScan) {
context.skillspector = {
status: skillSpectorScan.status,
verdict: skillSpectorScan.verdict,
checked_at: isoTime(skillSpectorScan.checked_at),
scanner_version: skillSpectorScan.scanner_version,
issue_codes: skillSpectorScan.reason_codes,
score: skillSpectorScan.score ?? null,
severity: skillSpectorScan.severity ?? null,
issue_count: skillSpectorScan.issues?.length ?? 0,
issues: (skillSpectorScan.issues ?? []).map((issue) => ({
code: issue.code,
category: issue.category,
severity: issue.severity,
confidence: issue.confidence,
explanation: issue.explanation_redacted,
})),
};
}
return context;
}
function flatArtifactId(artifact: ArtifactRow) {
if (artifact.artifact_sha256) return artifact.artifact_sha256;
return hashString(artifact.artifact_id);
}
function scanByName(scans: ScanResultRow[], scanner: ScanResultRow["scanner"]) {
return scans.find((row) => row.scanner === scanner);
}
function labelBySource(labels: LabelRow[], source: LabelRow["label_source"]) {
return labels.find((row) => row.label_source === source);
}
function groupBy<T>(items: T[], getKey: (item: T) => string) {
const map = new Map<string, T[]>();
for (const item of items) {
const key = getKey(item);
const group = map.get(key);
if (group) {
group.push(item);
} else {
map.set(key, [item]);
}
}
return map;
}
function uniqueSorted(values: string[]) {
return Array.from(new Set(values)).sort((left, right) => left.localeCompare(right));
}
function isoTime(value: number | null) {
return value === null ? null : new Date(value).toISOString();
}
scripts/security-dataset/manifest.ts
+15
View File
@@ -1,5 +1,6 @@
export type SnapshotManifestInput = {
snapshotId: string;
sourceSnapshotId?: string | null;
createdAt: string;
repoGitSha: string;
convexDeployment: string;
@@ -16,7 +17,9 @@ export type SnapshotManifestInput = {
clawScanFindings: number;
labels: number;
splits: number;
huggingFaceRows?: number;
};
outputSizes?: Record<string, number>;
scannerVersions: string[];
modelNames: string[];
redactionPolicyVersion: string;
@@ -25,11 +28,20 @@ export type SnapshotManifestInput = {
createdAtGte: number | null;
createdAtLt: number | null;
};
huggingFaceDataset?: {
repo: string;
revision: string;
commit: string | null;
configNames: string[];
splitNames: string[];
rowCountsBySplit: Record<string, number>;
};
};
export function buildSecurityDatasetManifest(input: SnapshotManifestInput) {
return {
snapshot_id: input.snapshotId,
source_snapshot_id: input.sourceSnapshotId ?? input.snapshotId,
created_at: input.createdAt,
repo_git_sha: input.repoGitSha,
convex_deployment: input.convexDeployment,
@@ -47,7 +59,9 @@ export function buildSecurityDatasetManifest(input: SnapshotManifestInput) {
clawscan_findings: input.rowCounts.clawScanFindings,
labels: input.rowCounts.labels,
splits: input.rowCounts.splits,
huggingface_rows: input.rowCounts.huggingFaceRows ?? 0,
},
output_sizes: input.outputSizes ?? {},
scanner_versions: input.scannerVersions,
model_names: input.modelNames,
redaction_policy_version: input.redactionPolicyVersion,
@@ -57,6 +71,7 @@ export function buildSecurityDatasetManifest(input: SnapshotManifestInput) {
created_at_gte: input.timeWindow?.createdAtGte ?? null,
created_at_lt: input.timeWindow?.createdAtLt ?? null,
},
huggingface_dataset: input.huggingFaceDataset ?? null,
};
}