From e82f0704c8626b6f5475a41831aaa52e9f48c7ad Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Mon, 22 Jun 2026 22:40:09 -0700 Subject: [PATCH] Publish security dataset from live Convex export (#2800) * feat(cli): restore skill sync command * feat: publish security dataset from live export --- .../workflows/security-dataset-snapshot.yml | 220 ++++++++++++++++ convex/securityDatasetNode.test.ts | 130 ++++++++++ convex/securityDatasetNode.ts | 137 +++++++--- scripts/security-dataset/export-snapshot.ts | 239 ++++++++++++++++-- .../exportSnapshotCli.test.ts | 151 +++++++++++ .../huggingFaceExport.test.ts | 176 +++++++++++++ scripts/security-dataset/huggingFaceExport.ts | 197 +++++++++++++++ scripts/security-dataset/manifest.ts | 15 ++ 8 files changed, 1215 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/security-dataset-snapshot.yml create mode 100644 convex/securityDatasetNode.test.ts create mode 100644 scripts/security-dataset/exportSnapshotCli.test.ts create mode 100644 scripts/security-dataset/huggingFaceExport.test.ts create mode 100644 scripts/security-dataset/huggingFaceExport.ts diff --git a/.github/workflows/security-dataset-snapshot.yml b/.github/workflows/security-dataset-snapshot.yml new file mode 100644 index 00000000..b10aa18d --- /dev/null +++ b/.github/workflows/security-dataset-snapshot.yml @@ -0,0 +1,220 @@ +name: Security Dataset Snapshot + +on: + workflow_dispatch: + inputs: + upload: + description: "Upload sanitized dataset files to Hugging Face" + required: true + default: "false" + type: choice + options: + - "false" + - "true" + limit: + description: "Optional source artifact cap for validation runs" + required: false + default: "" + hf-revision: + description: "Hugging Face branch/revision to upload to" + required: true + default: "main" + schedule: + - cron: "17 9 * * *" + +permissions: + contents: read + id-token: write + +concurrency: + group: clawhub-security-dataset-snapshot + cancel-in-progress: false + +jobs: + publish-security-dataset: + name: Publish sanitized security dataset + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 360 + environment: Production + env: + CONVEX_URL: ${{ vars.CONVEX_URL || vars.VITE_CONVEX_URL || 'https://wry-manatee-359.convex.cloud' }} + SECURITY_SCAN_WORKER_TOKEN: ${{ secrets.SECURITY_SCAN_WORKER_TOKEN }} + HF_DATASET_REPO: OpenClaw/clawhub-security-signals + HF_OIDC_RESOURCE: datasets/OpenClaw/clawhub-security-signals + HF_REVISION: ${{ inputs.hf-revision || 'main' }} + HF_UPLOAD: ${{ github.event_name == 'schedule' || inputs.upload == 'true' }} + SNAPSHOT_LIMIT: ${{ inputs.limit || '' }} + SANITIZED_OUT_DIR: ${{ runner.temp }}/clawhub-security-dataset/sanitized + WORK_DIR: ${{ runner.temp }}/clawhub-security-dataset + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup-bun + + - uses: actions/setup-python@v6 + if: ${{ env.HF_UPLOAD == 'true' }} + with: + python-version: "3.12" + + - name: Check configuration + run: | + set -euo pipefail + if [[ -z "$SECURITY_SCAN_WORKER_TOKEN" ]]; then + echo "::error::SECURITY_SCAN_WORKER_TOKEN is required" + exit 1 + fi + echo "Upload enabled: $HF_UPLOAD" + echo "Convex URL: $CONVEX_URL" + echo "Hugging Face repo: $HF_DATASET_REPO" + echo "Hugging Face OIDC resource: $HF_OIDC_RESOURCE" + echo "Hugging Face revision: $HF_REVISION" + + - name: Export sanitized live dataset + run: | + set -euo pipefail + mkdir -p "$SANITIZED_OUT_DIR" + source_snapshot_id="live-convex-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + args=( + --convex-url "$CONVEX_URL" + --worker-token "$SECURITY_SCAN_WORKER_TOKEN" + --source-snapshot-id "$source_snapshot_id" + --out-dir "$SANITIZED_OUT_DIR" + --hf-dataset + --hf-repo "$HF_DATASET_REPO" + --hf-revision "$HF_REVISION" + ) + if [[ -n "$SNAPSHOT_LIMIT" ]]; then + args+=(--limit "$SNAPSHOT_LIMIT") + fi + bun scripts/security-dataset/export-snapshot.ts "${args[@]}" | tee "$SANITIZED_OUT_DIR/summary.json" + snapshot_dir="$(jq -r '.snapshotDir' "$SANITIZED_OUT_DIR/summary.json")" + if [[ -z "$snapshot_dir" || "$snapshot_dir" == "null" ]]; then + echo "::error::export summary did not include snapshotDir" + exit 1 + fi + echo "SNAPSHOT_DIR=$snapshot_dir" >> "$GITHUB_ENV" + jq '.manifest.row_counts, .manifest.huggingface_dataset' "$SANITIZED_OUT_DIR/summary.json" + + - name: Validate sanitized output guardrails + run: | + set -euo pipefail + test -d "$SNAPSHOT_DIR/hf-dataset/data" + test -f "$SNAPSHOT_DIR/manifest.json" + for split in train validation test eval_holdout; do + test -f "$SNAPSHOT_DIR/hf-dataset/data/$split.jsonl" + done + if grep -R -E 'storageId|skillVersions:|packageReleases:|_storage/' "$SNAPSHOT_DIR/hf-dataset" "$SNAPSHOT_DIR/manifest.json"; then + echo "::error::sanitized output contains raw storage or internal document identifiers" + exit 1 + fi + if grep -R -E 'gh[pousr]_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9_-]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN [A-Z0-9 ]*(PRIVATE KEY|CERTIFICATE)-----' "$SNAPSHOT_DIR/hf-dataset"; then + echo "::error::sanitized output contains obvious secret-like values" + exit 1 + fi + + - name: Install Hugging Face uploader + if: ${{ env.HF_UPLOAD == 'true' }} + run: | + set -euo pipefail + python -m pip install --upgrade pip + python -m pip install 'huggingface_hub[hf_xet]' + + - name: Upload sanitized dataset to Hugging Face + if: ${{ env.HF_UPLOAD == 'true' }} + run: | + set -euo pipefail + python - <<'PY' + import json + import os + import urllib.request + from pathlib import Path + from huggingface_hub import HfApi + + def get_github_oidc_token() -> str: + request_url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"] + separator = "&" if "?" in request_url else "?" + request = urllib.request.Request( + f"{request_url}{separator}audience=https://huggingface.co", + headers={ + "Authorization": f"bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}", + "Accept": "application/json", + }, + ) + with urllib.request.urlopen(request) as response: + payload = json.loads(response.read().decode("utf-8")) + return payload["value"] + + def exchange_hugging_face_token(oidc_token: str, resource: str) -> str: + body = json.dumps({ + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "subject_token_type": "urn:ietf:params:oauth:token-type:id_token", + "subject_token": oidc_token, + "resource": resource, + }).encode("utf-8") + request = urllib.request.Request( + "https://huggingface.co/oauth/token", + data=body, + headers={ + "Content-Type": "application/json", + "Accept": "application/json", + }, + method="POST", + ) + with urllib.request.urlopen(request) as response: + payload = json.loads(response.read().decode("utf-8")) + return payload["access_token"] + + snapshot_dir = Path(os.environ["SNAPSHOT_DIR"]) + repo_id = os.environ["HF_DATASET_REPO"] + revision = os.environ["HF_REVISION"] + token = exchange_hugging_face_token( + get_github_oidc_token(), + os.environ["HF_OIDC_RESOURCE"], + ) + api = HfApi(token=token) + + data_commit = api.upload_folder( + folder_path=str(snapshot_dir / "hf-dataset" / "data"), + path_in_repo="data", + repo_id=repo_id, + repo_type="dataset", + revision=revision, + delete_patterns="data/*.jsonl", + commit_message="Update nightly ClawHub security dataset splits", + ) + + manifest_path = snapshot_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["huggingface_dataset"]["commit"] = data_commit.oid + manifest["huggingface_dataset"]["revision"] = revision + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + manifest_commit = api.upload_file( + path_or_fileobj=str(manifest_path), + path_in_repo="metadata/latest-manifest.json", + repo_id=repo_id, + repo_type="dataset", + revision=revision, + commit_message="Update nightly ClawHub security dataset manifest", + ) + print(json.dumps({ + "data_commit": data_commit.oid, + "manifest_commit": manifest_commit.oid, + "repo": repo_id, + "revision": revision, + }, indent=2)) + PY + + - name: Upload sanitized summary artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: security-dataset-summary-${{ github.run_id }} + path: | + ${{ env.SANITIZED_OUT_DIR }}/summary.json + ${{ env.SNAPSHOT_DIR }}/manifest.json + if-no-files-found: ignore + + - name: Cleanup transient dataset files + if: ${{ always() }} + run: rm -rf "$WORK_DIR" diff --git a/convex/securityDatasetNode.test.ts b/convex/securityDatasetNode.test.ts new file mode 100644 index 00000000..25b0daa6 --- /dev/null +++ b/convex/securityDatasetNode.test.ts @@ -0,0 +1,130 @@ +/* @vitest-environment node */ +import { gunzipSync } from "node:zlib"; +import { describe, expect, it, vi } from "vitest"; +import { listArtifactExportBatchCompressed } from "./securityDatasetNode"; + +type WrappedHandler = { + _handler: (ctx: unknown, args: TArgs) => Promise; +}; + +const listArtifactExportBatchCompressedHandler = ( + listArtifactExportBatchCompressed as unknown as WrappedHandler< + { + token: string; + sourceKind: "skill"; + paginationOpts: { cursor: string | null; numItems: number }; + pageCount: number; + }, + { encoding: "gzip-base64-json"; payload: string } + > +)._handler; + +describe("security dataset worker export", () => { + it("requires the shared worker token", async () => { + vi.stubEnv("SECURITY_SCAN_WORKER_TOKEN", "worker-secret"); + + await expect( + listArtifactExportBatchCompressedHandler(makeCtx(), { + token: "wrong", + sourceKind: "skill", + paginationOpts: { cursor: null, numItems: 1 }, + pageCount: 1, + }), + ).rejects.toThrow("Unauthorized"); + }); + + it("returns redacted storage-backed content without storage ids", async () => { + vi.stubEnv("SECURITY_SCAN_WORKER_TOKEN", "worker-secret"); + const result = await listArtifactExportBatchCompressedHandler(makeCtx(), { + token: "worker-secret", + sourceKind: "skill", + paginationOpts: { cursor: null, numItems: 1 }, + pageCount: 1, + }); + + const decoded = JSON.parse( + gunzipSync(Buffer.from(result.payload, "base64")).toString("utf8"), + ) as { + page: Array<{ + skillMdContentRedacted: string; + bundleFilesRedacted: Array<{ path: string; content: string }>; + files: Array>; + }>; + }; + + expect(decoded.page).toEqual([ + expect.objectContaining({ + skillMdContentRedacted: "Use this skill. [REDACTED_SECRET]", + bundleFilesRedacted: [ + { + path: "scripts/run.sh", + content: "echo [REDACTED_SECRET]\n", + }, + ], + files: [ + expect.not.objectContaining({ storageId: expect.anything() }), + expect.not.objectContaining({ storageId: expect.anything() }), + ], + }), + ]); + }); +}); + +function makeCtx() { + return { + runQuery: vi.fn(async () => ({ + page: [ + { + sourceKind: "skill", + sourceDocId: "skillVersions:1", + parentDocId: "skills:1", + publicName: "Demo", + publicOwnerHandle: "owner", + publicSlug: "demo", + version: "1.0.0", + artifactSha256: "a".repeat(64), + createdAt: 1, + softDeletedAt: null, + files: [ + { + path: "SKILL.md", + size: 42, + sha256: "skill-sha", + storageId: "storage:skill", + contentType: "text/markdown", + }, + { + path: "scripts/run.sh", + size: 32, + sha256: "script-sha", + storageId: "storage:script", + contentType: "text/x-shellscript", + }, + ], + packageFamily: null, + packageChannel: null, + sourceRepoHost: null, + vtAnalysis: null, + skillSpectorAnalysis: null, + staticScan: null, + llmAnalysis: null, + moderationConsensus: null, + }, + ], + isDone: true, + continueCursor: "", + exportMode: "public", + })), + storage: { + get: vi.fn(async (id: string) => { + if (id === "storage:skill") { + return new Blob(["Use this skill. token=supersecret123"]); + } + if (id === "storage:script") { + return new Blob(["echo password=scriptsecret123\n"]); + } + return null; + }), + }, + }; +} diff --git a/convex/securityDatasetNode.ts b/convex/securityDatasetNode.ts index 22c0b9ab..edcae614 100644 --- a/convex/securityDatasetNode.ts +++ b/convex/securityDatasetNode.ts @@ -2,10 +2,10 @@ import { gzipSync } from "node:zlib"; import { paginationOptsValidator } from "convex/server"; -import { v } from "convex/values"; +import { ConvexError, v } from "convex/values"; import { internal } from "./_generated/api"; import type { ActionCtx } from "./_generated/server"; -import { internalAction } from "./functions"; +import { action, internalAction } from "./functions"; const MAX_EXPORT_BATCH_PAGES = 20; const MAX_REDACTED_BUNDLE_FILE_BYTES = 192 * 1024; @@ -19,6 +19,21 @@ type ArtifactExportPage = { exportMode: "public"; }; +type DatasetLineage = { + exportMode: "public"; + generatedAt: number; + maxExportPageSize: number; + maxExportBatchPages: number; + redactionPolicyVersion: string; + sourceTables: readonly string[]; + scannerSources: readonly string[]; + sourceBounds: Array<{ + sourceKind: "skill" | "package"; + minCreatedAt: number | null; + maxCreatedAt: number | null; + }>; +}; + const SECRET_PATTERNS: RegExp[] = [ /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/g, @@ -31,6 +46,24 @@ const SECRET_PATTERNS: RegExp[] = [ /(["'`])(?=[A-Za-z0-9+/=_-]{32,}\1)(?=.*[A-Z])(?=.*[a-z])(?=.*\d)[A-Za-z0-9+/=_-]+\1/g, ]; +type ArtifactExportBatchArgs = { + sourceKind: "skill" | "package"; + mode?: "public"; + createdAtGte?: number; + createdAtLt?: number; + paginationOpts: { + cursor: string | null; + numItems: number; + }; + pageCount: number; +}; + +function assertDatasetExportWorkerToken(token: string) { + // Shared worker credential already used by the security and Skill Card GitHub workers. + const expected = process.env.SECURITY_SCAN_WORKER_TOKEN; + if (!expected || token !== expected) throw new ConvexError("Unauthorized"); +} + export const listArtifactExportBatchCompressedInternal = internalAction({ args: { sourceKind: v.union(v.literal("skill"), v.literal("package")), @@ -40,43 +73,77 @@ export const listArtifactExportBatchCompressedInternal = internalAction({ paginationOpts: paginationOptsValidator, pageCount: v.number(), }, + handler: async (ctx, args) => listArtifactExportBatchCompressedForWorker(ctx, args), +}); + +export const listArtifactExportBatchCompressed = action({ + args: { + token: v.string(), + sourceKind: v.union(v.literal("skill"), v.literal("package")), + mode: v.optional(v.literal("public")), + createdAtGte: v.optional(v.number()), + createdAtLt: v.optional(v.number()), + paginationOpts: paginationOptsValidator, + pageCount: v.number(), + }, handler: async (ctx, args) => { - const pageCount = Math.min(Math.max(1, Math.floor(args.pageCount)), MAX_EXPORT_BATCH_PAGES); - let cursor = args.paginationOpts.cursor; - const page: ArtifactExportPage["page"] = []; - let isDone = false; - for (let pageIndex = 0; pageIndex < pageCount; pageIndex += 1) { - const result: ArtifactExportPage = await ctx.runQuery( - internal.securityDataset.listArtifactExportPageInternal, - { - sourceKind: args.sourceKind, - mode: args.mode, - createdAtGte: args.createdAtGte, - createdAtLt: args.createdAtLt, - paginationOpts: { - cursor, - numItems: args.paginationOpts.numItems, - }, - }, - ); - page.push(...result.page); - cursor = result.continueCursor; - isDone = result.isDone; - if (isDone) break; - } - const json = JSON.stringify({ - page: await enrichAndSanitizeArtifactRows(ctx, page), - isDone, - continueCursor: cursor, - exportMode: args.mode ?? "public", - }); - return { - encoding: "gzip-base64-json" as const, - payload: gzipSync(json).toString("base64"), - }; + assertDatasetExportWorkerToken(args.token); + return await listArtifactExportBatchCompressedForWorker(ctx, args); }, }); +export const getDatasetLineage = action({ + args: { + token: v.string(), + mode: v.optional(v.literal("public")), + }, + handler: async (ctx, args): Promise => { + assertDatasetExportWorkerToken(args.token); + return await ctx.runQuery(internal.securityDataset.getDatasetLineageInternal, { + mode: args.mode, + }); + }, +}); + +async function listArtifactExportBatchCompressedForWorker( + ctx: ActionCtx, + args: ArtifactExportBatchArgs, +) { + const pageCount = Math.min(Math.max(1, Math.floor(args.pageCount)), MAX_EXPORT_BATCH_PAGES); + let cursor = args.paginationOpts.cursor; + const page: ArtifactExportPage["page"] = []; + let isDone = false; + for (let pageIndex = 0; pageIndex < pageCount; pageIndex += 1) { + const result: ArtifactExportPage = await ctx.runQuery( + internal.securityDataset.listArtifactExportPageInternal, + { + sourceKind: args.sourceKind, + mode: args.mode, + createdAtGte: args.createdAtGte, + createdAtLt: args.createdAtLt, + paginationOpts: { + cursor, + numItems: args.paginationOpts.numItems, + }, + }, + ); + page.push(...result.page); + cursor = result.continueCursor; + isDone = result.isDone; + if (isDone) break; + } + const json = JSON.stringify({ + page: await enrichAndSanitizeArtifactRows(ctx, page), + isDone, + continueCursor: cursor, + exportMode: args.mode ?? "public", + }); + return { + encoding: "gzip-base64-json" as const, + payload: gzipSync(json).toString("base64"), + }; +} + async function enrichAndSanitizeArtifactRows(ctx: ActionCtx, rows: unknown[]) { const enrichedRows = []; let remainingBundleBytes = MAX_REDACTED_BUNDLE_BYTES_PER_RESPONSE; diff --git a/scripts/security-dataset/export-snapshot.ts b/scripts/security-dataset/export-snapshot.ts index 33071f6a..d86655f3 100644 --- a/scripts/security-dataset/export-snapshot.ts +++ b/scripts/security-dataset/export-snapshot.ts @@ -1,17 +1,24 @@ import { execFile, spawnSync } from "node:child_process"; import { once } from "node:events"; import { createWriteStream, type WriteStream } from "node:fs"; -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, readdir, stat, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { promisify } from "node:util"; import { gunzipSync } from "node:zlib"; +import { ConvexHttpClient } from "convex/browser"; +import { api } from "../../convex/_generated/api"; import { artifactInputsFromConvexExportZip } from "./convexExport"; import { parseConvexJsonMatching } from "./convexOutput"; import { reserveExportInputs } from "./exportLimit"; +import { + buildHuggingFaceSecuritySignalRows, + type HuggingFaceSecuritySignalRow, +} from "./huggingFaceExport"; import { buildSecurityDatasetManifest } from "./manifest"; import { normalizeArtifactExport, type ArtifactExportInput, + type DatasetSplit, type NormalizedDatasetRows, type SourceKind, } from "./normalize"; @@ -38,6 +45,14 @@ type ConvexBounds = { maxCreatedAt: number | null; }; +type DatasetLineage = { + exportMode: "public"; + generatedAt: number; + redactionPolicyVersion: string; + sourceTables: string[]; + sourceBounds: ConvexBounds[]; +}; + type CompressedConvexPage = { encoding: "gzip-base64-json"; payload: string; @@ -45,6 +60,8 @@ type CompressedConvexPage = { type Options = { deployment: string | null; + convexUrl: string | null; + workerToken: string | null; prod: boolean; push: boolean; dryRun: boolean; @@ -58,6 +75,10 @@ type Options = { sourceKind: SourceKind | "all"; timeWindow: CreatedTimeWindow; convexExportZip: string | null; + sourceSnapshotId: string | null; + huggingFaceDataset: boolean; + huggingFaceRepo: string; + huggingFaceRevision: string; }; type ExportShard = { @@ -76,7 +97,9 @@ type SnapshotState = { clawScanFindings: number; labels: number; splits: number; + huggingFaceRows: number; }; + huggingFaceRowCountsBySplit: Record; scannerVersions: Set; modelNames: Set; }; @@ -88,6 +111,7 @@ type SnapshotWriters = { clawScanFindings: WriteStream; labels: WriteStream; splits: WriteStream; + huggingFaceSplits?: Record; }; const DEFAULT_PAGE_SIZE = 50; @@ -98,12 +122,13 @@ const DEFAULT_MAX_CONVEX_ATTEMPTS = 6; const DEFAULT_OUT_DIR = ".data/security-dataset/snapshots"; const CONVEX_RUN_MAX_BUFFER_BYTES = 128 * 1024 * 1024; const SOURCE_KINDS: SourceKind[] = ["skill", "package"]; +const HF_SPLITS: DatasetSplit[] = ["train", "validation", "test", "eval_holdout"]; async function main() { const options = parseArgs(process.argv.slice(2)); const snapshotId = buildSnapshotId(options); const snapshotDir = resolve(options.outDir, snapshotId); - const writers = options.dryRun ? null : await openSnapshotWriters(snapshotDir); + const writers = options.dryRun ? null : await openSnapshotWriters(snapshotDir, options); let writersClosed = false; const state = createSnapshotState(); @@ -111,9 +136,15 @@ async function main() { const shardCount = options.convexExportZip ? await exportConvexExportZip({ options, state, writers }) : await exportRemoteShards({ options, state, writers }); - const manifest = buildManifest({ options, snapshotId, state, shardCount }); if (options.dryRun) { + const manifest = buildManifest({ + options, + snapshotId, + state, + shardCount, + outputSizes: {}, + }); console.log(JSON.stringify({ snapshotId, dryRun: true, manifest }, null, 2)); return; } @@ -121,6 +152,8 @@ async function main() { if (!writers) throw new Error("Snapshot writers were not opened."); await closeSnapshotWriters(writers); writersClosed = true; + const outputSizes = await collectOutputSizes(snapshotDir); + const manifest = buildManifest({ options, snapshotId, state, shardCount, outputSizes }); await writeFile(join(snapshotDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); console.log(JSON.stringify({ snapshotId, snapshotDir, manifest }, null, 2)); @@ -231,6 +264,7 @@ async function runConvexPage( batchPages: number, ): Promise<{ page: ConvexPage; batchPages: number }> { const functionName = "securityDatasetNode:listArtifactExportBatchCompressedInternal"; + const workerFunctionName = "securityDatasetNode:listArtifactExportBatchCompressed"; let pageCount = batchPages; while (true) { @@ -246,12 +280,19 @@ async function runConvexPage( let lastError: unknown = null; for (let attempt = 1; attempt <= DEFAULT_MAX_CONVEX_ATTEMPTS; attempt += 1) { try { - const compressed = await runConvexJsonOnce( - options, - functionName, - args, - isCompressedConvexPage, - ); + const compressed = options.workerToken + ? await runWorkerAction( + options, + workerFunctionName, + { ...args, token: options.workerToken }, + isCompressedConvexPage, + ) + : await runConvexJsonOnce( + options, + functionName, + args, + isCompressedConvexPage, + ); return { page: decodeCompressedConvexPage(compressed), batchPages: pageCount }; } catch (error) { lastError = error; @@ -279,6 +320,19 @@ async function runConvexPage( } async function runConvexBounds(options: Options, sourceKind: SourceKind): Promise { + if (options.workerToken) { + const lineage = await runWorkerAction( + options, + "securityDatasetNode:getDatasetLineage", + { token: options.workerToken, mode: options.mode }, + isDatasetLineage, + ); + const bounds = lineage.sourceBounds.find((candidate) => candidate.sourceKind === sourceKind); + if (!bounds) { + return { sourceKind, minCreatedAt: null, maxCreatedAt: null }; + } + return bounds; + } return runConvexJson( options, "securityDataset:getArtifactExportBoundsInternal", @@ -287,6 +341,31 @@ async function runConvexBounds(options: Options, sourceKind: SourceKind): Promis ); } +async function runWorkerAction( + options: Options, + functionName: string, + args: unknown, + validate: (value: unknown) => value is T, +): Promise { + if (!options.convexUrl) { + throw new Error("--convex-url or CONVEX_URL is required when using --worker-token."); + } + const client = new ConvexHttpClient(options.convexUrl); + const result = await client.action(resolveWorkerAction(functionName), args as never); + if (validate(result)) return result; + throw new Error(`Invalid ${functionName} response.`); +} + +function resolveWorkerAction(functionName: string) { + if (functionName === "securityDatasetNode:listArtifactExportBatchCompressed") { + return api.securityDatasetNode.listArtifactExportBatchCompressed; + } + if (functionName === "securityDatasetNode:getDatasetLineage") { + return api.securityDatasetNode.getDatasetLineage; + } + throw new Error(`Unsupported worker action: ${functionName}`); +} + async function runConvexJson( options: Options, functionName: string, @@ -352,14 +431,19 @@ function buildManifest(input: { snapshotId: string; state: SnapshotState; shardCount: number; + outputSizes: Record; }) { - const { options, snapshotId, state, shardCount } = input; + const { options, snapshotId, state, shardCount, outputSizes } = input; const repoGitSha = gitSha(); return buildSecurityDatasetManifest({ snapshotId, + sourceSnapshotId: options.sourceSnapshotId ?? snapshotId, createdAt: new Date().toISOString(), repoGitSha, - convexDeployment: options.deployment ?? (options.prod ? "prod" : "configured-dev"), + convexDeployment: + options.deployment ?? + inferDeploymentFromConvexUrl(options.convexUrl) ?? + (options.prod ? "prod" : "configured-dev"), exportMode: options.mode, pageSize: options.pageSize, concurrency: options.concurrency, @@ -373,22 +457,45 @@ function buildManifest(input: { clawScanFindings: state.rowCounts.clawScanFindings, labels: state.rowCounts.labels, splits: state.rowCounts.splits, + huggingFaceRows: state.rowCounts.huggingFaceRows, }, + outputSizes, scannerVersions: Array.from(state.scannerVersions).sort(), modelNames: Array.from(state.modelNames).sort(), redactionPolicyVersion: "public-signals-v2-bundle-files", sourceTables: ["skillVersions", "packageReleases"], timeWindow: options.timeWindow, + huggingFaceDataset: options.huggingFaceDataset + ? { + repo: options.huggingFaceRepo, + revision: options.huggingFaceRevision, + commit: null, + configNames: ["default"], + splitNames: HF_SPLITS, + rowCountsBySplit: state.huggingFaceRowCountsBySplit, + } + : undefined, }); } +function inferDeploymentFromConvexUrl(convexUrl: string | null) { + if (!convexUrl) return null; + try { + return new URL(convexUrl).hostname.split(".")[0] || null; + } catch { + return null; + } +} + function buildSnapshotId(options: Options) { const timestamp = new Date() .toISOString() .replace(/[-:]/g, "") .replace(/\.\d{3}Z$/, "Z"); const deployment = - options.deployment?.replace(/[^a-zA-Z0-9]+/g, "-") ?? (options.prod ? "prod" : "dev"); + options.deployment?.replace(/[^a-zA-Z0-9]+/g, "-") ?? + inferDeploymentFromConvexUrl(options.convexUrl)?.replace(/[^a-zA-Z0-9]+/g, "-") ?? + (options.prod ? "prod" : "dev"); return `clawhub-${deployment}-${timestamp}-${gitSha().slice(0, 8)}`; } @@ -402,6 +509,13 @@ function createSnapshotState(): SnapshotState { clawScanFindings: 0, labels: 0, splits: 0, + huggingFaceRows: 0, + }, + huggingFaceRowCountsBySplit: { + train: 0, + validation: 0, + test: 0, + eval_holdout: 0, }, scannerVersions: new Set(), modelNames: new Set(), @@ -415,12 +529,17 @@ async function processArtifactInputs(input: { }) { const { inputs, state, writers } = input; const rows = normalizeArtifactExport(inputs); + const hfRows = buildHuggingFaceSecuritySignalRows(rows); state.rowCounts.artifacts += rows.artifacts.length; state.rowCounts.scanResults += rows.scanResults.length; state.rowCounts.staticFindings += rows.staticFindings.length; state.rowCounts.clawScanFindings += rows.clawScanFindings.length; state.rowCounts.labels += rows.labels.length; state.rowCounts.splits += rows.splits.length; + state.rowCounts.huggingFaceRows += hfRows.length; + for (const row of hfRows) { + state.huggingFaceRowCountsBySplit[row.split] += 1; + } for (const row of rows.scanResults) { if (row.scanner_version) state.scannerVersions.add(row.scanner_version); if (row.model) state.modelNames.add(row.model); @@ -428,11 +547,17 @@ async function processArtifactInputs(input: { if (!writers) return; await writeNormalizedRows(writers, rows); + if (writers.huggingFaceSplits) { + await writeHuggingFaceRows(writers.huggingFaceSplits, hfRows); + } } -async function openSnapshotWriters(snapshotDir: string): Promise { +async function openSnapshotWriters( + snapshotDir: string, + options: Options, +): Promise { await mkdir(snapshotDir, { recursive: true }); - return { + const writers: SnapshotWriters = { artifacts: createWriteStream(join(snapshotDir, "artifacts.jsonl"), { encoding: "utf8" }), scanResults: createWriteStream(join(snapshotDir, "scan_results.jsonl"), { encoding: "utf8" }), staticFindings: createWriteStream(join(snapshotDir, "static_findings.jsonl"), { @@ -444,10 +569,30 @@ async function openSnapshotWriters(snapshotDir: string): Promise endStream(stream))); + const streams = [ + writers.artifacts, + writers.scanResults, + writers.staticFindings, + writers.clawScanFindings, + writers.labels, + writers.splits, + ...Object.values(writers.huggingFaceSplits ?? {}), + ]; + await Promise.all(streams.map((stream) => endStream(stream))); } async function endStream(stream: WriteStream) { @@ -470,6 +615,37 @@ async function writeJsonlRows(stream: WriteStream, rows: unknown[]) { if (!stream.write(chunk)) await once(stream, "drain"); } +async function writeHuggingFaceRows( + streams: Record, + rows: HuggingFaceSecuritySignalRow[], +) { + for (const split of HF_SPLITS) { + await writeJsonlRows( + streams[split], + rows.filter((row) => row.split === split), + ); + } +} + +async function collectOutputSizes(root: string) { + const sizes: Record = {}; + await collectOutputSizesInto(root, root, sizes); + return sizes; +} + +async function collectOutputSizesInto(root: string, dir: string, sizes: Record) { + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const path = join(dir, entry.name); + if (entry.isDirectory()) { + await collectOutputSizesInto(root, path, sizes); + } else if (entry.isFile()) { + const relativePath = path.slice(root.length + 1); + sizes[relativePath] = (await stat(path)).size; + } + } +} + function boundsToShards(bounds: ConvexBounds, shardCount: number): ExportShard[] { if (bounds.minCreatedAt === null || bounds.maxCreatedAt === null) return []; const start = bounds.minCreatedAt; @@ -547,6 +723,18 @@ function isCompressedConvexPage(value: unknown): value is CompressedConvexPage { ); } +function isDatasetLineage(value: unknown): value is DatasetLineage { + return ( + isRecord(value) && + value.exportMode === "public" && + typeof value.generatedAt === "number" && + typeof value.redactionPolicyVersion === "string" && + Array.isArray(value.sourceTables) && + Array.isArray(value.sourceBounds) && + value.sourceBounds.every(isConvexBounds) + ); +} + function decodeCompressedConvexPage(value: CompressedConvexPage) { const json = gunzipSync(Buffer.from(value.payload, "base64")).toString("utf8"); const parsed: unknown = JSON.parse(json); @@ -588,6 +776,8 @@ function gitSha() { function parseArgs(args: string[]): Options { const options: Options = { deployment: null, + convexUrl: null, + workerToken: null, prod: false, push: false, dryRun: false, @@ -601,6 +791,10 @@ function parseArgs(args: string[]): Options { sourceKind: "all", timeWindow: emptyCreatedTimeWindow(), convexExportZip: null, + sourceSnapshotId: null, + huggingFaceDataset: false, + huggingFaceRepo: process.env.HF_DATASET_REPO ?? "OpenClaw/clawhub-security-signals", + huggingFaceRevision: process.env.HF_REVISION ?? "main", }; for (let index = 0; index < args.length; index += 1) { @@ -613,6 +807,10 @@ function parseArgs(args: string[]): Options { options.dryRun = true; } else if (arg === "--deployment") { options.deployment = readValue(args, ++index, arg); + } else if (arg === "--convex-url") { + options.convexUrl = readValue(args, ++index, arg); + } else if (arg === "--worker-token") { + options.workerToken = readValue(args, ++index, arg); } else if (arg === "--limit") { options.limit = readPositiveInt(readValue(args, ++index, arg), arg); } else if (arg === "--page-size") { @@ -633,6 +831,14 @@ function parseArgs(args: string[]): Options { options.timeWindow.createdAtLt = parseCreatedTimestamp(readValue(args, ++index, arg), arg); } else if (arg === "--convex-export-zip" || arg === "--from-convex-export") { options.convexExportZip = readValue(args, ++index, arg); + } else if (arg === "--source-snapshot-id") { + options.sourceSnapshotId = readValue(args, ++index, arg); + } else if (arg === "--hf-dataset") { + options.huggingFaceDataset = true; + } else if (arg === "--hf-repo") { + options.huggingFaceRepo = readValue(args, ++index, arg); + } else if (arg === "--hf-revision") { + options.huggingFaceRevision = readValue(args, ++index, arg); } else if (arg === "--mode") { const mode = readValue(args, ++index, arg); if (mode !== "public") throw new Error(`Unsupported mode: ${mode}`); @@ -645,6 +851,9 @@ function parseArgs(args: string[]): Options { if (options.prod && options.deployment) { throw new Error("Use either --prod or --deployment, not both."); } + if (options.workerToken && (options.prod || options.deployment || options.push)) { + throw new Error("Use --worker-token with --convex-url instead of --prod/--deployment/--push."); + } assertCreatedTimeWindow(options.timeWindow); return options; } diff --git a/scripts/security-dataset/exportSnapshotCli.test.ts b/scripts/security-dataset/exportSnapshotCli.test.ts new file mode 100644 index 00000000..ace7de22 --- /dev/null +++ b/scripts/security-dataset/exportSnapshotCli.test.ts @@ -0,0 +1,151 @@ +/* @vitest-environment node */ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { strToU8, zipSync } from "fflate"; +import { describe, expect, it } from "vitest"; + +const execFileAsync = promisify(execFile); + +describe("security dataset snapshot CLI", () => { + it("writes flat Hugging Face split files from a tiny sanitized Convex export", async () => { + const directory = await mkdtemp(join(tmpdir(), "clawhub-security-dataset-cli-")); + try { + const snapshotZip = join(directory, "snapshot.zip"); + const outDir = join(directory, "out"); + await writeFile(snapshotZip, Buffer.from(buildTinyConvexSnapshotZip())); + + const result = await execFileAsync( + "bun", + [ + "scripts/security-dataset/export-snapshot.ts", + "--convex-export-zip", + snapshotZip, + "--source-snapshot-id", + "live-export-prod-123-1", + "--out-dir", + outDir, + "--hf-dataset", + ], + { + cwd: process.cwd(), + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }, + ); + const summary: { + snapshotDir: string; + manifest: { + source_snapshot_id: string; + row_counts: { huggingface_rows: number }; + huggingface_dataset: { repo: string; rowCountsBySplit: Record }; + }; + } = JSON.parse(result.stdout); + + expect(summary.manifest.source_snapshot_id).toBe("live-export-prod-123-1"); + expect(summary.manifest.row_counts.huggingface_rows).toBe(1); + expect(summary.manifest.huggingface_dataset.repo).toBe("OpenClaw/clawhub-security-signals"); + const dataDir = join(summary.snapshotDir, "hf-dataset", "data"); + const files = await readdir(dataDir); + expect(files.sort()).toEqual([ + "eval_holdout.jsonl", + "test.jsonl", + "train.jsonl", + "validation.jsonl", + ]); + + const splitContents = await Promise.all( + files.map(async (file) => readFile(join(dataDir, file), "utf8")), + ); + const flatRows = splitContents + .join("") + .split(/\r?\n/) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as Record); + + expect(flatRows).toEqual([ + expect.objectContaining({ + id: "a".repeat(64), + skill_slug: "owner/stored-skill", + skill_version: "1.0.0", + skill_md_content: "Stored SKILL.md [REDACTED_SECRET]", + skill_bundle_content: [ + expect.objectContaining({ + path: "scripts/run.sh", + content: "echo [REDACTED_SECRET]\n", + }), + ], + }), + ]); + const serializedRows = JSON.stringify(flatRows); + expect(serializedRows).not.toContain("storageId"); + expect(serializedRows).not.toContain("skillVersions:1"); + expect(serializedRows).not.toContain("supersecret123"); + expect(serializedRows).not.toContain("scriptsecret123"); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); +}); + +function buildTinyConvexSnapshotZip() { + return zipSync({ + "skills/documents.jsonl": strToU8( + `${JSON.stringify({ + _id: "skills:1", + displayName: "Stored Skill", + slug: "stored-skill", + ownerUserId: "users:owner", + })}\n`, + ), + "skillVersions/documents.jsonl": strToU8( + `${JSON.stringify({ + _id: "skillVersions:1", + skillId: "skills:1", + version: "1.0.0", + createdAt: Date.UTC(2026, 5, 23), + sha256hash: "a".repeat(64), + files: [ + { + path: "SKILL.md", + size: 31, + sha256: "skill-md-sha", + content: "Stored SKILL.md token=supersecret123", + contentType: "text/markdown", + }, + { + path: "scripts/run.sh", + size: 35, + sha256: "script-sha", + content: "echo password=scriptsecret123\n", + contentType: "text/x-shellscript", + }, + ], + staticScan: { + status: "clean", + reasonCodes: [], + findings: [], + summary: "No suspicious patterns detected.", + engineVersion: "static-v1", + checkedAt: Date.UTC(2026, 5, 23), + }, + llmAnalysis: { + status: "completed", + verdict: "suspicious", + confidence: "medium", + summary: "Review before trusting.", + agenticRiskFindings: [], + model: "gpt-test", + checkedAt: Date.UTC(2026, 5, 23), + }, + })}\n`, + ), + "packages/documents.jsonl": strToU8(""), + "packageReleases/documents.jsonl": strToU8(""), + "users/documents.jsonl": strToU8( + `${JSON.stringify({ _id: "users:owner", handle: "owner" })}\n`, + ), + }); +} diff --git a/scripts/security-dataset/huggingFaceExport.test.ts b/scripts/security-dataset/huggingFaceExport.test.ts new file mode 100644 index 00000000..0de2a6c8 --- /dev/null +++ b/scripts/security-dataset/huggingFaceExport.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, it } from "vitest"; +import { buildHuggingFaceSecuritySignalRows } from "./huggingFaceExport"; +import { hashString, type NormalizedDatasetRows } from "./normalize"; + +describe("Hugging Face security dataset export", () => { + it("builds flat researcher rows from normalized security dataset sidecars", () => { + const artifactId = `skill:${"a".repeat(64)}`; + const normalized: NormalizedDatasetRows = { + artifacts: [ + { + artifact_id: artifactId, + source_kind: "skill", + source_table: "skillVersions", + source_doc_id_hash: hashString("skillVersions:1"), + parent_doc_id_hash: hashString("skills:1"), + public_name: "Demo Skill", + public_owner_handle: "openclaw", + public_slug: "demo-skill", + public_qualified_slug: "openclaw/demo-skill", + version: "1.0.0", + artifact_sha256: "a".repeat(64), + skill_md_content_redacted: "Use this skill with [REDACTED_SECRET]", + bundle_files_redacted: [ + { + path: "scripts/run.sh", + content: "echo [REDACTED_SECRET]\n", + sha256: "bundle-content-sha", + size_bytes: 23, + }, + ], + created_at: Date.UTC(2026, 5, 23), + created_month: "2026-06", + soft_deleted: false, + is_public: true, + file_count: 2, + total_bytes: 42, + file_ext_counts: { ".md": 1, ".sh": 1 }, + package_family: null, + package_channel: null, + source_repo_host: null, + has_vt_scan: true, + has_skillspector_scan: true, + has_static_scan: true, + has_llm_scan: true, + }, + ], + scanResults: [ + { + artifact_id: artifactId, + scanner: "static", + scanner_version: "static-v1", + model: null, + status: "clean", + verdict: "clean", + confidence: null, + checked_at: Date.UTC(2026, 5, 23), + reason_codes: [], + engine_stats: null, + summary_redacted: "No suspicious patterns detected.", + raw_status_family: "clean", + }, + { + artifact_id: artifactId, + scanner: "virustotal", + scanner_version: "vt-v3", + model: null, + status: "clean", + verdict: "clean", + confidence: null, + checked_at: Date.UTC(2026, 5, 23), + reason_codes: [], + engine_stats: { malicious: 0, suspicious: 0, harmless: 1, undetected: 65 }, + summary_redacted: null, + raw_status_family: "clean", + }, + { + artifact_id: artifactId, + scanner: "skillspector", + scanner_version: "2.0.0", + model: null, + status: "suspicious", + verdict: "CAUTION", + confidence: null, + checked_at: Date.UTC(2026, 5, 23), + score: 35, + severity: "MEDIUM", + reason_codes: ["SQP-1"], + issues: [ + { + code: "SQP-1", + category: "Skill Quality", + severity: "MEDIUM", + confidence: 0.9, + explanation_redacted: "Broad trigger scope.", + }, + ], + engine_stats: null, + summary_redacted: "Found broad trigger scope.", + raw_status_family: "suspicious", + }, + { + artifact_id: artifactId, + scanner: "llm", + scanner_version: null, + model: "gpt-test", + status: "completed", + verdict: "suspicious", + confidence: "medium", + checked_at: Date.UTC(2026, 5, 23), + reason_codes: [], + engine_stats: null, + summary_redacted: "Review before trusting.", + raw_status_family: "suspicious", + }, + ], + staticFindings: [ + { + artifact_id: artifactId, + finding_id: "finding-1", + code: "suspicious.env_credential_access", + severity: "warn", + file_path_hash: hashString("scripts/run.sh"), + file_ext: ".sh", + line_bucket: "1-20", + message: "Reads credentials", + evidence_redacted: "[REDACTED_SECRET]", + }, + ], + clawScanFindings: [], + labels: [ + { + artifact_id: artifactId, + label: "suspicious", + label_source: "moderation_consensus", + label_confidence: "derived_consensus", + reason_codes: ["suspicious.env_credential_access"], + scanner_agreement: 1, + notes_redacted: null, + }, + ], + splits: [ + { + artifact_id: artifactId, + split: "train", + split_version: "sha256-v1", + split_key: hashString("a".repeat(64)), + }, + ], + }; + + expect(buildHuggingFaceSecuritySignalRows(normalized)).toEqual([ + expect.objectContaining({ + id: "a".repeat(64), + skill_slug: "openclaw/demo-skill", + skill_version: "1.0.0", + skill_md_content: "Use this skill with [REDACTED_SECRET]", + skill_bundle_content: [ + { + path: "scripts/run.sh", + content: "echo [REDACTED_SECRET]\n", + sha256: "bundle-content-sha", + sizeBytes: 23, + }, + ], + clawscan_verdict: "suspicious", + clawscan_confidence: "medium", + clawscan_model: "gpt-test", + static_status: "clean", + static_finding_count: 1, + virustotal_malicious_count: 0, + skillspector_issue_categories: ["Skill Quality"], + split: "train", + }), + ]); + }); +}); diff --git a/scripts/security-dataset/huggingFaceExport.ts b/scripts/security-dataset/huggingFaceExport.ts new file mode 100644 index 00000000..584a56f0 --- /dev/null +++ b/scripts/security-dataset/huggingFaceExport.ts @@ -0,0 +1,197 @@ +import { + hashString, + type ArtifactRow, + type DatasetSplit, + type LabelRow, + type NormalizedDatasetRows, + type ScanResultRow, + type StaticFindingRow, +} from "./normalize"; + +export type HuggingFaceSecuritySignalRow = { + id: string; + skill_slug: string | null; + skill_version: string; + skill_md_content: string | null; + skill_bundle_content: Array<{ + path: string; + content: string; + sha256: string; + sizeBytes: number; + }>; + clawscan_verdict: string; + clawscan_confidence: string | null; + clawscan_model: string | null; + clawscan_summary: string | null; + static_status: string | null; + static_finding_count: number; + static_reason_codes: string[]; + virustotal_status: string | null; + virustotal_malicious_count: number | null; + virustotal_suspicious_count: number | null; + virustotal_harmless_count: number | null; + virustotal_undetected_count: number | null; + skillspector_status: string | null; + skillspector_score: number | null; + skillspector_severity: string | null; + skillspector_issue_count: number; + skillspector_issue_codes: string[]; + skillspector_issue_categories: string[]; + clawscan_context: Record; + split: DatasetSplit; +}; + +export function buildHuggingFaceSecuritySignalRows( + rows: NormalizedDatasetRows, +): HuggingFaceSecuritySignalRow[] { + const scansByArtifact = groupBy(rows.scanResults, (row) => row.artifact_id); + const labelsByArtifact = groupBy(rows.labels, (row) => row.artifact_id); + const staticFindingsByArtifact = groupBy(rows.staticFindings, (row) => row.artifact_id); + const splitByArtifact = new Map(rows.splits.map((row) => [row.artifact_id, row.split])); + + return rows.artifacts.flatMap((artifact) => { + if (artifact.source_kind !== "skill" || !artifact.is_public || artifact.soft_deleted) { + return []; + } + const scans = scansByArtifact.get(artifact.artifact_id) ?? []; + const labels = labelsByArtifact.get(artifact.artifact_id) ?? []; + const staticFindings = staticFindingsByArtifact.get(artifact.artifact_id) ?? []; + const split = splitByArtifact.get(artifact.artifact_id); + if (!split) return []; + + const staticScan = scanByName(scans, "static"); + const vtScan = scanByName(scans, "virustotal"); + const skillSpectorScan = scanByName(scans, "skillspector"); + const llmScan = scanByName(scans, "llm"); + const verdict = labelBySource(labels, "moderation_consensus")?.label ?? "unknown"; + + return [ + { + id: flatArtifactId(artifact), + skill_slug: artifact.public_qualified_slug ?? artifact.public_slug, + skill_version: artifact.version, + skill_md_content: artifact.skill_md_content_redacted ?? null, + skill_bundle_content: (artifact.bundle_files_redacted ?? []).map((file) => ({ + path: file.path, + content: file.content, + sha256: file.sha256, + sizeBytes: file.size_bytes, + })), + clawscan_verdict: verdict, + clawscan_confidence: llmScan?.confidence ?? null, + clawscan_model: llmScan?.model ?? null, + clawscan_summary: llmScan?.summary_redacted ?? null, + static_status: staticScan?.status ?? null, + static_finding_count: staticFindings.length, + static_reason_codes: staticScan?.reason_codes ?? [], + virustotal_status: vtScan?.status ?? null, + virustotal_malicious_count: vtScan?.engine_stats?.malicious ?? null, + virustotal_suspicious_count: vtScan?.engine_stats?.suspicious ?? null, + virustotal_harmless_count: vtScan?.engine_stats?.harmless ?? null, + virustotal_undetected_count: vtScan?.engine_stats?.undetected ?? null, + skillspector_status: skillSpectorScan?.status ?? null, + skillspector_score: skillSpectorScan?.score ?? null, + skillspector_severity: skillSpectorScan?.severity ?? null, + skillspector_issue_count: skillSpectorScan?.issues?.length ?? 0, + skillspector_issue_codes: skillSpectorScan?.reason_codes ?? [], + skillspector_issue_categories: uniqueSorted( + (skillSpectorScan?.issues ?? []).flatMap((issue) => + issue.category ? [issue.category] : [], + ), + ), + clawscan_context: buildClawScanContext({ + staticScan, + vtScan, + skillSpectorScan, + staticFindings, + }), + split, + }, + ]; + }); +} + +function buildClawScanContext(input: { + staticScan: ScanResultRow | undefined; + vtScan: ScanResultRow | undefined; + skillSpectorScan: ScanResultRow | undefined; + staticFindings: StaticFindingRow[]; +}) { + const context: Record = {}; + const { staticScan, vtScan, skillSpectorScan, staticFindings } = input; + if (staticScan) { + context.static = { + status: staticScan.status, + verdict: staticScan.verdict, + reason_codes: staticScan.reason_codes, + summary: staticScan.summary_redacted, + checked_at: isoTime(staticScan.checked_at), + scanner_version: staticScan.scanner_version, + finding_count: staticFindings.length, + }; + } + if (vtScan) { + context.virustotal = { + status: vtScan.status, + verdict: vtScan.verdict, + checked_at: isoTime(vtScan.checked_at), + scanner_version: vtScan.scanner_version, + engine_stats: vtScan.engine_stats, + }; + } + if (skillSpectorScan) { + context.skillspector = { + status: skillSpectorScan.status, + verdict: skillSpectorScan.verdict, + checked_at: isoTime(skillSpectorScan.checked_at), + scanner_version: skillSpectorScan.scanner_version, + issue_codes: skillSpectorScan.reason_codes, + score: skillSpectorScan.score ?? null, + severity: skillSpectorScan.severity ?? null, + issue_count: skillSpectorScan.issues?.length ?? 0, + issues: (skillSpectorScan.issues ?? []).map((issue) => ({ + code: issue.code, + category: issue.category, + severity: issue.severity, + confidence: issue.confidence, + explanation: issue.explanation_redacted, + })), + }; + } + return context; +} + +function flatArtifactId(artifact: ArtifactRow) { + if (artifact.artifact_sha256) return artifact.artifact_sha256; + return hashString(artifact.artifact_id); +} + +function scanByName(scans: ScanResultRow[], scanner: ScanResultRow["scanner"]) { + return scans.find((row) => row.scanner === scanner); +} + +function labelBySource(labels: LabelRow[], source: LabelRow["label_source"]) { + return labels.find((row) => row.label_source === source); +} + +function groupBy(items: T[], getKey: (item: T) => string) { + const map = new Map(); + for (const item of items) { + const key = getKey(item); + const group = map.get(key); + if (group) { + group.push(item); + } else { + map.set(key, [item]); + } + } + return map; +} + +function uniqueSorted(values: string[]) { + return Array.from(new Set(values)).sort((left, right) => left.localeCompare(right)); +} + +function isoTime(value: number | null) { + return value === null ? null : new Date(value).toISOString(); +} diff --git a/scripts/security-dataset/manifest.ts b/scripts/security-dataset/manifest.ts index cedae81f..a11b289d 100644 --- a/scripts/security-dataset/manifest.ts +++ b/scripts/security-dataset/manifest.ts @@ -1,5 +1,6 @@ export type SnapshotManifestInput = { snapshotId: string; + sourceSnapshotId?: string | null; createdAt: string; repoGitSha: string; convexDeployment: string; @@ -16,7 +17,9 @@ export type SnapshotManifestInput = { clawScanFindings: number; labels: number; splits: number; + huggingFaceRows?: number; }; + outputSizes?: Record; scannerVersions: string[]; modelNames: string[]; redactionPolicyVersion: string; @@ -25,11 +28,20 @@ export type SnapshotManifestInput = { createdAtGte: number | null; createdAtLt: number | null; }; + huggingFaceDataset?: { + repo: string; + revision: string; + commit: string | null; + configNames: string[]; + splitNames: string[]; + rowCountsBySplit: Record; + }; }; export function buildSecurityDatasetManifest(input: SnapshotManifestInput) { return { snapshot_id: input.snapshotId, + source_snapshot_id: input.sourceSnapshotId ?? input.snapshotId, created_at: input.createdAt, repo_git_sha: input.repoGitSha, convex_deployment: input.convexDeployment, @@ -47,7 +59,9 @@ export function buildSecurityDatasetManifest(input: SnapshotManifestInput) { clawscan_findings: input.rowCounts.clawScanFindings, labels: input.rowCounts.labels, splits: input.rowCounts.splits, + huggingface_rows: input.rowCounts.huggingFaceRows ?? 0, }, + output_sizes: input.outputSizes ?? {}, scanner_versions: input.scannerVersions, model_names: input.modelNames, redaction_policy_version: input.redactionPolicyVersion, @@ -57,6 +71,7 @@ export function buildSecurityDatasetManifest(input: SnapshotManifestInput) { created_at_gte: input.timeWindow?.createdAtGte ?? null, created_at_lt: input.timeWindow?.createdAtLt ?? null, }, + huggingface_dataset: input.huggingFaceDataset ?? null, }; }