feat(security-dataset): ingest Convex exports locally

scripts/security-dataset/convexExport.test.ts

@@ -0,0 +1,137 @@
+import { writeFile } from "node:fs/promises";
+import { strToU8, zipSync } from "fflate";
+import { describe, expect, it } from "vitest";
+import {
+	artifactInputsFromConvexExportTables,
+	artifactInputsFromConvexExportZip,
+} from "./convexExport";
+
+describe("Convex export dataset ingestion", () => {
+	it("maps exported skill versions and package releases to artifact inputs", () => {
+		const rows = artifactInputsFromConvexExportTables({
+			skills: [
+				{
+					_id: "skills:1",
+					displayName: "Demo Skill",
+					slug: "demo-skill",
+					capabilityTags: ["filesystem"],
+					moderationSourceVersionId: "skillVersions:1",
+					moderationVerdict: "suspicious",
+					moderationReasonCodes: ["network.exfiltration"],
+					moderationSummary: "uses a token",
+					moderationEngineVersion: "v2",
+					moderationEvaluatedAt: 10,
+				},
+			],
+			skillVersions: [
+				{
+					_id: "skillVersions:1",
+					skillId: "skills:1",
+					version: "1.0.0",
+					createdAt: 2,
+					sha256hash: "skill-sha",
+					files: [{ path: "SKILL.md", size: 12, sha256: "file-sha" }],
+					llmAnalysis: {
+						status: "suspicious",
+						verdict: "suspicious",
+						confidence: "high",
+						summary: "asks for secrets",
+						model: "gpt-test",
+						checkedAt: 3,
+					},
+				},
+			],
+			packages: [
+				{
+					_id: "packages:public",
+					displayName: "Demo Package",
+					name: "@demo/pkg",
+					channel: "community",
+					family: "code-plugin",
+					sourceRepo: "git@github.com:demo/pkg.git",
+					executesCode: true,
+					capabilityTags: ["executes-code"],
+				},
+				{
+					_id: "packages:private",
+					displayName: "Private Package",
+					name: "@demo/private",
+					channel: "private",
+					family: "code-plugin",
+				},
+			],
+			packageReleases: [
+				{
+					_id: "packageReleases:1",
+					packageId: "packages:public",
+					version: "2.0.0",
+					createdAt: 1,
+					integritySha256: "pkg-sha",
+					files: [{ path: "package/index.js", size: 24, sha256: "pkg-file-sha" }],
+					staticScan: {
+						status: "clean",
+						reasonCodes: [],
+						findings: [],
+						summary: "ok",
+						engineVersion: "static-v1",
+						checkedAt: 4,
+					},
+				},
+				{
+					_id: "packageReleases:private",
+					packageId: "packages:private",
+					version: "1.0.0",
+					createdAt: 5,
+					integritySha256: "private-sha",
+					files: [],
+				},
+			],
+		});
+
+		expect(rows).toHaveLength(2);
+		expect(rows.map((row) => row.sourceKind)).toEqual(["package", "skill"]);
+		expect(rows[0]).toMatchObject({
+			sourceKind: "package",
+			sourceDocId: "packageReleases:1",
+			artifactSha256: "pkg-sha",
+			packageChannel: "community",
+			sourceRepoHost: "github.com",
+			staticScan: { status: "clean" },
+		});
+		expect(rows[1]).toMatchObject({
+			sourceKind: "skill",
+			sourceDocId: "skillVersions:1",
+			artifactSha256: "skill-sha",
+			moderationConsensus: {
+				verdict: "suspicious",
+				reasonCodes: ["network.exfiltration"],
+			},
+			llmAnalysis: { model: "gpt-test" },
+		});
+	});
+
+	it("reads table JSONL files from a Convex export zip", async () => {
+		const zip = zipSync({
+			"tables/skills.jsonl": strToU8(
+				`${JSON.stringify({ _id: "skills:1", displayName: "S", slug: "s" })}\n`,
+			),
+			"tables/skillVersions.jsonl": strToU8(
+				`${JSON.stringify({
+					_id: "skillVersions:1",
+					skillId: "skills:1",
+					version: "1.0.0",
+					createdAt: 1,
+					files: [],
+				})}\n`,
+			),
+			"tables/packages.jsonl": strToU8(""),
+			"tables/packageReleases.jsonl": strToU8(""),
+		});
+		const path = "/tmp/clawhub-convex-export-test.zip";
+		await writeFile(path, zip);
+
+		await expect(artifactInputsFromConvexExportZip(path)).resolves.toMatchObject([
+			{ sourceKind: "skill", sourceDocId: "skillVersions:1" },
+		]);
+	});
+});

scripts/security-dataset/convexExport.ts

@@ -0,0 +1,329 @@
+import { readFile } from "node:fs/promises";
+import { strFromU8, unzipSync } from "fflate";
+import type {
+	ArtifactExportInput,
+	DatasetLabel,
+	ExportFileInput,
+	LlmAnalysisInput,
+	ModerationConsensusInput,
+	SourceKind,
+	StaticScanInput,
+	VtAnalysisInput,
+} from "./normalize";
+
+type ConvexDoc = Record<string, unknown> & { _id?: unknown };
+
+type ConvexExportTables = {
+	skills: ConvexDoc[];
+	skillVersions: ConvexDoc[];
+	packages: ConvexDoc[];
+	packageReleases: ConvexDoc[];
+};
+
+const REQUIRED_TABLES = ["skills", "skillVersions", "packages", "packageReleases"] as const;
+
+export async function artifactInputsFromConvexExportZip(
+	zipPath: string,
+): Promise<ArtifactExportInput[]> {
+	const zipBytes = new Uint8Array(await readFile(zipPath));
+	const entries = unzipSync(zipBytes);
+	const tables = Object.fromEntries(
+		REQUIRED_TABLES.map((table) => [table, readExportTable(entries, table)]),
+	) as ConvexExportTables;
+	return artifactInputsFromConvexExportTables(tables);
+}
+
+export function artifactInputsFromConvexExportTables(
+	tables: ConvexExportTables,
+): ArtifactExportInput[] {
+	const skillsById = buildIdMap(tables.skills);
+	const packagesById = buildIdMap(tables.packages);
+	const rows = [
+		...tables.skillVersions.flatMap((version) => skillVersionToExportRow(version, skillsById)),
+		...tables.packageReleases.flatMap((release) =>
+			packageReleaseToExportRow(release, packagesById),
+		),
+	];
+	return rows.sort((left, right) => {
+		const createdDelta = left.createdAt - right.createdAt;
+		if (createdDelta !== 0) return createdDelta;
+		return `${left.sourceKind}:${left.sourceDocId}`.localeCompare(
+			`${right.sourceKind}:${right.sourceDocId}`,
+		);
+	});
+}
+
+function readExportTable(
+	entries: Record<string, Uint8Array>,
+	table: (typeof REQUIRED_TABLES)[number],
+): ConvexDoc[] {
+	const entryName = findExportTableEntry(Object.keys(entries), table);
+	const bytes = entries[entryName];
+	if (!bytes) throw new Error(`Convex export table entry disappeared: ${entryName}`);
+	const text = strFromU8(bytes);
+	return text
+		.split(/\r?\n/)
+		.filter((line) => line.trim().length > 0)
+		.map((line) => JSON.parse(line) as ConvexDoc);
+}
+
+function findExportTableEntry(entryNames: string[], table: string) {
+	const normalized = entryNames.map((name) => ({ name, parts: name.split("/") }));
+	const exactJsonl = normalized.find(({ name }) => name === `${table}.jsonl`);
+	if (exactJsonl) return exactJsonl.name;
+	const basenameJsonl = normalized.find(({ parts }) => parts.at(-1) === `${table}.jsonl`);
+	if (basenameJsonl) return basenameJsonl.name;
+	const documentsJsonl = normalized.find(
+		({ parts }) => parts.at(-2) === table && parts.at(-1) === "documents.jsonl",
+	);
+	if (documentsJsonl) return documentsJsonl.name;
+	const tableJsonl = normalized.find(
+		({ parts }) => parts.includes(table) && parts.at(-1)?.endsWith(".jsonl"),
+	);
+	if (tableJsonl) return tableJsonl.name;
+	throw new Error(`Missing ${table} JSONL table in Convex export.`);
+}
+
+function buildIdMap(rows: ConvexDoc[]) {
+	const map = new Map<string, ConvexDoc>();
+	for (const row of rows) {
+		const id = stringValue(row._id);
+		if (id) map.set(id, row);
+	}
+	return map;
+}
+
+function skillVersionToExportRow(
+	version: ConvexDoc,
+	skillsById: Map<string, ConvexDoc>,
+): ArtifactExportInput[] {
+	if (numberOrNull(version.softDeletedAt) !== null) return [];
+	const skill = skillsById.get(stringValue(version.skillId));
+	if (!skill || numberOrNull(skill.softDeletedAt) !== null) return [];
+	const versionId = requiredString(version._id, "skillVersions._id");
+	const moderationConsensus =
+		stringValue(skill.moderationSourceVersionId) === versionId
+			? moderationConsensusFromSkill(skill)
+			: null;
+	return [
+		{
+			sourceKind: "skill",
+			sourceDocId: versionId,
+			parentDocId: requiredString(skill._id, "skills._id"),
+			publicName: requiredString(skill.displayName, "skills.displayName"),
+			publicSlug: stringOrNull(skill.slug),
+			version: requiredString(version.version, "skillVersions.version"),
+			artifactSha256: stringOrNull(version.sha256hash),
+			createdAt: numberValue(version.createdAt, "skillVersions.createdAt"),
+			softDeletedAt: numberOrNull(version.softDeletedAt),
+			files: filesFromExport(version.files),
+			capabilityTags: stringArray(version.capabilityTags ?? skill.capabilityTags),
+			packageFamily: null,
+			packageChannel: null,
+			packageExecutesCode: null,
+			sourceRepoHost: null,
+			vtAnalysis: vtAnalysisFromExport(version.vtAnalysis),
+			staticScan: staticScanFromExport(version.staticScan),
+			llmAnalysis: llmAnalysisFromExport(version.llmAnalysis),
+			moderationConsensus,
+		},
+	];
+}
+
+function packageReleaseToExportRow(
+	release: ConvexDoc,
+	packagesById: Map<string, ConvexDoc>,
+): ArtifactExportInput[] {
+	if (numberOrNull(release.softDeletedAt) !== null) return [];
+	const pkg = packagesById.get(stringValue(release.packageId));
+	if (!pkg || numberOrNull(pkg.softDeletedAt) !== null || pkg.channel === "private") return [];
+	return [
+		{
+			sourceKind: "package",
+			sourceDocId: requiredString(release._id, "packageReleases._id"),
+			parentDocId: requiredString(pkg._id, "packages._id"),
+			publicName: requiredString(pkg.displayName, "packages.displayName"),
+			publicSlug: stringOrNull(pkg.name),
+			version: requiredString(release.version, "packageReleases.version"),
+			artifactSha256: stringOrNull(release.sha256hash) ?? stringOrNull(release.integritySha256),
+			createdAt: numberValue(release.createdAt, "packageReleases.createdAt"),
+			softDeletedAt: numberOrNull(release.softDeletedAt),
+			files: filesFromExport(release.files),
+			capabilityTags: stringArray(pkg.capabilityTags),
+			packageFamily: stringOrNull(pkg.family),
+			packageChannel: stringOrNull(pkg.channel),
+			packageExecutesCode: booleanOrNull(pkg.executesCode),
+			sourceRepoHost: sourceRepoHost(stringOrNull(pkg.sourceRepo)),
+			vtAnalysis: vtAnalysisFromExport(release.vtAnalysis),
+			staticScan: staticScanFromExport(release.staticScan),
+			llmAnalysis: llmAnalysisFromExport(release.llmAnalysis),
+			moderationConsensus: null,
+		},
+	];
+}
+
+function filesFromExport(value: unknown): ExportFileInput[] {
+	if (!Array.isArray(value)) return [];
+	return value.flatMap((file) => {
+		if (!isRecord(file)) return [];
+		const path = stringValue(file.path);
+		const sha256 = stringValue(file.sha256);
+		const size = typeof file.size === "number" ? file.size : null;
+		if (!path || !sha256 || size === null) return [];
+		return [{ path, size, sha256, contentType: stringOrNull(file.contentType) }];
+	});
+}
+
+function vtAnalysisFromExport(value: unknown): VtAnalysisInput | null {
+	if (!isRecord(value)) return null;
+	return {
+		status: requiredString(value.status, "vtAnalysis.status"),
+		verdict: stringOrNull(value.verdict),
+		analysis: stringOrNull(value.analysis),
+		source: stringOrNull(value.source),
+		scanner: stringOrNull(value.scanner),
+		engineStats: engineStatsFromExport(value.engineStats),
+		checkedAt: numberValue(value.checkedAt, "vtAnalysis.checkedAt"),
+	};
+}
+
+function staticScanFromExport(value: unknown): StaticScanInput | null {
+	if (!isRecord(value)) return null;
+	const status = datasetLabelOrNull(value.status);
+	if (!status || status === "unknown") return null;
+	return {
+		status,
+		reasonCodes: stringArray(value.reasonCodes),
+		findings: staticFindingsFromExport(value.findings),
+		summary: requiredString(value.summary, "staticScan.summary"),
+		engineVersion: requiredString(value.engineVersion, "staticScan.engineVersion"),
+		checkedAt: numberValue(value.checkedAt, "staticScan.checkedAt"),
+	};
+}
+
+function staticFindingsFromExport(value: unknown): StaticScanInput["findings"] {
+	if (!Array.isArray(value)) return [];
+	return value.flatMap((finding) => {
+		if (!isRecord(finding)) return [];
+		const severity = finding.severity;
+		if (severity !== "info" && severity !== "warn" && severity !== "critical") return [];
+		return [
+			{
+				code: requiredString(finding.code, "staticScan.finding.code"),
+				severity,
+				file: requiredString(finding.file, "staticScan.finding.file"),
+				line: numberValue(finding.line, "staticScan.finding.line"),
+				message: requiredString(finding.message, "staticScan.finding.message"),
+				evidence: requiredString(finding.evidence, "staticScan.finding.evidence"),
+			},
+		];
+	});
+}
+
+function llmAnalysisFromExport(value: unknown): LlmAnalysisInput | null {
+	if (!isRecord(value)) return null;
+	return {
+		status: requiredString(value.status, "llmAnalysis.status"),
+		verdict: stringOrNull(value.verdict),
+		confidence: stringOrNull(value.confidence),
+		summary: stringOrNull(value.summary),
+		dimensions: llmDimensionsFromExport(value.dimensions),
+		guidance: stringOrNull(value.guidance),
+		findings: stringOrNull(value.findings),
+		model: stringOrNull(value.model),
+		checkedAt: numberValue(value.checkedAt, "llmAnalysis.checkedAt"),
+	};
+}
+
+function llmDimensionsFromExport(value: unknown): LlmAnalysisInput["dimensions"] {
+	if (!Array.isArray(value)) return null;
+	return value.flatMap((dimension) => {
+		if (!isRecord(dimension)) return [];
+		return [
+			{
+				name: requiredString(dimension.name, "llmAnalysis.dimension.name"),
+				label: requiredString(dimension.label, "llmAnalysis.dimension.label"),
+				rating: requiredString(dimension.rating, "llmAnalysis.dimension.rating"),
+				detail: requiredString(dimension.detail, "llmAnalysis.dimension.detail"),
+			},
+		];
+	});
+}
+
+function moderationConsensusFromSkill(skill: ConvexDoc): ModerationConsensusInput | null {
+	const verdict = datasetLabelOrNull(skill.moderationVerdict);
+	return {
+		verdict,
+		reasonCodes: stringArray(skill.moderationReasonCodes),
+		summary: stringOrNull(skill.moderationSummary),
+		engineVersion: stringOrNull(skill.moderationEngineVersion),
+		evaluatedAt: numberOrNull(skill.moderationEvaluatedAt),
+	};
+}
+
+function engineStatsFromExport(value: unknown): VtAnalysisInput["engineStats"] {
+	if (!isRecord(value)) return null;
+	return {
+		malicious: optionalNumber(value.malicious),
+		suspicious: optionalNumber(value.suspicious),
+		undetected: optionalNumber(value.undetected),
+		harmless: optionalNumber(value.harmless),
+	};
+}
+
+function sourceRepoHost(sourceRepo: string | null) {
+	if (!sourceRepo) return null;
+	try {
+		return new URL(sourceRepo).host.toLowerCase();
+	} catch {
+		const match = sourceRepo.match(/^[^/:]+[:/](?<owner>[^/]+)\/(?<repo>[^/]+)$/);
+		return match?.groups?.owner && match.groups.repo ? "github.com" : null;
+	}
+}
+
+function datasetLabelOrNull(value: unknown): DatasetLabel | null {
+	if (value === "clean" || value === "suspicious" || value === "malicious" || value === "unknown")
+		return value;
+	return null;
+}
+
+function stringArray(value: unknown) {
+	return Array.isArray(value)
+		? value.filter((item): item is string => typeof item === "string")
+		: [];
+}
+
+function requiredString(value: unknown, field: string) {
+	const result = stringValue(value);
+	if (!result) throw new Error(`Missing string field in Convex export: ${field}`);
+	return result;
+}
+
+function stringOrNull(value: unknown) {
+	return typeof value === "string" ? value : null;
+}
+
+function stringValue(value: unknown) {
+	return typeof value === "string" ? value : "";
+}
+
+function numberValue(value: unknown, field: string) {
+	if (typeof value === "number") return value;
+	throw new Error(`Missing number field in Convex export: ${field}`);
+}
+
+function optionalNumber(value: unknown) {
+	return typeof value === "number" ? value : undefined;
+}
+
+function numberOrNull(value: unknown) {
+	return typeof value === "number" ? value : null;
+}
+
+function booleanOrNull(value: unknown) {
+	return typeof value === "boolean" ? value : null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null;
+}

scripts/security-dataset/export-snapshot.ts

@@ -4,6 +4,7 @@ import { createWriteStream, type WriteStream } from "node:fs";
 import { mkdir, writeFile } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { promisify } from "node:util";
+import { artifactInputsFromConvexExportZip } from "./convexExport";
 import { parseConvexJsonMatching } from "./convexOutput";
 import { reserveExportInputs } from "./exportLimit";
 import { buildSecurityDatasetManifest } from "./manifest";
@@ -49,6 +50,7 @@ type Options = {
 	outDir: string;
 	sourceKind: SourceKind | "all";
 	timeWindow: CreatedTimeWindow;
+	convexExportZip: string | null;
 };
 
 type ExportShard = {
@@ -96,9 +98,10 @@ async function main() {
 	const state = createSnapshotState();
 
 	try {
-		const shards = await buildExportShards(options);
-		await exportShards({ options, shards, state, writers });
-		const manifest = buildManifest({ options, snapshotId, state, shardCount: shards.length });
+		const shardCount = options.convexExportZip
+			? await exportConvexExportZip({ options, state, writers })
+			: await exportRemoteShards({ options, state, writers });
+		const manifest = buildManifest({ options, snapshotId, state, shardCount });
 
 		if (options.dryRun) {
 			console.log(JSON.stringify({ snapshotId, dryRun: true, manifest }, null, 2));
@@ -117,6 +120,52 @@ async function main() {
 	}
 }
 
+async function exportRemoteShards(input: {
+	options: Options;
+	state: SnapshotState;
+	writers: SnapshotWriters | null;
+}) {
+	const { options, state, writers } = input;
+	const shards = await buildExportShards(options);
+	await exportShards({ options, shards, state, writers });
+	return shards.length;
+}
+
+async function exportConvexExportZip(input: {
+	options: Options;
+	state: SnapshotState;
+	writers: SnapshotWriters | null;
+}) {
+	const { options, state, writers } = input;
+	if (!options.convexExportZip) throw new Error("Missing Convex export ZIP path.");
+	const inputs = await artifactInputsFromConvexExportZip(options.convexExportZip);
+	const reserved = reserveExportInputs(
+		filterExportInputs(inputs, options.sourceKind, options.timeWindow),
+		state,
+		options.limit,
+	);
+	await processArtifactInputs({ inputs: reserved, state, writers });
+	console.error(
+		`[snapshot] convex-export +${reserved.length} artifacts (${state.sourceArtifacts} total)`,
+	);
+	return 0;
+}
+
+function filterExportInputs(
+	inputs: ArtifactExportInput[],
+	sourceKind: SourceKind | "all",
+	timeWindow: CreatedTimeWindow,
+) {
+	return inputs.filter((input) => {
+		if (sourceKind !== "all" && input.sourceKind !== sourceKind) return false;
+		if (timeWindow.createdAtGte !== undefined && input.createdAt < timeWindow.createdAtGte)
+			return false;
+		if (timeWindow.createdAtLt !== undefined && input.createdAt >= timeWindow.createdAtLt)
+			return false;
+		return true;
+	});
+}
+
 async function buildExportShards(options: Options) {
 	const sourceKinds = options.sourceKind === "all" ? SOURCE_KINDS : [options.sourceKind];
 	const shards: ExportShard[] = [];
@@ -457,6 +506,7 @@ function parseArgs(args: string[]): Options {
 		outDir: DEFAULT_OUT_DIR,
 		sourceKind: "all",
 		timeWindow: emptyCreatedTimeWindow(),
+		convexExportZip: null,
 	};
 
 	for (let index = 0; index < args.length; index += 1) {
@@ -485,6 +535,8 @@ function parseArgs(args: string[]): Options {
 			options.timeWindow.createdAtGte = parseCreatedTimestamp(readValue(args, ++index, arg), arg);
 		} else if (arg === "--created-before") {
 			options.timeWindow.createdAtLt = parseCreatedTimestamp(readValue(args, ++index, arg), arg);
+		} else if (arg === "--convex-export-zip" || arg === "--from-convex-export") {
+			options.convexExportZip = readValue(args, ++index, arg);
 		} else if (arg === "--mode") {
 			const mode = readValue(args, ++index, arg);
 			if (mode !== "public") throw new Error(`Unsupported mode: ${mode}`);