From 49abba7747a30e9f1bfdaff2b86b77677c963151 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Wed, 15 Jul 2026 17:23:28 -0700 Subject: [PATCH] feat: add security scan comparison modes (#3095) --- .github/workflows/security-scan-codex.yml | 6 +- .../run-codex-scan-worker-clawscan.test.ts | 89 +++- .../run-codex-scan-worker-modes.test.ts | 453 +++++++++++++++++ .../security/run-codex-scan-worker.test.ts | 479 ++---------------- scripts/security/run-codex-scan-worker.ts | 378 ++++++-------- .../security-scan-worker-workflow.test.ts | 8 +- specs/security-moderation.md | 15 +- 7 files changed, 729 insertions(+), 699 deletions(-) create mode 100644 scripts/security/run-codex-scan-worker-modes.test.ts diff --git a/.github/workflows/security-scan-codex.yml b/.github/workflows/security-scan-codex.yml index f5b4ad73..056a6ed0 100644 --- a/.github/workflows/security-scan-codex.yml +++ b/.github/workflows/security-scan-codex.yml @@ -65,11 +65,9 @@ jobs: CODEX_SECURITY_SCAN_MAX_JOBS: ${{ github.event.client_payload.max_jobs || inputs['max-jobs'] || '' }} CODEX_SECURITY_SCAN_MAX_RUNTIME_MINUTES: ${{ github.event.client_payload.max_runtime_minutes || inputs['max-runtime-minutes'] || '12' }} CODEX_SECURITY_SCAN_LANE: ${{ matrix.lane }} + CODEX_SECURITY_SCAN_MODE: ${{ vars.CODEX_SECURITY_SCAN_MODE || 'legacy' }} + CODEX_SECURITY_SCAN_CLAWSCAN_TIMEOUT_MS: ${{ vars.CODEX_SECURITY_SCAN_CLAWSCAN_TIMEOUT_MS || '240000' }} CODEX_SECURITY_SCAN_TIMEOUT_MS: ${{ vars.CODEX_SECURITY_SCAN_TIMEOUT_MS || '240000' }} - CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN: ${{ vars.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN || '0' }} - CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX: ${{ vars.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX || 'docker' }} - CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX_IMAGE: ${{ vars.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX_IMAGE || 'ghcr.io/openclaw/clawscan-runtime@sha256:012f045c77e5adce3ff9d00ed37fe04ed020fdb555ebdcebab0e0dca48e19d7a' }} - CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_TIMEOUT_MS: ${{ vars.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_TIMEOUT_MS || '300000' }} CODEX_SECURITY_SCAN_LEASE_MINUTES: "60" CODEX_SECURITY_SCAN_DIAGNOSTICS_DIR: codex-security-scan-diagnostics-${{ matrix.shard }} CODEX_SECURITY_SCAN_SHARD: ${{ matrix.shard }} diff --git a/scripts/security/run-codex-scan-worker-clawscan.test.ts b/scripts/security/run-codex-scan-worker-clawscan.test.ts index bdebfc97..c3ecb070 100644 --- a/scripts/security/run-codex-scan-worker-clawscan.test.ts +++ b/scripts/security/run-codex-scan-worker-clawscan.test.ts @@ -101,6 +101,57 @@ async function writeFakeClawScanCommand(path: string, body: string) { await chmod(path, 0o755); } +async function withFakeLegacySecondary(run: () => Promise) { + const binDir = await tempDir(); + await writeFakeClawScanCommand( + join(binDir, "skillspector"), + `out="" +while [[ $# -gt 0 ]]; do + case "$1" in + --output) + out="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +mkdir -p "$(dirname "$out")" +cat > "$out" <<'JSON' +{"status":"clean","issue_count":0,"issues":[]} +JSON`, + ); + await writeFakeClawScanCommand( + join(binDir, "codex"), + `out="" +while [[ $# -gt 0 ]]; do + case "$1" in + --output-last-message) + out="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +mkdir -p "$(dirname "$out")" +cat > "$out" <<'JSON' +{"verdict":"benign","confidence":"high","summary":"legacy diagnostic","dimensions":{"purpose_capability":{"status":"ok","detail":"ok"}},"scan_findings_in_context":[],"user_guidance":"guidance"} +JSON`, + ); + + const previousPath = process.env.PATH; + process.env.PATH = `${binDir}:${previousPath ?? ""}`; + try { + return await run(); + } finally { + if (previousPath === undefined) delete process.env.PATH; + else process.env.PATH = previousPath; + } +} + type ClawScanVerdict = "benign" | "suspicious" | "malicious"; function completeJudgeDimensions() { @@ -317,12 +368,14 @@ JSON`, const client = { action: vi.fn(async (..._args: unknown[]) => ({})), }; - const result = await processJob( - client, - "worker-auth", - skillVersionJob(`securityScanJobs:${verdict}`), - undefined, - "clawscan", + const result = await withFakeLegacySecondary(async () => + processJob( + client, + "worker-auth", + skillVersionJob(`securityScanJobs:${verdict}`), + undefined, + "clawscan", + ), ); expect(result).toEqual({ @@ -443,17 +496,19 @@ JSON`, const client = { action: vi.fn(async (..._args: unknown[]) => ({})), }; - const result = await processJob( - client, - "worker-auth", - claimedJob({ - jobId: `securityScanJobs:${targetKind}-${source}`, - source, - target: await target(), - targetKind, - }), - undefined, - "clawscan", + const result = await withFakeLegacySecondary(async () => + processJob( + client, + "worker-auth", + claimedJob({ + jobId: `securityScanJobs:${targetKind}-${source}`, + source, + target: await target(), + targetKind, + }), + undefined, + "clawscan", + ), ); expect(result).toEqual({ diff --git a/scripts/security/run-codex-scan-worker-modes.test.ts b/scripts/security/run-codex-scan-worker-modes.test.ts new file mode 100644 index 00000000..28148e61 --- /dev/null +++ b/scripts/security/run-codex-scan-worker-modes.test.ts @@ -0,0 +1,453 @@ +/* @vitest-environment node */ +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + type ClaimedJob, + processJob, + resolveSecurityScanMode, + type SecurityScanMode, +} from "./run-codex-scan-worker"; + +const tempDirs: string[] = []; +const jobFixture = "lease-fixture"; + +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { force: true, recursive: true }))); +}); + +async function tempDir() { + const dir = await mkdtemp(join(tmpdir(), "clawhub-scan-mode-test-")); + tempDirs.push(dir); + return dir; +} + +async function writeCommand(path: string, body: string) { + await writeFile(path, `#!/usr/bin/env bash\nset -euo pipefail\n${body}\n`); + await chmod(path, 0o755); +} + +function claimedJob(id: string): ClaimedJob { + return { + job: { + _id: `securityScanJobs:${id}`, + hasMaliciousSignal: false, + leaseToken: jobFixture, + source: "publish", + targetKind: "skillVersion", + waitForVtUntil: 0, + }, + target: { + files: [ + { + path: "SKILL.md", + sha256: "artifact-sha", + size: 8, + url: "data:text/plain,%23%20Skill", + }, + ], + }, + }; +} + +function completeClawScanArtifact(verdict: "benign" | "suspicious" | "malicious") { + return JSON.stringify({ + schemaVersion: "clawscan-run-v1", + profile: "clawhub", + completedAt: "2026-07-15T00:00:00Z", + scanners: { + "clawscan-static": { + status: "completed", + raw: { status: "clean" }, + }, + skillspector: { + status: "completed", + raw: { + status: "clean", + issue_count: 0, + issues: [], + }, + }, + virustotal: { + status: "completed", + raw: { status: "clean" }, + }, + }, + judge: { + status: "completed", + promptSha256: "prompt-sha", + outputSchemaSha256: "schema-sha", + result: { + verdict, + confidence: "high", + summary: "ClawScan result", + dimensions: { + purpose_capability: { status: "ok", detail: "ok" }, + instruction_scope: { status: "ok", detail: "ok" }, + install_mechanism: { status: "ok", detail: "ok" }, + environment_proportionality: { status: "ok", detail: "ok" }, + persistence_privilege: { status: "ok", detail: "ok" }, + }, + scan_findings_in_context: [], + user_guidance: "guidance", + }, + }, + }); +} + +async function setupCommands(options?: { + clawscanFailure?: string; + clawscanVerdict?: "benign" | "suspicious" | "malicious"; + legacyFailure?: string; + legacyVerdict?: "benign" | "suspicious" | "malicious"; +}) { + const root = await tempDir(); + const binDir = join(root, "bin"); + const marker = join(root, "invocations.log"); + await mkdir(binDir, { recursive: true }); + + await writeCommand( + join(binDir, "skillspector"), + `echo "legacy-skillspector:$PWD" >> ${JSON.stringify(marker)} +out="" +while [[ $# -gt 0 ]]; do + case "$1" in + --output) + out="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +mkdir -p "$(dirname "$out")" +cat > "$out" <<'JSON' +{"status":"clean","issue_count":0,"issues":[]} +JSON`, + ); + + await writeCommand( + join(binDir, "codex"), + `echo "legacy-codex:$PWD" >> ${JSON.stringify(marker)} +${options?.legacyFailure ? `echo ${JSON.stringify(options.legacyFailure)} >&2\nexit 19` : ""} +out="" +while [[ $# -gt 0 ]]; do + case "$1" in + --output-last-message) + out="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +mkdir -p "$(dirname "$out")" +cat > "$out" <<'JSON' +${JSON.stringify({ + verdict: options?.legacyVerdict ?? "suspicious", + confidence: "medium", + summary: "Legacy result", + dimensions: { + purpose_capability: { status: "ok", detail: "ok" }, + }, + scan_findings_in_context: [], + user_guidance: "guidance", +})} +JSON`, + ); + + const clawscan = join(root, "clawscan"); + await writeCommand( + clawscan, + `echo "clawscan:$PWD:$1" >> ${JSON.stringify(marker)} +${options?.clawscanFailure ? `echo ${JSON.stringify(options.clawscanFailure)} >&2\nexit 17` : ""} +out="" +while [[ $# -gt 0 ]]; do + case "$1" in + --output) + out="$2" + shift 2 + ;; + *) + shift + ;; + esac +done +mkdir -p "$(dirname "$out")" +cat > "$out" <<'JSON' +${completeClawScanArtifact(options?.clawscanVerdict ?? "malicious")} +JSON`, + ); + + return { binDir, clawscan, marker, root }; +} + +async function withCommands( + commands: Awaited>, + run: () => Promise, +) { + const previousCommand = process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND; + const previousPath = process.env.PATH; + process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND = commands.clawscan; + process.env.PATH = `${commands.binDir}:${previousPath ?? ""}`; + try { + return await run(); + } finally { + if (previousCommand === undefined) delete process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND; + else process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND = previousCommand; + if (previousPath === undefined) delete process.env.PATH; + else process.env.PATH = previousPath; + } +} + +function completionPayload(client: { action: ReturnType }) { + return client.action.mock.calls.find((call) => { + const payload = call[1] as { llmAnalysis?: unknown } | undefined; + return payload?.llmAnalysis !== undefined; + })?.[1] as + | { + llmAnalysis?: { status?: string; verdict?: string }; + skillSpectorAnalysis?: { status?: string }; + } + | undefined; +} + +async function invocationLines(marker: string) { + try { + return (await readFile(marker, "utf8")).trim().split("\n").filter(Boolean); + } catch { + return []; + } +} + +describe("security scan rollout modes", () => { + it.each([ + { + mode: "legacy" as const, + expectedClawScan: 0, + expectedLegacyCodex: 1, + expectedLegacySkillSpector: 1, + expectedStatus: "suspicious", + expectedVerdict: "suspicious", + }, + { + mode: "shadow" as const, + expectedClawScan: 1, + expectedLegacyCodex: 1, + expectedLegacySkillSpector: 1, + expectedStatus: "suspicious", + expectedVerdict: "suspicious", + }, + { + mode: "clawscan" as const, + expectedClawScan: 1, + expectedLegacyCodex: 1, + expectedLegacySkillSpector: 1, + expectedStatus: "malicious", + expectedVerdict: "malicious", + }, + ])( + "$mode invokes the expected implementations and persists only its authoritative result", + async ({ + mode, + expectedClawScan, + expectedLegacyCodex, + expectedLegacySkillSpector, + expectedStatus, + expectedVerdict, + }) => { + const commands = await setupCommands(); + const diagnosticsRoot = await tempDir(); + const client = { + action: vi.fn(async (..._args: unknown[]) => ({})), + }; + + const result = await withCommands(commands, () => + processJob(client, "worker-token", claimedJob(mode), diagnosticsRoot, mode), + ); + + expect(result).toEqual({ + completed: true, + hardFailed: false, + retryableFailed: false, + }); + expect(client.action).toHaveBeenCalledTimes(1); + expect(completionPayload(client)).toMatchObject({ + llmAnalysis: { + status: expectedStatus, + verdict: expectedVerdict, + }, + }); + + const lines = await invocationLines(commands.marker); + expect(lines.filter((line) => line.startsWith("clawscan:"))).toHaveLength(expectedClawScan); + expect(lines.filter((line) => line.startsWith("legacy-codex:"))).toHaveLength( + expectedLegacyCodex, + ); + expect(lines.filter((line) => line.startsWith("legacy-skillspector:"))).toHaveLength( + expectedLegacySkillSpector, + ); + + const workspaces = new Set( + lines.map((line) => { + const [, workspace] = line.split(":"); + return workspace; + }), + ); + expect([...workspaces]).toHaveLength(1); + + const comparisonPath = join( + diagnosticsRoot, + `securityScanJobs_${mode}`, + "scan-comparison.json", + ); + if (mode === "legacy") { + await expect(readFile(comparisonPath, "utf8")).rejects.toThrow(); + } else { + const comparison = JSON.parse(await readFile(comparisonPath, "utf8")); + expect(comparison).toMatchObject({ + authoritative: { + implementation: mode === "shadow" ? "legacy" : "clawscan", + verdict: expectedVerdict, + }, + secondary: { + implementation: mode === "shadow" ? "clawscan" : "legacy", + }, + status: "completed", + }); + } + }, + ); + + it.each([ + { + mode: "shadow" as const, + options: { clawscanFailure: "diagnostic ClawScan failure" }, + expectedVerdict: "suspicious", + }, + { + mode: "clawscan" as const, + options: { legacyFailure: "diagnostic legacy failure" }, + expectedVerdict: "malicious", + }, + ])( + "$mode ignores secondary failures after authoritative completion", + async ({ mode, options, expectedVerdict }) => { + const commands = await setupCommands(options); + const diagnosticsRoot = await tempDir(); + const client = { + action: vi.fn(async (..._args: unknown[]) => ({})), + }; + + const result = await withCommands(commands, () => + processJob( + client, + "worker-token", + claimedJob(`${mode}-secondary-failed`), + diagnosticsRoot, + mode, + ), + ); + + expect(result).toEqual({ + completed: true, + hardFailed: false, + retryableFailed: false, + }); + expect(client.action).toHaveBeenCalledTimes(1); + expect(completionPayload(client)?.llmAnalysis?.verdict).toBe(expectedVerdict); + const comparison = JSON.parse( + await readFile( + join( + diagnosticsRoot, + `securityScanJobs_${mode}-secondary-failed`, + "scan-comparison.json", + ), + "utf8", + ), + ); + expect(comparison).toMatchObject({ + status: "failed", + error: expect.stringContaining(mode === "shadow" ? "exited 17" : "exited 19"), + }); + }, + ); + + it("fails authoritative ClawScan through the retry lifecycle without invoking legacy", async () => { + const commands = await setupCommands({ clawscanFailure: "authoritative failure" }); + const client = { + action: vi.fn(async (...args: unknown[]) => { + const payload = args[1] as { error?: string } | undefined; + return payload?.error ? { retry: true } : {}; + }), + }; + + const result = await withCommands(commands, () => + processJob( + client, + "worker-token", + claimedJob("clawscan-authority-failed"), + undefined, + "clawscan", + ), + ); + + expect(result).toEqual({ + completed: false, + hardFailed: false, + retryableFailed: true, + }); + expect(client.action).toHaveBeenCalledTimes(1); + expect(client.action.mock.calls[0]?.[1]).toMatchObject({ + error: expect.stringContaining("exited 17"), + }); + const lines = await invocationLines(commands.marker); + expect(lines.filter((line) => line.startsWith("clawscan:"))).toHaveLength(1); + expect(lines.some((line) => line.startsWith("legacy-"))).toBe(false); + }); + + it("rolls the whole route back by changing the mode to legacy", async () => { + const commands = await setupCommands(); + const client = { + action: vi.fn(async (..._args: unknown[]) => ({})), + }; + + await withCommands(commands, async () => { + await processJob( + client, + "worker-token", + claimedJob("before-rollback"), + undefined, + "clawscan", + ); + await processJob(client, "worker-token", claimedJob("after-rollback"), undefined, "legacy"); + }); + + const persistedVerdicts = client.action.mock.calls.map( + (call) => (call[1] as { llmAnalysis?: { verdict?: string } }).llmAnalysis?.verdict, + ); + expect(persistedVerdicts).toEqual(["malicious", "suspicious"]); + const lines = await invocationLines(commands.marker); + expect(lines.filter((line) => line.startsWith("clawscan:"))).toHaveLength(1); + expect(lines.filter((line) => line.startsWith("legacy-codex:"))).toHaveLength(2); + }); + + it("defaults safely to legacy and accepts only the three rollout values", () => { + expect(resolveSecurityScanMode(undefined)).toBe("legacy"); + expect(resolveSecurityScanMode("")).toBe("legacy"); + expect(["legacy", "shadow", "clawscan"].map((mode) => resolveSecurityScanMode(mode))).toEqual([ + "legacy", + "shadow", + "clawscan", + ] satisfies SecurityScanMode[]); + for (const invalid of ["codex", "ClawScan", " shadow ", "0", "true"]) { + expect(() => resolveSecurityScanMode(invalid)).toThrow( + `CODEX_SECURITY_SCAN_MODE must be one of legacy, shadow, or clawscan; received ${JSON.stringify(invalid)}`, + ); + } + }); +}); diff --git a/scripts/security/run-codex-scan-worker.test.ts b/scripts/security/run-codex-scan-worker.test.ts index f93416e8..aca98fc3 100644 --- a/scripts/security/run-codex-scan-worker.test.ts +++ b/scripts/security/run-codex-scan-worker.test.ts @@ -1,5 +1,5 @@ /* @vitest-environment node */ -import { chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -16,7 +16,6 @@ import { resolveSkillSpectorScanInput, resolveSkillSpectorScanInputs, runContinuouslyRefilledWorkerPool, - runClawScanShadow, writeArtifactWorkspace, writeJobDiagnostic, } from "./run-codex-scan-worker"; @@ -875,409 +874,6 @@ describe("run-codex-scan-worker diagnostics", () => { else process.env.GITHUB_ACTIONS = previousGitHubActions; }); - it("runs OSS ClawScan shadow mode with recorded VirusTotal evidence", async () => { - const workspace = await tempDir(); - await mkdir(join(workspace, "artifact"), { recursive: true }); - await writeFile(join(workspace, "artifact", "SKILL.md"), "# Demo\n"); - const fakeClawScan = join(workspace, "fake-clawscan"); - await writeFile( - fakeClawScan, - `#!/usr/bin/env bash -set -euo pipefail -out="" -while [[ $# -gt 0 ]]; do - case "$1" in - --output) - out="$2" - shift 2 - ;; - *) - shift - ;; - esac -done -mkdir -p "$(dirname "$out")" -cat > "$out" <<'JSON' -{"schemaVersion":"clawscan-run-v1","profile":"clawhub","scanners":{"skillspector":{"status":"completed"},"virustotal":{"status":"completed"},"clawscan-static":{"status":"completed"}},"judge":{"status":"completed","result":{"verdict":"benign","confidence":"high"}}} -JSON -echo "targets: 1" -`, - ); - await chmod(fakeClawScan, 0o755); - const previousEnabled = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - const previousCommand = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND; - const previousSandbox = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = "1"; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND = fakeClawScan; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX = "docker"; - - const shadow = await runClawScanShadow( - { - job: { - _id: "securityScanJobs:shadow", - hasMaliciousSignal: false, - leaseToken: "lease-secret", - source: "vt-update", - targetKind: "skillVersion", - waitForVtUntil: 0, - }, - target: { - version: { - vtAnalysis: { - status: "clean", - engineStats: { malicious: 0, suspicious: 0 }, - }, - }, - }, - }, - workspace, - { - checkedAt: 123, - confidence: "medium", - status: "clean", - verdict: "benign", - }, - ); - - expect(shadow).toMatchObject({ - prod: { confidence: "medium", status: "clean", verdict: "benign" }, - shadow: { - confidence: "high", - judgeStatus: "completed", - profile: "clawhub", - scannerStatuses: { - "clawscan-static": "completed", - skillspector: "completed", - virustotal: "completed", - }, - status: "clean", - verdict: "benign", - }, - status: "completed", - }); - expect(shadow.command).toEqual( - expect.arrayContaining([ - fakeClawScan, - "./artifact", - "--profile", - "clawhub", - "--scanner-result", - expect.stringMatching(/^virustotal=/), - "--sandbox", - "docker", - "--sandbox-image", - expect.stringContaining("@sha256:"), - ]), - ); - expect(shadow.vtFixturePath).toBeDefined(); - const vtFixture = JSON.parse(await readFile(String(shadow.vtFixturePath), "utf8")); - expect(vtFixture).toMatchObject({ status: "clean" }); - - if (previousEnabled === undefined) delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = previousEnabled; - if (previousCommand === undefined) - delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND = previousCommand; - if (previousSandbox === undefined) - delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX = previousSandbox; - }); - - it("runs OSS ClawScan shadow mode for skill scan request jobs", async () => { - const workspace = await tempDir(); - await mkdir(join(workspace, "artifact"), { recursive: true }); - await writeFile(join(workspace, "artifact", "SKILL.md"), "# Pending publish\n"); - const fakeClawScan = join(workspace, "fake-clawscan"); - await writeFile( - fakeClawScan, - `#!/usr/bin/env bash -set -euo pipefail -target="$1" -out="" -while [[ $# -gt 0 ]]; do - case "$1" in - --output) - out="$2" - shift 2 - ;; - *) - shift - ;; - esac -done -if [[ "$target" != "./artifact" ]]; then - echo "unexpected target: $target" >&2 - exit 9 -fi -mkdir -p "$(dirname "$out")" -cat > "$out" <<'JSON' -{"schemaVersion":"clawscan-run-v1","profile":"clawhub","scanners":{"skillspector":{"status":"completed"},"virustotal":{"status":"completed"},"clawscan-static":{"status":"completed"}},"judge":{"status":"completed","result":{"verdict":"suspicious","confidence":"medium"}}} -JSON -`, - ); - await chmod(fakeClawScan, 0o755); - const previousEnabled = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - const previousCommand = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND; - const previousSandbox = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = "1"; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND = fakeClawScan; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX = "off"; - - const shadow = await runClawScanShadow( - { - job: { - _id: "securityScanJobs:shadow-scan-request", - hasMaliciousSignal: false, - leaseToken: "lease-secret", - source: "publish", - targetKind: "skillScanRequest", - waitForVtUntil: 0, - }, - target: {}, - }, - workspace, - { - checkedAt: 123, - confidence: "medium", - status: "suspicious", - verdict: "suspicious", - }, - ); - - expect(shadow).toMatchObject({ - prod: { confidence: "medium", status: "suspicious", verdict: "suspicious" }, - shadow: { - confidence: "medium", - judgeStatus: "completed", - profile: "clawhub", - scannerStatuses: { - "clawscan-static": "completed", - skillspector: "completed", - virustotal: "completed", - }, - status: "suspicious", - verdict: "suspicious", - }, - status: "completed", - }); - expect(shadow.command).toEqual( - expect.arrayContaining([ - fakeClawScan, - "./artifact", - "--profile", - "clawhub", - "--scanner-result", - expect.stringMatching(/^virustotal=/), - ]), - ); - expect(JSON.parse(await readFile(String(shadow.vtFixturePath), "utf8"))).toBeNull(); - - if (previousEnabled === undefined) delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = previousEnabled; - if (previousCommand === undefined) - delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND = previousCommand; - if (previousSandbox === undefined) - delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX = previousSandbox; - }); - - it("marks OSS ClawScan shadow mode failed when the judge fails", async () => { - const workspace = await tempDir(); - await mkdir(join(workspace, "artifact"), { recursive: true }); - await writeFile(join(workspace, "artifact", "SKILL.md"), "# Demo\n"); - const fakeClawScan = join(workspace, "fake-clawscan"); - await writeFile( - fakeClawScan, - `#!/usr/bin/env bash -set -euo pipefail -out="" -while [[ $# -gt 0 ]]; do - case "$1" in - --output) - out="$2" - shift 2 - ;; - *) - shift - ;; - esac -done -mkdir -p "$(dirname "$out")" -cat > "$out" <<'JSON' -{"schemaVersion":"clawscan-run-v1","profile":"clawhub","scanners":{"skillspector":{"status":"completed"},"virustotal":{"status":"completed"},"clawscan-static":{"status":"completed"}},"judge":{"status":"failed","error":"schema mismatch"}} -JSON -`, - ); - await chmod(fakeClawScan, 0o755); - const previousEnabled = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - const previousCommand = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND; - const previousSandbox = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = "1"; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND = fakeClawScan; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX = "off"; - - const shadow = await runClawScanShadow( - { - job: { - _id: "securityScanJobs:shadow-failed", - hasMaliciousSignal: false, - leaseToken: "lease-secret", - source: "publish", - targetKind: "skillVersion", - waitForVtUntil: 0, - }, - target: {}, - }, - workspace, - { - checkedAt: 123, - confidence: "medium", - status: "clean", - verdict: "benign", - }, - ); - - expect(shadow).toMatchObject({ - error: "ClawScan judge status was failed", - shadow: { - judgeStatus: "failed", - profile: "clawhub", - scannerStatuses: { - "clawscan-static": "completed", - skillspector: "completed", - virustotal: "completed", - }, - }, - status: "failed", - }); - - if (previousEnabled === undefined) delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = previousEnabled; - if (previousCommand === undefined) - delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND = previousCommand; - if (previousSandbox === undefined) - delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX = previousSandbox; - }); - - it("marks OSS ClawScan shadow mode failed when an expected scanner fails", async () => { - const workspace = await tempDir(); - await mkdir(join(workspace, "artifact"), { recursive: true }); - await writeFile(join(workspace, "artifact", "SKILL.md"), "# Demo\n"); - const fakeClawScan = join(workspace, "fake-clawscan"); - await writeFile( - fakeClawScan, - `#!/usr/bin/env bash -set -euo pipefail -out="" -while [[ $# -gt 0 ]]; do - case "$1" in - --output) - out="$2" - shift 2 - ;; - *) - shift - ;; - esac -done -mkdir -p "$(dirname "$out")" -cat > "$out" <<'JSON' -{"schemaVersion":"clawscan-run-v1","profile":"clawhub","scanners":{"skillspector":{"status":"failed"},"virustotal":{"status":"completed"},"clawscan-static":{"status":"completed"}},"judge":{"status":"completed","result":{"verdict":"benign","confidence":"high"}}} -JSON -`, - ); - await chmod(fakeClawScan, 0o755); - const previousEnabled = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - const previousCommand = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND; - const previousSandbox = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = "1"; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND = fakeClawScan; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX = "off"; - - const shadow = await runClawScanShadow( - { - job: { - _id: "securityScanJobs:shadow-scanner-failed", - hasMaliciousSignal: false, - leaseToken: "lease-secret", - source: "publish", - targetKind: "skillVersion", - waitForVtUntil: 0, - }, - target: {}, - }, - workspace, - { - checkedAt: 123, - confidence: "medium", - status: "clean", - verdict: "benign", - }, - ); - - expect(shadow).toMatchObject({ - error: "ClawScan scanner skillspector status was failed", - shadow: { - judgeStatus: "completed", - scannerStatuses: { - "clawscan-static": "completed", - skillspector: "failed", - virustotal: "completed", - }, - verdict: "benign", - }, - status: "failed", - }); - - if (previousEnabled === undefined) delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = previousEnabled; - if (previousCommand === undefined) - delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND = previousCommand; - if (previousSandbox === undefined) - delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX = previousSandbox; - }); - - it("skips OSS ClawScan shadow mode for unsupported parity contexts", async () => { - const workspace = await tempDir(); - const previousEnabled = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = "1"; - - const shadow = await runClawScanShadow( - { - job: { - _id: "securityScanJobs:shadow-skipped", - hasMaliciousSignal: false, - leaseToken: "lease-secret", - source: "publish", - targetKind: "packageRelease", - waitForVtUntil: 0, - }, - target: {}, - }, - workspace, - { - checkedAt: 123, - confidence: "medium", - status: "clean", - verdict: "benign", - }, - ); - - expect(shadow).toMatchObject({ - error: - "ClawScan shadow parity is only enabled for skillVersion or skillScanRequest jobs, got packageRelease", - status: "skipped", - }); - - if (previousEnabled === undefined) delete process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN; - else process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN = previousEnabled; - }); - it("writes redacted Codex diagnostics without copying submitted artifact files or signed URLs", async () => { const diagnosticsRoot = await tempDir(); const artifactWorkspace = await tempDir(); @@ -1324,19 +920,16 @@ JSON }, }, llmAnalysis: { confidence: "low", status: "clean", verdict: "benign" }, - clawscanShadow: { - command: ["clawscan", "./artifact", "--profile", "clawhub"], - prod: { confidence: "low", status: "clean", verdict: "benign" }, - shadow: { + secondaryScan: { + authoritative: { + confidence: "low", + implementation: "legacy", + status: "clean", + verdict: "benign", + }, + secondary: { confidence: "high", - judgeStatus: "completed", - profile: "clawhub", - scannerStatuses: { - "clawscan-static": "completed", - skillspector: "completed", - virustotal: "completed", - }, - schemaVersion: "clawscan-run-v1", + implementation: "clawscan", status: "clean", verdict: "benign", }, @@ -1419,17 +1012,16 @@ JSON status: "failed", }); expect(diagnostic.job.leaseToken).toBeUndefined(); - expect(diagnostic.clawscanShadow).toMatchObject({ - prod: { confidence: "low", status: "clean", verdict: "benign" }, - shadow: { + expect(diagnostic.secondaryScan).toMatchObject({ + authoritative: { + confidence: "low", + implementation: "legacy", + status: "clean", + verdict: "benign", + }, + secondary: { confidence: "high", - judgeStatus: "completed", - profile: "clawhub", - scannerStatuses: { - "clawscan-static": "completed", - skillspector: "completed", - virustotal: "completed", - }, + implementation: "clawscan", status: "clean", verdict: "benign", }, @@ -1455,12 +1047,10 @@ JSON expect(allDiagnosticText).not.toContain("sk-short-fixture"); expect(allDiagnosticText).not.toContain("quoted artifact payload"); expect(allDiagnosticText).not.toContain("SkillSpector artifact payload"); - const comparison = JSON.parse( - await readFile(join(jobDir, "clawscan-shadow-comparison.json"), "utf8"), - ); + const comparison = JSON.parse(await readFile(join(jobDir, "scan-comparison.json"), "utf8")); expect(comparison).toMatchObject({ - prod: { status: "clean", verdict: "benign" }, - shadow: { status: "clean", verdict: "benign" }, + authoritative: { implementation: "legacy", status: "clean", verdict: "benign" }, + secondary: { implementation: "clawscan", status: "clean", verdict: "benign" }, status: "completed", }); expect(await readdir(jobDir)).not.toContain("artifact"); @@ -1652,8 +1242,9 @@ JSON ]); }); - it("preserves sanitized ClawScan shadow failure reasons in comparison artifacts", async () => { + it("preserves sanitized secondary failure reasons in comparison artifacts", async () => { const diagnosticsRoot = await tempDir(); + const redactionFixture = "sk-short-fixture"; await writeJobDiagnostic({ completedAt: 2, @@ -1669,29 +1260,31 @@ JSON }, target: {}, }, - clawscanShadow: { - error: "clawscan timed out with api_key=sk-shadow-secret", + secondaryScan: { + authoritative: { + implementation: "legacy", + status: "clean", + verdict: "benign", + }, + error: `clawscan timed out with api_key=${redactionFixture}`, + secondary: { + implementation: "clawscan", + }, status: "failed", - stderr: "provider stderr token=shadow-secret", }, startedAt: 1, status: "completed", }); const comparison = JSON.parse( - await readFile( - join(diagnosticsRoot, "job-shadow-failed", "clawscan-shadow-comparison.json"), - "utf8", - ), + await readFile(join(diagnosticsRoot, "job-shadow-failed", "scan-comparison.json"), "utf8"), ); expect(comparison).toMatchObject({ error: expect.stringContaining("clawscan timed out"), status: "failed", - stderr: expect.stringMatching(/^\[redacted \d+ chars\]$/), }); const comparisonText = JSON.stringify(comparison); - expect(comparisonText).not.toContain("sk-shadow-secret"); - expect(comparisonText).not.toContain("shadow-secret"); + expect(comparisonText).not.toContain(redactionFixture); }); }); diff --git a/scripts/security/run-codex-scan-worker.ts b/scripts/security/run-codex-scan-worker.ts index 6c49c595..e48efd71 100644 --- a/scripts/security/run-codex-scan-worker.ts +++ b/scripts/security/run-codex-scan-worker.ts @@ -117,44 +117,41 @@ type ClawScanCommandDiagnostic = { }; }; -type ClawScanShadowDiagnostic = { - artifactPath?: string; - command?: string[]; - completedAt?: number; - durationMs?: number; - error?: string; - exitCode?: number | null; - prod?: { +type ScanImplementation = "legacy" | "clawscan"; + +export type SecurityScanMode = "legacy" | "shadow" | "clawscan"; + +type SecondaryScanDiagnostic = { + authoritative: { confidence?: string; + implementation: ScanImplementation; status?: string; verdict?: string; }; - shadow?: { + completedAt?: number; + durationMs?: number; + error?: string; + secondary: { confidence?: string; - judgeStatus?: string; - profile?: string; - scannerStatuses: Record; - schemaVersion?: string; + implementation: ScanImplementation; status?: string; verdict?: string; }; startedAt?: number; - status: "completed" | "failed" | "skipped"; - stdout?: string; - stderr?: string; - vtFixturePath?: string; + status: "completed" | "failed"; }; type JobDiagnosticInput = { clawscan?: ClawScanCommandDiagnostic; - clawscanShadow?: ClawScanShadowDiagnostic; codex?: CodexCommandDiagnostic; completedAt: number; diagnosticsRoot?: string; error?: string; job: ClaimedJob; llmAnalysis?: unknown; + mode?: SecurityScanMode; runId?: string; + secondaryScan?: SecondaryScanDiagnostic; skillSpector?: CodexCommandDiagnostic; skillSpectorAnalysis?: unknown; startedAt: number; @@ -173,10 +170,7 @@ const DEFAULT_BATCH_LIMIT = 4; const DEFAULT_MAX_RUNTIME_MS = 40 * 60 * 1000; const DEFAULT_CODEX_SCAN_TIMEOUT_MS = 20 * 60 * 1000; const DEFAULT_CLAWSCAN_TIMEOUT_MS = 20 * 60 * 1000; -const DEFAULT_CLAWSCAN_SHADOW_TIMEOUT_MS = 20 * 60 * 1000; -const DEFAULT_CLAWSCAN_SHADOW_SANDBOX_IMAGE = - "ghcr.io/openclaw/clawscan-runtime@sha256:d85bfe671fe597edc6802f9d6a07dd91b59c69cec4faa6e8f89778037507dc3b"; -const EXPECTED_CLAWHUB_SHADOW_SCANNERS = ["clawscan-static", "skillspector", "virustotal"]; +const REQUIRED_CLAWHUB_SCANNERS = ["clawscan-static", "skillspector", "virustotal"]; const MAX_DIAGNOSTIC_TEXT_CHARS = 20_000; const MAX_STORED_SKILLSPECTOR_ISSUES = 25; const MAX_STORED_SKILLSPECTOR_TEXT_CHARS = 2_000; @@ -282,6 +276,16 @@ function loadClawHubOutputSchemaContract(path: string): ClawHubOutputSchemaContr const CLAWHUB_OUTPUT_SCHEMA_CONTRACT = loadClawHubOutputSchemaContract(schemaPath); +export function resolveSecurityScanMode( + value = process.env.CODEX_SECURITY_SCAN_MODE, +): SecurityScanMode { + if (value === undefined || value === "") return "legacy"; + if (value === "legacy" || value === "shadow" || value === "clawscan") return value; + throw new Error( + `CODEX_SECURITY_SCAN_MODE must be one of legacy, shadow, or clawscan; received ${JSON.stringify(value)}`, + ); +} + function parseArgs() { const args = process.argv.slice(2); const get = (name: string) => { @@ -427,19 +431,18 @@ const DIAGNOSTIC_PUBLIC_TEXT_PATHS = new Set([ "codexstdout.item.type", "codexstdout.status", "codexstdout.type", - "clawscanshadow.prod.confidence", - "clawscanshadow.prod.status", - "clawscanshadow.prod.verdict", - "clawscanshadow.shadow.confidence", - "clawscanshadow.shadow.judgestatus", - "clawscanshadow.shadow.profile", - "clawscanshadow.shadow.schemaversion", - "clawscanshadow.shadow.status", - "clawscanshadow.shadow.verdict", - "clawscanshadow.status", "llmanalysis.confidence", "llmanalysis.status", "llmanalysis.verdict", + "secondaryscan.authoritative.confidence", + "secondaryscan.authoritative.implementation", + "secondaryscan.authoritative.status", + "secondaryscan.authoritative.verdict", + "secondaryscan.secondary.confidence", + "secondaryscan.secondary.implementation", + "secondaryscan.secondary.status", + "secondaryscan.secondary.verdict", + "secondaryscan.status", "skillspectoranalysis.issues.*.issueid", "skillspectoranalysis.issues.*.severity", "skillspectoranalysis.recommendation", @@ -469,12 +472,10 @@ function isDiagnosticSecretPath(path: string[]) { function shouldPreserveDiagnosticText(path: string[], original: string, redacted: string) { const key = diagnosticPathKey(path); - if (key === "clawscanshadow.error") return true; + if (key === "secondaryscan.error") return true; return ( original === redacted && - (DIAGNOSTIC_PUBLIC_TEXT_PATHS.has(key) || - key.startsWith("clawscanartifact.env.") || - key.startsWith("clawscanshadow.shadow.scannerstatuses.")) && + (DIAGNOSTIC_PUBLIC_TEXT_PATHS.has(key) || key.startsWith("clawscanartifact.env.")) && DIAGNOSTIC_PUBLIC_TEXT_VALUE_PATTERN.test(redacted) ); } @@ -782,6 +783,7 @@ export async function writeJobDiagnostic(input: JobDiagnosticInput) { waitForVtUntil: input.job.job.waitForVtUntil, }, llmAnalysis: redactDiagnosticValue(input.llmAnalysis, ["llmAnalysis"]), + mode: input.mode, runId: input.runId, clawscan: input.clawscan ? { @@ -791,8 +793,8 @@ export async function writeJobDiagnostic(input: JobDiagnosticInput) { : undefined, } : undefined, - clawscanShadow: input.clawscanShadow - ? redactDiagnosticValue(input.clawscanShadow, ["clawscanShadow"]) + secondaryScan: input.secondaryScan + ? redactDiagnosticValue(input.secondaryScan, ["secondaryScan"]) : undefined, skillSpectorAnalysis: redactDiagnosticValue(input.skillSpectorAnalysis, [ "skillSpectorAnalysis", @@ -829,10 +831,10 @@ export async function writeJobDiagnostic(input: JobDiagnosticInput) { await writeFile(join(jobDir, "diagnostic.json"), `${JSON.stringify(diagnostic, null, 2)}\n`); - if (input.clawscanShadow) { + if (input.secondaryScan) { await writeFile( - join(jobDir, "clawscan-shadow-comparison.json"), - `${JSON.stringify(redactDiagnosticValue(input.clawscanShadow, ["clawscanShadow"]), null, 2)}\n`, + join(jobDir, "scan-comparison.json"), + `${JSON.stringify(redactDiagnosticValue(input.secondaryScan, ["secondaryScan"]), null, 2)}\n`, ); } } @@ -1531,18 +1533,6 @@ function clawScanTimeoutMs() { return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CLAWSCAN_TIMEOUT_MS; } -function clawScanShadowTimeoutMs() { - const parsed = Number(process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_TIMEOUT_MS); - return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CLAWSCAN_SHADOW_TIMEOUT_MS; -} - -function clawScanShadowEnabled() { - const value = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN?.trim().toLowerCase(); - return value === "1" || value === "true" || value === "yes"; -} - -type ScanImplementation = "codex" | "clawscan"; - const REQUIRED_CLAWHUB_RESULT_KEYS = CLAWHUB_OUTPUT_SCHEMA_CONTRACT.requiredResultKeys; const REQUIRED_CLAWHUB_DIMENSION_KEYS = CLAWHUB_OUTPUT_SCHEMA_CONTRACT.requiredDimensionKeys; const REQUIRED_CLAWHUB_DIMENSION_FIELD_KEYS = @@ -1639,7 +1629,7 @@ function artifactCompletedAtMs(artifact: Record) { function readClawScanScannerStatuses( artifact: Record, - scannerSet = EXPECTED_CLAWHUB_SHADOW_SCANNERS, + scannerSet = REQUIRED_CLAWHUB_SCANNERS, ) { const scanners = asRecord(artifact.scanners); const scannerStatuses: Record = {}; @@ -1835,14 +1825,6 @@ export async function runCodex( return toStoredLlmAnalysis(parsed); } -function targetVirusTotalAnalysis(job: ClaimedJob) { - return ( - (job.target.version as Record | undefined)?.vtAnalysis ?? - (job.target.release as Record | undefined)?.vtAnalysis ?? - null - ); -} - async function resolveClawScanTarget(workspace: string, job: ClaimedJob) { if (job.job.targetKind === "packageRelease") { const packageRoot = join(workspace, "artifact", "package"); @@ -1851,161 +1833,82 @@ async function resolveClawScanTarget(workspace: string, job: ClaimedJob) { return "./artifact"; } -function clawScanShadowUnsupportedContext(job: ClaimedJob) { - if (job.job.targetKind !== "skillVersion" && job.job.targetKind !== "skillScanRequest") { - return `ClawScan shadow parity is only enabled for skillVersion or skillScanRequest jobs, got ${job.job.targetKind}`; - } - if (job.job.source !== "publish" && job.job.source !== "vt-update") { - return `ClawScan shadow parity is only enabled for publish or vt-update jobs, got ${job.job.source}`; - } - if (job.target.trustedOpenClawPlugin) { - return "ClawScan 0.1.1 cannot preserve trusted OpenClaw plugin context"; - } - return undefined; -} - -function clawScanShadowVerdictFromArtifact(artifact: unknown) { - const record = asRecord(artifact); - const judge = asRecord(record?.judge); - const result = asRecord(judge?.result); - const scanners = asRecord(record?.scanners); - const scannerStatuses: Record = {}; - if (scanners) { - for (const [scanner, value] of Object.entries(scanners)) { - const scannerRecord = asRecord(value); - const status = readString(scannerRecord ?? {}, ["status"]); - scannerStatuses[scanner] = status ?? "unknown"; - } - } - const verdict = readString(result ?? {}, ["verdict", "status"]); - return { - confidence: readString(result ?? {}, ["confidence"]), - judgeStatus: readString(judge ?? {}, ["status"]), - profile: readString(record ?? {}, ["profile"]), - scannerStatuses, - schemaVersion: readString(record ?? {}, ["schemaVersion"]), - status: verdict ? verdictToStatus(verdict) : undefined, - verdict, - }; -} - -function validateClawScanShadowResult(shadow: NonNullable) { - for (const scanner of EXPECTED_CLAWHUB_SHADOW_SCANNERS) { - if (shadow.scannerStatuses[scanner] !== "completed") { - return `ClawScan scanner ${scanner} status was ${shadow.scannerStatuses[scanner] ?? "missing"}`; - } - } - if (shadow.judgeStatus !== "completed") { - return `ClawScan judge status was ${shadow.judgeStatus ?? "missing"}`; - } - if (!shadow.verdict) { - return "ClawScan judge did not return a verdict"; - } - return undefined; -} - -export async function runClawScanShadow( +async function runLegacyScan( job: ClaimedJob, workspace: string, - llmAnalysis: StoredLlmAnalysis | undefined, -): Promise { - if (!clawScanShadowEnabled()) { - return { status: "skipped" }; - } - const unsupportedContext = clawScanShadowUnsupportedContext(job); - if (unsupportedContext) { - return { - status: "skipped", - error: unsupportedContext, - }; - } - if (!llmAnalysis) { - return { - status: "skipped", - error: "authoritative ClawHub scan did not produce llmAnalysis", - }; - } + codexDiagnostic: CodexCommandDiagnostic, + skillSpectorDiagnostic: CodexCommandDiagnostic, +) { + const skillSpectorInputs = await resolveSkillSpectorScanInputs(workspace, job); + const skillSpectorAnalysis = + skillSpectorInputs.length > 0 + ? await runSkillSpector(workspace, skillSpectorInputs, (next) => { + Object.assign(skillSpectorDiagnostic, next); + }) + : undefined; + const llmAnalysis = await runCodex(job, workspace, skillSpectorAnalysis, (next) => { + Object.assign(codexDiagnostic, next); + }); + return { llmAnalysis, skillSpectorAnalysis }; +} +async function runSecondaryScan(input: { + authoritativeAnalysis: StoredLlmAnalysis; + authoritativeImplementation: ScanImplementation; + clawscanDiagnostic: ClawScanCommandDiagnostic; + codexDiagnostic: CodexCommandDiagnostic; + job: ClaimedJob; + secondaryImplementation: ScanImplementation; + skillSpectorDiagnostic: CodexCommandDiagnostic; + workspace: string; +}): Promise { const startedAt = Date.now(); - const diagnostic: ClawScanShadowDiagnostic = { - prod: { - confidence: llmAnalysis.confidence, - status: llmAnalysis.status, - verdict: llmAnalysis.verdict, + const diagnostic: SecondaryScanDiagnostic = { + authoritative: { + confidence: input.authoritativeAnalysis.confidence, + implementation: input.authoritativeImplementation, + status: input.authoritativeAnalysis.status, + verdict: input.authoritativeAnalysis.verdict, + }, + secondary: { + implementation: input.secondaryImplementation, }, startedAt, status: "failed", }; try { - const shadowDir = join(workspace, "clawscan-shadow"); - await mkdir(shadowDir, { recursive: true }); - const vtFixturePath = join(shadowDir, "virustotal-prod.json"); - const artifactPath = join(shadowDir, "artifact.json"); - await writeFile(vtFixturePath, `${JSON.stringify(targetVirusTotalAnalysis(job), null, 2)}\n`); - const target = await resolveClawScanTarget(workspace, job); - const command = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_COMMAND ?? "clawscan"; - const sandboxMode = process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX ?? "docker"; - const sandboxImage = - process.env.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN_SANDBOX_IMAGE ?? - DEFAULT_CLAWSCAN_SHADOW_SANDBOX_IMAGE; - const args = [ - target, - "--profile", - "clawhub", - "--scanner-result", - `virustotal=${vtFixturePath}`, - "--output", - artifactPath, - "--sandbox", - sandboxMode, - ]; - if (sandboxMode === "docker") { - args.push("--sandbox-image", sandboxImage); - } - diagnostic.artifactPath = artifactPath; - diagnostic.command = [command, ...args]; - diagnostic.vtFixturePath = vtFixturePath; - - const output = await runCommand(command, args, { - cwd: workspace, - timeoutMs: clawScanShadowTimeoutMs(), - }); - const raw = await readFile(artifactPath, "utf8"); - const artifact = JSON.parse(raw) as unknown; + const result = + input.secondaryImplementation === "clawscan" + ? await runClawScan(input.job, input.workspace, (next) => { + Object.assign(input.clawscanDiagnostic, next); + }) + : await runLegacyScan( + input.job, + input.workspace, + input.codexDiagnostic, + input.skillSpectorDiagnostic, + ); const completedAt = Date.now(); - const shadow = clawScanShadowVerdictFromArtifact(artifact); - const resultError = validateClawScanShadowResult(shadow); return { ...diagnostic, completedAt, durationMs: completedAt - startedAt, - ...(resultError ? { error: resultError } : {}), - shadow, - status: resultError ? "failed" : "completed", - stderr: output.stderr, - stdout: output.stdout, + secondary: { + confidence: result.llmAnalysis.confidence, + implementation: input.secondaryImplementation, + status: result.llmAnalysis.status, + verdict: result.llmAnalysis.verdict, + }, + status: "completed", }; } catch (error) { const completedAt = Date.now(); - let exitCode: number | null | undefined; - let stderr: string | undefined; - let stdout: string | undefined; - let publicError = error instanceof Error ? error.message : String(error); - if (error instanceof CommandFailure) { - exitCode = error.exitCode; - stderr = error.stderr; - stdout = error.stdout; - publicError = error.message; - } return { ...diagnostic, completedAt, durationMs: completedAt - startedAt, - error: sanitizeWorkerErrorMessage(publicError), - exitCode, - stderr, - stdout, + error: sanitizeWorkerErrorMessage(error instanceof Error ? error.message : String(error)), status: "failed", }; } @@ -2016,37 +1919,39 @@ export async function processJob( token: string, job: ClaimedJob, diagnosticsRoot: string | undefined, - implementation: ScanImplementation = "codex", + mode: SecurityScanMode = "legacy", ): Promise { const workspace = await mkdtemp(join(tmpdir(), `clawhub-codex-scan-${basename(job.job._id)}-`)); const startedAt = Date.now(); - const scanImplementation = implementation; + // Authority is global: private trust metadata cannot create a per-job legacy exception + // to the artifact-only ClawScan contract. + const authoritativeImplementation: ScanImplementation = + mode === "clawscan" ? "clawscan" : "legacy"; + const secondaryImplementation: ScanImplementation | undefined = + mode === "shadow" ? "clawscan" : mode === "clawscan" ? "legacy" : undefined; const clawscan: ClawScanCommandDiagnostic = {}; const codex: CodexCommandDiagnostic = {}; const skillSpector: CodexCommandDiagnostic = {}; let errorMessage: string | undefined; let llmAnalysis: StoredLlmAnalysis | undefined; let skillSpectorAnalysis: SkillSpectorAnalysis | undefined; - let clawscanShadow: ClawScanShadowDiagnostic | undefined; + let secondaryScan: SecondaryScanDiagnostic | undefined; let status: JobDiagnosticInput["status"] = "failed"; try { await writeArtifactWorkspace(job, workspace); - if (scanImplementation === "clawscan") { + if (authoritativeImplementation === "clawscan") { const mapped = await runClawScan(job, workspace, (next) => { Object.assign(clawscan, next); }); llmAnalysis = mapped.llmAnalysis; skillSpectorAnalysis = mapped.skillSpectorAnalysis; } else { - const skillSpectorInputs = await resolveSkillSpectorScanInputs(workspace, job); - if (skillSpectorInputs.length > 0) { - skillSpectorAnalysis = await runSkillSpector(workspace, skillSpectorInputs, (next) => { - Object.assign(skillSpector, next); - }); - } - llmAnalysis = await runCodex(job, workspace, skillSpectorAnalysis, (next) => { - Object.assign(codex, next); - }); + ({ llmAnalysis, skillSpectorAnalysis } = await runLegacyScan( + job, + workspace, + codex, + skillSpector, + )); } if (!llmAnalysis) throw new Error("Security scan did not produce llmAnalysis"); await client.action(api.securityScan.completeCodexScanJob, { @@ -2058,31 +1963,42 @@ export async function processJob( runId: process.env.GITHUB_RUN_ID, }); status = "completed"; - if (scanImplementation === "codex") { - clawscanShadow = await runClawScanShadow(job, workspace, llmAnalysis); - if (clawscanShadow.status !== "skipped") { - logger.info( - { - durationMs: clawscanShadow.durationMs, - event: "security_scan_clawscan_shadow_completed", - jobId: job.job._id, - prodStatus: llmAnalysis.status, - prodVerdict: llmAnalysis.verdict, - shadowStatus: clawscanShadow.shadow?.status, - shadowVerdict: clawscanShadow.shadow?.verdict, - shadowRunStatus: clawscanShadow.status, - targetKind: job.job.targetKind, - }, - "ClawScan shadow run completed", - ); - } + if (secondaryImplementation) { + secondaryScan = await runSecondaryScan({ + authoritativeAnalysis: llmAnalysis, + authoritativeImplementation, + clawscanDiagnostic: clawscan, + codexDiagnostic: codex, + job, + secondaryImplementation, + skillSpectorDiagnostic: skillSpector, + workspace, + }); + logger.info( + { + authoritativeImplementation, + authoritativeStatus: llmAnalysis.status, + authoritativeVerdict: llmAnalysis.verdict, + durationMs: secondaryScan.durationMs, + event: "security_scan_secondary_completed", + jobId: job.job._id, + mode, + secondaryImplementation, + secondaryRunStatus: secondaryScan.status, + secondaryStatus: secondaryScan.secondary.status, + secondaryVerdict: secondaryScan.secondary.verdict, + targetKind: job.job.targetKind, + }, + "secondary security scan completed", + ); } logger.info( { durationMs: Date.now() - startedAt, event: "security_scan_job_completed", - implementation: scanImplementation, + implementation: authoritativeImplementation, jobId: job.job._id, + mode, scannerPhase: "complete", status: llmAnalysis.status, targetKind: job.job.targetKind, @@ -2123,12 +2039,13 @@ export async function processJob( codex, completedAt: Date.now(), clawscan, - clawscanShadow, diagnosticsRoot, error: errorMessage, job, llmAnalysis, + mode, runId: process.env.GITHUB_RUN_ID, + secondaryScan, skillSpector, skillSpectorAnalysis, startedAt, @@ -2260,6 +2177,7 @@ export async function runContinuouslyRefilledWorkerPool(options: { async function main() { const { batchLimit, maxJobs, maxRuntimeMs, leaseMs, lane, diagnosticsRoot } = parseArgs(); + const mode = resolveSecurityScanMode(); assertCodexWorkerExecutionAllowed(process.env); maskKnownWorkerSecrets(); const convexUrl = process.env.CONVEX_URL ?? process.env.VITE_CONVEX_URL; @@ -2275,7 +2193,7 @@ async function main() { const claimDeadline = startedAt + maxRuntimeMs; logger.info( - { diagnosticsRoot, event: "security_scan_diagnostics_directory", lane, workerId }, + { diagnosticsRoot, event: "security_scan_diagnostics_directory", lane, mode, workerId }, "security scan diagnostics directory", ); @@ -2352,7 +2270,7 @@ async function main() { ); return { claimedCount: leases.length, jobs }; }, - processClaimedJob: (job) => processJob(client, token, job, diagnosticsRoot), + processClaimedJob: (job) => processJob(client, token, job, diagnosticsRoot, mode), idlePollMs: lane === "priority" ? 15_000 : undefined, }); diff --git a/scripts/security/security-scan-worker-workflow.test.ts b/scripts/security/security-scan-worker-workflow.test.ts index fd7f6495..d718964c 100644 --- a/scripts/security/security-scan-worker-workflow.test.ts +++ b/scripts/security/security-scan-worker-workflow.test.ts @@ -101,9 +101,13 @@ describe("security-scan-codex workflow", () => { expect(jobEnv.CODEX_SECURITY_SCAN_TIMEOUT_MS).toBe( "${{ vars.CODEX_SECURITY_SCAN_TIMEOUT_MS || '240000' }}", ); - expect(jobEnv.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN).toBe( - "${{ vars.CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN || '0' }}", + expect(jobEnv.CODEX_SECURITY_SCAN_MODE).toBe( + "${{ vars.CODEX_SECURITY_SCAN_MODE || 'legacy' }}", ); + expect(jobEnv.CODEX_SECURITY_SCAN_CLAWSCAN_TIMEOUT_MS).toBe( + "${{ vars.CODEX_SECURITY_SCAN_CLAWSCAN_TIMEOUT_MS || '240000' }}", + ); + expect(jobEnv).not.toHaveProperty("CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN"); expect(jobEnv).not.toHaveProperty("OPENAI_API_KEY"); expect(jobEnv).not.toHaveProperty("CODEX_API_KEY"); expect(jobEnv).not.toHaveProperty("SECURITY_SCAN_WORKER_TOKEN"); diff --git a/specs/security-moderation.md b/specs/security-moderation.md index a9d05639..cf61b749 100644 --- a/specs/security-moderation.md +++ b/specs/security-moderation.md @@ -279,9 +279,18 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic - The worker's explicit artifact-only OSS ClawScan route accepts every claimed target kind and source through the same completion/failure contract. Skill versions and scan requests use the isolated `artifact` root; extracted - ClawPack releases use `artifact/package`. Production remains on the legacy - default until the separate whole-system rollout changes it, and ClawScan - failures never trigger per-job legacy fallback. + ClawPack releases use `artifact/package`. +- The temporary whole-system rollout mode accepts exactly `legacy`, `shadow`, + and `clawscan`, with `legacy` as the safe default until the production + cutover is explicitly approved. `legacy` runs only the legacy implementation. + `shadow` persists legacy authoritatively, then runs ClawScan diagnostically + against the same isolated artifact. `clawscan` persists ClawScan + authoritatively, then runs legacy diagnostically against that same artifact + during the soak. Secondary execution cannot change stored verdicts, + publication or moderation behavior, retries, or authoritative job success. + Authoritative ClawScan failures use the existing failure/retry lifecycle and + never trigger per-job legacy fallback. Rollback is a manual whole-system mode + change to `legacy`. - Claimable queue work edge-triggers a coalesced GitHub Actions worker dispatch. Successful completion requests another dispatch while queued work remains; a five-minute Convex cron is only a recovery watchdog for lost dispatch signals.