admin/clawhub
1
0
Fork 0
mirror of https://github.com/openclaw/clawhub.git synced 2026-08-14 00:47:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(security): remove legacy scan implementation (#3124)

Browse Source
This commit is contained in:
Patrick Erichsen
2026-07-16 16:49:36 -07:00
committed by GitHub
parent 97905c81b0
commit bcf33f04ee
9 changed files with 108 additions and 2365 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/security-scan-codex.yml
-2
View File
@@ -65,9 +65,7 @@ jobs:
CODEX_SECURITY_SCAN_MAX_JOBS: ${{ github.event.client_payload.max_jobs || inputs['max-jobs'] || '' }}
CODEX_SECURITY_SCAN_MAX_RUNTIME_MINUTES: ${{ github.event.client_payload.max_runtime_minutes || inputs['max-runtime-minutes'] || '12' }}
CODEX_SECURITY_SCAN_LANE: ${{ matrix.lane }}
CODEX_SECURITY_SCAN_MODE: ${{ vars.CODEX_SECURITY_SCAN_MODE || 'legacy' }}
CODEX_SECURITY_SCAN_CLAWSCAN_TIMEOUT_MS: ${{ vars.CODEX_SECURITY_SCAN_CLAWSCAN_TIMEOUT_MS || '240000' }}
CODEX_SECURITY_SCAN_TIMEOUT_MS: ${{ vars.CODEX_SECURITY_SCAN_TIMEOUT_MS || '240000' }}
CODEX_SECURITY_SCAN_LEASE_MINUTES: "60"
CODEX_SECURITY_SCAN_DIAGNOSTICS_DIR: codex-security-scan-diagnostics-${{ matrix.shard }}
CODEX_SECURITY_SCAN_SHARD: ${{ matrix.shard }}
scripts/security/run-codex-scan-worker-clawscan.test.ts
+36 -190
View File
@@ -113,57 +113,6 @@ async function writeFakeClawScanCommand(path: string, body: string) {
await chmod(path, 0o755);
}
async function withFakeLegacySecondary<T>(run: () => Promise<T>) {
const binDir = await tempDir();
await writeFakeClawScanCommand(
join(binDir, "skillspector"),
`out=""
while [[ $# -gt 0 ]]; do
case "$1" in
--output)
out="$2"
shift 2
;;
*)
shift
;;
esac
done
mkdir -p "$(dirname "$out")"
cat > "$out" <<'JSON'
{"status":"clean","issue_count":0,"issues":[]}
JSON`,
);
await writeFakeClawScanCommand(
join(binDir, "codex"),
`out=""
while [[ $# -gt 0 ]]; do
case "$1" in
--output-last-message)
out="$2"
shift 2
;;
*)
shift
;;
esac
done
mkdir -p "$(dirname "$out")"
cat > "$out" <<'JSON'
{"verdict":"benign","confidence":"high","summary":"legacy diagnostic","dimensions":{"purpose_capability":{"status":"ok","detail":"ok"}},"scan_findings_in_context":[],"user_guidance":"guidance"}
JSON`,
);
const previousPath = process.env.PATH;
process.env.PATH = `${binDir}:${previousPath ?? ""}`;
try {
return await run();
} finally {
if (previousPath === undefined) delete process.env.PATH;
else process.env.PATH = previousPath;
}
}
type ClawScanVerdict = "benign" | "suspicious" | "malicious";
function completeJudgeDimensions() {
@@ -265,21 +214,13 @@ function clawScanArtifactJson(options?: {
}
describe("run-codex-scan-worker clawscan authority", () => {
it("defaults skillVersion jobs to the legacy codex path unless clawscan is explicitly selected", async () => {
it("uses ClawScan as the only skillVersion scan implementation", async () => {
const workspace = await tempDir();
const fakeClawScan = join(workspace, "fake-clawscan");
const clawscanMarker = join(workspace, "clawscan-called.log");
await writeFakeClawScanCommand(
fakeClawScan,
`echo "called" > ${JSON.stringify(clawscanMarker)}
exit 0`,
);
const binDir = await tempDir();
const legacyMarker = join(workspace, "legacy-called.log");
await writeFakeClawScanCommand(
join(binDir, "skillspector"),
`echo "skillspector" >> ${JSON.stringify(legacyMarker)}
out=""
while [[ $# -gt 0 ]]; do
case "$1" in
@@ -294,34 +235,12 @@ while [[ $# -gt 0 ]]; do
done
mkdir -p "$(dirname "$out")"
cat > "$out" <<'JSON'
{"status":"clean","issue_count":0,"issues":[]}
JSON`,
);
await writeFakeClawScanCommand(
join(binDir, "codex"),
`echo "codex" >> ${JSON.stringify(legacyMarker)}
out=""
while [[ $# -gt 0 ]]; do
case "$1" in
--output-last-message)
out="$2"
shift 2
;;
*)
shift
;;
esac
done
mkdir -p "$(dirname "$out")"
cat > "$out" <<'JSON'
{"verdict":"benign","confidence":"high","summary":"summary","dimensions":{"purpose_capability":{"status":"ok","detail":"ok"}},"scan_findings_in_context":[],"user_guidance":"guidance"}
${clawScanArtifactJson({ verdict: "benign" })}
JSON`,
);
const previousCommand = process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND;
const previousPath = process.env.PATH;
process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND = fakeClawScan;
process.env.PATH = `${binDir}:${previousPath ?? ""}`;
try {
const client = {
action: vi.fn(async (..._args: unknown[]) => ({})),
@@ -329,7 +248,7 @@ JSON`,
const result = await processJob(
client,
"worker-auth",
skillVersionJob("securityScanJobs:default-legacy"),
skillVersionJob("securityScanJobs:clawscan-only"),
undefined,
);
@@ -338,13 +257,13 @@ JSON`,
hardFailed: false,
retryableFailed: false,
});
expect(await readFile(legacyMarker, "utf8")).toContain("codex");
await expect(readFile(clawscanMarker, "utf8")).rejects.toThrow();
expect(await readFile(clawscanMarker, "utf8")).toContain("called");
expect(client.action.mock.calls[0]?.[1]).toMatchObject({
llmAnalysis: { status: "clean", verdict: "benign" },
});
} finally {
if (previousCommand === undefined) delete process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND;
else process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND = previousCommand;
if (previousPath === undefined) delete process.env.PATH;
else process.env.PATH = previousPath;
}
});
@@ -397,14 +316,11 @@ JSON`,
const client = {
action: vi.fn(async (..._args: unknown[]) => ({})),
};
const result = await withFakeLegacySecondary(async () =>
processJob(
client,
"worker-auth",
skillVersionJob(`securityScanJobs:${verdict}`),
undefined,
"clawscan",
),
const result = await processJob(
client,
"worker-auth",
skillVersionJob(`securityScanJobs:${verdict}`),
undefined,
);
expect(result).toEqual({
@@ -540,23 +456,20 @@ JSON`,
const client = {
action: vi.fn(async (..._args: unknown[]) => ({})),
};
const result = await withFakeLegacySecondary(async () =>
processJob(
client,
"worker-auth",
claimedJob({
jobId: `securityScanJobs:${targetKind}-${source}`,
source,
target: await target(),
targetKind,
vtAnalysis: {
status: "completed",
source: `${targetKind}-${source}`,
},
}),
undefined,
"clawscan",
),
const result = await processJob(
client,
"worker-auth",
claimedJob({
jobId: `securityScanJobs:${targetKind}-${source}`,
source,
target: await target(),
targetKind,
vtAnalysis: {
status: "completed",
source: `${targetKind}-${source}`,
},
}),
undefined,
);
expect(result).toEqual({
@@ -634,19 +547,16 @@ JSON`,
const client = {
action: vi.fn(async (..._args: unknown[]) => ({})),
};
const result = await withFakeLegacySecondary(async () =>
processJob(
client,
"worker-auth",
claimedJob({
jobId: "securityScanJobs:no-cached-vt",
source: "manual",
target: fileTarget("SKILL.md", "# Uploaded skill\n"),
targetKind: "skillScanRequest",
}),
undefined,
"clawscan",
),
const result = await processJob(
client,
"worker-auth",
claimedJob({
jobId: "securityScanJobs:no-cached-vt",
source: "manual",
target: fileTarget("SKILL.md", "# Uploaded skill\n"),
targetKind: "skillScanRequest",
}),
undefined,
);
expect(result).toEqual({
@@ -706,7 +616,6 @@ JSON`,
targetKind,
}),
undefined,
"clawscan",
);
expect(result).toEqual({
@@ -765,7 +674,6 @@ JSON`,
"worker-auth",
skillVersionJob("securityScanJobs:vt-skipped"),
undefined,
"clawscan",
);
expect(result).toEqual({
@@ -820,7 +728,6 @@ echo "not json" > "$out"`,
"worker-auth",
skillVersionJob("securityScanJobs:malformed"),
undefined,
"clawscan",
onHealth,
);
@@ -878,7 +785,6 @@ JSON`,
"worker-auth",
skillVersionJob("securityScanJobs:judge-no-inspection"),
undefined,
"clawscan",
);
expect(result).toEqual({
@@ -956,7 +862,6 @@ JSON`,
"worker-auth",
skillVersionJob("securityScanJobs:judge-incomplete"),
undefined,
"clawscan",
onHealth,
);
@@ -1032,7 +937,6 @@ JSON`,
"worker-auth",
skillVersionJob(`securityScanJobs:completed-at-${name}`),
undefined,
"clawscan",
);
expect(result).toEqual({
@@ -1093,7 +997,6 @@ JSON`,
"worker-auth",
skillVersionJob("securityScanJobs:scanner-failed"),
undefined,
"clawscan",
onHealth,
);
@@ -1146,7 +1049,6 @@ echo "this should never complete"`,
"worker-auth",
skillVersionJob("securityScanJobs:timeout"),
undefined,
"clawscan",
onHealth,
);
@@ -1172,60 +1074,4 @@ echo "this should never complete"`,
else process.env.CODEX_SECURITY_SCAN_CLAWSCAN_TIMEOUT_MS = previousTimeout;
}
});
it("does not fall back to legacy Codex/SkillSpector commands when ClawScan fails", async () => {
const workspace = await tempDir();
const fakeClawScan = join(workspace, "fake-clawscan");
await writeFakeClawScanCommand(
fakeClawScan,
`echo "clawscan failed intentionally" >&2
exit 7`,
);
const binDir = await tempDir();
const markerPath = join(binDir, "legacy-commands-called.log");
await writeFakeClawScanCommand(
join(binDir, "codex"),
`echo codex >> ${JSON.stringify(markerPath)}
exit 0`,
);
await writeFakeClawScanCommand(
join(binDir, "skillspector"),
`echo skillspector >> ${JSON.stringify(markerPath)}
exit 0`,
);
const previousCommand = process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND;
const previousPath = process.env.PATH;
process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND = fakeClawScan;
process.env.PATH = `${binDir}:${previousPath ?? ""}`;
try {
const client = {
action: vi.fn(async (...args: unknown[]) => {
const payload = args[1] as { error?: string } | undefined;
return payload?.error ? { retry: false } : {};
}),
};
const result = await processJob(
client,
"worker-auth",
skillVersionJob("securityScanJobs:no-fallback"),
undefined,
"clawscan",
);
expect(result).toEqual({
completed: false,
hardFailed: true,
retryableFailed: false,
});
await expect(readFile(markerPath, "utf8")).rejects.toThrow();
} finally {
if (previousCommand === undefined) delete process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND;
else process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND = previousCommand;
if (previousPath === undefined) delete process.env.PATH;
else process.env.PATH = previousPath;
}
});
});
scripts/security/run-codex-scan-worker-modes.test.ts
-470
View File
@@ -1,470 +0,0 @@
/* @vitest-environment node */
import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
type ClaimedJob,
processJob,
resolveSecurityScanMode,
type SecurityScanMode,
} from "./run-codex-scan-worker";
const tempDirs: string[] = [];
const jobFixture = "lease-fixture";
afterEach(async () => {
vi.restoreAllMocks();
await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { force: true, recursive: true })));
});
async function tempDir() {
const dir = await mkdtemp(join(tmpdir(), "clawhub-scan-mode-test-"));
tempDirs.push(dir);
return dir;
}
async function writeCommand(path: string, body: string) {
await writeFile(path, `#!/usr/bin/env bash\nset -euo pipefail\n${body}\n`);
await chmod(path, 0o755);
}
function claimedJob(id: string): ClaimedJob {
return {
job: {
_id: `securityScanJobs:${id}`,
hasMaliciousSignal: false,
leaseToken: jobFixture,
source: "publish",
targetKind: "skillVersion",
waitForVtUntil: 0,
},
target: {
files: [
{
path: "SKILL.md",
sha256: "artifact-sha",
size: 8,
url: "data:text/plain,%23%20Skill",
},
],
},
};
}
function completeClawScanArtifact(verdict: "benign" | "suspicious" | "malicious") {
return JSON.stringify({
schemaVersion: "clawscan-run-v1",
profile: "clawhub",
completedAt: "2026-07-15T00:00:00Z",
scanners: {
"clawscan-static": {
status: "completed",
raw: { status: "clean" },
},
skillspector: {
status: "completed",
raw: {
status: "clean",
issue_count: 0,
issues: [],
},
},
virustotal: {
status: "completed",
raw: { status: "clean" },
},
},
judge: {
status: "completed",
promptSha256: "prompt-sha",
outputSchemaSha256: "schema-sha",
result: {
verdict,
confidence: "high",
summary: "ClawScan result",
dimensions: {
purpose_capability: { status: "ok", detail: "ok" },
instruction_scope: { status: "ok", detail: "ok" },
install_mechanism: { status: "ok", detail: "ok" },
environment_proportionality: { status: "ok", detail: "ok" },
persistence_privilege: { status: "ok", detail: "ok" },
},
scan_findings_in_context: [],
user_guidance: "guidance",
artifact_inspection: {
status: "completed",
challenge: "inspection-challenge",
required_file_sha256: "a".repeat(64),
files_inspected: ["artifact/SKILL.md"],
},
},
},
});
}
async function setupCommands(options?: {
clawscanFailure?: string;
clawscanVerdict?: "benign" | "suspicious" | "malicious";
legacyFailure?: string;
legacyVerdict?: "benign" | "suspicious" | "malicious";
}) {
const root = await tempDir();
const binDir = join(root, "bin");
const marker = join(root, "invocations.log");
await mkdir(binDir, { recursive: true });
await writeCommand(
join(binDir, "skillspector"),
`echo "legacy-skillspector:$PWD" >> ${JSON.stringify(marker)}
out=""
while [[ $# -gt 0 ]]; do
case "$1" in
--output)
out="$2"
shift 2
;;
*)
shift
;;
esac
done
mkdir -p "$(dirname "$out")"
cat > "$out" <<'JSON'
{"status":"clean","issue_count":0,"issues":[]}
JSON`,
);
await writeCommand(
join(binDir, "codex"),
`echo "legacy-codex:$PWD" >> ${JSON.stringify(marker)}
${options?.legacyFailure ? `echo ${JSON.stringify(options.legacyFailure)} >&2\nexit 19` : ""}
out=""
while [[ $# -gt 0 ]]; do
case "$1" in
--output-last-message)
out="$2"
shift 2
;;
*)
shift
;;
esac
done
mkdir -p "$(dirname "$out")"
cat > "$out" <<'JSON'
${JSON.stringify({
verdict: options?.legacyVerdict ?? "suspicious",
confidence: "medium",
summary: "Legacy result",
dimensions: {
purpose_capability: { status: "ok", detail: "ok" },
},
scan_findings_in_context: [],
user_guidance: "guidance",
})}
JSON`,
);
const clawscan = join(root, "clawscan");
await writeCommand(
clawscan,
`echo "clawscan:$PWD:$1" >> ${JSON.stringify(marker)}
${options?.clawscanFailure ? `echo ${JSON.stringify(options.clawscanFailure)} >&2\nexit 17` : ""}
out=""
while [[ $# -gt 0 ]]; do
case "$1" in
--output)
out="$2"
shift 2
;;
*)
shift
;;
esac
done
mkdir -p "$(dirname "$out")"
cat > "$out" <<'JSON'
${completeClawScanArtifact(options?.clawscanVerdict ?? "malicious")}
JSON`,
);
return { binDir, clawscan, marker, root };
}
async function withCommands<T>(
commands: Awaited<ReturnType<typeof setupCommands>>,
run: () => Promise<T>,
) {
const previousCommand = process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND;
const previousPath = process.env.PATH;
process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND = commands.clawscan;
process.env.PATH = `${commands.binDir}:${previousPath ?? ""}`;
try {
return await run();
} finally {
if (previousCommand === undefined) delete process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND;
else process.env.CODEX_SECURITY_SCAN_CLAWSCAN_COMMAND = previousCommand;
if (previousPath === undefined) delete process.env.PATH;
else process.env.PATH = previousPath;
}
}
function completionPayload(client: { action: ReturnType<typeof vi.fn> }) {
return client.action.mock.calls.find((call) => {
const payload = call[1] as { llmAnalysis?: unknown } | undefined;
return payload?.llmAnalysis !== undefined;
})?.[1] as
| {
llmAnalysis?: { status?: string; verdict?: string };
skillSpectorAnalysis?: { status?: string };
}
| undefined;
}
async function invocationLines(marker: string) {
try {
return (await readFile(marker, "utf8")).trim().split("\n").filter(Boolean);
} catch {
return [];
}
}
describe("security scan rollout modes", () => {
it.each([
{
mode: "legacy" as const,
expectedClawScan: 0,
expectedLegacyCodex: 1,
expectedLegacySkillSpector: 1,
expectedStatus: "suspicious",
expectedVerdict: "suspicious",
},
{
mode: "shadow" as const,
expectedClawScan: 1,
expectedLegacyCodex: 1,
expectedLegacySkillSpector: 1,
expectedStatus: "suspicious",
expectedVerdict: "suspicious",
},
{
mode: "clawscan" as const,
expectedClawScan: 1,
expectedLegacyCodex: 1,
expectedLegacySkillSpector: 1,
expectedStatus: "malicious",
expectedVerdict: "malicious",
},
])(
"$mode invokes the expected implementations and persists only its authoritative result",
async ({
mode,
expectedClawScan,
expectedLegacyCodex,
expectedLegacySkillSpector,
expectedStatus,
expectedVerdict,
}) => {
const commands = await setupCommands();
const diagnosticsRoot = await tempDir();
const client = {
action: vi.fn(async (..._args: unknown[]) => ({})),
};
const result = await withCommands(commands, () =>
processJob(client, "worker-token", claimedJob(mode), diagnosticsRoot, mode),
);
expect(result).toEqual({
completed: true,
hardFailed: false,
retryableFailed: false,
});
expect(client.action).toHaveBeenCalledTimes(1);
expect(completionPayload(client)).toMatchObject({
llmAnalysis: {
status: expectedStatus,
verdict: expectedVerdict,
},
});
const lines = await invocationLines(commands.marker);
expect(lines.filter((line) => line.startsWith("clawscan:"))).toHaveLength(expectedClawScan);
expect(lines.filter((line) => line.startsWith("legacy-codex:"))).toHaveLength(
expectedLegacyCodex,
);
expect(lines.filter((line) => line.startsWith("legacy-skillspector:"))).toHaveLength(
expectedLegacySkillSpector,
);
const workspaces = new Set(
lines.map((line) => {
const [, workspace] = line.split(":");
return workspace;
}),
);
expect([...workspaces]).toHaveLength(1);
const comparisonPath = join(
diagnosticsRoot,
`securityScanJobs_${mode}`,
"scan-comparison.json",
);
if (mode === "legacy") {
await expect(readFile(comparisonPath, "utf8")).rejects.toThrow();
} else {
const comparison = JSON.parse(await readFile(comparisonPath, "utf8"));
expect(comparison).toMatchObject({
authoritative: {
implementation: mode === "shadow" ? "legacy" : "clawscan",
verdict: expectedVerdict,
},
secondary: {
implementation: mode === "shadow" ? "clawscan" : "legacy",
},
status: "completed",
});
}
},
);
it.each([
{
mode: "shadow" as const,
options: { clawscanFailure: "diagnostic ClawScan failure" },
expectedVerdict: "suspicious",
},
{
mode: "clawscan" as const,
options: { legacyFailure: "diagnostic legacy failure" },
expectedVerdict: "malicious",
},
])(
"$mode ignores secondary failures after authoritative completion",
async ({ mode, options, expectedVerdict }) => {
const commands = await setupCommands(options);
const diagnosticsRoot = await tempDir();
const client = {
action: vi.fn(async (..._args: unknown[]) => ({})),
};
const onHealth = vi.fn();
const result = await withCommands(commands, () =>
processJob(
client,
"worker-token",
claimedJob(`${mode}-secondary-failed`),
diagnosticsRoot,
mode,
onHealth,
),
);
expect(result).toEqual({
completed: true,
hardFailed: false,
retryableFailed: false,
});
expect(client.action).toHaveBeenCalledTimes(1);
expect(completionPayload(client)?.llmAnalysis?.verdict).toBe(expectedVerdict);
const comparison = JSON.parse(
await readFile(
join(
diagnosticsRoot,
`securityScanJobs_${mode}-secondary-failed`,
"scan-comparison.json",
),
"utf8",
),
);
expect(comparison).toMatchObject({
status: "failed",
error: expect.stringContaining(mode === "shadow" ? "exited 17" : "exited 19"),
});
expect(onHealth).toHaveBeenCalledWith(
expect.objectContaining({
completed: true,
comparison: expect.objectContaining({
secondaryFailureStage: mode === "shadow" ? "unclassified" : "judge",
secondaryStatus: "failed",
}),
}),
);
},
);
it("fails authoritative ClawScan through the retry lifecycle without invoking legacy", async () => {
const commands = await setupCommands({ clawscanFailure: "authoritative failure" });
const client = {
action: vi.fn(async (...args: unknown[]) => {
const payload = args[1] as { error?: string } | undefined;
return payload?.error ? { retry: true } : {};
}),
};
const result = await withCommands(commands, () =>
processJob(
client,
"worker-token",
claimedJob("clawscan-authority-failed"),
undefined,
"clawscan",
),
);
expect(result).toEqual({
completed: false,
hardFailed: false,
retryableFailed: true,
});
expect(client.action).toHaveBeenCalledTimes(1);
expect(client.action.mock.calls[0]?.[1]).toMatchObject({
error: expect.stringContaining("exited 17"),
});
const lines = await invocationLines(commands.marker);
expect(lines.filter((line) => line.startsWith("clawscan:"))).toHaveLength(1);
expect(lines.some((line) => line.startsWith("legacy-"))).toBe(false);
});
it("rolls the whole route back by changing the mode to legacy", async () => {
const commands = await setupCommands();
const client = {
action: vi.fn(async (..._args: unknown[]) => ({})),
};
await withCommands(commands, async () => {
await processJob(
client,
"worker-token",
claimedJob("before-rollback"),
undefined,
"clawscan",
);
await processJob(client, "worker-token", claimedJob("after-rollback"), undefined, "legacy");
});
const persistedVerdicts = client.action.mock.calls.map(
(call) => (call[1] as { llmAnalysis?: { verdict?: string } }).llmAnalysis?.verdict,
);
expect(persistedVerdicts).toEqual(["malicious", "suspicious"]);
const lines = await invocationLines(commands.marker);
expect(lines.filter((line) => line.startsWith("clawscan:"))).toHaveLength(1);
expect(lines.filter((line) => line.startsWith("legacy-codex:"))).toHaveLength(2);
});
it("defaults safely to legacy and accepts only the three rollout values", () => {
expect(resolveSecurityScanMode(undefined)).toBe("legacy");
expect(resolveSecurityScanMode("")).toBe("legacy");
expect(["legacy", "shadow", "clawscan"].map((mode) => resolveSecurityScanMode(mode))).toEqual([
"legacy",
"shadow",
"clawscan",
] satisfies SecurityScanMode[]);
for (const invalid of ["codex", "ClawScan", " shadow ", "0", "true"]) {
expect(() => resolveSecurityScanMode(invalid)).toThrow(
`CODEX_SECURITY_SCAN_MODE must be one of legacy, shadow, or clawscan; received ${JSON.stringify(invalid)}`,
);
}
});
});
scripts/security/run-codex-scan-worker.test.ts
+5 -682
View File
@@ -1,5 +1,5 @@
/* @vitest-environment node */
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
@@ -10,14 +10,10 @@ import {
resolveCodexWorkerHome,
} from "../codex-worker-guard";
import {
buildPrompt,
normalizeSkillSpectorAnalysis,
publishWorkerHealthSummary,
processJob,
resolveSkillSpectorScanInput,
resolveSkillSpectorScanInputs,
runContinuouslyRefilledWorkerPool,
scanHealthClassification,
writeArtifactWorkspace,
writeJobDiagnostic,
} from "./run-codex-scan-worker";
@@ -35,22 +31,6 @@ async function tempDir() {
return dir;
}
async function readAllFilesText(dir: string) {
const texts: string[] = [];
async function visit(current: string) {
for (const entry of await readdir(current, { withFileTypes: true })) {
const path = join(current, entry.name);
if (entry.isDirectory()) {
await visit(path);
} else if (entry.isFile()) {
texts.push(await readFile(path, "utf8"));
}
}
}
await visit(dir);
return texts.join("\n");
}
function unsafeFixtureLabels() {
return {
label: ["API", "key"].join(" "),
@@ -61,189 +41,6 @@ function unsafeFixtureLabels() {
}
describe("run-codex-scan-worker diagnostics", () => {
describe("legacy SkillSpector health classification", () => {
const baseInput = {
clawscan: {},
codex: {},
implementation: "legacy" as const,
skillSpector: {
args: ["scan", "./artifact", "--format", "json"],
exitCode: 1,
},
status: "completed" as const,
};
it("does not treat exit 1 with a valid suspicious report as a scanner failure", () => {
expect(
scanHealthClassification({
...baseInput,
skillSpectorAnalysis: {
status: "suspicious",
score: 93,
issueCount: 1,
issues: [
{
issueId: "SDI-1",
severity: "HIGH",
explanation: "Detected suspicious behavior.",
},
],
checkedAt: 1,
},
}),
).toMatchObject({
scannerStageFailed: false,
timedOut: false,
});
});
it("recognizes a valid captured report when a later stage prevents returning the analysis", () => {
expect(
scanHealthClassification({
...baseInput,
skillSpector: {
...baseInput.skillSpector,
rawResult: JSON.stringify({
status: "suspicious",
issue_count: 1,
issues: [
{
id: "SDI-1",
severity: "HIGH",
explanation: "Detected suspicious behavior.",
},
],
}),
},
}),
).toMatchObject({
scannerStageFailed: false,
});
});
it.each(["error", "failed"] as const)(
"treats a parsed %s report as a scanner failure",
(status) => {
expect(
scanHealthClassification({
...baseInput,
skillSpectorAnalysis: {
status,
issueCount: 0,
issues: [],
checkedAt: 1,
},
}),
).toMatchObject({
scannerStageFailed: true,
});
},
);
it("treats other nonzero exits as failures even with a parseable report", () => {
expect(
scanHealthClassification({
...baseInput,
skillSpector: {
...baseInput.skillSpector,
exitCode: 2,
},
skillSpectorAnalysis: {
status: "suspicious",
issueCount: 1,
issues: [],
checkedAt: 1,
},
}),
).toMatchObject({
scannerStageFailed: true,
});
});
it("treats a positive issue count without parsed findings as a scanner failure", () => {
expect(
scanHealthClassification({
...baseInput,
skillSpectorAnalysis: {
status: "suspicious",
issueCount: 1,
issues: [],
checkedAt: 1,
},
}),
).toMatchObject({
scannerStageFailed: true,
});
});
it("treats exit 1 with a clean zero-findings report as a scanner failure", () => {
expect(
scanHealthClassification({
...baseInput,
skillSpectorAnalysis: {
status: "clean",
issueCount: 0,
issues: [],
checkedAt: 1,
},
}),
).toMatchObject({
scannerStageFailed: true,
});
});
it("treats a missing process exit status as a scanner failure", () => {
expect(
scanHealthClassification({
...baseInput,
skillSpector: {
...baseInput.skillSpector,
exitCode: undefined,
rawResult: JSON.stringify({
status: "suspicious",
issue_count: 1,
issues: [],
}),
},
}),
).toMatchObject({
scannerStageFailed: true,
});
});
it.each([undefined, "{malformed"])(
"treats a nonzero exit with %s captured output as a scanner failure",
(rawResult) => {
expect(
scanHealthClassification({
...baseInput,
skillSpector: {
...baseInput.skillSpector,
rawResult,
},
}),
).toMatchObject({
scannerStageFailed: true,
});
},
);
it("treats a SkillSpector timeout as a scanner failure", () => {
expect(
scanHealthClassification({
...baseInput,
skillSpector: {
...baseInput.skillSpector,
timedOut: true,
},
}),
).toMatchObject({
scannerStageFailed: true,
timedOut: true,
});
});
});
it("publishes the worker report to the Actions summary and diagnostics artifact", async () => {
const diagnosticsRoot = await tempDir();
const stepSummaryPath = join(await tempDir(), "step-summary.md");
@@ -252,7 +49,7 @@ describe("run-codex-scan-worker diagnostics", () => {
process.env.GITHUB_STEP_SUMMARY = stepSummaryPath;
try {
await publishWorkerHealthSummary(diagnosticsRoot, {
authoritative: {
clawscan: {
averageDurationMs: 30_000,
completed: 1,
failed: 0,
@@ -269,7 +66,6 @@ describe("run-codex-scan-worker diagnostics", () => {
},
claimFailures: 0,
durationMs: 30_000,
mode: "clawscan",
queueHealth: {
snapshotAt: 1,
queueDepth: 2,
@@ -295,7 +91,7 @@ describe("run-codex-scan-worker diagnostics", () => {
await readFile(join(diagnosticsRoot, "worker-summary.json"), "utf8"),
);
expect(artifact).toMatchObject({
mode: "clawscan",
clawscan: { completed: 1 },
workerId: "fixture-worker",
queueHealth: { queueDepth: 2 },
});
@@ -330,7 +126,8 @@ describe("run-codex-scan-worker diagnostics", () => {
processClaimedJob,
});
await vi.waitFor(() => expect(started).toEqual(["slow", "fast", "next"]));
while (!started.includes("next")) await new Promise((resolve) => setTimeout(resolve, 0));
expect(started).toEqual(["slow", "fast", "next"]);
expect(releaseSlowJob).toBeTypeOf("function");
releaseSlowJob?.();
@@ -500,120 +297,6 @@ describe("run-codex-scan-worker diagnostics", () => {
).toBe("/tmp/custom-codex-home");
});
it("frames workspace inspection as discretionary Codex research", () => {
const prompt = buildPrompt(
{
job: {
_id: "job123",
hasMaliciousSignal: false,
leaseToken: "lease-secret",
source: "publish",
targetKind: "skillVersion",
waitForVtUntil: 0,
},
target: {},
},
[],
);
expect(prompt).toContain("Do your own security research");
expect(prompt).toContain("Inspect workspace files when needed");
expect(prompt).toContain("SkillSpector findings are advisory research-preview evidence");
expect(prompt).toContain("not validated ground truth");
expect(prompt).toContain("artifact-backed evidence");
expect(prompt).toContain("totality of evidence");
expect(prompt).not.toContain("incomplete_artifact_inspection");
expect(prompt).not.toContain("Return the required JSON object only after those reads complete");
});
it("does not expose incomplete artifact inspection as an output-schema field", async () => {
const raw = await readFile("scripts/security/codex-scan-output.schema.json", "utf8");
const schema = JSON.parse(raw) as {
required?: string[];
properties?: Record<string, unknown>;
};
expect(schema.required).not.toContain("incomplete_artifact_inspection");
expect(schema.properties).not.toHaveProperty("incomplete_artifact_inspection");
});
it("passes SkillSpector findings to Codex without asking for OWASP finding output", () => {
const prompt = buildPrompt(
{
job: {
_id: "job123",
hasMaliciousSignal: false,
leaseToken: "lease-secret",
source: "publish",
targetKind: "skillVersion",
waitForVtUntil: 0,
},
target: {
version: {
skillSpectorAnalysis: {
status: "suspicious",
score: 55,
recommendation: "DO_NOT_INSTALL",
issueCount: 1,
checkedAt: 123,
issues: [
{
issueId: "SDI-1",
severity: "HIGH",
confidence: 0.98,
file: "SKILL.md",
startLine: 3,
endLine: 6,
explanation:
"The manifest advertises a generic benchmark while the skill body executes shell commands.",
remediation: "Make the manifest and skill body describe the same behavior.",
},
],
},
},
},
},
[],
);
expect(prompt).toContain("SkillSpector findings supplied to Codex");
expect(prompt).toContain("SDI-1");
expect(prompt).toContain("DO_NOT_INSTALL");
expect(prompt).not.toContain("agentic_risk_findings");
expect(prompt).not.toContain("OWASP");
});
it("does not reuse plugin-level SkillSpector findings when no bundled skills are declared", () => {
const prompt = buildPrompt(
{
job: {
_id: "plugin-job",
hasMaliciousSignal: false,
leaseToken: "lease-secret",
source: "publish",
targetKind: "packageRelease",
waitForVtUntil: 0,
},
target: {
release: {
skillSpectorAnalysis: {
status: "suspicious",
issueCount: 1,
checkedAt: 123,
issues: [{ issueId: "SDI-1", severity: "HIGH", explanation: "plugin root" }],
},
pluginManifestSummary: {
bundledSkills: [],
},
},
},
},
[],
);
expect(prompt).not.toContain("plugin root");
});
it("normalizes real SkillSpector JSON risk assessment fields", () => {
const analysis = normalizeSkillSpectorAnalysis(
JSON.stringify({
@@ -685,81 +368,6 @@ describe("run-codex-scan-worker diagnostics", () => {
expect(analysis.issues[0]?.codeSnippet?.length).toBeLessThan(longSnippet.length);
});
it("scans the extracted package root for ClawPack artifacts", async () => {
const workspace = await tempDir();
await mkdir(join(workspace, "artifact", "package"), { recursive: true });
await writeFile(join(workspace, "artifact.tgz"), "packed artifact");
await writeFile(join(workspace, "artifact", "package", "package.json"), "{}");
await writeFile(join(workspace, "artifact", "package.json"), "{}");
await expect(resolveSkillSpectorScanInput(workspace)).resolves.toBe("artifact/package");
});
it("scans the artifact root when there is no ClawPack extraction", async () => {
const workspace = await tempDir();
await mkdir(join(workspace, "artifact"), { recursive: true });
await writeFile(join(workspace, "artifact", "SKILL.md"), "# Skill");
await expect(resolveSkillSpectorScanInput(workspace)).resolves.toBe("artifact");
});
it("scans only bundled skill roots for plugin releases", async () => {
const workspace = await tempDir();
await mkdir(join(workspace, "artifact", "package"), { recursive: true });
await writeFile(join(workspace, "artifact.tgz"), "packed artifact");
await writeFile(join(workspace, "artifact", "package", "package.json"), "{}");
await expect(
resolveSkillSpectorScanInputs(workspace, {
job: {
_id: "package-job",
hasMaliciousSignal: false,
leaseToken: "lease-secret",
source: "publish",
targetKind: "packageRelease",
waitForVtUntil: 0,
},
target: {
release: {
pluginManifestSummary: {
bundledSkills: [
{ rootPath: "skills/first" },
{ rootPath: "./skills/second/" },
{ rootPath: "../package-code" },
],
},
},
},
}),
).resolves.toEqual(["artifact/package/skills/first", "artifact/package/skills/second"]);
});
it("skips SkillSpector for plugin releases without bundled skills", async () => {
const workspace = await tempDir();
await mkdir(join(workspace, "artifact"), { recursive: true });
await writeFile(join(workspace, "artifact", "openclaw.plugin.json"), "{}");
await expect(
resolveSkillSpectorScanInputs(workspace, {
job: {
_id: "plugin-job",
hasMaliciousSignal: false,
leaseToken: "lease-secret",
source: "publish",
targetKind: "packageRelease",
waitForVtUntil: 0,
},
target: {
release: {
pluginManifestSummary: {
bundledSkills: [],
},
},
},
}),
).resolves.toEqual([]);
});
it("writes scanner metadata without lease tokens or signed file URLs", async () => {
const workspace = await tempDir();
@@ -1116,241 +724,6 @@ describe("run-codex-scan-worker diagnostics", () => {
else process.env.GITHUB_ACTIONS = previousGitHubActions;
});
it("writes redacted Codex diagnostics without copying submitted artifact files or signed URLs", async () => {
const diagnosticsRoot = await tempDir();
const artifactWorkspace = await tempDir();
await mkdir(join(artifactWorkspace, "artifact"), { recursive: true });
await writeJobDiagnostic({
codex: {
args: ["exec", "--sandbox", "read-only"],
exitCode: 0,
rawResult:
'{"verdict":"benign","scan_findings_in_context":[{"ruleId":"x","expected_for_purpose":true,"note":"quoted artifact payload should not persist"}]}',
stderr: "workspace read failed https://signed.example.invalid/file?token=secret",
stdout:
'{"type":"error","message":"Codex CLI provider returned HTTP 429 for https://signed.example.invalid/file?token=secret with api_key=sk-short-fixture"}\n{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I could not inspect the artifact because the provider returned a transient error."}}\n{"type":"tool_call","status":"failed","source":"artifact controlled source string","api_key":"sk-short-fixture","output":"read https://signed.example.invalid/file?token=secret","content":["quoted array artifact payload should not persist"],"code-snippet":["hyphenated artifact payload should not persist"],"raw_result":["snake artifact payload should not persist"],"userImpact":["camel artifact payload should not persist"],"token":123456,"headers":{"authorization":["Bearer numeric-secret"]}}\n',
},
skillSpector: {
args: ["scan", "artifact", "--format", "json"],
exitCode: 0,
rawResult:
'{"issues":[{"id":"SDI-1","code_snippet":"quoted SkillSpector artifact payload should not persist","finding":"matched SkillSpector artifact payload should not persist","explanation":"safe to redact"}]}',
},
completedAt: 2000,
diagnosticsRoot,
error:
"Codex result did not match ClawScan schema: quoted artifact payload should not persist https://signed.example.invalid/file?token=secret",
job: {
job: {
_id: "job123",
hasMaliciousSignal: false,
leaseToken: "lease-secret",
source: "publish",
targetKind: "skillVersion",
waitForVtUntil: 0,
},
target: {
files: [
{
path: "artifacts/token=artifact-path-secret.md",
sha256: "abc123",
size: 42,
url: "https://signed.example.invalid/file?token=secret",
},
],
},
},
llmAnalysis: { confidence: "low", status: "clean", verdict: "benign" },
secondaryScan: {
authoritative: {
confidence: "low",
implementation: "legacy",
status: "clean",
verdict: "benign",
},
judgeStageFailed: false,
scannerStageFailed: false,
secondary: {
confidence: "high",
implementation: "clawscan",
status: "clean",
verdict: "benign",
},
status: "completed",
timedOut: false,
},
skillSpectorAnalysis: {
status: "suspicious",
issueCount: 1,
checkedAt: 123,
issues: [
{
issueId: "SDI-1",
severity: "HIGH",
explanation: "safe to redact",
finding: "matched SkillSpector artifact payload should not persist",
codeSnippet: "quoted SkillSpector artifact payload should not persist",
},
],
},
runId: "26127771775",
startedAt: 1000,
status: "failed",
});
const jobDir = join(diagnosticsRoot, "job123");
const stdoutText = await readFile(join(jobDir, "codex.stdout.redacted.jsonl"), "utf8");
expect(stdoutText).toContain('"tool_call"');
expect(stdoutText).not.toContain("Codex CLI provider returned HTTP 429");
expect(stdoutText).not.toContain(
"I could not inspect the artifact because the provider returned a transient error.",
);
expect(stdoutText).not.toContain("token=secret");
expect(stdoutText).not.toContain("signed.example.invalid");
expect(stdoutText).not.toContain("sk-short-fixture");
expect(stdoutText).not.toContain("123456");
expect(stdoutText).not.toContain("numeric-secret");
expect(stdoutText).not.toContain("quoted array artifact payload");
expect(stdoutText).not.toContain("hyphenated artifact payload");
expect(stdoutText).not.toContain("snake artifact payload");
expect(stdoutText).not.toContain("camel artifact payload");
expect(stdoutText).toContain('"api_key":"[redacted-secret]"');
expect(stdoutText).toContain('"token":"[redacted-secret]"');
expect(stdoutText).toContain('"authorization":"[redacted-secret]"');
expect(stdoutText).toContain('"source":"[redacted ');
expect(stdoutText).not.toContain("artifact controlled source");
expect(stdoutText).toContain('"content":"[redacted 1 item(s)]"');
expect(stdoutText).toContain('"code-snippet":"[redacted 1 item(s)]"');
expect(stdoutText).toContain('"raw_result":"[redacted 1 item(s)]"');
expect(stdoutText).toContain('"userImpact":"[redacted 1 item(s)]"');
await expect(readFile(join(jobDir, "codex.stderr.redacted.log"), "utf8")).resolves.toContain(
"workspace read failed",
);
const stderrText = await readFile(join(jobDir, "codex.stderr.redacted.log"), "utf8");
expect(stderrText).not.toContain("token=secret");
const resultText = await readFile(join(jobDir, "codex-result.redacted.json"), "utf8");
expect(resultText).toContain('"verdict"');
expect(resultText).toContain('"note": "[redacted');
expect(resultText).not.toContain("quoted artifact payload");
const skillSpectorResultText = await readFile(
join(jobDir, "skillspector-result.redacted.json"),
"utf8",
);
expect(skillSpectorResultText).toContain('"code_snippet": "[redacted');
expect(skillSpectorResultText).toContain('"finding": "[redacted');
expect(skillSpectorResultText).not.toContain("SkillSpector artifact payload");
const diagnostic = JSON.parse(await readFile(join(jobDir, "diagnostic.json"), "utf8"));
expect(diagnostic).toMatchObject({
job: {
id: "job123",
source: "publish",
targetKind: "skillVersion",
},
llmAnalysis: {
confidence: "low",
status: "clean",
verdict: "benign",
},
runId: "26127771775",
status: "failed",
});
expect(diagnostic.job.leaseToken).toBeUndefined();
expect(diagnostic.secondaryScan).toMatchObject({
authoritative: {
confidence: "low",
implementation: "legacy",
status: "clean",
verdict: "benign",
},
secondary: {
confidence: "high",
implementation: "clawscan",
status: "clean",
verdict: "benign",
},
status: "completed",
});
expect(diagnostic.error).toBe(
"Codex result did not match ClawScan schema: [redacted result body]",
);
expect(diagnostic.target.files).toEqual([
{ path: "[redacted-path]", sha256: "abc123", size: 42 },
]);
const diagnosticText = await readFile(join(jobDir, "diagnostic.json"), "utf8");
expect(diagnosticText).not.toContain("lease-secret");
expect(diagnosticText).not.toContain("artifact-path-secret");
expect(diagnosticText).not.toContain("token=secret");
expect(diagnosticText).not.toContain("quoted artifact payload");
expect(diagnosticText).not.toContain("SkillSpector artifact payload");
const allDiagnosticText = await readAllFilesText(jobDir);
expect(allDiagnosticText).not.toContain("lease-secret");
expect(allDiagnosticText).not.toContain("token=secret");
expect(allDiagnosticText).not.toContain("signed.example.invalid");
expect(allDiagnosticText).not.toContain("sk-short-fixture");
expect(allDiagnosticText).not.toContain("quoted artifact payload");
expect(allDiagnosticText).not.toContain("SkillSpector artifact payload");
const comparison = JSON.parse(await readFile(join(jobDir, "scan-comparison.json"), "utf8"));
expect(comparison).toMatchObject({
authoritative: { implementation: "legacy", status: "clean", verdict: "benign" },
secondary: { implementation: "clawscan", status: "clean", verdict: "benign" },
status: "completed",
});
expect(await readdir(jobDir)).not.toContain("artifact");
});
it("retains complete redacted legacy diagnostics beyond the former file and bundle caps", async () => {
const diagnosticsRoot = await tempDir();
const jsonl = Array.from({ length: 4_000 }, (_, index) =>
JSON.stringify({
item: { id: `item-${index}`, type: "agent_message" },
status: "completed",
type: "item.completed",
}),
).join("\n");
const stderr = `STDERR-BEGIN-${"a".repeat(70_000)}-STDERR-END`;
await writeJobDiagnostic({
codex: {
exitCode: 0,
stderr,
stdout: jsonl,
},
completedAt: 2,
diagnosticsRoot,
job: {
job: {
_id: "job-complete-legacy-diagnostics",
hasMaliciousSignal: false,
leaseToken: "fixture",
source: "publish",
targetKind: "skillVersion",
waitForVtUntil: 0,
},
target: {},
},
startedAt: 1,
status: "completed",
});
const jobDir = join(diagnosticsRoot, "job-complete-legacy-diagnostics");
const stdout = await readFile(join(jobDir, "codex.stdout.redacted.jsonl"), "utf8");
const retainedLines = stdout.trim().split("\n");
expect(Buffer.byteLength(stdout)).toBeGreaterThan(256 * 1_024);
expect(retainedLines).toHaveLength(4_000);
expect(retainedLines[0]).toContain("item-0");
expect(retainedLines.at(-1)).toContain("item-3999");
expect(stdout).not.toContain("...[truncated ");
const retainedStderr = await readFile(join(jobDir, "codex.stderr.redacted.log"), "utf8");
expect(Buffer.byteLength(retainedStderr)).toBeGreaterThan(64 * 1_024);
expect(retainedStderr).toContain("STDERR-BEGIN");
expect(retainedStderr).toContain("STDERR-END");
expect(retainedStderr).not.toContain("...[truncated ");
});
it("retains full ClawScan artifact and scanner outputs with secret-safe redaction", async () => {
const diagnosticsRoot = await tempDir();
const workspace = await tempDir();
@@ -1536,54 +909,4 @@ describe("run-codex-scan-worker diagnostics", () => {
},
]);
});
it("preserves sanitized secondary failure reasons in comparison artifacts", async () => {
const diagnosticsRoot = await tempDir();
const redactionFixture = "sk-short-fixture";
await writeJobDiagnostic({
completedAt: 2,
diagnosticsRoot,
job: {
job: {
_id: "job-shadow-failed",
hasMaliciousSignal: false,
leaseToken: "lease-secret",
source: "publish",
targetKind: "skillVersion",
waitForVtUntil: 0,
},
target: {},
},
secondaryScan: {
authoritative: {
implementation: "legacy",
status: "clean",
verdict: "benign",
},
error: `clawscan timed out with api_key=${redactionFixture}`,
failureStage: "unclassified",
judgeStageFailed: false,
scannerStageFailed: false,
secondary: {
implementation: "clawscan",
},
status: "failed",
timedOut: true,
},
startedAt: 1,
status: "completed",
});
const comparison = JSON.parse(
await readFile(join(diagnosticsRoot, "job-shadow-failed", "scan-comparison.json"), "utf8"),
);
expect(comparison).toMatchObject({
error: expect.stringContaining("clawscan timed out"),
status: "failed",
});
const comparisonText = JSON.stringify(comparison);
expect(comparisonText).not.toContain(redactionFixture);
});
});
scripts/security/run-codex-scan-worker.ts
+27 -750
View File
File diff suppressed because it is too large Load Diff
scripts/security/security-scan-worker-summary.test.ts
+8 -91
View File
@@ -8,35 +8,34 @@ import {
function outcome(overrides: Partial<SecurityScanJobHealth> = {}): SecurityScanJobHealth {
return {
authoritativeVerdict: "benign",
completed: true,
durationMs: 30_000,
judgeStageFailed: false,
scannerStageFailed: false,
timedOut: false,
verdict: "benign",
...overrides,
};
}
describe("security scan worker summary", () => {
it("calculates authoritative health, throughput, queue age, and verdict totals", () => {
it("calculates ClawScan health, throughput, queue age, and verdict totals", () => {
const summary = calculateSecurityScanWorkerHealthSummary({
durationMs: 120_000,
mode: "legacy",
outcomes: [
outcome(),
outcome({
authoritativeVerdict: "suspicious",
durationMs: 60_000,
scannerStageFailed: true,
verdict: "suspicious",
}),
outcome({
authoritativeVerdict: undefined,
completed: false,
durationMs: 90_000,
failureStage: "judge",
judgeStageFailed: true,
timedOut: true,
verdict: undefined,
}),
],
pool: {
@@ -59,7 +58,7 @@ describe("security scan worker summary", () => {
});
expect(summary).toMatchObject({
authoritative: {
clawscan: {
averageDurationMs: 60_000,
completed: 2,
failed: 1,
@@ -77,99 +76,17 @@ describe("security scan worker summary", () => {
throughputPerMinute: 1.5,
});
const markdown = renderSecurityScanWorkerSummaryMarkdown(summary);
expect(markdown).toContain("**Scanner:** `clawscan`");
expect(markdown).toContain("| Completed | 2 |");
expect(markdown).toContain("| Timed out | 1 |");
expect(markdown).toContain("- Queued: >=512");
expect(markdown).toContain("- Oldest ready job age: 15.0 min");
});
it("calculates verdict pairs, exact match rate, failures, and disagreement direction", () => {
const summary = calculateSecurityScanWorkerHealthSummary({
durationMs: 60_000,
mode: "clawscan",
outcomes: [
outcome({
authoritativeVerdict: "benign",
comparison: {
authoritativeVerdict: "benign",
secondaryJudgeStageFailed: false,
secondaryScannerStageFailed: false,
secondaryStatus: "completed",
secondaryTimedOut: false,
secondaryVerdict: "benign",
},
}),
outcome({
authoritativeVerdict: "malicious",
comparison: {
authoritativeVerdict: "malicious",
secondaryJudgeStageFailed: false,
secondaryScannerStageFailed: false,
secondaryStatus: "completed",
secondaryTimedOut: false,
secondaryVerdict: "suspicious",
},
}),
outcome({
authoritativeVerdict: "suspicious",
comparison: {
authoritativeVerdict: "suspicious",
secondaryJudgeStageFailed: false,
secondaryScannerStageFailed: false,
secondaryStatus: "completed",
secondaryTimedOut: false,
secondaryVerdict: "malicious",
},
}),
outcome({
authoritativeVerdict: "benign",
comparison: {
authoritativeVerdict: "benign",
secondaryFailureStage: "scanner",
secondaryJudgeStageFailed: false,
secondaryScannerStageFailed: true,
secondaryStatus: "failed",
secondaryTimedOut: true,
},
}),
],
pool: {
totalClaimed: 4,
totalClaimFailures: 0,
totalCompleted: 4,
totalFailed: 0,
totalRetryableFailed: 0,
},
workerId: "fixture-worker",
});
expect(summary.comparison).toEqual({
authoritativeMoreSevere: 1,
completedPairs: 3,
exactMatchRate: 33.33,
exactMatches: 1,
pairs: {
"benign -> benign": 1,
"malicious -> suspicious": 1,
"suspicious -> malicious": 1,
},
secondaryFailures: 1,
secondaryJudgeStageFailures: 0,
secondaryMoreSevere: 1,
secondaryScannerStageFailures: 1,
secondaryTimedOut: 1,
unknownDirection: 0,
});
const markdown = renderSecurityScanWorkerSummaryMarkdown(summary);
expect(markdown).toContain("Exact matches: 1 (33.33%)");
expect(markdown).toContain("| `malicious -> suspicious` | 1 |");
expect(markdown).toContain("Secondary scanner-stage failures: 1");
expect(markdown).not.toContain("secondary");
});
it("reports unavailable queue diagnostics without changing scan health", () => {
const summary = calculateSecurityScanWorkerHealthSummary({
durationMs: 60_000,
mode: "clawscan",
outcomes: [outcome()],
pool: {
totalClaimed: 1,
@@ -182,7 +99,7 @@ describe("security scan worker summary", () => {
workerId: "fixture-worker",
});
expect(summary.authoritative).toMatchObject({ completed: 1, failed: 0 });
expect(summary.clawscan).toMatchObject({ completed: 1, failed: 0 });
expect(renderSecurityScanWorkerSummaryMarkdown(summary)).toContain(
"- Unavailable: queue health request failed",
);
scripts/security/security-scan-worker-summary.ts
+19 -148
View File
@@ -1,5 +1,3 @@
export type SecurityScanMode = "legacy" | "shadow" | "clawscan";
export type SecurityScanQueueHealth = {
snapshotAt: number;
queueDepth: number;
@@ -10,25 +8,14 @@ export type SecurityScanQueueHealth = {
oldestReadyJobNextRunAt: number | null;
};
export type SecurityScanComparisonOutcome = {
authoritativeVerdict?: string;
secondaryFailureStage?: "scanner" | "judge" | "unclassified";
secondaryScannerStageFailed: boolean;
secondaryJudgeStageFailed: boolean;
secondaryStatus: "completed" | "failed";
secondaryTimedOut: boolean;
secondaryVerdict?: string;
};
export type SecurityScanJobHealth = {
authoritativeVerdict?: string;
comparison?: SecurityScanComparisonOutcome;
completed: boolean;
durationMs: number;
failureStage?: "scanner" | "judge" | "unclassified";
judgeStageFailed: boolean;
scannerStageFailed: boolean;
timedOut: boolean;
verdict?: string;
};
export type SecurityScanWorkerPoolStats = {
@@ -47,7 +34,7 @@ type VerdictTotals = {
};
export type SecurityScanWorkerHealthSummary = {
authoritative: {
clawscan: {
averageDurationMs: number;
completed: number;
failed: number;
@@ -58,21 +45,7 @@ export type SecurityScanWorkerHealthSummary = {
verdicts: VerdictTotals;
};
claimFailures: number;
comparison?: {
authoritativeMoreSevere: number;
completedPairs: number;
exactMatchRate: number | null;
exactMatches: number;
pairs: Record<string, number>;
secondaryFailures: number;
secondaryJudgeStageFailures: number;
secondaryMoreSevere: number;
secondaryScannerStageFailures: number;
secondaryTimedOut: number;
unknownDirection: number;
};
durationMs: number;
mode: SecurityScanMode;
queueHealth?: SecurityScanQueueHealth;
queueHealthError?: string;
throughputPerMinute: number;
@@ -94,17 +67,8 @@ function incrementVerdict(totals: VerdictTotals, verdict: string | undefined) {
}
}
function verdictSeverity(verdict: string | undefined) {
const normalized = normalizedVerdict(verdict);
if (normalized === "benign") return 0;
if (normalized === "suspicious") return 1;
if (normalized === "malicious") return 2;
return undefined;
}
export function calculateSecurityScanWorkerHealthSummary(input: {
durationMs: number;
mode: SecurityScanMode;
outcomes: SecurityScanJobHealth[];
pool: SecurityScanWorkerPoolStats;
queueHealth?: SecurityScanQueueHealth;
@@ -121,11 +85,11 @@ export function calculateSecurityScanWorkerHealthSummary(input: {
unknown: 0,
};
for (const outcome of input.outcomes) {
if (outcome.completed) incrementVerdict(verdicts, outcome.authoritativeVerdict);
if (outcome.completed) incrementVerdict(verdicts, outcome.verdict);
}
const summary: SecurityScanWorkerHealthSummary = {
authoritative: {
return {
clawscan: {
averageDurationMs:
input.outcomes.length > 0 ? Math.round(durationTotal / input.outcomes.length) : 0,
completed,
@@ -140,7 +104,6 @@ export function calculateSecurityScanWorkerHealthSummary(input: {
},
claimFailures: input.pool.totalClaimFailures,
durationMs: input.durationMs,
mode: input.mode,
queueHealth: input.queueHealth,
queueHealthError: input.queueHealthError,
throughputPerMinute:
@@ -148,66 +111,6 @@ export function calculateSecurityScanWorkerHealthSummary(input: {
totalClaimed: input.pool.totalClaimed,
workerId: input.workerId,
};
if (input.mode === "legacy") return summary;
const comparisons = input.outcomes
.map((outcome) => outcome.comparison)
.filter((comparison): comparison is SecurityScanComparisonOutcome => Boolean(comparison));
const pairs: Record<string, number> = {};
let exactMatches = 0;
let authoritativeMoreSevere = 0;
let secondaryMoreSevere = 0;
let unknownDirection = 0;
const completedPairs = comparisons.filter(
(comparison) =>
comparison.secondaryStatus === "completed" &&
Boolean(normalizedVerdict(comparison.authoritativeVerdict)) &&
Boolean(normalizedVerdict(comparison.secondaryVerdict)),
);
for (const comparison of completedPairs) {
const authoritative = normalizedVerdict(comparison.authoritativeVerdict) ?? "unknown";
const secondary = normalizedVerdict(comparison.secondaryVerdict) ?? "unknown";
const pair = `${authoritative} -> ${secondary}`;
pairs[pair] = (pairs[pair] ?? 0) + 1;
if (authoritative === secondary) {
exactMatches += 1;
continue;
}
const authoritativeSeverity = verdictSeverity(authoritative);
const secondarySeverity = verdictSeverity(secondary);
if (authoritativeSeverity === undefined || secondarySeverity === undefined) {
unknownDirection += 1;
} else if (authoritativeSeverity > secondarySeverity) {
authoritativeMoreSevere += 1;
} else {
secondaryMoreSevere += 1;
}
}
summary.comparison = {
authoritativeMoreSevere,
completedPairs: completedPairs.length,
exactMatchRate:
completedPairs.length > 0
? Math.round((exactMatches / completedPairs.length) * 10_000) / 100
: null,
exactMatches,
pairs,
secondaryFailures: comparisons.filter((comparison) => comparison.secondaryStatus === "failed")
.length,
secondaryJudgeStageFailures: comparisons.filter(
(comparison) => comparison.secondaryJudgeStageFailed,
).length,
secondaryMoreSevere,
secondaryScannerStageFailures: comparisons.filter(
(comparison) => comparison.secondaryScannerStageFailed,
).length,
secondaryTimedOut: comparisons.filter((comparison) => comparison.secondaryTimedOut).length,
unknownDirection,
};
return summary;
}
function formatDuration(durationMs: number) {
@@ -224,28 +127,28 @@ export function renderSecurityScanWorkerSummaryMarkdown(summary: SecurityScanWor
const lines = [
"## Security scan worker health",
"",
`**Mode:** \`${summary.mode}\` `,
`**Scanner:** \`clawscan\` `,
`**Worker:** \`${summary.workerId}\` `,
`**Run duration:** ${formatDuration(summary.durationMs)} `,
`**Throughput:** ${summary.throughputPerMinute.toFixed(2)} scans/min`,
"",
"| Authoritative scans | Count |",
"| ClawScan scans | Count |",
"| --- | ---: |",
`| Completed | ${summary.authoritative.completed} |`,
`| Failed | ${summary.authoritative.failed} |`,
`| Timed out | ${summary.authoritative.timedOut} |`,
`| Scanner-stage failures | ${summary.authoritative.scannerStageFailures} |`,
`| Judge-stage failures | ${summary.authoritative.judgeStageFailures} |`,
`| Unclassified failures | ${summary.authoritative.unclassifiedFailures} |`,
`| Average duration | ${formatDuration(summary.authoritative.averageDurationMs)} |`,
`| Completed | ${summary.clawscan.completed} |`,
`| Failed | ${summary.clawscan.failed} |`,
`| Timed out | ${summary.clawscan.timedOut} |`,
`| Scanner-stage failures | ${summary.clawscan.scannerStageFailures} |`,
`| Judge-stage failures | ${summary.clawscan.judgeStageFailures} |`,
`| Unclassified failures | ${summary.clawscan.unclassifiedFailures} |`,
`| Average duration | ${formatDuration(summary.clawscan.averageDurationMs)} |`,
`| Claim failures | ${summary.claimFailures} |`,
"",
"| Authoritative verdict | Count |",
"| ClawScan verdict | Count |",
"| --- | ---: |",
`| Benign | ${summary.authoritative.verdicts.benign} |`,
`| Suspicious | ${summary.authoritative.verdicts.suspicious} |`,
`| Malicious | ${summary.authoritative.verdicts.malicious} |`,
`| Unknown | ${summary.authoritative.verdicts.unknown} |`,
`| Benign | ${summary.clawscan.verdicts.benign} |`,
`| Suspicious | ${summary.clawscan.verdicts.suspicious} |`,
`| Malicious | ${summary.clawscan.verdicts.malicious} |`,
`| Unknown | ${summary.clawscan.verdicts.unknown} |`,
];
if (summary.queueHealth) {
@@ -261,37 +164,5 @@ export function renderSecurityScanWorkerSummaryMarkdown(summary: SecurityScanWor
lines.push("", "### Queue health", "", `- Unavailable: ${summary.queueHealthError}`);
}
if (summary.comparison) {
const rate =
summary.comparison.exactMatchRate === null
? "n/a"
: `${summary.comparison.exactMatchRate.toFixed(2)}%`;
lines.push(
"",
"### Authoritative vs secondary",
"",
`- Completed pairs: ${summary.comparison.completedPairs}`,
`- Exact matches: ${summary.comparison.exactMatches} (${rate})`,
`- Secondary failures: ${summary.comparison.secondaryFailures}`,
`- Secondary timeouts: ${summary.comparison.secondaryTimedOut}`,
`- Secondary scanner-stage failures: ${summary.comparison.secondaryScannerStageFailures}`,
`- Secondary judge-stage failures: ${summary.comparison.secondaryJudgeStageFailures}`,
`- Authoritative more severe: ${summary.comparison.authoritativeMoreSevere}`,
`- Secondary more severe: ${summary.comparison.secondaryMoreSevere}`,
`- Unknown disagreement direction: ${summary.comparison.unknownDirection}`,
"",
"| Verdict pair | Count |",
"| --- | ---: |",
);
const pairs = Object.entries(summary.comparison.pairs).sort(([left], [right]) =>
left.localeCompare(right),
);
if (pairs.length === 0) {
lines.push("| No completed pairs | 0 |");
} else {
for (const [pair, count] of pairs) lines.push(`| \`${pair}\` | ${count} |`);
}
}
return `${lines.join("\n")}\n`;
}
scripts/security/security-scan-worker-workflow.test.ts
+2 -6
View File
@@ -104,15 +104,11 @@ describe("security-scan-codex workflow", () => {
expect(jobEnv.CODEX_SECURITY_SCAN_MAX_RUNTIME_MINUTES).toBe(
"${{ github.event.client_payload.max_runtime_minutes || inputs['max-runtime-minutes'] || '12' }}",
);
expect(jobEnv.CODEX_SECURITY_SCAN_TIMEOUT_MS).toBe(
"${{ vars.CODEX_SECURITY_SCAN_TIMEOUT_MS || '240000' }}",
);
expect(jobEnv.CODEX_SECURITY_SCAN_MODE).toBe(
"${{ vars.CODEX_SECURITY_SCAN_MODE || 'legacy' }}",
);
expect(jobEnv.CODEX_SECURITY_SCAN_CLAWSCAN_TIMEOUT_MS).toBe(
"${{ vars.CODEX_SECURITY_SCAN_CLAWSCAN_TIMEOUT_MS || '240000' }}",
);
expect(jobEnv).not.toHaveProperty("CODEX_SECURITY_SCAN_MODE");
expect(jobEnv).not.toHaveProperty("CODEX_SECURITY_SCAN_TIMEOUT_MS");
expect(jobEnv).not.toHaveProperty("CODEX_SECURITY_SCAN_SHADOW_CLAWSCAN");
expect(jobEnv).not.toHaveProperty("OPENAI_API_KEY");
expect(jobEnv).not.toHaveProperty("CODEX_API_KEY");
specs/security-moderation.md
+11 -26
View File
@@ -284,38 +284,23 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
target kind and source through the same completion/failure contract. Skill
versions and scan requests use the isolated `artifact` root; extracted
ClawPack releases use `artifact/package`.
- The temporary whole-system rollout mode accepts exactly `legacy`, `shadow`,
and `clawscan`, with `legacy` as the safe default until the production
cutover is explicitly approved. `legacy` runs only the legacy implementation.
`shadow` persists legacy authoritatively, then runs ClawScan diagnostically
against the same isolated artifact. `clawscan` persists ClawScan
authoritatively, then runs legacy diagnostically against that same artifact
during the soak. Secondary execution cannot change stored verdicts,
publication or moderation behavior, retries, or authoritative job success.
Authoritative ClawScan failures use the existing failure/retry lifecycle and
never trigger per-job legacy fallback. Rollback is a manual whole-system mode
change to `legacy`.
- An authoritative ClawScan judge result is complete only when ClawScan verifies
- OSS ClawScan is the only security-scan implementation. Every claimed target
kind and source runs through the same ClawScan profile and completion/failure
contract. ClawScan failures use the existing failure/retry lifecycle; there
is no per-job fallback or alternate legacy route.
- A ClawScan judge result is complete only when ClawScan verifies
a workspace-only inspection challenge and the SHA-256 of a required artifact
file. Missing or mismatched inspection receipts fail the judge and use the
normal scan failure/retry lifecycle; a low-confidence verdict cannot replace
successful artifact inspection.
- Every worker run publishes a GitHub Actions summary and uploads a structured
summary with its secret-scanned diagnostics. The summary reports authoritative
summary with its secret-scanned diagnostics. The summary reports ClawScan
completions, failures, timeouts, scanner-stage and judge-stage failures,
duration, throughput, queue health, and verdict totals. `shadow` and
`clawscan` additionally report authoritative/secondary verdict pairs, exact
matches, secondary failures, and disagreement direction. Queue-health lookup
failures are diagnostic-only and must not change authoritative persistence,
retries, or the worker exit result. A valid parsed scanner report with
findings is a completed scanner stage when SkillSpector uses exit code `1`
with a `suspicious` or `malicious` report containing a positive normalized
issue count and at least one parsed finding. Other nonzero exits remain
failures. Scanner-stage failures also include a timeout, missing process exit
status, missing or unparseable report, or parsed `error`/`failed` status.
- Retained worker diagnostics preserve every redacted Codex/legacy output
record plus complete redacted ClawScan artifacts, per-scanner outputs, and
comparison records without per-file or aggregate-size truncation. Bounded
duration, throughput, queue health, and verdict totals. Queue-health lookup
failures are diagnostic-only and must not change persistence, retries, or the
worker exit result.
- Retained worker diagnostics preserve complete redacted ClawScan artifacts and
per-scanner outputs without per-file or aggregate-size truncation. Bounded
metadata/error fields may remain capped. Artifact upload still requires the
existing verified-secret scan to pass.
- Claimable queue work edge-triggers a coalesced GitHub Actions worker dispatch.