admin/clawhub
1
0
Fork 0
mirror of https://github.com/openclaw/clawhub.git synced 2026-08-14 00:47:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): bound security dataset page exports (#2900)

Browse Source
This commit is contained in:
Vincent Koc
2026-06-27 05:37:09 +08:00
committed by GitHub
parent 7f0350d96f
commit a3cc536f08
3 changed files with 144 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/security-dataset-snapshot.yml
+17
View File
@@ -31,6 +31,10 @@ on:
description: "Live export pages per Convex request"
required: true
default: "1"
page-timeout-ms:
description: "Maximum milliseconds for a single Convex export page request"
required: true
default: "600000"
concurrency:
description: "Concurrent live export shards"
required: true
@@ -69,6 +73,7 @@ jobs:
SNAPSHOT_PAGE_SIZE: ${{ inputs['page-size'] || '25' }}
SNAPSHOT_MIN_PAGE_SIZE: ${{ inputs['min-page-size'] || '1' }}
SNAPSHOT_BATCH_PAGES: ${{ inputs['batch-pages'] || '1' }}
SNAPSHOT_PAGE_TIMEOUT_MS: ${{ inputs['page-timeout-ms'] || '600000' }}
SNAPSHOT_CONCURRENCY: ${{ inputs.concurrency || '1' }}
SNAPSHOT_SHARDS: ${{ inputs.shards || '12' }}
SNAPSHOT_MAX_SHARDS_PER_SOURCE: ${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '16' }}
@@ -92,6 +97,7 @@ jobs:
echo "Shard count per source kind: $SNAPSHOT_SHARDS"
echo "Maximum shards per source kind: $SNAPSHOT_MAX_SHARDS_PER_SOURCE"
echo "Maximum matrix jobs: $SNAPSHOT_MAX_MATRIX_JOBS"
echo "Convex page timeout ms: $SNAPSHOT_PAGE_TIMEOUT_MS"
if [[ ! "$SNAPSHOT_SHARDS" =~ ^[1-9][0-9]*$ ]]; then
echo "::error::shards must be a positive integer"
exit 1
@@ -104,6 +110,14 @@ jobs:
echo "::error::SECURITY_DATASET_MAX_MATRIX_JOBS must be a positive integer"
exit 1
fi
if [[ ! "$SNAPSHOT_PAGE_TIMEOUT_MS" =~ ^[1-9][0-9]*$ ]]; then
echo "::error::page-timeout-ms must be a positive integer"
exit 1
fi
if (( SNAPSHOT_PAGE_TIMEOUT_MS > 2147483647 )); then
echo "::error::page-timeout-ms must be at most 2147483647"
exit 1
fi
bun -e '
const [shards, maxShards] = process.argv.slice(1);
if (BigInt(shards) > BigInt(maxShards)) {
@@ -127,6 +141,7 @@ jobs:
--page-size "$SNAPSHOT_PAGE_SIZE" \
--min-page-size "$SNAPSHOT_MIN_PAGE_SIZE" \
--batch-pages "$SNAPSHOT_BATCH_PAGES" \
--page-timeout-ms "$SNAPSHOT_PAGE_TIMEOUT_MS" \
--concurrency "$SNAPSHOT_CONCURRENCY" \
--shards "$SNAPSHOT_SHARDS" \
--write-shard-matrix "$matrix_path"
@@ -160,6 +175,7 @@ jobs:
SNAPSHOT_PAGE_SIZE: ${{ inputs['page-size'] || '25' }}
SNAPSHOT_MIN_PAGE_SIZE: ${{ inputs['min-page-size'] || '1' }}
SNAPSHOT_BATCH_PAGES: ${{ inputs['batch-pages'] || '1' }}
SNAPSHOT_PAGE_TIMEOUT_MS: ${{ inputs['page-timeout-ms'] || '600000' }}
SANITIZED_OUT_DIR: /tmp/clawhub-security-dataset/shard
steps:
- uses: actions/checkout@v7
@@ -184,6 +200,7 @@ jobs:
--page-size "$SNAPSHOT_PAGE_SIZE"
--min-page-size "$SNAPSHOT_MIN_PAGE_SIZE"
--batch-pages "$SNAPSHOT_BATCH_PAGES"
--page-timeout-ms "$SNAPSHOT_PAGE_TIMEOUT_MS"
--concurrency 1
--shards 1
)
scripts/security-dataset/export-snapshot.ts
+90 -1
View File
@@ -58,6 +58,11 @@ type CompressedConvexPage = {
payload: string;
};
type CommandOutputError = Error & {
stdout?: string;
stderr?: string;
};
type Options = {
deployment: string | null;
convexUrl: string | null;
@@ -72,6 +77,7 @@ type Options = {
batchPages: number;
concurrency: number;
shards: number;
pageTimeoutMs: number;
outDir: string;
sourceKind: SourceKind | "all";
timeWindow: CreatedTimeWindow;
@@ -121,6 +127,8 @@ const DEFAULT_BATCH_PAGES = 5;
const DEFAULT_CONCURRENCY = 6;
const DEFAULT_SHARDS = 12;
const DEFAULT_MAX_CONVEX_ATTEMPTS = 6;
const DEFAULT_CONVEX_PAGE_TIMEOUT_MS = 10 * 60 * 1000;
const MAX_TIMER_TIMEOUT_MS = 2_147_483_647;
const DEFAULT_OUT_DIR = ".data/security-dataset/snapshots";
const CONVEX_RUN_MAX_BUFFER_BYTES = 128 * 1024 * 1024;
const SOURCE_KINDS: SourceKind[] = ["skill", "package"];
@@ -273,12 +281,21 @@ async function exportShard(input: {
let cursor: string | null = null;
let pageSize = options.pageSize;
let batchPages = options.batchPages;
let pageNumber = 1;
while (!isLimitReached(options, state)) {
const startedAt = Date.now();
console.error(
`[snapshot] ${shard.label} page ${pageNumber} request page-size=${pageSize} batch-pages=${batchPages} cursor=${cursorSummary(cursor)} timeout-ms=${options.pageTimeoutMs}`,
);
const result = await runConvexPage(options, shard, cursor, pageSize, batchPages);
const elapsedMs = Date.now() - startedAt;
pageSize = result.pageSize;
batchPages = result.batchPages;
const page = result.page;
const inputs = reserveExportInputs(page.page, state, options.limit);
console.error(
`[snapshot] ${shard.label} page ${pageNumber} response artifacts=${page.page.length} reserved=${inputs.length} done=${page.isDone} next-cursor=${cursorSummary(page.continueCursor)} elapsed-ms=${elapsedMs}`,
);
if (inputs.length > 0) {
await processArtifactInputs({ inputs, state, writers });
console.error(
@@ -286,7 +303,13 @@ async function exportShard(input: {
);
}
if (page.isDone || inputs.length < page.page.length) return;
if (page.continueCursor === cursor) {
throw new Error(
`Convex pagination for ${shard.label} did not advance cursor ${cursorSummary(cursor)}.`,
);
}
cursor = page.continueCursor;
pageNumber += 1;
}
}
@@ -335,6 +358,10 @@ async function runConvexPage(
};
} catch (error) {
lastError = error;
if (isLocalConvexPageTimeout(error)) {
writeCommandErrorOutput(error);
throw error;
}
if (
isLikelyOversizedConvexBatch(error) &&
canReduceConvexBatch(options, pageSize, pageCount)
@@ -404,7 +431,11 @@ async function runWorkerAction<T>(
throw new Error("--convex-url or CONVEX_URL is required when using --worker-token.");
}
const client = new ConvexHttpClient(options.convexUrl);
const result = await client.action(resolveWorkerAction(functionName), args as never);
const result = await withTimeout(
client.action(resolveWorkerAction(functionName), args as never),
options.pageTimeoutMs,
`Convex action ${functionName} exceeded ${options.pageTimeoutMs}ms`,
);
if (validate(result)) return result;
throw new Error(`Invalid ${functionName} response.`);
}
@@ -431,6 +462,10 @@ async function runConvexJson<T>(
return await runConvexJsonOnce(options, functionName, args, validate);
} catch (error) {
lastError = error;
if (isLocalConvexPageTimeout(error)) {
writeCommandErrorOutput(error);
throw error;
}
if (attempt === DEFAULT_MAX_CONVEX_ATTEMPTS) break;
console.error(
`[snapshot] retrying ${functionName} after attempt ${attempt}: ${errorMessage(error)}`,
@@ -454,7 +489,14 @@ async function runConvexJsonOnce<T>(
cwd: process.cwd(),
encoding: "utf8",
env: convexRunEnv(),
timeout: options.pageTimeoutMs,
maxBuffer: CONVEX_RUN_MAX_BUFFER_BYTES,
}).catch((error: unknown) => {
if (!isExecFileTimeout(error)) throw error;
throw commandOutputError(
`Convex run ${functionName} exceeded ${options.pageTimeoutMs}ms`,
error,
);
});
try {
return parseConvexJsonMatching(result.stdout, validate);
@@ -803,6 +845,21 @@ function delay(ms: number) {
return new Promise<void>((done) => setTimeout(done, ms));
}
function withTimeout<T>(promise: Promise<T>, timeoutMs: number, message: string): Promise<T> {
let timeout: ReturnType<typeof setTimeout> | null = null;
const timeoutPromise = new Promise<never>((_resolve, reject) => {
timeout = setTimeout(() => reject(new Error(message)), timeoutMs);
});
return Promise.race([promise, timeoutPromise]).finally(() => {
if (timeout) clearTimeout(timeout);
});
}
function cursorSummary(cursor: string | null) {
if (cursor === null || cursor.length === 0) return "start";
return cursor.length <= 12 ? cursor : `${cursor.slice(0, 6)}...${cursor.slice(-4)}`;
}
function errorMessage(error: unknown) {
return error instanceof Error ? error.message : String(error);
}
@@ -817,6 +874,18 @@ function isLikelyConvexOperationTimeout(error: unknown) {
);
}
function isLocalConvexPageTimeout(error: unknown) {
return /Convex (?:action|run) .* exceeded \d+ms/.test(errorMessage(error));
}
function isExecFileTimeout(error: unknown) {
return (
isRecord(error) &&
error.killed === true &&
(error.signal === "SIGTERM" || error.signal === "SIGKILL")
);
}
function isLikelyOversizedConvexBatch(error: unknown) {
return isLikelyTruncatedConvexOutput(error) || isLikelyConvexOperationTimeout(error);
}
@@ -834,6 +903,15 @@ function writeCommandErrorOutput(error: unknown) {
}
}
function commandOutputError(message: string, cause: unknown) {
const error: CommandOutputError = new Error(message);
if (isRecord(cause)) {
if (typeof cause.stderr === "string") error.stderr = cause.stderr;
if (typeof cause.stdout === "string") error.stdout = cause.stdout;
}
return error;
}
function gitSha() {
const result = spawnSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" });
if (result.status !== 0) return "unknown";
@@ -855,6 +933,7 @@ function parseArgs(args: string[]): Options {
batchPages: DEFAULT_BATCH_PAGES,
concurrency: DEFAULT_CONCURRENCY,
shards: DEFAULT_SHARDS,
pageTimeoutMs: DEFAULT_CONVEX_PAGE_TIMEOUT_MS,
outDir: DEFAULT_OUT_DIR,
sourceKind: "all",
timeWindow: emptyCreatedTimeWindow(),
@@ -892,6 +971,8 @@ function parseArgs(args: string[]): Options {
options.concurrency = readPositiveInt(readValue(args, ++index, arg), arg);
} else if (arg === "--shards") {
options.shards = readPositiveInt(readValue(args, ++index, arg), arg);
} else if (arg === "--page-timeout-ms") {
options.pageTimeoutMs = readTimerTimeoutMs(readValue(args, ++index, arg), arg);
} else if (arg === "--out-dir") {
options.outDir = readValue(args, ++index, arg);
} else if (arg === "--source-kind") {
@@ -947,6 +1028,14 @@ function readPositiveInt(value: string, flag: string) {
return parsed;
}
function readTimerTimeoutMs(value: string, flag: string) {
const parsed = readPositiveInt(value, flag);
if (parsed > MAX_TIMER_TIMEOUT_MS) {
throw new Error(`Expected ${flag} to be at most ${MAX_TIMER_TIMEOUT_MS}.`);
}
return parsed;
}
function readSourceKind(value: string): SourceKind | "all" {
if (value === "all" || value === "skill" || value === "package") return value;
throw new Error(`Unsupported source kind: ${value}`);
scripts/security-dataset/exportSnapshotCli.test.ts
+37
View File
@@ -111,6 +111,43 @@ describe("security dataset snapshot CLI", () => {
stderr: expect.stringContaining("--min-page-size must be less than or equal to --page-size."),
});
});
it("rejects a non-positive page timeout", async () => {
await expect(
execFileAsync(
"bun",
["scripts/security-dataset/export-snapshot.ts", "--page-timeout-ms", "0", "--dry-run"],
{
cwd: process.cwd(),
encoding: "utf8",
maxBuffer: 16 * 1024 * 1024,
},
),
).rejects.toMatchObject({
stderr: expect.stringContaining("Expected positive integer for --page-timeout-ms"),
});
});
it("rejects a page timeout above the runtime timer maximum", async () => {
await expect(
execFileAsync(
"bun",
[
"scripts/security-dataset/export-snapshot.ts",
"--page-timeout-ms",
"2147483648",
"--dry-run",
],
{
cwd: process.cwd(),
encoding: "utf8",
maxBuffer: 16 * 1024 * 1024,
},
),
).rejects.toMatchObject({
stderr: expect.stringContaining("Expected --page-timeout-ms to be at most 2147483647."),
});
});
});
function buildTinyConvexSnapshotZip() {