From a3cc536f080d75106893f269f84f63d9ea21b85c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 27 Jun 2026 05:37:09 +0800 Subject: [PATCH] fix(ci): bound security dataset page exports (#2900) --- .../workflows/security-dataset-snapshot.yml | 17 ++++ scripts/security-dataset/export-snapshot.ts | 91 ++++++++++++++++++- .../exportSnapshotCli.test.ts | 37 ++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/.github/workflows/security-dataset-snapshot.yml b/.github/workflows/security-dataset-snapshot.yml index a8014c67..345926ed 100644 --- a/.github/workflows/security-dataset-snapshot.yml +++ b/.github/workflows/security-dataset-snapshot.yml @@ -31,6 +31,10 @@ on: description: "Live export pages per Convex request" required: true default: "1" + page-timeout-ms: + description: "Maximum milliseconds for a single Convex export page request" + required: true + default: "600000" concurrency: description: "Concurrent live export shards" required: true @@ -69,6 +73,7 @@ jobs: SNAPSHOT_PAGE_SIZE: ${{ inputs['page-size'] || '25' }} SNAPSHOT_MIN_PAGE_SIZE: ${{ inputs['min-page-size'] || '1' }} SNAPSHOT_BATCH_PAGES: ${{ inputs['batch-pages'] || '1' }} + SNAPSHOT_PAGE_TIMEOUT_MS: ${{ inputs['page-timeout-ms'] || '600000' }} SNAPSHOT_CONCURRENCY: ${{ inputs.concurrency || '1' }} SNAPSHOT_SHARDS: ${{ inputs.shards || '12' }} SNAPSHOT_MAX_SHARDS_PER_SOURCE: ${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '16' }} @@ -92,6 +97,7 @@ jobs: echo "Shard count per source kind: $SNAPSHOT_SHARDS" echo "Maximum shards per source kind: $SNAPSHOT_MAX_SHARDS_PER_SOURCE" echo "Maximum matrix jobs: $SNAPSHOT_MAX_MATRIX_JOBS" + echo "Convex page timeout ms: $SNAPSHOT_PAGE_TIMEOUT_MS" if [[ ! "$SNAPSHOT_SHARDS" =~ ^[1-9][0-9]*$ ]]; then echo "::error::shards must be a positive integer" exit 1 @@ -104,6 +110,14 @@ jobs: echo "::error::SECURITY_DATASET_MAX_MATRIX_JOBS must be a positive integer" exit 1 fi + if [[ ! "$SNAPSHOT_PAGE_TIMEOUT_MS" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::page-timeout-ms must be a positive integer" + exit 1 + fi + if (( SNAPSHOT_PAGE_TIMEOUT_MS > 2147483647 )); then + echo "::error::page-timeout-ms must be at most 2147483647" + exit 1 + fi bun -e ' const [shards, maxShards] = process.argv.slice(1); if (BigInt(shards) > BigInt(maxShards)) { @@ -127,6 +141,7 @@ jobs: --page-size "$SNAPSHOT_PAGE_SIZE" \ --min-page-size "$SNAPSHOT_MIN_PAGE_SIZE" \ --batch-pages "$SNAPSHOT_BATCH_PAGES" \ + --page-timeout-ms "$SNAPSHOT_PAGE_TIMEOUT_MS" \ --concurrency "$SNAPSHOT_CONCURRENCY" \ --shards "$SNAPSHOT_SHARDS" \ --write-shard-matrix "$matrix_path" @@ -160,6 +175,7 @@ jobs: SNAPSHOT_PAGE_SIZE: ${{ inputs['page-size'] || '25' }} SNAPSHOT_MIN_PAGE_SIZE: ${{ inputs['min-page-size'] || '1' }} SNAPSHOT_BATCH_PAGES: ${{ inputs['batch-pages'] || '1' }} + SNAPSHOT_PAGE_TIMEOUT_MS: ${{ inputs['page-timeout-ms'] || '600000' }} SANITIZED_OUT_DIR: /tmp/clawhub-security-dataset/shard steps: - uses: actions/checkout@v7 @@ -184,6 +200,7 @@ jobs: --page-size "$SNAPSHOT_PAGE_SIZE" --min-page-size "$SNAPSHOT_MIN_PAGE_SIZE" --batch-pages "$SNAPSHOT_BATCH_PAGES" + --page-timeout-ms "$SNAPSHOT_PAGE_TIMEOUT_MS" --concurrency 1 --shards 1 ) diff --git a/scripts/security-dataset/export-snapshot.ts b/scripts/security-dataset/export-snapshot.ts index 43aa1442..0bd42c88 100644 --- a/scripts/security-dataset/export-snapshot.ts +++ b/scripts/security-dataset/export-snapshot.ts @@ -58,6 +58,11 @@ type CompressedConvexPage = { payload: string; }; +type CommandOutputError = Error & { + stdout?: string; + stderr?: string; +}; + type Options = { deployment: string | null; convexUrl: string | null; @@ -72,6 +77,7 @@ type Options = { batchPages: number; concurrency: number; shards: number; + pageTimeoutMs: number; outDir: string; sourceKind: SourceKind | "all"; timeWindow: CreatedTimeWindow; @@ -121,6 +127,8 @@ const DEFAULT_BATCH_PAGES = 5; const DEFAULT_CONCURRENCY = 6; const DEFAULT_SHARDS = 12; const DEFAULT_MAX_CONVEX_ATTEMPTS = 6; +const DEFAULT_CONVEX_PAGE_TIMEOUT_MS = 10 * 60 * 1000; +const MAX_TIMER_TIMEOUT_MS = 2_147_483_647; const DEFAULT_OUT_DIR = ".data/security-dataset/snapshots"; const CONVEX_RUN_MAX_BUFFER_BYTES = 128 * 1024 * 1024; const SOURCE_KINDS: SourceKind[] = ["skill", "package"]; @@ -273,12 +281,21 @@ async function exportShard(input: { let cursor: string | null = null; let pageSize = options.pageSize; let batchPages = options.batchPages; + let pageNumber = 1; while (!isLimitReached(options, state)) { + const startedAt = Date.now(); + console.error( + `[snapshot] ${shard.label} page ${pageNumber} request page-size=${pageSize} batch-pages=${batchPages} cursor=${cursorSummary(cursor)} timeout-ms=${options.pageTimeoutMs}`, + ); const result = await runConvexPage(options, shard, cursor, pageSize, batchPages); + const elapsedMs = Date.now() - startedAt; pageSize = result.pageSize; batchPages = result.batchPages; const page = result.page; const inputs = reserveExportInputs(page.page, state, options.limit); + console.error( + `[snapshot] ${shard.label} page ${pageNumber} response artifacts=${page.page.length} reserved=${inputs.length} done=${page.isDone} next-cursor=${cursorSummary(page.continueCursor)} elapsed-ms=${elapsedMs}`, + ); if (inputs.length > 0) { await processArtifactInputs({ inputs, state, writers }); console.error( @@ -286,7 +303,13 @@ async function exportShard(input: { ); } if (page.isDone || inputs.length < page.page.length) return; + if (page.continueCursor === cursor) { + throw new Error( + `Convex pagination for ${shard.label} did not advance cursor ${cursorSummary(cursor)}.`, + ); + } cursor = page.continueCursor; + pageNumber += 1; } } @@ -335,6 +358,10 @@ async function runConvexPage( }; } catch (error) { lastError = error; + if (isLocalConvexPageTimeout(error)) { + writeCommandErrorOutput(error); + throw error; + } if ( isLikelyOversizedConvexBatch(error) && canReduceConvexBatch(options, pageSize, pageCount) @@ -404,7 +431,11 @@ async function runWorkerAction( throw new Error("--convex-url or CONVEX_URL is required when using --worker-token."); } const client = new ConvexHttpClient(options.convexUrl); - const result = await client.action(resolveWorkerAction(functionName), args as never); + const result = await withTimeout( + client.action(resolveWorkerAction(functionName), args as never), + options.pageTimeoutMs, + `Convex action ${functionName} exceeded ${options.pageTimeoutMs}ms`, + ); if (validate(result)) return result; throw new Error(`Invalid ${functionName} response.`); } @@ -431,6 +462,10 @@ async function runConvexJson( return await runConvexJsonOnce(options, functionName, args, validate); } catch (error) { lastError = error; + if (isLocalConvexPageTimeout(error)) { + writeCommandErrorOutput(error); + throw error; + } if (attempt === DEFAULT_MAX_CONVEX_ATTEMPTS) break; console.error( `[snapshot] retrying ${functionName} after attempt ${attempt}: ${errorMessage(error)}`, @@ -454,7 +489,14 @@ async function runConvexJsonOnce( cwd: process.cwd(), encoding: "utf8", env: convexRunEnv(), + timeout: options.pageTimeoutMs, maxBuffer: CONVEX_RUN_MAX_BUFFER_BYTES, + }).catch((error: unknown) => { + if (!isExecFileTimeout(error)) throw error; + throw commandOutputError( + `Convex run ${functionName} exceeded ${options.pageTimeoutMs}ms`, + error, + ); }); try { return parseConvexJsonMatching(result.stdout, validate); @@ -803,6 +845,21 @@ function delay(ms: number) { return new Promise((done) => setTimeout(done, ms)); } +function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { + let timeout: ReturnType | null = null; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs); + }); + return Promise.race([promise, timeoutPromise]).finally(() => { + if (timeout) clearTimeout(timeout); + }); +} + +function cursorSummary(cursor: string | null) { + if (cursor === null || cursor.length === 0) return "start"; + return cursor.length <= 12 ? cursor : `${cursor.slice(0, 6)}...${cursor.slice(-4)}`; +} + function errorMessage(error: unknown) { return error instanceof Error ? error.message : String(error); } @@ -817,6 +874,18 @@ function isLikelyConvexOperationTimeout(error: unknown) { ); } +function isLocalConvexPageTimeout(error: unknown) { + return /Convex (?:action|run) .* exceeded \d+ms/.test(errorMessage(error)); +} + +function isExecFileTimeout(error: unknown) { + return ( + isRecord(error) && + error.killed === true && + (error.signal === "SIGTERM" || error.signal === "SIGKILL") + ); +} + function isLikelyOversizedConvexBatch(error: unknown) { return isLikelyTruncatedConvexOutput(error) || isLikelyConvexOperationTimeout(error); } @@ -834,6 +903,15 @@ function writeCommandErrorOutput(error: unknown) { } } +function commandOutputError(message: string, cause: unknown) { + const error: CommandOutputError = new Error(message); + if (isRecord(cause)) { + if (typeof cause.stderr === "string") error.stderr = cause.stderr; + if (typeof cause.stdout === "string") error.stdout = cause.stdout; + } + return error; +} + function gitSha() { const result = spawnSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }); if (result.status !== 0) return "unknown"; @@ -855,6 +933,7 @@ function parseArgs(args: string[]): Options { batchPages: DEFAULT_BATCH_PAGES, concurrency: DEFAULT_CONCURRENCY, shards: DEFAULT_SHARDS, + pageTimeoutMs: DEFAULT_CONVEX_PAGE_TIMEOUT_MS, outDir: DEFAULT_OUT_DIR, sourceKind: "all", timeWindow: emptyCreatedTimeWindow(), @@ -892,6 +971,8 @@ function parseArgs(args: string[]): Options { options.concurrency = readPositiveInt(readValue(args, ++index, arg), arg); } else if (arg === "--shards") { options.shards = readPositiveInt(readValue(args, ++index, arg), arg); + } else if (arg === "--page-timeout-ms") { + options.pageTimeoutMs = readTimerTimeoutMs(readValue(args, ++index, arg), arg); } else if (arg === "--out-dir") { options.outDir = readValue(args, ++index, arg); } else if (arg === "--source-kind") { @@ -947,6 +1028,14 @@ function readPositiveInt(value: string, flag: string) { return parsed; } +function readTimerTimeoutMs(value: string, flag: string) { + const parsed = readPositiveInt(value, flag); + if (parsed > MAX_TIMER_TIMEOUT_MS) { + throw new Error(`Expected ${flag} to be at most ${MAX_TIMER_TIMEOUT_MS}.`); + } + return parsed; +} + function readSourceKind(value: string): SourceKind | "all" { if (value === "all" || value === "skill" || value === "package") return value; throw new Error(`Unsupported source kind: ${value}`); diff --git a/scripts/security-dataset/exportSnapshotCli.test.ts b/scripts/security-dataset/exportSnapshotCli.test.ts index 36a447dd..921a4002 100644 --- a/scripts/security-dataset/exportSnapshotCli.test.ts +++ b/scripts/security-dataset/exportSnapshotCli.test.ts @@ -111,6 +111,43 @@ describe("security dataset snapshot CLI", () => { stderr: expect.stringContaining("--min-page-size must be less than or equal to --page-size."), }); }); + + it("rejects a non-positive page timeout", async () => { + await expect( + execFileAsync( + "bun", + ["scripts/security-dataset/export-snapshot.ts", "--page-timeout-ms", "0", "--dry-run"], + { + cwd: process.cwd(), + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }, + ), + ).rejects.toMatchObject({ + stderr: expect.stringContaining("Expected positive integer for --page-timeout-ms"), + }); + }); + + it("rejects a page timeout above the runtime timer maximum", async () => { + await expect( + execFileAsync( + "bun", + [ + "scripts/security-dataset/export-snapshot.ts", + "--page-timeout-ms", + "2147483648", + "--dry-run", + ], + { + cwd: process.cwd(), + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }, + ), + ).rejects.toMatchObject({ + stderr: expect.stringContaining("Expected --page-timeout-ms to be at most 2147483647."), + }); + }); }); function buildTinyConvexSnapshotZip() {