From 91dc9b1486f772d77aced80db8682d0bdd7a46d1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 27 Jun 2026 06:45:22 -0700 Subject: [PATCH] fix(ci): right-size security dataset snapshots Right-size the ClawHub security dataset snapshot workflow after the scheduled run exposed a 120-minute shard timeout. Raises the default shard split while preserving hosted-runner max parallelism, tightens override caps, and cancels only superseded manual dry-run exports.\n\nProof: bun test scripts/security-dataset/security-dataset-snapshot-workflow.test.ts; bun run format:check -- .github/workflows/security-dataset-snapshot.yml scripts/security-dataset/security-dataset-snapshot-workflow.test.ts; bun run ci:static; autoreview clean; PR CI 22 successful / 1 skipped. --- .github/workflows/security-dataset-snapshot.yml | 16 ++++++++-------- .../security-dataset-snapshot-workflow.test.ts | 14 ++++++++++---- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/security-dataset-snapshot.yml b/.github/workflows/security-dataset-snapshot.yml index 03274a19..c9aba501 100644 --- a/.github/workflows/security-dataset-snapshot.yml +++ b/.github/workflows/security-dataset-snapshot.yml @@ -42,7 +42,7 @@ on: shards: description: "Created-at shards per source kind" required: true - default: "12" + default: "48" reuse-shards-run-id: description: "Optional previous successful shard run id to merge/publish without re-exporting" required: false @@ -55,8 +55,8 @@ permissions: contents: read concurrency: - group: clawhub-security-dataset-snapshot - cancel-in-progress: false + group: clawhub-security-dataset-snapshot-${{ github.event_name == 'workflow_dispatch' && inputs.upload != 'true' && inputs['reuse-shards-run-id'] == '' && 'manual-dry-run' || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.upload != 'true' && inputs['reuse-shards-run-id'] == '' }} jobs: plan-security-dataset: @@ -80,11 +80,11 @@ jobs: SNAPSHOT_BATCH_PAGES: ${{ inputs['batch-pages'] || '1' }} SNAPSHOT_PAGE_TIMEOUT_MS: ${{ inputs['page-timeout-ms'] || '600000' }} SNAPSHOT_CONCURRENCY: ${{ inputs.concurrency || '1' }} - SNAPSHOT_SHARDS: ${{ inputs.shards || '12' }} - SNAPSHOT_MAX_SHARDS_PER_SOURCE: ${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '128' }} - SNAPSHOT_MAX_MATRIX_JOBS: ${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '256' }} - SNAPSHOT_HARD_MAX_SHARDS_PER_SOURCE: "128" - SNAPSHOT_HARD_MAX_MATRIX_JOBS: "256" + SNAPSHOT_SHARDS: ${{ inputs.shards || '48' }} + SNAPSHOT_MAX_SHARDS_PER_SOURCE: ${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '64' }} + SNAPSHOT_MAX_MATRIX_JOBS: ${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '128' }} + SNAPSHOT_HARD_MAX_SHARDS_PER_SOURCE: "64" + SNAPSHOT_HARD_MAX_MATRIX_JOBS: "128" steps: - uses: actions/checkout@v7 diff --git a/scripts/security-dataset/security-dataset-snapshot-workflow.test.ts b/scripts/security-dataset/security-dataset-snapshot-workflow.test.ts index 39a75c55..0c529b0a 100644 --- a/scripts/security-dataset/security-dataset-snapshot-workflow.test.ts +++ b/scripts/security-dataset/security-dataset-snapshot-workflow.test.ts @@ -28,6 +28,10 @@ describe("security-dataset-snapshot workflow", () => { }; }; permissions?: Record; + concurrency?: { + group?: string; + "cancel-in-progress"?: string | boolean; + }; }; const planJob = workflow.jobs["plan-security-dataset"]; @@ -36,16 +40,18 @@ describe("security-dataset-snapshot workflow", () => { const planStep = planJob.steps.find((step) => step.id === "plan"); expect(workflow.permissions?.actions).toBe("read"); - expect(workflow.on?.workflow_dispatch?.inputs?.shards?.default).toBe("12"); + expect(workflow.on?.workflow_dispatch?.inputs?.shards?.default).toBe("48"); expect(workflow.on?.workflow_dispatch?.inputs?.["reuse-shards-run-id"]?.default).toBe(""); + expect(workflow.concurrency?.group).toContain("manual-dry-run"); + expect(workflow.concurrency?.["cancel-in-progress"]).toContain("inputs.upload != 'true'"); expect(planJob.if).toContain("reuse-shards-run-id"); - expect(planJob.env?.SNAPSHOT_SHARDS).toBe("${{ inputs.shards || '12' }}"); + expect(planJob.env?.SNAPSHOT_SHARDS).toBe("${{ inputs.shards || '48' }}"); expect(planJob.env?.HF_DATASET_REPO).toBe("OpenClaw/clawhub-security-signals-live"); expect(planJob.env?.SNAPSHOT_MAX_SHARDS_PER_SOURCE).toBe( - "${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '128' }}", + "${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '64' }}", ); expect(planJob.env?.SNAPSHOT_MAX_MATRIX_JOBS).toBe( - "${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '256' }}", + "${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '128' }}", ); expect(planJob.steps.find((step) => step.run?.includes("Maximum matrix jobs"))?.run).toContain( "requested ${shards} shards per source kind",