diff --git a/.github/workflows/security-dataset-snapshot.yml b/.github/workflows/security-dataset-snapshot.yml index 03274a19..c9aba501 100644 --- a/.github/workflows/security-dataset-snapshot.yml +++ b/.github/workflows/security-dataset-snapshot.yml @@ -42,7 +42,7 @@ on: shards: description: "Created-at shards per source kind" required: true - default: "12" + default: "48" reuse-shards-run-id: description: "Optional previous successful shard run id to merge/publish without re-exporting" required: false @@ -55,8 +55,8 @@ permissions: contents: read concurrency: - group: clawhub-security-dataset-snapshot - cancel-in-progress: false + group: clawhub-security-dataset-snapshot-${{ github.event_name == 'workflow_dispatch' && inputs.upload != 'true' && inputs['reuse-shards-run-id'] == '' && 'manual-dry-run' || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.upload != 'true' && inputs['reuse-shards-run-id'] == '' }} jobs: plan-security-dataset: @@ -80,11 +80,11 @@ jobs: SNAPSHOT_BATCH_PAGES: ${{ inputs['batch-pages'] || '1' }} SNAPSHOT_PAGE_TIMEOUT_MS: ${{ inputs['page-timeout-ms'] || '600000' }} SNAPSHOT_CONCURRENCY: ${{ inputs.concurrency || '1' }} - SNAPSHOT_SHARDS: ${{ inputs.shards || '12' }} - SNAPSHOT_MAX_SHARDS_PER_SOURCE: ${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '128' }} - SNAPSHOT_MAX_MATRIX_JOBS: ${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '256' }} - SNAPSHOT_HARD_MAX_SHARDS_PER_SOURCE: "128" - SNAPSHOT_HARD_MAX_MATRIX_JOBS: "256" + SNAPSHOT_SHARDS: ${{ inputs.shards || '48' }} + SNAPSHOT_MAX_SHARDS_PER_SOURCE: ${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '64' }} + SNAPSHOT_MAX_MATRIX_JOBS: ${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '128' }} + SNAPSHOT_HARD_MAX_SHARDS_PER_SOURCE: "64" + SNAPSHOT_HARD_MAX_MATRIX_JOBS: "128" steps: - uses: actions/checkout@v7 diff --git a/scripts/security-dataset/security-dataset-snapshot-workflow.test.ts b/scripts/security-dataset/security-dataset-snapshot-workflow.test.ts index 39a75c55..0c529b0a 100644 --- a/scripts/security-dataset/security-dataset-snapshot-workflow.test.ts +++ b/scripts/security-dataset/security-dataset-snapshot-workflow.test.ts @@ -28,6 +28,10 @@ describe("security-dataset-snapshot workflow", () => { }; }; permissions?: Record; + concurrency?: { + group?: string; + "cancel-in-progress"?: string | boolean; + }; }; const planJob = workflow.jobs["plan-security-dataset"]; @@ -36,16 +40,18 @@ describe("security-dataset-snapshot workflow", () => { const planStep = planJob.steps.find((step) => step.id === "plan"); expect(workflow.permissions?.actions).toBe("read"); - expect(workflow.on?.workflow_dispatch?.inputs?.shards?.default).toBe("12"); + expect(workflow.on?.workflow_dispatch?.inputs?.shards?.default).toBe("48"); expect(workflow.on?.workflow_dispatch?.inputs?.["reuse-shards-run-id"]?.default).toBe(""); + expect(workflow.concurrency?.group).toContain("manual-dry-run"); + expect(workflow.concurrency?.["cancel-in-progress"]).toContain("inputs.upload != 'true'"); expect(planJob.if).toContain("reuse-shards-run-id"); - expect(planJob.env?.SNAPSHOT_SHARDS).toBe("${{ inputs.shards || '12' }}"); + expect(planJob.env?.SNAPSHOT_SHARDS).toBe("${{ inputs.shards || '48' }}"); expect(planJob.env?.HF_DATASET_REPO).toBe("OpenClaw/clawhub-security-signals-live"); expect(planJob.env?.SNAPSHOT_MAX_SHARDS_PER_SOURCE).toBe( - "${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '128' }}", + "${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '64' }}", ); expect(planJob.env?.SNAPSHOT_MAX_MATRIX_JOBS).toBe( - "${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '256' }}", + "${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '128' }}", ); expect(planJob.steps.find((step) => step.run?.includes("Maximum matrix jobs"))?.run).toContain( "requested ${shards} shards per source kind",