admin/clawhub
1
0
Fork 0
mirror of https://github.com/openclaw/clawhub.git synced 2026-08-14 08:52:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): right-size security dataset snapshots

Browse Source 
Right-size the ClawHub security dataset snapshot workflow after the scheduled run exposed a 120-minute shard timeout. Raises the default shard split while preserving hosted-runner max parallelism, tightens override caps, and cancels only superseded manual dry-run exports.\n\nProof: bun test scripts/security-dataset/security-dataset-snapshot-workflow.test.ts; bun run format:check -- .github/workflows/security-dataset-snapshot.yml scripts/security-dataset/security-dataset-snapshot-workflow.test.ts; bun run ci:static; autoreview clean; PR CI 22 successful / 1 skipped.
This commit is contained in:
Vincent Koc
2026-06-27 06:45:22 -07:00
committed by GitHub
parent 6a18bea576
commit 91dc9b1486
2 changed files with 18 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/security-dataset-snapshot.yml
+8 -8
View File
@@ -42,7 +42,7 @@ on:
shards:
description: "Created-at shards per source kind"
required: true
default: "12"
default: "48"
reuse-shards-run-id:
description: "Optional previous successful shard run id to merge/publish without re-exporting"
required: false
@@ -55,8 +55,8 @@ permissions:
contents: read
concurrency:
group: clawhub-security-dataset-snapshot
cancel-in-progress: false
group: clawhub-security-dataset-snapshot-${{ github.event_name == 'workflow_dispatch' && inputs.upload != 'true' && inputs['reuse-shards-run-id'] == '' && 'manual-dry-run' || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.upload != 'true' && inputs['reuse-shards-run-id'] == '' }}
jobs:
plan-security-dataset:
@@ -80,11 +80,11 @@ jobs:
SNAPSHOT_BATCH_PAGES: ${{ inputs['batch-pages'] || '1' }}
SNAPSHOT_PAGE_TIMEOUT_MS: ${{ inputs['page-timeout-ms'] || '600000' }}
SNAPSHOT_CONCURRENCY: ${{ inputs.concurrency || '1' }}
SNAPSHOT_SHARDS: ${{ inputs.shards || '12' }}
SNAPSHOT_MAX_SHARDS_PER_SOURCE: ${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '128' }}
SNAPSHOT_MAX_MATRIX_JOBS: ${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '256' }}
SNAPSHOT_HARD_MAX_SHARDS_PER_SOURCE: "128"
SNAPSHOT_HARD_MAX_MATRIX_JOBS: "256"
SNAPSHOT_SHARDS: ${{ inputs.shards || '48' }}
SNAPSHOT_MAX_SHARDS_PER_SOURCE: ${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '64' }}
SNAPSHOT_MAX_MATRIX_JOBS: ${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '128' }}
SNAPSHOT_HARD_MAX_SHARDS_PER_SOURCE: "64"
SNAPSHOT_HARD_MAX_MATRIX_JOBS: "128"
steps:
- uses: actions/checkout@v7
scripts/security-dataset/security-dataset-snapshot-workflow.test.ts
+10 -4
View File
@@ -28,6 +28,10 @@ describe("security-dataset-snapshot workflow", () => {
};
};
permissions?: Record<string, string>;
concurrency?: {
group?: string;
"cancel-in-progress"?: string | boolean;
};
};
const planJob = workflow.jobs["plan-security-dataset"];
@@ -36,16 +40,18 @@ describe("security-dataset-snapshot workflow", () => {
const planStep = planJob.steps.find((step) => step.id === "plan");
expect(workflow.permissions?.actions).toBe("read");
expect(workflow.on?.workflow_dispatch?.inputs?.shards?.default).toBe("12");
expect(workflow.on?.workflow_dispatch?.inputs?.shards?.default).toBe("48");
expect(workflow.on?.workflow_dispatch?.inputs?.["reuse-shards-run-id"]?.default).toBe("");
expect(workflow.concurrency?.group).toContain("manual-dry-run");
expect(workflow.concurrency?.["cancel-in-progress"]).toContain("inputs.upload != 'true'");
expect(planJob.if).toContain("reuse-shards-run-id");
expect(planJob.env?.SNAPSHOT_SHARDS).toBe("${{ inputs.shards || '12' }}");
expect(planJob.env?.SNAPSHOT_SHARDS).toBe("${{ inputs.shards || '48' }}");
expect(planJob.env?.HF_DATASET_REPO).toBe("OpenClaw/clawhub-security-signals-live");
expect(planJob.env?.SNAPSHOT_MAX_SHARDS_PER_SOURCE).toBe(
"${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '128' }}",
"${{ vars.SECURITY_DATASET_MAX_SHARDS_PER_SOURCE || '64' }}",
);
expect(planJob.env?.SNAPSHOT_MAX_MATRIX_JOBS).toBe(
"${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '256' }}",
"${{ vars.SECURITY_DATASET_MAX_MATRIX_JOBS || '128' }}",
);
expect(planJob.steps.find((step) => step.run?.includes("Maximum matrix jobs"))?.run).toContain(
"requested ${shards} shards per source kind",