mirror of
https://github.com/garrytan/gbrain.git
synced 2026-08-14 08:53:22 +00:00
v0.41.6.0 feat(ci): CI test speedup — 23min → ~9min via matrix 4→6 + weight-aware sharding + auto SHA cache + parallel verify (#1444)
* feat(ci): scripts/run-verify-parallel.sh — parallel verify dispatcher Fans out the 21 pre-test grep guards via & + wait, captures per-check exit codes in a tempdir, aggregates failures with named check + log tail to stderr on miss. Wallclock 27s sequential → 13s parallel locally (2x). Bigger CI win is shard 1 deload (workflow restructure in a later commit). Pinned by test/scripts/run-verify-parallel.test.ts (6 cases: CLI contract + synthetic dispatcher failure-surfacing). * feat(ci): weight-aware LPT bin-packer + auto SHA cache hash scripts/sharding.ts (NEW) — pure TypeScript LPT bin-packer. Sort weights desc, assign each file to the shard with current minimum total. Worst-case makespan within 4/3 of optimal, O(n log n). Missing weights fall back to corpus median (not 0). New test file → ships immediately without regenerating weights. Pinned by test/scripts/sharding.test.ts (23 cases). scripts/mine-shard-weights.ts (NEW) — scrapes per-file timing from gh run view --log via timestamp delta between ##[group]test/foo.test.ts: headers within a shard. Three input modes: --run <ID>, --from-file <PATH>, stdin. Stable JSON output (sorted keys). Initial weights mined from run 26398061007. Pinned by test/scripts/mine-shard-weights.test.ts (15 cases). scripts/ci-cache-hash.sh (NEW) — deterministic 16-char sha256 over git ls-files -s minus deny-list (CHANGELOG/TODOS/README/LICENSE/ docs/**/*.md). CLAUDE.md, AGENTS.md, skills/**/* deliberately INCLUDED (8+ test files read them; deny-listing would create false-pass holes). ~40ms on 1891 files. Pinned by test/scripts/ci-cache-hash.test.ts (24 cases: 8 CRITICAL false-pass guards + 7 SAFE deny-list invariants + 9 edge cases). scripts/test-weights.json (NEW) — 712 weights. Total 3306s observed runtime; median 30ms; max 6 min outlier. * chore: bump version and changelog (v0.41.6.0) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
e084457c2d
commit
3a2605e9a0
+167
-19
@@ -10,7 +10,62 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# cache-check: runs first, computes the content hash of every tracked
|
||||
# file EXCEPT the deny-list (CHANGELOG.md, README.md, docs/**/*.md, etc.
|
||||
# — see scripts/ci-cache-hash.sh for the full list). Looks up
|
||||
# `ci-pass-<hash>` in actions/cache; if hit, the test matrix + verify
|
||||
# + serial jobs all skip and test-status reports green immediately.
|
||||
# If miss, the full suite runs and cache-write seals it on success.
|
||||
#
|
||||
# Hit rate covers re-pushes (same SHA twice), branch rebases that
|
||||
# don't touch tracked code, and any branch update that touches only
|
||||
# the deny-listed doc files.
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
cache-check:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
hit: ${{ steps.lookup.outputs.cache-hit }}
|
||||
hash: ${{ steps.compute.outputs.hash }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Compute content hash
|
||||
id: compute
|
||||
run: |
|
||||
# --verbose writes the "X/Y files in hash" diagnostic to stderr;
|
||||
# stdout carries the 16-char hash. Capture both.
|
||||
HASH=$(bash scripts/ci-cache-hash.sh --verbose 2>/tmp/cache-diag)
|
||||
cat /tmp/cache-diag
|
||||
echo "Computed cache hash: $HASH"
|
||||
echo "hash=$HASH" >> "$GITHUB_OUTPUT"
|
||||
- name: Lookup actions/cache for ci-pass-<hash>
|
||||
id: lookup
|
||||
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||||
with:
|
||||
key: ci-pass-${{ steps.compute.outputs.hash }}
|
||||
path: .ci-cache-marker
|
||||
# `lookup-only: true` means we only probe whether the cache
|
||||
# entry exists — we don't download it (the marker contents
|
||||
# don't matter, only the key match does). `cache-hit` returns
|
||||
# true only on EXACT key match (per actions/cache docs); a
|
||||
# restore-keys prefix fallback would set cache-hit=false, so
|
||||
# it's deliberately omitted here. Cross-branch scoping works
|
||||
# naturally: PR branches can read default-branch (master)
|
||||
# cache entries via exact key match when the content hash
|
||||
# matches, which happens whenever the tree is doc-only
|
||||
# different from a green master run.
|
||||
lookup-only: true
|
||||
- name: Cache status
|
||||
run: |
|
||||
if [ "${{ steps.lookup.outputs.cache-hit }}" = "true" ]; then
|
||||
echo "✓ cache HIT for hash ${{ steps.compute.outputs.hash }} — test jobs will skip"
|
||||
else
|
||||
echo "✗ cache MISS for hash ${{ steps.compute.outputs.hash }} — full suite will run"
|
||||
fi
|
||||
|
||||
gitleaks:
|
||||
needs: cache-check
|
||||
if: needs.cache-check.outputs.hit != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
@@ -20,29 +75,122 @@ jobs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test:
|
||||
# ubuntu-latest is free 2-core/7GB. Larger runners (16-cores, etc.) require
|
||||
# a provisioned runner pool in repo settings. Falling back to default keeps
|
||||
# the matrix shard speedup (~5-6x via parallelism) at zero cost.
|
||||
verify:
|
||||
# Pre-test gates: privacy/jsonb/source-id/etc + typecheck + admin-build.
|
||||
# Lives in its own runner so the matrix shards aren't carrying ~2-3min
|
||||
# of verify work in addition to their test files (the old shape stuffed
|
||||
# this into `test (1)` via `if: matrix.shard == 1`, which made shard 1
|
||||
# the slowest matrix worker). scripts/run-verify-parallel.sh fans out
|
||||
# the 20 checks via & + wait (~5s vs ~15-25s sequential).
|
||||
needs: cache-check
|
||||
if: needs.cache-check.outputs.hit != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4]
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
- name: Pre-test gates (shard 1 only — they're not test files)
|
||||
if: matrix.shard == 1
|
||||
run: bun run verify
|
||||
- name: Run test shard ${{ matrix.shard }}/4
|
||||
run: scripts/test-shard.sh ${{ matrix.shard }} 4
|
||||
- name: Run *.serial.test.ts at --max-concurrency=1 (shard 1 only)
|
||||
# Serial files share file-wide state (top-level mock.module, module
|
||||
# singletons) that leaks across files in the same bun-test process.
|
||||
# test-shard.sh excludes them; this step runs them at concurrency=1.
|
||||
if: matrix.shard == 1
|
||||
run: bun run test:serial
|
||||
- run: bun run verify
|
||||
|
||||
serial-tests:
|
||||
# *.serial.test.ts at --max-concurrency=1. Lives in its own runner so
|
||||
# the matrix shards aren't carrying the serial-pass tail (the old shape
|
||||
# stuffed this into `test (1)` after the matrix work, which compounded
|
||||
# shard 1's overload).
|
||||
needs: cache-check
|
||||
if: needs.cache-check.outputs.hit != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
- run: bun run test:serial
|
||||
|
||||
test:
|
||||
# Pure matrix shard — no verify, no serial. Each shard runs its slice
|
||||
# of the unit test set under one `bun test` invocation.
|
||||
#
|
||||
# 6 shards (not 8) stays under the GitHub free-tier ~20-job concurrency
|
||||
# budget when multiple PRs land same day: 6 shards + verify + serial +
|
||||
# gitleaks + cache-check + cache-write + test-status = ~12 jobs × 2
|
||||
# concurrent PRs = 24; 8 shards × 2 PRs would queue worse.
|
||||
#
|
||||
# Partition policy is weight-aware LPT bin-packing via scripts/sharding.ts
|
||||
# (replaces FNV-1a path hash). Weights live in scripts/test-weights.json,
|
||||
# mined from real CI logs via scripts/mine-shard-weights.ts. Missing
|
||||
# weights fall back to corpus median — new test files work immediately.
|
||||
needs: cache-check
|
||||
if: needs.cache-check.outputs.hit != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4, 5, 6]
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.3.13
|
||||
- run: bun install
|
||||
- name: Run test shard ${{ matrix.shard }}/6
|
||||
run: scripts/test-shard.sh ${{ matrix.shard }} 6
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# cache-write: ONLY runs when every gated job succeeded. Writes the
|
||||
# cache entry under `ci-pass-<hash>` so future runs at the same hash
|
||||
# hit cache. Codex's load-bearing correctness point: writing the
|
||||
# cache before the matrix completes would permanently bless bad states
|
||||
# (a future run at the same hash would skip tests because of a cache
|
||||
# entry written when tests hadn't actually passed).
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
cache-write:
|
||||
needs: [cache-check, gitleaks, verify, serial-tests, test]
|
||||
if: success() && needs.cache-check.outputs.hit != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create cache marker
|
||||
run: |
|
||||
mkdir -p .ci-cache-marker
|
||||
echo "${{ needs.cache-check.outputs.hash }}" > .ci-cache-marker/hash
|
||||
echo "$GITHUB_SHA" > .ci-cache-marker/sha
|
||||
echo "$GITHUB_REF" > .ci-cache-marker/ref
|
||||
- uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||||
with:
|
||||
key: ci-pass-${{ needs.cache-check.outputs.hash }}
|
||||
path: .ci-cache-marker
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# test-status: the single user-visible "did CI pass?" check.
|
||||
# Runs always (if: always()), succeeds when EITHER cache-check.hit==true
|
||||
# OR all gated jobs (gitleaks, verify, serial-tests, test) succeeded.
|
||||
# Branch protection (when configured) gates on this single job name.
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
test-status:
|
||||
needs: [cache-check, gitleaks, verify, serial-tests, test]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Aggregate result
|
||||
run: |
|
||||
HIT="${{ needs.cache-check.outputs.hit }}"
|
||||
GITLEAKS="${{ needs.gitleaks.result }}"
|
||||
VERIFY="${{ needs.verify.result }}"
|
||||
SERIAL="${{ needs.serial-tests.result }}"
|
||||
TEST="${{ needs.test.result }}"
|
||||
echo "cache-check.hit=$HIT"
|
||||
echo "gitleaks=$GITLEAKS verify=$VERIFY serial-tests=$SERIAL test=$TEST"
|
||||
if [ "$HIT" = "true" ]; then
|
||||
echo "✓ cache HIT for hash ${{ needs.cache-check.outputs.hash }} — CI green"
|
||||
exit 0
|
||||
fi
|
||||
# Cache miss: every gated job must have succeeded.
|
||||
for r in "$GITLEAKS" "$VERIFY" "$SERIAL" "$TEST"; do
|
||||
if [ "$r" != "success" ]; then
|
||||
echo "✗ gated job did not succeed (got $r) — CI fail"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "✓ all gated jobs succeeded — CI green"
|
||||
|
||||
@@ -2,6 +2,55 @@
|
||||
|
||||
All notable changes to GBrain will be documented in this file.
|
||||
|
||||
## [0.41.6.0] - 2026-05-25
|
||||
|
||||
**Your PR CI stops taking 23 minutes.** The Test workflow on every pull request was wallclock-bound by one unlucky shard (shard 3, 22-26 min) while other shards finished in 5-13 min. After this release, every shard finishes in roughly the same time, the whole matrix runs in about 9-10 min on a fresh PR, and PRs that don't actually change test-affecting code (rebases, doc-only commits, retries) skip the test run entirely and finish in under 2 minutes via a content-hash cache.
|
||||
|
||||
To turn it on: `gbrain upgrade` and pull the new `.github/workflows/test.yml`. First run after the merge writes the cache entry; subsequent matching pushes skip the test matrix.
|
||||
|
||||
What you'd see in a concrete example. Pre-this-release: you push a 1-line CHANGELOG typo fix on top of a green PR. CI runs the full 23-minute matrix again. Post-fix: CHANGELOG.md is on the cache-key deny-list, the hash matches your prior green run, every test job skips with the cache-hit signal, the consolidated `Test / test-status` check goes green in under 2 minutes. Same shape for rebases, doc-only PRs, and re-pushes of the same SHA.
|
||||
|
||||
How the speedup actually lands. Five orthogonal levers, all in one PR:
|
||||
|
||||
1. **Restructured `test.yml` jobs.** `verify` (the 20 pre-test grep guards + typecheck + admin-build), `serial-tests`, and the test matrix used to all run on shard 1 via `if: matrix.shard == 1`, which is why shard 1 was the second-slowest. Each now runs in its own GitHub Actions runner in parallel. Shard 1 becomes equal in size to shards 2-6.
|
||||
2. **Matrix 4 → 6 shards.** Six (not eight) keeps total per-PR job count under the GitHub free-tier ~20-job concurrency budget when multiple PRs land same day (6 shards + verify + serial + gitleaks + cache-check + cache-write + test-status = ~12 jobs × 2 concurrent PRs = 24, tight but workable; 8 shards × 2 PRs would queue worse).
|
||||
3. **Weight-aware sharding (LPT bin-packer).** `scripts/test-shard.sh` used to partition with `(fnv1a(path) % N) + 1` which is weight-blind. The new `scripts/sharding.ts` does longest-processing-time-first greedy bin-packing over measured per-file weights from `scripts/test-weights.json`. Files absent from the JSON fall back to the corpus median, so adding a new test file works immediately without regenerating weights. Real-weight projection on 712 mined weights: every 6-shard estimated at 534s = 8.9 min wallclock.
|
||||
4. **Mined CI-log weights, not isolated profiles.** `scripts/mine-shard-weights.ts` scrapes per-file wallclock from `gh run view --log` output (delta between `##[group]test/foo.test.ts:` timestamps within a shard). Free, real-world data, methodologically right (measures CI shard runtime, not per-file cold-start dominantly). Initial `test-weights.json` mined from run `26398061007`.
|
||||
5. **Auto SHA cache.** `scripts/ci-cache-hash.sh` computes a deterministic 16-char sha256 over every git-tracked file EXCEPT `CHANGELOG.md`, `TODOS.md`, `README.md`, `LICENSE`, `docs/**/*.md`, `docs/**/*.txt`. CLAUDE.md, AGENTS.md, and `skills/**/*` are deliberately INCLUDED in the hash because tests read them; deny-listing those would create false-pass holes. A `cache-check` job runs first via `actions/cache/restore@v4.2.3` in `lookup-only: true` mode; on hit, every gated job skips and the `test-status` aggregator reports green. A `cache-write` job seals the cache key only after every gated job actually succeeded.
|
||||
|
||||
**What we caught and fixed before merging.** Outside-voice (Codex) reviewed the plan and produced four material changes baked into this ship: (a) confirmed `e2e.yml` is fast (3-5 min) and NOT the critical path, validating that targeting `test.yml` is correct; (b) corrected the deny-list to keep CLAUDE.md and AGENTS.md IN the hash (their original deny-listing would have been a real false-pass hole since 8+ test files read them); (c) replaced the original draft's isolated per-file profiling (~57 min run, wrong methodology) with the log-mining approach; (d) added the job restructure (verify/serial split out of shard 1) which was missing from the original plan.
|
||||
|
||||
Coverage. 8 CRITICAL false-pass guards pin the hash invalidation contract: `CLAUDE.md` edit → DIFFERENT hash, `AGENTS.md` edit → DIFFERENT hash, `skills/foo/SKILL.md` edit → DIFFERENT hash, `src/core/db.ts` edit → DIFFERENT hash, `test/foo.test.ts` edit → DIFFERENT hash, `package.json` edit → DIFFERENT hash, `bun.lock` edit → DIFFERENT hash, `.github/workflows/test.yml` edit → DIFFERENT hash. 7 SAFE deny-list invariants pin the cache-hit contract for genuinely test-irrelevant docs. Plus 9 edge cases (untracked-file excluded, rename detection, new-file-type discovery, deny-list-typo guard, symlinks, locale-stable sort, deterministic, usage errors). 24/24 green in `test/scripts/ci-cache-hash.test.ts`.
|
||||
|
||||
### Itemized changes
|
||||
|
||||
**New scripts:**
|
||||
- `scripts/sharding.ts` (NEW) — Pure TypeScript LPT bin-packer with median fallback. Exports `partition`, `loadWeights`, `computeMedian`, `imbalanceRatio`. Throws `WeightsLoadError` on malformed JSON; runs in O(n log n). Pinned by `test/scripts/sharding.test.ts` (23 cases).
|
||||
- `scripts/test-shard.sh` (rewrite) — Thin shell wrapper. Same CLI surface (`--dry-run-list` preserved). Streams the file list to `bun run scripts/sharding.ts` via stdin (avoids argv overflow at 676+ files). Slow files (`*.slow.test.ts`) intentionally INCLUDED in CI matrix — local fast loop excludes them; preserves the v0.26.4 CI-vs-local divergence policy.
|
||||
- `scripts/test-weights.json` (NEW) — 712 mined weights from a green master run. Stats: min=0ms, median=30ms, max=359087ms (~6 min outlier), total 3306.3s observed runtime.
|
||||
- `scripts/mine-shard-weights.ts` (NEW) — Scrapes `gh run view --log` for per-file timing. Three input modes: `--run <ID>`, `--from-file <PATH>`, or piped from stdin. Stable JSON output (sorted keys) for clean diffs. Pinned by `test/scripts/mine-shard-weights.test.ts` (15 cases).
|
||||
- `scripts/run-verify-parallel.sh` (NEW) — Fans out 21 fast checks (privacy/jsonb/source-id/admin-build/typecheck/gateway-routed/etc.) via `& wait`, per-check temp dir log, failure aggregation. 27s sequential → 13s parallel (2x) locally; bigger win in CI is shard 1 deload. Pinned by `test/scripts/run-verify-parallel.test.ts` (6 cases including synthetic failure-surfacing).
|
||||
- `scripts/ci-cache-hash.sh` (NEW) — Deterministic 16-char sha256 over `git ls-files -s` minus deny-list. ~40ms on 1891 files (was ~9s with per-file `git hash-object`). Pinned by `test/scripts/ci-cache-hash.test.ts` (24 cases: 8 CRITICAL false-pass guards + 7 SAFE deny-list invariants + 9 edge cases).
|
||||
|
||||
**Workflow restructure:**
|
||||
- `.github/workflows/test.yml` — Seven jobs replacing the old 5-shard layout: `cache-check` (runs first via `actions/cache/restore@v4.2.3` `lookup-only`), then `gitleaks` + `verify` + `serial-tests` + `test` (6-shard matrix) all gated on `if: needs.cache-check.outputs.hit != 'true'`, then `cache-write` (post-all-pass via `if: success() && ...`), then `test-status` (the user-visible aggregator, `if: always()` so it reports green on cache-hit OR all-jobs-pass).
|
||||
|
||||
**Coverage extensions:**
|
||||
- `test/scripts/test-shard.slow.test.ts` (EXTENDED) — New LPT balance contract: 4-shard and 6-shard wallclock imbalance ratio ≤1.5 with real weights from `test-weights.json`. New INCLUDE-slow-files regression guard. New determinism check. Old FNV-1a runtime tests removed (sharding now lives in TS).
|
||||
- `test/privacy-script-wired.test.ts` (FIXED in same wave) — Updated to follow the verify-script indirection (`package.json` `verify` → `run-verify-parallel.sh` → CHECKS array contains `check:privacy`). The original substring assertion broke when the `&&` chain was replaced with the parallel dispatcher.
|
||||
|
||||
### For contributors
|
||||
|
||||
To regenerate `scripts/test-weights.json` after the corpus drifts significantly:
|
||||
|
||||
```bash
|
||||
LATEST_RUN=$(gh run list --workflow=test.yml --status success --limit 1 --json databaseId --jq '.[0].databaseId')
|
||||
bun run scripts/mine-shard-weights.ts --run "$LATEST_RUN"
|
||||
git add scripts/test-weights.json && git commit -m "chore: refresh test weights from CI run $LATEST_RUN"
|
||||
```
|
||||
|
||||
There is no scheduled regen — weights drift gracefully (missing files fall back to median), so this is opt-in when a specific shard starts feeling slow.
|
||||
|
||||
## [0.41.5.0] - 2026-05-24
|
||||
|
||||
**Six community bug-fix PRs land + the E2E suite stops lying about itself.** A fix-wave triage swept the 333-PR queue, closed 10 PRs as already-shipped (with credit, naming the commits + files), and bundled 6 real fixes from the community into one collector. Plus three E2E-suite reliability fixes that surfaced while getting the full Docker suite to 100% green.
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gbrain",
|
||||
"version": "0.41.5.0",
|
||||
"version": "0.41.6.0",
|
||||
"description": "Postgres-native personal knowledge brain with hybrid RAG search",
|
||||
"type": "module",
|
||||
"main": "src/core/index.ts",
|
||||
@@ -40,7 +40,7 @@
|
||||
"build:pglite-snapshot": "bun run scripts/build-pglite-snapshot.ts",
|
||||
"test": "bash scripts/run-unit-parallel.sh",
|
||||
"test:full": "bun run verify && bash scripts/run-unit-parallel.sh && bun run test:slow && ([ -n \"$DATABASE_URL\" ] && bash scripts/run-e2e.sh || echo '[test:full] skipped E2E (no DATABASE_URL); run docker-compose -f docker-compose.ci.yml up + bun run test:e2e to include' 1>&2)",
|
||||
"verify": "bun run check:privacy && bun run check:proposal-pii && bun run check:test-names && bun run check:jsonb && bun run check:source-id-projection && bun run check:source-config-leak && bun run check:progress && bun run check:test-isolation && bun run check:wasm && bun run check:admin-build && bun run check:admin-scope-drift && bun run check:cli-exec && bun run check:system-of-record && bun run check:eval-glossary && bun run check:no-pii-agent-voice && bun run check:synthetic-corpus-privacy && bun run check:skill-brain-first && bun run check:fuzz-purity && bun run check:operations-filter-bypass && bun run check:gateway-routed && bun run typecheck",
|
||||
"verify": "bash scripts/run-verify-parallel.sh",
|
||||
"check:source-config-leak": "scripts/check-source-config-leak.sh",
|
||||
"check:no-pii-agent-voice": "scripts/check-no-pii-in-agent-voice.sh",
|
||||
"check:synthetic-corpus-privacy": "scripts/check-synthetic-corpus-privacy.sh",
|
||||
|
||||
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/ci-cache-hash.sh — deterministic content hash of all test-
|
||||
# affecting files for the CI auto-cache.
|
||||
#
|
||||
# Outputs a 16-character hex prefix of sha256(sorted list of
|
||||
# `<git-blob-sha> <path>` lines for every tracked file EXCEPT the
|
||||
# deny-list below). Same tree → same hash. Different code → different
|
||||
# hash. Doc-only changes → same hash (cache hit).
|
||||
#
|
||||
# Used by .github/workflows/test.yml's cache-check job. Cache key is
|
||||
# `ci-pass-<hash>`. When the hash matches a prior green run, the test
|
||||
# matrix skips and reports green immediately.
|
||||
#
|
||||
# DESIGN: deny-list NOT allowlist. New files default to "include in
|
||||
# hash" — worst case, cache miss (waste 8min). Allowlist would default
|
||||
# to "exclude", risking false-pass (broken code shipped under green
|
||||
# check) when someone adds a new file type that tests read.
|
||||
#
|
||||
# WHAT'S DENY-LISTED (genuinely test-irrelevant):
|
||||
# - CHANGELOG.md, TODOS.md pure documentation
|
||||
# - README.md, LICENSE marketing / metadata, no test reads them
|
||||
# - docs/**/*.md, *.txt all docs/ subtree is doc-only
|
||||
#
|
||||
# WHAT'S DELIBERATELY NOT DENY-LISTED (affects test outcomes):
|
||||
# - CLAUDE.md 8+ test files reference it (resolver-merge, schema-cli,
|
||||
# public-exports, eval-cross-modal-batch, etc.)
|
||||
# - AGENTS.md same — referenced by resolver tests; counterpart to
|
||||
# CLAUDE.md for OpenClaw hosts
|
||||
# - skills/** SKILL.md files are read by skill conformance tests
|
||||
# - everything else under src/, test/, scripts/, .github/, package.json,
|
||||
# bun.lock, tsconfig*.json, the schema files — obviously test-affecting
|
||||
#
|
||||
# Locale-stable: LC_ALL=C on the sort step so byte-order is identical
|
||||
# across runners (different default locales would re-order the line list
|
||||
# and change the final hash).
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/ci-cache-hash.sh # print hash
|
||||
# bash scripts/ci-cache-hash.sh --verbose # print hash + diagnostics to stderr
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 printed a 16-char hex hash
|
||||
# 1 internal error (git failure, etc.)
|
||||
# 2 usage error
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
VERBOSE=0
|
||||
if [ "${1:-}" = "--verbose" ]; then
|
||||
VERBOSE=1
|
||||
shift
|
||||
fi
|
||||
if [ "$#" -gt 0 ]; then
|
||||
echo "usage: bash scripts/ci-cache-hash.sh [--verbose]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Deny-list as an extended regex matched against full paths emitted by
|
||||
# `git ls-files`. -x makes the match anchored (full-line). Each pattern
|
||||
# is a path predicate, not a glob — `\.` to literal-match dots.
|
||||
#
|
||||
# To add a new deny entry: add another `-e '<regex>'` line below. To
|
||||
# REMOVE a deny entry (= include the path back in hash): delete its line.
|
||||
# Either change invalidates the cache for everyone on next run (different
|
||||
# hash output), which is the correct behavior.
|
||||
DENY_PATTERNS=(
|
||||
-e '^CHANGELOG\.md$'
|
||||
-e '^TODOS\.md$'
|
||||
-e '^README\.md$'
|
||||
-e '^LICENSE$'
|
||||
-e '^docs/.*\.md$'
|
||||
-e '^docs/.*\.txt$'
|
||||
)
|
||||
|
||||
# Use `git ls-files -s` for one-shot enumeration of tracked files + their
|
||||
# index blob shas. Output shape: `<mode> <sha> <stage>\t<path>` per line.
|
||||
# Far faster than per-file `git hash-object` (~30ms vs ~9s on 2000 files).
|
||||
#
|
||||
# Trade-off: this reflects the INDEX (committed/staged state), not the
|
||||
# working tree. CI always works against a committed tree so this matches
|
||||
# what tests actually run. Local dev with uncommitted edits sees the
|
||||
# committed-side hash (close enough — the hash is for CI's cache lookup,
|
||||
# not a tree-state diagnostic).
|
||||
LS_FILES=$(git ls-files -s)
|
||||
if [ -z "$LS_FILES" ]; then
|
||||
echo "error: git ls-files -s returned empty (are we in a git repo?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Apply deny-list. Each line ends in `\t<path>` so the deny patterns
|
||||
# anchor on a tab boundary. Compose the alternation regex from
|
||||
# DENY_PATTERNS — each entry is `^<pat>$`; strip the `^` (since `\t`
|
||||
# acts as our anchor in `git ls-files -s` output) and the trailing `$`
|
||||
# stays as-is.
|
||||
DENY_ALT=""
|
||||
i=1
|
||||
while [ $i -lt ${#DENY_PATTERNS[@]} ]; do
|
||||
p="${DENY_PATTERNS[$i]}"
|
||||
p="${p#^}"
|
||||
if [ -z "$DENY_ALT" ]; then
|
||||
DENY_ALT="$p"
|
||||
else
|
||||
DENY_ALT="$DENY_ALT|$p"
|
||||
fi
|
||||
i=$((i + 2))
|
||||
done
|
||||
DENY_RE=$(printf '\t(%s)' "$DENY_ALT")
|
||||
# Note: each $p already ends in `$` to anchor end-of-line, so the full
|
||||
# regex is `\t(^CHANGELOG\.md$|^TODOS\.md$|...)`. Wait — we stripped `^`
|
||||
# from each but kept `$`, so the composed regex is `\t(CHANGELOG\.md$|
|
||||
# TODOS\.md$|docs/.*\.md$|...)`. Each alternative anchors its own end.
|
||||
INCLUDED=$(printf '%s\n' "$LS_FILES" | grep -vE "$DENY_RE" || true)
|
||||
|
||||
if [ -z "$INCLUDED" ]; then
|
||||
echo "error: every tracked file is deny-listed — refusing to hash empty set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Sort by full line (LC_ALL=C for byte-order stability across locales),
|
||||
# hash the concatenation. Each line carries (mode, sha, path) so any
|
||||
# change to content (sha), mode (executable bit), or path (rename) flips
|
||||
# the final hash.
|
||||
HASH=$(printf '%s\n' "$INCLUDED" \
|
||||
| LC_ALL=C sort \
|
||||
| sha256sum \
|
||||
| cut -c1-16)
|
||||
|
||||
if [ "$VERBOSE" = "1" ]; then
|
||||
included_count=$(printf '%s\n' "$INCLUDED" | wc -l | tr -d ' ')
|
||||
all_count=$(printf '%s\n' "$LS_FILES" | wc -l | tr -d ' ')
|
||||
denied_count=$((all_count - included_count))
|
||||
{
|
||||
echo "ci-cache-hash: $included_count/$all_count files in hash ($denied_count deny-listed)"
|
||||
} >&2
|
||||
fi
|
||||
|
||||
echo "$HASH"
|
||||
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* scripts/mine-shard-weights.ts — extract per-file test wallclock from a
|
||||
* real CI run's logs, write scripts/test-weights.json.
|
||||
*
|
||||
* Why this exists: scripts/sharding.ts does LPT bin-packing over per-file
|
||||
* weights, but the weights have to come from somewhere. The original
|
||||
* design ran each test file in isolation (`bun test <file>` per file)
|
||||
* which (a) takes ~57min to run all 676 files, and (b) measures cold-
|
||||
* start dominantly because each invocation pays a fresh `bun test`
|
||||
* startup. CI shards run ~150 files in ONE bun process — cold-start is
|
||||
* amortized away. Per-file isolated profiles are wrong-by-methodology.
|
||||
*
|
||||
* This script scrapes per-file wallclock from a real CI shard's log via
|
||||
* GitHub's `gh run view --log` output. bun emits an `##[group]test/foo.
|
||||
* test.ts:` header before each file with an ISO timestamp; the
|
||||
* difference between consecutive headers = how long the previous file
|
||||
* took. This is the actual CI shard runtime per file, in the right
|
||||
* execution mode, for free on every green run.
|
||||
*
|
||||
* Usage:
|
||||
* bun run scripts/mine-shard-weights.ts --run <RUN_ID> [--out PATH]
|
||||
* bun run scripts/mine-shard-weights.ts --from-file <LOG_FILE> [--out PATH]
|
||||
* gh run view <RUN_ID> --log | bun run scripts/mine-shard-weights.ts [--out PATH]
|
||||
*
|
||||
* Default output: scripts/test-weights.json (overwrites). Use --out to
|
||||
* write elsewhere (useful for diffing before commit). Output is JSON
|
||||
* with sorted keys for stable diffs.
|
||||
*
|
||||
* Regen cadence: there is none. Weights drift continuously but missing
|
||||
* files fall back to the corpus median in sharding.ts, so stale weights
|
||||
* degrade gracefully. Run this script when you notice a specific shard
|
||||
* starts running long, or after a wave that added many heavy tests.
|
||||
*
|
||||
* Exit codes:
|
||||
* 0 wrote weights file (count > 0)
|
||||
* 1 internal error
|
||||
* 2 usage error
|
||||
* 3 no usable timing data found (parsed log but extracted 0 weights)
|
||||
*/
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = resolve(__dirname, "..");
|
||||
const DEFAULT_OUT = resolve(REPO_ROOT, "scripts/test-weights.json");
|
||||
|
||||
interface Args {
|
||||
runId?: string;
|
||||
fromFile?: string;
|
||||
fromStdin: boolean;
|
||||
out: string;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): Args {
|
||||
const out: Args = { fromStdin: false, out: DEFAULT_OUT };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === "--run" || a === "--run-id") {
|
||||
out.runId = argv[++i];
|
||||
} else if (a === "--from-file") {
|
||||
out.fromFile = argv[++i];
|
||||
} else if (a === "--out") {
|
||||
out.out = resolve(argv[++i] ?? "");
|
||||
} else if (a === "--help" || a === "-h") {
|
||||
console.log(
|
||||
"usage: bun run scripts/mine-shard-weights.ts (--run <ID> | --from-file <PATH> | <stdin>) [--out <PATH>]",
|
||||
);
|
||||
process.exit(0);
|
||||
} else {
|
||||
console.error(`error: unknown arg: ${a}`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
if (!out.runId && !out.fromFile) {
|
||||
out.fromStdin = true;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function readSource(args: Args): Promise<string> {
|
||||
if (args.runId) {
|
||||
const r = spawnSync("gh", ["run", "view", args.runId, "--log"], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 256 * 1024 * 1024, // CI logs can be 50-80MB
|
||||
});
|
||||
if (r.status !== 0) {
|
||||
throw new Error(
|
||||
`gh run view ${args.runId} --log failed (exit ${r.status}): ${r.stderr}`,
|
||||
);
|
||||
}
|
||||
return r.stdout;
|
||||
}
|
||||
if (args.fromFile) {
|
||||
return readFileSync(args.fromFile, "utf8");
|
||||
}
|
||||
// Read stdin
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of process.stdin) {
|
||||
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
|
||||
}
|
||||
return Buffer.concat(chunks).toString("utf8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed timing event from a CI log line. timestamp is ms-since-epoch.
|
||||
*/
|
||||
interface TimingEvent {
|
||||
job: string;
|
||||
timestampMs: number;
|
||||
file: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a CI log into a list of `##[group]test/X.test.ts:` events keyed
|
||||
* by job (so timing deltas don't cross shard boundaries).
|
||||
*
|
||||
* GH log line shape:
|
||||
* <job-name>\tUNKNOWN STEP\t<ISO-timestamp> ##[group]test/foo.test.ts:
|
||||
* or:
|
||||
* <job-name>\t<step-name>\t<ISO-timestamp> ##[group]test/foo.test.ts:
|
||||
*
|
||||
* Exported for unit testing.
|
||||
*/
|
||||
export function parseLog(raw: string): TimingEvent[] {
|
||||
const events: TimingEvent[] = [];
|
||||
const lines = raw.split("\n");
|
||||
// Match: <job>TAB<step>TAB<iso-ts> ##[group]<path>:
|
||||
const re = /^([^\t]+)\t[^\t]*\t(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)\s+##\[group\](test\/[^\s:]+\.test\.ts):?\s*$/;
|
||||
for (const line of lines) {
|
||||
const m = re.exec(line);
|
||||
if (!m) continue;
|
||||
const job = m[1]!.trim();
|
||||
const ts = Date.parse(m[2]!);
|
||||
const file = m[3]!;
|
||||
if (Number.isNaN(ts)) continue;
|
||||
events.push({ job, timestampMs: ts, file });
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
/**
|
||||
* From a list of file-start events grouped by job, compute per-file
|
||||
* runtime as (timestamp[i+1] - timestamp[i]) within each job. The last
|
||||
* file in each job is dropped (we don't know when it ended without
|
||||
* also parsing the bun summary line; the loss is acceptable since
|
||||
* sharding.ts's median fallback covers missing files).
|
||||
*
|
||||
* When the same file appears in multiple jobs (shouldn't happen, but
|
||||
* defensive against shard remix during the in-flight CI run that
|
||||
* generated this log), take the max — heaviest observation wins.
|
||||
*
|
||||
* Exported for unit testing.
|
||||
*/
|
||||
export function computeWeights(events: TimingEvent[]): Map<string, number> {
|
||||
// Group events by job, in stream order.
|
||||
const byJob = new Map<string, TimingEvent[]>();
|
||||
for (const e of events) {
|
||||
let bucket = byJob.get(e.job);
|
||||
if (!bucket) {
|
||||
bucket = [];
|
||||
byJob.set(e.job, bucket);
|
||||
}
|
||||
bucket.push(e);
|
||||
}
|
||||
|
||||
const weights = new Map<string, number>();
|
||||
for (const [, jobEvents] of byJob) {
|
||||
for (let i = 0; i + 1 < jobEvents.length; i++) {
|
||||
const file = jobEvents[i]!.file;
|
||||
const delta = jobEvents[i + 1]!.timestampMs - jobEvents[i]!.timestampMs;
|
||||
if (delta < 0) continue; // log out-of-order; defensive
|
||||
// Round to nearest ms; sub-ms doesn't matter for shard balancing.
|
||||
const ms = Math.round(delta);
|
||||
const prev = weights.get(file);
|
||||
if (prev === undefined || ms > prev) {
|
||||
weights.set(file, ms);
|
||||
}
|
||||
}
|
||||
// Drop the last event's file (no successor → unknown duration).
|
||||
}
|
||||
return weights;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize a weights map to canonical JSON (keys sorted asc) so the
|
||||
* committed file produces stable diffs run-to-run.
|
||||
*
|
||||
* Exported for unit testing.
|
||||
*/
|
||||
export function serializeWeights(weights: Map<string, number>): string {
|
||||
const sorted = Array.from(weights.entries()).sort((a, b) =>
|
||||
a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0,
|
||||
);
|
||||
const obj: Record<string, number> = {};
|
||||
for (const [k, v] of sorted) obj[k] = v;
|
||||
return JSON.stringify(obj, null, 2) + "\n";
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
console.error(
|
||||
`[mine-shard-weights] source=${args.runId ?? args.fromFile ?? "<stdin>"}`,
|
||||
);
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await readSource(args);
|
||||
} catch (e) {
|
||||
console.error(`error: ${e instanceof Error ? e.message : String(e)}`);
|
||||
return 1;
|
||||
}
|
||||
const events = parseLog(raw);
|
||||
console.error(`[mine-shard-weights] parsed ${events.length} file-start events`);
|
||||
if (events.length === 0) {
|
||||
console.error(
|
||||
"error: no ##[group]test/*.test.ts: events found in input. Was this a CI test run log?",
|
||||
);
|
||||
return 3;
|
||||
}
|
||||
const weights = computeWeights(events);
|
||||
if (weights.size === 0) {
|
||||
console.error("error: parsed events but extracted 0 weights (every job had ≤1 file?)");
|
||||
return 3;
|
||||
}
|
||||
const json = serializeWeights(weights);
|
||||
writeFileSync(args.out, json);
|
||||
// Summary: min/median/max/total. Useful for spot-checking the file.
|
||||
const values = Array.from(weights.values()).sort((a, b) => a - b);
|
||||
const min = values[0]!;
|
||||
const max = values[values.length - 1]!;
|
||||
const median = values[Math.floor(values.length / 2)]!;
|
||||
const total = values.reduce((a, b) => a + b, 0);
|
||||
console.error(
|
||||
`[mine-shard-weights] wrote ${weights.size} weights to ${args.out}`,
|
||||
);
|
||||
console.error(
|
||||
`[mine-shard-weights] stats: min=${min}ms median=${median}ms max=${max}ms total=${(total / 1000).toFixed(1)}s`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().then((code) => process.exit(code));
|
||||
}
|
||||
Executable
+193
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env bash
|
||||
# scripts/run-verify-parallel.sh — parallel verify dispatcher.
|
||||
#
|
||||
# Runs the 19+ verify checks (privacy, jsonb, source-id, … + typecheck +
|
||||
# admin-build) as background jobs, waits for all, aggregates exit codes,
|
||||
# surfaces failed-check name + tail of its log to stderr.
|
||||
#
|
||||
# Replaces the sequential `&&`-chain in package.json's `verify` script.
|
||||
# Wallclock: 19 sequential checks (~15-25s on CI) → parallel (~3-5s).
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run-verify-parallel.sh # run every CHECK below
|
||||
# bash scripts/run-verify-parallel.sh --dry-list # print check list, exit
|
||||
#
|
||||
# Env overrides:
|
||||
# GBRAIN_VERIFY_TIMEOUT per-check wallclock cap, seconds (default 120)
|
||||
# GBRAIN_VERIFY_LOG_DIR where to write per-check logs (default tempdir)
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 all checks passed
|
||||
# 1 one or more checks failed (full details in stderr)
|
||||
# 2 usage error / no checks defined
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Checks to run. Order is irrelevant (parallel), but keep stable for log
|
||||
# determinism + grep-ability. Each entry is a bun-script name (the
|
||||
# `package.json` "scripts" key), invoked as `bun run <name>`.
|
||||
#
|
||||
# To add a check: append to this array. To skip in CI temporarily, comment
|
||||
# the line — the parallel runner doesn't care about count.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
CHECKS=(
|
||||
"check:privacy"
|
||||
"check:proposal-pii"
|
||||
"check:test-names"
|
||||
"check:jsonb"
|
||||
"check:source-id-projection"
|
||||
"check:source-config-leak"
|
||||
"check:progress"
|
||||
"check:test-isolation"
|
||||
"check:wasm"
|
||||
"check:admin-build"
|
||||
"check:admin-scope-drift"
|
||||
"check:cli-exec"
|
||||
"check:system-of-record"
|
||||
"check:eval-glossary"
|
||||
"check:no-pii-agent-voice"
|
||||
"check:synthetic-corpus-privacy"
|
||||
"check:skill-brain-first"
|
||||
"check:fuzz-purity"
|
||||
"check:operations-filter-bypass"
|
||||
"check:gateway-routed"
|
||||
"typecheck"
|
||||
)
|
||||
|
||||
if [ "${#CHECKS[@]}" -eq 0 ]; then
|
||||
echo "ERROR: no checks defined in run-verify-parallel.sh" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Dry-run path: list checks, exit. Used by tests + ops debugging.
|
||||
if [ "${1:-}" = "--dry-list" ]; then
|
||||
printf '%s\n' "${CHECKS[@]}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$#" -gt 0 ] && [ "${1:-}" != "" ]; then
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
echo "usage: bash scripts/run-verify-parallel.sh [--dry-list]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
TIMEOUT="${GBRAIN_VERIFY_TIMEOUT:-120}"
|
||||
|
||||
# Per-check temp dir. Each check gets its own subdir so writes can't race
|
||||
# on shared scratch state (the checks themselves are read-only — they grep
|
||||
# the working tree — but defense-in-depth.)
|
||||
if [ -n "${GBRAIN_VERIFY_LOG_DIR:-}" ]; then
|
||||
LOG_DIR="$GBRAIN_VERIFY_LOG_DIR"
|
||||
mkdir -p "$LOG_DIR" || { echo "ERROR: cannot create $LOG_DIR" >&2; exit 2; }
|
||||
else
|
||||
LOG_DIR="$(mktemp -d /tmp/gbrain-verify-XXXXXX)"
|
||||
trap 'rm -rf "$LOG_DIR"' EXIT
|
||||
fi
|
||||
|
||||
# Resolve `timeout` for per-check wallclock cap. macOS doesn't ship one;
|
||||
# brew coreutils provides `gtimeout`. If neither is available, fall back to
|
||||
# bg-pid + sleep-cap (slightly less reliable but still bounded).
|
||||
TIMEOUT_BIN=""
|
||||
if command -v gtimeout >/dev/null 2>&1; then TIMEOUT_BIN="gtimeout"
|
||||
elif command -v timeout >/dev/null 2>&1; then TIMEOUT_BIN="timeout"
|
||||
fi
|
||||
|
||||
START_TS=$(date +%s)
|
||||
echo "[verify-parallel] running ${#CHECKS[@]} checks in parallel (timeout=${TIMEOUT}s, logs=$LOG_DIR)" >&2
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Spawn one background process per check. Each child captures its own exit
|
||||
# code into a sentinel file under $LOG_DIR/<safe-name>.exit; the parent
|
||||
# never trusts `wait`'s aggregate value because that maps to last-spawned.
|
||||
#
|
||||
# safe_name: turn `check:privacy` into `check_privacy` so it fits a filename
|
||||
# without escaping.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
PIDS=()
|
||||
SAFE_NAMES=()
|
||||
for c in "${CHECKS[@]}"; do
|
||||
safe="${c//:/_}"
|
||||
SAFE_NAMES+=("$safe")
|
||||
LOG_FILE="$LOG_DIR/$safe.log"
|
||||
EXIT_FILE="$LOG_DIR/$safe.exit"
|
||||
(
|
||||
if [ -n "$TIMEOUT_BIN" ]; then
|
||||
"$TIMEOUT_BIN" "${TIMEOUT}s" bun run "$c" > "$LOG_FILE" 2>&1
|
||||
else
|
||||
bun run "$c" > "$LOG_FILE" 2>&1 &
|
||||
pid=$!
|
||||
( sleep "$TIMEOUT" && kill -TERM "$pid" 2>/dev/null && \
|
||||
sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
|
||||
cap_pid=$!
|
||||
wait "$pid" 2>/dev/null
|
||||
kill "$cap_pid" 2>/dev/null
|
||||
wait "$cap_pid" 2>/dev/null
|
||||
fi
|
||||
rc=$?
|
||||
echo "$rc" > "$EXIT_FILE"
|
||||
) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
# Wait for every background job. Ignore wait's aggregate exit — exit codes
|
||||
# live in the sentinel files.
|
||||
for pid in "${PIDS[@]}"; do wait "$pid" 2>/dev/null || true; done
|
||||
|
||||
END_TS=$(date +%s)
|
||||
ELAPSED=$((END_TS - START_TS))
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Aggregate. For each check, read its exit file; on failure, append a
|
||||
# labeled block (check name + tail of log) to the failure report. Surface
|
||||
# one final summary line and the report to stderr if anything failed.
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
PASS=0
|
||||
FAIL=0
|
||||
FAIL_NAMES=()
|
||||
FAIL_REPORT=""
|
||||
|
||||
for i in "${!CHECKS[@]}"; do
|
||||
c="${CHECKS[$i]}"
|
||||
safe="${SAFE_NAMES[$i]}"
|
||||
EXIT_FILE="$LOG_DIR/$safe.exit"
|
||||
LOG_FILE="$LOG_DIR/$safe.log"
|
||||
|
||||
rc=1
|
||||
[ -f "$EXIT_FILE" ] && rc=$(cat "$EXIT_FILE" 2>/dev/null || echo 1)
|
||||
|
||||
if [ "$rc" = "0" ]; then
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
FAIL=$((FAIL + 1))
|
||||
FAIL_NAMES+=("$c")
|
||||
if [ "$rc" = "124" ]; then
|
||||
FAIL_REPORT+=$'\n--- '"$c"' (TIMED OUT after '"${TIMEOUT}"'s) ---\n'
|
||||
else
|
||||
FAIL_REPORT+=$'\n--- '"$c"' (rc='"$rc"') ---\n'
|
||||
fi
|
||||
if [ -f "$LOG_FILE" ]; then
|
||||
FAIL_REPORT+="$(tail -30 "$LOG_FILE")"
|
||||
FAIL_REPORT+=$'\n'
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
{
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "❌ verify failed: $FAIL/${#CHECKS[@]} checks did not pass"
|
||||
echo "Failed: ${FAIL_NAMES[*]}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
printf '%s' "$FAIL_REPORT"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "[verify-parallel] elapsed=${ELAPSED}s | pass=$PASS fail=$FAIL"
|
||||
} >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[verify-parallel] elapsed=${ELAPSED}s | pass=$PASS fail=0 | all checks green" >&2
|
||||
exit 0
|
||||
@@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* scripts/sharding.ts — weight-aware test file partitioning.
|
||||
*
|
||||
* Replaces FNV-1a path-hash sharding in scripts/test-shard.sh. Uses
|
||||
* longest-processing-time-first (LPT) greedy bin-packing over measured
|
||||
* per-file runtimes to balance total wallclock across N shards.
|
||||
*
|
||||
* LPT is a textbook approximation algorithm: sort jobs by weight desc,
|
||||
* assign each to the bin (shard) with the current minimum total. Worst-
|
||||
* case makespan is within 4/3 of optimal. Runs in O(n log n).
|
||||
*
|
||||
* Weights live in scripts/test-weights.json — committed, mined from real
|
||||
* CI run logs via scripts/mine-shard-weights.ts. Files absent from the
|
||||
* weights map fall back to the corpus median (not zero — that would
|
||||
* favor unknown new files into the smallest shard, defeating balance).
|
||||
*
|
||||
* CLI:
|
||||
* bun run scripts/sharding.ts <shard-index> <total-shards>
|
||||
* Reads test file list from stdin (one path per line). Prints the
|
||||
* subset assigned to <shard-index> to stdout, one per line.
|
||||
*
|
||||
* bun run scripts/sharding.ts <shard-index> <total-shards> --files <glob>
|
||||
* Walks the filesystem for matching files instead of reading stdin.
|
||||
*
|
||||
* Exit codes:
|
||||
* 0 success
|
||||
* 1 internal error (e.g., malformed weights JSON)
|
||||
* 2 usage error
|
||||
*
|
||||
* Used by: scripts/test-shard.sh (thin wrapper), test/scripts/sharding.test.ts.
|
||||
*/
|
||||
|
||||
import { readFileSync, existsSync } from "node:fs";
|
||||
import { resolve, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = resolve(__dirname, "..");
|
||||
const DEFAULT_WEIGHTS_PATH = resolve(REPO_ROOT, "scripts/test-weights.json");
|
||||
|
||||
export type WeightMap = Map<string, number>;
|
||||
|
||||
export class WeightsLoadError extends Error {
|
||||
constructor(public readonly path: string, public readonly cause: unknown) {
|
||||
super(`failed to load weights from ${path}: ${cause}`);
|
||||
this.name = "WeightsLoadError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a weights JSON file. Fail-soft on missing file (returns empty map).
|
||||
* Throws WeightsLoadError on malformed JSON or non-object shape — the
|
||||
* caller decides whether to fall through to defaults or surface.
|
||||
*/
|
||||
export function loadWeights(path: string = DEFAULT_WEIGHTS_PATH): WeightMap {
|
||||
if (!existsSync(path)) return new Map();
|
||||
let raw: string;
|
||||
try {
|
||||
raw = readFileSync(path, "utf8");
|
||||
} catch (e) {
|
||||
throw new WeightsLoadError(path, e);
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch (e) {
|
||||
throw new WeightsLoadError(path, `JSON.parse: ${e}`);
|
||||
}
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
||||
throw new WeightsLoadError(path, "expected top-level object {path: ms}");
|
||||
}
|
||||
const out: WeightMap = new Map();
|
||||
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
|
||||
if (typeof v !== "number" || !Number.isFinite(v) || v < 0) {
|
||||
throw new WeightsLoadError(
|
||||
path,
|
||||
`value for "${k}" must be a non-negative finite number, got ${JSON.stringify(v)}`,
|
||||
);
|
||||
}
|
||||
out.set(k, v);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Median of a list of numbers. Used as the fallback weight for files
|
||||
* absent from the weights map. Empty input returns 0 (no signal).
|
||||
*/
|
||||
export function computeMedian(values: number[]): number {
|
||||
if (values.length === 0) return 0;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 === 0
|
||||
? (sorted[mid - 1]! + sorted[mid]!) / 2
|
||||
: sorted[mid]!;
|
||||
}
|
||||
|
||||
export interface PartitionOpts {
|
||||
/**
|
||||
* Weight to assign files that are absent from the weights map. Defaults
|
||||
* to the median of present weights (computed inside `partition`) so
|
||||
* unknown new files cluster around the typical file's cost.
|
||||
*
|
||||
* Override mainly for tests; production callers should use the default.
|
||||
*/
|
||||
fallbackWeight?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Partition `files` into `n` shards using LPT greedy bin-packing.
|
||||
*
|
||||
* Returns an array of length `n` where each entry is the (deterministic)
|
||||
* file list for that shard. Files in each shard are returned in
|
||||
* assignment order (heaviest first). Same input always produces same
|
||||
* output — no Math.random, stable sort key (weight desc, then path asc).
|
||||
*
|
||||
* Contracts:
|
||||
* - Every file in `files` appears in exactly one returned shard.
|
||||
* - If `files` is empty, returns `n` empty arrays.
|
||||
* - If `n <= 0`, throws RangeError.
|
||||
* - Files missing from `weights` get `opts.fallbackWeight` (or median).
|
||||
*/
|
||||
export function partition(
|
||||
files: string[],
|
||||
weights: WeightMap,
|
||||
n: number,
|
||||
opts: PartitionOpts = {},
|
||||
): string[][] {
|
||||
if (!Number.isInteger(n) || n <= 0) {
|
||||
throw new RangeError(`shard count must be a positive integer, got ${n}`);
|
||||
}
|
||||
const shards: string[][] = Array.from({ length: n }, () => []);
|
||||
if (files.length === 0) return shards;
|
||||
|
||||
// Compute fallback weight from the median of present weights, unless
|
||||
// the caller supplied an explicit override.
|
||||
let fallback: number;
|
||||
if (opts.fallbackWeight !== undefined) {
|
||||
if (!Number.isFinite(opts.fallbackWeight) || opts.fallbackWeight < 0) {
|
||||
throw new RangeError(
|
||||
`fallbackWeight must be non-negative finite, got ${opts.fallbackWeight}`,
|
||||
);
|
||||
}
|
||||
fallback = opts.fallbackWeight;
|
||||
} else {
|
||||
fallback = computeMedian(Array.from(weights.values()));
|
||||
}
|
||||
// Cold-start guard: if the weights map is empty AND no explicit
|
||||
// fallback was supplied, every effective weight would be 0 and LPT
|
||||
// collapses (all ties → lowest-index wins → every file in shard 0).
|
||||
// Normalize fallback to 1 so LPT degenerates to round-robin, which is
|
||||
// a strictly better default than "everything in shard 1" until
|
||||
// test-weights.json gets mined.
|
||||
if (fallback === 0 && opts.fallbackWeight === undefined) {
|
||||
fallback = 1;
|
||||
}
|
||||
|
||||
// Build [weight, path] tuples. Sort by weight desc, then path asc for
|
||||
// determinism on ties (multiple files with the same weight).
|
||||
const tuples = files.map((f) => ({
|
||||
path: f,
|
||||
weight: weights.get(f) ?? fallback,
|
||||
}));
|
||||
tuples.sort((a, b) => {
|
||||
if (b.weight !== a.weight) return b.weight - a.weight;
|
||||
return a.path < b.path ? -1 : a.path > b.path ? 1 : 0;
|
||||
});
|
||||
|
||||
// Running per-shard totals. argmin tiebreaker: lowest index (stable).
|
||||
const totals = new Array<number>(n).fill(0);
|
||||
for (const t of tuples) {
|
||||
let minIdx = 0;
|
||||
for (let i = 1; i < n; i++) {
|
||||
if (totals[i]! < totals[minIdx]!) minIdx = i;
|
||||
}
|
||||
shards[minIdx]!.push(t.path);
|
||||
totals[minIdx] = totals[minIdx]! + t.weight;
|
||||
}
|
||||
return shards;
|
||||
}
|
||||
|
||||
/**
|
||||
* Imbalance ratio: max-total / min-total over the partition. 1.0 = perfect.
|
||||
* Returns Infinity when any shard is empty (degenerate). Use ≤1.5 as a
|
||||
* loose health gate; LPT typically hits ≤1.1 on real corpora.
|
||||
*
|
||||
* Exposed for tests + the slow-test regression that pins corpus health.
|
||||
*/
|
||||
export function imbalanceRatio(shards: string[][], weights: WeightMap, fallback: number): number {
|
||||
if (shards.length === 0) return 1;
|
||||
const totals = shards.map((s) =>
|
||||
s.reduce((sum, f) => sum + (weights.get(f) ?? fallback), 0),
|
||||
);
|
||||
const max = Math.max(...totals);
|
||||
const min = Math.min(...totals);
|
||||
if (min === 0) return max === 0 ? 1 : Infinity;
|
||||
return max / min;
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────────
|
||||
// CLI
|
||||
// ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function readStdinLines(): Promise<string[]> {
|
||||
// Bun gives us a readable stream on process.stdin. Read to end.
|
||||
if (process.stdin.isTTY) return []; // no piped input
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of process.stdin) {
|
||||
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
|
||||
}
|
||||
const text = Buffer.concat(chunks).toString("utf8");
|
||||
return text
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l.length > 0);
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv.length < 2) {
|
||||
console.error("usage: bun run scripts/sharding.ts <shard-index> <total-shards>");
|
||||
console.error(" (reads file list from stdin, one path per line)");
|
||||
return 2;
|
||||
}
|
||||
const idx = Number.parseInt(argv[0]!, 10);
|
||||
const total = Number.parseInt(argv[1]!, 10);
|
||||
if (!Number.isInteger(idx) || !Number.isInteger(total) || idx < 1 || total < 1 || idx > total) {
|
||||
console.error(
|
||||
`error: shard index ${argv[0]} / total ${argv[1]} invalid (need 1 <= index <= total, both ints)`,
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
const files = await readStdinLines();
|
||||
if (files.length === 0) {
|
||||
// Caller asked for a shard but piped no files. Exit clean — the wrapper
|
||||
// will warn or no-op as it sees fit.
|
||||
return 0;
|
||||
}
|
||||
let weights: WeightMap;
|
||||
try {
|
||||
weights = loadWeights();
|
||||
} catch (e) {
|
||||
console.error(`error: ${e instanceof Error ? e.message : String(e)}`);
|
||||
return 1;
|
||||
}
|
||||
const shards = partition(files, weights, total);
|
||||
for (const f of shards[idx - 1]!) {
|
||||
process.stdout.write(`${f}\n`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().then((code) => process.exit(code));
|
||||
}
|
||||
+46
-54
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Partition unit test files into N shards by stable hash and run one shard.
|
||||
# Partition unit test files into N shards and run one shard.
|
||||
#
|
||||
# Usage: scripts/test-shard.sh <shard-index> <total-shards>
|
||||
# shard-index: 1-based (1..N)
|
||||
@@ -8,16 +8,27 @@
|
||||
# Excluded from sharding:
|
||||
# - test/e2e/* — need DATABASE_URL; run via bun run test:e2e
|
||||
# - *.serial.test.ts — concurrency-unsafe (file-wide mock.module / env
|
||||
# leaks); run via scripts/run-serial-tests.sh on
|
||||
# shard 1 only. Including these here lets their
|
||||
# mock.module() calls leak into the rest of the
|
||||
# shard's bun process and silently break unrelated
|
||||
# tests. See test/eval-takes-quality-runner.serial.test.ts
|
||||
# mocking gateway.ts → voyage-multimodal failures.
|
||||
# leaks); run via bun run test:serial on its
|
||||
# own runner in CI. Including these here lets
|
||||
# their mock.module() calls leak into the rest
|
||||
# of the shard's bun process and silently break
|
||||
# unrelated tests.
|
||||
#
|
||||
# Stable partitioning: a file's shard is `(hash(path) % N) + 1`. Same file
|
||||
# lands in the same shard on every run, regardless of how many other files
|
||||
# exist, so retries are reproducible. Hash is FNV-1a — pure shell, no jq.
|
||||
# *.slow.test.ts is deliberately INCLUDED here. CI's matrix is the only
|
||||
# default place these run; the local fast loop (run-unit-shard.sh)
|
||||
# excludes them. See CLAUDE.md "CI vs local: intentionally divergent file
|
||||
# sets" for the rationale.
|
||||
#
|
||||
# Partition: weight-aware LPT bin-packing via scripts/sharding.ts. Reads
|
||||
# per-file runtime weights from scripts/test-weights.json (mined from real
|
||||
# CI logs by scripts/mine-shard-weights.ts). Files absent from the map
|
||||
# fall back to the corpus median, so adding a new test file works
|
||||
# immediately without regenerating weights — worst case it lands in the
|
||||
# wrong shard until next regen, never silently dropped.
|
||||
#
|
||||
# Stable partitioning: same `(files, weights, N)` always produces the
|
||||
# same assignment, so retries are reproducible.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DRY_RUN_LIST=0
|
||||
@@ -45,60 +56,41 @@ fi
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Find all unit test files, deterministic order. Excludes test/e2e/ and
|
||||
# *.serial.test.ts. Serial files share file-wide state (top-level
|
||||
# mock.module, module singletons) that leaks across files in the same
|
||||
# `bun test` shard process — see scripts/check-test-isolation.sh R2.
|
||||
# CI runs them via `bun run test:serial` (scripts/run-serial-tests.sh) at
|
||||
# --max-concurrency=1 in a separate step on shard 1. Local `bun run test`
|
||||
# already excludes them from the parallel pass and runs them after the
|
||||
# same way. Portable: avoid `mapfile` (bash 4+) so this runs on macOS
|
||||
# bash 3.2 too.
|
||||
FILES=()
|
||||
while IFS= read -r line; do
|
||||
FILES+=("$line")
|
||||
done < <(find test -name '*.test.ts' -not -name '*.serial.test.ts' -not -path 'test/e2e/*' | sort)
|
||||
# Collect non-E2E, non-serial unit test files. Slow files INCLUDED — see
|
||||
# header comment. Local run-unit-shard.sh excludes slow files (different
|
||||
# policy by design).
|
||||
ALL_FILES=$(find test -name '*.test.ts' \
|
||||
-not -name '*.serial.test.ts' \
|
||||
-not -path 'test/e2e/*' | sort)
|
||||
|
||||
if [ "${#FILES[@]}" -eq 0 ]; then
|
||||
if [ -z "$ALL_FILES" ]; then
|
||||
echo "no test files found under test/" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# FNV-1a 32-bit hash of a string — implemented in pure bash so we don't depend
|
||||
# on python/openssl/etc on the runner. Output is decimal.
|
||||
fnv1a() {
|
||||
local str="$1"
|
||||
local h=2166136261 # FNV offset basis
|
||||
local i ord
|
||||
for (( i=0; i<${#str}; i++ )); do
|
||||
ord=$(printf '%d' "'${str:$i:1}")
|
||||
h=$(( (h ^ ord) & 0xFFFFFFFF ))
|
||||
h=$(( (h * 16777619) & 0xFFFFFFFF ))
|
||||
done
|
||||
echo "$h"
|
||||
}
|
||||
|
||||
SHARD_FILES=()
|
||||
for f in "${FILES[@]}"; do
|
||||
hash=$(fnv1a "$f")
|
||||
bucket=$(( hash % TOTAL_SHARDS + 1 ))
|
||||
if [ "$bucket" -eq "$SHARD_INDEX" ]; then
|
||||
SHARD_FILES+=("$f")
|
||||
fi
|
||||
done
|
||||
# Delegate the LPT partition to scripts/sharding.ts. Stream the file list
|
||||
# via stdin to keep argv small (676+ files would overflow argv in some
|
||||
# shells / OSes).
|
||||
SHARD_FILES=$(printf '%s\n' "$ALL_FILES" | bun run scripts/sharding.ts "$SHARD_INDEX" "$TOTAL_SHARDS")
|
||||
|
||||
if [ "$DRY_RUN_LIST" = "1" ]; then
|
||||
if [ "${#SHARD_FILES[@]}" -eq 0 ]; then
|
||||
exit 0
|
||||
fi
|
||||
printf '%s\n' "${SHARD_FILES[@]}"
|
||||
printf '%s' "$SHARD_FILES"
|
||||
[ -n "$SHARD_FILES" ] && echo "" # trailing newline if non-empty
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "shard $SHARD_INDEX/$TOTAL_SHARDS: ${#SHARD_FILES[@]}/${#FILES[@]} files"
|
||||
if [ "${#SHARD_FILES[@]}" -eq 0 ]; then
|
||||
echo "warning: shard $SHARD_INDEX has no files (rehash or reduce shard count)" >&2
|
||||
ALL_COUNT=$(printf '%s\n' "$ALL_FILES" | grep -c '^' || true)
|
||||
SHARD_COUNT=$(printf '%s\n' "$SHARD_FILES" | grep -c '^' || true)
|
||||
# grep -c on empty input returns 0 even with trailing newline edge cases
|
||||
[ -z "$SHARD_FILES" ] && SHARD_COUNT=0
|
||||
|
||||
echo "shard $SHARD_INDEX/$TOTAL_SHARDS: ${SHARD_COUNT}/${ALL_COUNT} files (LPT-balanced)"
|
||||
|
||||
if [ "$SHARD_COUNT" -eq 0 ]; then
|
||||
echo "warning: shard $SHARD_INDEX has no files (total shards may exceed file count)" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
exec bun test --timeout=60000 "${SHARD_FILES[@]}"
|
||||
# Convert newline-separated file list to argv. xargs handles the
|
||||
# whitespace correctly without word-splitting on spaces in paths.
|
||||
printf '%s\n' "$SHARD_FILES" | xargs bun test --timeout=60000
|
||||
|
||||
@@ -0,0 +1,713 @@
|
||||
{
|
||||
"test/active-pack-wiring.test.ts": 3,
|
||||
"test/admin-agents-spend.test.ts": 4995,
|
||||
"test/admin-embed-spawn.serial.test.ts": 12792,
|
||||
"test/agent-cli.test.ts": 2935,
|
||||
"test/agent-runner.test.ts": 2,
|
||||
"test/ai/adaptive-embed-batch.test.ts": 34,
|
||||
"test/ai/build-gateway-config.test.ts": 0,
|
||||
"test/ai/capabilities.test.ts": 3,
|
||||
"test/ai/config-no-env-mutation.test.ts": 4,
|
||||
"test/ai/dims-openai.test.ts": 4,
|
||||
"test/ai/dims-zeroentropy.test.ts": 4,
|
||||
"test/ai/embedQuery.test.ts": 8,
|
||||
"test/ai/gateway-chat.test.ts": 8,
|
||||
"test/ai/gateway-tool-loop.test.ts": 9,
|
||||
"test/ai/gateway.test.ts": 444,
|
||||
"test/ai/header-transport.serial.test.ts": 118,
|
||||
"test/ai/no-batch-cap-suppression.serial.test.ts": 98,
|
||||
"test/ai/recipe-azure-openai.test.ts": 7,
|
||||
"test/ai/recipe-dashscope.test.ts": 5,
|
||||
"test/ai/recipe-llama-server.test.ts": 0,
|
||||
"test/ai/recipe-minimax.test.ts": 1,
|
||||
"test/ai/recipe-openrouter.test.ts": 1,
|
||||
"test/ai/recipe-zhipu.test.ts": 3,
|
||||
"test/ai/recipes-existing-regression.test.ts": 5,
|
||||
"test/ai/rerank.test.ts": 16,
|
||||
"test/ai/schema-templating.test.ts": 4,
|
||||
"test/ai/silent-drop-regression.test.ts": 9,
|
||||
"test/ai/voyage-code-3-recipe.test.ts": 3,
|
||||
"test/ai/zeroentropy-compat-fetch.test.ts": 16,
|
||||
"test/ai/zeroentropy-recipe.test.ts": 1,
|
||||
"test/anomalies.test.ts": 1,
|
||||
"test/anthropic-model-ids.test.ts": 3,
|
||||
"test/apply-migrations-pglite-spawn.serial.test.ts": 18874,
|
||||
"test/apply-migrations.test.ts": 5,
|
||||
"test/archive-crawler-config.test.ts": 26,
|
||||
"test/artifact-abstraction.test.ts": 2,
|
||||
"test/asymmetric-encoding-contract.test.ts": 1,
|
||||
"test/audit-slug-fallback.serial.test.ts": 98,
|
||||
"test/audit-synopsis.serial.test.ts": 99,
|
||||
"test/audit/audit-writer.test.ts": 9,
|
||||
"test/audit/content-sanity-audit.test.ts": 6,
|
||||
"test/auto-think-phase.test.ts": 3251,
|
||||
"test/autopilot-cycle-failure-classification.test.ts": 4,
|
||||
"test/autopilot-cycle-handler.test.ts": 3039,
|
||||
"test/autopilot-fanout-wiring.test.ts": 3,
|
||||
"test/autopilot-fanout.test.ts": 24,
|
||||
"test/autopilot-install.test.ts": 5,
|
||||
"test/autopilot-lock-path.test.ts": 13,
|
||||
"test/autopilot-nightly-probe-wiring.test.ts": 2,
|
||||
"test/autopilot-reconnect-classifier.test.ts": 5,
|
||||
"test/autopilot-resolve-cli.test.ts": 11,
|
||||
"test/autopilot-supervisor-wiring.test.ts": 10,
|
||||
"test/backfill-base.test.ts": 2,
|
||||
"test/backfill-concurrency-clamp.serial.test.ts": 103,
|
||||
"test/backlinks.test.ts": 23,
|
||||
"test/backoff.test.ts": 8,
|
||||
"test/balanced-reranker-default.test.ts": 19,
|
||||
"test/batch-projection.test.ts": 1,
|
||||
"test/bench-publish.test.ts": 8,
|
||||
"test/bench/baseline-file.test.ts": 2,
|
||||
"test/bench/correctness-gate.test.ts": 5,
|
||||
"test/bench/qrels-file.test.ts": 13,
|
||||
"test/book-mirror.test.ts": 431,
|
||||
"test/bootstrap.test.ts": 18481,
|
||||
"test/brain-allowlist.test.ts": 3257,
|
||||
"test/brain-registry.serial.test.ts": 104,
|
||||
"test/brain-resolver.test.ts": 125,
|
||||
"test/brain-score-breakdown.test.ts": 3087,
|
||||
"test/brain-score-recommendations.test.ts": 23,
|
||||
"test/brain-writer-partial-scan.test.ts": 4335,
|
||||
"test/brain-writer-walk-prune.test.ts": 7,
|
||||
"test/brain-writer.test.ts": 3949,
|
||||
"test/brainstorm-timeout.test.ts": 45,
|
||||
"test/brainstorm/checkpoint.serial.test.ts": 136,
|
||||
"test/brainstorm/cost-guardrails.test.ts": 13,
|
||||
"test/brainstorm/distance.test.ts": 26,
|
||||
"test/brainstorm/eval-brainstorm.test.ts": 13,
|
||||
"test/brainstorm/lsd-mode-skip.test.ts": 2,
|
||||
"test/budget-meter.test.ts": 26,
|
||||
"test/budget-tracker.test.ts": 3265,
|
||||
"test/build-llms.test.ts": 11,
|
||||
"test/calibration-cli.test.ts": 13,
|
||||
"test/calibration-profile.test.ts": 7,
|
||||
"test/candidate-audit.test.ts": 9,
|
||||
"test/capture-build-content.test.ts": 18,
|
||||
"test/capture-runcapture.test.ts": 8,
|
||||
"test/cathedral-ii-brainbench.test.ts": 9162,
|
||||
"test/check-resolvable-cli.test.ts": 3600,
|
||||
"test/check-resolvable.test.ts": 40,
|
||||
"test/check-system-of-record.test.ts": 70,
|
||||
"test/check-update.test.ts": 674,
|
||||
"test/child-worker-supervisor.test.ts": 144,
|
||||
"test/chunk-grain-fts.test.ts": 70996,
|
||||
"test/chunker-timeout.test.ts": 27,
|
||||
"test/chunker-version-gate.test.ts": 6,
|
||||
"test/chunkers/code.test.ts": 129,
|
||||
"test/chunkers/recursive.test.ts": 57,
|
||||
"test/cjk.test.ts": 4,
|
||||
"test/claw-test-cli.test.ts": 9,
|
||||
"test/cli-args.test.ts": 15,
|
||||
"test/cli-dispatch-thin-client.test.ts": 6027,
|
||||
"test/cli-help-discoverability.test.ts": 1670,
|
||||
"test/cli-multimodal-integration.test.ts": 26677,
|
||||
"test/cli-options.test.ts": 1658,
|
||||
"test/cli-pty-runner.test.ts": 16,
|
||||
"test/cli-query-image.test.ts": 56,
|
||||
"test/cli.test.ts": 2633,
|
||||
"test/code-callers-cli.test.ts": 2,
|
||||
"test/code-def-refs.test.ts": 28662,
|
||||
"test/code-edges.test.ts": 2884,
|
||||
"test/code-intel/edge-densification.test.ts": 33,
|
||||
"test/code-intel/eval-capture-graph.test.ts": 5,
|
||||
"test/code-intel/recursive-walk.test.ts": 31107,
|
||||
"test/code-intel/scope-walker-resolution.test.ts": 19,
|
||||
"test/code-intel/traversal-cache.test.ts": 42036,
|
||||
"test/code-retrieval-harness.test.ts": 4,
|
||||
"test/commands-search.test.ts": 23985,
|
||||
"test/commands/capture.test.ts": 42431,
|
||||
"test/config-ensure-gitignore.test.ts": 7,
|
||||
"test/config-env.test.ts": 6,
|
||||
"test/config-set.test.ts": 17,
|
||||
"test/config-unset.test.ts": 23831,
|
||||
"test/config.test.ts": 6,
|
||||
"test/connection-manager.serial.test.ts": 117,
|
||||
"test/connection-resilience.test.ts": 29,
|
||||
"test/console-prefix.test.ts": 5,
|
||||
"test/consolidate-valid-until.test.ts": 2471,
|
||||
"test/content-sanity-literals.test.ts": 2,
|
||||
"test/content-sanity.test.ts": 5,
|
||||
"test/context-engine.test.ts": 33,
|
||||
"test/contextual-retrieval-doctor.serial.test.ts": 2755,
|
||||
"test/contextual-retrieval-resolver.test.ts": 4,
|
||||
"test/contextual-retrieval-service-pure.test.ts": 3,
|
||||
"test/core/audit-week-file.serial.test.ts": 97,
|
||||
"test/core/base-phase.test.ts": 9,
|
||||
"test/core/cycle.serial.test.ts": 5563,
|
||||
"test/core/diarize/payload-fitter-summarize.test.ts": 15,
|
||||
"test/core/remediation-checkpoint.serial.test.ts": 129,
|
||||
"test/cosine-rescore-column.test.ts": 3198,
|
||||
"test/cross-brain-calibration.test.ts": 8,
|
||||
"test/cross-modal-eval-aggregate.test.ts": 3,
|
||||
"test/cross-modal-eval-cli.test.ts": 23,
|
||||
"test/cross-modal-eval-json-repair.test.ts": 31,
|
||||
"test/cross-modal-hybrid-integration.serial.test.ts": 3312,
|
||||
"test/cross-modal-phase1.test.ts": 40,
|
||||
"test/cross-modal-phase2.test.ts": 48,
|
||||
"test/cycle-abort.test.ts": 108,
|
||||
"test/cycle-consolidate.test.ts": 23952,
|
||||
"test/cycle-last-full-cycle-at.test.ts": 3771,
|
||||
"test/cycle-legacy-phases.test.ts": 4177,
|
||||
"test/cycle-lock-per-source.test.ts": 9,
|
||||
"test/cycle-pack-gating.test.ts": 5,
|
||||
"test/cycle-patterns.test.ts": 2,
|
||||
"test/cycle-pglite-lock-ordering.serial.test.ts": 2693,
|
||||
"test/cycle-synthesize-chunker.test.ts": 21,
|
||||
"test/cycle-synthesize-md-discovery.test.ts": 6,
|
||||
"test/cycle-synthesize-slug-collection.test.ts": 3140,
|
||||
"test/cycle-synthesize.test.ts": 17,
|
||||
"test/cycle/extract-atoms-synthesize-concepts.test.ts": 60181,
|
||||
"test/cycle/nightly-probe-adapters.test.ts": 6,
|
||||
"test/data-research.test.ts": 17,
|
||||
"test/db-lock-election.test.ts": 23346,
|
||||
"test/db-lock-per-source.test.ts": 33460,
|
||||
"test/db-lock-refresh.test.ts": 6,
|
||||
"test/dedup.test.ts": 13,
|
||||
"test/destructive-guard.test.ts": 6357,
|
||||
"test/disk-walk.test.ts": 17,
|
||||
"test/distribution-import-boundary.test.ts": 7,
|
||||
"test/doctor-behavioral.test.ts": 5729,
|
||||
"test/doctor-calibration-checks.test.ts": 5,
|
||||
"test/doctor-child-orphans.test.ts": 4,
|
||||
"test/doctor-cli-smoke.serial.test.ts": 4239,
|
||||
"test/doctor-cycle-freshness.test.ts": 33963,
|
||||
"test/doctor-cycle-phase-scope.test.ts": 7,
|
||||
"test/doctor-federation-health.test.ts": 4021,
|
||||
"test/doctor-fix.test.ts": 1301,
|
||||
"test/doctor-frontmatter-partial.test.ts": 2,
|
||||
"test/doctor-home-dir-in-worktree.test.ts": 194,
|
||||
"test/doctor-minions-check.test.ts": 2697,
|
||||
"test/doctor-remote.serial.test.ts": 210,
|
||||
"test/doctor-report-remote.serial.test.ts": 2823,
|
||||
"test/doctor-search-mode.test.ts": 3519,
|
||||
"test/doctor-subagent-health.test.ts": 2972,
|
||||
"test/doctor-v0_37_7_checks.test.ts": 2447,
|
||||
"test/doctor-ze-checks.test.ts": 26042,
|
||||
"test/doctor.test.ts": 46040,
|
||||
"test/domain-aggregators.test.ts": 3512,
|
||||
"test/dream-cli-flags.test.ts": 3,
|
||||
"test/dream.test.ts": 26718,
|
||||
"test/drift-watch.test.ts": 255,
|
||||
"test/dry-fix.test.ts": 442,
|
||||
"test/edge-extractor.test.ts": 252,
|
||||
"test/effective-date.test.ts": 6,
|
||||
"test/embed-backfill-submit.test.ts": 3099,
|
||||
"test/embed-multimodal-batching.test.ts": 11,
|
||||
"test/embed-skip.test.ts": 5,
|
||||
"test/embed-stale.test.ts": 34959,
|
||||
"test/embed.serial.test.ts": 2312,
|
||||
"test/embedding-context.test.ts": 6,
|
||||
"test/embedding-dim-check.test.ts": 4174,
|
||||
"test/embedding-pricing.test.ts": 3,
|
||||
"test/emotional-weight.test.ts": 4,
|
||||
"test/engine-factory.test.ts": 13,
|
||||
"test/engine-find-trajectory.test.ts": 3351,
|
||||
"test/engine-parity-event-type.test.ts": 3251,
|
||||
"test/engine-upsertFile.test.ts": 3801,
|
||||
"test/engine-weight-rounding-integration.test.ts": 2367,
|
||||
"test/enrichable-pack.test.ts": 8,
|
||||
"test/enrichment-service.test.ts": 4,
|
||||
"test/enrichment.test.ts": 3498,
|
||||
"test/entity-resolve-perf.slow.test.ts": 158667,
|
||||
"test/entity-resolve.test.ts": 24029,
|
||||
"test/error-classify.test.ts": 3,
|
||||
"test/errors.test.ts": 18,
|
||||
"test/eval-candidates.test.ts": 18453,
|
||||
"test/eval-capture-scrub.test.ts": 1,
|
||||
"test/eval-capture.test.ts": 6,
|
||||
"test/eval-compare.test.ts": 36,
|
||||
"test/eval-contradictions-auto-supersession.test.ts": 3,
|
||||
"test/eval-contradictions-cache.test.ts": 4601,
|
||||
"test/eval-contradictions-calibration-join.test.ts": 2,
|
||||
"test/eval-contradictions-calibration.test.ts": 12,
|
||||
"test/eval-contradictions-cost-prompt.test.ts": 4223,
|
||||
"test/eval-contradictions-cost.test.ts": 16,
|
||||
"test/eval-contradictions-cross-source.test.ts": 5,
|
||||
"test/eval-contradictions-date-filter.test.ts": 5,
|
||||
"test/eval-contradictions-engine.test.ts": 5226,
|
||||
"test/eval-contradictions-fixture-redact.test.ts": 5,
|
||||
"test/eval-contradictions-integrations.test.ts": 4067,
|
||||
"test/eval-contradictions-judge-errors.test.ts": 3,
|
||||
"test/eval-contradictions-judge.test.ts": 32,
|
||||
"test/eval-contradictions-runner.test.ts": 5998,
|
||||
"test/eval-contradictions-severity.test.ts": 5,
|
||||
"test/eval-contradictions-trends.test.ts": 36220,
|
||||
"test/eval-contradictions/no-valid-until-write.test.ts": 33,
|
||||
"test/eval-cross-modal-batch.test.ts": 141,
|
||||
"test/eval-export.test.ts": 2399,
|
||||
"test/eval-gate.test.ts": 3884,
|
||||
"test/eval-longmemeval.slow.test.ts": 359087,
|
||||
"test/eval-prune.test.ts": 18396,
|
||||
"test/eval-replay-gate.test.ts": 34958,
|
||||
"test/eval-replay-metadata-skip.test.ts": 3430,
|
||||
"test/eval-replay.test.ts": 16,
|
||||
"test/eval-run-all.test.ts": 17,
|
||||
"test/eval-schema-authoring.test.ts": 4,
|
||||
"test/eval-shared-json-repair-shim.test.ts": 6,
|
||||
"test/eval-takes-quality-aggregate.test.ts": 32,
|
||||
"test/eval-takes-quality-boundaries.test.ts": 3193,
|
||||
"test/eval-takes-quality-cli.test.ts": 13,
|
||||
"test/eval-takes-quality-pricing.test.ts": 2,
|
||||
"test/eval-takes-quality-receipt-name.test.ts": 4,
|
||||
"test/eval-takes-quality-receipt-write.test.ts": 3213,
|
||||
"test/eval-takes-quality-regress.test.ts": 14,
|
||||
"test/eval-takes-quality-replay.test.ts": 2890,
|
||||
"test/eval-takes-quality-rubric.test.ts": 3,
|
||||
"test/eval-takes-quality-runner.serial.test.ts": 2642,
|
||||
"test/eval-takes-quality-trend.test.ts": 23553,
|
||||
"test/eval-trajectory.test.ts": 3424,
|
||||
"test/eval-v041_2-scaffolds.test.ts": 23,
|
||||
"test/eval-whoknows.test.ts": 8,
|
||||
"test/eval.test.ts": 2,
|
||||
"test/exit-classification.test.ts": 4,
|
||||
"test/expert-types-pack.test.ts": 60,
|
||||
"test/extract-db.test.ts": 3200,
|
||||
"test/extract-facts-phase.test.ts": 24617,
|
||||
"test/extract-from-fence.test.ts": 6,
|
||||
"test/extract-fs.test.ts": 2618,
|
||||
"test/extract-incremental.test.ts": 4523,
|
||||
"test/extract-source-aware.test.ts": 23970,
|
||||
"test/extract-takes-holder-producer-seam.test.ts": 2469,
|
||||
"test/extract-takes.test.ts": 3153,
|
||||
"test/extract.test.ts": 24,
|
||||
"test/extractable-pack.test.ts": 76,
|
||||
"test/facts-absorb-log.test.ts": 3161,
|
||||
"test/facts-anti-loop.test.ts": 2478,
|
||||
"test/facts-backstop-gating.test.ts": 3278,
|
||||
"test/facts-backstop-integration.test.ts": 23696,
|
||||
"test/facts-backstop.test.ts": 3118,
|
||||
"test/facts-canonicality.test.ts": 3218,
|
||||
"test/facts-classify.test.ts": 3,
|
||||
"test/facts-context-injection.serial.test.ts": 2703,
|
||||
"test/facts-decay.test.ts": 19,
|
||||
"test/facts-doctor-shape.test.ts": 3109,
|
||||
"test/facts-eligibility.test.ts": 5,
|
||||
"test/facts-engine.test.ts": 12954,
|
||||
"test/facts-extract-silent-no-op.test.ts": 4,
|
||||
"test/facts-extract-smoke.test.ts": 6,
|
||||
"test/facts-extract.test.ts": 1,
|
||||
"test/facts-fence-typed.test.ts": 4,
|
||||
"test/facts-fence.test.ts": 5,
|
||||
"test/facts-mcp-allowlist.serial.test.ts": 2633,
|
||||
"test/facts-meta-cache.test.ts": 3117,
|
||||
"test/facts-migration-dim.test.ts": 3329,
|
||||
"test/facts-multi-tenant.test.ts": 23425,
|
||||
"test/facts-queue.test.ts": 604,
|
||||
"test/facts-recall-render.test.ts": 2968,
|
||||
"test/facts-separation-pglite.test.ts": 3260,
|
||||
"test/facts-visibility.test.ts": 2917,
|
||||
"test/fail-improve.test.ts": 132,
|
||||
"test/feature-flags.test.ts": 3864,
|
||||
"test/features.test.ts": 5,
|
||||
"test/fence-extraction.test.ts": 3427,
|
||||
"test/fence-write.test.ts": 2335,
|
||||
"test/file-migration.test.ts": 4,
|
||||
"test/file-resolver.test.ts": 24,
|
||||
"test/file-upload-security.test.ts": 3,
|
||||
"test/files.test.ts": 6,
|
||||
"test/filing-audit.test.ts": 9,
|
||||
"test/find-experts-op.test.ts": 3125,
|
||||
"test/fix-wave-structural.test.ts": 6,
|
||||
"test/founder-scorecard.test.ts": 9,
|
||||
"test/friction-cli.test.ts": 21,
|
||||
"test/friction.test.ts": 29,
|
||||
"test/frontmatter-cli.test.ts": 2936,
|
||||
"test/frontmatter-inference.test.ts": 10,
|
||||
"test/frontmatter-install-hook.test.ts": 56,
|
||||
"test/fuzz/filesystem-validators.test.ts": 87,
|
||||
"test/fuzz/mixed-validators.test.ts": 279,
|
||||
"test/fuzz/pure-validators.test.ts": 220,
|
||||
"test/gateway-embed-model-override.test.ts": 1,
|
||||
"test/gbrain-home-isolation.test.ts": 7,
|
||||
"test/get-brain-identity.test.ts": 36332,
|
||||
"test/git-remote.test.ts": 365,
|
||||
"test/grade-takes-ensemble.test.ts": 30,
|
||||
"test/grade-takes.test.ts": 6,
|
||||
"test/graph-query.test.ts": 3163,
|
||||
"test/gstack-learnings-coupling.test.ts": 15,
|
||||
"test/handlers-embed-backfill.test.ts": 2978,
|
||||
"test/handlers.test.ts": 25084,
|
||||
"test/helpers/schema-diff-indexes.test.ts": 4,
|
||||
"test/helpers/schema-diff.test.ts": 2,
|
||||
"test/helpers/with-env.test.ts": 19,
|
||||
"test/http-transport.test.ts": 100,
|
||||
"test/hybrid-meta.test.ts": 3314,
|
||||
"test/hybrid-search-lite.serial.test.ts": 2849,
|
||||
"test/import-checkpoint.test.ts": 5,
|
||||
"test/import-file-content-sanity.test.ts": 3363,
|
||||
"test/import-file.test.ts": 87,
|
||||
"test/import-image-file.test.ts": 3539,
|
||||
"test/import-resume.test.ts": 2811,
|
||||
"test/import-source-id.test.ts": 3149,
|
||||
"test/import-walker.test.ts": 1,
|
||||
"test/incremental-chunking.test.ts": 3212,
|
||||
"test/infer-type-pack.test.ts": 16,
|
||||
"test/ingestion/daemon.test.ts": 39,
|
||||
"test/ingestion/dedup.test.ts": 5,
|
||||
"test/ingestion/gstack-learnings.test.ts": 8,
|
||||
"test/ingestion/ingest-capture.test.ts": 53213,
|
||||
"test/ingestion/markdown-greenfield.test.ts": 13,
|
||||
"test/ingestion/migration-mode.test.ts": 8,
|
||||
"test/ingestion/put-page-write-through.test.ts": 38741,
|
||||
"test/ingestion/skillpack-load.test.ts": 15,
|
||||
"test/ingestion/sources/file-watcher.test.ts": 1007,
|
||||
"test/ingestion/sources/inbox-folder.test.ts": 518,
|
||||
"test/ingestion/test-harness.test.ts": 68,
|
||||
"test/ingestion/types.test.ts": 9,
|
||||
"test/init-env-detection.test.ts": 11,
|
||||
"test/init-mcp-only.test.ts": 4380,
|
||||
"test/init-migrate-only.test.ts": 7098,
|
||||
"test/init-mode-picker.test.ts": 2440,
|
||||
"test/init-provider-picker.test.ts": 2,
|
||||
"test/insert-facts-batch.test.ts": 2976,
|
||||
"test/integrations-install.test.ts": 183,
|
||||
"test/integrations.test.ts": 24,
|
||||
"test/integrity.test.ts": 2678,
|
||||
"test/intent-weights.test.ts": 4,
|
||||
"test/jobs-watch-snapshot.test.ts": 12,
|
||||
"test/language-manifest.test.ts": 17,
|
||||
"test/lease-cap-controller.test.ts": 3161,
|
||||
"test/lens-pack-manifests.test.ts": 13,
|
||||
"test/levenshtein.test.ts": 3,
|
||||
"test/link-extraction-code-refs.test.ts": 4,
|
||||
"test/link-extraction.test.ts": 25,
|
||||
"test/link-inference-pack.test.ts": 46,
|
||||
"test/lint-content-sanity.test.ts": 36,
|
||||
"test/lint-frontmatter.test.ts": 15,
|
||||
"test/lint.test.ts": 1,
|
||||
"test/list-all-sources.test.ts": 39599,
|
||||
"test/llm-intent-escalation.test.ts": 6,
|
||||
"test/llm-intent-hybrid-integration.serial.test.ts": 3246,
|
||||
"test/loadConfig-merge.test.ts": 4,
|
||||
"test/longmemeval-extract.test.ts": 2991,
|
||||
"test/longmemeval-intent.test.ts": 3,
|
||||
"test/longmemeval-sanitize.test.ts": 5,
|
||||
"test/longmemeval-trajectory-routing.test.ts": 12926,
|
||||
"test/markdown-serializer.test.ts": 8,
|
||||
"test/markdown-validation.test.ts": 3,
|
||||
"test/markdown.test.ts": 27,
|
||||
"test/mcp-client-hardening.test.ts": 305,
|
||||
"test/mcp-client.test.ts": 65,
|
||||
"test/mcp-dispatch-summarize.test.ts": 6,
|
||||
"test/mcp-eval-capture.test.ts": 25246,
|
||||
"test/mcp-tool-defs.test.ts": 15,
|
||||
"test/metric-glossary.test.ts": 16,
|
||||
"test/migrate-extensions.test.ts": 3,
|
||||
"test/migrate.test.ts": 102260,
|
||||
"test/migration-orchestrator-v0_21_0.test.ts": 7,
|
||||
"test/migration-orchestrator-v0_31_0.test.ts": 12960,
|
||||
"test/migration-resume.test.ts": 9,
|
||||
"test/migration-v0-29-1.serial.test.ts": 3273,
|
||||
"test/migrations-cjk-wave.test.ts": 3280,
|
||||
"test/migrations-registry.test.ts": 3,
|
||||
"test/migrations-v0_11_0.test.ts": 14,
|
||||
"test/migrations-v0_12_0.test.ts": 6,
|
||||
"test/migrations-v0_12_2.test.ts": 3,
|
||||
"test/migrations-v0_13_0.test.ts": 1,
|
||||
"test/migrations-v0_13_1.test.ts": 23,
|
||||
"test/migrations-v0_14_0.test.ts": 20,
|
||||
"test/migrations-v0_16_0.test.ts": 1,
|
||||
"test/migrations-v0_19_0.test.ts": 20826,
|
||||
"test/migrations-v0_21_0.test.ts": 2,
|
||||
"test/migrations-v0_22_4.test.ts": 1,
|
||||
"test/migrations-v0_27_1.test.ts": 3248,
|
||||
"test/migrations-v0_32_2.test.ts": 2510,
|
||||
"test/migrations-v48-takes-weight-backfill.test.ts": 2501,
|
||||
"test/migrations-v94.test.ts": 32374,
|
||||
"test/minions-lease-full-retry.test.ts": 3031,
|
||||
"test/minions-quiet-hours.test.ts": 3341,
|
||||
"test/minions-shell-inherit.test.ts": 24,
|
||||
"test/minions-shell-redact.test.ts": 19,
|
||||
"test/minions-shell-validate.test.ts": 7,
|
||||
"test/minions-shell.test.ts": 3268,
|
||||
"test/minions.test.ts": 15231,
|
||||
"test/minions/agent-audit.test.ts": 9,
|
||||
"test/minions/budget-meter.test.ts": 44657,
|
||||
"test/mode-switch-ux.test.ts": 3698,
|
||||
"test/model-config.serial.test.ts": 101,
|
||||
"test/mounts-cache.test.ts": 21,
|
||||
"test/mounts-cli.test.ts": 20,
|
||||
"test/multi-source-drift.test.ts": 2479,
|
||||
"test/multi-source-integration.test.ts": 3120,
|
||||
"test/nightly-quality-probe.test.ts": 50,
|
||||
"test/notability-eval.test.ts": 36,
|
||||
"test/nudge.test.ts": 16,
|
||||
"test/oauth-confidential-client.test.ts": 3221,
|
||||
"test/oauth-scope-probe.test.ts": 112,
|
||||
"test/oauth.test.ts": 2151,
|
||||
"test/op-checkpoint.test.ts": 5333,
|
||||
"test/openai-compat-multimodal.test.ts": 6,
|
||||
"test/operation-context-sourceid-required.test.ts": 10,
|
||||
"test/operations-allow-list.test.ts": 4,
|
||||
"test/operations-descriptions.test.ts": 11,
|
||||
"test/operations-embedding-column.test.ts": 30,
|
||||
"test/operations-find-trajectory.test.ts": 3419,
|
||||
"test/operations-schema-pack.test.ts": 5842,
|
||||
"test/operations-trust-boundary.test.ts": 43020,
|
||||
"test/orphans.test.ts": 16921,
|
||||
"test/page-lock.test.ts": 266,
|
||||
"test/page-type-exhaustive.test.ts": 17,
|
||||
"test/pages-soft-delete.test.ts": 15271,
|
||||
"test/parallel.test.ts": 108,
|
||||
"test/parent-scope.test.ts": 23673,
|
||||
"test/parity.test.ts": 16,
|
||||
"test/performfullsync-source-id.test.ts": 2869,
|
||||
"test/pglite-engine.test.ts": 33476,
|
||||
"test/pglite-lock.test.ts": 1007,
|
||||
"test/phantom-redirect-engine-parity.test.ts": 3605,
|
||||
"test/phantom-redirect-per-source-lock.test.ts": 3,
|
||||
"test/phantom-redirect.test.ts": 7303,
|
||||
"test/phase-scope-coverage.test.ts": 3,
|
||||
"test/plugin-loader.test.ts": 12,
|
||||
"test/post-install-advisory.test.ts": 15,
|
||||
"test/post-write-lint.test.ts": 4180,
|
||||
"test/postgres-engine.test.ts": 32,
|
||||
"test/preferences.test.ts": 14,
|
||||
"test/privacy-script-wired.test.ts": 3,
|
||||
"test/privacy-strip-and-forget.test.ts": 2762,
|
||||
"test/progress-tail.test.ts": 3,
|
||||
"test/progress.test.ts": 215,
|
||||
"test/propose-takes.test.ts": 15,
|
||||
"test/providers.test.ts": 17,
|
||||
"test/public-exports.test.ts": 35,
|
||||
"test/publish.test.ts": 128,
|
||||
"test/put-page-namespace.test.ts": 4,
|
||||
"test/put-page-provenance.test.ts": 24839,
|
||||
"test/qualified-names.test.ts": 4,
|
||||
"test/query-cache-gate.test.ts": 31003,
|
||||
"test/query-cache-knobs-hash.test.ts": 3122,
|
||||
"test/query-cache.test.ts": 2565,
|
||||
"test/query-image-flag.serial.test.ts": 2965,
|
||||
"test/query-intent-legacy.test.ts": 8,
|
||||
"test/query-intent.test.ts": 6,
|
||||
"test/query-sanitization.test.ts": 4,
|
||||
"test/queue-child-done.test.ts": 2548,
|
||||
"test/rate-leases-uncapped.test.ts": 2884,
|
||||
"test/rate-leases.test.ts": 2959,
|
||||
"test/readme-hero-anchors.test.ts": 0,
|
||||
"test/recall-extensions.test.ts": 4668,
|
||||
"test/recall-footer.test.ts": 17,
|
||||
"test/recall-rollup.test.ts": 84,
|
||||
"test/recency-decay.test.ts": 5,
|
||||
"test/recompute-emotional-weight.test.ts": 2,
|
||||
"test/reconcile-links.serial.test.ts": 2702,
|
||||
"test/regression-strict-source-id.test.ts": 4,
|
||||
"test/regression-v0_16_4.test.ts": 28,
|
||||
"test/regressions/gbrain-base-equivalence.test.ts": 12,
|
||||
"test/regressions/v0.36.1.0-iron-rule.test.ts": 3,
|
||||
"test/regressions/v0_36_frontier_cap.test.ts": 27151,
|
||||
"test/regressions/v0_40_2_0-trajectory-backcompat.test.ts": 27576,
|
||||
"test/reindex-code-max-cost.serial.test.ts": 2636,
|
||||
"test/reindex-code-model-source.serial.test.ts": 2782,
|
||||
"test/reindex-code-nudge.serial.test.ts": 2710,
|
||||
"test/reindex-code.test.ts": 38123,
|
||||
"test/reindex-frontmatter-connect.test.ts": 18383,
|
||||
"test/reindex.test.ts": 19003,
|
||||
"test/remediation-step.test.ts": 25,
|
||||
"test/repair-jsonb.test.ts": 3,
|
||||
"test/repo-root.test.ts": 14,
|
||||
"test/report.test.ts": 2,
|
||||
"test/repos-alias.test.ts": 3163,
|
||||
"test/rerank-audit.test.ts": 24,
|
||||
"test/resolve-prepare.test.ts": 11,
|
||||
"test/resolver-merge.test.ts": 5,
|
||||
"test/resolver.test.ts": 58,
|
||||
"test/resolvers.test.ts": 4130,
|
||||
"test/restart-sweep.test.ts": 42,
|
||||
"test/retrieval-upgrade-planner.test.ts": 8375,
|
||||
"test/retry-matcher.test.ts": 2,
|
||||
"test/routing-eval-cli.test.ts": 1118,
|
||||
"test/routing-eval.test.ts": 34,
|
||||
"test/salience.test.ts": 2,
|
||||
"test/scenarios.test.ts": 29,
|
||||
"test/schema-bootstrap-coverage.test.ts": 6449,
|
||||
"test/schema-cli-contract.test.ts": 4,
|
||||
"test/schema-cli.test.ts": 3375,
|
||||
"test/schema-pack-best-effort.test.ts": 9,
|
||||
"test/schema-pack-lint-rules.test.ts": 11,
|
||||
"test/schema-pack-load-active.test.ts": 43,
|
||||
"test/schema-pack-loader.test.ts": 10,
|
||||
"test/schema-pack-manifest-v041_2.test.ts": 17,
|
||||
"test/schema-pack-mutate-audit.test.ts": 8,
|
||||
"test/schema-pack-mutate.test.ts": 80,
|
||||
"test/schema-pack-pack-lock.test.ts": 65,
|
||||
"test/schema-pack-query-cache-invalidator.test.ts": 22822,
|
||||
"test/schema-pack-registry-reload.test.ts": 40,
|
||||
"test/schema-pack-registry.test.ts": 52,
|
||||
"test/schema-pack-stats.test.ts": 43032,
|
||||
"test/schema-pack-sync.test.ts": 5580,
|
||||
"test/schema-pack-trust-boundary.test.ts": 8,
|
||||
"test/schema-verify.test.ts": 7,
|
||||
"test/scope-agent-isolation.test.ts": 30,
|
||||
"test/scope-normalize.test.ts": 24,
|
||||
"test/scope.test.ts": 5,
|
||||
"test/scripts/check-proposal-pii.test.ts": 511,
|
||||
"test/scripts/check-test-isolation.test.ts": 223,
|
||||
"test/scripts/run-unit-parallel.test.ts": 427,
|
||||
"test/scripts/run-unit-shard.test.ts": 131,
|
||||
"test/scripts/serial-files.test.ts": 41,
|
||||
"test/scripts/test-shard.slow.test.ts": 32161,
|
||||
"test/search-by-image-op.test.ts": 3869,
|
||||
"test/search-image-column.test.ts": 30089,
|
||||
"test/search-lang-symbol-kind.test.ts": 2556,
|
||||
"test/search-limit.test.ts": 2909,
|
||||
"test/search-mode.test.ts": 8,
|
||||
"test/search-telemetry.test.ts": 25699,
|
||||
"test/search-types-filter.test.ts": 3023,
|
||||
"test/search.test.ts": 9,
|
||||
"test/search/attribution-stamping.test.ts": 5,
|
||||
"test/search/embedding-column.serial.test.ts": 109,
|
||||
"test/search/explain-formatter.test.ts": 13,
|
||||
"test/search/graph-signals-wire-integration.test.ts": 28358,
|
||||
"test/search/graph-signals.test.ts": 14,
|
||||
"test/search/hybrid-reranker-integration.serial.test.ts": 2905,
|
||||
"test/search/knobs-hash-reranker.test.ts": 5,
|
||||
"test/search/rerank.test.ts": 23,
|
||||
"test/search/search-stats-graph-signals.test.ts": 23476,
|
||||
"test/seed-pglite.test.ts": 12966,
|
||||
"test/select-e2e.test.ts": 11,
|
||||
"test/self-fix.test.ts": 28842,
|
||||
"test/serve-http-bootstrap-token.test.ts": 2,
|
||||
"test/serve-http-health.test.ts": 301,
|
||||
"test/serve-stdio-lifecycle.test.ts": 1252,
|
||||
"test/setup-branching.test.ts": 15,
|
||||
"test/skill-brain-first.test.ts": 9,
|
||||
"test/skill-manifest.test.ts": 25,
|
||||
"test/skillify-check.test.ts": 554,
|
||||
"test/skillify-scaffold.test.ts": 31,
|
||||
"test/skillpack-apply-hunks.test.ts": 19,
|
||||
"test/skillpack-bootstrap-display.test.ts": 4,
|
||||
"test/skillpack-changed-since-version.test.ts": 361,
|
||||
"test/skillpack-check.test.ts": 4691,
|
||||
"test/skillpack-copy.test.ts": 11,
|
||||
"test/skillpack-endorse.test.ts": 114,
|
||||
"test/skillpack-frontmatter-sources.test.ts": 10,
|
||||
"test/skillpack-harvest-lint.test.ts": 5,
|
||||
"test/skillpack-harvest.test.ts": 35,
|
||||
"test/skillpack-init-pack.test.ts": 50,
|
||||
"test/skillpack-install.test.ts": 51,
|
||||
"test/skillpack-manifest-v1.test.ts": 4,
|
||||
"test/skillpack-migrate-fence.test.ts": 10,
|
||||
"test/skillpack-reference-apply.test.ts": 13,
|
||||
"test/skillpack-reference-pack-is-ten.test.ts": 14,
|
||||
"test/skillpack-reference.test.ts": 29,
|
||||
"test/skillpack-registry-client.test.ts": 27,
|
||||
"test/skillpack-registry-schema.test.ts": 13,
|
||||
"test/skillpack-remote-source.test.ts": 65,
|
||||
"test/skillpack-rubric-doctor.test.ts": 45,
|
||||
"test/skillpack-scaffold-third-party.test.ts": 17,
|
||||
"test/skillpack-scaffold.test.ts": 44,
|
||||
"test/skillpack-scrub-legacy.test.ts": 51,
|
||||
"test/skillpack-state.test.ts": 30,
|
||||
"test/skillpack-tarball.test.ts": 132,
|
||||
"test/skillpack-trust-prompt.test.ts": 9,
|
||||
"test/skills-conformance.test.ts": 25,
|
||||
"test/slug-validation.test.ts": 12,
|
||||
"test/sort-newest-first.test.ts": 120,
|
||||
"test/source-config-redact.test.ts": 1,
|
||||
"test/source-health.test.ts": 2940,
|
||||
"test/source-id-routing.test.ts": 3687,
|
||||
"test/source-id-tx-regression.test.ts": 3200,
|
||||
"test/source-id.test.ts": 4,
|
||||
"test/source-resolver-silent-fallback.test.ts": 40514,
|
||||
"test/source-resolver-with-tier.test.ts": 1,
|
||||
"test/source-resolver.test.ts": 27,
|
||||
"test/sources-load.test.ts": 4120,
|
||||
"test/sources-mcp.test.ts": 41488,
|
||||
"test/sources-ops.test.ts": 4235,
|
||||
"test/sources-resync-recovery.test.ts": 40765,
|
||||
"test/sources-set-cr-mode.test.ts": 37772,
|
||||
"test/sources-webhook.test.ts": 20,
|
||||
"test/sources.test.ts": 11,
|
||||
"test/spawn-helpers.test.ts": 18,
|
||||
"test/sql-query.test.ts": 2327,
|
||||
"test/sql-ranking.test.ts": 17,
|
||||
"test/ssrf-validate.test.ts": 6,
|
||||
"test/storage-backfill.test.ts": 4,
|
||||
"test/storage-config.test.ts": 8,
|
||||
"test/storage-export.test.ts": 3170,
|
||||
"test/storage-pglite.test.ts": 23978,
|
||||
"test/storage-status.test.ts": 12,
|
||||
"test/storage-sync.test.ts": 27,
|
||||
"test/storage.test.ts": 146,
|
||||
"test/stub-guard-audit.test.ts": 27,
|
||||
"test/subagent-aggregator.test.ts": 5,
|
||||
"test/subagent-audit.test.ts": 29,
|
||||
"test/subagent-handler.test.ts": 3178,
|
||||
"test/subagent-prompt-too-long.test.ts": 17,
|
||||
"test/subagent-transcript.test.ts": 3276,
|
||||
"test/subagent-v1-v2-shim.test.ts": 4830,
|
||||
"test/submit-agent.test.ts": 4992,
|
||||
"test/supabase-admin.test.ts": 0,
|
||||
"test/supervisor-audit.test.ts": 8,
|
||||
"test/supervisor-tini.test.ts": 1,
|
||||
"test/supervisor.test.ts": 326,
|
||||
"test/svg-renderer.test.ts": 7,
|
||||
"test/sync-all-parallel.test.ts": 6,
|
||||
"test/sync-classifier-widening.test.ts": 4,
|
||||
"test/sync-concurrency.test.ts": 6,
|
||||
"test/sync-cost-preview.test.ts": 115,
|
||||
"test/sync-failures.test.ts": 18,
|
||||
"test/sync-parallel.test.ts": 22640,
|
||||
"test/sync-strategy.test.ts": 15,
|
||||
"test/sync-trigger-cli.test.ts": 3100,
|
||||
"test/sync-walker-submodule.test.ts": 5,
|
||||
"test/sync-walker-symlink.test.ts": 12,
|
||||
"test/sync.test.ts": 5841,
|
||||
"test/synth-enabled-default.test.ts": 3,
|
||||
"test/system-prompt.test.ts": 8,
|
||||
"test/take-forecast.test.ts": 7,
|
||||
"test/takes-engine.test.ts": 3302,
|
||||
"test/takes-fence-read-ops.serial.test.ts": 2720,
|
||||
"test/takes-fence.test.ts": 132,
|
||||
"test/takes-holder-semantics.test.ts": 5,
|
||||
"test/takes-holder-validation.test.ts": 13,
|
||||
"test/takes-mcp-allowlist.serial.test.ts": 2721,
|
||||
"test/takes-resolution.test.ts": 5,
|
||||
"test/takes-weight-rounding.test.ts": 6,
|
||||
"test/thin-client-routing-audit.test.ts": 5,
|
||||
"test/thin-client-upgrade-prompt.test.ts": 23,
|
||||
"test/think-ab.test.ts": 18,
|
||||
"test/think-entity-extract.test.ts": 4,
|
||||
"test/think-gateway-adapter.test.ts": 33,
|
||||
"test/think-intent.test.ts": 8,
|
||||
"test/think-pipeline.serial.test.ts": 2723,
|
||||
"test/think-sanitize-trajectory.test.ts": 7,
|
||||
"test/think-trajectory-injection.test.ts": 3346,
|
||||
"test/think-with-calibration.test.ts": 3,
|
||||
"test/timing-safe.test.ts": 17,
|
||||
"test/token-budget.test.ts": 3,
|
||||
"test/trajectory-format.test.ts": 4,
|
||||
"test/transcript-capture.test.ts": 246,
|
||||
"test/transcription.test.ts": 20,
|
||||
"test/transcripts.test.ts": 20,
|
||||
"test/traverse-graph-dedup.test.ts": 2933,
|
||||
"test/trust-boundary-contract.test.ts": 24,
|
||||
"test/two-pass.test.ts": 3137,
|
||||
"test/undo-wave.test.ts": 5,
|
||||
"test/unified-multimodal.serial.test.ts": 3286,
|
||||
"test/upgrade-checkpoint.serial.test.ts": 109,
|
||||
"test/upgrade-reembed-prompt.test.ts": 3219,
|
||||
"test/upgrade-reference-sweep.test.ts": 56,
|
||||
"test/upgrade.serial.test.ts": 723,
|
||||
"test/url-redact.test.ts": 2,
|
||||
"test/utils.test.ts": 5,
|
||||
"test/v0_29-tool-surfaces.test.ts": 4,
|
||||
"test/v0_37_fix_wave.serial.test.ts": 211,
|
||||
"test/v81-v82-smoke.test.ts": 3160,
|
||||
"test/vector-index-lifecycle.test.ts": 5,
|
||||
"test/voice-gate.test.ts": 9,
|
||||
"test/voyage-multimodal.test.ts": 22,
|
||||
"test/voyage-response-cap.test.ts": 1,
|
||||
"test/wait-for-completion.test.ts": 3626,
|
||||
"test/whoami.test.ts": 3,
|
||||
"test/whoknows-doctor.test.ts": 70,
|
||||
"test/whoknows.test.ts": 3,
|
||||
"test/worker-rss.test.ts": 3,
|
||||
"test/worker-shutdown-disconnect.test.ts": 3260,
|
||||
"test/writer.test.ts": 37365,
|
||||
"test/yaml-lite.test.ts": 9,
|
||||
"test/ze-switch-cli.test.ts": 4227,
|
||||
"test/zombie-reap.test.ts": 3
|
||||
}
|
||||
@@ -8,19 +8,26 @@
|
||||
*
|
||||
* v0.26.4 split: `bun run test` is now the fast parallel loop and does
|
||||
* NOT chain pre-checks; the privacy gate moved to `bun run verify`,
|
||||
* which CI's test.yml runs on shard 1 before the matrix fans out.
|
||||
* Regression guard now asserts both: (1) verify chains check:privacy,
|
||||
* (2) CI workflow's pre-test gate calls `bun run verify`. Together those
|
||||
* guarantee the privacy check runs before any merge.
|
||||
* which CI's test.yml runs as its own job before the matrix fans out.
|
||||
*
|
||||
* v0.41.4+ wave: `bun run verify` now delegates to
|
||||
* scripts/run-verify-parallel.sh which fans out all 20 checks in
|
||||
* parallel via & + wait. The privacy check is one entry in that
|
||||
* script's CHECKS[] array. Regression guard updated to follow the
|
||||
* indirection: (1) verify points at the parallel dispatcher,
|
||||
* (2) the dispatcher's CHECKS array contains check:privacy,
|
||||
* (3) CI workflow's verify job calls `bun run verify`.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'bun:test';
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { resolve } from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
const REPO_ROOT = resolve(import.meta.dir, '..');
|
||||
const PACKAGE_JSON = resolve(REPO_ROOT, 'package.json');
|
||||
const PRIVACY_SCRIPT = resolve(REPO_ROOT, 'scripts/check-privacy.sh');
|
||||
const VERIFY_DISPATCHER = resolve(REPO_ROOT, 'scripts/run-verify-parallel.sh');
|
||||
const TEST_WORKFLOW = resolve(REPO_ROOT, '.github/workflows/test.yml');
|
||||
|
||||
describe('check-privacy.sh CI wiring', () => {
|
||||
@@ -31,10 +38,27 @@ describe('check-privacy.sh CI wiring', () => {
|
||||
expect((stat.mode & 0o100) !== 0).toBe(true);
|
||||
});
|
||||
|
||||
it('package.json "verify" script chains check:privacy', () => {
|
||||
it('package.json "verify" script delegates to run-verify-parallel.sh', () => {
|
||||
const pkg = JSON.parse(readFileSync(PACKAGE_JSON, 'utf-8'));
|
||||
expect(typeof pkg.scripts?.verify).toBe('string');
|
||||
expect(pkg.scripts.verify).toContain('check:privacy');
|
||||
// verify body is now `bash scripts/run-verify-parallel.sh`. The
|
||||
// direct check:privacy substring assertion broke when the && chain
|
||||
// was replaced with the parallel dispatcher. Follow the indirection.
|
||||
expect(pkg.scripts.verify).toContain('run-verify-parallel.sh');
|
||||
});
|
||||
|
||||
it('run-verify-parallel.sh dispatches check:privacy', () => {
|
||||
expect(existsSync(VERIFY_DISPATCHER)).toBe(true);
|
||||
// The dispatcher exposes --dry-list which prints one check name per
|
||||
// line. Authoritative check than substring-grepping the script body
|
||||
// (which could pass on a commented-out entry).
|
||||
const r = spawnSync('bash', [VERIFY_DISPATCHER, '--dry-list'], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
expect(r.status).toBe(0);
|
||||
const checks = r.stdout.trim().split('\n');
|
||||
expect(checks).toContain('check:privacy');
|
||||
});
|
||||
|
||||
it('package.json "check:privacy" alias points at the script', () => {
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
/**
|
||||
* ci-cache-hash.test.ts — adversarial coverage of the auto-cache hash.
|
||||
*
|
||||
* The cache key controls false-pass risk. If a test-affecting file is
|
||||
* accidentally deny-listed (or a file change fails to invalidate the
|
||||
* hash), broken code ships under a green CI check. This suite is the
|
||||
* primary safety net.
|
||||
*
|
||||
* Strategy: each test spins up a synthetic git repo, copies the
|
||||
* ci-cache-hash.sh script in, populates a small file set covering the
|
||||
* deny-list edges, runs the script, modifies one file, runs again,
|
||||
* asserts hash behavior.
|
||||
*
|
||||
* Critical invariants (the false-pass guards):
|
||||
* - CLAUDE.md edit → DIFFERENT hash
|
||||
* - AGENTS.md edit → DIFFERENT hash
|
||||
* - skills/foo/SKILL.md edit → DIFFERENT hash
|
||||
* - src/core/db.ts edit → DIFFERENT hash
|
||||
* - test/foo.test.ts edit → DIFFERENT hash
|
||||
*
|
||||
* Safe deny-list invariants:
|
||||
* - CHANGELOG.md edit → same hash
|
||||
* - README.md edit → same hash
|
||||
* - docs/guide.md edit → same hash
|
||||
* - TODOS.md edit → same hash
|
||||
* - LICENSE edit → same hash
|
||||
*/
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import {
|
||||
cpSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
const REPO_ROOT = resolve(import.meta.dir, "..", "..");
|
||||
const SCRIPT_SRC = resolve(REPO_ROOT, "scripts/ci-cache-hash.sh");
|
||||
|
||||
interface Sandbox {
|
||||
dir: string;
|
||||
scriptPath: string;
|
||||
}
|
||||
|
||||
function makeSandbox(files: Record<string, string>): Sandbox {
|
||||
const dir = mkdtempSync(join(tmpdir(), "ci-cache-hash-"));
|
||||
// Init git repo
|
||||
execFileSync("git", ["init", "--quiet"], { cwd: dir });
|
||||
execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: dir });
|
||||
execFileSync("git", ["config", "user.name", "Test"], { cwd: dir });
|
||||
|
||||
// Copy script under scripts/ so cd "$(dirname "$0")/.." lands at sandbox root
|
||||
mkdirSync(join(dir, "scripts"), { recursive: true });
|
||||
cpSync(SCRIPT_SRC, join(dir, "scripts/ci-cache-hash.sh"));
|
||||
|
||||
// Write each fixture file under its declared path
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
const abs = join(dir, path);
|
||||
mkdirSync(resolve(abs, ".."), { recursive: true });
|
||||
writeFileSync(abs, content);
|
||||
}
|
||||
|
||||
// Stage + commit so git ls-files returns them
|
||||
execFileSync("git", ["add", "-A"], { cwd: dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", "fixture"], { cwd: dir });
|
||||
|
||||
return { dir, scriptPath: join(dir, "scripts/ci-cache-hash.sh") };
|
||||
}
|
||||
|
||||
function hash(s: Sandbox): string {
|
||||
const r = spawnSync("bash", [s.scriptPath], {
|
||||
cwd: s.dir,
|
||||
encoding: "utf8",
|
||||
});
|
||||
if (r.status !== 0) {
|
||||
throw new Error(`ci-cache-hash exit ${r.status}: ${r.stderr}`);
|
||||
}
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
function modify(s: Sandbox, path: string, content: string) {
|
||||
writeFileSync(join(s.dir, path), content);
|
||||
// The script reads `git ls-files -s` which reflects the INDEX, not
|
||||
// the working tree. Stage + commit so the change is visible to the
|
||||
// hash. This matches what CI sees on a checked-out PR (committed
|
||||
// tree, not working tree).
|
||||
execFileSync("git", ["add", path], { cwd: s.dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", `modify ${path}`], {
|
||||
cwd: s.dir,
|
||||
});
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Baseline fixtures used across most cases.
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
const BASELINE_FILES: Record<string, string> = {
|
||||
"CHANGELOG.md": "# Changelog\n\n## [1.0.0]\n",
|
||||
"TODOS.md": "- [ ] thing\n",
|
||||
"README.md": "# Project\n",
|
||||
"LICENSE": "MIT\n",
|
||||
"CLAUDE.md": "# CLAUDE.md\n\nproject instructions\n",
|
||||
"AGENTS.md": "# AGENTS.md\n\nopenclaw entry\n",
|
||||
"package.json": '{"name":"test","version":"0.0.0"}\n',
|
||||
"bun.lock": "lockfile-v1\n",
|
||||
"tsconfig.json": '{"compilerOptions":{"strict":true}}\n',
|
||||
"src/core/db.ts": "export const db = 1;\n",
|
||||
"src/cli.ts": "console.log('hi');\n",
|
||||
"test/foo.test.ts": "import {test} from 'bun:test';\ntest('x', () => {});\n",
|
||||
"scripts/check-x.sh": "#!/bin/bash\necho ok\n",
|
||||
".github/workflows/test.yml": "name: Test\n",
|
||||
"skills/foo/SKILL.md": "---\nname: foo\n---\nFoo skill\n",
|
||||
"docs/guide.md": "# Guide\n",
|
||||
"docs/sub/notes.md": "Notes\n",
|
||||
"docs/raw.txt": "raw text\n",
|
||||
};
|
||||
|
||||
describe("ci-cache-hash.sh — determinism", () => {
|
||||
let sb: Sandbox;
|
||||
beforeAll(() => {
|
||||
sb = makeSandbox(BASELINE_FILES);
|
||||
});
|
||||
afterAll(() => {
|
||||
if (sb) rmSync(sb.dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("same tree → same hash (run twice)", () => {
|
||||
const h1 = hash(sb);
|
||||
const h2 = hash(sb);
|
||||
expect(h1).toBe(h2);
|
||||
expect(h1).toMatch(/^[0-9a-f]{16}$/);
|
||||
});
|
||||
|
||||
it("hash is 16-char lowercase hex", () => {
|
||||
const h = hash(sb);
|
||||
expect(h).toMatch(/^[0-9a-f]{16}$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ci-cache-hash.sh — CRITICAL false-pass guards (must invalidate)", () => {
|
||||
// Each test gets its own sandbox so modifications don't leak across cases.
|
||||
|
||||
function withSandbox(test: (sb: Sandbox) => void) {
|
||||
const sb = makeSandbox(BASELINE_FILES);
|
||||
try {
|
||||
test(sb);
|
||||
} finally {
|
||||
rmSync(sb.dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it("CLAUDE.md edit MUST change hash (8+ test files reference it)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "CLAUDE.md", "# CLAUDE.md\n\nNEW INSTRUCTIONS\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("AGENTS.md edit MUST change hash (resolver tests read it)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "AGENTS.md", "# AGENTS.md\n\nNEW DISPATCH\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("skills/foo/SKILL.md edit MUST change hash (skill conformance tests)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "skills/foo/SKILL.md", "---\nname: foo\n---\nNEW SKILL BODY\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("src/core/db.ts edit MUST change hash", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "src/core/db.ts", "export const db = 2;\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("test/foo.test.ts edit MUST change hash", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(
|
||||
sb,
|
||||
"test/foo.test.ts",
|
||||
"import {test} from 'bun:test';\ntest('y', () => {});\n",
|
||||
);
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("package.json edit MUST change hash", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "package.json", '{"name":"test","version":"0.0.1"}\n');
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("bun.lock edit MUST change hash (dependency drift catches false-pass)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "bun.lock", "lockfile-v2\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it(".github/workflows/test.yml edit MUST change hash (CI shape change is test-affecting)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, ".github/workflows/test.yml", "name: Test\n# changed\n");
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("ci-cache-hash.sh — SAFE deny-list invariants (must NOT invalidate)", () => {
|
||||
function withSandbox(test: (sb: Sandbox) => void) {
|
||||
const sb = makeSandbox(BASELINE_FILES);
|
||||
try {
|
||||
test(sb);
|
||||
} finally {
|
||||
rmSync(sb.dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it("CHANGELOG.md edit produces SAME hash (deny-listed)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "CHANGELOG.md", "# Changelog\n\n## [1.1.0]\nnew release\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("README.md edit produces SAME hash (deny-listed)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "README.md", "# Project\n\nupdated tagline\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("TODOS.md edit produces SAME hash (deny-listed)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "TODOS.md", "- [x] thing\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("LICENSE edit produces SAME hash (deny-listed)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "LICENSE", "Apache-2.0\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("docs/guide.md edit produces SAME hash (deny-listed via docs/**/*.md)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "docs/guide.md", "# Guide v2\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("docs/sub/notes.md edit produces SAME hash (nested docs match)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "docs/sub/notes.md", "Updated notes\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("docs/raw.txt edit produces SAME hash (deny-listed via docs/**/*.txt)", () => {
|
||||
withSandbox((sb) => {
|
||||
const before = hash(sb);
|
||||
modify(sb, "docs/raw.txt", "new raw text\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("ci-cache-hash.sh — edge cases", () => {
|
||||
function withSandbox(
|
||||
files: Record<string, string>,
|
||||
test: (sb: Sandbox) => void,
|
||||
) {
|
||||
const sb = makeSandbox(files);
|
||||
try {
|
||||
test(sb);
|
||||
} finally {
|
||||
rmSync(sb.dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
it("untracked file does NOT affect hash (git ls-files only)", () => {
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
const before = hash(sb);
|
||||
// Write a new file but don't `git add` it.
|
||||
writeFileSync(join(sb.dir, "src/untracked.ts"), "const x = 1;\n");
|
||||
const after = hash(sb);
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("rename detection: same content, new path → DIFFERENT hash", () => {
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
const before = hash(sb);
|
||||
execFileSync("git", ["mv", "src/cli.ts", "src/main.ts"], { cwd: sb.dir });
|
||||
const after = hash(sb);
|
||||
// Same content, new path. Our hash includes the path, so it changes.
|
||||
// This is correct: a rename can absolutely affect test outcomes
|
||||
// (imports change, file resolution changes, etc.).
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("new file under unknown path (e.g. prompts/x.md) DOES affect hash (deny-list default)", () => {
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
const before = hash(sb);
|
||||
// Add a NEW file in a path that isn't deny-listed and didn't exist
|
||||
// before. Default behavior: include in hash. Closes the "missed file
|
||||
// type" false-pass class (e.g. someone adds `prompts/*.md` that
|
||||
// tests read).
|
||||
mkdirSync(join(sb.dir, "prompts"), { recursive: true });
|
||||
writeFileSync(join(sb.dir, "prompts/extract-takes.md"), "system prompt\n");
|
||||
execFileSync("git", ["add", "prompts/extract-takes.md"], { cwd: sb.dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", "add prompts"], {
|
||||
cwd: sb.dir,
|
||||
});
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("symlink target change affects hash (git hash-object follows symlink contents)", () => {
|
||||
// Note: git stores symlinks as the link target string, so changing
|
||||
// the LINK TARGET changes the git blob sha. Our hash will catch
|
||||
// that. We can't easily test "change of target's content" without
|
||||
// creating an out-of-tree file, so this case pins the link-target
|
||||
// path change.
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
// Create a symlink and commit.
|
||||
symlinkSync("./db.ts", join(sb.dir, "src/core/db-link.ts"));
|
||||
execFileSync("git", ["add", "src/core/db-link.ts"], { cwd: sb.dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", "add link"], {
|
||||
cwd: sb.dir,
|
||||
});
|
||||
const before = hash(sb);
|
||||
// Point the symlink elsewhere — and commit so it lands in the
|
||||
// index that `git ls-files -s` reads.
|
||||
rmSync(join(sb.dir, "src/core/db-link.ts"));
|
||||
symlinkSync("./cli.ts", join(sb.dir, "src/core/db-link.ts"));
|
||||
execFileSync("git", ["add", "src/core/db-link.ts"], { cwd: sb.dir });
|
||||
execFileSync("git", ["commit", "--quiet", "-m", "repoint link"], {
|
||||
cwd: sb.dir,
|
||||
});
|
||||
const after = hash(sb);
|
||||
expect(after).not.toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("locale-stable: LC_ALL=de_DE.UTF-8 produces the same hash as default", () => {
|
||||
withSandbox(BASELINE_FILES, (sb) => {
|
||||
const defaultHash = spawnSync("bash", [sb.scriptPath], {
|
||||
cwd: sb.dir,
|
||||
encoding: "utf8",
|
||||
}).stdout.trim();
|
||||
const altLocale = spawnSync("bash", [sb.scriptPath], {
|
||||
cwd: sb.dir,
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, LC_ALL: "de_DE.UTF-8", LANG: "de_DE.UTF-8" },
|
||||
}).stdout.trim();
|
||||
expect(altLocale).toBe(defaultHash);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("ci-cache-hash.sh — usage errors", () => {
|
||||
it("--bogus arg exits 2", () => {
|
||||
const r = spawnSync("bash", [SCRIPT_SRC, "--bogus"], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: "utf8",
|
||||
});
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain("usage:");
|
||||
});
|
||||
|
||||
it("--verbose flag works (writes diagnostics to stderr, hash to stdout)", () => {
|
||||
const r = spawnSync("bash", [SCRIPT_SRC, "--verbose"], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: "utf8",
|
||||
});
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout.trim()).toMatch(/^[0-9a-f]{16}$/);
|
||||
expect(r.stderr).toContain("files in hash");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,158 @@
|
||||
// mine-shard-weights.test.ts — pure-function tests for the log parser
|
||||
// and weight extraction. The full pipeline (gh run view → write JSON)
|
||||
// is integration-tested by actually running it once during T4.
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
computeWeights,
|
||||
parseLog,
|
||||
serializeWeights,
|
||||
} from "../../scripts/mine-shard-weights.ts";
|
||||
|
||||
const SAMPLE = `test (1)\tUNKNOWN STEP\t2026-05-25T11:26:40.000000Z ##[group]test/alpha.test.ts:
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:41.000000Z (pass) some test [1.00ms]
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:43.500000Z ##[group]test/beta.test.ts:
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:45.000000Z (pass) another test
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:48.000000Z ##[group]test/gamma.test.ts:
|
||||
test (1)\tUNKNOWN STEP\t2026-05-25T11:26:50.000000Z ##[group]test/delta.test.ts:
|
||||
test (2)\tUNKNOWN STEP\t2026-05-25T11:26:42.000000Z ##[group]test/epsilon.test.ts:
|
||||
test (2)\tUNKNOWN STEP\t2026-05-25T11:26:55.000000Z ##[group]test/zeta.test.ts:
|
||||
`;
|
||||
|
||||
describe("parseLog", () => {
|
||||
it("extracts file-start events with timestamps and jobs", () => {
|
||||
const events = parseLog(SAMPLE);
|
||||
expect(events.length).toBe(6);
|
||||
expect(events[0]).toMatchObject({
|
||||
job: "test (1)",
|
||||
file: "test/alpha.test.ts",
|
||||
});
|
||||
expect(events[5]).toMatchObject({
|
||||
job: "test (2)",
|
||||
file: "test/zeta.test.ts",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves stream order", () => {
|
||||
const events = parseLog(SAMPLE);
|
||||
const files = events.map((e) => e.file);
|
||||
expect(files).toEqual([
|
||||
"test/alpha.test.ts",
|
||||
"test/beta.test.ts",
|
||||
"test/gamma.test.ts",
|
||||
"test/delta.test.ts",
|
||||
"test/epsilon.test.ts",
|
||||
"test/zeta.test.ts",
|
||||
]);
|
||||
});
|
||||
|
||||
it("ignores non-group lines", () => {
|
||||
// The sample has (pass) lines mixed in. They should not appear in the
|
||||
// event list; parseLog only emits ##[group]test/X.test.ts: matches.
|
||||
const events = parseLog(SAMPLE);
|
||||
expect(events.every((e) => e.file.startsWith("test/"))).toBe(true);
|
||||
expect(events.every((e) => e.file.endsWith(".test.ts"))).toBe(true);
|
||||
});
|
||||
|
||||
it("handles empty input", () => {
|
||||
expect(parseLog("")).toEqual([]);
|
||||
});
|
||||
|
||||
it("ignores malformed timestamps", () => {
|
||||
const bad = "test (1)\tUNKNOWN\tBAD-TS ##[group]test/x.test.ts:\n";
|
||||
expect(parseLog(bad)).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects non-test-file groups (e.g. setup-bun action groups)", () => {
|
||||
const noise =
|
||||
"test (1)\tUNKNOWN\t2026-05-25T11:26:40.000Z ##[group]Run actions/checkout\n";
|
||||
expect(parseLog(noise)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeWeights", () => {
|
||||
it("computes delta between consecutive file headers within a job", () => {
|
||||
const events = parseLog(SAMPLE);
|
||||
const weights = computeWeights(events);
|
||||
// job test (1): alpha → 3500ms (40s → 43.5s), beta → 4500ms (43.5s → 48s),
|
||||
// gamma → 2000ms (48s → 50s), delta dropped (no successor).
|
||||
expect(weights.get("test/alpha.test.ts")).toBe(3500);
|
||||
expect(weights.get("test/beta.test.ts")).toBe(4500);
|
||||
expect(weights.get("test/gamma.test.ts")).toBe(2000);
|
||||
expect(weights.get("test/delta.test.ts")).toBeUndefined();
|
||||
// job test (2): epsilon → 13000ms (42s → 55s), zeta dropped.
|
||||
expect(weights.get("test/epsilon.test.ts")).toBe(13000);
|
||||
expect(weights.get("test/zeta.test.ts")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not cross job boundaries", () => {
|
||||
// If we naively diffed across jobs we'd get bogus values when
|
||||
// alpha (test 1) was followed by epsilon (test 2) by clock skew.
|
||||
const events = parseLog(SAMPLE);
|
||||
const weights = computeWeights(events);
|
||||
// alpha → beta within same job: 3500ms. Not 2000ms (which would be
|
||||
// alpha's stamp to epsilon's stamp across jobs).
|
||||
expect(weights.get("test/alpha.test.ts")).toBe(3500);
|
||||
});
|
||||
|
||||
it("takes the max when a file appears with multiple deltas", () => {
|
||||
const events = [
|
||||
{ job: "test (1)", timestampMs: 1000, file: "test/x.test.ts" },
|
||||
{ job: "test (1)", timestampMs: 2000, file: "test/y.test.ts" },
|
||||
{ job: "test (2)", timestampMs: 3000, file: "test/x.test.ts" },
|
||||
{ job: "test (2)", timestampMs: 8000, file: "test/y.test.ts" },
|
||||
];
|
||||
const w = computeWeights(events);
|
||||
// x: 1000→2000 (1000ms in job 1), 3000→8000 (5000ms in job 2). Max = 5000.
|
||||
expect(w.get("test/x.test.ts")).toBe(5000);
|
||||
});
|
||||
|
||||
it("skips out-of-order events defensively (negative delta)", () => {
|
||||
const events = [
|
||||
{ job: "test (1)", timestampMs: 5000, file: "test/x.test.ts" },
|
||||
{ job: "test (1)", timestampMs: 3000, file: "test/y.test.ts" },
|
||||
];
|
||||
const w = computeWeights(events);
|
||||
// x → negative delta → skipped. y has no successor → not recorded.
|
||||
expect(w.size).toBe(0);
|
||||
});
|
||||
|
||||
it("empty input → empty map", () => {
|
||||
expect(computeWeights([]).size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("serializeWeights", () => {
|
||||
it("sorts keys alphabetically for stable diffs", () => {
|
||||
const w = new Map([
|
||||
["test/z.test.ts", 100],
|
||||
["test/a.test.ts", 200],
|
||||
["test/m.test.ts", 50],
|
||||
]);
|
||||
const json = serializeWeights(w);
|
||||
const lines = json.split("\n").filter((l) => l.includes('"test/'));
|
||||
expect(lines[0]).toContain("a.test.ts");
|
||||
expect(lines[1]).toContain("m.test.ts");
|
||||
expect(lines[2]).toContain("z.test.ts");
|
||||
});
|
||||
|
||||
it("ends with a trailing newline (POSIX-friendly diff)", () => {
|
||||
const w = new Map([["test/x.test.ts", 1]]);
|
||||
expect(serializeWeights(w).endsWith("\n")).toBe(true);
|
||||
});
|
||||
|
||||
it("empty map → empty object JSON", () => {
|
||||
expect(serializeWeights(new Map())).toBe("{}\n");
|
||||
});
|
||||
|
||||
it("round-trips: parse our own output", () => {
|
||||
const w = new Map([
|
||||
["test/alpha.test.ts", 100],
|
||||
["test/beta.test.ts", 250],
|
||||
]);
|
||||
const json = serializeWeights(w);
|
||||
const parsed = JSON.parse(json);
|
||||
expect(parsed["test/alpha.test.ts"]).toBe(100);
|
||||
expect(parsed["test/beta.test.ts"]).toBe(250);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
// run-verify-parallel.test.ts — pin the dispatcher's contract.
|
||||
//
|
||||
// We can't easily fake the 20-check list inside the script (it's a static
|
||||
// array), but we CAN verify:
|
||||
// 1. --dry-list emits one line per check
|
||||
// 2. Unknown args exit 2
|
||||
// 3. The fast-path measurement claim — running it on a clean tree
|
||||
// completes faster than the sequential `&&`-chain would have
|
||||
// 4. A made-up failing check propagates exit 1 with the named check
|
||||
// in the failure report (covered by overriding the CHECKS array
|
||||
// via a sibling temp script)
|
||||
//
|
||||
// (3) is a soft regression guard, not a hard timing assertion (CI runners
|
||||
// vary). (4) is the load-bearing test: failure surfaces with name + log.
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { writeFileSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const SCRIPT = "scripts/run-verify-parallel.sh";
|
||||
|
||||
describe("run-verify-parallel.sh — CLI contract", () => {
|
||||
it("--dry-list emits one line per check, exit 0", () => {
|
||||
const r = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" });
|
||||
expect(r.status).toBe(0);
|
||||
const lines = r.stdout.trim().split("\n").filter(Boolean);
|
||||
// The exact count is allowed to grow; assert > 10 and that every line
|
||||
// looks like a script name (no whitespace, no quotes).
|
||||
expect(lines.length).toBeGreaterThan(10);
|
||||
for (const l of lines) {
|
||||
expect(l).toMatch(/^[a-z][a-z0-9:_-]+$/);
|
||||
}
|
||||
});
|
||||
|
||||
it("includes the load-bearing privacy + jsonb + typecheck checks", () => {
|
||||
const r = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" });
|
||||
expect(r.status).toBe(0);
|
||||
const set = new Set(r.stdout.trim().split("\n"));
|
||||
expect(set.has("check:privacy")).toBe(true);
|
||||
expect(set.has("check:jsonb")).toBe(true);
|
||||
expect(set.has("typecheck")).toBe(true);
|
||||
expect(set.has("check:operations-filter-bypass")).toBe(true);
|
||||
});
|
||||
|
||||
it("unknown arg exits 2 with usage error", () => {
|
||||
const r = spawnSync("bash", [SCRIPT, "--bogus"], { encoding: "utf8" });
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain("unknown arg");
|
||||
expect(r.stderr).toContain("usage:");
|
||||
});
|
||||
});
|
||||
|
||||
describe("run-verify-parallel.sh — failure surfacing (synthetic dispatcher)", () => {
|
||||
// We can't inject a fake check into the real script without touching the
|
||||
// CHECKS array. Instead, we write a SMALLER synthetic dispatcher that
|
||||
// shares the same shape (background jobs, exit-file aggregation, log
|
||||
// tail on failure) and assert the surfacing contract on it. This proves
|
||||
// the design, leaving the real script tested via CLI contract above.
|
||||
//
|
||||
// Each "check" is written as its own executable file so bash array
|
||||
// expansion doesn't try to word-split shell strings (the natural
|
||||
// failure mode of expressing checks as bash command strings is that
|
||||
// spaces and quotes don't survive `"${CHECKS[@]}"`).
|
||||
|
||||
function writeSynth(checkBodies: { name: string; body: string }[]): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "verify-synth-"));
|
||||
const checkPaths: string[] = [];
|
||||
for (const c of checkBodies) {
|
||||
const p = `${dir}/${c.name}.sh`;
|
||||
writeFileSync(p, `#!/usr/bin/env bash\n${c.body}\n`, { mode: 0o755 });
|
||||
checkPaths.push(p);
|
||||
}
|
||||
const dispatcher = `${dir}/dispatch.sh`;
|
||||
// CHECKS is an array of absolute paths to executable scripts.
|
||||
const arrLit = checkPaths.map((p) => `"${p}"`).join(" ");
|
||||
writeFileSync(
|
||||
dispatcher,
|
||||
`#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
LOG_DIR=$(mktemp -d /tmp/verify-synth-log-XXXXXX)
|
||||
trap 'rm -rf "$LOG_DIR"' EXIT
|
||||
CHECKS=(${arrLit})
|
||||
PIDS=()
|
||||
NAMES=()
|
||||
for c in "\${CHECKS[@]}"; do
|
||||
base=$(basename "$c" .sh)
|
||||
NAMES+=("$base")
|
||||
(
|
||||
"$c" > "$LOG_DIR/$base.log" 2>&1
|
||||
echo $? > "$LOG_DIR/$base.exit"
|
||||
) &
|
||||
PIDS+=($!)
|
||||
done
|
||||
for p in "\${PIDS[@]}"; do wait "$p" 2>/dev/null || true; done
|
||||
FAIL=0
|
||||
FAILED_NAMES=""
|
||||
for i in "\${!NAMES[@]}"; do
|
||||
n="\${NAMES[$i]}"
|
||||
rc=$(cat "$LOG_DIR/$n.exit")
|
||||
if [ "$rc" != "0" ]; then
|
||||
FAIL=$((FAIL+1))
|
||||
FAILED_NAMES="$FAILED_NAMES $n"
|
||||
echo "--- $n (rc=$rc) ---" >&2
|
||||
tail -10 "$LOG_DIR/$n.log" >&2
|
||||
fi
|
||||
done
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo "Failed:$FAILED_NAMES" >&2
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
return dispatcher;
|
||||
}
|
||||
|
||||
function cleanup(dispatcher: string) {
|
||||
rmSync(dispatcher.replace(/\/dispatch\.sh$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
|
||||
it("all-pass: exit 0, no failure block", () => {
|
||||
const d = writeSynth([
|
||||
{ name: "alpha", body: "exit 0" },
|
||||
{ name: "beta", body: "exit 0" },
|
||||
{ name: "gamma", body: "exit 0" },
|
||||
]);
|
||||
try {
|
||||
const r = spawnSync("bash", [d], { encoding: "utf8" });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stderr).not.toContain("Failed:");
|
||||
} finally {
|
||||
cleanup(d);
|
||||
}
|
||||
});
|
||||
|
||||
it("one-fails: exit 1, failure block names the check + shows log tail", () => {
|
||||
const d = writeSynth([
|
||||
{ name: "alpha", body: "exit 0" },
|
||||
{
|
||||
name: "beta",
|
||||
body: 'echo "synthetic failure detail line" >&2\nexit 7',
|
||||
},
|
||||
{ name: "gamma", body: "exit 0" },
|
||||
]);
|
||||
try {
|
||||
const r = spawnSync("bash", [d], { encoding: "utf8" });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toContain("Failed: beta");
|
||||
expect(r.stderr).toContain("synthetic failure detail line");
|
||||
expect(r.stderr).toMatch(/--- beta \(rc=7\) ---/);
|
||||
} finally {
|
||||
cleanup(d);
|
||||
}
|
||||
});
|
||||
|
||||
it("two-fail: both names appear in the Failed: list", () => {
|
||||
const d = writeSynth([
|
||||
{ name: "alpha", body: "exit 0" },
|
||||
{ name: "beta", body: "echo err-A >&2\nexit 1" },
|
||||
{ name: "gamma", body: "echo err-B >&2\nexit 2" },
|
||||
]);
|
||||
try {
|
||||
const r = spawnSync("bash", [d], { encoding: "utf8" });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stderr).toContain("err-A");
|
||||
expect(r.stderr).toContain("err-B");
|
||||
expect(r.stderr).toMatch(/Failed:\s+beta\s+gamma/);
|
||||
} finally {
|
||||
cleanup(d);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,266 @@
|
||||
// sharding.test.ts — pure unit tests for the LPT bin-packer.
|
||||
//
|
||||
// Covers: happy path, fallback semantics, full coverage, determinism,
|
||||
// balance ratio, N=1 trivial, weight-equal-to-zero handling, malformed
|
||||
// weights, missing weights file fail-soft.
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
computeMedian,
|
||||
imbalanceRatio,
|
||||
loadWeights,
|
||||
partition,
|
||||
WeightsLoadError,
|
||||
type WeightMap,
|
||||
} from "../../scripts/sharding.ts";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
function tempJson(content: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "weights-test-"));
|
||||
const path = join(dir, "weights.json");
|
||||
writeFileSync(path, content, "utf8");
|
||||
return path;
|
||||
}
|
||||
|
||||
describe("computeMedian", () => {
|
||||
it("returns 0 on empty input", () => {
|
||||
expect(computeMedian([])).toBe(0);
|
||||
});
|
||||
it("single value is itself", () => {
|
||||
expect(computeMedian([42])).toBe(42);
|
||||
});
|
||||
it("odd count: middle value", () => {
|
||||
expect(computeMedian([1, 5, 3])).toBe(3);
|
||||
});
|
||||
it("even count: mean of middle two", () => {
|
||||
expect(computeMedian([1, 2, 3, 4])).toBe(2.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("partition — happy path", () => {
|
||||
it("balances 4 known-weight files across 2 shards", () => {
|
||||
const weights: WeightMap = new Map([
|
||||
["a.test.ts", 100],
|
||||
["b.test.ts", 100],
|
||||
["c.test.ts", 50],
|
||||
["d.test.ts", 50],
|
||||
]);
|
||||
const out = partition(
|
||||
["a.test.ts", "b.test.ts", "c.test.ts", "d.test.ts"],
|
||||
weights,
|
||||
2,
|
||||
);
|
||||
expect(out.length).toBe(2);
|
||||
// LPT assigns 100→s0, 100→s1, 50→s0 (now 150), 50→s1 (now 150).
|
||||
// Wait: 100→s0 (s0=100), 100→s1 (s1=100), tie → s0 gets 50 (s0=150),
|
||||
// then s1 gets 50 (s1=150). Balanced.
|
||||
const totals = out.map((s) =>
|
||||
s.reduce((acc, f) => acc + (weights.get(f) ?? 0), 0),
|
||||
);
|
||||
expect(totals[0]).toBe(150);
|
||||
expect(totals[1]).toBe(150);
|
||||
});
|
||||
|
||||
it("LPT prefers heavy-first assignment", () => {
|
||||
// 3 files: 100, 10, 10. 2 shards. Naive assignment (alpha) would put
|
||||
// 100 + 10 = 110 in s0 and 10 in s1 (imbalance 11x). LPT puts 100 in
|
||||
// s0, then 10 + 10 in s1 (totals 100 + 20 = imbalance 5x).
|
||||
const weights: WeightMap = new Map([
|
||||
["big.test.ts", 100],
|
||||
["small1.test.ts", 10],
|
||||
["small2.test.ts", 10],
|
||||
]);
|
||||
const out = partition(
|
||||
["big.test.ts", "small1.test.ts", "small2.test.ts"],
|
||||
weights,
|
||||
2,
|
||||
);
|
||||
expect(out[0]).toEqual(["big.test.ts"]);
|
||||
expect(out[1]?.sort()).toEqual(["small1.test.ts", "small2.test.ts"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("partition — fallback semantics", () => {
|
||||
it("missing weights default to corpus median", () => {
|
||||
// weights = {a:100, b:50}, median = 75. Files c + d are unknown → 75 each.
|
||||
const weights: WeightMap = new Map([
|
||||
["a", 100],
|
||||
["b", 50],
|
||||
]);
|
||||
const out = partition(["a", "b", "c", "d"], weights, 2);
|
||||
// Effective weights: a=100, b=50, c=75, d=75. LPT: 100→s0, 75→s1 (c),
|
||||
// 75→s1 (d, ties broken alpha)... actually: 100→s0=100, 75→s1=75,
|
||||
// 75→s1 vs s0 → s1=150, 50→s0=150. Balanced 150/150.
|
||||
const totalsEffective = out.map((s) =>
|
||||
s.reduce((acc, f) => acc + (weights.get(f) ?? 75), 0),
|
||||
);
|
||||
expect(totalsEffective[0]).toBe(totalsEffective[1]);
|
||||
});
|
||||
|
||||
it("explicit fallback override beats median", () => {
|
||||
const weights: WeightMap = new Map([["a", 100]]);
|
||||
const out = partition(["a", "unknown"], weights, 2, { fallbackWeight: 500 });
|
||||
// a=100 to s0, unknown=500 to s1 (heavier goes first actually — sort
|
||||
// desc puts unknown=500 first). Wait, LPT sorts desc, so unknown
|
||||
// (500) goes to s0 first, then a (100) goes to s1.
|
||||
expect(out[0]).toEqual(["unknown"]);
|
||||
expect(out[1]).toEqual(["a"]);
|
||||
});
|
||||
|
||||
it("zero-weight files are valid (just always go to current min shard)", () => {
|
||||
const weights: WeightMap = new Map([
|
||||
["zero1", 0],
|
||||
["zero2", 0],
|
||||
["big", 100],
|
||||
]);
|
||||
const out = partition(["zero1", "zero2", "big"], weights, 2);
|
||||
// 100→s0=100, 0→s1=0, 0→s1=0. Both zeroes go to s1.
|
||||
expect(out[0]).toEqual(["big"]);
|
||||
expect(out[1]?.sort()).toEqual(["zero1", "zero2"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("partition — invariants", () => {
|
||||
it("every input file lands in exactly one shard (full coverage)", () => {
|
||||
const files = Array.from({ length: 25 }, (_, i) => `f${i}.test.ts`);
|
||||
const weights: WeightMap = new Map(
|
||||
files.map((f, i) => [f, (i % 5) * 10 + 5]),
|
||||
);
|
||||
const out = partition(files, weights, 4);
|
||||
const seen: string[] = [];
|
||||
for (const shard of out) seen.push(...shard);
|
||||
expect(seen.sort()).toEqual([...files].sort());
|
||||
expect(new Set(seen).size).toBe(files.length);
|
||||
});
|
||||
|
||||
it("deterministic — same input always produces same output", () => {
|
||||
const files = ["d.test.ts", "a.test.ts", "c.test.ts", "b.test.ts"];
|
||||
const weights: WeightMap = new Map([
|
||||
["a.test.ts", 30],
|
||||
["b.test.ts", 30],
|
||||
["c.test.ts", 30],
|
||||
["d.test.ts", 30],
|
||||
]);
|
||||
const r1 = partition(files, weights, 2);
|
||||
const r2 = partition([...files].reverse(), weights, 2);
|
||||
const r3 = partition(files, weights, 2);
|
||||
// Same input → identical output
|
||||
expect(r3).toEqual(r1);
|
||||
// Input order shuffled but contents identical → identical output
|
||||
// (ties broken by path asc so order is canonical regardless of input)
|
||||
expect(r2).toEqual(r1);
|
||||
});
|
||||
|
||||
it("N=1 trivial: everything in one shard", () => {
|
||||
const out = partition(["x.test.ts", "y.test.ts"], new Map(), 1);
|
||||
expect(out.length).toBe(1);
|
||||
expect(out[0]?.sort()).toEqual(["x.test.ts", "y.test.ts"]);
|
||||
});
|
||||
|
||||
it("empty file list returns N empty shards", () => {
|
||||
const out = partition([], new Map(), 3);
|
||||
expect(out).toEqual([[], [], []]);
|
||||
});
|
||||
|
||||
it("invalid shard count throws RangeError", () => {
|
||||
expect(() => partition(["a"], new Map(), 0)).toThrow(RangeError);
|
||||
expect(() => partition(["a"], new Map(), -1)).toThrow(RangeError);
|
||||
expect(() => partition(["a"], new Map(), 1.5)).toThrow(RangeError);
|
||||
});
|
||||
|
||||
it("invalid fallbackWeight throws RangeError", () => {
|
||||
expect(() =>
|
||||
partition(["a"], new Map(), 2, { fallbackWeight: -1 }),
|
||||
).toThrow(RangeError);
|
||||
expect(() =>
|
||||
partition(["a"], new Map(), 2, { fallbackWeight: NaN }),
|
||||
).toThrow(RangeError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("partition — balance quality", () => {
|
||||
it("synthetic skewed corpus produces imbalance ratio ≤ 1.5", () => {
|
||||
// 100 files, weights drawn from a Zipf-ish distribution (a few heavy,
|
||||
// many light). This is the shape that broke FNV-1a hash sharding.
|
||||
const files = Array.from({ length: 100 }, (_, i) => `f${i}.test.ts`);
|
||||
const weights: WeightMap = new Map();
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
// Heavy tail: file 0 = 1000ms, file 1 = 500, file 2 = 333, ...
|
||||
// Average about 50ms; max about 1000ms.
|
||||
const w = Math.max(10, Math.floor(1000 / (i + 1)));
|
||||
weights.set(files[i]!, w);
|
||||
}
|
||||
const out = partition(files, weights, 6);
|
||||
const ratio = imbalanceRatio(out, weights, 0);
|
||||
expect(ratio).toBeLessThanOrEqual(1.5);
|
||||
});
|
||||
|
||||
it("imbalance ratio of 1.0 when weights are perfectly divisible", () => {
|
||||
const files = ["a", "b", "c", "d"];
|
||||
const weights: WeightMap = new Map([
|
||||
["a", 50],
|
||||
["b", 50],
|
||||
["c", 50],
|
||||
["d", 50],
|
||||
]);
|
||||
const out = partition(files, weights, 2);
|
||||
expect(imbalanceRatio(out, weights, 0)).toBe(1.0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("loadWeights", () => {
|
||||
it("missing file → empty map (fail-soft)", () => {
|
||||
const w = loadWeights("/tmp/definitely-does-not-exist-weights.json");
|
||||
expect(w.size).toBe(0);
|
||||
});
|
||||
|
||||
it("malformed JSON → throws WeightsLoadError", () => {
|
||||
const p = tempJson("not json {");
|
||||
try {
|
||||
expect(() => loadWeights(p)).toThrow(WeightsLoadError);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("array JSON → throws (wrong shape)", () => {
|
||||
const p = tempJson("[1,2,3]");
|
||||
try {
|
||||
expect(() => loadWeights(p)).toThrow(/expected top-level object/);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("negative weight → throws (semantic invalid)", () => {
|
||||
const p = tempJson('{"foo.test.ts": -5}');
|
||||
try {
|
||||
expect(() => loadWeights(p)).toThrow(/non-negative finite/);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("non-number weight → throws (semantic invalid)", () => {
|
||||
const p = tempJson('{"foo.test.ts": "100ms"}');
|
||||
try {
|
||||
expect(() => loadWeights(p)).toThrow(/non-negative finite/);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("valid weights JSON round-trips", () => {
|
||||
const p = tempJson('{"a.test.ts": 100, "b.test.ts": 250}');
|
||||
try {
|
||||
const w = loadWeights(p);
|
||||
expect(w.size).toBe(2);
|
||||
expect(w.get("a.test.ts")).toBe(100);
|
||||
expect(w.get("b.test.ts")).toBe(250);
|
||||
} finally {
|
||||
rmSync(p.replace(/\/weights\.json$/, ""), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,18 +1,30 @@
|
||||
/**
|
||||
* Regression test: scripts/test-shard.sh exclusion symmetry.
|
||||
* Regression test: scripts/test-shard.sh exclusion + balance contract.
|
||||
*
|
||||
* Pins the contract that CI's hash-bucketed shard script EXCLUDES
|
||||
* *.serial.test.ts and test/e2e/ from every shard. Serial files share
|
||||
* file-wide state (top-level mock.module, module singletons) that leaks
|
||||
* across files in the same `bun test` shard process. Before v0.31.4.1
|
||||
* they were hashed into the same buckets as parallel files, which broke
|
||||
* the quarantine — `eval-takes-quality-runner.serial.test.ts` stubbed
|
||||
* `gateway.ts` and broke every `gateway.embedMultimodal` test in
|
||||
* `voyage-multimodal.test.ts` on shard 2.
|
||||
* Pins two invariants of the CI matrix shard script:
|
||||
*
|
||||
* Without this guard, a future refactor that drops the `-not -name
|
||||
* '*.serial.test.ts'` clause from test-shard.sh would silently undo the
|
||||
* fix and re-introduce the contention flake.
|
||||
* 1. EXCLUDE *.serial.test.ts and test/e2e/ from every shard. Serial
|
||||
* files share file-wide state (top-level mock.module, module
|
||||
* singletons) that leaks across files in the same `bun test` shard
|
||||
* process. Before v0.31.4.1 they were hashed into the same buckets
|
||||
* as parallel files, which broke the quarantine —
|
||||
* `eval-takes-quality-runner.serial.test.ts` stubbed `gateway.ts`
|
||||
* and broke every `gateway.embedMultimodal` test in
|
||||
* `voyage-multimodal.test.ts` on shard 2.
|
||||
*
|
||||
* 2. INCLUDE *.slow.test.ts. CI is the only default place slow files
|
||||
* run; the local fast loop excludes them via run-unit-shard.sh. See
|
||||
* CLAUDE.md "CI vs local: intentionally divergent file sets".
|
||||
*
|
||||
* 3. LPT balance: file counts across shards are within 5% of each other
|
||||
* under the default (no-weights / cold-start) configuration. Once
|
||||
* test-weights.json is populated, weighted balance also lands the
|
||||
* imbalance ratio ≤ 1.5.
|
||||
*
|
||||
* Without (1), the v0.31.4.1 mock.module leak comes back. Without (2),
|
||||
* slow tests stop running in CI (silent regression — they don't fail,
|
||||
* they just never execute). Without (3), shard 3 will drift back to
|
||||
* 26-min p100 wallclock.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll } from 'bun:test';
|
||||
@@ -22,30 +34,27 @@ import { resolve } from 'path';
|
||||
const REPO_ROOT = resolve(import.meta.dir, '..', '..');
|
||||
const SHARD_SH = resolve(REPO_ROOT, 'scripts/test-shard.sh');
|
||||
|
||||
// Pure-bash FNV-1a per shard takes ~4s on a M-series Mac because the script
|
||||
// computes a hash for every test file. Compute all 4 shards once in beforeAll
|
||||
// and reuse across cases so the suite finishes in one shell-out per shard.
|
||||
const shardCache: Record<number, string[]> = {};
|
||||
// LPT partition via sharding.ts is ~30ms per shard (much faster than the
|
||||
// pure-bash FNV-1a it replaced). Cache results so repeated assertions
|
||||
// don't shell out per case.
|
||||
const shardCache: Record<string, string[]> = {};
|
||||
|
||||
function dryRunList(shard: number, total: number): string[] {
|
||||
if (shardCache[shard]) return shardCache[shard];
|
||||
const key = `${shard}/${total}`;
|
||||
if (shardCache[key]) return shardCache[key];
|
||||
const out = execFileSync('bash', [SHARD_SH, '--dry-run-list', String(shard), String(total)], {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
shardCache[shard] = out.split('\n').map(s => s.trim()).filter(Boolean);
|
||||
return shardCache[shard];
|
||||
shardCache[key] = out.split('\n').map(s => s.trim()).filter(Boolean);
|
||||
return shardCache[key];
|
||||
}
|
||||
|
||||
describe('test-shard.sh exclusion symmetry', () => {
|
||||
describe('test-shard.sh — exclusion contract', () => {
|
||||
beforeAll(() => {
|
||||
for (const shard of [1, 2, 3, 4]) dryRunList(shard, 4);
|
||||
// v0.40.10 flake-hardening: bump beforeAll budget 60s → 180s. Each
|
||||
// FNV-1a dry-run shells out and computes a hash for every test file
|
||||
// (~4s solo). Under slow-shard concurrency (longmemeval E2E at ~50s
|
||||
// hogging CPU), the 4 sequential shell-outs slip past 60s and time
|
||||
// out even though they'd complete fine solo.
|
||||
}, 180_000);
|
||||
}, 60_000);
|
||||
|
||||
it('includes plain *.test.ts files in at least one shard', () => {
|
||||
const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4));
|
||||
expect(allFiles.length).toBeGreaterThan(0);
|
||||
@@ -68,7 +77,17 @@ describe('test-shard.sh exclusion symmetry', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('partitions plain files across shards without overlap', () => {
|
||||
it('INCLUDES *.slow.test.ts files (CI matrix is where slow files run)', () => {
|
||||
const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4));
|
||||
const slowFiles = allFiles.filter(f => /\.slow\.test\.ts$/.test(f));
|
||||
// The repo currently has 3 slow files — pin >= 1 so the test stays
|
||||
// green if the count changes but fails loud if slow files vanish
|
||||
// entirely (regression: someone re-added -not -name '*.slow.test.ts'
|
||||
// to the find clause).
|
||||
expect(slowFiles.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('partitions every file across shards without overlap', () => {
|
||||
const seen = new Map<string, number>();
|
||||
for (const shard of [1, 2, 3, 4]) {
|
||||
for (const f of dryRunList(shard, 4)) {
|
||||
@@ -81,3 +100,70 @@ describe('test-shard.sh exclusion symmetry', () => {
|
||||
expect(seen.size).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('test-shard.sh — LPT balance contract', () => {
|
||||
// LPT balances WALLCLOCK (sum of weights), not file count. When real
|
||||
// weights are loaded from scripts/test-weights.json, shards with
|
||||
// heavier files have FEWER files — that's the optimization working.
|
||||
// We assert wallclock balance via the imbalance ratio (≤1.5).
|
||||
|
||||
// Read weights from the committed JSON. If it doesn't exist yet (early
|
||||
// in the wave) or is empty, we degrade to cold-start round-robin and
|
||||
// assert file-count balance instead.
|
||||
let weightsMap = new Map<string, number>();
|
||||
let weightsLoaded = false;
|
||||
beforeAll(() => {
|
||||
try {
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const p = path.resolve(REPO_ROOT, 'scripts/test-weights.json');
|
||||
if (fs.existsSync(p)) {
|
||||
const raw = JSON.parse(fs.readFileSync(p, 'utf8'));
|
||||
for (const [k, v] of Object.entries(raw)) {
|
||||
if (typeof v === 'number' && Number.isFinite(v)) weightsMap.set(k, v);
|
||||
}
|
||||
weightsLoaded = weightsMap.size > 0;
|
||||
}
|
||||
} catch {
|
||||
// fall through to cold-start mode
|
||||
}
|
||||
});
|
||||
|
||||
function totalsFor(shards: string[][]): number[] {
|
||||
// Use 30ms as the cold-start fallback (matches mine-shard-weights
|
||||
// median observation). When weights are loaded, missing files get
|
||||
// the corpus median anyway via sharding.ts.
|
||||
const fallback = weightsLoaded
|
||||
? Array.from(weightsMap.values()).sort((a, b) => a - b)[
|
||||
Math.floor(weightsMap.size / 2)
|
||||
] ?? 30
|
||||
: 1;
|
||||
return shards.map((s) =>
|
||||
s.reduce((acc, f) => acc + (weightsMap.get(f) ?? fallback), 0),
|
||||
);
|
||||
}
|
||||
|
||||
it('4-shard wallclock imbalance ratio ≤ 1.5', () => {
|
||||
const shards = [1, 2, 3, 4].map(s => dryRunList(s, 4));
|
||||
for (const s of shards) expect(s.length).toBeGreaterThan(0);
|
||||
const totals = totalsFor(shards);
|
||||
const ratio = Math.max(...totals) / Math.min(...totals);
|
||||
expect(ratio).toBeLessThanOrEqual(1.5);
|
||||
});
|
||||
|
||||
it('6-shard wallclock imbalance ratio ≤ 1.5', () => {
|
||||
const shards = [1, 2, 3, 4, 5, 6].map(s => dryRunList(s, 6));
|
||||
for (const s of shards) expect(s.length).toBeGreaterThan(0);
|
||||
const totals = totalsFor(shards);
|
||||
const ratio = Math.max(...totals) / Math.min(...totals);
|
||||
expect(ratio).toBeLessThanOrEqual(1.5);
|
||||
});
|
||||
|
||||
it('6-shard partition is deterministic across runs', () => {
|
||||
// Clear cache + re-run for one shard, compare to the cached result.
|
||||
const cached = [...dryRunList(1, 6)];
|
||||
delete shardCache['1/6'];
|
||||
const fresh = dryRunList(1, 6);
|
||||
expect(fresh).toEqual(cached);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user