From 2ddaad62cc7852eb8274022ae8a6d7527d169ae8 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Tue, 12 May 2026 21:13:04 -0700 Subject: [PATCH] feat: add Crabbox UI proof workflow (#2192) * feat: add crabbox ui proof workflow * fix: render ui proof video previews inline --- .agents/skills/blacksmith-testbox/SKILL.md | 347 ---------- .agents/skills/clawhub-pr-maintainer/SKILL.md | 102 +++ .agents/skills/clawhub-ui-proof/SKILL.md | 75 +++ .agents/skills/crabbox/SKILL.md | 51 ++ .crabbox.yaml | 32 + .github/workflows/ci-check-testbox.yml | 2 +- .gitignore | 9 +- CONTRIBUTING.md | 27 +- package.json | 9 +- scripts/blacksmith-testbox-runner.mjs | 195 ------ scripts/blacksmith-testbox-runner.test.mjs | 76 --- scripts/blacksmith-testbox-state.mjs | 181 ----- scripts/blacksmith-testbox-state.test.mjs | 116 ---- scripts/crabbox-wrapper.mjs | 150 +++++ scripts/crabbox-wrapper.test.mjs | 73 ++ scripts/testbox-sync-sanity.mjs | 128 ---- scripts/ui-proof-publish.mjs | 506 ++++++++++++++ scripts/ui-proof-publish.test.mjs | 167 +++++ scripts/ui-proof-runtime.mjs | 138 ++++ scripts/ui-proof-runtime.test.mjs | 45 ++ scripts/ui-proof.mjs | 635 ++++++++++++++++++ scripts/ui-proof.test.mjs | 255 +++++++ 22 files changed, 2255 insertions(+), 1064 deletions(-) delete mode 100644 .agents/skills/blacksmith-testbox/SKILL.md create mode 100644 .agents/skills/clawhub-pr-maintainer/SKILL.md create mode 100644 .agents/skills/clawhub-ui-proof/SKILL.md create mode 100644 .agents/skills/crabbox/SKILL.md create mode 100644 .crabbox.yaml delete mode 100644 scripts/blacksmith-testbox-runner.mjs delete mode 100644 scripts/blacksmith-testbox-runner.test.mjs delete mode 100644 scripts/blacksmith-testbox-state.mjs delete mode 100644 scripts/blacksmith-testbox-state.test.mjs create mode 100644 scripts/crabbox-wrapper.mjs create mode 100644 scripts/crabbox-wrapper.test.mjs delete mode 100644 scripts/testbox-sync-sanity.mjs create mode 100644 scripts/ui-proof-publish.mjs create mode 100644 scripts/ui-proof-publish.test.mjs create mode 100644 scripts/ui-proof-runtime.mjs create mode 100644 scripts/ui-proof-runtime.test.mjs create mode 100644 scripts/ui-proof.mjs create mode 100644 scripts/ui-proof.test.mjs diff --git a/.agents/skills/blacksmith-testbox/SKILL.md b/.agents/skills/blacksmith-testbox/SKILL.md deleted file mode 100644 index 3adfbfce..00000000 --- a/.agents/skills/blacksmith-testbox/SKILL.md +++ /dev/null @@ -1,347 +0,0 @@ ---- -name: blacksmith-testbox -description: Run Blacksmith Testbox for ClawHub CI-parity checks, hosted services, broad Bun gates, or builds local cannot reproduce without hurting developer machines. ---- - -# Blacksmith Testbox - -## Scope - -Use Testbox when you need remote CI parity, injected secrets, hosted services, -or an OS/runtime image that your local machine cannot provide cheaply. - -Do not default to Testbox for every local test/build loop. If the repo has -documented local commands for normal iteration, use those first so you keep -warm caches, local build state, and fast feedback. - -Testbox is the expensive path. Reach for it deliberately. - -ClawHub maintainers can opt into Testbox-first validation by setting -`CLAWHUB_TESTBOX=1` in their environment or standing agent rules. This mode is -maintainers-only and requires Blacksmith access. - -When `CLAWHUB_TESTBOX=1` is set in ClawHub: - -- Pre-warm a Testbox early for longer, wider, or uncertain work. -- Prefer Testbox for broad Bun gates, e2e, Convex-ish deploy parity, package - proof, and expensive validation. -- Reuse the same Testbox ID for every run command in the same task/session. -- Use local commands only when the task explicitly sets - `CLAWHUB_LOCAL_CHECK_MODE=throttled|full`, or when the user asks for local - proof. - -## Install The CLI - -If `blacksmith` is not installed, install it: - -```bash -curl -fsSL https://get.blacksmith.sh | sh -``` - -For the canary channel: - -```bash -BLACKSMITH_CHANNEL=canary sh -c 'curl -fsSL https://get.blacksmith.sh | sh' -``` - -Then authenticate: - -```bash -blacksmith auth login -``` - -## Agent-Triggered Browser Auth - -When an agent needs to ensure the user is authenticated before running Testbox -commands, use browser-based auth with non-interactive mode. This opens the -browser for the user to sign in; the agent does not interact with the browser. - -`--organization` is required with `--non-interactive`: - -```bash -blacksmith auth login --non-interactive --organization -``` - -The org slug can come from `BLACKSMITH_ORG` or the `--org` global flag. Do not -use `--api-token` for this browser flow; that is for headless/token auth. - -## Decide First: Local Or Testbox - -Before warming anything up, check the repo's own instructions. - -Prefer local commands when: - -- the repo documents a supported local test/build workflow -- you are iterating on unit tests, lint, typecheck, formatting, or other - local-only validation -- the value comes from warm local caches and fast repeat runs -- the command does not need remote secrets, hosted services, or CI-only images - -Prefer Testbox when: - -- `CLAWHUB_TESTBOX=1` is set by the user, agent environment, or standing rules -- the repo explicitly requires CI-parity or remote validation -- the command needs secrets, service containers, or provisioned infra -- you are reproducing CI-only failures -- you need the exact workflow image/job environment from GitHub Actions - -For ClawHub specifically, normal local iteration stays local unless maintainer -Testbox mode is enabled with `CLAWHUB_TESTBOX=1`: - -- `bun run format:check` -- `bun run lint` -- `bun run test` -- `bun run coverage` -- `bunx tsc --noEmit` -- `bun run build` - -If `CLAWHUB_TESTBOX=1` is enabled, run those same repo commands inside the warm -Testbox. If the user wants laptop-friendly local proof for one command, use the -explicit escape hatch `CLAWHUB_LOCAL_CHECK_MODE=throttled`. - -In `.codex` worktrees without a `node_modules` symlink, do not run -`bun install` just to validate locally. Use syntax checks or Testbox. - -## Setup: Warmup Before Coding - -If you decided Testbox is warranted, warm one up early. This returns an ID -instantly and boots the CI environment in the background while you work: - -```bash -blacksmith testbox warmup ci-check-testbox.yml --ref main --idle-timeout 90 -# -> tbx_01jkz5b3t9... -``` - -Save this ID in the current session. You need it for every `run` command. -Treat `blacksmith testbox list` as diagnostics, not a reusable work queue. -Listed boxes can be visible at the org/repo level while still being unusable or -stale for the current local agent lane. - -For ClawHub maintainer Testbox mode, claim the ID in the current checkout: - -```bash -bun run testbox:claim -- --id -``` - -Warmup dispatches `.github/workflows/ci-check-testbox.yml`, which provisions a -VM with Bun, Node, dependency install/cache, and a clean checkout of the repo at -the chosen ref. - -Bootstrap note: GitHub only exposes `workflow_dispatch` workflows through the -Actions API after the workflow file exists on the default branch. If a brand-new -Testbox workflow exists only on a feature branch, `blacksmith testbox warmup -ci-check-testbox.yml --ref ` can return a GitHub 404 even though the -file exists on that branch. Land the workflow bootstrap first, then dispatch -branch refs normally. - -Options: - -```text ---ref Git ref to dispatch against ---job Specific job within the workflow, if it has multiple ---idle-timeout Idle timeout in minutes -``` - -## Critical: Always Run From The Repo Root - -Always invoke `blacksmith testbox` commands from the root of the git -repository. The CLI syncs the current working directory to the testbox using -rsync with `--delete`. If you run from a subdirectory, rsync mirrors only that -subdirectory and can delete everything else on the testbox. - -Correct: - -```bash -blacksmith testbox run --id "bun run test" -blacksmith testbox run --id "cd packages/clawhub && bun run verify" -``` - -Wrong: - -```bash -cd packages/clawhub && blacksmith testbox run --id "bun run verify" -``` - -If your shell is in a subdirectory, move back first: - -```bash -cd "$(git rev-parse --show-toplevel)" -``` - -## Running Commands - -Raw Blacksmith form: - -```bash -blacksmith testbox run --id "" -``` - -The `run` command waits for the testbox to become ready if it is still booting, -so you can call `run` immediately after warmup. - -In ClawHub, prefer the guarded runner wrapper so stale/reused ids fail before -the Blacksmith CLI spends time syncing or emits a confusing missing-key error: - -```bash -bun run testbox:run -- --id -- bun run lint -bun run testbox:run -- --id -- bun run test -bun run testbox:run -- --id -- bun run build -``` - -The wrapper refuses to run when the local per-Testbox key is missing or when -the id was not claimed by this ClawHub checkout with: - -```bash -bun run testbox:claim -- --id -``` - -Treat that as the expected remediation, not as a GitHub account or normal -SSH-key problem. A local key alone is not enough; a ready box may still carry -stale rsync state from another lane. - -If the agent crashes, the remote box relies on Blacksmith's idle timeout. The -local ClawHub claim marker is not deleted automatically, so the wrapper treats -claims older than 12 hours as stale. Override only for intentional long-running -work with: - -```bash -CLAWHUB_TESTBOX_CLAIM_TTL_MINUTES= -``` - -Before spending a broad gate on a manually assembled command, run: - -```bash -bun run testbox:sanity -- --id -``` - -## Downloading Files From A Testbox - -Use the `download` command to retrieve files or directories from a running -testbox to your local machine. This is useful for fetching build artifacts, -test results, coverage reports, or any output generated on the testbox. - -```bash -blacksmith testbox download --id [local-path] -``` - -The remote path is relative to the testbox working directory. If no local path -is specified, the file is saved to the current directory using the same base -name. - -Examples: - -```bash -blacksmith testbox download --id coverage/lcov-report/ ./coverage/ -blacksmith testbox download --id test-results/ ./test-results/ -blacksmith testbox download --id dist/ ./dist/ -``` - -## How File Sync Works - -Understanding this model is critical for using Testbox correctly. - -When you call `run`, the CLI performs a delta sync of your local changes to the -remote testbox before executing your command: - -1. The testbox VM starts from a clean checkout at the warmup ref. The workflow - setup steps run during warmup and populate dependency directories on the - remote VM. -2. On each `run`, the CLI uses git to detect which files changed locally since - the last sync. It syncs only tracked files and untracked non-ignored files. -3. `.gitignore`'d directories are never synced. `node_modules/`, `.bun/`, - `.vite/`, `dist/`, `.output/`, `.nitro/`, and coverage outputs stay local. - The testbox uses its own copies populated by the warmup workflow. -4. If nothing has changed since the last sync, the sync is skipped. - -Why this matters: - -- If you modify `package.json` or `bun.lock`, re-run install on the testbox: - - ```bash - bun run testbox:run -- --id -- bun install --frozen-lockfile - ``` - -- If tests depend on generated/build output, re-run the build on the testbox. -- New untracked files sync as long as they are not gitignored. -- Deleted files are also deleted on the remote testbox. - -## Critical: Do Not Ban Local Tests - -Do not assume local validation is forbidden. Many repos intentionally invest in -fast, warm local loops, and forcing every run through Testbox destroys that -advantage. - -Use Testbox for checks that actually need it: remote parity, secrets, services, -CI-only runners, expensive broad gates, or reproducibility against the workflow -image. - -ClawHub maintainer exception: if `CLAWHUB_TESTBOX=1` is set by the user or -agent environment, treat Testbox as the normal validation path for this repo. -Use `CLAWHUB_LOCAL_CHECK_MODE=throttled|full` as the explicit local escape -hatch. - -## Workflow - -1. Decide whether the repo's local loop is the right default. For ClawHub, - `CLAWHUB_TESTBOX=1` makes Testbox the maintainer default. -2. If Testbox is warranted, warm up early: - `blacksmith testbox warmup ci-check-testbox.yml --ref main --idle-timeout 90`. -3. Save the ID, then claim it: - `bun run testbox:claim -- --id `. -4. Write code while the testbox boots in the background. -5. Run sanity before broad checks: - `bun run testbox:sanity -- --id `. -6. Run the remote command: - `bun run testbox:run -- --id -- bun run lint`. -7. If tests fail, fix code and re-run against the same warm box. -8. If dependency manifests changed, run install in the box before testing. -9. If you need artifacts, download them with `blacksmith testbox download`. -10. Stop the box when done if it is no longer needed: - `blacksmith testbox stop --id `. - -## ClawHub Broad Gate - -For a broad ClawHub proof in maintainer Testbox mode, use the repo package -manager and keep the commands explicit: - -```bash -bun run testbox:run -- --id -- bun run format:check -bun run testbox:run -- --id -- bun run lint -bun run testbox:run -- --id -- bun run test -bun run testbox:run -- --id -- bunx tsc --noEmit -bun run testbox:run -- --id -- bunx tsc -p packages/schema/tsconfig.json --noEmit -bun run testbox:run -- --id -- bunx tsc -p packages/clawhub/tsconfig.json --noEmit -bun run testbox:run -- --id -- bun run build -``` - -For e2e: - -```bash -bun run testbox:run -- --id -- bun run test:e2e -bun run testbox:run -- --id -- bun run test:pw -``` - -## Waiting For Readiness - -The `run` command automatically waits for the testbox, so explicit waiting is -usually unnecessary. If you do need to check readiness separately, use -`--wait`. Do not use a sleep-and-recheck loop. - -```bash -blacksmith testbox status --id --wait --wait-timeout 5m -``` - -## Managing Testboxes - -```bash -blacksmith testbox status --id -blacksmith testbox list -blacksmith testbox stop --id -``` - -Testboxes automatically shut down after being idle. For ClawHub maintainer -work, use 90 minutes for long-running sessions: - -```bash -blacksmith testbox warmup ci-check-testbox.yml --idle-timeout 90 -``` diff --git a/.agents/skills/clawhub-pr-maintainer/SKILL.md b/.agents/skills/clawhub-pr-maintainer/SKILL.md new file mode 100644 index 00000000..577b8dda --- /dev/null +++ b/.agents/skills/clawhub-pr-maintainer/SKILL.md @@ -0,0 +1,102 @@ +--- +name: clawhub-pr-maintainer +description: Use when reviewing, triaging, validating, or discussing ClawHub GitHub issues or pull requests, including author context, CI, UI proof, evidence, labels, close decisions, and maintainer handoff. +--- + +# ClawHub PR Maintainer + +Use this skill for maintainer-facing ClawHub GitHub workflow, not for ordinary +implementation work. + +## Start With Live GitHub State + +- Use `gh pr view` or `gh issue view` against `openclaw/clawhub`; verify live + state before commenting, labeling, closing, or recommending merge. +- For PRs, read title, body, author, labels, comments, files, commits, status + checks, review state, and linked issues. +- Surface author identity briefly: GitHub name/login and account age when + useful. Treat identity as triage signal, never as proof by itself. + +Common read-only commands: + +```sh +gh pr view --repo openclaw/clawhub --json title,body,author,labels,comments,files,commits,statusCheckRollup,reviewDecision,url +gh issue view --repo openclaw/clawhub --json title,body,author,labels,comments,state,url +gh api users/ --jq '{login,name,created_at,type}' +``` + +## Review Evidence Bar + +- For bug fixes, require symptom evidence, a plausible root cause in the touched + code path, and either a regression test or focused manual proof. +- For UI changes, require screenshots or video when the behavior is meaningfully + visual. Use tests as supplemental evidence, not a substitute for visible proof. +- Do not merge or recommend merge based only on PR prose, AI rationale, or green + CI when the changed behavior has not been exercised. +- For contributor-provided screenshots/videos/logs, inspect the artifact + directly and state what it proves. Do not rerun `proof:ui` just to inspect + existing evidence. + +## Decide UI Proof Mode + +Use the `clawhub-ui-proof` skill when the maintainer/agent should generate new +visual evidence. + +- `before-after`: bug fixes, regressions, changed copy, changed layout, or any + PR where main-vs-candidate comparison clarifies the change. +- `feature`: new page, new flow, new UI state, or behavior that cannot exist on + `origin/main`. +- No generated proof: docs-only, backend-only, tests-only, metadata-only, or + already-sufficient contributor evidence. + +Write a temporary Playwright scenario under `.artifacts/proof-scenarios/`; do +not infer manual clicks. Keep screenshots and videos in `.artifacts/` until +publishing. Never commit proof artifacts. + +## Final Review Comment With Proof + +If this review generated `proof:ui` artifacts, publish them before the final PR +review comment. Do not leave only local `.artifacts/...` paths in a PR comment; +they are useful to the maintainer locally but invisible to GitHub readers. + +Use: + +```sh +bun run proof:publish -- --proof-dir .artifacts/clawhub-ui-proof/ --target-pr +``` + +`proof:publish` copies the selected files to the `qa-artifacts` branch and +upserts a marker-backed PR comment with a **ClawHub UI Proof** section. + +That comment includes: + +- the proof mode (`before-after` or `feature`) +- the `report.md` result summary +- the most relevant per-step screenshots +- inline video previews when GIF previews are present +- links to full-run MP4s +- links to raw proof files on the artifact branch + +Use `--dry-run` before publishing if you need to inspect the generated comment. +If publishing fails because credentials are missing, report the local proof +directory and the failed command instead of posting a comment that claims +evidence is attached. + +## ClawSweeper + +ClawSweeper is the bot control plane for automated PR/issue review once ClawHub +dispatch is configured. Until then, use this skill for manual maintainer review. +If ClawSweeper has posted a review, read it as evidence but verify live PR state +before acting. + +## Commenting And Labels + +- Use literal multiline comment bodies or `--body-file`; never pass escaped + `\n` strings. +- Keep maintainer comments short: finding, evidence, requested action, and + verification path. +- When no proof artifacts were generated, `gh pr comment --body-file` is fine. + When proof artifacts were generated, use `proof:publish` so screenshots/videos + are published before posting. +- Do not close more than five issues/PRs in one action without explicit + confirmation and the exact target list. diff --git a/.agents/skills/clawhub-ui-proof/SKILL.md b/.agents/skills/clawhub-ui-proof/SKILL.md new file mode 100644 index 00000000..22b8035f --- /dev/null +++ b/.agents/skills/clawhub-ui-proof/SKILL.md @@ -0,0 +1,75 @@ +--- +name: clawhub-ui-proof +description: Use when ClawHub UI changes need visual proof, before/after comparison, new-feature screenshots, temporary Playwright scenarios, or Crabbox desktop recordings. +--- + +# ClawHub UI Proof + +Use `proof:ui` for human-readable UI evidence. The agent should write a +temporary scenario for the feature instead of manually clicking through the UI. + +## Pick A Mode + +- Use `--mode before-after` for bug fixes, regressions, changed copy, changed + layout, or anything where main-vs-candidate comparison helps. This is the + default and runs baseline `origin/main` plus the candidate worktree. +- Use `--mode feature` for new pages, new workflows, or new UI states that do + not exist on main. This runs only the candidate lane. +- Do not use `proof:ui` to inspect contributor-provided screenshots, videos, or + logs. Review those artifacts directly and cite what they prove or fail to + prove. + +## Scenario Shape + +Create a temporary scenario under `.artifacts/proof-scenarios/`: + +```js +export default async function scenario({ baseURL, expect, page, proof }) { + await proof.step("01 skills list", async () => { + await page.goto(`${baseURL}/skills`); + await expect(page.getByText("Skills")).toBeVisible(); + }); +} +``` + +Each `proof.step()` captures a screenshot after the step. The runner compares +`origin/main` to the current worktree by default in `before-after` mode. + +## Commands + +Dry-run the plan first. Before/after mode is the default: + +```sh +bun run proof:ui -- --mode before-after --scenario .artifacts/proof-scenarios/my-fix.pw.ts --dry-run +``` + +For new feature proof, run candidate-only: + +```sh +bun run proof:ui -- --mode feature --scenario .artifacts/proof-scenarios/my-feature.pw.ts --dry-run +``` + +Run real desktop proof on a Crabbox-owned provider: + +```sh +bun run proof:ui -- --mode before-after --scenario .artifacts/proof-scenarios/my-fix.pw.ts --provider hetzner +``` + +Artifacts are written under `.artifacts/clawhub-ui-proof//` with +screenshots, videos when available, `summary.json`, and `report.md`. Feature +mode has only candidate artifacts. Promote only broadly useful scenarios into +committed `e2e/proofs/`. + +## Publish To A PR + +When UI proof should appear on a GitHub PR, publish the completed proof run +instead of posting local paths: + +```sh +bun run proof:publish -- --proof-dir .artifacts/clawhub-ui-proof/ --target-pr +``` + +`proof:publish` copies the selected screenshots, video preview GIFs when +present, MP4s, `summary.json`, and `report.md` to the `qa-artifacts` branch, +then upserts a marker-backed PR comment with inline screenshots/previews and +linked MP4s. Use `--dry-run` first when drafting or checking the comment body. diff --git a/.agents/skills/crabbox/SKILL.md b/.agents/skills/crabbox/SKILL.md new file mode 100644 index 00000000..63311151 --- /dev/null +++ b/.agents/skills/crabbox/SKILL.md @@ -0,0 +1,51 @@ +--- +name: crabbox +description: Use when ClawHub needs remote Linux validation, CI-parity checks, broad Bun gates, hosted-service checks, desktop/VNC inspection, or Crabbox lease cleanup. +--- + +# Crabbox + +Crabbox is ClawHub's agent-facing isolation layer. Use direct `blacksmith` +commands only as a backend emergency fallback; normal agents should go through +the repo scripts below. + +## Fast Checks + +Run from the repo root: + +```sh +bun run crabbox:run -- --help +bun run crabbox:warmup -- --provider blacksmith-testbox --blacksmith-org openclaw --blacksmith-workflow .github/workflows/ci-check-testbox.yml --blacksmith-job check +``` + +The wrapper prefers `../crabbox/bin/crabbox` when present and rejects stale +binaries that do not support the Blacksmith Testbox provider. For desktop UI +proof, use a Crabbox-owned provider such as `hetzner` or `aws`; the +`blacksmith-testbox` provider cannot expose VNC, screenshots, or desktop +artifacts. + +## Common Remote Validation + +Broad ClawHub gates: + +```sh +bun run crabbox:run -- --provider blacksmith-testbox --shell -- "bun run ci:static" +bun run crabbox:run -- --provider blacksmith-testbox --shell -- "VITE_CONVEX_URL=https://example.invalid bun run coverage" +``` + +Reusable desktop lease: + +```sh +bun run crabbox:warmup -- --provider hetzner --desktop --browser --class standard --idle-timeout 60m --ttl 120m +bun run crabbox:run -- --provider hetzner --id --keep --shell -- "bun run test" +bun run crabbox:stop -- --provider hetzner +``` + +## Cleanup + +Stop leases created for the task before handoff unless the user asked to keep +one open for WebVNC inspection: + +```sh +bun run crabbox:stop -- --provider +``` diff --git a/.crabbox.yaml b/.crabbox.yaml new file mode 100644 index 00000000..4d2eccc1 --- /dev/null +++ b/.crabbox.yaml @@ -0,0 +1,32 @@ +profile: clawhub-check +provider: blacksmith-testbox +blacksmith: + org: openclaw + workflow: .github/workflows/ci-check-testbox.yml + job: check + ref: main + idleTimeout: 90m + debug: false +sync: + delete: true + checksum: false + gitSeed: true + fingerprint: true + baseRef: main + exclude: + - .artifacts + - .codex + - .DS_Store + - coverage + - dist + - dist-ssr + - node_modules + - playwright-report + - test-results +env: + allow: + - CI + - NODE_OPTIONS + - CLAWHUB_* + - VITE_CONVEX_URL + - VITE_CONVEX_SITE_URL diff --git a/.github/workflows/ci-check-testbox.yml b/.github/workflows/ci-check-testbox.yml index eb8c1977..08e7bb76 100644 --- a/.github/workflows/ci-check-testbox.yml +++ b/.github/workflows/ci-check-testbox.yml @@ -1,4 +1,4 @@ -name: Blacksmith Testbox +name: Crabbox Testbox Backend on: workflow_dispatch: diff --git a/.gitignore b/.gitignore index 2d88d5ef..b40ede4f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules .DS_Store .bun-build *.bun-build +.artifacts/ .cache/ .data/ bin/docs-list @@ -40,8 +41,12 @@ skills-lock.json !.agents/skills/ !.agents/skills/convex*/ !.agents/skills/convex*/** -!.agents/skills/blacksmith-testbox/ -!.agents/skills/blacksmith-testbox/** +!.agents/skills/crabbox/ +!.agents/skills/crabbox/** +!.agents/skills/clawhub-ui-proof/ +!.agents/skills/clawhub-ui-proof/** +!.agents/skills/clawhub-pr-maintainer/ +!.agents/skills/clawhub-pr-maintainer/** skills/* .codex/* !.codex/environments/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f757c443..05095cac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -160,29 +160,26 @@ bun run --cwd packages/clawhub verify These are the same checks that run in CI (`.github/workflows/ci.yml`). -### Blacksmith Testbox checks +### Crabbox remote checks -Maintainers with Blacksmith access can run the same checks in a warmed Testbox -instead of spending local CPU: +Maintainers can run the same checks in a Crabbox lease instead of spending local +CPU. ClawHub uses Crabbox as the agent-facing command surface; the Testbox +workflow is only the backend for the default Blacksmith provider. ```bash -export CLAWHUB_TESTBOX=1 -blacksmith testbox warmup ci-check-testbox.yml --ref main --idle-timeout 90 -bun run testbox:claim -- --id -bun run testbox:sanity -- --id -bun run testbox:run -- --id -- bun run lint -bun run testbox:run -- --id -- bun run test -bun run testbox:run -- --id -- bun run build +bun run crabbox:warmup -- --provider blacksmith-testbox +bun run crabbox:run -- --provider blacksmith-testbox --shell -- "bun run lint" +bun run crabbox:run -- --provider blacksmith-testbox --shell -- "bun run test" +bun run crabbox:run -- --provider blacksmith-testbox --shell -- "bun run build" ``` -Use the `tbx_...` id from the current warmup output. The wrapper refuses ids -that are missing the local SSH key or were claimed by a different checkout. +Use `--id ` with `crabbox:run` when reusing an existing warmed lease, +and stop disposable leases with `bun run crabbox:stop -- --provider +`. Use `CLAWHUB_LOCAL_CHECK_MODE=throttled` or `CLAWHUB_LOCAL_CHECK_MODE=full` as the explicit local escape hatch when you intentionally want laptop-side proof. -If Blacksmith auth/org access is missing, report that instead of falling back +If Crabbox auth/provider access is missing, report that instead of falling back to a broad local gate that can bog down a dev machine. -For the initial bootstrap only, the Testbox workflow must land on `main` before -`blacksmith testbox warmup ci-check-testbox.yml --ref ` can dispatch it. **PR guidelines:** diff --git a/package.json b/package.json index 6d890fa8..559f51d2 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,10 @@ "clawscan:local": "bun scripts/local-clawscan-dry-run.ts", "convex:deploy": "bunx convex deploy --typecheck=disable --yes", "coverage": "vitest run --coverage", + "crabbox:hydrate": "node scripts/crabbox-wrapper.mjs actions hydrate", + "crabbox:run": "node scripts/crabbox-wrapper.mjs run", + "crabbox:stop": "node scripts/crabbox-wrapper.mjs stop", + "crabbox:warmup": "node scripts/crabbox-wrapper.mjs warmup", "dataset:snapshot": "bun scripts/security-dataset/export-snapshot.ts", "dataset:snapshot:prod:dry-run": "bun scripts/security-dataset/export-snapshot.ts --prod --limit 10 --dry-run", "deadcode:ci": "bun run deadcode:knip", @@ -41,6 +45,8 @@ "lint:oxlint": "oxlint --type-aware --tsconfig ./tsconfig.oxlint.json ./src ./convex ./packages/clawhub/src ./packages/clawhub-mod/src ./packages/schema/src", "preinstall": "bunx only-allow bun", "preview": "bun --bun vite preview", + "proof:publish": "node scripts/ui-proof-publish.mjs", + "proof:ui": "node scripts/ui-proof.mjs", "release:clawhub:cli:npm:check": "node scripts/clawhub-cli-npm-release-check.mjs", "seed:dev": "bun run setup:worktree -- --quiet && bun scripts/dev-worktree.ts --seed-only", "setup:worktree": "bun scripts/setup-worktree.ts", @@ -51,9 +57,6 @@ "test:pw": "playwright test", "test:ui-contract": "vitest run src/__tests__/ui-design-contract.test.ts src/__tests__/header.test.tsx src/__tests__/home-route.test.tsx src/components/Footer.test.tsx src/lib/theme.test.tsx src/routes/-settings.test.tsx", "test:watch": "vitest", - "testbox:claim": "node scripts/blacksmith-testbox-runner.mjs --claim", - "testbox:run": "node scripts/blacksmith-testbox-runner.mjs", - "testbox:sanity": "node scripts/testbox-sync-sanity.mjs", "verify:convex-contract": "bun scripts/verify-convex-contract.ts" }, "dependencies": { diff --git a/scripts/blacksmith-testbox-runner.mjs b/scripts/blacksmith-testbox-runner.mjs deleted file mode 100644 index 9e87b52a..00000000 --- a/scripts/blacksmith-testbox-runner.mjs +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env node - -import { execFileSync, spawn as nodeSpawn } from "node:child_process"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; -import { - evaluateClawHubTestboxClaim, - evaluateLocalTestboxKey, - resolveTestboxId, - writeClawHubTestboxClaim, -} from "./blacksmith-testbox-state.mjs"; - -function git(args, cwd) { - return execFileSync("git", args, { cwd, encoding: "utf8" }); -} - -export function splitRunnerArgs(argv = []) { - const normalizedArgv = stripPackageManagerSeparator(argv); - const separatorIndex = normalizedArgv.indexOf("--"); - if (separatorIndex === -1) { - return { runnerArgs: normalizedArgv, commandArgs: [] }; - } - return { - runnerArgs: normalizedArgv.slice(0, separatorIndex), - commandArgs: normalizedArgv.slice(separatorIndex + 1), - }; -} - -function stripPackageManagerSeparator(argv) { - if (argv[0] === "--") return argv.slice(1); - const separatorIndex = argv.indexOf("--"); - const next = argv[separatorIndex + 1]; - if ( - separatorIndex > 0 && - (next === "--id" || - next === "--testbox-id" || - next?.startsWith("--id=") || - next?.startsWith("--testbox-id=")) - ) { - return [...argv.slice(0, separatorIndex), ...argv.slice(separatorIndex + 1)]; - } - return argv; -} - -export function buildBlacksmithRunArgs({ commandArgs, testboxId }) { - const command = commandArgs.join(" ").trim(); - if (!command) return []; - return ["testbox", "run", "--id", testboxId, command]; -} - -export function resolveTestboxSyncTimeoutMs(env = process.env) { - const raw = env.CLAWHUB_TESTBOX_SYNC_TIMEOUT_MS; - if (raw === undefined || raw === "") return 5 * 60 * 1000; - const parsed = Number(raw); - return Number.isFinite(parsed) && parsed >= 0 ? parsed : 5 * 60 * 1000; -} - -function hasClaimFlag(runnerArgs) { - return runnerArgs.includes("--claim") || runnerArgs.includes("--claim-fresh"); -} - -function stripRunnerOnlyFlags(runnerArgs) { - return runnerArgs.filter((arg) => arg !== "--claim" && arg !== "--claim-fresh"); -} - -function pipeChunk(stream, chunk) { - if (chunk) stream.write(chunk); -} - -function runBlacksmithWithSyncGuard({ args, cwd, env, spawn, stderr, stdout, syncTimeoutMs }) { - return new Promise((resolve) => { - const child = spawn("blacksmith", args, { - cwd, - env, - stdio: ["inherit", "pipe", "pipe"], - }); - let settled = false; - let syncingSince = 0; - let timedOut = false; - let timer; - - const finish = (code) => { - if (settled) return; - settled = true; - clearInterval(timer); - resolve(timedOut ? 124 : typeof code === "number" ? code : 1); - }; - - const handleOutput = (stream, chunk) => { - const text = String(chunk); - pipeChunk(stream, chunk); - if (text.includes("Syncing...")) { - syncingSince ||= Date.now(); - } else if (syncingSince && /\b(running|executing|command|pnpm|npm|yarn|bun)\b/iu.test(text)) { - syncingSince = 0; - } - }; - - child.stdout?.on("data", (chunk) => handleOutput(stdout, chunk)); - child.stderr?.on("data", (chunk) => handleOutput(stderr, chunk)); - child.on("error", (error) => { - stderr.write(`Failed to start blacksmith: ${error.message}\n`); - finish(1); - }); - child.on("close", (code) => finish(code)); - - timer = setInterval( - () => { - if (!syncingSince || syncTimeoutMs <= 0) return; - if (Date.now() - syncingSince < syncTimeoutMs) return; - stderr.write( - `Blacksmith Testbox sync produced no post-sync output for ${syncTimeoutMs}ms; terminating local runner. ` + - "Rerun with CLAWHUB_TESTBOX_SYNC_TIMEOUT_MS=0 to disable this guard.\n", - ); - timedOut = true; - syncingSince = 0; - child.kill?.("SIGTERM"); - }, - Math.min(Math.max(syncTimeoutMs, 1), 1000), - ); - }); -} - -export async function runBlacksmithTestboxRunner({ - argv = process.argv.slice(2), - cwd = process.cwd(), - env = process.env, - spawn = nodeSpawn, - stderr = process.stderr, - stdout = process.stdout, -} = {}) { - const { runnerArgs, commandArgs } = splitRunnerArgs(argv); - const shouldClaim = hasClaimFlag(runnerArgs); - const testboxId = resolveTestboxId({ argv: stripRunnerOnlyFlags(runnerArgs), env }); - if (!testboxId) { - stderr.write( - "Missing Testbox id. Pass `--id ` or set CLAWHUB_TESTBOX_ID from this session's warmup output.\n", - ); - return 2; - } - - const keyResult = evaluateLocalTestboxKey({ env, testboxId }); - if (!keyResult.ok) { - stderr.write(`${keyResult.problems.join("\n")}\n`); - stderr.write( - "Refusing to reuse a remote-visible Testbox without the local private key. Run:\n" + - " blacksmith testbox warmup ci-check-testbox.yml --ref main --idle-timeout 90\n", - ); - return 2; - } - - const root = git(["rev-parse", "--show-toplevel"], cwd).trim(); - if (path.resolve(cwd) !== path.resolve(root)) { - stderr.write( - `Refusing to run Testbox sync from ${cwd}; run from repo root ${root} so rsync does not mirror a subdirectory.\n`, - ); - return 2; - } - - if (shouldClaim) { - const claim = writeClawHubTestboxClaim({ cwd: root, env, testboxId }); - stdout.write(`ClawHub Testbox claim written: ${testboxId} -> ${claim.claimPath}\n`); - } else { - const claimResult = evaluateClawHubTestboxClaim({ cwd: root, env, testboxId }); - if (!claimResult.ok) { - stderr.write(`${claimResult.problems.join("\n")}\n`); - stderr.write( - "Refusing to run a Testbox that was not claimed by this ClawHub checkout. Run:\n" + - " blacksmith testbox warmup ci-check-testbox.yml --ref main --idle-timeout 90\n" + - " bun run testbox:claim -- --id \n", - ); - return 2; - } - } - - const blacksmithArgs = buildBlacksmithRunArgs({ commandArgs, testboxId }); - if (blacksmithArgs.length === 0) { - stdout.write(`Testbox local key and ClawHub claim ok: ${testboxId}\n`); - return 0; - } - - return await runBlacksmithWithSyncGuard({ - args: blacksmithArgs, - cwd, - env, - spawn, - stderr, - stdout, - syncTimeoutMs: resolveTestboxSyncTimeoutMs(env), - }); -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { - process.exitCode = await runBlacksmithTestboxRunner(); -} diff --git a/scripts/blacksmith-testbox-runner.test.mjs b/scripts/blacksmith-testbox-runner.test.mjs deleted file mode 100644 index e5aa66bc..00000000 --- a/scripts/blacksmith-testbox-runner.test.mjs +++ /dev/null @@ -1,76 +0,0 @@ -/* @vitest-environment node */ - -import { describe, expect, it } from "vitest"; -import { - buildBlacksmithRunArgs, - resolveTestboxSyncTimeoutMs, - splitRunnerArgs, -} from "./blacksmith-testbox-runner.mjs"; -import { evaluateTestboxSyncSanity, parseGitShortStatus } from "./testbox-sync-sanity.mjs"; - -describe("blacksmith-testbox-runner", () => { - it("splits runner args from remote command args", () => { - expect(splitRunnerArgs(["--id", "tbx_abc", "--", "bun", "run", "lint"])).toEqual({ - runnerArgs: ["--id", "tbx_abc"], - commandArgs: ["bun", "run", "lint"], - }); - }); - - it("tolerates package-manager separators before runner args", () => { - expect(splitRunnerArgs(["--", "--id", "tbx_abc", "--", "bun", "run", "lint"])).toEqual({ - runnerArgs: ["--id", "tbx_abc"], - commandArgs: ["bun", "run", "lint"], - }); - expect(splitRunnerArgs(["--claim", "--", "--id", "tbx_abc"])).toEqual({ - runnerArgs: ["--claim", "--id", "tbx_abc"], - commandArgs: [], - }); - }); - - it("builds blacksmith run args from command args", () => { - expect( - buildBlacksmithRunArgs({ - commandArgs: ["bun", "run", "test", "--", "convex/lib/skills.test.ts"], - testboxId: "tbx_abc", - }), - ).toEqual(["testbox", "run", "--id", "tbx_abc", "bun run test -- convex/lib/skills.test.ts"]); - }); - - it("uses a five minute sync timeout by default", () => { - expect(resolveTestboxSyncTimeoutMs({})).toBe(300_000); - expect(resolveTestboxSyncTimeoutMs({ CLAWHUB_TESTBOX_SYNC_TIMEOUT_MS: "0" })).toBe(0); - }); -}); - -describe("testbox-sync-sanity", () => { - it("parses tracked deletions from git status", () => { - expect(parseGitShortStatus(" D src/a.ts\n?? scratch.txt\nR old.ts -> new.ts\n")).toEqual([ - { line: " D src/a.ts", path: "src/a.ts", status: " D", trackedDeletion: true }, - { line: "?? scratch.txt", path: "scratch.txt", status: "??", trackedDeletion: false }, - { line: "R old.ts -> new.ts", path: "new.ts", status: "R ", trackedDeletion: false }, - ]); - }); - - it("fails when required root files are missing", () => { - const result = evaluateTestboxSyncSanity({ - cwd: "/repo", - statusRaw: "", - exists: () => false, - }); - - expect(result.ok).toBe(false); - expect(result.problems.join("\n")).toContain("missing required root files"); - }); - - it("fails on mass tracked deletions", () => { - const result = evaluateTestboxSyncSanity({ - cwd: "/repo", - statusRaw: " D a.ts\n D b.ts\n", - exists: () => true, - deletionThreshold: 2, - }); - - expect(result.ok).toBe(false); - expect(result.trackedDeletionCount).toBe(2); - }); -}); diff --git a/scripts/blacksmith-testbox-state.mjs b/scripts/blacksmith-testbox-state.mjs deleted file mode 100644 index 363ebdb7..00000000 --- a/scripts/blacksmith-testbox-state.mjs +++ /dev/null @@ -1,181 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; - -const DEFAULT_CLAWHUB_TESTBOX_CLAIM_TTL_MINUTES = 12 * 60; -const TESTBOX_ID_PATTERN = /^tbx_[a-z0-9]+$/u; -const CLAWHUB_TESTBOX_CLAIM_FILE = "clawhub-runner.json"; - -function parsePositiveInteger(value, fallback) { - if (!value) return fallback; - const parsed = Number.parseInt(value, 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; -} - -export function parseTestboxIdArg(argv = []) { - for (let index = 0; index < argv.length; index += 1) { - const value = argv[index]; - if (value === "--id" || value === "--testbox-id") { - return argv[index + 1] ?? ""; - } - if (value?.startsWith("--id=")) { - return value.slice("--id=".length); - } - if (value?.startsWith("--testbox-id=")) { - return value.slice("--testbox-id=".length); - } - } - return ""; -} - -export function resolveTestboxId({ argv = [], env = process.env } = {}) { - return ( - parseTestboxIdArg(argv) || - env.CLAWHUB_TESTBOX_ID || - env.BLACKSMITH_TESTBOX_ID || - env.TESTBOX_ID || - "" - ).trim(); -} - -export function resolveBlacksmithTestboxStateDir({ env = process.env, homeDir } = {}) { - if (env.CLAWHUB_BLACKSMITH_TESTBOX_STATE_DIR) { - return env.CLAWHUB_BLACKSMITH_TESTBOX_STATE_DIR; - } - const blacksmithHome = - env.BLACKSMITH_HOME || path.join(homeDir || env.HOME || process.cwd(), ".blacksmith"); - return path.join(blacksmithHome, "testboxes"); -} - -export function evaluateLocalTestboxKey({ - testboxId, - env = process.env, - exists = fs.existsSync, - homeDir, -} = {}) { - if (!testboxId) { - return { ok: true, checked: false, problems: [] }; - } - - const problems = []; - if (!TESTBOX_ID_PATTERN.test(testboxId)) { - problems.push(`invalid Testbox id: ${testboxId}`); - return { ok: false, checked: true, keyPath: "", problems, testboxId }; - } - - const stateDir = resolveBlacksmithTestboxStateDir({ env, homeDir }); - const testboxDir = path.join(stateDir, testboxId); - const keyPath = path.join(testboxDir, "id_ed25519"); - if (!exists(keyPath)) { - problems.push( - `local Testbox SSH key missing for ${testboxId}: expected ${keyPath}. ` + - "This id may be visible in `blacksmith testbox list` but unusable by this operator; warm a fresh box instead.", - ); - } - - return { - ok: problems.length === 0, - checked: true, - keyPath, - problems, - testboxDir, - testboxId, - }; -} - -export function resolveClawHubTestboxClaimPath({ testboxId, env = process.env, homeDir } = {}) { - const stateDir = resolveBlacksmithTestboxStateDir({ env, homeDir }); - return path.join(stateDir, testboxId, CLAWHUB_TESTBOX_CLAIM_FILE); -} - -export function evaluateClawHubTestboxClaim({ - testboxId, - cwd, - env = process.env, - exists = fs.existsSync, - now = () => new Date(), - readFile = fs.readFileSync, - homeDir, -} = {}) { - if (!testboxId) { - return { ok: true, checked: false, problems: [] }; - } - - const claimPath = resolveClawHubTestboxClaimPath({ testboxId, env, homeDir }); - const expectedRepoRoot = path.resolve(cwd || process.cwd()); - const maxAgeMinutes = parsePositiveInteger( - env.CLAWHUB_TESTBOX_CLAIM_TTL_MINUTES, - DEFAULT_CLAWHUB_TESTBOX_CLAIM_TTL_MINUTES, - ); - const problems = []; - - if (!exists(claimPath)) { - problems.push( - `ClawHub Testbox claim missing for ${testboxId}: expected ${claimPath}. ` + - "Do not reuse ids from `blacksmith testbox list`; warm a fresh box and claim it with " + - "`bun run testbox:claim -- --id `.", - ); - return { ok: false, checked: true, claimPath, expectedRepoRoot, problems, testboxId }; - } - - let claim; - try { - claim = JSON.parse(readFile(claimPath, "utf8")); - } catch (error) { - problems.push(`ClawHub Testbox claim is unreadable for ${testboxId}: ${error.message}`); - } - - const claimedRepoRoot = claim?.repoRoot ? path.resolve(claim.repoRoot) : ""; - if (!claimedRepoRoot) { - problems.push(`ClawHub Testbox claim is missing repoRoot for ${testboxId}: ${claimPath}`); - } else if (claimedRepoRoot !== expectedRepoRoot) { - problems.push( - `ClawHub Testbox claim repo mismatch for ${testboxId}: ` + - `claimed ${claimedRepoRoot}, current ${expectedRepoRoot}. ` + - "Warm and claim a fresh box for this checkout.", - ); - } - - const claimedAtMs = Date.parse(claim?.claimedAt ?? ""); - if (!Number.isFinite(claimedAtMs)) { - problems.push(`ClawHub Testbox claim is missing claimedAt for ${testboxId}: ${claimPath}`); - } else { - const ageMinutes = Math.floor((now().getTime() - claimedAtMs) / 60000); - if (ageMinutes > maxAgeMinutes) { - problems.push( - `ClawHub Testbox claim is stale for ${testboxId}: ${ageMinutes}m old, limit ${maxAgeMinutes}m. ` + - "Warm and claim a fresh box after crashes or long pauses.", - ); - } - } - - return { - ok: problems.length === 0, - checked: true, - claim, - claimPath, - expectedRepoRoot, - problems, - testboxId, - }; -} - -export function writeClawHubTestboxClaim({ - testboxId, - cwd, - env = process.env, - homeDir, - mkdir = fs.mkdirSync, - writeFile = fs.writeFileSync, - now = () => new Date(), -} = {}) { - const claimPath = resolveClawHubTestboxClaimPath({ testboxId, env, homeDir }); - const repoRoot = path.resolve(cwd || process.cwd()); - const payload = { - claimedAt: now().toISOString(), - repoRoot, - runnerVersion: 1, - }; - mkdir(path.dirname(claimPath), { recursive: true }); - writeFile(claimPath, `${JSON.stringify(payload, null, 2)}\n`); - return { claimPath, payload, testboxId }; -} diff --git a/scripts/blacksmith-testbox-state.test.mjs b/scripts/blacksmith-testbox-state.test.mjs deleted file mode 100644 index 385a407a..00000000 --- a/scripts/blacksmith-testbox-state.test.mjs +++ /dev/null @@ -1,116 +0,0 @@ -/* @vitest-environment node */ - -import path from "node:path"; -import { describe, expect, it } from "vitest"; -import { - evaluateClawHubTestboxClaim, - evaluateLocalTestboxKey, - parseTestboxIdArg, - resolveBlacksmithTestboxStateDir, - resolveClawHubTestboxClaimPath, - resolveTestboxId, - writeClawHubTestboxClaim, -} from "./blacksmith-testbox-state.mjs"; - -describe("blacksmith-testbox-state", () => { - it("parses testbox ids from runner args", () => { - expect(parseTestboxIdArg(["--id", "tbx_abc123"])).toBe("tbx_abc123"); - expect(parseTestboxIdArg(["--testbox-id=tbx_def456"])).toBe("tbx_def456"); - expect(parseTestboxIdArg(["--other"])).toBe(""); - }); - - it("prefers CLI id over env ids", () => { - expect( - resolveTestboxId({ - argv: ["--id", "tbx_cli"], - env: { CLAWHUB_TESTBOX_ID: "tbx_env", TESTBOX_ID: "tbx_fallback" }, - }), - ).toBe("tbx_cli"); - }); - - it("builds state paths from BLACKSMITH_HOME", () => { - expect(resolveBlacksmithTestboxStateDir({ env: { BLACKSMITH_HOME: "/tmp/bs" } })).toBe( - path.join("/tmp/bs", "testboxes"), - ); - expect( - resolveClawHubTestboxClaimPath({ - testboxId: "tbx_abc", - env: { BLACKSMITH_HOME: "/tmp/bs" }, - }), - ).toBe(path.join("/tmp/bs", "testboxes", "tbx_abc", "clawhub-runner.json")); - }); - - it("rejects ids without a local private key", () => { - const result = evaluateLocalTestboxKey({ - testboxId: "tbx_missing", - env: { BLACKSMITH_HOME: "/tmp/bs" }, - exists: () => false, - }); - - expect(result.ok).toBe(false); - expect(result.problems.join("\n")).toContain("local Testbox SSH key missing"); - }); - - it("validates claim repo root and freshness", () => { - const claim = JSON.stringify({ - claimedAt: "2026-04-30T00:00:00.000Z", - repoRoot: "/repo", - runnerVersion: 1, - }); - const result = evaluateClawHubTestboxClaim({ - testboxId: "tbx_claimed", - cwd: "/repo", - env: { BLACKSMITH_HOME: "/tmp/bs", CLAWHUB_TESTBOX_CLAIM_TTL_MINUTES: "60" }, - exists: () => true, - readFile: () => claim, - now: () => new Date("2026-04-30T00:30:00.000Z"), - }); - - expect(result.ok).toBe(true); - }); - - it("flags stale claims", () => { - const claim = JSON.stringify({ - claimedAt: "2026-04-30T00:00:00.000Z", - repoRoot: "/repo", - runnerVersion: 1, - }); - const result = evaluateClawHubTestboxClaim({ - testboxId: "tbx_stale", - cwd: "/repo", - env: { BLACKSMITH_HOME: "/tmp/bs", CLAWHUB_TESTBOX_CLAIM_TTL_MINUTES: "10" }, - exists: () => true, - readFile: () => claim, - now: () => new Date("2026-04-30T00:30:00.000Z"), - }); - - expect(result.ok).toBe(false); - expect(result.problems.join("\n")).toContain("claim is stale"); - }); - - it("writes a claim payload", () => { - let writtenPath = ""; - let writtenBody = ""; - const result = writeClawHubTestboxClaim({ - testboxId: "tbx_write", - cwd: "/repo", - env: { BLACKSMITH_HOME: "/tmp/bs" }, - mkdir: () => {}, - writeFile: (target, body) => { - writtenPath = target; - writtenBody = body; - }, - now: () => new Date("2026-04-30T00:00:00.000Z"), - }); - - expect(result.claimPath).toBe( - path.join("/tmp/bs", "testboxes", "tbx_write", "clawhub-runner.json"), - ); - expect(writtenPath).toBe(result.claimPath); - expect(JSON.parse(writtenBody)).toMatchObject({ - claimedAt: "2026-04-30T00:00:00.000Z", - repoRoot: "/repo", - runnerVersion: 1, - }); - }); -}); diff --git a/scripts/crabbox-wrapper.mjs b/scripts/crabbox-wrapper.mjs new file mode 100644 index 00000000..74a7e14c --- /dev/null +++ b/scripts/crabbox-wrapper.mjs @@ -0,0 +1,150 @@ +#!/usr/bin/env node +import { spawn, spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { dirname, relative, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const DESKTOP_COMMANDS = new Set(["artifacts", "desktop", "screenshot", "webvnc"]); +const DESKTOP_FLAGS = new Set(["--browser", "--desktop", "--screenshot", "--webvnc"]); + +export function normalizeCrabboxArgs(rawArgs = []) { + const args = [...rawArgs]; + if (args[0] === "--") { + args.shift(); + } + if (args[0] === "actions" && args[1] === "hydrate" && args[2] === "--") { + args.splice(2, 1); + } + return args; +} + +export function selectCrabboxBinary({ exists = existsSync, repoRoot }) { + const repoLocal = resolve(repoRoot, "../crabbox/bin/crabbox"); + return exists(repoLocal) ? repoLocal : "crabbox"; +} + +function commandAdvertised(help, command) { + return new RegExp(`(?:^|\\n|\\s)${command}(?:\\s|$)`, "u").test(help); +} + +export function inspectCrabboxCapabilities({ runHelp = "", topLevelHelp = "", versionText = "" }) { + const providers = ["aws", "hetzner", "blacksmith-testbox"].filter((provider) => + runHelp.includes(provider), + ); + return { + commands: { + artifacts: commandAdvertised(topLevelHelp, "artifacts"), + desktop: commandAdvertised(topLevelHelp, "desktop"), + screenshot: commandAdvertised(topLevelHelp, "screenshot"), + webvnc: commandAdvertised(topLevelHelp, "webvnc"), + }, + providers, + runHelp, + topLevelHelp, + versionText: versionText.trim(), + }; +} + +export function requiresDesktopSupport(args = []) { + return args.some((arg) => DESKTOP_COMMANDS.has(arg) || DESKTOP_FLAGS.has(arg)); +} + +export function assertRequiredCrabboxCapabilities(capabilities, { requireDesktop }) { + if (!capabilities.providers.includes("blacksmith-testbox")) { + throw new Error( + "selected Crabbox binary does not advertise provider blacksmith-testbox; refusing stale Crabbox binary", + ); + } + if (!requireDesktop) { + return; + } + const missing = Object.entries(capabilities.commands) + .filter(([, present]) => !present) + .map(([command]) => command); + if (missing.length > 0) { + throw new Error( + `selected Crabbox binary is missing desktop/artifacts support (${missing.join(", ")}); update the sibling Crabbox checkout or PATH binary`, + ); + } +} + +function checkedOutput(command, commandArgs, { cwd }) { + const result = spawnSync(command, commandArgs, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + return { + status: result.status ?? 1, + text: `${result.stdout ?? ""}${result.stderr ?? ""}`.trim(), + }; +} + +export function runCrabboxWrapper({ + argv = process.argv.slice(2), + cwd = dirname(fileURLToPath(import.meta.url)), + spawnImpl = spawn, +} = {}) { + const repoRoot = resolve(cwd, ".."); + const args = normalizeCrabboxArgs(argv); + const binary = selectCrabboxBinary({ repoRoot }); + const displayBinary = binary === "crabbox" ? "crabbox" : relative(repoRoot, binary); + const version = checkedOutput(binary, ["--version"], { cwd: repoRoot }); + const runHelp = checkedOutput(binary, ["run", "--help"], { cwd: repoRoot }); + const topLevelHelp = checkedOutput(binary, ["--help"], { cwd: repoRoot }); + + console.error( + `[crabbox] bin=${displayBinary} version=${version.text || "unknown"} providers=${ + inspectCrabboxCapabilities({ + runHelp: runHelp.text, + topLevelHelp: topLevelHelp.text, + versionText: version.text, + }).providers.join(",") || "unknown" + }`, + ); + + if (version.status !== 0 || runHelp.status !== 0 || topLevelHelp.status !== 0) { + console.error("[crabbox] selected binary failed basic --version/--help sanity checks"); + return 2; + } + + try { + assertRequiredCrabboxCapabilities( + inspectCrabboxCapabilities({ + runHelp: runHelp.text, + topLevelHelp: topLevelHelp.text, + versionText: version.text, + }), + { requireDesktop: requiresDesktopSupport(args) }, + ); + } catch (error) { + console.error(`[crabbox] ${error.message}`); + return 2; + } + + const child = spawnImpl(binary, args, { + cwd: repoRoot, + stdio: "inherit", + }); + + child.on("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + process.exit(code ?? 1); + }); + + child.on("error", (error) => { + console.error(`[crabbox] failed to execute ${displayBinary}: ${error.message}`); + process.exit(2); + }); + return undefined; +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + const exitCode = runCrabboxWrapper(); + if (typeof exitCode === "number") { + process.exitCode = exitCode; + } +} diff --git a/scripts/crabbox-wrapper.test.mjs b/scripts/crabbox-wrapper.test.mjs new file mode 100644 index 00000000..3ae0c7ff --- /dev/null +++ b/scripts/crabbox-wrapper.test.mjs @@ -0,0 +1,73 @@ +/* @vitest-environment node */ + +import { describe, expect, it } from "vitest"; +import { + assertRequiredCrabboxCapabilities, + inspectCrabboxCapabilities, + normalizeCrabboxArgs, + requiresDesktopSupport, + selectCrabboxBinary, +} from "./crabbox-wrapper.mjs"; + +describe("crabbox-wrapper", () => { + it("normalizes package-manager separators without eating remote command separators", () => { + expect(normalizeCrabboxArgs(["--", "run", "--provider", "blacksmith-testbox"])).toEqual([ + "run", + "--provider", + "blacksmith-testbox", + ]); + expect(normalizeCrabboxArgs(["actions", "hydrate", "--", "--id", "cbx_123"])).toEqual([ + "actions", + "hydrate", + "--id", + "cbx_123", + ]); + expect(normalizeCrabboxArgs(["run", "--provider", "hetzner", "--", "bun", "test"])).toEqual([ + "run", + "--provider", + "hetzner", + "--", + "bun", + "test", + ]); + }); + + it("prefers a sibling Crabbox checkout when it exists", () => { + expect( + selectCrabboxBinary({ + exists: (candidate) => candidate === "/repo/crabbox/bin/crabbox", + repoRoot: "/repo/clawhub", + }), + ).toBe("/repo/crabbox/bin/crabbox"); + }); + + it("rejects stale Crabbox binaries that cannot wrap Blacksmith Testbox", () => { + const capabilities = inspectCrabboxCapabilities({ + runHelp: "Usage: crabbox run --provider aws|hetzner", + topLevelHelp: "Usage: crabbox run", + versionText: "crabbox v0.4.0", + }); + + expect(() => + assertRequiredCrabboxCapabilities(capabilities, { requireDesktop: false }), + ).toThrow(/blacksmith-testbox/u); + }); + + it("rejects stale Crabbox binaries for desktop UI proof commands", () => { + const capabilities = inspectCrabboxCapabilities({ + runHelp: "Usage: crabbox run --provider aws|hetzner|blacksmith-testbox", + topLevelHelp: "Usage: crabbox run\ncrabbox screenshot", + versionText: "crabbox v0.5.0", + }); + + expect(() => assertRequiredCrabboxCapabilities(capabilities, { requireDesktop: true })).toThrow( + /desktop.*artifacts/u, + ); + }); + + it("requires desktop support when run flags request browser or desktop leases", () => { + expect(requiresDesktopSupport(["run", "--provider", "hetzner", "--desktop"])).toBe(true); + expect(requiresDesktopSupport(["warmup", "--provider", "hetzner", "--browser"])).toBe(true); + expect(requiresDesktopSupport(["run", "--provider", "blacksmith-testbox"])).toBe(false); + }); +}); diff --git a/scripts/testbox-sync-sanity.mjs b/scripts/testbox-sync-sanity.mjs deleted file mode 100644 index d267fcb5..00000000 --- a/scripts/testbox-sync-sanity.mjs +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env node - -import { execFileSync } from "node:child_process"; -import fs from "node:fs"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; -import { - evaluateClawHubTestboxClaim, - evaluateLocalTestboxKey, - resolveTestboxId, -} from "./blacksmith-testbox-state.mjs"; - -const DEFAULT_DELETION_THRESHOLD = 200; -const REQUIRED_ROOT_FILES = ["package.json", "bun.lock", ".gitignore"]; - -function parseBooleanEnv(value) { - return ["1", "true", "yes", "on"].includes(value?.trim().toLowerCase() ?? ""); -} - -function parsePositiveInteger(value, fallback) { - if (!value) return fallback; - const parsed = Number.parseInt(value, 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; -} - -export function parseGitShortStatus(raw) { - return raw - .split(/\r?\n/u) - .map((line) => line.trimEnd()) - .filter(Boolean) - .map((line) => { - const status = line.slice(0, 2); - const rawPath = line.slice(3); - return { - line, - path: rawPath.includes(" -> ") ? (rawPath.split(" -> ").at(-1) ?? rawPath) : rawPath, - status, - trackedDeletion: status.includes("D") && status !== "??", - }; - }); -} - -export function evaluateTestboxSyncSanity({ - cwd, - statusRaw, - exists = fs.existsSync, - deletionThreshold = DEFAULT_DELETION_THRESHOLD, - allowMassDeletions = false, -}) { - const missingRootFiles = REQUIRED_ROOT_FILES.filter((file) => !exists(path.join(cwd, file))); - const statusEntries = parseGitShortStatus(statusRaw); - const trackedDeletions = statusEntries.filter((entry) => entry.trackedDeletion); - const problems = []; - - if (missingRootFiles.length > 0) { - problems.push(`missing required root files: ${missingRootFiles.join(", ")}`); - } - if (!allowMassDeletions && trackedDeletions.length >= deletionThreshold) { - const examples = trackedDeletions - .slice(0, 8) - .map((entry) => entry.path) - .join(", "); - problems.push( - `remote git status has ${trackedDeletions.length} tracked deletions ` + - `(threshold ${deletionThreshold}); examples: ${examples}`, - ); - } - - return { - ok: problems.length === 0, - missingRootFiles, - problems, - statusEntryCount: statusEntries.length, - trackedDeletionCount: trackedDeletions.length, - }; -} - -function git(args, cwd) { - return execFileSync("git", args, { cwd, encoding: "utf8" }); -} - -export function runTestboxSyncSanity({ - cwd = process.cwd(), - env = process.env, - argv = process.argv.slice(2), - stdout = process.stdout, - stderr = process.stderr, -} = {}) { - const root = git(["rev-parse", "--show-toplevel"], cwd).trim(); - const statusRaw = git(["status", "--short", "--untracked-files=all"], root); - const testboxId = resolveTestboxId({ argv, env }); - const keyResult = evaluateLocalTestboxKey({ env, testboxId }); - const claimResult = evaluateClawHubTestboxClaim({ cwd: root, env, testboxId }); - const result = evaluateTestboxSyncSanity({ - cwd: root, - statusRaw, - deletionThreshold: parsePositiveInteger( - env.CLAWHUB_TESTBOX_DELETION_THRESHOLD, - DEFAULT_DELETION_THRESHOLD, - ), - allowMassDeletions: parseBooleanEnv(env.CLAWHUB_TESTBOX_ALLOW_MASS_DELETIONS), - }); - result.problems.push(...keyResult.problems); - result.problems.push(...claimResult.problems); - result.ok = result.problems.length === 0; - - if (!result.ok) { - stderr.write(`Testbox sync sanity failed:\n- ${result.problems.join("\n- ")}\n`); - stderr.write( - "Warm a fresh box, keep using the id from this session, " + - "or rerun from a clean repo root before spending a gate.\n", - ); - return 1; - } - - if (keyResult.checked) { - stdout.write(`Testbox local key and ClawHub claim ok: ${keyResult.testboxId}\n`); - } - stdout.write( - `Testbox sync sanity ok: ${result.statusEntryCount} changed entries, ` + - `${result.trackedDeletionCount} tracked deletions.\n`, - ); - return 0; -} - -if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { - process.exitCode = runTestboxSyncSanity(); -} diff --git a/scripts/ui-proof-publish.mjs b/scripts/ui-proof-publish.mjs new file mode 100644 index 00000000..5de0ac4a --- /dev/null +++ b/scripts/ui-proof-publish.mjs @@ -0,0 +1,506 @@ +#!/usr/bin/env node +import { execFileSync, spawnSync } from "node:child_process"; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const DEFAULT_ARTIFACT_BRANCH = "qa-artifacts"; +const DEFAULT_MARKER = ""; +const DEFAULT_REPO = "openclaw/clawhub"; + +export function parseProofPublishArgs(argv = []) { + const opts = { + artifactBranch: DEFAULT_ARTIFACT_BRANCH, + marker: DEFAULT_MARKER, + repo: process.env.GITHUB_REPOSITORY || DEFAULT_REPO, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = argv[index + 1]; + if (arg === "--") { + continue; + } + if (arg === "--artifact-branch") { + opts.artifactBranch = requireValue(arg, next); + index += 1; + } else if (arg === "--artifact-root") { + opts.artifactRoot = requireValue(arg, next); + index += 1; + } else if (arg === "--artifact-url") { + opts.artifactUrl = requireValue(arg, next); + index += 1; + } else if (arg === "--dry-run") { + opts.dryRun = true; + } else if (arg === "--marker") { + opts.marker = requireValue(arg, next); + index += 1; + } else if (arg === "--proof-dir") { + opts.proofDir = requireValue(arg, next); + index += 1; + } else if (arg === "--repo") { + opts.repo = requireValue(arg, next); + index += 1; + } else if (arg === "--request-source") { + opts.requestSource = requireValue(arg, next); + index += 1; + } else if (arg === "--run-url") { + opts.runUrl = requireValue(arg, next); + index += 1; + } else if (arg === "--target-pr") { + opts.targetPr = requireValue(arg, next); + index += 1; + } else { + throw new Error(`Unknown proof:publish argument: ${arg}`); + } + } + if (!opts.proofDir) throw new Error("proof:publish requires --proof-dir "); + if (!opts.targetPr) throw new Error("proof:publish requires --target-pr "); + if (!/^[0-9]+$/u.test(opts.targetPr)) { + throw new Error(`--target-pr must be numeric, got ${opts.targetPr}`); + } + return opts; +} + +function requireValue(flag, value) { + if (!value || value.startsWith("--")) { + throw new Error(`${flag} requires a value`); + } + return value; +} + +function readJson(filePath) { + return JSON.parse(readFileSync(filePath, "utf8")); +} + +function assertInside(parentDir, candidatePath, label) { + const relative = path.relative(parentDir, candidatePath); + if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) { + return candidatePath; + } + throw new Error(`${label} escapes proof directory: ${candidatePath}`); +} + +function normalizeTargetPath(targetPath) { + const normalized = path.posix.normalize(String(targetPath).replaceAll("\\", "/")); + if ( + normalized === "." || + normalized === "" || + normalized.startsWith("../") || + normalized.includes("/../") || + normalized.startsWith("/") || + /^[A-Za-z]:/u.test(normalized) + ) { + throw new Error(`Invalid artifact target path: ${targetPath}`); + } + return normalized; +} + +function encodePathForUrl(input) { + return input + .split("/") + .filter(Boolean) + .map((part) => encodeURIComponent(part)) + .join("/"); +} + +function artifactUrl(rawBase, artifact) { + return `${rawBase}/${encodePathForUrl(artifact.targetPath)}`; +} + +function relativeArtifactPath(proofDir, sourcePath, label) { + const source = assertInside(proofDir, path.resolve(sourcePath), label); + if (!existsSync(source)) return undefined; + if (!statSync(source).isFile()) { + throw new Error(`${label} is not a file: ${sourcePath}`); + } + return source; +} + +function pushArtifact(artifacts, proofDir, artifact) { + const source = relativeArtifactPath( + proofDir, + path.resolve(proofDir, artifact.path), + artifact.label ?? artifact.path, + ); + if (!source) { + if (artifact.required === false) return; + throw new Error(`Missing required artifact: ${artifact.path}`); + } + artifacts.push({ + ...artifact, + label: artifact.label ?? artifact.path, + source, + targetPath: normalizeTargetPath(artifact.targetPath ?? artifact.path), + }); +} + +export async function buildUiProofEvidence({ proofDir }) { + const resolvedProofDir = path.resolve(proofDir); + const summaryPath = path.join(resolvedProofDir, "summary.json"); + const reportPath = path.join(resolvedProofDir, "report.md"); + const summary = readJson(summaryPath); + if (summary.status === "dry-run") { + throw new Error("proof:publish requires a non-dry-run proof directory"); + } + + const artifacts = []; + pushArtifact(artifacts, resolvedProofDir, { + kind: "metadata", + label: "ClawHub UI proof summary", + path: "summary.json", + targetPath: "summary.json", + }); + pushArtifact(artifacts, resolvedProofDir, { + kind: "report", + label: "ClawHub UI proof report", + path: "report.md", + required: existsSync(reportPath), + targetPath: "report.md", + }); + + for (const lane of summary.lanes ?? []) { + for (const [index, step] of (lane.steps ?? []).entries()) { + if (!step.screenshot) continue; + const targetName = `${step.slug || `step-${index + 1}`}.png`; + pushArtifact(artifacts, resolvedProofDir, { + alt: step.name, + index, + kind: "screenshot", + label: step.name, + lane: lane.name, + path: path.posix.join(lane.name, step.screenshot.replaceAll("\\", "/")), + status: step.status, + targetPath: path.posix.join(lane.name, targetName), + width: 420, + }); + } + const videoPath = lane.videoPath + ? path.relative(resolvedProofDir, lane.videoPath).replaceAll("\\", "/") + : path.posix.join(lane.name, "full-run.mp4"); + pushArtifact(artifacts, resolvedProofDir, { + kind: "fullVideo", + label: `${lane.name} full run`, + lane: lane.name, + path: videoPath, + required: false, + targetPath: path.posix.join(lane.name, "full-run.mp4"), + }); + pushArtifact(artifacts, resolvedProofDir, { + alt: `${lane.name} full run preview`, + kind: "videoPreview", + label: `${lane.name} full run preview`, + lane: lane.name, + path: path.posix.join(lane.name, "full-run.gif"), + required: false, + targetPath: path.posix.join(lane.name, "full-run.gif"), + width: 720, + }); + } + + return { + artifacts, + proofDir: resolvedProofDir, + summary, + }; +} + +function artifactsByLane(evidence, kind) { + const lanes = new Map(); + for (const artifact of evidence.artifacts) { + if (artifact.kind !== kind || !artifact.lane) continue; + const lane = lanes.get(artifact.lane) ?? []; + lane.push(artifact); + lanes.set(artifact.lane, lane); + } + for (const lane of lanes.values()) { + lane.sort((left, right) => Number(left.index ?? 0) - Number(right.index ?? 0)); + } + return lanes; +} + +function renderBeforeAfterScreenshots({ evidence, rawBase }) { + const lanes = artifactsByLane(evidence, "screenshot"); + const baseline = lanes.get("baseline") ?? []; + const candidate = lanes.get("candidate") ?? []; + const rows = []; + const count = Math.max(baseline.length, candidate.length); + for (let index = 0; index < count; index += 1) { + const left = baseline[index]; + const right = candidate[index]; + if (!left || !right) continue; + const width = Math.min(Number(left.width ?? right.width ?? 420) || 420, 720); + rows.push( + `| ${left.label} | ${right.label} |`, + "| --- | --- |", + `| ${left.alt ?? left.label} | ${right.alt ?? right.label} |`, + "", + ); + } + return rows.join("\n"); +} + +function renderFeatureScreenshots({ evidence, rawBase }) { + return evidence.artifacts + .filter((artifact) => artifact.kind === "screenshot") + .map((artifact) => { + const width = Math.min(Number(artifact.width ?? 720) || 720, 900); + return [ + `**${artifact.label}**`, + "", + `${artifact.alt ?? artifact.label}`, + "", + ].join("\n"); + }) + .join("\n"); +} + +function renderVideoLinks({ evidence, rawBase }) { + const links = evidence.artifacts + .filter((artifact) => artifact.kind === "fullVideo") + .map((artifact) => `- [${artifact.label}](${artifactUrl(rawBase, artifact)})`); + return links.length ? ["Full videos:", ...links, ""].join("\n") : ""; +} + +function renderVideoPreviews({ evidence, rawBase }) { + const previews = evidence.artifacts.filter((artifact) => artifact.kind === "videoPreview"); + if (!previews.length) return ""; + return [ + "Inline video previews:", + "", + ...previews.map((artifact) => { + const width = Math.min(Number(artifact.width ?? 720) || 720, 900); + return [ + `**${artifact.label}**`, + "", + `${artifact.alt ?? artifact.label}`, + "", + ].join("\n"); + }), + ].join("\n"); +} + +export function renderUiProofComment({ + artifactRoot, + artifactUrl: actionsArtifactUrl, + evidence, + marker, + rawBase, + requestSource, + runUrl, + treeUrl, +}) { + const { summary } = evidence; + const lines = [ + marker, + "## ClawHub UI Proof", + "", + `Status: \`${summary.status ?? "unknown"}\``, + `Mode: \`${summary.mode ?? "before-after"}\``, + `Scenario: \`${summary.scenario ?? "unknown"}\``, + `Provider: \`${summary.provider ?? "unknown"}\``, + ]; + if (summary.mode === "feature") { + lines.push("Baseline: not run for feature proof."); + } else { + lines.push(`Baseline: \`${summary.baseline ?? "origin/main"}\``); + } + lines.push(`Candidate: \`${summary.candidate ?? "worktree"}\``); + if (requestSource) lines.push(`Trigger: \`${requestSource}\``); + if (runUrl) lines.push(`Run: ${runUrl}`); + if (actionsArtifactUrl) lines.push(`Actions artifact: ${actionsArtifactUrl}`); + lines.push(""); + + const screenshotSection = + summary.mode === "feature" + ? renderFeatureScreenshots({ evidence, rawBase }) + : renderBeforeAfterScreenshots({ evidence, rawBase }); + if (screenshotSection) lines.push(screenshotSection); + const videoPreviews = renderVideoPreviews({ evidence, rawBase }); + if (videoPreviews) lines.push(videoPreviews); + const videoLinks = renderVideoLinks({ evidence, rawBase }); + if (videoLinks) lines.push(videoLinks); + lines.push( + `Raw proof files: ${treeUrl ?? `https://github.com/${process.env.GITHUB_REPOSITORY ?? DEFAULT_REPO}/tree/qa-artifacts/${artifactRoot}`}`, + ); + return `${lines.join("\n").replace(/\n{3,}/gu, "\n\n")}\n`; +} + +function run(command, args, options = {}) { + return execFileSync(command, args, { + encoding: "utf8", + stdio: options.stdio ?? ["ignore", "pipe", "inherit"], + ...options, + }); +} + +function runStatus(command, args, options = {}) { + const result = spawnSync(command, args, { + stdio: "ignore", + ...options, + }); + if (result.error) throw result.error; + return result.status ?? 1; +} + +function remoteUrl(repo) { + const token = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN; + return token + ? `https://x-access-token:${token}@github.com/${repo}.git` + : `https://github.com/${repo}.git`; +} + +function publishArtifactFiles({ artifactBranch, artifactRoot, evidence, repo }) { + const worktree = mkdtempSync(path.join(tmpdir(), "clawhub-ui-proof-artifacts-")); + const safeArtifactRoot = normalizeTargetPath(artifactRoot); + try { + run("git", ["init", "--quiet", worktree]); + run("git", ["-C", worktree, "config", "user.name", "github-actions[bot]"]); + run("git", [ + "-C", + worktree, + "config", + "user.email", + "41898282+github-actions[bot]@users.noreply.github.com", + ]); + run("git", ["-C", worktree, "remote", "add", "origin", remoteUrl(repo)]); + try { + run("git", ["-C", worktree, "fetch", "--quiet", "origin", artifactBranch]); + run("git", ["-C", worktree, "checkout", "--quiet", "-B", artifactBranch, "FETCH_HEAD"]); + } catch { + run("git", ["-C", worktree, "checkout", "--quiet", "--orphan", artifactBranch]); + } + + const destinationRoot = path.join(worktree, safeArtifactRoot); + for (const artifact of evidence.artifacts) { + const destination = assertInside( + destinationRoot, + path.resolve(destinationRoot, artifact.targetPath), + `Artifact target ${artifact.targetPath}`, + ); + mkdirSync(path.dirname(destination), { recursive: true }); + copyFileSync(artifact.source, destination); + } + + run("git", ["-C", worktree, "add", safeArtifactRoot]); + const hasChanges = runStatus("git", ["-C", worktree, "diff", "--cached", "--quiet"]) !== 0; + if (hasChanges) { + run("git", [ + "-C", + worktree, + "commit", + "--quiet", + "-m", + `qa: publish ClawHub UI proof for ${safeArtifactRoot}`, + ]); + run("git", ["-C", worktree, "push", "--quiet", "origin", `HEAD:${artifactBranch}`]); + } else { + console.log("No ClawHub UI proof artifact changes to publish."); + } + } finally { + rmSync(worktree, { force: true, recursive: true }); + } + return safeArtifactRoot; +} + +function upsertPrComment({ body, marker, prNumber, repo }) { + run("gh", ["api", `repos/${repo}/pulls/${prNumber}`, "--jq", ".number"]); + const commentId = run("gh", [ + "api", + "--paginate", + `repos/${repo}/issues/${prNumber}/comments`, + "--jq", + `.[] | select(.body | contains("${marker}")) | .id`, + ]) + .trim() + .split("\n") + .findLast((line) => line.length > 0); + const bodyDir = mkdtempSync(path.join(tmpdir(), "clawhub-ui-proof-comment-")); + const bodyFile = path.join(bodyDir, "body.md"); + writeFileSync(bodyFile, body); + try { + if (commentId) { + const payloadFile = `${bodyFile}.json`; + writeFileSync(payloadFile, JSON.stringify({ body })); + try { + run("gh", [ + "api", + "--method", + "PATCH", + `repos/${repo}/issues/comments/${commentId}`, + "--input", + payloadFile, + ]); + console.log(`Updated ClawHub UI proof comment on PR #${prNumber}.`); + return; + } catch { + console.warn( + `Could not update existing ClawHub UI proof comment ${commentId}; creating a new one.`, + ); + } + } + run("gh", ["pr", "comment", prNumber, "--repo", repo, "--body-file", bodyFile], { + stdio: "inherit", + }); + console.log(`Created ClawHub UI proof comment on PR #${prNumber}.`); + } finally { + rmSync(bodyDir, { force: true, recursive: true }); + } +} + +function defaultArtifactRoot({ proofDir, targetPr }) { + return normalizeTargetPath( + path.posix.join("clawhub-ui-proof", `pr-${targetPr}`, path.basename(path.resolve(proofDir))), + ); +} + +export async function publishUiProof(rawArgs = process.argv.slice(2)) { + const opts = parseProofPublishArgs(rawArgs); + const evidence = await buildUiProofEvidence({ proofDir: opts.proofDir }); + const artifactRoot = opts.artifactRoot ?? defaultArtifactRoot(opts); + const rawBase = `https://raw.githubusercontent.com/${opts.repo}/${opts.artifactBranch}/${encodePathForUrl(artifactRoot)}`; + const treeUrl = `https://github.com/${opts.repo}/tree/${opts.artifactBranch}/${encodePathForUrl(artifactRoot)}`; + const body = renderUiProofComment({ + artifactRoot, + artifactUrl: opts.artifactUrl, + evidence, + marker: opts.marker, + rawBase, + requestSource: opts.requestSource, + runUrl: opts.runUrl, + treeUrl, + }); + if (opts.dryRun) { + console.log(body); + return { body, status: "dry-run" }; + } + const publishedRoot = publishArtifactFiles({ + artifactBranch: opts.artifactBranch, + artifactRoot, + evidence, + repo: opts.repo, + }); + upsertPrComment({ + body, + marker: opts.marker, + prNumber: opts.targetPr, + repo: opts.repo, + }); + return { artifactRoot: publishedRoot, body, status: "published" }; +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + publishUiProof().catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/ui-proof-publish.test.mjs b/scripts/ui-proof-publish.test.mjs new file mode 100644 index 00000000..f02ad742 --- /dev/null +++ b/scripts/ui-proof-publish.test.mjs @@ -0,0 +1,167 @@ +/* @vitest-environment node */ + +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + buildUiProofEvidence, + parseProofPublishArgs, + renderUiProofComment, +} from "./ui-proof-publish.mjs"; + +const tempDirs = []; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { force: true, recursive: true }); + } +}); + +async function fixtureProof({ mode = "before-after", status = "pass" } = {}) { + const dir = await fsp.mkdtemp(path.join(os.tmpdir(), "clawhub-proof-publish-")); + tempDirs.push(dir); + await fsp.mkdir(path.join(dir, "baseline", "screenshots"), { recursive: true }); + await fsp.mkdir(path.join(dir, "candidate", "screenshots"), { recursive: true }); + await fsp.writeFile(path.join(dir, "baseline", "screenshots", "skills.png"), "baseline"); + await fsp.writeFile(path.join(dir, "candidate", "screenshots", "skills.png"), "candidate"); + await fsp.writeFile(path.join(dir, "baseline", "full-run.gif"), "baseline gif"); + await fsp.writeFile(path.join(dir, "candidate", "full-run.gif"), "candidate gif"); + await fsp.writeFile(path.join(dir, "candidate", "full-run.mp4"), "video"); + await fsp.writeFile(path.join(dir, "report.md"), "# ClawHub UI Proof\nStatus: pass\n"); + await fsp.writeFile( + path.join(dir, "summary.json"), + `${JSON.stringify( + { + baseline: "origin/main", + candidate: "worktree", + generatedAt: "2026-05-13T12:00:00.000Z", + lanes: + mode === "feature" + ? [ + { + name: "candidate", + ref: "worktree", + status, + steps: [ + { + lane: "candidate", + name: "candidate skills page", + screenshot: "screenshots/skills.png", + slug: "skills", + status: "pass", + }, + ], + videoPath: path.join(dir, "candidate", "full-run.mp4"), + }, + ] + : [ + { + name: "baseline", + ref: "origin/main", + status, + steps: [ + { + lane: "baseline", + name: "baseline skills page", + screenshot: "screenshots/skills.png", + slug: "skills", + status: "pass", + }, + ], + }, + { + name: "candidate", + ref: "worktree", + status, + steps: [ + { + lane: "candidate", + name: "candidate skills page", + screenshot: "screenshots/skills.png", + slug: "skills", + status: "pass", + }, + ], + videoPath: path.join(dir, "candidate", "full-run.mp4"), + }, + ], + mode, + outputDir: dir, + provider: "hetzner", + scenario: ".artifacts/proof-scenarios/demo.pw.ts", + status, + }, + null, + 2, + )}\n`, + ); + return dir; +} + +describe("ui-proof-publish", () => { + it("parses publish defaults", () => { + expect( + parseProofPublishArgs(["--proof-dir", ".artifacts/proof", "--target-pr", "123"]), + ).toMatchObject({ + artifactBranch: "qa-artifacts", + marker: "", + proofDir: ".artifacts/proof", + repo: "openclaw/clawhub", + targetPr: "123", + }); + }); + + it("renders before/after proof comments with inline screenshots and video links", async () => { + const proofDir = await fixtureProof(); + const evidence = await buildUiProofEvidence({ proofDir }); + const body = renderUiProofComment({ + artifactRoot: "clawhub-ui-proof/pr-123/run", + evidence, + marker: "", + rawBase: + "https://raw.githubusercontent.com/openclaw/clawhub/qa-artifacts/clawhub-ui-proof/pr-123/run", + treeUrl: "https://github.com/openclaw/clawhub/tree/qa-artifacts/clawhub-ui-proof/pr-123/run", + }); + + expect(body).toContain(""); + expect(body).toContain("Mode: `before-after`"); + expect(body).toContain("| baseline skills page | candidate skills page |"); + expect(body).toContain( + ' { + const proofDir = await fixtureProof({ mode: "feature" }); + const evidence = await buildUiProofEvidence({ proofDir }); + const body = renderUiProofComment({ + artifactRoot: "clawhub-ui-proof/pr-123/run", + evidence, + marker: "", + rawBase: + "https://raw.githubusercontent.com/openclaw/clawhub/qa-artifacts/clawhub-ui-proof/pr-123/run", + treeUrl: "https://github.com/openclaw/clawhub/tree/qa-artifacts/clawhub-ui-proof/pr-123/run", + }); + + expect(body).toContain("Mode: `feature`"); + expect(body).toContain("**candidate skills page**"); + expect(body).toContain(" { + const proofDir = await fixtureProof({ status: "dry-run" }); + await expect(buildUiProofEvidence({ proofDir })).rejects.toThrow( + "proof:publish requires a non-dry-run proof directory", + ); + }); +}); diff --git a/scripts/ui-proof-runtime.mjs b/scripts/ui-proof-runtime.mjs new file mode 100644 index 00000000..df8ed125 --- /dev/null +++ b/scripts/ui-proof-runtime.mjs @@ -0,0 +1,138 @@ +#!/usr/bin/env node +import fs from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export function sanitizeStepName(name) { + const slug = String(name) + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/gu, "-") + .replace(/^-+|-+$/gu, ""); + return slug || "step"; +} + +function uniqueSlug(base, used) { + let slug = base; + let suffix = 2; + while (used.has(slug)) { + slug = `${base}-${suffix}`; + suffix += 1; + } + used.add(slug); + return slug; +} + +export function createProofContext({ lane, outputDir, page }) { + const steps = []; + const used = new Set(); + return { + get steps() { + return steps; + }, + async step(name, fn) { + const slug = uniqueSlug(sanitizeStepName(name), used); + const screenshot = path.join("screenshots", `${slug}.png`); + const screenshotPath = path.join(outputDir, screenshot); + await fs.mkdir(path.dirname(screenshotPath), { recursive: true }); + const entry = { + lane, + name, + screenshot, + slug, + status: "pass", + }; + try { + await fn(); + await page.screenshot({ fullPage: true, path: screenshotPath }); + } catch (error) { + entry.status = "fail"; + entry.error = error instanceof Error ? error.message : String(error); + try { + await page.screenshot({ fullPage: true, path: screenshotPath }); + } catch { + // Keep the original failure. Missing screenshots are obvious in the manifest. + } + steps.push(entry); + throw error; + } + steps.push(entry); + }, + }; +} + +function parseRuntimeArgs(argv) { + const opts = {}; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = argv[index + 1]; + if (arg === "run-scenario") { + continue; + } + if (arg === "--base-url") { + opts.baseURL = next; + index += 1; + } else if (arg === "--lane") { + opts.lane = next; + index += 1; + } else if (arg === "--output-dir") { + opts.outputDir = next; + index += 1; + } else if (arg === "--scenario") { + opts.scenario = next; + index += 1; + } else { + throw new Error(`Unknown ui-proof-runtime argument: ${arg}`); + } + } + return opts; +} + +export async function runUiProofScenario({ baseURL, lane, outputDir, scenario }) { + if (!baseURL || !lane || !outputDir || !scenario) { + throw new Error("run-scenario requires --base-url, --lane, --output-dir, and --scenario"); + } + const { chromium, expect } = await import("@playwright/test"); + await fs.mkdir(outputDir, { recursive: true }); + const browser = await chromium.launch({ + args: ["--window-position=0,0", "--window-size=1280,900"], + headless: false, + }); + const page = await browser.newPage({ viewport: { height: 900, width: 1280 } }); + const proof = createProofContext({ lane, outputDir, page }); + let status = "pass"; + let error; + try { + const imported = await import(pathToFileURL(path.resolve(scenario)).href); + const scenarioFn = imported.default ?? imported.run; + if (typeof scenarioFn !== "function") { + throw new Error("Proof scenario must export a default function or named run function."); + } + await scenarioFn({ baseURL, expect, lane, page, proof }); + } catch (caught) { + status = "fail"; + error = caught instanceof Error ? caught.message : String(caught); + throw caught; + } finally { + await browser.close().catch(() => {}); + const summary = { + baseURL, + error, + lane, + scenario: path.resolve(scenario), + status, + steps: proof.steps, + }; + await fs.writeFile( + path.join(outputDir, "proof-steps.json"), + `${JSON.stringify(summary, null, 2)}\n`, + ); + } +} + +if (process.argv[2] === "run-scenario") { + runUiProofScenario(parseRuntimeArgs(process.argv.slice(2))).catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/ui-proof-runtime.test.mjs b/scripts/ui-proof-runtime.test.mjs new file mode 100644 index 00000000..66ac2dec --- /dev/null +++ b/scripts/ui-proof-runtime.test.mjs @@ -0,0 +1,45 @@ +/* @vitest-environment node */ + +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { createProofContext, sanitizeStepName } from "./ui-proof-runtime.mjs"; + +describe("ui-proof-runtime", () => { + it("sanitizes step names into stable screenshot filenames", () => { + expect(sanitizeStepName("01 Skills / List")).toBe("01-skills-list"); + expect(sanitizeStepName(" ")).toBe("step"); + }); + + it("captures a screenshot and manifest entry for every proof step", async () => { + const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "clawhub-proof-runtime-")); + const calls = []; + const page = { + async screenshot(options) { + calls.push(options); + await fs.writeFile(options.path, "png"); + }, + }; + const proof = createProofContext({ lane: "candidate", outputDir, page }); + + await proof.step("01 Skills / List", async () => { + calls.push({ action: "visited" }); + }); + + expect(proof.steps).toEqual([ + { + lane: "candidate", + name: "01 Skills / List", + screenshot: "screenshots/01-skills-list.png", + slug: "01-skills-list", + status: "pass", + }, + ]); + expect(calls[0]).toEqual({ action: "visited" }); + expect(calls[1]).toMatchObject({ fullPage: true }); + await expect( + fs.readFile(path.join(outputDir, "screenshots", "01-skills-list.png"), "utf8"), + ).resolves.toBe("png"); + }); +}); diff --git a/scripts/ui-proof.mjs b/scripts/ui-proof.mjs new file mode 100644 index 00000000..a3a49a0a --- /dev/null +++ b/scripts/ui-proof.mjs @@ -0,0 +1,635 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const DEFAULT_BASELINE = "origin/main"; +const DEFAULT_CANDIDATE = "worktree"; +const DEFAULT_MODE = "before-after"; +const DEFAULT_PROVIDER = "hetzner"; +const DEFAULT_CLASS = "standard"; +const DEFAULT_IDLE_TIMEOUT = "60m"; +const DEFAULT_TTL = "120m"; +const DEFAULT_VIDEO_DURATION = "60"; +const DEFAULT_PORTS = { + baseline: 4317, + candidate: 4318, +}; +const DEFAULT_PUBLIC_ENV = { + VITE_CONVEX_SITE_URL: "https://wry-manatee-359.convex.site", + VITE_CONVEX_URL: "https://wry-manatee-359.convex.cloud", +}; + +export function parseProofUiArgs(argv = []) { + const opts = { + baseline: DEFAULT_BASELINE, + candidate: DEFAULT_CANDIDATE, + dryRun: false, + idleTimeout: DEFAULT_IDLE_TIMEOUT, + keepLease: false, + machineClass: DEFAULT_CLASS, + mode: DEFAULT_MODE, + provider: DEFAULT_PROVIDER, + scenario: "", + skipInstall: false, + ttl: DEFAULT_TTL, + videoDuration: DEFAULT_VIDEO_DURATION, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = argv[index + 1]; + if (arg === "--") { + continue; + } + if (arg === "--baseline") { + opts.baseline = requireValue(arg, next); + index += 1; + } else if (arg === "--candidate") { + opts.candidate = requireValue(arg, next); + index += 1; + } else if (arg === "--class" || arg === "--machine-class") { + opts.machineClass = requireValue(arg, next); + index += 1; + } else if (arg === "--crabbox-bin") { + opts.crabboxBin = requireValue(arg, next); + index += 1; + } else if (arg === "--dry-run") { + opts.dryRun = true; + } else if (arg === "--idle-timeout") { + opts.idleTimeout = requireValue(arg, next); + index += 1; + } else if (arg === "--keep-lease") { + opts.keepLease = true; + } else if (arg === "--lease-id") { + opts.leaseId = requireValue(arg, next); + index += 1; + } else if (arg === "--mode") { + opts.mode = requireValue(arg, next); + index += 1; + } else if (arg === "--output-dir") { + opts.outputDir = requireValue(arg, next); + index += 1; + } else if (arg === "--provider") { + opts.provider = requireValue(arg, next); + index += 1; + } else if (arg === "--scenario") { + opts.scenario = requireValue(arg, next); + index += 1; + } else if (arg === "--skip-install") { + opts.skipInstall = true; + } else if (arg === "--ttl") { + opts.ttl = requireValue(arg, next); + index += 1; + } else if (arg === "--video-duration") { + opts.videoDuration = requireValue(arg, next); + index += 1; + } else { + throw new Error(`Unknown proof:ui argument: ${arg}`); + } + } + if (!opts.scenario) { + throw new Error("proof:ui requires --scenario "); + } + if (!["before-after", "feature"].includes(opts.mode)) { + throw new Error(`Unknown proof:ui mode: ${opts.mode}`); + } + return opts; +} + +function requireValue(flag, value) { + if (!value || value.startsWith("--")) { + throw new Error(`${flag} requires a value`); + } + return value; +} + +function timestamp(now) { + return now().toISOString().replace(/[:.]/gu, "-"); +} + +export function buildProofUiPlan({ now = () => new Date(), opts, repoRoot }) { + const outputDir = path.resolve( + repoRoot, + opts.outputDir ?? path.join(".artifacts", "clawhub-ui-proof", timestamp(now)), + ); + const candidateLane = { + name: "candidate", + outputDir: path.join(outputDir, "candidate"), + port: DEFAULT_PORTS.candidate, + ref: opts.candidate, + }; + const lanes = + opts.mode === "feature" + ? [candidateLane] + : [ + { + name: "baseline", + outputDir: path.join(outputDir, "baseline"), + port: DEFAULT_PORTS.baseline, + ref: opts.baseline, + }, + candidateLane, + ]; + return { + baseline: opts.baseline, + candidate: opts.candidate, + mode: opts.mode, + outputDir, + provider: opts.provider, + scenario: path.resolve(repoRoot, opts.scenario), + lanes, + }; +} + +async function defaultCommandRunner(command, args, options = {}) { + return await new Promise((resolve, reject) => { + const child = spawn(command, args, { + ...options, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk) => { + const text = chunk.toString(); + stdout += text; + if (options.stdio === "inherit") { + process.stdout.write(text); + } + }); + child.stderr?.on("data", (chunk) => { + const text = chunk.toString(); + stderr += text; + if (options.stdio === "inherit") { + process.stderr.write(text); + } + }); + child.on("error", reject); + child.on("close", (code, signal) => { + if (code === 0) { + resolve({ stdout, stderr }); + return; + } + const detail = signal ? `signal ${signal}` : `exit code ${code ?? "unknown"}`; + const error = new Error(`${command} ${args.join(" ")} failed with ${detail}`); + error.stdout = stdout; + error.stderr = stderr; + reject(error); + }); + }); +} + +function crabboxInvocation({ opts, repoRoot }) { + if (opts.crabboxBin) { + return { argsPrefix: [], command: opts.crabboxBin }; + } + return { + argsPrefix: [path.join(repoRoot, "scripts", "crabbox-wrapper.mjs")], + command: "node", + }; +} + +function extractLeaseId(output) { + return output.match(/\b(?:cbx_[a-f0-9]+|tbx_[A-Za-z0-9_-]+)\b/u)?.[0]; +} + +function extractRemoteOutputDir(output) { + return output.match(/^__CLAWHUB_UI_PROOF_REMOTE_OUTPUT__=(.+)$/mu)?.[1]?.trim(); +} + +function shellQuote(value) { + return `'${String(value).replaceAll("'", "'\\''")}'`; +} + +export function renderRemoteLaneScript({ lane, opts, plan, scenarioText }) { + const scenarioB64 = Buffer.from(scenarioText, "utf8").toString("base64"); + const runtimePath = path.join("scripts", "ui-proof-runtime.mjs"); + const laneRemoteDir = `.artifacts/clawhub-ui-proof/remote-${path.basename(plan.outputDir)}/${lane.name}`; + const appRootSetup = + lane.name === "baseline" + ? [ + `git fetch --no-tags origin "+refs/heads/main:refs/remotes/origin/main" || true`, + `git worktree remove -f .artifacts/clawhub-ui-proof/worktrees/${lane.name} >/dev/null 2>&1 || true`, + `rm -rf .artifacts/clawhub-ui-proof/worktrees/${lane.name}`, + `git worktree add --detach .artifacts/clawhub-ui-proof/worktrees/${lane.name} ${shellQuote(lane.ref)}`, + `app_root="$PWD/.artifacts/clawhub-ui-proof/worktrees/${lane.name}"`, + ].join("\n") + : `app_root="$PWD"`; + const envExports = Object.entries(DEFAULT_PUBLIC_ENV) + .map(([key, value]) => `export ${key}=${shellQuote(value)}`) + .join("\n"); + + return `set -euo pipefail +export DISPLAY="\${DISPLAY:-:99}" +remote_out="$PWD/${laneRemoteDir}" +rm -rf "$remote_out" +mkdir -p "$remote_out" +echo "__CLAWHUB_UI_PROOF_REMOTE_OUTPUT__=$remote_out" +video_pid="" +cleanup_proof_processes() { + if [ -n "$video_pid" ]; then + kill -INT "$video_pid" >/dev/null 2>&1 || true + wait "$video_pid" >/dev/null 2>&1 || true + video_pid="" + fi + if [ -f "$remote_out/preview.pid" ]; then + kill "$(cat "$remote_out/preview.pid")" >/dev/null 2>&1 || true + rm -f "$remote_out/preview.pid" + fi + return 0 +} +trap cleanup_proof_processes EXIT +scenario_file="$remote_out/scenario.mjs" +printf %s ${shellQuote(scenarioB64)} | base64 -d > "$scenario_file" +${appRootSetup} +${envExports} +export CLAWHUB_UI_PROOF_LANE=${shellQuote(lane.name)} +export BUN_INSTALL="\${BUN_INSTALL:-$HOME/.bun}" +export PATH="$BUN_INSTALL/bin:$PATH" +if ! command -v bun >/dev/null 2>&1; then + if ! command -v unzip >/dev/null 2>&1; then + if command -v apt-get >/dev/null 2>&1; then + if command -v sudo >/dev/null 2>&1; then + sudo env DEBIAN_FRONTEND=noninteractive apt-get update >/dev/null + sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y unzip >/dev/null + else + DEBIAN_FRONTEND=noninteractive apt-get update >/dev/null + DEBIAN_FRONTEND=noninteractive apt-get install -y unzip >/dev/null + fi + else + echo "bun is not installed on this Crabbox image and unzip is unavailable." >&2 + exit 127 + fi + fi + if ! command -v curl >/dev/null 2>&1; then + echo "bun is not installed on this Crabbox image and curl is unavailable to install it." >&2 + exit 127 + fi + curl -fsSL https://bun.sh/install | bash +fi +export PATH="$BUN_INSTALL/bin:$PATH" +if [ ${opts.skipInstall ? "1" : "0"} -ne 1 ]; then + bun install --frozen-lockfile + if [ "$app_root" != "$PWD" ]; then + (cd "$app_root" && bun install --frozen-lockfile) + fi + bunx playwright install chromium > "$remote_out/playwright-install.log" 2>&1 +fi +(cd "$app_root" && bun run build > "$remote_out/build.log" 2>&1) +(cd "$app_root" && bun run preview -- --host 127.0.0.1 --port ${lane.port} > "$remote_out/preview.log" 2>&1 & echo $! > "$remote_out/preview.pid") +bun -e ${shellQuote(`const url = "http://127.0.0.1:${lane.port}"; +const started = Date.now(); +async function tick() { + try { + const res = await fetch(url); + if (res.status < 500) process.exit(0); + } catch {} + if (Date.now() - started > 60000) { + console.error("preview did not become ready: " + url); + process.exit(1); + } + setTimeout(tick, 500); +} +tick();`)} +if command -v ffmpeg >/dev/null 2>&1; then + display_input="$DISPLAY" + case "$display_input" in + *.*) ;; + *) display_input="$display_input.0" ;; + esac + ffmpeg -hide_banner -loglevel error -y -f x11grab -framerate 15 -i "$display_input" -t ${shellQuote( + opts.videoDuration, + )} -pix_fmt yuv420p "$remote_out/full-run.mp4" > "$remote_out/ffmpeg.log" 2>&1 & + video_pid=$! +else + echo "ffmpeg missing; full-run.mp4 skipped" > "$remote_out/ffmpeg.log" +fi +status=0 +bun ${shellQuote(runtimePath)} run-scenario \ + --scenario "$scenario_file" \ + --base-url ${shellQuote(`http://127.0.0.1:${lane.port}`)} \ + --lane ${shellQuote(lane.name)} \ + --output-dir "$remote_out" || status=$? +manifest_status="" +if [ -f "$remote_out/proof-steps.json" ]; then + manifest_status="$(CLAWHUB_UI_PROOF_MANIFEST="$remote_out/proof-steps.json" bun -e 'const fs = require("fs"); const path = process.env.CLAWHUB_UI_PROOF_MANIFEST; process.stdout.write(JSON.parse(fs.readFileSync(path, "utf8")).status || "unknown");' 2>/dev/null || true)" + if [ "$manifest_status" = "pass" ]; then + status=0 + elif [ "$manifest_status" = "fail" ] && [ "$status" -eq 0 ]; then + status=1 + fi +fi +if [ -n "$video_pid" ]; then + kill -INT "$video_pid" >/dev/null 2>&1 || true + wait "$video_pid" >/dev/null 2>&1 || true + video_pid="" +fi +if [ -f "$remote_out/preview.pid" ]; then + kill "$(cat "$remote_out/preview.pid")" >/dev/null 2>&1 || true + rm -f "$remote_out/preview.pid" +fi +cat > "$remote_out/lane-summary.json" < { + console.error(`warning: failed to stop Crabbox lease ${leaseId}: ${error.message}`); + }); +} + +export async function runProofUi({ + args = process.argv.slice(2), + commandRunner = defaultCommandRunner, + now = () => new Date(), + repoRoot = process.cwd(), +} = {}) { + const opts = parseProofUiArgs(args); + const plan = buildProofUiPlan({ now, opts, repoRoot }); + const scenarioText = await fs.readFile(plan.scenario, "utf8"); + const summary = { + baseline: plan.baseline, + candidate: plan.candidate, + generatedAt: now().toISOString(), + lanes: plan.lanes.map((lane) => ({ + localOutputDir: lane.outputDir, + name: lane.name, + ref: lane.ref, + status: opts.dryRun ? "planned" : "pending", + })), + mode: plan.mode, + outputDir: plan.outputDir, + provider: plan.provider, + scenario: plan.scenario, + status: opts.dryRun ? "dry-run" : "pending", + }; + if (opts.dryRun) { + await writeSummaryAndReport({ outputDir: plan.outputDir, summary }); + return { + outputDir: plan.outputDir, + status: "dry-run", + summaryPath: path.join(plan.outputDir, "summary.json"), + }; + } + + const invocation = crabboxInvocation({ opts, repoRoot }); + const { created, leaseId } = await warmupLease({ commandRunner, invocation, opts, repoRoot }); + summary.crabbox = { createdLease: created, leaseId }; + try { + const lanes = []; + for (const lane of plan.lanes) { + lanes.push( + await runLane({ + commandRunner, + invocation, + lane, + leaseId, + opts, + plan, + repoRoot, + scenarioText, + }), + ); + } + summary.lanes = lanes; + summary.status = lanes.every((lane) => lane.status === "pass") ? "pass" : "fail"; + } finally { + if (!opts.keepLease && created) { + await stopLease({ commandRunner, invocation, leaseId, opts, repoRoot }); + } + } + await writeSummaryAndReport({ outputDir: plan.outputDir, summary }); + return { + outputDir: plan.outputDir, + status: summary.status, + summaryPath: path.join(plan.outputDir, "summary.json"), + }; +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + runProofUi() + .then((result) => { + console.log(`ClawHub UI proof ${result.status}: ${result.outputDir}`); + if (result.status === "fail") { + process.exitCode = 1; + } + }) + .catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/ui-proof.test.mjs b/scripts/ui-proof.test.mjs new file mode 100644 index 00000000..fffa5abe --- /dev/null +++ b/scripts/ui-proof.test.mjs @@ -0,0 +1,255 @@ +/* @vitest-environment node */ + +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + buildProofUiPlan, + buildRsyncSshCommand, + parseProofUiArgs, + renderRemoteLaneScript, + runProofUi, +} from "./ui-proof.mjs"; + +describe("ui-proof", () => { + it("parses proof defaults for temporary scenarios", () => { + expect(parseProofUiArgs(["--scenario", ".artifacts/proof-scenarios/demo.pw.ts"])).toMatchObject( + { + baseline: "origin/main", + candidate: "worktree", + mode: "before-after", + provider: "hetzner", + scenario: ".artifacts/proof-scenarios/demo.pw.ts", + }, + ); + }); + + it("parses explicit proof modes and rejects unknown modes", () => { + expect( + parseProofUiArgs([ + "--mode", + "before-after", + "--scenario", + ".artifacts/proof-scenarios/demo.pw.ts", + ]), + ).toMatchObject({ + mode: "before-after", + }); + expect( + parseProofUiArgs([ + "--mode", + "feature", + "--scenario", + ".artifacts/proof-scenarios/demo.pw.ts", + ]), + ).toMatchObject({ + mode: "feature", + }); + expect(() => + parseProofUiArgs(["--mode", "smoke", "--scenario", ".artifacts/proof-scenarios/demo.pw.ts"]), + ).toThrow("Unknown proof:ui mode: smoke"); + }); + + it("builds a before/after plan with stable lane output directories", () => { + const plan = buildProofUiPlan({ + now: () => new Date("2026-05-12T12:34:56.000Z"), + opts: parseProofUiArgs(["--scenario", ".artifacts/proof-scenarios/demo.pw.ts"]), + repoRoot: "/repo/clawhub", + }); + + expect(plan.mode).toBe("before-after"); + expect(plan.outputDir).toBe( + "/repo/clawhub/.artifacts/clawhub-ui-proof/2026-05-12T12-34-56-000Z", + ); + expect(plan.lanes.map((lane) => [lane.name, lane.ref, lane.outputDir])).toEqual([ + ["baseline", "origin/main", `${plan.outputDir}/baseline`], + ["candidate", "worktree", `${plan.outputDir}/candidate`], + ]); + }); + + it("builds a feature plan with candidate-only lane output", () => { + const plan = buildProofUiPlan({ + now: () => new Date("2026-05-12T12:34:56.000Z"), + opts: parseProofUiArgs([ + "--mode", + "feature", + "--scenario", + ".artifacts/proof-scenarios/demo.pw.ts", + ]), + repoRoot: "/repo/clawhub", + }); + + expect(plan.mode).toBe("feature"); + expect(plan.lanes.map((lane) => [lane.name, lane.ref, lane.outputDir])).toEqual([ + ["candidate", "worktree", `${plan.outputDir}/candidate`], + ]); + }); + + it("dry-runs without invoking Crabbox and writes the planned report", async () => { + const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "clawhub-proof-")); + const scenario = path.join(repoRoot, ".artifacts/proof-scenarios/demo.pw.ts"); + await fs.mkdir(path.dirname(scenario), { recursive: true }); + await fs.writeFile(scenario, "export default async function demo() {}\n"); + const commands = []; + + const result = await runProofUi({ + args: ["--scenario", scenario, "--dry-run"], + commandRunner: async (command, commandArgs) => { + commands.push([command, commandArgs]); + return { stdout: "", stderr: "" }; + }, + now: () => new Date("2026-05-12T12:34:56.000Z"), + repoRoot, + }); + + expect(commands).toEqual([]); + expect(result.status).toBe("dry-run"); + const report = await fs.readFile(path.join(result.outputDir, "report.md"), "utf8"); + expect(report).toContain("Mode: `before-after`"); + expect(report).toContain("Baseline: `origin/main`"); + expect(report).toContain("Candidate: `worktree`"); + expect(report).toContain("Dry run"); + await expect( + fs.readFile(path.join(result.outputDir, "summary.json"), "utf8"), + ).resolves.toContain('"scenario"'); + }); + + it("dry-runs feature proof with candidate-only report language", async () => { + const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "clawhub-proof-")); + const scenario = path.join(repoRoot, ".artifacts/proof-scenarios/demo.pw.ts"); + await fs.mkdir(path.dirname(scenario), { recursive: true }); + await fs.writeFile(scenario, "export default async function demo() {}\n"); + + const result = await runProofUi({ + args: ["--mode", "feature", "--scenario", scenario, "--dry-run"], + commandRunner: async () => { + throw new Error("Crabbox should not run during dry-run"); + }, + now: () => new Date("2026-05-12T12:34:56.000Z"), + repoRoot, + }); + + const report = await fs.readFile(path.join(result.outputDir, "report.md"), "utf8"); + expect(report).toContain("Mode: `feature`"); + expect(report).toContain("Baseline: not run for feature proof."); + expect(report).toContain("Candidate: `worktree`"); + expect(report).not.toContain("### baseline"); + expect(report).toContain("### candidate"); + await expect( + fs.readFile(path.join(result.outputDir, "summary.json"), "utf8"), + ).resolves.toContain('"mode": "feature"'); + }); + + it("treats a passing proof manifest as authoritative after a Crabbox transport error", async () => { + const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "clawhub-proof-")); + const scenario = path.join(repoRoot, ".artifacts/proof-scenarios/demo.pw.ts"); + await fs.mkdir(path.dirname(scenario), { recursive: true }); + await fs.writeFile(scenario, "export default async function demo() {}\n"); + let laneIndex = 0; + + const result = await runProofUi({ + args: ["--scenario", scenario], + commandRunner: async (command, commandArgs) => { + if (commandArgs.includes("warmup")) { + return { stdout: "leased cbx_deadbeef", stderr: "" }; + } + if (commandArgs.includes("run")) { + const lane = laneIndex === 0 ? "baseline" : "candidate"; + laneIndex += 1; + const error = new Error(`transport failed for ${lane}`); + error.stdout = `__CLAWHUB_UI_PROOF_REMOTE_OUTPUT__=/remote/${lane}\n`; + error.stderr = ""; + throw error; + } + if (commandArgs.includes("inspect")) { + return { + stdout: JSON.stringify({ + sshHost: "203.0.113.10", + sshKey: "/tmp/crabbox key", + sshPort: 22, + sshUser: "crabbox", + }), + stderr: "", + }; + } + if (command === "rsync") { + const localOutputDir = commandArgs.at(-1).replace(/\/$/u, ""); + const lane = localOutputDir.endsWith("baseline") ? "baseline" : "candidate"; + await fs.mkdir(localOutputDir, { recursive: true }); + await fs.writeFile( + path.join(localOutputDir, "proof-steps.json"), + `${JSON.stringify({ + lane, + status: "pass", + steps: [ + { + name: `${lane} /skills`, + screenshot: "screenshots/skills.png", + status: "pass", + }, + ], + })}\n`, + ); + return { stdout: "", stderr: "" }; + } + if (commandArgs.includes("stop")) { + return { stdout: "", stderr: "" }; + } + throw new Error(`unexpected command: ${command} ${commandArgs.join(" ")}`); + }, + now: () => new Date("2026-05-12T12:34:56.000Z"), + repoRoot, + }); + + expect(result.status).toBe("pass"); + await expect( + fs.readFile(path.join(result.outputDir, "summary.json"), "utf8"), + ).resolves.toContain('"status": "pass"'); + }); + + it("quotes Crabbox ssh key paths with spaces for rsync artifact copying", () => { + const { ssh } = buildRsyncSshCommand({ + sshHost: "203.0.113.10", + sshKey: "/Users/patrick/Library/Application Support/crabbox/testboxes/cbx_123/id_ed25519", + sshPort: 22, + sshUser: "crabbox", + }); + + expect(ssh).toContain( + "-i '/Users/patrick/Library/Application Support/crabbox/testboxes/cbx_123/id_ed25519'", + ); + }); + + it("bootstraps Bun on desktop Crabbox images before running proof commands", () => { + const script = renderRemoteLaneScript({ + lane: { + name: "candidate", + outputDir: "/tmp/out/candidate", + port: 4318, + ref: "worktree", + }, + opts: { + skipInstall: false, + videoDuration: "1", + }, + plan: { + outputDir: "/tmp/out", + }, + scenarioText: "export default async function scenario() {}\n", + }); + + expect(script).toContain("command -v bun"); + expect(script).toContain("command -v unzip"); + expect(script).toContain("curl -fsSL https://bun.sh/install | bash"); + expect(script).toContain('export PATH="$BUN_INSTALL/bin:$PATH"'); + expect(script).toContain("bunx playwright install chromium"); + expect(script).toContain("trap cleanup_proof_processes EXIT"); + expect(script).toContain("return 0"); + expect(script).toContain("bun -e"); + expect(script).toContain("bun 'scripts/ui-proof-runtime.mjs' run-scenario"); + expect(script).toContain( + 'manifest_status="$(CLAWHUB_UI_PROOF_MANIFEST="$remote_out/proof-steps.json" bun -e', + ); + }); +});