diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index cbfebda1..33ac43f6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -17,6 +17,11 @@ on: required: true default: false type: boolean + active_rollout_deploy_confirm: + description: "For backend-only deploys with active external-skill rollouts, enter: pause-and-restore-active-rollouts" + required: false + default: "" + type: string concurrency: group: deploy-production @@ -44,9 +49,20 @@ jobs: - name: Resolve deploy mode id: mode + env: + ACTIVE_ROLLOUT_DEPLOY_CONFIRM: ${{ inputs.active_rollout_deploy_confirm }} run: | set -euo pipefail target="${{ inputs.target }}" + rollout_confirmation="$ACTIVE_ROLLOUT_DEPLOY_CONFIRM" + if [[ -n "$rollout_confirmation" && "$target" != "backend" ]]; then + echo "::error::Active rollout pause/restore is supported only for backend deploys." + exit 1 + fi + if [[ -n "$rollout_confirmation" && "$rollout_confirmation" != "pause-and-restore-active-rollouts" ]]; then + echo "::error::Invalid active rollout deploy confirmation." + exit 1 + fi case "$target" in full) echo "deploy_backend=true" >> "$GITHUB_OUTPUT" @@ -113,26 +129,74 @@ jobs: - name: Install run: bun install --frozen-lockfile - - name: Require dark rollout modes + - name: Inspect external skill rollout modes + id: rollout if: needs.validate-deploy-request.outputs.deploy_backend == 'true' env: + ACTIVE_ROLLOUT_DEPLOY_CONFIRM: ${{ inputs.active_rollout_deploy_confirm }} CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }} run: | set -euo pipefail + names="$(bunx convex env list --names-only --prod)" + pause_required=false + for name in \ CLAWHUB_SKILLS_SH_ROLLOUT_MODE \ CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE do - value="$(bunx convex env get "$name" --prod 2>/dev/null || true)" + value="" + if grep -Fxq "$name" <<< "$names"; then + value="$(bunx convex env get "$name" --prod)" + fi case "$value" in - ""|off) ;; + ""|off) + expected_mode=off + restore_mode="" + ;; + test|production) + expected_mode="$value" + restore_mode="$value" + pause_required=true + ;; *) - echo "::error::$name must be missing or off before an ordinary production deploy" + echo "::error::$name has unsupported rollout mode '$value'" exit 1 ;; esac + + case "$name" in + CLAWHUB_SKILLS_SH_ROLLOUT_MODE) + echo "skills_sh_expected_mode=$expected_mode" >> "$GITHUB_OUTPUT" + echo "skills_sh_restore_mode=$restore_mode" >> "$GITHUB_OUTPUT" + ;; + CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE) + echo "github_skill_sync_expected_mode=$expected_mode" >> "$GITHUB_OUTPUT" + echo "github_skill_sync_restore_mode=$restore_mode" >> "$GITHUB_OUTPUT" + ;; + esac done + echo "pause_required=$pause_required" >> "$GITHUB_OUTPUT" + if [[ "$pause_required" == "true" && "$ACTIVE_ROLLOUT_DEPLOY_CONFIRM" != "pause-and-restore-active-rollouts" ]]; then + echo "::error::Active external-skill rollouts require a backend deploy with active_rollout_deploy_confirm=pause-and-restore-active-rollouts." + exit 1 + fi + + - name: Pause external skill rollouts + if: steps.rollout.outputs.pause_required == 'true' + env: + CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }} + GITHUB_SKILL_SYNC_RESTORE_MODE: ${{ steps.rollout.outputs.github_skill_sync_restore_mode }} + SKILLS_SH_RESTORE_MODE: ${{ steps.rollout.outputs.skills_sh_restore_mode }} + run: | + set -euo pipefail + if [[ -n "$SKILLS_SH_RESTORE_MODE" ]]; then + bunx convex env set CLAWHUB_SKILLS_SH_ROLLOUT_MODE off --prod + fi + if [[ -n "$GITHUB_SKILL_SYNC_RESTORE_MODE" ]]; then + bunx convex env set CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE off --prod + fi + - name: Stamp Convex runtime environment if: needs.validate-deploy-request.outputs.deploy_backend == 'true' env: @@ -193,6 +257,47 @@ jobs: .githubSkillSync.selfServiceEnabled == false ' <<< "$capabilities" + - name: Restore external skill rollouts + if: always() && steps.rollout.outputs.pause_required == 'true' + env: + CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }} + GITHUB_SKILL_SYNC_RESTORE_MODE: ${{ steps.rollout.outputs.github_skill_sync_restore_mode }} + SKILLS_SH_RESTORE_MODE: ${{ steps.rollout.outputs.skills_sh_restore_mode }} + run: | + set -uo pipefail + restore_failed=0 + if [[ -n "$SKILLS_SH_RESTORE_MODE" ]] && + ! bunx convex env set CLAWHUB_SKILLS_SH_ROLLOUT_MODE "$SKILLS_SH_RESTORE_MODE" --prod + then + echo "::error::Failed to restore CLAWHUB_SKILLS_SH_ROLLOUT_MODE." + restore_failed=1 + fi + if [[ -n "$GITHUB_SKILL_SYNC_RESTORE_MODE" ]] && + ! bunx convex env set CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE "$GITHUB_SKILL_SYNC_RESTORE_MODE" --prod + then + echo "::error::Failed to restore CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE." + restore_failed=1 + fi + exit "$restore_failed" + + - name: Verify restored external skill rollouts + if: always() && steps.rollout.outputs.pause_required == 'true' + env: + CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }} + GITHUB_SKILL_SYNC_EXPECTED_MODE: ${{ steps.rollout.outputs.github_skill_sync_expected_mode }} + SKILLS_SH_EXPECTED_MODE: ${{ steps.rollout.outputs.skills_sh_expected_mode }} + run: | + set -euo pipefail + capabilities="$(bunx convex run rolloutCapabilities:getPublicCapabilities --prod)" + jq -e \ + --arg skills_sh_mode "$SKILLS_SH_EXPECTED_MODE" \ + --arg github_skill_sync_mode "$GITHUB_SKILL_SYNC_EXPECTED_MODE" \ + ' + .environment == "production" and + .skillsSh.mode == $skills_sh_mode and + .githubSkillSync.mode == $github_skill_sync_mode + ' <<< "$capabilities" + - name: Wait for Vercel production deployment id: vercel if: needs.validate-deploy-request.outputs.deploy_frontend == 'true' diff --git a/CHANGELOG.md b/CHANGELOG.md index e0d80c36..f0099386 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Fixes +- Deploy: allow an explicitly confirmed backend-only deploy to pause and reliably restore active external-skill rollouts instead of requiring a manual dashboard toggle. - GitHub Actions/CLI: trigger exact pre-publication checks immediately and wait for package publication to finish, so a pending staged upload no longer reports a successful release. - API: keep publish-time Plugin Inspector target preparation inside its disposable workspace when hosted runtimes expose an unusable home directory. - API: keep older code-plugin and Claw backports from replacing the highest-semver `latest` release while preserving custom distribution tags. diff --git a/specs/ci.md b/specs/ci.md index bdd06b08..aabe0d3a 100644 --- a/specs/ci.md +++ b/specs/ci.md @@ -117,6 +117,11 @@ Production-only checks stay in the manual deploy workflow: - `bun run test:e2e:prod-http` - production Playwright smoke tests +The default backend deploy remains fail-closed when external-skill rollouts are active. Maintainers +can explicitly select the backend-only pause/deploy/restore path with +`active_rollout_deploy_confirm=pause-and-restore-active-rollouts`; the workflow restores the exact +prior modes under `always()` and verifies the restored public capability modes before smoke. + Successful `full` and `frontend` production deploys create two annotated Git tags: diff --git a/specs/deploy.md b/specs/deploy.md index ca17b234..7e0f792c 100644 --- a/specs/deploy.md +++ b/specs/deploy.md @@ -49,6 +49,12 @@ Production deploy notes: - `full`: deploy Convex, verify contract, wait for the matching Vercel production deploy, then run smoke tests - `backend`: deploy Convex, verify contract, then run smoke tests against current production - `frontend`: wait for the Vercel production deploy for the selected `main` SHA, then run smoke tests +- Ordinary backend deploys require both external-skill rollout modes to be missing or `off`. + When either rollout is intentionally active, use a backend-only deploy and set + `active_rollout_deploy_confirm=pause-and-restore-active-rollouts`. The workflow records the + exact active modes, pauses them before deploying Convex, and restores and verifies them before + production HTTP smoke. The restore steps run even when deployment or dark-state verification + fails. - `frontend` does not call `vercel deploy` directly yet. It relies on the existing Vercel Git-based production deploy for that SHA. - The real deploy job uses the GitHub `Production` environment for deploy secrets, but it does not wait for a separate approval. - Required `Production` environment secret: `CONVEX_DEPLOY_KEY`. diff --git a/specs/github-backed-skills.md b/specs/github-backed-skills.md index ec909522..a138b3cf 100644 --- a/specs/github-backed-skills.md +++ b/specs/github-backed-skills.md @@ -178,8 +178,10 @@ capability is enabled. Automation must identify its runtime explicitly. The scheduled live canary runs the generic sync path against an in-memory database with `CLAWHUB_ENV=test` and the GitHub Skill Sync rollout in `test` mode. Production deploys stamp -`CLAWHUB_ENV=production` before deploying Convex, while ordinary production -deploys still require both external-skill rollout modes to remain `off`. +`CLAWHUB_ENV=production` before deploying Convex. Ordinary production deploys +still require both external-skill rollout modes to remain `off`; the explicit +backend-only maintenance path temporarily pauses active modes and restores and +verifies their exact prior values before production smoke. When enabled, repository enrollment requires: diff --git a/src/__tests__/deploy-workflow.test.ts b/src/__tests__/deploy-workflow.test.ts index b5806f7e..0685b5b2 100644 --- a/src/__tests__/deploy-workflow.test.ts +++ b/src/__tests__/deploy-workflow.test.ts @@ -17,6 +17,24 @@ describe("production deploy workflow", () => { steps?: WorkflowStep[]; }; + type DeployWorkflow = { + jobs?: Record; + on?: { + workflow_dispatch?: { + inputs?: Record< + string, + { + default?: string; + description?: string; + required?: boolean; + type?: string; + } + >; + }; + }; + permissions?: Record; + }; + it("queues active deploys instead of cancelling them", async () => { const workflow = parseYaml(await readFile(".github/workflows/deploy.yml", "utf8")) as { concurrency?: { @@ -51,7 +69,8 @@ describe("production deploy workflow", () => { expect(deployJob?.env).toEqual({ PLAYWRIGHT_BASE_URL: "https://clawhub.ai" }); expect(convexSecretSteps).toEqual([ "Check deploy configuration", - "Require dark rollout modes", + "Inspect external skill rollout modes", + "Pause external skill rollouts", "Stamp Convex runtime environment", "Stamp Convex build SHA", "Stamp Convex deploy time", @@ -59,27 +78,81 @@ describe("production deploy workflow", () => { "Publish promotions feed snapshot", "Verify Convex contract", "Verify dark rollout capabilities", + "Restore external skill rollouts", + "Verify restored external skill rollouts", ]); expect(authSecretSteps).toEqual(["Write authenticated storage state"]); expect(tagJob?.permissions).toEqual({ contents: "write" }); expect(tagJob?.needs).toEqual(["validate-deploy-request", "deploy-production"]); }); - it("refuses production deploys unless both rollout modes are off and reads them back dark", async () => { - const workflow = parseYaml(await readFile(".github/workflows/deploy.yml", "utf8")) as { - jobs?: Record; - }; + it("fails closed on active rollouts unless a backend deploy explicitly pauses and restores them", async () => { + const workflow = parseYaml( + await readFile(".github/workflows/deploy.yml", "utf8"), + ) as DeployWorkflow; const steps = workflow.jobs?.["deploy-production"]?.steps ?? []; - const requireDark = steps.find((step) => step.name === "Require dark rollout modes"); + const validateSteps = workflow.jobs?.["validate-deploy-request"]?.steps ?? []; + const resolveMode = validateSteps.find((step) => step.name === "Resolve deploy mode"); + const inspect = steps.find((step) => step.name === "Inspect external skill rollout modes"); + const pause = steps.find((step) => step.name === "Pause external skill rollouts"); const verifyDark = steps.find((step) => step.name === "Verify dark rollout capabilities"); + const restore = steps.find((step) => step.name === "Restore external skill rollouts"); + const verifyRestored = steps.find( + (step) => step.name === "Verify restored external skill rollouts", + ); + const input = workflow.on?.workflow_dispatch?.inputs?.active_rollout_deploy_confirm; - expect(requireDark?.run).toContain("CLAWHUB_SKILLS_SH_ROLLOUT_MODE"); - expect(requireDark?.run).toContain("CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE"); - expect(requireDark?.run).toContain('""|off'); + expect(input).toMatchObject({ + required: false, + default: "", + type: "string", + }); + expect(input?.description).toContain("pause-and-restore-active-rollouts"); + expect(resolveMode?.run).toContain('"$target" != "backend"'); + expect(resolveMode?.env?.ACTIVE_ROLLOUT_DEPLOY_CONFIRM).toBe( + "${{ inputs.active_rollout_deploy_confirm }}", + ); + expect(resolveMode?.run).not.toContain("${{ inputs.active_rollout_deploy_confirm }}"); + expect(inspect?.run).toContain("convex env list --names-only --prod"); + expect(inspect?.run).toContain("CLAWHUB_SKILLS_SH_ROLLOUT_MODE"); + expect(inspect?.run).toContain("CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE"); + expect(inspect?.run).toContain("test|production"); + expect(inspect?.run).toContain("pause-and-restore-active-rollouts"); + expect(inspect?.env?.ACTIVE_ROLLOUT_DEPLOY_CONFIRM).toBe( + "${{ inputs.active_rollout_deploy_confirm }}", + ); + expect(inspect?.run).not.toContain("${{ inputs.active_rollout_deploy_confirm }}"); + expect(pause?.if).toBe("steps.rollout.outputs.pause_required == 'true'"); + expect(pause?.run).toContain("convex env set CLAWHUB_SKILLS_SH_ROLLOUT_MODE off --prod"); + expect(pause?.run).toContain( + "convex env set CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE off --prod", + ); expect(verifyDark?.run).toContain("rolloutCapabilities:getPublicCapabilities"); expect(verifyDark?.run).toContain('.environment == "production"'); expect(verifyDark?.run).toContain(".skillsSh.runtimeEnabled == false"); expect(verifyDark?.run).toContain(".githubSkillSync.selfServiceEnabled == false"); + expect(restore?.if).toContain("always()"); + expect(restore?.run).toContain( + 'convex env set CLAWHUB_SKILLS_SH_ROLLOUT_MODE "$SKILLS_SH_RESTORE_MODE" --prod', + ); + expect(restore?.run).toContain( + 'convex env set CLAWHUB_GITHUB_SKILL_SYNC_ROLLOUT_MODE "$GITHUB_SKILL_SYNC_RESTORE_MODE" --prod', + ); + expect(verifyRestored?.if).toContain("always()"); + expect(verifyRestored?.run).toContain(".skillsSh.mode == $skills_sh_mode"); + expect(verifyRestored?.run).toContain(".githubSkillSync.mode == $github_skill_sync_mode"); + + const pauseIndex = steps.indexOf(pause!); + const deployIndex = steps.findIndex((step) => step.name === "Deploy Convex"); + const verifyDarkIndex = steps.indexOf(verifyDark!); + const restoreIndex = steps.indexOf(restore!); + const verifyRestoredIndex = steps.indexOf(verifyRestored!); + const smokeIndex = steps.findIndex((step) => step.name === "Smoke test production HTTP"); + expect(pauseIndex).toBeLessThan(deployIndex); + expect(deployIndex).toBeLessThan(verifyDarkIndex); + expect(verifyDarkIndex).toBeLessThan(restoreIndex); + expect(restoreIndex).toBeLessThan(verifyRestoredIndex); + expect(verifyRestoredIndex).toBeLessThan(smokeIndex); }); it("stamps the production runtime identity before deploying Convex", async () => {