diff --git a/.github/workflows/skill-publish.yml b/.github/workflows/skill-publish.yml new file mode 100644 index 00000000..dc06681e --- /dev/null +++ b/.github/workflows/skill-publish.yml @@ -0,0 +1,292 @@ +name: Skill Publish + +on: + workflow_call: + inputs: + skill_path: + description: Optional path to one skill folder. When set, only this skill is processed. + required: false + type: string + default: "" + root: + description: Directory containing skill folders for bulk catalog publishing. + required: false + type: string + default: skills + dry_run: + description: Preview only. When true, no publish mutation is performed. + required: false + type: boolean + default: true + owner: + description: Optional owner/publisher handle for org publishing. + required: false + type: string + default: "" + tags: + description: Optional comma-separated tags override. + required: false + type: string + default: latest + bump: + description: Version bump for updated skills. One of patch, minor, or major. + required: false + type: string + default: patch + registry: + description: ClawHub registry URL. + required: false + type: string + default: https://clawhub.ai + site: + description: ClawHub site URL. + required: false + type: string + default: https://clawhub.ai + ref: + description: Optional caller repository ref to check out. + required: false + type: string + default: "" + secrets: + clawhub_token: + required: false + outputs: + publish_json: + description: Structured JSON output from clawhub sync. + value: ${{ jobs.publish.outputs.publish_json }} + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + +permissions: {} + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + outputs: + publish_json: ${{ steps.capture.outputs.publish_json }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.sha }} + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 + with: + bun-version: 1.3.10 + + - name: Resolve ClawHub workflow source + id: clawhub_source + run: | + python3 - <<'PY' + import base64 + import json + import os + from pathlib import Path + from urllib.request import Request, urlopen + + request_token = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "").strip() + request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL", "").strip() + if not request_token or not request_url: + raise SystemExit("GitHub OIDC token request env vars are missing; id-token: write is required.") + + audience = "clawhub-workflow-source" + joiner = "&" if "?" in request_url else "?" + token_url = f"{request_url}{joiner}audience={audience}" + request = Request(token_url, headers={"Authorization": f"Bearer {request_token}"}) + with urlopen(request) as response: + payload = json.load(response) + + token = str(payload.get("value", "")).strip() + if not token: + raise SystemExit("GitHub OIDC token response did not include a token value.") + + try: + encoded_payload = token.split(".")[1] + except IndexError as exc: + raise SystemExit("GitHub OIDC token was not a valid JWT.") from exc + padding = "=" * (-len(encoded_payload) % 4) + claims = json.loads(base64.urlsafe_b64decode(encoded_payload + padding).decode("utf-8")) + + workflow_ref = str(claims.get("job_workflow_ref", "")).strip() + workflow_sha = str(claims.get("job_workflow_sha", "")).strip() + repo, marker, _ = workflow_ref.partition("/.github/workflows/") + if not marker or not repo or not workflow_sha: + raise SystemExit( + "Unable to resolve reusable workflow source from GitHub OIDC claims: " + f"job_workflow_ref={workflow_ref!r} job_workflow_sha={workflow_sha!r}" + ) + + output_path = Path(os.environ["GITHUB_OUTPUT"]) + with output_path.open("a", encoding="utf-8") as fh: + fh.write(f"repository={repo}\n") + fh.write(f"ref={workflow_sha}\n") + PY + + - uses: actions/checkout@v6 + with: + repository: ${{ steps.clawhub_source.outputs.repository }} + ref: ${{ steps.clawhub_source.outputs.ref }} + path: clawhub-source + + - name: Install ClawHub CLI dependencies + working-directory: clawhub-source + run: bun install --frozen-lockfile + + - name: Validate publish mode inputs + env: + DRY_RUN: ${{ inputs.dry_run }} + CLAWHUB_TOKEN: ${{ secrets.clawhub_token }} + run: | + if [[ "$DRY_RUN" == "true" ]]; then + exit 0 + fi + if [[ -n "$CLAWHUB_TOKEN" ]]; then + exit 0 + fi + echo "::error::Real skill publishes need secrets.clawhub_token. GitHub OIDC trusted publishing for skills is not supported yet." + exit 1 + + - name: Write ClawHub config + env: + CLAWHUB_TOKEN: ${{ secrets.clawhub_token }} + CLAWHUB_REGISTRY: ${{ inputs.registry }} + run: | + if [[ -z "$CLAWHUB_TOKEN" ]]; then + echo "No ClawHub token provided, skipping config file creation." + exit 0 + fi + + python3 - <<'PY' + import json + import os + from pathlib import Path + + path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-config.json" + path.write_text( + json.dumps( + { + "registry": os.environ["CLAWHUB_REGISTRY"], + "token": os.environ["CLAWHUB_TOKEN"], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print(path) + PY + echo "CLAWHUB_CONFIG_PATH=$RUNNER_TEMP/clawhub-config.json" >> "$GITHUB_ENV" + + - name: Resolve sync command + env: + INPUT_SKILL_PATH: ${{ inputs.skill_path }} + INPUT_ROOT: ${{ inputs.root }} + INPUT_DRY_RUN: ${{ inputs.dry_run }} + INPUT_OWNER: ${{ inputs.owner }} + INPUT_TAGS: ${{ inputs.tags }} + INPUT_BUMP: ${{ inputs.bump }} + INPUT_SITE: ${{ inputs.site }} + INPUT_REGISTRY: ${{ inputs.registry }} + INPUT_REF: ${{ inputs.ref }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_REF: ${{ github.ref }} + run: | + python3 - <<'PY' + import os + import shlex + import subprocess + from pathlib import Path + + skill_path = os.environ["INPUT_SKILL_PATH"].strip() + root = os.environ["INPUT_ROOT"].strip() or "skills" + scan_root = skill_path or root + source_commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + source_ref = os.environ["INPUT_REF"].strip() or os.environ["GITHUB_REF"].strip() + + cli_entry = ( + Path(os.environ["GITHUB_WORKSPACE"]) + / "clawhub-source" + / "packages" + / "clawhub" + / "src" + / "cli.ts" + ) + if not cli_entry.exists(): + raise SystemExit(f"Missing ClawHub CLI entrypoint at {cli_entry}") + + cmd = [ + "bun", + str(cli_entry), + "--workdir", + scan_root, + "--dir", + ".", + "sync", + "--all", + "--json", + "--no-clawdbot-roots", + "--site", + os.environ["INPUT_SITE"], + "--registry", + os.environ["INPUT_REGISTRY"], + "--bump", + os.environ["INPUT_BUMP"].strip() or "patch", + "--source-repo", + os.environ["GITHUB_REPOSITORY"], + "--source-commit", + source_commit, + ] + + if os.environ["INPUT_DRY_RUN"] == "true": + cmd.append("--dry-run") + owner = os.environ["INPUT_OWNER"].strip() + tags = os.environ["INPUT_TAGS"].strip() + if owner: + cmd += ["--owner", owner] + if tags: + cmd += ["--tags", tags] + if source_ref: + cmd += ["--source-ref", source_ref] + + path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-skill-publish-command.sh" + shell_line = " ".join(shlex.quote(part) for part in cmd) + path.write_text("#!/usr/bin/env bash\nset -euo pipefail\n" + shell_line + "\n", encoding="utf-8") + path.chmod(0o755) + print(shell_line) + PY + + - name: Run skill sync + run: | + set -euo pipefail + "$RUNNER_TEMP/clawhub-skill-publish-command.sh" | tee "$RUNNER_TEMP/skill-publish.json" + + - name: Capture workflow outputs + id: capture + run: | + python3 - <<'PY' + import json + import os + from pathlib import Path + + output_path = Path(os.environ["RUNNER_TEMP"]) / "skill-publish.json" + raw = output_path.read_text(encoding="utf-8").strip() + parsed = json.loads(raw) + + github_output = Path(os.environ["GITHUB_OUTPUT"]) + with github_output.open("a", encoding="utf-8") as fh: + fh.write("publish_json<<__CLAWHUB_JSON__\n") + fh.write(json.dumps(parsed, indent=2)) + fh.write("\n__CLAWHUB_JSON__\n") + PY + + - name: Upload publish JSON artifact + uses: actions/upload-artifact@v7 + with: + name: clawhub-skill-publish-json + path: ${{ runner.temp }}/skill-publish.json + if-no-files-found: error diff --git a/docs/cli.md b/docs/cli.md index fdd29f9f..ab2c925a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -184,6 +184,46 @@ Stores your API token + cached registry URL. clawhub skill publish ./my-skill --version 1.0.0 ``` +#### GitHub Actions + +ClawHub ships an official reusable workflow at +[`/.github/workflows/skill-publish.yml`](../.github/workflows/skill-publish.yml) +for skill repos and catalog repos. + +Typical catalog setup: + +```yaml +name: Skill Publish + +on: + pull_request: + workflow_dispatch: + +jobs: + dry-run: + if: github.event_name == 'pull_request' + uses: openclaw/clawhub/.github/workflows/skill-publish.yml@v1 + with: + owner: nvidia + dry_run: true + + publish: + if: github.event_name == 'workflow_dispatch' + uses: openclaw/clawhub/.github/workflows/skill-publish.yml@v1 + with: + owner: nvidia + dry_run: false + secrets: + clawhub_token: ${{ secrets.CLAWHUB_TOKEN }} +``` + +Notes: + +- `root` defaults to `skills` for catalog repos. +- Pass `skill_path: skills/review-helper` to process one skill folder. +- `owner` maps to the CLI `--owner` flag; omit it to publish as the authenticated user. +- V1 skill publishing uses `clawhub_token`; GitHub OIDC trusted publishing is package-only for now. + ### `delete ` - Soft-delete a skill (owner, moderator, or admin). @@ -609,10 +649,13 @@ Notes: - `--root ` extra scan roots - `--all` upload without prompting - `--dry-run` show plan only + - `--json` machine-readable summary for CI + - `--owner ` publish under a user or org publisher - `--bump patch|minor|major` (default: patch) - `--changelog ` (non-interactive) - `--tags a,b,c` (default: latest) - `--concurrency ` (default: 4) + - `--source-repo `, `--source-commit `, `--source-ref ` for GitHub provenance Telemetry: diff --git a/docs/publishing.md b/docs/publishing.md index a3c01098..593d5d13 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -67,6 +67,26 @@ and the destination owner. It preserves the skill, version history, stats, comments, forks, aliases, and audit trail; old owner URLs continue through the alias/redirect path. +### GitHub Actions for Skills + +Use the reusable skill workflow for catalog repos that keep many skills under a +directory such as `skills/`. Pull requests should run dry-run previews, and real +publishes should start with manual `workflow_dispatch` runs. + +```yaml +jobs: + publish: + uses: openclaw/clawhub/.github/workflows/skill-publish.yml@v1 + with: + owner: nvidia + dry_run: false + secrets: + clawhub_token: ${{ secrets.CLAWHUB_TOKEN }} +``` + +`root` defaults to `skills`. To publish or preview one folder, pass +`skill_path`, for example `skill_path: skills/review-helper`. + ## Plugins Plugins use npm-style package names. Scoped package names include the owner in diff --git a/docs/quickstart.md b/docs/quickstart.md index 2ec25665..c5648dbb 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -126,6 +126,18 @@ clawhub sync --all --dry-run clawhub sync --all ``` +For catalog repos, ClawHub also provides a reusable GitHub workflow. By +default it scans `skills/`; pass `skill_path` to process one folder. + +```yaml +jobs: + dry-run: + uses: openclaw/clawhub/.github/workflows/skill-publish.yml@v1 + with: + owner: nvidia + dry_run: true +``` + When you are signed in, `sync` may also send a minimal install snapshot for aggregate install counts. See [Telemetry](./telemetry.md) for what is reported and how to opt out. diff --git a/packages/clawhub/src/cli.ts b/packages/clawhub/src/cli.ts index 1fca4caa..f6a100d8 100644 --- a/packages/clawhub/src/cli.ts +++ b/packages/clawhub/src/cli.ts @@ -747,10 +747,16 @@ registerCommand(program, ["sync"]) .option("--root ", "Extra scan roots (one or more)") .option("--all", "Upload all new/updated skills without prompting") .option("--dry-run", "Show what would be uploaded") + .option("--json", "Output JSON") + .option("--owner ", "Publish under an org/user publisher handle") .option("--bump ", "Version bump for updates (patch|minor|major)", "patch") .option("--changelog ", "Changelog to use for updates (non-interactive)") .option("--tags ", "Comma-separated tags", "latest") .option("--concurrency ", "Concurrent registry checks (default: 4)", "4") + .option("--no-clawdbot-roots", "Only scan the configured workdir/dir and --root values") + .option("--source-repo ", "GitHub repo (owner/repo or URL)") + .option("--source-commit ", "Git commit SHA") + .option("--source-ref ", "Git ref/tag/branch") .action(async (options) => { const opts = await resolveGlobalOpts(); const bump = String(options.bump ?? "patch") as "patch" | "minor" | "major"; @@ -764,10 +770,16 @@ registerCommand(program, ["sync"]) root: options.root, all: options.all, dryRun: options.dryRun, + json: options.json, + owner: options.owner, bump, changelog: options.changelog, tags: options.tags, concurrency, + clawdbotRoots: options.clawdbotRoots, + sourceRepo: options.sourceRepo, + sourceCommit: options.sourceCommit, + sourceRef: options.sourceRef, }, isInputAllowed(), ); diff --git a/packages/clawhub/src/cli/commands/publish.test.ts b/packages/clawhub/src/cli/commands/publish.test.ts index 1d439bd2..19680bf5 100644 --- a/packages/clawhub/src/cli/commands/publish.test.ts +++ b/packages/clawhub/src/cli/commands/publish.test.ts @@ -227,6 +227,54 @@ describe("cmdPublish", () => { } }); + it("includes GitHub source provenance for CI publishes", async () => { + const workdir = await makeTmpWorkdir(); + const dateSpy = vi.spyOn(Date, "now").mockReturnValue(123_456_789); + try { + const folder = join(workdir, "source-skill"); + await mkdir(folder, { recursive: true }); + await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8"); + + httpMocks.apiRequestForm.mockResolvedValueOnce({ + ok: true, + skillId: "skill_1", + versionId: "ver_1", + }); + + await cmdPublish(makeOpts(workdir), "source-skill", { + slug: "source-skill", + name: "Source Skill", + version: "1.0.0", + sourceRepo: "https://github.com/NVIDIA/skills", + sourceCommit: "abc123", + sourceRef: "refs/heads/main", + sourcePath: "skills/source-skill", + }); + + const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => { + const req = call[1] as { path?: string } | undefined; + return req?.path === "/api/v1/skills"; + }); + if (!publishCall) throw new Error("Missing publish call"); + const publishForm = (publishCall[1] as { form?: FormData }).form as FormData; + const payloadEntry = publishForm.get("payload"); + if (typeof payloadEntry !== "string") throw new Error("Missing publish payload"); + const payload = JSON.parse(payloadEntry); + expect(payload.source).toEqual({ + kind: "github", + url: "https://github.com/NVIDIA/skills", + repo: "NVIDIA/skills", + ref: "refs/heads/main", + commit: "abc123", + path: "skills/source-skill", + importedAt: 123_456_789, + }); + dateSpy.mockRestore(); + } finally { + await rm(workdir, { recursive: true, force: true }); + } + }); + it('rejects plugin folders with guidance to use "clawhub package publish"', async () => { const workdir = await makeTmpWorkdir(); try { diff --git a/packages/clawhub/src/cli/commands/publish.ts b/packages/clawhub/src/cli/commands/publish.ts index 72729648..3c681086 100644 --- a/packages/clawhub/src/cli/commands/publish.ts +++ b/packages/clawhub/src/cli/commands/publish.ts @@ -9,6 +9,7 @@ import { getRegistry } from "../registry.js"; import { sanitizeSlug, titleCase } from "../slug.js"; import type { GlobalOpts } from "../types.js"; import { createSpinner, fail, formatError } from "../ui.js"; +import { normalizeGitHubRepo } from "./github.js"; export async function cmdPublish( opts: GlobalOpts, @@ -22,6 +23,10 @@ export async function cmdPublish( tags?: string; forkOf?: string; migrateOwner?: boolean; + sourceRepo?: string; + sourceCommit?: string; + sourceRef?: string; + sourcePath?: string; }, ) { const folder = folderArg ? resolve(opts.workdir, folderArg) : null; @@ -48,6 +53,7 @@ export async function cmdPublish( const forkOfRaw = options.forkOf?.trim(); const forkOf = forkOfRaw ? parseForkOf(forkOfRaw) : undefined; + const source = buildPublishSource(options); if (!slug) fail("--slug required"); if (!displayName) fail("--name required"); @@ -80,6 +86,7 @@ export async function cmdPublish( changelog, acceptLicenseTerms: true, tags, + ...(source ? { source } : {}), ...(forkOf ? { forkOf } : {}), }), ); @@ -174,3 +181,34 @@ function parseForkOf(value: string) { if (version && !semver.valid(version)) fail("--fork-of version must be valid semver"); return { slug, version: version || undefined }; } + +function buildPublishSource(options: { + sourceRepo?: string; + sourceCommit?: string; + sourceRef?: string; + sourcePath?: string; +}) { + const rawRepo = options.sourceRepo?.trim(); + const commit = options.sourceCommit?.trim(); + const ref = options.sourceRef?.trim(); + const path = normalizeSourcePath(options.sourcePath); + if (!rawRepo && !commit && !ref && !options.sourcePath?.trim()) return undefined; + if (!rawRepo || !commit) fail("--source-repo and --source-commit must be provided together"); + const repo = normalizeGitHubRepo(rawRepo); + if (!repo) fail("--source-repo must be a GitHub repo or URL"); + return { + kind: "github" as const, + url: `https://github.com/${repo}`, + repo, + ref: ref || commit, + commit, + path, + importedAt: Date.now(), + }; +} + +function normalizeSourcePath(value: string | undefined) { + const normalized = (value?.trim() || ".").replaceAll("\\", "/").replace(/^\.\/+/, ""); + if (!normalized || normalized === ".") return "."; + return normalized.replace(/\/+$/, "") || "."; +} diff --git a/packages/clawhub/src/cli/commands/sync.test.ts b/packages/clawhub/src/cli/commands/sync.test.ts index 14aac957..669d9f10 100644 --- a/packages/clawhub/src/cli/commands/sync.test.ts +++ b/packages/clawhub/src/cli/commands/sync.test.ts @@ -116,7 +116,6 @@ afterEach(async () => { vi.spyOn(console, "log").mockImplementation((...args) => { mockLog(args.map(String).join(" ")); }); - describe("cmdSync", () => { it("classifies skills as new/update/synced (dry-run, mocked HTTP)", async () => { interactive = false; @@ -151,6 +150,84 @@ describe("cmdSync", () => { expect(String(dryRunOutro)).toMatch(/Dry run: would upload 2 skill/); }); + it("emits CI JSON dry-run without requiring auth", async () => { + interactive = false; + const stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => { + if (args.path.startsWith("/api/v1/resolve?")) { + const u = new URL(`https://x.test${args.path}`); + const slug = u.searchParams.get("slug"); + if (slug === "new-skill") { + throw new Error("Skill not found"); + } + if (slug === "synced-skill") { + return { match: { version: "1.2.3" }, latestVersion: { version: "1.2.3" } }; + } + if (slug === "update-skill") { + return { match: null, latestVersion: { version: "1.0.0" } }; + } + } + throw new Error(`Unexpected apiRequest: ${args.path}`); + }); + + let output = ""; + try { + await cmdSync( + makeOpts(), + { + root: ["/scan"], + all: true, + dryRun: true, + json: true, + owner: "nvidia", + sourceRepo: "NVIDIA/skills", + sourceCommit: "abc123", + sourceRef: "refs/heads/main", + }, + false, + ); + output = String(stdoutWrite.mock.calls.at(-1)?.[0] ?? "").trim(); + } finally { + stdoutWrite.mockRestore(); + } + + expect(authTokenMocks.requireAuthToken).not.toHaveBeenCalled(); + expect(mockCmdPublish).not.toHaveBeenCalled(); + expect(mockLog).not.toHaveBeenCalled(); + expect(mockIntro).not.toHaveBeenCalled(); + expect(mockOutro).not.toHaveBeenCalled(); + + const parsed = JSON.parse(output) as { + ok: boolean; + dryRun: boolean; + owner?: string; + summary: { wouldPublish: number; alreadySynced: number; failed: number }; + wouldPublish: Array<{ + slug: string; + version: string; + status: string; + source?: { repo: string }; + }>; + alreadySynced: Array<{ slug: string; version: string }>; + published: unknown[]; + failed: unknown[]; + }; + expect(parsed.ok).toBe(true); + expect(parsed.dryRun).toBe(true); + expect(parsed.owner).toBe("nvidia"); + expect(parsed.summary).toMatchObject({ wouldPublish: 2, alreadySynced: 1, failed: 0 }); + expect(parsed.wouldPublish.map((entry) => [entry.slug, entry.version, entry.status])).toEqual([ + ["new-skill", "1.0.0", "new"], + ["update-skill", "1.0.1", "update"], + ]); + expect(parsed.wouldPublish[0]?.source?.repo).toBe("NVIDIA/skills"); + expect(parsed.alreadySynced).toEqual([ + expect.objectContaining({ slug: "synced-skill", version: "1.2.3" }), + ]); + expect(parsed.published).toEqual([]); + expect(parsed.failed).toEqual([]); + }); + it("prints bullet lists and selects all actionable by default", async () => { interactive = true; mockMultiselect.mockImplementation(async (args?: unknown) => { @@ -191,6 +268,108 @@ describe("cmdSync", () => { expect(mockCmdPublish).toHaveBeenCalledTimes(2); }); + it("passes owner and source provenance into real bulk publishes", async () => { + interactive = false; + const opts = makeGlobalOpts("/repo"); + const { findSkillFolders } = await import("../scanSkills.js"); + mocked(findSkillFolders).mockImplementation(async (root: string) => { + if (root !== "/repo/skills") return []; + return [ + { folder: "/repo/skills/new-skill", slug: "new-skill", displayName: "New Skill" }, + { folder: "/repo/skills/update-skill", slug: "update-skill", displayName: "Update Skill" }, + ]; + }); + mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => { + if (args.path === "/api/v1/whoami") return { user: { handle: "mikehollinger" } }; + if (args.path === "/api/cli/telemetry/sync") return { ok: true }; + if (args.path.startsWith("/api/v1/resolve?")) { + const u = new URL(`https://x.test${args.path}`); + const slug = u.searchParams.get("slug"); + if (slug === "new-skill") throw new Error("Skill not found"); + if (slug === "update-skill") { + return { match: null, latestVersion: { version: "1.0.0" } }; + } + } + throw new Error(`Unexpected apiRequest: ${args.path}`); + }); + + await cmdSync( + opts, + { + root: ["/repo/skills"], + all: true, + dryRun: false, + owner: "nvidia", + tags: "latest,catalog", + sourceRepo: "https://github.com/NVIDIA/skills", + sourceCommit: "abc123", + sourceRef: "refs/heads/main", + }, + false, + ); + + expect(mockCmdPublish).toHaveBeenCalledTimes(2); + expect(mockCmdPublish.mock.calls.map((call) => call[2])).toEqual([ + expect.objectContaining({ + slug: "new-skill", + owner: "nvidia", + version: "1.0.0", + tags: "latest,catalog", + sourceRepo: "https://github.com/NVIDIA/skills", + sourceCommit: "abc123", + sourceRef: "refs/heads/main", + sourcePath: "skills/new-skill", + }), + expect.objectContaining({ + slug: "update-skill", + owner: "nvidia", + version: "1.0.1", + tags: "latest,catalog", + sourceRepo: "https://github.com/NVIDIA/skills", + sourceCommit: "abc123", + sourceRef: "refs/heads/main", + sourcePath: "skills/update-skill", + }), + ]); + }); + + it("uses dot source path for a root skill publish", async () => { + interactive = false; + const opts = makeGlobalOpts("/repo"); + const { findSkillFolders } = await import("../scanSkills.js"); + mocked(findSkillFolders).mockImplementation(async (root: string) => { + if (root !== "/repo") return []; + return [{ folder: "/repo", slug: "root-skill", displayName: "Root Skill" }]; + }); + mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => { + if (args.path === "/api/v1/whoami") return { user: { handle: "mikehollinger" } }; + if (args.path === "/api/cli/telemetry/sync") return { ok: true }; + if (args.path.startsWith("/api/v1/resolve?")) { + throw new Error("Skill not found"); + } + throw new Error(`Unexpected apiRequest: ${args.path}`); + }); + + await cmdSync( + opts, + { + all: true, + dryRun: false, + sourceRepo: "NVIDIA/root-skill", + sourceCommit: "abc123", + }, + false, + ); + + expect(mockCmdPublish).toHaveBeenCalledTimes(1); + expect(mockCmdPublish.mock.calls[0]?.[2]).toEqual( + expect.objectContaining({ + slug: "root-skill", + sourcePath: ".", + }), + ); + }); + it("labels unmatched local content as proposed publish versions, not registry updates", async () => { interactive = false; mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => { @@ -300,6 +479,71 @@ describe("cmdSync", () => { expect(output).toMatch(/Agent: Work/); }); + it("can disable auto-discovered clawdbot roots for CI exact scans", async () => { + interactive = false; + const stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + const scannedRoots: string[] = []; + const { findSkillFolders } = await import("../scanSkills.js"); + mocked(findSkillFolders).mockImplementation(async (root: string) => { + scannedRoots.push(root); + if (root === "/scan") { + return [{ folder: "/scan/ci-skill", slug: "ci-skill", displayName: "CI Skill" }]; + } + if (root === "/auto") { + return [{ folder: "/auto/auto-skill", slug: "auto-skill", displayName: "Auto Skill" }]; + } + return []; + }); + mockResolveClawdbotSkillRoots.mockResolvedValueOnce({ + roots: ["/auto"], + labels: { "/auto": "Agent: Work" }, + }); + mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => { + if (args.path.startsWith("/api/v1/resolve?")) { + throw new Error("Skill not found"); + } + throw new Error(`Unexpected apiRequest: ${args.path}`); + }); + + let output = ""; + try { + await cmdSync( + makeOpts(), + { root: ["/scan"], all: true, dryRun: true, json: true, clawdbotRoots: false }, + false, + ); + output = String(stdoutWrite.mock.calls.at(-1)?.[0] ?? "").trim(); + } finally { + stdoutWrite.mockRestore(); + } + + expect(mockResolveClawdbotSkillRoots).not.toHaveBeenCalled(); + expect(scannedRoots).not.toContain("/auto"); + const parsed = JSON.parse(output) as { + roots: string[]; + wouldPublish: Array<{ slug: string }>; + }; + expect(parsed.roots).toEqual(["/work", "/work/skills", "/scan"]); + expect(parsed.wouldPublish.map((entry) => entry.slug)).toEqual(["ci-skill"]); + }); + + it("does not fall back to ambient roots when exact CI scans find no skills", async () => { + interactive = false; + const { findSkillFolders, getFallbackSkillRoots } = await import("../scanSkills.js"); + mocked(findSkillFolders).mockImplementation(async () => []); + + await expect( + cmdSync( + makeOpts(), + { root: ["/scan"], all: true, dryRun: true, json: true, clawdbotRoots: false }, + false, + ), + ).rejects.toThrow("No skills found (checked configured roots)"); + + expect(mockResolveClawdbotSkillRoots).not.toHaveBeenCalled(); + expect(getFallbackSkillRoots).not.toHaveBeenCalled(); + }); + it("allows empty changelog for updates (interactive)", async () => { interactive = true; mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => { diff --git a/packages/clawhub/src/cli/commands/sync.ts b/packages/clawhub/src/cli/commands/sync.ts index 230659ad..56b4ccae 100644 --- a/packages/clawhub/src/cli/commands/sync.ts +++ b/packages/clawhub/src/cli/commands/sync.ts @@ -1,10 +1,13 @@ +import { isAbsolute, relative } from "node:path"; import { intro, outro } from "@clack/prompts"; import { hashSkillFiles, listTextFiles, readSkillOrigin } from "../../skills.js"; -import { requireAuthToken } from "../authToken.js"; +import { getOptionalAuthToken, requireAuthToken } from "../authToken.js"; import { resolveClawdbotSkillRoots } from "../clawdbotConfig.js"; +import { getRegistry } from "../registry.js"; import { getFallbackSkillRoots } from "../scanSkills.js"; import type { GlobalOpts } from "../types.js"; import { createSpinner, fail, formatError, isInteractive } from "../ui.js"; +import { normalizeGitHubRepo } from "./github.js"; import { cmdPublish } from "./publish.js"; import { buildScanRoots, @@ -29,53 +32,64 @@ import { import type { Candidate, LocalSkill, SyncOptions } from "./syncTypes.js"; export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllowed: boolean) { - const allowPrompt = isInteractive() && inputAllowed !== false; - intro("ClawHub sync"); + const jsonMode = options.json === true; + const allowPrompt = !jsonMode && isInteractive() && inputAllowed !== false; + if (!jsonMode) intro("ClawHub sync"); - const token = await requireAuthToken(); + const token = options.dryRun ? await getOptionalAuthToken() : await requireAuthToken(); - const registry = await getRegistryWithAuth(opts, token); + const registry = token + ? await getRegistryWithAuth(opts, token) + : await getRegistry(opts, { cache: true }); const selectedRoots = buildScanRoots(opts, options.root); - const clawdbotRoots = await resolveClawdbotSkillRoots(); + const includeClawdbotRoots = options.clawdbotRoots !== false; + const clawdbotRoots = includeClawdbotRoots + ? await resolveClawdbotSkillRoots() + : { roots: [], labels: {} }; const combinedRoots = Array.from( new Set([...selectedRoots, ...clawdbotRoots.roots].map((root) => root.trim()).filter(Boolean)), ); const concurrency = normalizeConcurrency(options.concurrency); - const spinner = createSpinner("Scanning for local skills"); + const spinner = jsonMode ? null : createSpinner("Scanning for local skills"); const primaryScan = await scanRootsWithLabels(combinedRoots, clawdbotRoots.labels); let scan = primaryScan; let telemetryScan = primaryScan; if (primaryScan.skills.length === 0) { + if (!includeClawdbotRoots) { + fail("No skills found (checked configured roots)"); + } const fallback = getFallbackSkillRoots(opts.workdir); const fallbackScan = await scanRootsWithLabels(fallback); - spinner.stop(); + spinner?.stop(); telemetryScan = mergeScan(primaryScan, fallbackScan); scan = fallbackScan; if (fallbackScan.skills.length === 0) fail("No skills found (checked workdir and known Clawdis/Clawd locations)"); - printSection( - `No skills in workdir. Found ${fallbackScan.skills.length} in fallback locations.`, - formatList(fallbackScan.rootsWithSkills, 10), - ); + if (!jsonMode) { + printSection( + `No skills in workdir. Found ${fallbackScan.skills.length} in fallback locations.`, + formatList(fallbackScan.rootsWithSkills, 10), + ); + } } else { - spinner.stop(); + spinner?.stop(); const labeledRoots = primaryScan.rootsWithSkills .map((root) => { const label = primaryScan.rootLabels?.[root]; return label ? `${label} (${root})` : root; }) .filter(Boolean); - if (labeledRoots.length > 0) { + if (!jsonMode && labeledRoots.length > 0) { printSection("Roots with skills", formatList(labeledRoots, 10)); } } const deduped = dedupeSkillsBySlug(scan.skills); const skills = deduped.skills; - if (deduped.duplicates.length > 0) { + if (!jsonMode && deduped.duplicates.length > 0) { printSection("Skipped duplicate slugs", formatCommaList(deduped.duplicates, 16)); } - const parsingSpinner = createSpinner("Parsing local skills"); + const parsingSpinner = jsonMode ? null : createSpinner("Parsing local skills"); const locals: LocalSkill[] = []; try { let done = 0; @@ -84,7 +98,7 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow const hashed = hashSkillFiles(filesOnDisk); const origin = await readSkillOrigin(skill.folder); done += 1; - parsingSpinner.text = `Parsing local skills ${done}/${skills.length}`; + if (parsingSpinner) parsingSpinner.text = `Parsing local skills ${done}/${skills.length}`; return { ...skill, fingerprint: hashed.fingerprint, @@ -94,13 +108,13 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow }); locals.push(...parsed); } catch (error) { - parsingSpinner.fail(formatError(error)); + parsingSpinner?.fail(formatError(error)); throw error; } finally { - parsingSpinner.stop(); + parsingSpinner?.stop(); } - const candidatesSpinner = createSpinner("Checking registry sync state"); + const candidatesSpinner = jsonMode ? null : createSpinner("Checking registry sync state"); const candidates: Candidate[] = []; const resolveSupport: { value: boolean | null } = { value: null }; try { @@ -110,29 +124,50 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow return await checkRegistrySyncState(registry, skill, resolveSupport, token); } finally { done += 1; - candidatesSpinner.text = `Checking registry sync state ${done}/${locals.length}`; + if (candidatesSpinner) { + candidatesSpinner.text = `Checking registry sync state ${done}/${locals.length}`; + } } }); candidates.push(...resolved); } catch (error) { - candidatesSpinner.fail(formatError(error)); + candidatesSpinner?.fail(formatError(error)); throw error; } finally { - candidatesSpinner.stop(); + candidatesSpinner?.stop(); } - await reportTelemetryIfEnabled({ - token, - registry, - scan: telemetryScan, - candidates, - }); + if (token) { + await reportTelemetryIfEnabled({ + token, + registry, + scan: telemetryScan, + candidates, + }); + } const synced = candidates.filter((candidate) => candidate.status === "synced"); const actionable = candidates.filter((candidate) => candidate.status !== "synced"); const bump = options.bump ?? "patch"; if (actionable.length === 0) { + if (jsonMode) { + writeSyncJson( + buildSyncJsonOutput({ + ok: true, + dryRun: Boolean(options.dryRun), + registry, + roots: combinedRoots, + owner: normalizeOwner(options.owner), + duplicates: deduped.duplicates, + alreadySynced: synced.map(formatSyncedJson), + wouldPublish: [], + published: [], + failed: [], + }), + ); + return; + } if (synced.length > 0) { printSection("Already synced", formatCommaList(synced.map(formatSyncedSummary), 16)); } @@ -140,14 +175,16 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow return; } - printSection( - "To sync", - formatBulletList( - actionable.map((candidate) => formatActionableLine(candidate, bump)), - 20, - ), - ); - if (synced.length > 0) { + if (!jsonMode) { + printSection( + "To sync", + formatBulletList( + actionable.map((candidate) => formatActionableLine(candidate, bump)), + 20, + ), + ); + } + if (!jsonMode && synced.length > 0) { printSection("Already synced", formatSyncedDisplay(synced)); } @@ -157,11 +194,60 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow bump, }); if (selected.length === 0) { + if (jsonMode) { + writeSyncJson( + buildSyncJsonOutput({ + ok: true, + dryRun: Boolean(options.dryRun), + registry, + roots: combinedRoots, + owner: normalizeOwner(options.owner), + duplicates: deduped.duplicates, + alreadySynced: synced.map(formatSyncedJson), + wouldPublish: [], + published: [], + failed: [], + }), + ); + return; + } outro("Nothing selected."); return; } + const plannedPublishes = selected.map((skill) => { + const source = buildSourceProvenance(opts, skill, options); + return { skill, source }; + }); + if (options.dryRun) { + const wouldPublish = await Promise.all( + plannedPublishes.map(async ({ skill, source }) => { + const { publishVersion } = await resolvePublishMeta(skill, { + bump, + allowPrompt, + changelogFlag: options.changelog, + }); + return formatPublishJson(skill, publishVersion, source); + }), + ); + if (jsonMode) { + writeSyncJson( + buildSyncJsonOutput({ + ok: true, + dryRun: true, + registry, + roots: combinedRoots, + owner: normalizeOwner(options.owner), + duplicates: deduped.duplicates, + alreadySynced: synced.map(formatSyncedJson), + wouldPublish, + published: [], + failed: [], + }), + ); + return; + } outro(`Dry run: would upload ${selected.length} skill(s).`); return; } @@ -170,7 +256,8 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow const failedUploads: Array<{ slug: string; message: string }> = []; let uploaded = 0; - for (const skill of selected) { + const published: Array<{ slug: string; folder: string; version: string }> = []; + for (const { skill, source } of plannedPublishes) { const { publishVersion, changelog } = await resolvePublishMeta(skill, { bump, allowPrompt, @@ -186,18 +273,46 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow await cmdPublish(opts, skill.folder, { slug: skill.slug, name: skill.displayName, + owner: normalizeOwner(options.owner), version: publishVersion, changelog, tags, forkOf, + ...(source + ? { + sourceRepo: options.sourceRepo, + sourceCommit: options.sourceCommit, + sourceRef: options.sourceRef, + sourcePath: source.path, + } + : {}), }); uploaded += 1; + published.push({ slug: skill.slug, folder: skill.folder, version: publishVersion }); } catch (error) { failedUploads.push({ slug: skill.slug, message: formatError(error) }); } } if (failedUploads.length > 0) { + if (jsonMode) { + writeSyncJson( + buildSyncJsonOutput({ + ok: false, + dryRun: false, + registry, + roots: combinedRoots, + owner: normalizeOwner(options.owner), + duplicates: deduped.duplicates, + alreadySynced: synced.map(formatSyncedJson), + wouldPublish: [], + published, + failed: failedUploads, + }), + ); + process.exitCode = 1; + return; + } printSection( "Failed to upload", formatBulletList( @@ -210,9 +325,140 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow return; } + if (jsonMode) { + writeSyncJson( + buildSyncJsonOutput({ + ok: true, + dryRun: false, + registry, + roots: combinedRoots, + owner: normalizeOwner(options.owner), + duplicates: deduped.duplicates, + alreadySynced: synced.map(formatSyncedJson), + wouldPublish: [], + published, + failed: [], + }), + ); + return; + } + outro(`Uploaded ${selected.length} skill(s).`); } function normalizeRegistry(value: string) { return value.trim().replace(/\/+$/, "").toLowerCase(); } + +function normalizeOwner(value: string | undefined) { + return value?.trim().replace(/^@+/, "") || undefined; +} + +function formatSyncedJson(candidate: Candidate) { + return { + slug: candidate.slug, + folder: candidate.folder, + version: candidate.matchVersion ?? candidate.latestVersion ?? "unknown", + }; +} + +function formatPublishJson( + candidate: Candidate, + version: string, + source: ReturnType, +) { + return { + slug: candidate.slug, + displayName: candidate.displayName, + folder: candidate.folder, + status: candidate.status, + version, + latestVersion: candidate.latestVersion, + fileCount: candidate.fileCount, + fingerprint: candidate.fingerprint, + ...(source ? { source } : {}), + }; +} + +function buildSourceProvenance(opts: GlobalOpts, skill: Candidate, options: SyncOptions) { + const rawRepo = options.sourceRepo?.trim(); + const commit = options.sourceCommit?.trim(); + if (!rawRepo && !commit && !options.sourceRef?.trim()) return undefined; + if (!rawRepo || !commit) fail("--source-repo and --source-commit must be provided together"); + const repo = normalizeGitHubRepo(rawRepo); + if (!repo) fail("--source-repo must be a GitHub repo or URL"); + return { + kind: "github" as const, + url: `https://github.com/${repo}`, + repo, + ref: options.sourceRef?.trim() || commit, + commit, + path: sourcePathForSkill(opts, skill.folder), + }; +} + +function sourcePathForSkill(opts: GlobalOpts, folder: string) { + return ( + relativeInside(process.cwd(), folder) ?? + relativeInside(opts.workdir, folder) ?? + relativeInside(opts.dir, folder) ?? + normalizeSourcePath(folder) + ); +} + +function relativeInside(base: string, target: string) { + const rel = relative(base, target); + if (!rel) return "."; + if (rel.startsWith("..") || isAbsolute(rel)) return null; + return normalizeSourcePath(rel); +} + +function normalizeSourcePath(value: string) { + const normalized = value + .replaceAll("\\", "/") + .replace(/^\.\/+/, "") + .replace(/\/+$/, ""); + return normalized || "."; +} + +function buildSyncJsonOutput(params: { + ok: boolean; + dryRun: boolean; + registry: string; + roots: string[]; + owner?: string; + duplicates: string[]; + alreadySynced: Array<{ slug: string; folder: string; version: string }>; + wouldPublish: Array>; + published: Array<{ slug: string; folder: string; version: string }>; + failed: Array<{ slug: string; message: string }>; +}) { + const skipped = params.duplicates.map((duplicate) => ({ + slug: duplicate.replace(/\s+\(\d+\)$/, ""), + reason: "duplicate-slug", + detail: duplicate, + })); + return { + ok: params.ok, + dryRun: params.dryRun, + registry: params.registry, + roots: params.roots, + ...(params.owner ? { owner: params.owner } : {}), + summary: { + wouldPublish: params.wouldPublish.length, + published: params.published.length, + alreadySynced: params.alreadySynced.length, + skipped: skipped.length, + failed: params.failed.length, + }, + wouldPublish: params.wouldPublish, + published: params.published, + alreadySynced: params.alreadySynced, + skipped, + failed: params.failed, + }; +} + +function writeSyncJson(value: unknown) { + process.stdout.write(`${JSON.stringify(value, null, 2)}\n`); +} diff --git a/packages/clawhub/src/cli/commands/syncTypes.ts b/packages/clawhub/src/cli/commands/syncTypes.ts index 86948a2d..4fe715a9 100644 --- a/packages/clawhub/src/cli/commands/syncTypes.ts +++ b/packages/clawhub/src/cli/commands/syncTypes.ts @@ -5,10 +5,16 @@ export type SyncOptions = { root?: string[]; all?: boolean; dryRun?: boolean; + json?: boolean; + owner?: string; bump?: "patch" | "minor" | "major"; changelog?: string; tags?: string; concurrency?: number; + clawdbotRoots?: boolean; + sourceRepo?: string; + sourceCommit?: string; + sourceRef?: string; }; export type Candidate = SkillFolder & {