mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-17 10:22:12 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
909a47106e | ||
|
|
e8cfbddf17 | ||
|
|
162528abe4 | ||
|
|
74aa61086e | ||
|
|
858a121d33 | ||
|
|
953358a322 | ||
|
|
0a79612fe5 | ||
|
|
9b5d2e088d | ||
|
|
ce62df9d08 | ||
|
|
0abdbf4a50 | ||
|
|
dcbc38999f | ||
|
|
01aa28ccda | ||
|
|
cb6ced7906 | ||
|
|
ded9ff4235 | ||
|
|
9fc2da4dc4 | ||
|
|
05d5fc1151 | ||
|
|
9aaab158cb | ||
|
|
6fc5bb7cd8 | ||
|
|
9aa3f37ee1 |
@@ -0,0 +1,292 @@
|
||||
name: Skill Publish
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
skill_path:
|
||||
description: Optional path to one skill folder. When set, only this skill is processed.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
root:
|
||||
description: Directory containing skill folders for bulk catalog publishing.
|
||||
required: false
|
||||
type: string
|
||||
default: skills
|
||||
dry_run:
|
||||
description: Preview only. When true, no publish mutation is performed.
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
owner:
|
||||
description: Optional owner/publisher handle for org publishing.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
tags:
|
||||
description: Optional comma-separated tags override.
|
||||
required: false
|
||||
type: string
|
||||
default: latest
|
||||
bump:
|
||||
description: Version bump for updated skills. One of patch, minor, or major.
|
||||
required: false
|
||||
type: string
|
||||
default: patch
|
||||
registry:
|
||||
description: ClawHub registry URL.
|
||||
required: false
|
||||
type: string
|
||||
default: https://clawhub.ai
|
||||
site:
|
||||
description: ClawHub site URL.
|
||||
required: false
|
||||
type: string
|
||||
default: https://clawhub.ai
|
||||
ref:
|
||||
description: Optional caller repository ref to check out.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
secrets:
|
||||
clawhub_token:
|
||||
required: false
|
||||
outputs:
|
||||
publish_json:
|
||||
description: Structured JSON output from clawhub sync.
|
||||
value: ${{ jobs.publish.outputs.publish_json }}
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
outputs:
|
||||
publish_json: ${{ steps.capture.outputs.publish_json }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
|
||||
with:
|
||||
bun-version: 1.3.10
|
||||
|
||||
- name: Resolve ClawHub workflow source
|
||||
id: clawhub_source
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
request_token = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "").strip()
|
||||
request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL", "").strip()
|
||||
if not request_token or not request_url:
|
||||
raise SystemExit("GitHub OIDC token request env vars are missing; id-token: write is required.")
|
||||
|
||||
audience = "clawhub-workflow-source"
|
||||
joiner = "&" if "?" in request_url else "?"
|
||||
token_url = f"{request_url}{joiner}audience={audience}"
|
||||
request = Request(token_url, headers={"Authorization": f"Bearer {request_token}"})
|
||||
with urlopen(request) as response:
|
||||
payload = json.load(response)
|
||||
|
||||
token = str(payload.get("value", "")).strip()
|
||||
if not token:
|
||||
raise SystemExit("GitHub OIDC token response did not include a token value.")
|
||||
|
||||
try:
|
||||
encoded_payload = token.split(".")[1]
|
||||
except IndexError as exc:
|
||||
raise SystemExit("GitHub OIDC token was not a valid JWT.") from exc
|
||||
padding = "=" * (-len(encoded_payload) % 4)
|
||||
claims = json.loads(base64.urlsafe_b64decode(encoded_payload + padding).decode("utf-8"))
|
||||
|
||||
workflow_ref = str(claims.get("job_workflow_ref", "")).strip()
|
||||
workflow_sha = str(claims.get("job_workflow_sha", "")).strip()
|
||||
repo, marker, _ = workflow_ref.partition("/.github/workflows/")
|
||||
if not marker or not repo or not workflow_sha:
|
||||
raise SystemExit(
|
||||
"Unable to resolve reusable workflow source from GitHub OIDC claims: "
|
||||
f"job_workflow_ref={workflow_ref!r} job_workflow_sha={workflow_sha!r}"
|
||||
)
|
||||
|
||||
output_path = Path(os.environ["GITHUB_OUTPUT"])
|
||||
with output_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(f"repository={repo}\n")
|
||||
fh.write(f"ref={workflow_sha}\n")
|
||||
PY
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
repository: ${{ steps.clawhub_source.outputs.repository }}
|
||||
ref: ${{ steps.clawhub_source.outputs.ref }}
|
||||
path: clawhub-source
|
||||
|
||||
- name: Install ClawHub CLI dependencies
|
||||
working-directory: clawhub-source
|
||||
run: bun install --frozen-lockfile
|
||||
|
||||
- name: Validate publish mode inputs
|
||||
env:
|
||||
DRY_RUN: ${{ inputs.dry_run }}
|
||||
CLAWHUB_TOKEN: ${{ secrets.clawhub_token }}
|
||||
run: |
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
if [[ -n "$CLAWHUB_TOKEN" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::Real skill publishes need secrets.clawhub_token. GitHub OIDC trusted publishing for skills is not supported yet."
|
||||
exit 1
|
||||
|
||||
- name: Write ClawHub config
|
||||
env:
|
||||
CLAWHUB_TOKEN: ${{ secrets.clawhub_token }}
|
||||
CLAWHUB_REGISTRY: ${{ inputs.registry }}
|
||||
run: |
|
||||
if [[ -z "$CLAWHUB_TOKEN" ]]; then
|
||||
echo "No ClawHub token provided, skipping config file creation."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-config.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"registry": os.environ["CLAWHUB_REGISTRY"],
|
||||
"token": os.environ["CLAWHUB_TOKEN"],
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(path)
|
||||
PY
|
||||
echo "CLAWHUB_CONFIG_PATH=$RUNNER_TEMP/clawhub-config.json" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Resolve sync command
|
||||
env:
|
||||
INPUT_SKILL_PATH: ${{ inputs.skill_path }}
|
||||
INPUT_ROOT: ${{ inputs.root }}
|
||||
INPUT_DRY_RUN: ${{ inputs.dry_run }}
|
||||
INPUT_OWNER: ${{ inputs.owner }}
|
||||
INPUT_TAGS: ${{ inputs.tags }}
|
||||
INPUT_BUMP: ${{ inputs.bump }}
|
||||
INPUT_SITE: ${{ inputs.site }}
|
||||
INPUT_REGISTRY: ${{ inputs.registry }}
|
||||
INPUT_REF: ${{ inputs.ref }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_REF: ${{ github.ref }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
skill_path = os.environ["INPUT_SKILL_PATH"].strip()
|
||||
root = os.environ["INPUT_ROOT"].strip() or "skills"
|
||||
scan_root = skill_path or root
|
||||
source_commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
||||
source_ref = os.environ["INPUT_REF"].strip() or os.environ["GITHUB_REF"].strip()
|
||||
|
||||
cli_entry = (
|
||||
Path(os.environ["GITHUB_WORKSPACE"])
|
||||
/ "clawhub-source"
|
||||
/ "packages"
|
||||
/ "clawhub"
|
||||
/ "src"
|
||||
/ "cli.ts"
|
||||
)
|
||||
if not cli_entry.exists():
|
||||
raise SystemExit(f"Missing ClawHub CLI entrypoint at {cli_entry}")
|
||||
|
||||
cmd = [
|
||||
"bun",
|
||||
str(cli_entry),
|
||||
"--workdir",
|
||||
scan_root,
|
||||
"--dir",
|
||||
".",
|
||||
"sync",
|
||||
"--all",
|
||||
"--json",
|
||||
"--no-clawdbot-roots",
|
||||
"--site",
|
||||
os.environ["INPUT_SITE"],
|
||||
"--registry",
|
||||
os.environ["INPUT_REGISTRY"],
|
||||
"--bump",
|
||||
os.environ["INPUT_BUMP"].strip() or "patch",
|
||||
"--source-repo",
|
||||
os.environ["GITHUB_REPOSITORY"],
|
||||
"--source-commit",
|
||||
source_commit,
|
||||
]
|
||||
|
||||
if os.environ["INPUT_DRY_RUN"] == "true":
|
||||
cmd.append("--dry-run")
|
||||
owner = os.environ["INPUT_OWNER"].strip()
|
||||
tags = os.environ["INPUT_TAGS"].strip()
|
||||
if owner:
|
||||
cmd += ["--owner", owner]
|
||||
if tags:
|
||||
cmd += ["--tags", tags]
|
||||
if source_ref:
|
||||
cmd += ["--source-ref", source_ref]
|
||||
|
||||
path = Path(os.environ["RUNNER_TEMP"]) / "clawhub-skill-publish-command.sh"
|
||||
shell_line = " ".join(shlex.quote(part) for part in cmd)
|
||||
path.write_text("#!/usr/bin/env bash\nset -euo pipefail\n" + shell_line + "\n", encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
print(shell_line)
|
||||
PY
|
||||
|
||||
- name: Run skill sync
|
||||
run: |
|
||||
set -euo pipefail
|
||||
"$RUNNER_TEMP/clawhub-skill-publish-command.sh" | tee "$RUNNER_TEMP/skill-publish.json"
|
||||
|
||||
- name: Capture workflow outputs
|
||||
id: capture
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
output_path = Path(os.environ["RUNNER_TEMP"]) / "skill-publish.json"
|
||||
raw = output_path.read_text(encoding="utf-8").strip()
|
||||
parsed = json.loads(raw)
|
||||
|
||||
github_output = Path(os.environ["GITHUB_OUTPUT"])
|
||||
with github_output.open("a", encoding="utf-8") as fh:
|
||||
fh.write("publish_json<<__CLAWHUB_JSON__\n")
|
||||
fh.write(json.dumps(parsed, indent=2))
|
||||
fh.write("\n__CLAWHUB_JSON__\n")
|
||||
PY
|
||||
|
||||
- name: Upload publish JSON artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: clawhub-skill-publish-json
|
||||
path: ${{ runner.temp }}/skill-publish.json
|
||||
if-no-files-found: error
|
||||
@@ -2,8 +2,15 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
## 0.19.0 - 2026-06-03
|
||||
|
||||
### Changes
|
||||
|
||||
- CLI/API: add authenticated `clawhub scan` submit/poll support for ephemeral local skill bundles and owner-authorized published skill scans, including JSON output and report ZIP downloads (#2479).
|
||||
|
||||
### Fixes
|
||||
|
||||
- Auth/Ops: keep GitHub account-age lookups on immutable numeric IDs, retry without auth when a configured GitHub token is rejected, and add an operator backfill for missing cached account ages.
|
||||
- API/CLI: report Skill Card verification with flattened skill/version metadata, ClawScan verdict fields at `security.*`, and supporting scanner evidence under `security.signals`.
|
||||
|
||||
## 0.18.0 - 2026-05-25
|
||||
|
||||
@@ -103,6 +103,24 @@ CLAWHUB_WORKTREE_SOURCE=/path/to/source/worktree bun run setup:worktree
|
||||
|
||||
The detached server writes runtime state under `.codex/runtime/`. Stop it with `wt --yes stop` before removing the worktree.
|
||||
|
||||
### Local Codex workers
|
||||
|
||||
Local dev does not start Codex-backed workers by default, so `dev:worktree` does
|
||||
not spend Codex quota.
|
||||
|
||||
To process local ClawScan or Skill Card jobs, opt in for that shell:
|
||||
|
||||
```bash
|
||||
CLAWHUB_ALLOW_LOCAL_CODEX_SCAN=1 bun run dev:workers -- --workers security-scan --once
|
||||
CLAWHUB_ALLOW_LOCAL_CODEX_SCAN=1 bun run dev:workers -- --workers skill-card --once
|
||||
```
|
||||
|
||||
Opted-in local runs use an ignored worktree-local `CODEX_HOME` unless you provide
|
||||
one.
|
||||
|
||||
Without those workers, local ClawScan and Skill Card jobs stay pending until you
|
||||
opt in, seed/mock results, or use the production workflows.
|
||||
|
||||
### Seed the database
|
||||
|
||||
Populate local QA fixtures and the committed public corpus so the UI isn't empty:
|
||||
|
||||
@@ -81,7 +81,7 @@
|
||||
},
|
||||
"packages/clawhub": {
|
||||
"name": "clawhub",
|
||||
"version": "0.18.0",
|
||||
"version": "0.19.0",
|
||||
"bin": {
|
||||
"clawdhub": "bin/clawdhub.js",
|
||||
"clawhub": "bin/clawdhub.js",
|
||||
|
||||
Vendored
+6
@@ -19,6 +19,7 @@ import type * as devSeed from "../devSeed.js";
|
||||
import type * as devSeedExtra from "../devSeedExtra.js";
|
||||
import type * as downloads from "../downloads.js";
|
||||
import type * as functions from "../functions.js";
|
||||
import type * as githubAccountAgeBackfill from "../githubAccountAgeBackfill.js";
|
||||
import type * as githubBackups from "../githubBackups.js";
|
||||
import type * as githubBackupsNode from "../githubBackupsNode.js";
|
||||
import type * as githubIdentity from "../githubIdentity.js";
|
||||
@@ -58,6 +59,7 @@ import type * as lib_embeddingVisibility from "../lib/embeddingVisibility.js";
|
||||
import type * as lib_embeddings from "../lib/embeddings.js";
|
||||
import type * as lib_githubAccount from "../lib/githubAccount.js";
|
||||
import type * as lib_githubActionsOidc from "../lib/githubActionsOidc.js";
|
||||
import type * as lib_githubAuth from "../lib/githubAuth.js";
|
||||
import type * as lib_githubBackup from "../lib/githubBackup.js";
|
||||
import type * as lib_githubIdentity from "../lib/githubIdentity.js";
|
||||
import type * as lib_githubImport from "../lib/githubImport.js";
|
||||
@@ -93,6 +95,7 @@ import type * as lib_securityPrompt from "../lib/securityPrompt.js";
|
||||
import type * as lib_skillBackfill from "../lib/skillBackfill.js";
|
||||
import type * as lib_skillCapabilityTags from "../lib/skillCapabilityTags.js";
|
||||
import type * as lib_skillCards from "../lib/skillCards.js";
|
||||
import type * as lib_skillFileAccess from "../lib/skillFileAccess.js";
|
||||
import type * as lib_skillIcon from "../lib/skillIcon.js";
|
||||
import type * as lib_skillPublish from "../lib/skillPublish.js";
|
||||
import type * as lib_skillQuality from "../lib/skillQuality.js";
|
||||
@@ -158,6 +161,7 @@ declare const fullApi: ApiFromModules<{
|
||||
devSeedExtra: typeof devSeedExtra;
|
||||
downloads: typeof downloads;
|
||||
functions: typeof functions;
|
||||
githubAccountAgeBackfill: typeof githubAccountAgeBackfill;
|
||||
githubBackups: typeof githubBackups;
|
||||
githubBackupsNode: typeof githubBackupsNode;
|
||||
githubIdentity: typeof githubIdentity;
|
||||
@@ -197,6 +201,7 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/embeddings": typeof lib_embeddings;
|
||||
"lib/githubAccount": typeof lib_githubAccount;
|
||||
"lib/githubActionsOidc": typeof lib_githubActionsOidc;
|
||||
"lib/githubAuth": typeof lib_githubAuth;
|
||||
"lib/githubBackup": typeof lib_githubBackup;
|
||||
"lib/githubIdentity": typeof lib_githubIdentity;
|
||||
"lib/githubImport": typeof lib_githubImport;
|
||||
@@ -232,6 +237,7 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/skillBackfill": typeof lib_skillBackfill;
|
||||
"lib/skillCapabilityTags": typeof lib_skillCapabilityTags;
|
||||
"lib/skillCards": typeof lib_skillCards;
|
||||
"lib/skillFileAccess": typeof lib_skillFileAccess;
|
||||
"lib/skillIcon": typeof lib_skillIcon;
|
||||
"lib/skillPublish": typeof lib_skillPublish;
|
||||
"lib/skillQuality": typeof lib_skillQuality;
|
||||
|
||||
@@ -79,6 +79,13 @@ crons.interval(
|
||||
{ batchSize: 100 },
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"skill-scan-request-prune",
|
||||
{ hours: 6 },
|
||||
internal.securityScan.pruneExpiredSkillScanRequestsInternal,
|
||||
{ batchSize: 250 },
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"download-dedupe-prune",
|
||||
{ hours: 24 },
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Doc, Id, TableNames } from "./_generated/dataModel";
|
||||
import {
|
||||
internalMutation,
|
||||
isGitHubMirrorEligibleSkillDoc,
|
||||
repointPackageLatestRelease,
|
||||
scheduleGitHubBackupDeletionForSkill,
|
||||
@@ -13,6 +15,35 @@ import {
|
||||
syncSkillSearchDigestsForOwnerPublisherId,
|
||||
} from "./functions";
|
||||
|
||||
type WrappedHandler = {
|
||||
_handler: (ctx: unknown, args: Record<string, never>) => Promise<unknown>;
|
||||
};
|
||||
|
||||
function hasWrappedHandler(value: unknown): value is WrappedHandler {
|
||||
return typeof value === "function" && "_handler" in value && typeof value._handler === "function";
|
||||
}
|
||||
|
||||
function getWrappedHandler(value: unknown): WrappedHandler["_handler"] {
|
||||
if (!hasWrappedHandler(value)) {
|
||||
throw new Error("Expected a Convex function with a test-callable _handler");
|
||||
}
|
||||
return value._handler;
|
||||
}
|
||||
|
||||
function testId<TableName extends TableNames>(
|
||||
tableName: TableName,
|
||||
value: `${TableName}:${string}`,
|
||||
): Id<TableName> {
|
||||
if (!value.startsWith(`${tableName}:`)) {
|
||||
throw new Error(`Expected ${value} to be a ${tableName} id`);
|
||||
}
|
||||
return value as Id<TableName>;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
describe("package digest sync", () => {
|
||||
it("identifies GitHub mirror eligibility from skill visibility fields", () => {
|
||||
expect(isGitHubMirrorEligibleSkillDoc({ softDeletedAt: undefined })).toBe(true);
|
||||
@@ -677,4 +708,159 @@ describe("publisher digest scheduling", () => {
|
||||
{ ownerPublisherId: "publishers:demo", cursor: "next-skills" },
|
||||
);
|
||||
});
|
||||
|
||||
it("syncs recommended rank stats into the skill search digest after wrapped skill patches", async () => {
|
||||
const skillId = testId("skills", "skills:demo");
|
||||
const ownerUserId = testId("users", "users:owner");
|
||||
const publisherId = testId("publishers", "publishers:owner");
|
||||
const digestId = testId("skillSearchDigest", "skillSearchDigest:demo");
|
||||
|
||||
const skill = {
|
||||
_id: skillId,
|
||||
_creationTime: 1,
|
||||
slug: "demo-skill",
|
||||
displayName: "Demo Skill",
|
||||
summary: "Demo summary",
|
||||
ownerUserId,
|
||||
ownerPublisherId: publisherId,
|
||||
tags: {},
|
||||
statsDownloads: 3,
|
||||
statsStars: 2,
|
||||
statsInstallsCurrent: 4,
|
||||
statsInstallsAllTime: 5,
|
||||
stats: {
|
||||
downloads: 3,
|
||||
stars: 2,
|
||||
installsCurrent: 4,
|
||||
installsAllTime: 5,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
createdAt: 10,
|
||||
updatedAt: 20,
|
||||
} satisfies Doc<"skills">;
|
||||
const publisher = {
|
||||
_id: publisherId,
|
||||
_creationTime: 2,
|
||||
kind: "user",
|
||||
handle: "owner",
|
||||
displayName: "Owner",
|
||||
linkedUserId: ownerUserId,
|
||||
publishedSkills: 1,
|
||||
publishedPackages: 0,
|
||||
totalInstalls: 5,
|
||||
totalDownloads: 3,
|
||||
totalStars: 2,
|
||||
skillTotalInstalls: 5,
|
||||
skillTotalDownloads: 3,
|
||||
skillTotalStars: 2,
|
||||
createdAt: 10,
|
||||
updatedAt: 20,
|
||||
} satisfies Doc<"publishers">;
|
||||
const digest = {
|
||||
_id: digestId,
|
||||
_creationTime: 3,
|
||||
skillId,
|
||||
slug: "demo-skill",
|
||||
displayName: "Demo Skill",
|
||||
summary: "Demo summary",
|
||||
ownerUserId,
|
||||
ownerPublisherId: publisherId,
|
||||
ownerHandle: "owner",
|
||||
ownerKind: "user",
|
||||
ownerDisplayName: "Owner",
|
||||
tags: {},
|
||||
statsDownloads: 3,
|
||||
statsStars: 2,
|
||||
statsInstallsCurrent: 4,
|
||||
statsInstallsAllTime: 5,
|
||||
stats: {
|
||||
downloads: 3,
|
||||
stars: 2,
|
||||
installsCurrent: 4,
|
||||
installsAllTime: 5,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
createdAt: 10,
|
||||
updatedAt: 20,
|
||||
} satisfies Doc<"skillSearchDigest">;
|
||||
const docs = new Map<string, unknown>([
|
||||
[skillId, skill],
|
||||
[publisherId, publisher],
|
||||
[digestId, digest],
|
||||
]);
|
||||
const patchSkillRankStats = internalMutation({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
await ctx.db.patch(skillId, {
|
||||
statsDownloads: 13,
|
||||
statsStars: 7,
|
||||
statsInstallsAllTime: 11,
|
||||
stats: {
|
||||
downloads: 13,
|
||||
stars: 7,
|
||||
installsCurrent: 4,
|
||||
installsAllTime: 11,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
const handler = getWrappedHandler(patchSkillRankStats);
|
||||
const db = {
|
||||
system: {},
|
||||
normalizeId: vi.fn((tableName: string, id: string) =>
|
||||
id.startsWith(`${tableName}:`) ? id : null,
|
||||
),
|
||||
get: vi.fn(async (first: string, second?: string) => docs.get(second ?? first) ?? null),
|
||||
insert: vi.fn(async (tableName: string, value: unknown) => {
|
||||
if (!isRecord(value))
|
||||
throw new Error(`Expected inserted ${tableName} value to be an object`);
|
||||
const insertedId = `${tableName}:inserted`;
|
||||
docs.set(insertedId, { ...value, _id: insertedId, _creationTime: 0 });
|
||||
return insertedId;
|
||||
}),
|
||||
patch: vi.fn(
|
||||
async (first: string, second: string | Record<string, unknown>, third?: unknown) => {
|
||||
const id = typeof second === "string" ? second : first;
|
||||
const patch = typeof second === "string" ? third : second;
|
||||
if (!isRecord(patch)) throw new Error(`Expected patch for ${id} to be an object`);
|
||||
const existing = docs.get(id);
|
||||
if (!isRecord(existing)) throw new Error(`Missing test doc ${id}`);
|
||||
docs.set(id, { ...existing, ...patch });
|
||||
},
|
||||
),
|
||||
delete: vi.fn(async (first: string, second?: string) => {
|
||||
docs.delete(second ?? first);
|
||||
}),
|
||||
query: vi.fn((tableName: string) => ({
|
||||
withIndex: vi.fn(() => ({
|
||||
unique: vi.fn(async () => {
|
||||
if (tableName === "skillSearchDigest") return docs.get(digestId) ?? null;
|
||||
return null;
|
||||
}),
|
||||
collect: vi.fn(async () => []),
|
||||
paginate: vi.fn(async () => ({ page: [], isDone: true, continueCursor: "" })),
|
||||
take: vi.fn(async () => []),
|
||||
})),
|
||||
})),
|
||||
};
|
||||
|
||||
await expect(handler({ db }, {})).resolves.toBeUndefined();
|
||||
|
||||
expect(docs.get(digestId)).toEqual(
|
||||
expect.objectContaining({
|
||||
statsDownloads: 13,
|
||||
statsStars: 7,
|
||||
statsInstallsAllTime: 11,
|
||||
stats: expect.objectContaining({
|
||||
downloads: 13,
|
||||
stars: 7,
|
||||
installsAllTime: 11,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
import { ConvexError, v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { internalAction, internalMutation, internalQuery } from "./functions";
|
||||
import { fetchGitHubCreatedAtByProviderAccountId } from "./lib/githubAccount";
|
||||
import { getGitHubProviderAccountId } from "./lib/githubIdentity";
|
||||
import { getUserByHandleOrPersonalPublisher } from "./lib/publishers";
|
||||
|
||||
const DEFAULT_BATCH_SIZE = 25;
|
||||
const MAX_ACTION_BATCH_SIZE = 50;
|
||||
const MAX_LIST_BATCH_SIZE = 500;
|
||||
const DEFAULT_MAX_PAGES = 1;
|
||||
const MAX_MAX_PAGES = 20;
|
||||
|
||||
type BackfillCandidate = {
|
||||
userId: Id<"users">;
|
||||
providerAccountId: string;
|
||||
handle: string | null;
|
||||
};
|
||||
|
||||
type BackfillStats = {
|
||||
scanned: number;
|
||||
candidates: number;
|
||||
fetched: number;
|
||||
patched: number;
|
||||
failed: number;
|
||||
missingHandles: string[];
|
||||
errors: Array<{ userId: string; handle: string | null; message: string }>;
|
||||
};
|
||||
|
||||
type BackfillPageResult = {
|
||||
candidates: BackfillCandidate[];
|
||||
scanned: number;
|
||||
cursor: string | null;
|
||||
isDone: boolean;
|
||||
};
|
||||
|
||||
type BackfillHandlesResult = {
|
||||
candidates: BackfillCandidate[];
|
||||
missingHandles: string[];
|
||||
};
|
||||
|
||||
type BackfillResult =
|
||||
| { ok: true; stats: BackfillStats; cursor: string | null; isDone: boolean }
|
||||
| { ok: false; rateLimited: true; stats: BackfillStats; cursor: string | null; isDone: false };
|
||||
|
||||
function clampPositiveInteger(value: number | undefined, fallback: number, max: number) {
|
||||
if (!value || !Number.isFinite(value)) return fallback;
|
||||
return Math.max(1, Math.min(max, Math.floor(value)));
|
||||
}
|
||||
|
||||
async function candidateForUser(
|
||||
ctx: Parameters<typeof getGitHubProviderAccountId>[0],
|
||||
userId: Id<"users">,
|
||||
): Promise<BackfillCandidate | null> {
|
||||
const user = await ctx.db.get(userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) return null;
|
||||
const providerAccountId = await getGitHubProviderAccountId(ctx, userId);
|
||||
if (!providerAccountId || !/^\d+$/.test(providerAccountId)) return null;
|
||||
return { userId, providerAccountId, handle: user.handle ?? null };
|
||||
}
|
||||
|
||||
export const listGitHubCreatedAtBackfillPageInternal = internalQuery({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const batchSize = clampPositiveInteger(args.batchSize, DEFAULT_BATCH_SIZE, MAX_LIST_BATCH_SIZE);
|
||||
const page = await ctx.db
|
||||
.query("authAccounts")
|
||||
.withIndex("providerAndAccountId", (q) => q.eq("provider", "github"))
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
|
||||
|
||||
const candidates: BackfillCandidate[] = [];
|
||||
for (const account of page.page) {
|
||||
if (!/^\d+$/.test(account.providerAccountId)) continue;
|
||||
const user = await ctx.db.get(account.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) continue;
|
||||
candidates.push({
|
||||
userId: account.userId,
|
||||
providerAccountId: account.providerAccountId,
|
||||
handle: user.handle ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
candidates,
|
||||
scanned: page.page.length,
|
||||
cursor: page.continueCursor,
|
||||
isDone: page.isDone,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const listGitHubCreatedAtBackfillHandlesInternal = internalQuery({
|
||||
args: { handles: v.array(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
const seen = new Set<string>();
|
||||
const candidates: BackfillCandidate[] = [];
|
||||
const missingHandles: string[] = [];
|
||||
for (const handle of args.handles) {
|
||||
const user = await getUserByHandleOrPersonalPublisher(ctx, handle);
|
||||
if (!user) {
|
||||
missingHandles.push(handle);
|
||||
continue;
|
||||
}
|
||||
if (seen.has(user._id)) continue;
|
||||
seen.add(user._id);
|
||||
const candidate = await candidateForUser(ctx, user._id);
|
||||
if (candidate) candidates.push(candidate);
|
||||
}
|
||||
return { candidates, missingHandles };
|
||||
},
|
||||
});
|
||||
|
||||
export const applyGitHubCreatedAtBackfillInternal = internalMutation({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
githubCreatedAt: v.number(),
|
||||
fetchedAt: v.number(),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const user = await ctx.db.get(args.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) {
|
||||
return { patched: false };
|
||||
}
|
||||
if (args.dryRun) return { patched: false };
|
||||
await ctx.db.patch(args.userId, {
|
||||
githubCreatedAt: args.githubCreatedAt,
|
||||
githubFetchedAt: args.fetchedAt,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
return { patched: true };
|
||||
},
|
||||
});
|
||||
|
||||
export const applyGitHubCreatedAtBackfillBatchInternal = internalMutation({
|
||||
args: {
|
||||
items: v.array(
|
||||
v.object({
|
||||
userId: v.id("users"),
|
||||
githubCreatedAt: v.number(),
|
||||
}),
|
||||
),
|
||||
fetchedAt: v.number(),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
let patched = 0;
|
||||
let skipped = 0;
|
||||
for (const item of args.items) {
|
||||
const user = await ctx.db.get(item.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
if (!args.dryRun) {
|
||||
await ctx.db.patch(item.userId, {
|
||||
githubCreatedAt: item.githubCreatedAt,
|
||||
githubFetchedAt: args.fetchedAt,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
}
|
||||
patched += 1;
|
||||
}
|
||||
return { patched, skipped };
|
||||
},
|
||||
});
|
||||
|
||||
export const backfillGitHubCreatedAtInternal = internalAction({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxPages: v.optional(v.number()),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
handles: v.optional(v.array(v.string())),
|
||||
},
|
||||
handler: async (ctx: ActionCtx, args): Promise<BackfillResult> => {
|
||||
const batchSize = clampPositiveInteger(
|
||||
args.batchSize,
|
||||
DEFAULT_BATCH_SIZE,
|
||||
MAX_ACTION_BATCH_SIZE,
|
||||
);
|
||||
const maxPages = clampPositiveInteger(args.maxPages, DEFAULT_MAX_PAGES, MAX_MAX_PAGES);
|
||||
const dryRun = args.dryRun ?? false;
|
||||
const fetchedAt = Date.now();
|
||||
const stats = {
|
||||
scanned: 0,
|
||||
candidates: 0,
|
||||
fetched: 0,
|
||||
patched: 0,
|
||||
failed: 0,
|
||||
missingHandles: [] as string[],
|
||||
errors: [] as Array<{ userId: string; handle: string | null; message: string }>,
|
||||
};
|
||||
|
||||
let cursor = args.cursor ?? null;
|
||||
let isDone = true;
|
||||
let pages = 0;
|
||||
|
||||
while (pages < maxPages) {
|
||||
pages += 1;
|
||||
const page: BackfillPageResult | BackfillHandlesResult = args.handles
|
||||
? ((await ctx.runQuery(
|
||||
internal.githubAccountAgeBackfill.listGitHubCreatedAtBackfillHandlesInternal,
|
||||
{
|
||||
handles: args.handles,
|
||||
},
|
||||
)) as BackfillHandlesResult)
|
||||
: ((await ctx.runQuery(
|
||||
internal.githubAccountAgeBackfill.listGitHubCreatedAtBackfillPageInternal,
|
||||
{
|
||||
cursor: cursor ?? undefined,
|
||||
batchSize,
|
||||
},
|
||||
)) as BackfillPageResult);
|
||||
|
||||
const candidates = page.candidates;
|
||||
stats.scanned += "scanned" in page ? page.scanned : (args.handles?.length ?? 0);
|
||||
if ("missingHandles" in page) stats.missingHandles.push(...page.missingHandles);
|
||||
stats.candidates += candidates.length;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const githubCreatedAt = await fetchGitHubCreatedAtByProviderAccountId(
|
||||
candidate.providerAccountId,
|
||||
);
|
||||
stats.fetched += 1;
|
||||
const result: { patched: boolean } = await ctx.runMutation(
|
||||
internal.githubAccountAgeBackfill.applyGitHubCreatedAtBackfillInternal,
|
||||
{
|
||||
userId: candidate.userId,
|
||||
githubCreatedAt,
|
||||
fetchedAt,
|
||||
dryRun,
|
||||
},
|
||||
);
|
||||
if (result.patched) stats.patched += 1;
|
||||
} catch (error) {
|
||||
stats.failed += 1;
|
||||
const message = error instanceof ConvexError ? String(error.data) : String(error);
|
||||
if (stats.errors.length < 10) {
|
||||
stats.errors.push({
|
||||
userId: candidate.userId,
|
||||
handle: candidate.handle,
|
||||
message,
|
||||
});
|
||||
}
|
||||
if (/rate limit/i.test(message)) {
|
||||
return { ok: false as const, rateLimited: true as const, stats, cursor, isDone: false };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (args.handles) return { ok: true as const, stats, cursor: null, isDone: true };
|
||||
cursor = "cursor" in page ? page.cursor : null;
|
||||
isDone = "isDone" in page ? page.isDone : true;
|
||||
if (isDone) break;
|
||||
}
|
||||
|
||||
if (!dryRun && !isDone && cursor) {
|
||||
await ctx.scheduler.runAfter(
|
||||
0,
|
||||
internal.githubAccountAgeBackfill.backfillGitHubCreatedAtInternal,
|
||||
{
|
||||
cursor,
|
||||
batchSize,
|
||||
maxPages,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return { ok: true as const, stats, cursor, isDone };
|
||||
},
|
||||
});
|
||||
@@ -35,6 +35,10 @@ import {
|
||||
publishSoulV1Http,
|
||||
resolveSkillVersionV1Http,
|
||||
searchSkillsV1Http,
|
||||
skillScanBatchStatusV1Http,
|
||||
skillScanBatchSubmitV1Http,
|
||||
skillScanGetRouterV1Http,
|
||||
skillScanSubmitV1Http,
|
||||
skillSecurityVerdictsV1Http,
|
||||
skillsDeleteRouterV1Http,
|
||||
skillsGetRouterV1Http,
|
||||
@@ -87,6 +91,12 @@ http.route({
|
||||
handler: listSkillsV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
pathPrefix: `${ApiRoutes.skillScans}/`,
|
||||
method: "GET",
|
||||
handler: skillScanGetRouterV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.packages,
|
||||
method: "GET",
|
||||
@@ -141,6 +151,24 @@ http.route({
|
||||
handler: publishSkillV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.skillScans,
|
||||
method: "POST",
|
||||
handler: skillScanSubmitV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: `${ApiRoutes.skillScans}/batch`,
|
||||
method: "POST",
|
||||
handler: skillScanBatchSubmitV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: `${ApiRoutes.skillScans}/batch/status`,
|
||||
method: "POST",
|
||||
handler: skillScanBatchStatusV1Http,
|
||||
});
|
||||
|
||||
http.route({
|
||||
path: ApiRoutes.packages,
|
||||
method: "POST",
|
||||
|
||||
@@ -4,13 +4,15 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
vi.mock("./lib/apiTokenAuth", () => ({
|
||||
getOptionalApiTokenUser: vi.fn(),
|
||||
requireApiTokenUser: vi.fn(),
|
||||
requirePackagePublishAuth: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./skills", () => ({
|
||||
publishVersionForUser: vi.fn(),
|
||||
}));
|
||||
|
||||
const { getOptionalApiTokenUser, requireApiTokenUser } = await import("./lib/apiTokenAuth");
|
||||
const { getOptionalApiTokenUser, requireApiTokenUser, requirePackagePublishAuth } =
|
||||
await import("./lib/apiTokenAuth");
|
||||
const { publishVersionForUser } = await import("./skills");
|
||||
const { __handlers } = await import("./httpApi");
|
||||
const { hashSkillFiles } = await import("./lib/skills");
|
||||
@@ -23,6 +25,7 @@ describe("httpApi handlers", () => {
|
||||
afterEach(() => {
|
||||
vi.mocked(getOptionalApiTokenUser).mockReset();
|
||||
vi.mocked(requireApiTokenUser).mockReset();
|
||||
vi.mocked(requirePackagePublishAuth).mockReset();
|
||||
vi.mocked(publishVersionForUser).mockReset();
|
||||
});
|
||||
|
||||
@@ -444,18 +447,51 @@ describe("httpApi handlers", () => {
|
||||
});
|
||||
|
||||
it("cliUploadUrlHttp returns uploadUrl", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockResolvedValueOnce({ userId: "user1" } as never);
|
||||
const runMutation = vi.fn().mockResolvedValue("https://upload.local");
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValueOnce({
|
||||
kind: "user",
|
||||
userId: "user1",
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
});
|
||||
const response = await __handlers.cliUploadUrlHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request("https://x/api/cli/upload-url", { method: "POST" }),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({ uploadUrl: "https://upload.local" });
|
||||
expect(await response.json()).toEqual({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
});
|
||||
});
|
||||
|
||||
it("cliUploadUrlHttp accepts package publish tokens", async () => {
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValueOnce({
|
||||
kind: "github-actions",
|
||||
publishToken: { _id: "packagePublishTokens:1" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue({
|
||||
uploadUrl: "https://upload.local/package",
|
||||
uploadTicket: "packagePublishUploadTickets:2",
|
||||
});
|
||||
const response = await __handlers.cliUploadUrlHandler(
|
||||
makeCtx({ runMutation }),
|
||||
new Request("https://x/api/cli/upload-url", { method: "POST" }),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({
|
||||
uploadUrl: "https://upload.local/package",
|
||||
uploadTicket: "packagePublishUploadTickets:2",
|
||||
});
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({ publishTokenId: "packagePublishTokens:1" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("cliUploadUrlHttp returns 401 when unauthorized", async () => {
|
||||
vi.mocked(requireApiTokenUser).mockRejectedValueOnce(new Error("Unauthorized"));
|
||||
vi.mocked(requirePackagePublishAuth).mockRejectedValueOnce(new Error("Unauthorized"));
|
||||
const response = await __handlers.cliUploadUrlHandler(
|
||||
makeCtx({}),
|
||||
new Request("https://x/api/cli/upload-url", { method: "POST" }),
|
||||
|
||||
+11
-6
@@ -10,7 +10,7 @@ import { api, internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { httpAction } from "./functions";
|
||||
import { requireApiTokenUser } from "./lib/apiTokenAuth";
|
||||
import { requireApiTokenUser, requirePackagePublishAuth } from "./lib/apiTokenAuth";
|
||||
import { corsHeaders, mergeHeaders } from "./lib/httpHeaders";
|
||||
import { applyRateLimit } from "./lib/httpRateLimit";
|
||||
import { parseBooleanQueryParam, resolveBooleanQueryParam } from "./lib/httpUtils";
|
||||
@@ -148,11 +148,16 @@ export const cliWhoamiHttp = httpAction(cliWhoamiHandler);
|
||||
|
||||
async function cliUploadUrlHandler(ctx: ActionCtx, request: Request) {
|
||||
try {
|
||||
const { userId } = await requireApiTokenUser(ctx, request);
|
||||
const uploadUrl = await ctx.runMutation(internal.uploads.generateUploadUrlForUserInternal, {
|
||||
userId,
|
||||
});
|
||||
return json({ uploadUrl });
|
||||
const auth = await requirePackagePublishAuth(ctx, request);
|
||||
const upload =
|
||||
auth.kind === "user"
|
||||
? await ctx.runMutation(internal.uploads.createPackagePublishUploadForUserInternal, {
|
||||
userId: auth.userId,
|
||||
})
|
||||
: await ctx.runMutation(internal.uploads.createPackagePublishUploadForTokenInternal, {
|
||||
publishTokenId: auth.publishToken._id,
|
||||
});
|
||||
return json(upload);
|
||||
} catch (error) {
|
||||
return text(formatAuthFailure(error), 401);
|
||||
}
|
||||
|
||||
@@ -99,8 +99,8 @@ function writeTarString(target: Uint8Array, offset: number, width: number, value
|
||||
target.set(encoded.subarray(0, width), offset);
|
||||
}
|
||||
|
||||
function tarFile(path: string, content: string) {
|
||||
const bytes = new TextEncoder().encode(content);
|
||||
function tarFile(path: string, content: string | Uint8Array) {
|
||||
const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content;
|
||||
const header = new Uint8Array(TAR_BLOCK_SIZE);
|
||||
writeTarString(header, 0, 100, path);
|
||||
writeTarString(header, 100, 8, tarOctal(0o644, 8));
|
||||
@@ -123,7 +123,7 @@ function tarFile(path: string, content: string) {
|
||||
return [header, body];
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
function npmPackFixture(files: Record<string, string | Uint8Array>) {
|
||||
const parts: Uint8Array[] = [];
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
parts.push(...tarFile(path, content));
|
||||
@@ -145,13 +145,35 @@ function bytesToArrayBuffer(bytes: Uint8Array): ArrayBuffer {
|
||||
return copy.buffer;
|
||||
}
|
||||
|
||||
function packagePublishMetadata(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
name: "demo-plugin",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function packagePublishForm(payload: Record<string, unknown>) {
|
||||
const form = new FormData();
|
||||
form.set("payload", JSON.stringify(payload));
|
||||
return form;
|
||||
}
|
||||
|
||||
function makeCtx(partial: Record<string, unknown>) {
|
||||
const rateLimitStatus =
|
||||
typeof partial.rateLimitStatus === "function"
|
||||
? (partial.rateLimitStatus as (args: RateLimitArgs) => unknown)
|
||||
: null;
|
||||
const partialRunQuery =
|
||||
typeof partial.runQuery === "function"
|
||||
? (partial.runQuery as (query: unknown, args: Record<string, unknown>) => unknown)
|
||||
: null;
|
||||
const runQuery = vi.fn(async (query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return { ...okRate(), limit: args.limit };
|
||||
if (isRateLimitArgs(args)) {
|
||||
return rateLimitStatus?.(args) ?? { ...okRate(), limit: args.limit };
|
||||
}
|
||||
return partialRunQuery ? await partialRunQuery(query, args) : null;
|
||||
});
|
||||
const runMutation =
|
||||
@@ -1174,6 +1196,23 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(json.items[0].tags.latest).toBe("1.0.0");
|
||||
});
|
||||
|
||||
it("lists skills keeps the v1 no-sort default on updated ranking", async () => {
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("cursor" in args || "numItems" in args) {
|
||||
expect(args.sort).toBe("updated");
|
||||
return { page: [], nextCursor: null };
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const response = await __handlers.listSkillsV1Handler(
|
||||
makeCtx({ runQuery, runMutation }),
|
||||
new Request("https://example.com/api/v1/skills"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
|
||||
it("batches tag resolution across multiple skills into single query", async () => {
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("cursor" in args || "numItems" in args) {
|
||||
@@ -1432,6 +1471,8 @@ describe("httpApiV1 handlers", () => {
|
||||
|
||||
it("lists skills supports sort aliases", async () => {
|
||||
const checks: Array<[string, string | null]> = [
|
||||
["default", "recommended"],
|
||||
["recommended", "recommended"],
|
||||
["createdAt", "newest"],
|
||||
["created-at", "newest"],
|
||||
["newest", "newest"],
|
||||
@@ -1476,6 +1517,18 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(runQuery).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("lists skills rejects empty sort", async () => {
|
||||
const runQuery = vi.fn();
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const response = await __handlers.listSkillsV1Handler(
|
||||
makeCtx({ runQuery, runMutation }),
|
||||
new Request("https://example.com/api/v1/skills?sort="),
|
||||
);
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.text()).toBe("Invalid sort query parameter");
|
||||
expect(runQuery).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("lists skills forwards nonSuspiciousOnly", async () => {
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("sort" in args || "cursor" in args || "numItems" in args) {
|
||||
@@ -9464,37 +9517,26 @@ describe("httpApiV1 handlers", () => {
|
||||
const runAction = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
ownerHandle: "openclaw",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
}),
|
||||
);
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation }),
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: {
|
||||
store: vi.fn(async (entry: File) => `storage:${entry.name}`),
|
||||
},
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: "Bearer clh_test",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: "demo-plugin",
|
||||
ownerHandle: "openclaw",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
files: [
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
{
|
||||
path: ".codex-plugin/plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
],
|
||||
}),
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -9532,6 +9574,53 @@ describe("httpApiV1 handlers", () => {
|
||||
'Documents read from or written to the "publishers" table changed while this mutation was being run and on every subsequent retry.',
|
||||
),
|
||||
);
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
ownerHandle: "openclaw",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
}),
|
||||
);
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
});
|
||||
form.append(
|
||||
"clawpack",
|
||||
new File([bytesToArrayBuffer(pack)], "demo-plugin-1.0.0.tgz", {
|
||||
type: "application/octet-stream",
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { store: vi.fn(async (_entry: Blob) => "storage:1") },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: "Bearer clh_test",
|
||||
},
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(503);
|
||||
expect(response.headers.get("Retry-After")).toBe("1");
|
||||
await expect(response.text()).resolves.toContain("Transient ClawHub write contention");
|
||||
});
|
||||
|
||||
it("package publish rejects JSON request bodies before publish actions run", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation }),
|
||||
@@ -9541,34 +9630,39 @@ describe("httpApiV1 handlers", () => {
|
||||
Authorization: "Bearer clh_test",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: "demo-plugin",
|
||||
ownerHandle: "openclaw",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
files: [
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
{
|
||||
path: ".codex-plugin/plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
],
|
||||
}),
|
||||
body: JSON.stringify(packagePublishMetadata()),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(503);
|
||||
expect(response.headers.get("Retry-After")).toBe("1");
|
||||
await expect(response.text()).resolves.toContain("Transient ClawHub write contention");
|
||||
expect(response.status).toBe(415);
|
||||
expect(await response.text()).toBe("Package publish requires multipart/form-data");
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("package publish rejects browser session auth when token auth is not an API token", async () => {
|
||||
vi.mocked(getAuthUserId).mockResolvedValue("users:session" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockRejectedValue(new Error("Unauthorized"));
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const form = packagePublishForm(packagePublishMetadata());
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { store: vi.fn(async () => "storage:plugin") },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer convex-session-token" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(401);
|
||||
expect(await response.text()).toBe("Unauthorized");
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("multipart package publish ignores macOS junk files", async () => {
|
||||
@@ -9582,6 +9676,7 @@ describe("httpApiV1 handlers", () => {
|
||||
const runAction = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
|
||||
const storageStore = vi.fn(async () => "storage:plugin");
|
||||
const form = new FormData();
|
||||
form.set(
|
||||
"payload",
|
||||
@@ -9600,9 +9695,7 @@ describe("httpApiV1 handlers", () => {
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: {
|
||||
store: vi.fn(async (entry: File) => `storage:${entry.name}`),
|
||||
},
|
||||
storage: { store: storageStore },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
@@ -9612,18 +9705,30 @@ describe("httpApiV1 handlers", () => {
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(storageStore).toHaveBeenCalledTimes(1);
|
||||
expect(runAction).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
payload: expect.objectContaining({
|
||||
files: [
|
||||
expect.objectContaining({
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
}),
|
||||
size: 2,
|
||||
storageId: "storage:plugin",
|
||||
sha256: "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
|
||||
contentType: "application/json",
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const actionCall = runAction.mock.calls[0];
|
||||
expect(actionCall).toBeTruthy();
|
||||
expect(actionCall[1]).toEqual(
|
||||
expect.objectContaining({
|
||||
payload: expect.not.objectContaining({ artifact: expect.anything() }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("multipart ClawPack publish stores the tarball and extracted file metadata", async () => {
|
||||
@@ -9705,6 +9810,234 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(payload?.files?.map((file) => file.path)).toContain("dist/index.js");
|
||||
});
|
||||
|
||||
it("staged ClawPack publish derives artifact metadata from stored bytes", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
});
|
||||
const storageGet = vi.fn(async (storageId: string) =>
|
||||
storageId === "storage:clawpack"
|
||||
? new Blob([bytesToArrayBuffer(pack)], { type: "application/octet-stream" })
|
||||
: null,
|
||||
);
|
||||
const storageStore = vi.fn(async (_entry: Blob) => `storage:${storageStore.mock.calls.length}`);
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
family: "code-plugin",
|
||||
}),
|
||||
);
|
||||
form.set("clawpack", "storage:clawpack");
|
||||
form.set("clawpackUploadTicket", "packagePublishUploadTickets:1");
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { get: storageGet, store: storageStore },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:clawpack",
|
||||
auth: { kind: "user", userId: "users:1" },
|
||||
}),
|
||||
);
|
||||
expect(storageGet).toHaveBeenCalledWith("storage:clawpack");
|
||||
expect(storageStore).toHaveBeenCalledTimes(3);
|
||||
expect(runAction).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
payload: expect.objectContaining({
|
||||
artifact: expect.objectContaining({
|
||||
kind: "npm-pack",
|
||||
storageId: "storage:clawpack",
|
||||
size: pack.byteLength,
|
||||
npmFileCount: 3,
|
||||
}),
|
||||
files: [
|
||||
expect.objectContaining({ path: "package.json", storageId: "storage:1" }),
|
||||
expect.objectContaining({ path: "openclaw.plugin.json", storageId: "storage:2" }),
|
||||
expect.objectContaining({ path: "dist/index.js", storageId: "storage:3" }),
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("staged ClawPack publish rejects storage ids without upload tickets", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const storageGet = vi.fn();
|
||||
const form = packagePublishForm(packagePublishMetadata({ family: "code-plugin" }));
|
||||
form.set("clawpack", "storage:clawpack");
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { get: storageGet, store: vi.fn() },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.text()).toBe("Package tarball upload ticket required");
|
||||
expect(storageGet).not.toHaveBeenCalled();
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("multipart package publish rejects files and tarball together", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
|
||||
});
|
||||
const form = packagePublishForm(packagePublishMetadata({ family: "code-plugin" }));
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
form.append(
|
||||
"clawpack",
|
||||
new File([bytesToArrayBuffer(pack)], "demo-plugin-1.0.0.tgz", {
|
||||
type: "application/octet-stream",
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation, storage: { store: vi.fn() } }),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.text()).toBe(
|
||||
"Upload either a package tarball or individual files, not both",
|
||||
);
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["files[]", "tarball", "artifact", "extraMetadata"])(
|
||||
"multipart package publish rejects unsupported field %s",
|
||||
async (field) => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const form = packagePublishForm(packagePublishMetadata());
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
form.append(field, new File(["{}"], "ignored.json", { type: "application/json" }));
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation, storage: { store: vi.fn() } }),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.text()).toBe(`Unsupported package publish form field: ${field}`);
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["files", "artifact"])(
|
||||
"multipart package publish rejects caller-supplied %s metadata",
|
||||
async (field) => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
user: { _id: "users:1", handle: "p" },
|
||||
} as never);
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runAction = vi.fn();
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
[field]:
|
||||
field === "files"
|
||||
? [
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:attacker",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
]
|
||||
: {
|
||||
kind: "npm-pack",
|
||||
storageId: "storage:attacker",
|
||||
sha256: "a".repeat(64),
|
||||
size: 2,
|
||||
format: "tgz",
|
||||
npmIntegrity: "sha512-attacker",
|
||||
npmShasum: "a".repeat(40),
|
||||
npmTarballName: "demo-plugin-1.0.0.tgz",
|
||||
npmUnpackedSize: 2,
|
||||
npmFileCount: 1,
|
||||
},
|
||||
}),
|
||||
);
|
||||
form.append("files", new File(["{}"], "openclaw.plugin.json", { type: "application/json" }));
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation, storage: { store: vi.fn() } }),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: { Authorization: "Bearer clh_test" },
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(await response.text()).toContain(`Package publish payload: ${field}`);
|
||||
expect(runAction).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("package publish routes GitHub Actions auth through the trusted publisher action", async () => {
|
||||
vi.mocked(getOptionalApiTokenUserId).mockResolvedValue("users:1" as never);
|
||||
vi.mocked(requirePackagePublishAuth).mockResolvedValue({
|
||||
@@ -9715,36 +10048,35 @@ describe("httpApiV1 handlers", () => {
|
||||
const runAction = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ ok: true, packageId: "pkg:1", releaseId: "rel:1" });
|
||||
const form = packagePublishForm(
|
||||
packagePublishMetadata({
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
}),
|
||||
);
|
||||
const pack = npmPackFixture({
|
||||
"package/package.json": JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "demo.plugin" }),
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
});
|
||||
form.append(
|
||||
"clawpack",
|
||||
new File([bytesToArrayBuffer(pack)], "demo-plugin-1.0.0.tgz", {
|
||||
type: "application/octet-stream",
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await __handlers.publishPackageV1Handler(
|
||||
makeCtx({ runAction, runMutation }),
|
||||
makeCtx({
|
||||
runAction,
|
||||
runMutation,
|
||||
storage: { store: vi.fn(async (_entry: Blob) => "storage:1") },
|
||||
}),
|
||||
new Request("https://example.com/api/v1/packages", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: "Bearer clh_publish",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: "demo-plugin",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
bundle: { hostTargets: ["desktop"] },
|
||||
files: [
|
||||
{
|
||||
path: "openclaw.plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
{
|
||||
path: ".codex-plugin/plugin.json",
|
||||
size: 2,
|
||||
storageId: "storage:1",
|
||||
sha256: "a".repeat(64),
|
||||
},
|
||||
],
|
||||
}),
|
||||
body: form,
|
||||
}),
|
||||
);
|
||||
|
||||
|
||||
@@ -2,7 +2,11 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { formatUserFacingErrorMessage, resolveVersionTagsBatch } from "./httpApiV1/shared";
|
||||
import {
|
||||
formatUserFacingErrorMessage,
|
||||
parseMultipartSkillScan,
|
||||
resolveVersionTagsBatch,
|
||||
} from "./httpApiV1/shared";
|
||||
|
||||
function makeCtx() {
|
||||
return {
|
||||
@@ -82,4 +86,28 @@ describe("http API v1 shared helpers", () => {
|
||||
|
||||
expect(result).toEqual([{ stable: "1.5.0" }]);
|
||||
});
|
||||
|
||||
it("validates skill scan multipart payloads before storing uploaded files", async () => {
|
||||
const form = new FormData();
|
||||
form.set("payload", JSON.stringify({ source: { kind: "upload" }, update: true }));
|
||||
form.append("files", new Blob(["# Demo"], { type: "text/markdown" }), "SKILL.md");
|
||||
const request = new Request("https://clawhub.ai/api/v1/skills/-/scan", {
|
||||
method: "POST",
|
||||
body: form,
|
||||
});
|
||||
const store = vi.fn();
|
||||
const ctx = {
|
||||
storage: {
|
||||
store,
|
||||
delete: vi.fn(),
|
||||
},
|
||||
} as unknown as ActionCtx;
|
||||
|
||||
await expect(
|
||||
parseMultipartSkillScan(ctx, request, () => {
|
||||
throw new Error("update is not valid for uploaded scans");
|
||||
}),
|
||||
).rejects.toThrow("update is not valid for uploaded scans");
|
||||
expect(store).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,6 +20,10 @@ import {
|
||||
publishSkillV1Handler,
|
||||
resolveSkillVersionV1Handler,
|
||||
searchSkillsV1Handler,
|
||||
skillScanBatchStatusV1Handler,
|
||||
skillScanBatchSubmitV1Handler,
|
||||
skillScanGetRouterV1Handler,
|
||||
skillScanSubmitV1Handler,
|
||||
skillSecurityVerdictsV1Handler,
|
||||
skillsDeleteRouterV1Handler,
|
||||
skillsGetRouterV1Handler,
|
||||
@@ -61,6 +65,10 @@ export const listSkillsV1Http = httpAction(listSkillsV1Handler);
|
||||
export const skillsGetRouterV1Http = httpAction(skillsGetRouterV1Handler);
|
||||
export const publishSkillV1Http = httpAction(publishSkillV1Handler);
|
||||
export const skillSecurityVerdictsV1Http = httpAction(skillSecurityVerdictsV1Handler);
|
||||
export const skillScanSubmitV1Http = httpAction(skillScanSubmitV1Handler);
|
||||
export const skillScanGetRouterV1Http = httpAction(skillScanGetRouterV1Handler);
|
||||
export const skillScanBatchSubmitV1Http = httpAction(skillScanBatchSubmitV1Handler);
|
||||
export const skillScanBatchStatusV1Http = httpAction(skillScanBatchStatusV1Handler);
|
||||
export const skillsPostRouterV1Http = httpAction(skillsPostRouterV1Handler);
|
||||
export const skillsDeleteRouterV1Http = httpAction(skillsDeleteRouterV1Handler);
|
||||
export const exportSkillsV1Http = httpAction(exportSkillsV1Handler);
|
||||
|
||||
+245
-139
@@ -11,16 +11,18 @@ import {
|
||||
PackageReportRequestSchema,
|
||||
PackageReportTriageRequestSchema,
|
||||
PackageReleaseModerationRequestSchema,
|
||||
PackagePublishRequestSchema,
|
||||
PackagePublishMetadataSchema,
|
||||
PackageTransferRequestSchema,
|
||||
PackageTrustedPublisherUpsertRequestSchema,
|
||||
PublishTokenMintRequestSchema,
|
||||
isPluginCategorySlug,
|
||||
parseArk,
|
||||
type PackagePublishMetadata,
|
||||
type PackageAppealListStatus,
|
||||
type PackageModerationQueueStatus,
|
||||
type PackageOfficialMigrationListPhase,
|
||||
type PackageReportListStatus,
|
||||
type ServerPackagePublishRequest,
|
||||
} from "clawhub-schema";
|
||||
import { api, internal } from "../_generated/api";
|
||||
import type { Doc, Id } from "../_generated/dataModel";
|
||||
@@ -43,9 +45,13 @@ import {
|
||||
} from "../lib/packageSecurity";
|
||||
import {
|
||||
getClawPackSizeError,
|
||||
getPackageMultipartSizeError,
|
||||
getPublishFileSizeError,
|
||||
getPublishTotalSizeError,
|
||||
isPackageMultipartUploadTooLarge,
|
||||
MAX_CLAWPACK_BYTES,
|
||||
MAX_PUBLISH_FILE_BYTES,
|
||||
MAX_PUBLISH_TOTAL_BYTES,
|
||||
} from "../lib/publishLimits";
|
||||
import { getPublicSkillFileAccessBlock, isSkillVersionForSkill } from "../lib/skillFileAccess";
|
||||
import { isMacJunkPath, isTextFile } from "../lib/skills";
|
||||
@@ -120,6 +126,9 @@ const internalRefs = internal as unknown as {
|
||||
packagePublishTokens: {
|
||||
createInternal: unknown;
|
||||
};
|
||||
uploads: {
|
||||
consumePackagePublishUploadTicketInternal: unknown;
|
||||
};
|
||||
skills: {
|
||||
getSkillBySlugInternal: unknown;
|
||||
searchPackageCatalogForHttpInternal: unknown;
|
||||
@@ -1018,75 +1027,27 @@ function skillVersionTags(tags: Record<string, string>, version: string) {
|
||||
.map(([tag]) => tag);
|
||||
}
|
||||
|
||||
function parsePackagePublishBody(body: unknown) {
|
||||
const parsed = parseArk(PackagePublishRequestSchema, body, "Package publish payload") as {
|
||||
name: string;
|
||||
displayName?: string;
|
||||
ownerHandle?: string;
|
||||
family: "skill" | "code-plugin" | "bundle-plugin";
|
||||
version: string;
|
||||
changelog: string;
|
||||
manualOverrideReason?: string;
|
||||
channel?: "official" | "community" | "private";
|
||||
tags?: string[];
|
||||
source?: Record<string, unknown>;
|
||||
bundle?: Record<string, unknown>;
|
||||
files: Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}>;
|
||||
artifact?: {
|
||||
kind: "npm-pack";
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
size: number;
|
||||
format: "tgz";
|
||||
npmIntegrity: string;
|
||||
npmShasum: string;
|
||||
npmTarballName: string;
|
||||
npmUnpackedSize: number;
|
||||
npmFileCount: number;
|
||||
type StoredPackagePublishFile = ServerPackagePublishRequest["files"][number];
|
||||
type PackagePublishTarballArtifact = NonNullable<ServerPackagePublishRequest["artifact"]>;
|
||||
type ParsedPackageClawPack = Awaited<ReturnType<typeof parseClawPack>>;
|
||||
type PackagePublishAuth =
|
||||
| { kind: "user"; userId: Id<"users"> }
|
||||
| { kind: "github-actions"; publishToken: Doc<"packagePublishTokens"> };
|
||||
type PackagePublishTarballPart =
|
||||
| { kind: "file"; file: File }
|
||||
| {
|
||||
kind: "storage";
|
||||
storageId: Id<"_storage">;
|
||||
uploadTicket: Id<"packagePublishUploadTickets">;
|
||||
};
|
||||
};
|
||||
if (parsed.files.length === 0) throw new Error("files required");
|
||||
return {
|
||||
name: parsed.name,
|
||||
displayName: parsed.displayName ?? undefined,
|
||||
ownerHandle: parsed.ownerHandle?.trim().replace(/^@+/, "") || undefined,
|
||||
family: parsed.family,
|
||||
version: parsed.version,
|
||||
changelog: parsed.changelog,
|
||||
manualOverrideReason: parsed.manualOverrideReason?.trim() || undefined,
|
||||
channel: parsed.channel ?? undefined,
|
||||
tags: parsed.tags?.filter(Boolean) ?? undefined,
|
||||
source: parsed.source ?? undefined,
|
||||
bundle: parsed.bundle ?? undefined,
|
||||
files: parsed.files.map((file) => ({
|
||||
...file,
|
||||
storageId: file.storageId as Id<"_storage">,
|
||||
})),
|
||||
artifact: parsed.artifact
|
||||
? {
|
||||
...parsed.artifact,
|
||||
storageId: parsed.artifact.storageId as Id<"_storage">,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function inferStoredPackageContentType(path: string) {
|
||||
const lower = path.toLowerCase();
|
||||
if (lower.endsWith(".json")) return "application/json";
|
||||
if (lower.endsWith(".md") || lower.endsWith(".mdx") || lower.endsWith(".txt")) {
|
||||
return "text/plain; charset=utf-8";
|
||||
}
|
||||
if (lower.endsWith(".js") || lower.endsWith(".mjs") || lower.endsWith(".cjs")) {
|
||||
return "text/javascript; charset=utf-8";
|
||||
}
|
||||
if (lower.endsWith(".ts") || lower.endsWith(".tsx")) return "text/plain; charset=utf-8";
|
||||
if (isTextFile(path)) return "text/plain; charset=utf-8";
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
@@ -1096,9 +1057,10 @@ function bytesToArrayBuffer(bytes: Uint8Array) {
|
||||
return copy.buffer;
|
||||
}
|
||||
|
||||
async function storeClawPackFile(ctx: ActionCtx, entry: { path: string; bytes: Uint8Array }) {
|
||||
// npm-pack artifacts are bounded by the tarball and total package limits; the
|
||||
// legacy per-file cap only applies to raw file uploads.
|
||||
async function storeClawPackFile(
|
||||
ctx: ActionCtx,
|
||||
entry: { path: string; bytes: Uint8Array },
|
||||
): Promise<StoredPackagePublishFile> {
|
||||
const contentType = inferStoredPackageContentType(entry.path);
|
||||
const storageId = await ctx.storage.store(
|
||||
new Blob([bytesToArrayBuffer(entry.bytes)], { type: contentType }),
|
||||
@@ -1116,91 +1078,234 @@ async function storeClawPackFiles(
|
||||
ctx: ActionCtx,
|
||||
entries: Array<{ path: string; bytes: Uint8Array }>,
|
||||
) {
|
||||
const files: Awaited<ReturnType<typeof storeClawPackFile>>[] = [];
|
||||
// Convex HTTP actions have a tight memory ceiling; concurrent Blob/storage
|
||||
// work can duplicate large npm-pack entries enough to OOM the action.
|
||||
const files: StoredPackagePublishFile[] = [];
|
||||
// Convex HTTP actions have a tight memory ceiling; avoid concurrent Blob work.
|
||||
for (const entry of entries) {
|
||||
files.push(await storeClawPackFile(ctx, entry));
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
async function parseMultipartPackagePublish(ctx: ActionCtx, request: Request) {
|
||||
async function storeUploadedPackageFile(
|
||||
ctx: ActionCtx,
|
||||
entry: File,
|
||||
): Promise<StoredPackagePublishFile> {
|
||||
if (entry.size > MAX_PUBLISH_FILE_BYTES) {
|
||||
throw new Error(getPublishFileSizeError(entry.name));
|
||||
}
|
||||
const buffer = new Uint8Array(await entry.arrayBuffer());
|
||||
const contentType = inferStoredPackageContentType(entry.name);
|
||||
const storageId = await ctx.storage.store(
|
||||
new Blob([bytesToArrayBuffer(buffer)], { type: contentType }),
|
||||
);
|
||||
return {
|
||||
path: entry.name,
|
||||
size: entry.size,
|
||||
storageId,
|
||||
sha256: await sha256Hex(buffer),
|
||||
contentType,
|
||||
};
|
||||
}
|
||||
|
||||
function getFileParts(form: FormData, fields: readonly string[], stringPartError: string) {
|
||||
const parts = fields.flatMap((field) => form.getAll(field));
|
||||
if (parts.some((entry) => typeof entry === "string")) {
|
||||
throw new Error(stringPartError);
|
||||
}
|
||||
return parts.filter((entry): entry is File => typeof entry !== "string");
|
||||
}
|
||||
|
||||
function getTarballPart(form: FormData): PackagePublishTarballPart | null {
|
||||
const parts = form.getAll("clawpack");
|
||||
if (parts.length > 1) throw new Error("Upload one package tarball");
|
||||
const ticketParts = form.getAll("clawpackUploadTicket");
|
||||
if (ticketParts.length > 1) throw new Error("Upload one package tarball ticket");
|
||||
const ticketPart = ticketParts[0];
|
||||
if (ticketPart && typeof ticketPart !== "string") {
|
||||
throw new Error("Package tarball upload ticket must be a string");
|
||||
}
|
||||
const part = parts[0];
|
||||
if (!part) {
|
||||
if (ticketPart) throw new Error("Package tarball upload ticket requires a staged ClawPack");
|
||||
return null;
|
||||
}
|
||||
if (typeof part !== "string") {
|
||||
if (ticketPart) throw new Error("Package tarball upload ticket requires a staged ClawPack");
|
||||
return { kind: "file", file: part };
|
||||
}
|
||||
|
||||
const storageId = part.trim();
|
||||
if (!storageId) throw new Error("Package tarball storage id required");
|
||||
const uploadTicket = ticketPart?.trim();
|
||||
if (!uploadTicket) throw new Error("Package tarball upload ticket required");
|
||||
return {
|
||||
kind: "storage",
|
||||
storageId: storageId as Id<"_storage">,
|
||||
uploadTicket: uploadTicket as Id<"packagePublishUploadTickets">,
|
||||
};
|
||||
}
|
||||
|
||||
async function consumePackageTarballUploadTicket(
|
||||
ctx: ActionCtx,
|
||||
auth: PackagePublishAuth,
|
||||
part: Extract<PackagePublishTarballPart, { kind: "storage" }>,
|
||||
) {
|
||||
await ctx.runMutation(
|
||||
internalRefs.uploads.consumePackagePublishUploadTicketInternal as never,
|
||||
{
|
||||
uploadTicket: part.uploadTicket,
|
||||
storageId: part.storageId,
|
||||
auth:
|
||||
auth.kind === "user"
|
||||
? { kind: "user", userId: auth.userId }
|
||||
: { kind: "github-actions", publishTokenId: auth.publishToken._id },
|
||||
} as never,
|
||||
);
|
||||
}
|
||||
|
||||
async function readStoredPackageTarball(ctx: ActionCtx, storageId: Id<"_storage">) {
|
||||
const blob = await ctx.storage.get(storageId);
|
||||
if (!blob) throw new Error("Package tarball upload no longer exists");
|
||||
if (blob.size > MAX_CLAWPACK_BYTES) {
|
||||
throw new Error(getClawPackSizeError("uploaded ClawPack"));
|
||||
}
|
||||
return new Uint8Array(await blob.arrayBuffer());
|
||||
}
|
||||
|
||||
async function buildPackagePublishRequestFromClawPack(
|
||||
ctx: ActionCtx,
|
||||
metadata: PackagePublishMetadata,
|
||||
parsed: ParsedPackageClawPack,
|
||||
artifactBytes: Uint8Array,
|
||||
artifactStorageId: Id<"_storage">,
|
||||
): Promise<ServerPackagePublishRequest> {
|
||||
if (parsed.unpackedSize > MAX_PUBLISH_TOTAL_BYTES) {
|
||||
throw new Error(getPublishTotalSizeError("package"));
|
||||
}
|
||||
const artifact: PackagePublishTarballArtifact = {
|
||||
kind: "npm-pack",
|
||||
storageId: artifactStorageId,
|
||||
sha256: parsed.artifactSha256,
|
||||
size: artifactBytes.byteLength,
|
||||
format: "tgz",
|
||||
npmIntegrity: parsed.npmIntegrity,
|
||||
npmShasum: parsed.npmShasum,
|
||||
npmTarballName: parsed.npmTarballName,
|
||||
npmUnpackedSize: parsed.unpackedSize,
|
||||
npmFileCount: parsed.fileCount,
|
||||
};
|
||||
const files = await storeClawPackFiles(ctx, parsed.entries);
|
||||
return { ...metadata, files, artifact };
|
||||
}
|
||||
|
||||
const PACKAGE_PUBLISH_FILE_FIELDS = ["files"] as const;
|
||||
const PACKAGE_PUBLISH_TARBALL_FIELDS = ["clawpack"] as const;
|
||||
const PACKAGE_PUBLISH_FORM_FIELDS = new Set([
|
||||
"payload",
|
||||
...PACKAGE_PUBLISH_FILE_FIELDS,
|
||||
...PACKAGE_PUBLISH_TARBALL_FIELDS,
|
||||
"clawpackUploadTicket",
|
||||
]);
|
||||
|
||||
function multipartUploadPart(file: File) {
|
||||
return {
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type: file.type || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function parseMultipartPackagePublish(
|
||||
ctx: ActionCtx,
|
||||
auth: PackagePublishAuth,
|
||||
request: Request,
|
||||
): Promise<ServerPackagePublishRequest> {
|
||||
const form = await request.formData();
|
||||
const payloadRaw = form.get("payload");
|
||||
if (!payloadRaw || typeof payloadRaw !== "string") throw new Error("Missing payload");
|
||||
const payload = JSON.parse(payloadRaw) as Record<string, unknown>;
|
||||
const files: Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}> = [];
|
||||
let artifact:
|
||||
| {
|
||||
kind: "npm-pack";
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
size: number;
|
||||
format: "tgz";
|
||||
npmIntegrity: string;
|
||||
npmShasum: string;
|
||||
npmTarballName: string;
|
||||
npmUnpackedSize: number;
|
||||
npmFileCount: number;
|
||||
}
|
||||
| undefined;
|
||||
for (const field of form.keys()) {
|
||||
if (!PACKAGE_PUBLISH_FORM_FIELDS.has(field)) {
|
||||
throw new Error(`Unsupported package publish form field: ${field}`);
|
||||
}
|
||||
}
|
||||
|
||||
const clawpackEntry = form.get("clawpack") ?? form.get("artifact");
|
||||
if (clawpackEntry && typeof clawpackEntry !== "string") {
|
||||
if (form.getAll("files").some((entry) => typeof entry !== "string")) {
|
||||
throw new Error("Upload either a ClawPack tarball or individual files, not both");
|
||||
const payloadParts = form.getAll("payload");
|
||||
const payloadRaw = payloadParts[0];
|
||||
if (payloadParts.length !== 1 || typeof payloadRaw !== "string") {
|
||||
throw new Error("Package publish payload must be one JSON string");
|
||||
}
|
||||
const parsedPayload: unknown = JSON.parse(payloadRaw);
|
||||
const metadata: PackagePublishMetadata = parseArk(
|
||||
PackagePublishMetadataSchema,
|
||||
parsedPayload,
|
||||
"Package publish payload",
|
||||
);
|
||||
|
||||
const tarballPart = getTarballPart(form);
|
||||
const fileParts = getFileParts(
|
||||
form,
|
||||
PACKAGE_PUBLISH_FILE_FIELDS,
|
||||
"Package publish file uploads must be files",
|
||||
);
|
||||
|
||||
if (tarballPart) {
|
||||
if (fileParts.length > 0) {
|
||||
throw new Error("Upload either a package tarball or individual files, not both");
|
||||
}
|
||||
if (clawpackEntry.size > MAX_CLAWPACK_BYTES) {
|
||||
throw new Error(getClawPackSizeError(clawpackEntry.name));
|
||||
if (tarballPart.kind === "storage") {
|
||||
await consumePackageTarballUploadTicket(ctx, auth, tarballPart);
|
||||
const artifactBytes = await readStoredPackageTarball(ctx, tarballPart.storageId);
|
||||
const parsed = await parseClawPack(artifactBytes);
|
||||
return await buildPackagePublishRequestFromClawPack(
|
||||
ctx,
|
||||
metadata,
|
||||
parsed,
|
||||
artifactBytes,
|
||||
tarballPart.storageId,
|
||||
);
|
||||
}
|
||||
const artifactBytes = new Uint8Array(await clawpackEntry.arrayBuffer());
|
||||
|
||||
const tarballEntry = tarballPart.file;
|
||||
if (tarballEntry.size > MAX_CLAWPACK_BYTES) {
|
||||
throw new Error(getClawPackSizeError(tarballEntry.name));
|
||||
}
|
||||
if (
|
||||
isPackageMultipartUploadTooLarge({
|
||||
payloadJson: payloadRaw,
|
||||
fileFieldName: "clawpack",
|
||||
files: [multipartUploadPart(tarballEntry)],
|
||||
})
|
||||
) {
|
||||
throw new Error(getPackageMultipartSizeError());
|
||||
}
|
||||
const artifactBytes = new Uint8Array(await tarballEntry.arrayBuffer());
|
||||
const parsed = await parseClawPack(artifactBytes);
|
||||
const artifactBlob = new Blob([artifactBytes], { type: "application/octet-stream" });
|
||||
const artifactStorageId = await ctx.storage.store(artifactBlob);
|
||||
artifact = {
|
||||
kind: "npm-pack",
|
||||
storageId: artifactStorageId,
|
||||
sha256: parsed.artifactSha256,
|
||||
size: artifactBytes.byteLength,
|
||||
format: "tgz",
|
||||
npmIntegrity: parsed.npmIntegrity,
|
||||
npmShasum: parsed.npmShasum,
|
||||
npmTarballName: parsed.npmTarballName,
|
||||
npmUnpackedSize: parsed.unpackedSize,
|
||||
npmFileCount: parsed.fileCount,
|
||||
};
|
||||
files.push(...(await storeClawPackFiles(ctx, parsed.entries)));
|
||||
return parsePackagePublishBody({ ...payload, files, artifact });
|
||||
const artifactStorageId = await ctx.storage.store(
|
||||
new Blob([bytesToArrayBuffer(artifactBytes)], { type: "application/octet-stream" }),
|
||||
);
|
||||
return await buildPackagePublishRequestFromClawPack(
|
||||
ctx,
|
||||
metadata,
|
||||
parsed,
|
||||
artifactBytes,
|
||||
artifactStorageId,
|
||||
);
|
||||
}
|
||||
|
||||
for (const entry of form.getAll("files")) {
|
||||
if (typeof entry === "string") continue;
|
||||
if (isMacJunkPath(entry.name)) continue;
|
||||
if (entry.size > MAX_PUBLISH_FILE_BYTES) {
|
||||
throw new Error(getPublishFileSizeError(entry.name));
|
||||
}
|
||||
const buffer = new Uint8Array(await entry.arrayBuffer());
|
||||
const digest = await crypto.subtle.digest("SHA-256", buffer);
|
||||
const sha256 = Array.from(new Uint8Array(digest), (byte) =>
|
||||
byte.toString(16).padStart(2, "0"),
|
||||
).join("");
|
||||
const storageId = await ctx.storage.store(entry);
|
||||
files.push({
|
||||
path: entry.name,
|
||||
size: entry.size,
|
||||
storageId,
|
||||
sha256,
|
||||
contentType: entry.type || undefined,
|
||||
});
|
||||
if (
|
||||
isPackageMultipartUploadTooLarge({
|
||||
payloadJson: payloadRaw,
|
||||
fileFieldName: "files",
|
||||
files: fileParts.map(multipartUploadPart),
|
||||
})
|
||||
) {
|
||||
throw new Error(getPackageMultipartSizeError());
|
||||
}
|
||||
return parsePackagePublishBody({ ...payload, files });
|
||||
|
||||
const packageFileParts = fileParts.filter((entry) => !isMacJunkPath(entry.name));
|
||||
const files = await Promise.all(
|
||||
packageFileParts.map((entry) => storeUploadedPackageFile(ctx, entry)),
|
||||
);
|
||||
if (files.length === 0) throw new Error("files required");
|
||||
return { ...metadata, files };
|
||||
}
|
||||
|
||||
async function listPackages(
|
||||
@@ -1481,9 +1586,10 @@ export async function publishPackageV1Handler(ctx: ActionCtx, request: Request)
|
||||
|
||||
try {
|
||||
const contentType = request.headers.get("content-type") ?? "";
|
||||
const payload = contentType.includes("multipart/form-data")
|
||||
? await parseMultipartPackagePublish(ctx, request)
|
||||
: parsePackagePublishBody(await request.json());
|
||||
if (!contentType.includes("multipart/form-data")) {
|
||||
return text("Package publish requires multipart/form-data", 415, rate.headers);
|
||||
}
|
||||
const payload = await parseMultipartPackagePublish(ctx, auth.auth, request);
|
||||
const result =
|
||||
auth.auth.kind === "user"
|
||||
? await runActionRef(ctx, internalRefs.packages.publishPackageForUserInternal, {
|
||||
|
||||
@@ -422,6 +422,71 @@ export async function parseMultipartPublish(
|
||||
return parsePublishBody(body);
|
||||
}
|
||||
|
||||
export async function parseMultipartSkillScan(
|
||||
ctx: ActionCtx,
|
||||
request: Request,
|
||||
validatePayload?: (payload: Record<string, unknown>) => Record<string, unknown>,
|
||||
): Promise<{
|
||||
payload: Record<string, unknown>;
|
||||
files: Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}>;
|
||||
}> {
|
||||
const form = await request.formData();
|
||||
const payloadRaw = form.get("payload");
|
||||
if (!payloadRaw || typeof payloadRaw !== "string") {
|
||||
throw new Error("Missing payload");
|
||||
}
|
||||
let payload: Record<string, unknown>;
|
||||
try {
|
||||
payload = JSON.parse(payloadRaw) as Record<string, unknown>;
|
||||
} catch {
|
||||
throw new Error("Invalid JSON payload");
|
||||
}
|
||||
const validatedPayload = validatePayload ? validatePayload(payload) : payload;
|
||||
|
||||
const fileEntries = form
|
||||
.getAll("files")
|
||||
.map((entry) => toFileLike(entry))
|
||||
.filter((file): file is FileLikeEntry => Boolean(file))
|
||||
.filter((file) => !isMacJunkPath(file.name));
|
||||
if (fileEntries.length === 0) throw new Error("files required");
|
||||
if (!fileEntries.some((file) => file.name.trim().toLowerCase() === "skill.md")) {
|
||||
throw new Error("SKILL.md required");
|
||||
}
|
||||
const oversized = fileEntries.find((file) => file.size > MAX_PUBLISH_FILE_BYTES);
|
||||
if (oversized) throw new Error(getPublishFileSizeError(oversized.name));
|
||||
|
||||
const files: Array<{
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: Id<"_storage">;
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}> = [];
|
||||
|
||||
try {
|
||||
for (const file of fileEntries) {
|
||||
const path = file.name;
|
||||
const size = file.size;
|
||||
const contentType = file.type || undefined;
|
||||
const buffer = new Uint8Array(await file.arrayBuffer());
|
||||
const sha256 = await sha256Hex(buffer);
|
||||
const storageId = await ctx.storage.store(file as Blob);
|
||||
files.push({ path, size, storageId, sha256, contentType });
|
||||
}
|
||||
} catch (error) {
|
||||
await Promise.allSettled(files.map((file) => ctx.storage.delete(file.storageId)));
|
||||
throw error;
|
||||
}
|
||||
|
||||
return { payload: validatedPayload, files };
|
||||
}
|
||||
|
||||
export function parsePublishBody(body: unknown) {
|
||||
const parsed = parseArk(CliPublishRequestSchema, body, "Publish payload");
|
||||
if (parsed.files.length === 0) throw new Error("files required");
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import {
|
||||
ApiRoutes,
|
||||
ApiV1SkillBulkRescanBatchRequestSchema,
|
||||
ApiV1SkillBulkRescanStatusRequestSchema,
|
||||
ApiV1SkillRepairVtPendingRequestSchema,
|
||||
ApiV1SkillScanBatchRequestSchema,
|
||||
ApiV1SkillScanBatchStatusRequestSchema,
|
||||
ApiV1SkillScanSubmitRequestSchema,
|
||||
SkillAppealRequestSchema,
|
||||
SkillAppealResolveRequestSchema,
|
||||
SkillReportTriageRequestSchema,
|
||||
@@ -25,6 +29,7 @@ import type {
|
||||
import { selectGeneratedSkillCardFile, sourceSkillVersionFiles } from "../lib/skillCards";
|
||||
import { getPublicSkillFileAccessBlock, isSkillVersionForSkill } from "../lib/skillFileAccess";
|
||||
import {
|
||||
buildDeterministicZip,
|
||||
buildMergedExportZip,
|
||||
type MergedExportManifestEntry,
|
||||
validateSlug,
|
||||
@@ -37,6 +42,7 @@ import {
|
||||
getPathSegments,
|
||||
json,
|
||||
parseJsonPayload,
|
||||
parseMultipartSkillScan,
|
||||
parseMultipartPublish,
|
||||
parsePublishBody,
|
||||
publicApiOrigin,
|
||||
@@ -279,7 +285,10 @@ type SkillSecuritySnapshot = {
|
||||
|
||||
const internalRefs = internal as unknown as {
|
||||
securityScan: {
|
||||
createUploadedSkillScanRequestInternal: unknown;
|
||||
createPublishedSkillScanRequestInternal: unknown;
|
||||
enqueueBulkSkillRescanBatchForAdminInternal: unknown;
|
||||
getSkillScanRequestForUserInternal: unknown;
|
||||
getBulkSkillRescanBatchStatusForAdminInternal: unknown;
|
||||
requestSkillRescanForUserInternal: unknown;
|
||||
};
|
||||
@@ -309,6 +318,138 @@ async function runActionRef<T>(ctx: ActionCtx, ref: unknown, args: unknown): Pro
|
||||
return (await ctx.runAction(ref as never, args as never)) as T;
|
||||
}
|
||||
|
||||
function isMultipartRequest(request: Request) {
|
||||
return (
|
||||
request.headers.get("content-type")?.toLowerCase().includes("multipart/form-data") === true
|
||||
);
|
||||
}
|
||||
|
||||
async function deleteStoredScanFiles(ctx: ActionCtx, files: Array<{ storageId: Id<"_storage"> }>) {
|
||||
await Promise.allSettled(files.map((file) => ctx.storage.delete(file.storageId)));
|
||||
}
|
||||
|
||||
function encodeJsonEntry(value: unknown) {
|
||||
return new TextEncoder().encode(`${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function encodeTextEntry(value: string) {
|
||||
return new TextEncoder().encode(value);
|
||||
}
|
||||
|
||||
function scanReportPart(status: Record<string, unknown>, key: string) {
|
||||
const report = status.report;
|
||||
if (!report || typeof report !== "object" || Array.isArray(report)) return null;
|
||||
return (report as Record<string, unknown>)[key] ?? null;
|
||||
}
|
||||
|
||||
function buildSkillScanReportZip(status: Record<string, unknown>) {
|
||||
const manifest = {
|
||||
scanId: status.scanId,
|
||||
sourceKind: status.sourceKind,
|
||||
update: status.update,
|
||||
status: status.status,
|
||||
artifact: status.artifact ?? null,
|
||||
createdAt: status.createdAt,
|
||||
updatedAt: status.updatedAt,
|
||||
completedAt: status.completedAt ?? null,
|
||||
writtenBack: status.writtenBack === true,
|
||||
};
|
||||
const scanIdText = typeof status.scanId === "string" ? status.scanId : "";
|
||||
const statusText = typeof status.status === "string" ? status.status : "";
|
||||
const readme = [
|
||||
"# ClawHub Scan Report",
|
||||
"",
|
||||
`Scan ID: ${scanIdText}`,
|
||||
`Status: ${statusText}`,
|
||||
"",
|
||||
"This archive uses the ClawHub security-audit export shape:",
|
||||
"",
|
||||
"- manifest.json",
|
||||
"- clawscan.json",
|
||||
"- skillspector.json",
|
||||
"- static-analysis.json",
|
||||
"- virustotal.json",
|
||||
"- README.md",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
return buildDeterministicZip([
|
||||
{ path: "manifest.json", bytes: encodeJsonEntry(manifest) },
|
||||
{ path: "clawscan.json", bytes: encodeJsonEntry(scanReportPart(status, "clawscan")) },
|
||||
{ path: "skillspector.json", bytes: encodeJsonEntry(scanReportPart(status, "skillspector")) },
|
||||
{
|
||||
path: "static-analysis.json",
|
||||
bytes: encodeJsonEntry(scanReportPart(status, "staticAnalysis")),
|
||||
},
|
||||
{ path: "virustotal.json", bytes: encodeJsonEntry(scanReportPart(status, "virustotal")) },
|
||||
{ path: "README.md", bytes: encodeTextEntry(readme) },
|
||||
]);
|
||||
}
|
||||
|
||||
async function handleSkillScanBatchSubmit(ctx: ActionCtx, request: Request, headers: HeadersInit) {
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
const admin = requireAdminOrResponse(auth.user, headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
try {
|
||||
const body = parseArk(
|
||||
ApiV1SkillScanBatchRequestSchema,
|
||||
await request.json(),
|
||||
"Skill scan batch payload",
|
||||
) as {
|
||||
mode?: "all-active-latest";
|
||||
cursor?: string | null;
|
||||
batchSize?: number;
|
||||
dryRun?: boolean;
|
||||
};
|
||||
const result = await runMutationRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.enqueueBulkSkillRescanBatchForAdminInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
...(body.mode ? { mode: body.mode } : {}),
|
||||
cursor: body.cursor ?? null,
|
||||
...(body.batchSize !== undefined ? { batchSize: body.batchSize } : {}),
|
||||
...(body.dryRun !== undefined ? { dryRun: body.dryRun } : {}),
|
||||
},
|
||||
);
|
||||
return json(result, 200, headers);
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) return text("Invalid JSON", 400, headers);
|
||||
return text(error instanceof Error ? error.message : "Skill scan batch failed", 400, headers);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSkillScanBatchStatus(ctx: ActionCtx, request: Request, headers: HeadersInit) {
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
const admin = requireAdminOrResponse(auth.user, headers);
|
||||
if (!admin.ok) return admin.response;
|
||||
try {
|
||||
const body = parseArk(
|
||||
ApiV1SkillScanBatchStatusRequestSchema,
|
||||
await request.json(),
|
||||
"Skill scan batch status payload",
|
||||
) as { jobIds: string[] };
|
||||
const result = await runQueryRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.getBulkSkillRescanBatchStatusForAdminInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
jobIds: body.jobIds,
|
||||
},
|
||||
);
|
||||
return json(result, 200, headers);
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) return text("Invalid JSON", 400, headers);
|
||||
return text(
|
||||
error instanceof Error ? error.message : "Skill scan batch status failed",
|
||||
400,
|
||||
headers,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isDefinitiveSecurityStatus(
|
||||
status: NormalizedSecurityStatus | null | undefined,
|
||||
): status is "clean" | "suspicious" | "malicious" {
|
||||
@@ -964,6 +1105,127 @@ export async function skillSecurityVerdictsV1Handler(ctx: ActionCtx, request: Re
|
||||
return json({ schema: "clawhub.skill.security-verdicts.v1", items }, 200, rate.headers);
|
||||
}
|
||||
|
||||
export async function skillScanSubmitV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "write");
|
||||
if (!rate.ok) return rate.response;
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
|
||||
try {
|
||||
if (isMultipartRequest(request)) {
|
||||
const multipart = await parseMultipartSkillScan(ctx, request, (payload) => {
|
||||
const parsed = parseArk(
|
||||
ApiV1SkillScanSubmitRequestSchema,
|
||||
payload,
|
||||
"Skill scan payload",
|
||||
) as {
|
||||
source: { kind: "upload" } | { kind: "published"; slug: string; version?: string };
|
||||
update?: boolean;
|
||||
};
|
||||
if (parsed.source.kind !== "upload") {
|
||||
throw new Error("multipart scan payload must use source.kind=upload");
|
||||
}
|
||||
if (parsed.update === true) {
|
||||
throw new Error("update is not valid for uploaded scans");
|
||||
}
|
||||
return parsed;
|
||||
});
|
||||
const result = await runMutationRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.createUploadedSkillScanRequestInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
files: multipart.files,
|
||||
},
|
||||
).catch(async (error) => {
|
||||
await deleteStoredScanFiles(ctx, multipart.files);
|
||||
throw error;
|
||||
});
|
||||
return json(result, 202, rate.headers);
|
||||
}
|
||||
|
||||
const body = parseArk(
|
||||
ApiV1SkillScanSubmitRequestSchema,
|
||||
await request.json(),
|
||||
"Skill scan payload",
|
||||
) as {
|
||||
source: { kind: "upload" } | { kind: "published"; slug: string; version?: string };
|
||||
update?: boolean;
|
||||
};
|
||||
if (body.source.kind === "upload") {
|
||||
return text("uploaded scans must use multipart/form-data", 400, rate.headers);
|
||||
}
|
||||
const result = await runMutationRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.createPublishedSkillScanRequestInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
slug: body.source.slug,
|
||||
...(body.source.version ? { version: body.source.version } : {}),
|
||||
update: body.update === true,
|
||||
},
|
||||
);
|
||||
return json(result, 202, rate.headers);
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) return text("Invalid JSON", 400, rate.headers);
|
||||
return text(
|
||||
error instanceof Error ? error.message : "Skill scan submit failed",
|
||||
400,
|
||||
rate.headers,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function skillScanGetRouterV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "read");
|
||||
if (!rate.ok) return rate.response;
|
||||
const auth = await requireApiTokenUserOrResponse(ctx, request, rate.headers);
|
||||
if (!auth.ok) return auth.response;
|
||||
|
||||
const segments = getPathSegments(request, `${ApiRoutes.skillScans}/`);
|
||||
const scanId = segments[0];
|
||||
if (!scanId) return text("scanId required", 400, rate.headers);
|
||||
|
||||
try {
|
||||
const status = (await runQueryRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.getSkillScanRequestForUserInternal,
|
||||
{
|
||||
actorUserId: auth.userId,
|
||||
scanId: scanId as Id<"skillScanRequests">,
|
||||
},
|
||||
)) as Record<string, unknown>;
|
||||
|
||||
if (segments.length === 1) return json(status, 200, rate.headers);
|
||||
|
||||
if (segments.length === 2 && segments[1] === "download") {
|
||||
if (status.status !== "succeeded") return text("Scan is not complete", 409, rate.headers);
|
||||
const zip = buildSkillScanReportZip(status);
|
||||
const headers = mergeHeaders(rate.headers, {
|
||||
"Content-Type": "application/zip",
|
||||
"Content-Disposition": `attachment; filename="clawhub-scan-${scanId}.zip"`,
|
||||
});
|
||||
return new Response(zip, { status: 200, headers });
|
||||
}
|
||||
|
||||
return text("Not found", 404, rate.headers);
|
||||
} catch (error) {
|
||||
return text(error instanceof Error ? error.message : "Skill scan failed", 400, rate.headers);
|
||||
}
|
||||
}
|
||||
|
||||
export async function skillScanBatchSubmitV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "write");
|
||||
if (!rate.ok) return rate.response;
|
||||
return handleSkillScanBatchSubmit(ctx, request, rate.headers);
|
||||
}
|
||||
|
||||
export async function skillScanBatchStatusV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "write");
|
||||
if (!rate.ok) return rate.response;
|
||||
return handleSkillScanBatchStatus(ctx, request, rate.headers);
|
||||
}
|
||||
|
||||
export async function searchSkillsV1Handler(ctx: ActionCtx, request: Request) {
|
||||
const rate = await applyRateLimit(ctx, request, "read");
|
||||
if (!rate.ok) return rate.response;
|
||||
@@ -1034,6 +1296,7 @@ export async function resolveSkillVersionV1Handler(ctx: ActionCtx, request: Requ
|
||||
}
|
||||
|
||||
type SkillListSort =
|
||||
| "recommended"
|
||||
| "createdAt"
|
||||
| "updated"
|
||||
| "downloads"
|
||||
@@ -1042,11 +1305,14 @@ type SkillListSort =
|
||||
| "installsAllTime"
|
||||
| "trending";
|
||||
|
||||
type PublicListSort = "newest" | "updated" | "downloads" | "stars" | "installs";
|
||||
type PublicListSort = "recommended" | "newest" | "updated" | "downloads" | "stars" | "installs";
|
||||
|
||||
function parseListSort(value: string | null): SkillListSort | null {
|
||||
if (value === null) return "updated";
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized === "default" || normalized === "recommended") {
|
||||
return "recommended";
|
||||
}
|
||||
if (normalized === "createdat" || normalized === "created-at" || normalized === "newest") {
|
||||
return "createdAt";
|
||||
}
|
||||
@@ -1069,6 +1335,7 @@ function parseListSort(value: string | null): SkillListSort | null {
|
||||
}
|
||||
|
||||
function toPublicListSort(sort: Exclude<SkillListSort, "trending">): PublicListSort {
|
||||
if (sort === "recommended") return "recommended";
|
||||
if (sort === "createdAt") return "newest";
|
||||
if (sort === "updated") return "updated";
|
||||
if (sort === "downloads" || sort === "stars") return sort;
|
||||
|
||||
@@ -37,9 +37,9 @@ function tarFile(path: string, content: string) {
|
||||
return [header, body];
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
function npmPackFixtureEntries(files: Array<[string, string]>) {
|
||||
const parts: Uint8Array[] = [];
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
for (const [path, content] of files) {
|
||||
parts.push(...tarFile(path, content));
|
||||
}
|
||||
parts.push(new Uint8Array(BLOCK_SIZE), new Uint8Array(BLOCK_SIZE));
|
||||
@@ -53,6 +53,10 @@ function npmPackFixture(files: Record<string, string>) {
|
||||
return gzipSync(tar);
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
return npmPackFixtureEntries(Object.entries(files));
|
||||
}
|
||||
|
||||
describe("clawpack", () => {
|
||||
it("parses npm pack tarballs and computes npm integrity fields", async () => {
|
||||
const pack = npmPackFixture({
|
||||
@@ -95,6 +99,18 @@ describe("clawpack", () => {
|
||||
await expect(parseClawPack(pack)).rejects.toThrow("rooted under package");
|
||||
});
|
||||
|
||||
it("rejects duplicate normalized archive paths", async () => {
|
||||
const pack = npmPackFixtureEntries([
|
||||
["package/package.json", JSON.stringify({ name: "demo", version: "1.0.0" })],
|
||||
["package/openclaw.plugin.json", JSON.stringify({ id: "demo" })],
|
||||
["package/package.json", JSON.stringify({ name: "other", version: "9.9.9" })],
|
||||
]);
|
||||
|
||||
await expect(parseClawPack(pack)).rejects.toThrow(
|
||||
"ClawPack contains duplicate path: package.json",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses npm-style tarball names", () => {
|
||||
expect(npmTarballName("demo", "1.0.0")).toBe("demo-1.0.0.tgz");
|
||||
expect(npmTarballName("@scope/demo", "1.0.0")).toBe("scope-demo-1.0.0.tgz");
|
||||
|
||||
@@ -67,6 +67,7 @@ function isZeroBlock(block: Uint8Array) {
|
||||
|
||||
function parseTarEntries(bytes: Uint8Array): ClawPackEntry[] {
|
||||
const entries: ClawPackEntry[] = [];
|
||||
const paths = new Set<string>();
|
||||
let offset = 0;
|
||||
|
||||
while (offset + TAR_BLOCK_SIZE <= bytes.byteLength) {
|
||||
@@ -93,6 +94,10 @@ function parseTarEntries(bytes: Uint8Array): ClawPackEntry[] {
|
||||
offset = nextTarOffset(payloadOffset, size);
|
||||
continue;
|
||||
}
|
||||
if (paths.has(relPath)) {
|
||||
throw new Error(`ClawPack contains duplicate path: ${relPath}`);
|
||||
}
|
||||
paths.add(relPath);
|
||||
entries.push({
|
||||
path: relPath,
|
||||
bytes: Uint8Array.from(tarEntryPayload(bytes, payloadOffset, size)),
|
||||
|
||||
@@ -282,13 +282,128 @@ describe("requireGitHubAccountAge", () => {
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
headers: expect.objectContaining({
|
||||
"User-Agent": "clawhub",
|
||||
Authorization: "Bearer ghp_test123",
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("omits Authorization header when GITHUB_TOKEN is blank", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = new Date("2026-02-02T12:00:00Z");
|
||||
vi.setSystemTime(now);
|
||||
|
||||
vi.stubEnv("GITHUB_TOKEN", " ");
|
||||
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
_id: "users:1",
|
||||
githubCreatedAt: undefined,
|
||||
})
|
||||
.mockResolvedValueOnce("12345");
|
||||
const runMutation = vi.fn();
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
created_at: "2020-01-01T00:00:00Z",
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({ "User-Agent": "clawhub" }),
|
||||
}),
|
||||
);
|
||||
expect(fetchMock.mock.calls[0]?.[1]?.headers).not.toHaveProperty("Authorization");
|
||||
});
|
||||
|
||||
it("retries without Authorization when GITHUB_TOKEN is rejected", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = new Date("2026-02-02T12:00:00Z");
|
||||
vi.setSystemTime(now);
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghp_expired");
|
||||
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
_id: "users:1",
|
||||
githubCreatedAt: undefined,
|
||||
})
|
||||
.mockResolvedValueOnce("12345");
|
||||
const runMutation = vi.fn();
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ ok: false, status: 401 })
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
created_at: "2020-01-01T00:00:00Z",
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
|
||||
|
||||
expect(fetchMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
"User-Agent": "clawhub",
|
||||
Authorization: "Bearer ghp_expired",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(fetchMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: { "User-Agent": "clawhub" },
|
||||
}),
|
||||
);
|
||||
expect(runMutation).toHaveBeenCalledWith(internal.users.setGitHubCreatedAtInternal, {
|
||||
userId: "users:1",
|
||||
githubCreatedAt: Date.parse("2020-01-01T00:00:00Z"),
|
||||
});
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
"[githubAccount] GitHub API auth was rejected; retrying lookup without auth",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not retry unauthenticated 401 responses", async () => {
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
_id: "users:1",
|
||||
githubCreatedAt: undefined,
|
||||
})
|
||||
.mockResolvedValueOnce("12345");
|
||||
const runMutation = vi.fn();
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 401 });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await expect(
|
||||
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
|
||||
).rejects.toThrow(/GitHub account lookup failed/i);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({ "User-Agent": "clawhub" }),
|
||||
}),
|
||||
);
|
||||
expect(fetchMock.mock.calls[0]?.[1]?.headers).not.toHaveProperty("Authorization");
|
||||
});
|
||||
});
|
||||
|
||||
describe("syncGitHubProfile", () => {
|
||||
|
||||
+30
-26
@@ -2,6 +2,7 @@ import { ConvexError } from "convex/values";
|
||||
import { internal } from "../_generated/api";
|
||||
import type { Id } from "../_generated/dataModel";
|
||||
import type { ActionCtx } from "../_generated/server";
|
||||
import { buildGitHubApiHeaders } from "./githubAuth";
|
||||
import { GITHUB_PROFILE_SYNC_WINDOW_MS } from "./githubProfileSync";
|
||||
|
||||
const GITHUB_API = "https://api.github.com";
|
||||
@@ -22,13 +23,34 @@ function assertGitHubNumericId(providerAccountId: string) {
|
||||
}
|
||||
}
|
||||
|
||||
function buildGitHubHeaders() {
|
||||
const headers: Record<string, string> = { "User-Agent": "clawhub" };
|
||||
const token = process.env.GITHUB_TOKEN;
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
async function fetchGitHubUserByNumericId(providerAccountId: string) {
|
||||
assertGitHubNumericId(providerAccountId);
|
||||
const url = `${GITHUB_API}/user/${providerAccountId}`;
|
||||
const headers = await buildGitHubApiHeaders({ userAgent: "clawhub" });
|
||||
const response = await fetch(url, {
|
||||
headers,
|
||||
});
|
||||
if (response.status !== 401 || !headers.Authorization) return response;
|
||||
|
||||
console.warn("[githubAccount] GitHub API auth was rejected; retrying lookup without auth");
|
||||
return await fetch(url, {
|
||||
headers: { "User-Agent": "clawhub" },
|
||||
});
|
||||
}
|
||||
|
||||
export async function fetchGitHubCreatedAtByProviderAccountId(providerAccountId: string) {
|
||||
const response = await fetchGitHubUserByNumericId(providerAccountId);
|
||||
if (!response.ok) {
|
||||
if (response.status === 403 || response.status === 429) {
|
||||
throw new ConvexError("GitHub API rate limit exceeded — please try again in a few minutes");
|
||||
}
|
||||
throw new ConvexError("GitHub account lookup failed");
|
||||
}
|
||||
return headers;
|
||||
|
||||
const payload = (await response.json()) as GitHubUser;
|
||||
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN;
|
||||
if (!Number.isFinite(parsed)) throw new ConvexError("GitHub account lookup failed");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId: Id<"users">) {
|
||||
@@ -48,24 +70,8 @@ export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId:
|
||||
// Invariant: GitHub is our only auth provider, so this should never happen.
|
||||
throw new ConvexError("GitHub account required");
|
||||
}
|
||||
assertGitHubNumericId(providerAccountId);
|
||||
|
||||
// Fetch by immutable GitHub numeric ID to avoid username swap attacks entirely.
|
||||
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
|
||||
headers: buildGitHubHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
if (response.status === 403 || response.status === 429) {
|
||||
throw new ConvexError("GitHub API rate limit exceeded — please try again in a few minutes");
|
||||
}
|
||||
throw new ConvexError("GitHub account lookup failed");
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as GitHubUser;
|
||||
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN;
|
||||
if (!Number.isFinite(parsed)) throw new ConvexError("GitHub account lookup failed");
|
||||
|
||||
createdAt = parsed;
|
||||
createdAt = await fetchGitHubCreatedAtByProviderAccountId(providerAccountId);
|
||||
await ctx.runMutation(internal.users.setGitHubCreatedAtInternal, {
|
||||
userId,
|
||||
githubCreatedAt: createdAt,
|
||||
@@ -107,9 +113,7 @@ export async function syncGitHubProfile(ctx: ActionCtx, userId: Id<"users">) {
|
||||
|
||||
assertGitHubNumericId(providerAccountId);
|
||||
|
||||
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
|
||||
headers: buildGitHubHeaders(),
|
||||
});
|
||||
const response = await fetchGitHubUserByNumericId(providerAccountId);
|
||||
if (!response.ok) {
|
||||
// Silently fail - this is a best-effort sync, not critical path
|
||||
console.warn(`[syncGitHubProfile] GitHub API error for user ${userId}: ${response.status}`);
|
||||
|
||||
@@ -76,6 +76,34 @@ describe("fetchGitHubRepositoryIdentity", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("does not use GitHub App auth for arbitrary repository lookup", async () => {
|
||||
vi.stubEnv("GITHUB_APP_ID", "123");
|
||||
vi.stubEnv("GITHUB_APP_INSTALLATION_ID", "456");
|
||||
vi.stubEnv("GITHUB_APP_PRIVATE_KEY", "not-needed-for-this-test");
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghs_test_token");
|
||||
const fetchMock = vi.fn(async () =>
|
||||
Response.json({
|
||||
id: 123,
|
||||
full_name: "openclaw/clawhub",
|
||||
owner: { login: "openclaw", id: 456 },
|
||||
}),
|
||||
);
|
||||
|
||||
await fetchGitHubRepositoryIdentity("openclaw/clawhub", fetchMock);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/repos/openclaw/clawhub",
|
||||
expect.objectContaining({
|
||||
headers: expect.objectContaining({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: "Bearer ghs_test_token",
|
||||
"User-Agent": "clawhub/package-trusted-publisher",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("omits Authorization for repository lookup when GITHUB_TOKEN is blank", async () => {
|
||||
vi.stubEnv("GITHUB_TOKEN", " ");
|
||||
const fetchMock = vi.fn(async () =>
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { buildGitHubApiHeaders } from "./githubAuth";
|
||||
|
||||
type JwtHeader = {
|
||||
alg?: unknown;
|
||||
kid?: unknown;
|
||||
@@ -217,7 +219,7 @@ export async function fetchGitHubRepositoryIdentity(
|
||||
throw new Error(`Invalid GitHub repository: ${repository}`);
|
||||
}
|
||||
const response = await fetchImpl(`https://api.github.com/repos/${normalizedRepository}`, {
|
||||
headers: buildGitHubRepositoryLookupHeaders(),
|
||||
headers: await buildGitHubRepositoryLookupHeaders(fetchImpl),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
@@ -239,16 +241,16 @@ export async function fetchGitHubRepositoryIdentity(
|
||||
};
|
||||
}
|
||||
|
||||
function buildGitHubRepositoryLookupHeaders() {
|
||||
const headers: Record<string, string> = {
|
||||
Accept: "application/vnd.github+json",
|
||||
"User-Agent": "clawhub/package-trusted-publisher",
|
||||
};
|
||||
const token = process.env.GITHUB_TOKEN?.trim();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
}
|
||||
return headers;
|
||||
async function buildGitHubRepositoryLookupHeaders(fetchImpl: typeof fetch) {
|
||||
return await buildGitHubApiHeaders({
|
||||
accept: "application/vnd.github+json",
|
||||
fetchImpl,
|
||||
userAgent: "clawhub/package-trusted-publisher",
|
||||
// This lookup accepts arbitrary public repositories. GitHub App installation
|
||||
// tokens only see repositories where the App is installed, so prefer PAT or
|
||||
// anonymous auth here.
|
||||
useGitHubApp: false,
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeGitHubRepository(repository: string) {
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { generateKeyPairSync } from "node:crypto";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildGitHubApiHeaders, createGitHubAppInstallationToken } from "./githubAuth";
|
||||
|
||||
function stubGitHubAppEnv() {
|
||||
const { privateKey } = generateKeyPairSync("rsa", {
|
||||
modulusLength: 2048,
|
||||
privateKeyEncoding: { type: "pkcs1", format: "pem" },
|
||||
publicKeyEncoding: { type: "spki", format: "pem" },
|
||||
});
|
||||
vi.stubEnv("GITHUB_APP_ID", "3536245");
|
||||
vi.stubEnv("GITHUB_APP_INSTALLATION_ID", "987654");
|
||||
vi.stubEnv("GITHUB_APP_PRIVATE_KEY", privateKey);
|
||||
}
|
||||
|
||||
describe("githubAuth", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("mints a GitHub App installation token from app credentials", async () => {
|
||||
stubGitHubAppEnv();
|
||||
const fetchMock = vi.fn(async () =>
|
||||
Response.json({
|
||||
token: "ghs_app_token",
|
||||
expires_at: "2026-02-02T13:00:00Z",
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
createGitHubAppInstallationToken({ fetchImpl: fetchMock, userAgent: "clawhub/test" }),
|
||||
).resolves.toEqual({
|
||||
token: "ghs_app_token",
|
||||
expiresAt: Date.parse("2026-02-02T13:00:00Z"),
|
||||
});
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/app/installations/987654/access_tokens",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: expect.objectContaining({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: expect.stringMatching(/^Bearer [^.]+\.[^.]+\.[^.]+$/),
|
||||
"User-Agent": "clawhub/test",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("builds API headers with GitHub App auth before PAT fallback", async () => {
|
||||
stubGitHubAppEnv();
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghp_pat_token");
|
||||
const fetchMock = vi.fn(async () =>
|
||||
Response.json({
|
||||
token: "ghs_app_token",
|
||||
expires_at: "2026-02-02T13:00:00Z",
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
buildGitHubApiHeaders({ fetchImpl: fetchMock, userAgent: "clawhub/test" }),
|
||||
).resolves.toEqual({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: "Bearer ghs_app_token",
|
||||
"User-Agent": "clawhub/test",
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to GITHUB_TOKEN when GitHub App credentials are absent", async () => {
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghp_pat_token");
|
||||
|
||||
await expect(buildGitHubApiHeaders({ userAgent: "clawhub/test" })).resolves.toEqual({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: "Bearer ghp_pat_token",
|
||||
"User-Agent": "clawhub/test",
|
||||
});
|
||||
});
|
||||
|
||||
it("can skip GitHub App auth for arbitrary public resources", async () => {
|
||||
stubGitHubAppEnv();
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghp_pat_token");
|
||||
const fetchMock = vi.fn();
|
||||
|
||||
await expect(
|
||||
buildGitHubApiHeaders({
|
||||
fetchImpl: fetchMock,
|
||||
userAgent: "clawhub/test",
|
||||
useGitHubApp: false,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: "Bearer ghp_pat_token",
|
||||
"User-Agent": "clawhub/test",
|
||||
});
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,272 @@
|
||||
const GITHUB_API = "https://api.github.com";
|
||||
const DEFAULT_ACCEPT = "application/vnd.github+json";
|
||||
const DEFAULT_USER_AGENT = "clawhub/github-api";
|
||||
const APP_TOKEN_CACHE_BUFFER_MS = 60 * 1000;
|
||||
|
||||
type FetchImpl = typeof fetch;
|
||||
|
||||
type GitHubAppConfig = {
|
||||
appId: string;
|
||||
installationId: string;
|
||||
privateKey: string;
|
||||
};
|
||||
|
||||
type InstallationToken = {
|
||||
token: string;
|
||||
expiresAt: number;
|
||||
};
|
||||
|
||||
type CachedInstallationToken = InstallationToken & {
|
||||
cacheKey: string;
|
||||
};
|
||||
|
||||
let cachedInstallationToken: CachedInstallationToken | null = null;
|
||||
|
||||
export function isGitHubAppConfigured(env: NodeJS.ProcessEnv = process.env) {
|
||||
return Boolean(readGitHubAppConfig(env));
|
||||
}
|
||||
|
||||
export async function buildGitHubApiHeaders(options: {
|
||||
userAgent: string;
|
||||
accept?: string;
|
||||
fetchImpl?: FetchImpl;
|
||||
allowAnonymous?: boolean;
|
||||
useGitHubApp?: boolean;
|
||||
}): Promise<Record<string, string>> {
|
||||
const headers = buildGitHubHeaders({
|
||||
userAgent: options.userAgent,
|
||||
accept: options.accept,
|
||||
});
|
||||
|
||||
if (options.useGitHubApp !== false) {
|
||||
const appToken = await getCachedGitHubAppInstallationToken({
|
||||
fetchImpl: options.fetchImpl,
|
||||
userAgent: options.userAgent,
|
||||
});
|
||||
if (appToken) {
|
||||
headers.Authorization = `Bearer ${appToken}`;
|
||||
return headers;
|
||||
}
|
||||
}
|
||||
|
||||
const token = process.env.GITHUB_TOKEN?.trim();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
return headers;
|
||||
}
|
||||
|
||||
if (options.allowAnonymous === false) {
|
||||
throw new Error("GitHub API authentication is not configured");
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
export function buildGitHubHeaders(options: {
|
||||
userAgent: string;
|
||||
accept?: string;
|
||||
token?: string;
|
||||
isAppJwt?: boolean;
|
||||
}) {
|
||||
const headers: Record<string, string> = {
|
||||
Accept: options.accept ?? DEFAULT_ACCEPT,
|
||||
"User-Agent": options.userAgent,
|
||||
};
|
||||
if (options.token) {
|
||||
headers.Authorization = `Bearer ${options.token}`;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
export async function createGitHubAppInstallationToken(
|
||||
options: {
|
||||
fetchImpl?: FetchImpl;
|
||||
userAgent?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
now?: number;
|
||||
} = {},
|
||||
): Promise<InstallationToken> {
|
||||
const env = options.env ?? process.env;
|
||||
const config = readGitHubAppConfig(env);
|
||||
if (!config) throw new Error("GitHub App credentials missing");
|
||||
|
||||
const jwt = await createGitHubAppJwt(config.appId, config.privateKey, options.now ?? Date.now());
|
||||
const response = await (options.fetchImpl ?? fetch)(
|
||||
`${GITHUB_API}/app/installations/${config.installationId}/access_tokens`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: buildGitHubHeaders({
|
||||
userAgent: options.userAgent ?? DEFAULT_USER_AGENT,
|
||||
token: jwt,
|
||||
isAppJwt: true,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const message = await response.text();
|
||||
throw new Error(`GitHub App token failed: ${message}`);
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as { token?: string; expires_at?: string };
|
||||
const token = payload.token?.trim();
|
||||
if (!token) throw new Error("GitHub App token missing");
|
||||
const expiresAt = payload.expires_at ? Date.parse(payload.expires_at) : Number.NaN;
|
||||
if (!Number.isFinite(expiresAt)) throw new Error("GitHub App token expiry missing");
|
||||
return { token, expiresAt };
|
||||
}
|
||||
|
||||
async function getCachedGitHubAppInstallationToken(options: {
|
||||
fetchImpl?: FetchImpl;
|
||||
userAgent: string;
|
||||
}) {
|
||||
const config = readGitHubAppConfig(process.env);
|
||||
if (!config) return null;
|
||||
|
||||
const now = Date.now();
|
||||
const cacheKey = `${config.appId}:${config.installationId}:${hashCacheKey(config.privateKey)}`;
|
||||
if (
|
||||
cachedInstallationToken?.cacheKey === cacheKey &&
|
||||
cachedInstallationToken.expiresAt - APP_TOKEN_CACHE_BUFFER_MS > now
|
||||
) {
|
||||
return cachedInstallationToken.token;
|
||||
}
|
||||
|
||||
try {
|
||||
const next = await createGitHubAppInstallationToken({
|
||||
fetchImpl: options.fetchImpl,
|
||||
userAgent: options.userAgent,
|
||||
now,
|
||||
});
|
||||
cachedInstallationToken = { ...next, cacheKey };
|
||||
return next.token;
|
||||
} catch (error) {
|
||||
console.warn(`[githubAuth] GitHub App token unavailable: ${errorMessage(error)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readGitHubAppConfig(env: NodeJS.ProcessEnv): GitHubAppConfig | null {
|
||||
const appId = env.GITHUB_APP_ID?.trim();
|
||||
const installationId = env.GITHUB_APP_INSTALLATION_ID?.trim();
|
||||
const privateKey = env.GITHUB_APP_PRIVATE_KEY?.trim();
|
||||
if (!appId || !installationId || !privateKey) return null;
|
||||
return { appId, installationId, privateKey };
|
||||
}
|
||||
|
||||
async function createGitHubAppJwt(appId: string, rawPrivateKey: string, nowMs: number) {
|
||||
const now = Math.floor(nowMs / 1000);
|
||||
const header = { alg: "RS256", typ: "JWT" };
|
||||
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId };
|
||||
const signingInput = `${base64UrlString(JSON.stringify(header))}.${base64UrlString(
|
||||
JSON.stringify(payload),
|
||||
)}`;
|
||||
const key = await importPrivateKey(rawPrivateKey);
|
||||
const signature = await crypto.subtle.sign(
|
||||
"RSASSA-PKCS1-v1_5",
|
||||
key,
|
||||
new TextEncoder().encode(signingInput),
|
||||
);
|
||||
return `${signingInput}.${base64UrlBytes(new Uint8Array(signature))}`;
|
||||
}
|
||||
|
||||
async function importPrivateKey(rawPrivateKey: string) {
|
||||
const { label, der } = parsePem(rawPrivateKey);
|
||||
const pkcs8 = label === "RSA PRIVATE KEY" ? wrapPkcs1PrivateKeyAsPkcs8(der) : der;
|
||||
return await crypto.subtle.importKey(
|
||||
"pkcs8",
|
||||
pkcs8,
|
||||
{ name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
|
||||
false,
|
||||
["sign"],
|
||||
);
|
||||
}
|
||||
|
||||
function parsePem(raw: string) {
|
||||
const normalized = raw.replace(/\\n/g, "\n").trim();
|
||||
const match = /^-----BEGIN ([A-Z0-9 ]+)-----\s*([A-Za-z0-9+/=\s]+)\s*-----END \1-----$/m.exec(
|
||||
normalized,
|
||||
);
|
||||
if (!match) throw new Error("Invalid GitHub App private key");
|
||||
const label = match[1];
|
||||
if (label !== "PRIVATE KEY" && label !== "RSA PRIVATE KEY") {
|
||||
throw new Error(`Unsupported GitHub App private key type: ${label}`);
|
||||
}
|
||||
return { label, der: base64ToBytes(match[2]) };
|
||||
}
|
||||
|
||||
function wrapPkcs1PrivateKeyAsPkcs8(pkcs1: Uint8Array) {
|
||||
const version = derInteger(0);
|
||||
const rsaEncryptionAlgorithm = derSequence(
|
||||
new Uint8Array([0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01]),
|
||||
new Uint8Array([0x05, 0x00]),
|
||||
);
|
||||
return derSequence(version, rsaEncryptionAlgorithm, derOctetString(pkcs1));
|
||||
}
|
||||
|
||||
function derSequence(...parts: Uint8Array[]) {
|
||||
return derTagged(0x30, concatBytes(parts));
|
||||
}
|
||||
|
||||
function derInteger(value: number) {
|
||||
return derTagged(0x02, new Uint8Array([value]));
|
||||
}
|
||||
|
||||
function derOctetString(value: Uint8Array) {
|
||||
return derTagged(0x04, value);
|
||||
}
|
||||
|
||||
function derTagged(tag: number, value: Uint8Array) {
|
||||
return concatBytes([new Uint8Array([tag]), derLength(value.length), value]);
|
||||
}
|
||||
|
||||
function derLength(length: number) {
|
||||
if (length < 0x80) return new Uint8Array([length]);
|
||||
const bytes: number[] = [];
|
||||
let remaining = length;
|
||||
while (remaining > 0) {
|
||||
bytes.unshift(remaining & 0xff);
|
||||
remaining >>= 8;
|
||||
}
|
||||
return new Uint8Array([0x80 | bytes.length, ...bytes]);
|
||||
}
|
||||
|
||||
function concatBytes(parts: Uint8Array[]) {
|
||||
const total = parts.reduce((sum, part) => sum + part.length, 0);
|
||||
const out = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
out.set(part, offset);
|
||||
offset += part.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function base64UrlString(value: string) {
|
||||
return base64UrlBytes(new TextEncoder().encode(value));
|
||||
}
|
||||
|
||||
function base64UrlBytes(value: Uint8Array) {
|
||||
let binary = "";
|
||||
for (const byte of value) binary += String.fromCharCode(byte);
|
||||
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
|
||||
}
|
||||
|
||||
function base64ToBytes(value: string) {
|
||||
const binary = atob(value.replace(/\s/g, ""));
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i += 1) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function hashCacheKey(value: string) {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < value.length; i += 1) {
|
||||
hash = (hash * 31 + value.charCodeAt(i)) | 0;
|
||||
}
|
||||
return String(hash);
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
"use node";
|
||||
|
||||
import { createPrivateKey, createSign } from "node:crypto";
|
||||
import type { Id } from "../_generated/dataModel";
|
||||
import type { ActionCtx } from "../_generated/server";
|
||||
import { buildGitHubHeaders, createGitHubAppInstallationToken } from "./githubAuth";
|
||||
|
||||
const GITHUB_API = "https://api.github.com";
|
||||
const DEFAULT_REPO = "clawdbot/skills";
|
||||
@@ -93,7 +93,7 @@ export async function getGitHubBackupContext(): Promise<GitHubBackupContext> {
|
||||
const repo = process.env.GITHUB_SKILLS_REPO ?? DEFAULT_REPO;
|
||||
const root = process.env.GITHUB_SKILLS_ROOT ?? DEFAULT_ROOT;
|
||||
const [repoOwner, repoName] = parseRepo(repo);
|
||||
const token = await createInstallationToken();
|
||||
const { token } = await createGitHubAppInstallationToken({ userAgent: USER_AGENT });
|
||||
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`);
|
||||
const branch = repoInfo.default_branch ?? "main";
|
||||
|
||||
@@ -439,48 +439,6 @@ async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<"_storage">) {
|
||||
return buffer.toString("base64");
|
||||
}
|
||||
|
||||
async function createInstallationToken() {
|
||||
const appId = process.env.GITHUB_APP_ID;
|
||||
const installationId = process.env.GITHUB_APP_INSTALLATION_ID;
|
||||
if (!appId || !installationId) {
|
||||
throw new Error("GitHub App credentials missing");
|
||||
}
|
||||
const jwt = createAppJwt(appId);
|
||||
const response = await fetch(`${GITHUB_API}/app/installations/${installationId}/access_tokens`, {
|
||||
method: "POST",
|
||||
headers: buildHeaders(jwt, true),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const message = await response.text();
|
||||
throw new Error(`GitHub App token failed: ${message}`);
|
||||
}
|
||||
const payload = (await response.json()) as { token?: string };
|
||||
if (!payload.token) throw new Error("GitHub App token missing");
|
||||
return payload.token;
|
||||
}
|
||||
|
||||
function createAppJwt(appId: string) {
|
||||
const privateKey = loadPrivateKey();
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const header = { alg: "RS256", typ: "JWT" };
|
||||
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId };
|
||||
const encodedHeader = base64Url(JSON.stringify(header));
|
||||
const encodedPayload = base64Url(JSON.stringify(payload));
|
||||
const signingInput = `${encodedHeader}.${encodedPayload}`;
|
||||
const sign = createSign("RSA-SHA256");
|
||||
sign.update(signingInput);
|
||||
sign.end();
|
||||
const signature = sign.sign(privateKey);
|
||||
return `${signingInput}.${base64Url(signature)}`;
|
||||
}
|
||||
|
||||
function loadPrivateKey() {
|
||||
const raw = process.env.GITHUB_APP_PRIVATE_KEY;
|
||||
if (!raw) throw new Error("GITHUB_APP_PRIVATE_KEY is not configured");
|
||||
const normalized = raw.replace(/\\n/g, "\n");
|
||||
return createPrivateKey(normalized);
|
||||
}
|
||||
|
||||
async function createBlob(token: string, repoOwner: string, repoName: string, content: string) {
|
||||
const result = await githubPost<{ sha: string }>(
|
||||
token,
|
||||
@@ -531,11 +489,7 @@ async function githubPatch(token: string, path: string, body: unknown) {
|
||||
}
|
||||
|
||||
function buildHeaders(token: string, isAppJwt = false) {
|
||||
return {
|
||||
Authorization: `${isAppJwt ? "Bearer" : "token"} ${token}`,
|
||||
Accept: "application/vnd.github+json",
|
||||
"User-Agent": USER_AGENT,
|
||||
};
|
||||
return buildGitHubHeaders({ token, isAppJwt, userAgent: USER_AGENT });
|
||||
}
|
||||
|
||||
function parseRepo(repo: string) {
|
||||
@@ -570,11 +524,6 @@ function encodePath(path: string) {
|
||||
.join("/");
|
||||
}
|
||||
|
||||
function base64Url(value: string | Uint8Array) {
|
||||
const buffer = typeof value === "string" ? Buffer.from(value) : Buffer.from(value);
|
||||
return buffer.toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
|
||||
}
|
||||
|
||||
function toBase64(value: string) {
|
||||
return Buffer.from(value).toString("base64");
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"use node";
|
||||
|
||||
import { createPrivateKey, createSign } from "node:crypto";
|
||||
import type { Id } from "../_generated/dataModel";
|
||||
import type { ActionCtx } from "../_generated/server";
|
||||
import { buildGitHubHeaders, createGitHubAppInstallationToken } from "./githubAuth";
|
||||
|
||||
const GITHUB_API = "https://api.github.com";
|
||||
const DEFAULT_REPO = "clawdbot/souls";
|
||||
@@ -86,7 +86,7 @@ export async function getGitHubSoulBackupContext(): Promise<GitHubBackupContext>
|
||||
const repo = process.env.GITHUB_SOULS_REPO ?? DEFAULT_REPO;
|
||||
const root = process.env.GITHUB_SOULS_ROOT ?? DEFAULT_ROOT;
|
||||
const [repoOwner, repoName] = parseRepo(repo);
|
||||
const token = await createInstallationToken();
|
||||
const { token } = await createGitHubAppInstallationToken({ userAgent: USER_AGENT });
|
||||
const repoInfo = await githubGet<RepoInfo>(token, `/repos/${repoOwner}/${repoName}`);
|
||||
const branch = repoInfo.default_branch ?? "main";
|
||||
|
||||
@@ -297,48 +297,6 @@ async function fetchStorageBase64(ctx: ActionCtx, storageId: Id<"_storage">) {
|
||||
return buffer.toString("base64");
|
||||
}
|
||||
|
||||
async function createInstallationToken() {
|
||||
const appId = process.env.GITHUB_APP_ID;
|
||||
const installationId = process.env.GITHUB_APP_INSTALLATION_ID;
|
||||
if (!appId || !installationId) {
|
||||
throw new Error("GitHub App credentials missing");
|
||||
}
|
||||
const jwt = createAppJwt(appId);
|
||||
const response = await fetch(`${GITHUB_API}/app/installations/${installationId}/access_tokens`, {
|
||||
method: "POST",
|
||||
headers: buildHeaders(jwt, true),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const message = await response.text();
|
||||
throw new Error(`GitHub App token failed: ${message}`);
|
||||
}
|
||||
const payload = (await response.json()) as { token?: string };
|
||||
if (!payload.token) throw new Error("GitHub App token missing");
|
||||
return payload.token;
|
||||
}
|
||||
|
||||
function createAppJwt(appId: string) {
|
||||
const privateKey = loadPrivateKey();
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const header = { alg: "RS256", typ: "JWT" };
|
||||
const payload = { iat: now - 60, exp: now + 9 * 60, iss: appId };
|
||||
const encodedHeader = base64Url(JSON.stringify(header));
|
||||
const encodedPayload = base64Url(JSON.stringify(payload));
|
||||
const signingInput = `${encodedHeader}.${encodedPayload}`;
|
||||
const sign = createSign("RSA-SHA256");
|
||||
sign.update(signingInput);
|
||||
sign.end();
|
||||
const signature = sign.sign(privateKey);
|
||||
return `${signingInput}.${base64Url(signature)}`;
|
||||
}
|
||||
|
||||
function loadPrivateKey() {
|
||||
const raw = process.env.GITHUB_APP_PRIVATE_KEY;
|
||||
if (!raw) throw new Error("GITHUB_APP_PRIVATE_KEY is not configured");
|
||||
const normalized = raw.replace(/\\n/g, "\n");
|
||||
return createPrivateKey(normalized);
|
||||
}
|
||||
|
||||
async function createBlob(token: string, repoOwner: string, repoName: string, content: string) {
|
||||
const result = await githubPost<{ sha: string }>(
|
||||
token,
|
||||
@@ -389,11 +347,7 @@ async function githubPatch(token: string, path: string, body: unknown) {
|
||||
}
|
||||
|
||||
function buildHeaders(token: string, isAppJwt = false) {
|
||||
return {
|
||||
Authorization: `${isAppJwt ? "Bearer" : "token"} ${token}`,
|
||||
Accept: "application/vnd.github+json",
|
||||
"User-Agent": USER_AGENT,
|
||||
};
|
||||
return buildGitHubHeaders({ token, isAppJwt, userAgent: USER_AGENT });
|
||||
}
|
||||
|
||||
function parseRepo(repo: string) {
|
||||
@@ -428,11 +382,6 @@ function encodePath(path: string) {
|
||||
.join("/");
|
||||
}
|
||||
|
||||
function base64Url(value: string | Uint8Array) {
|
||||
const buffer = typeof value === "string" ? Buffer.from(value) : Buffer.from(value);
|
||||
return buffer.toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
|
||||
}
|
||||
|
||||
function toBase64(value: string) {
|
||||
return Buffer.from(value).toString("base64");
|
||||
}
|
||||
|
||||
@@ -271,6 +271,15 @@ function buildVerification(source: SourceInfo | undefined): PackageVerificationS
|
||||
scanStatus: "not-run",
|
||||
};
|
||||
}
|
||||
// `source.path` is the package directory inside the source repo (e.g.
|
||||
// "examples/openclaw-plugin"). When the package lives at the repo root the
|
||||
// CLI sends "." (or empty), and there's nothing useful to serialize. Only
|
||||
// promote real subpaths into `verification.sourcePath` so consumers can
|
||||
// build a `raw.githubusercontent.com/<repo>/<sha>/<path>/` base URL for
|
||||
// resolving relative README asset references.
|
||||
const rawPath = typeof source.path === "string" ? source.path.trim() : "";
|
||||
const sourcePath =
|
||||
rawPath && rawPath !== "." ? rawPath.replace(/^\/+/, "").replace(/\/+$/, "") : undefined;
|
||||
return {
|
||||
tier: "source-linked",
|
||||
scope: "artifact-only",
|
||||
@@ -278,6 +287,7 @@ function buildVerification(source: SourceInfo | undefined): PackageVerificationS
|
||||
sourceRepo: source.repo || source.url,
|
||||
sourceCommit: source.commit,
|
||||
sourceTag: source.ref,
|
||||
sourcePath: sourcePath || undefined,
|
||||
hasProvenance: false,
|
||||
scanStatus: "not-run",
|
||||
};
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
getPublishFileSizeError,
|
||||
getPublishTotalSizeError,
|
||||
MAX_CLAWPACK_BYTES,
|
||||
MAX_PACKAGE_MULTIPART_BYTES,
|
||||
MAX_PUBLISH_FILE_BYTES,
|
||||
} from "./publishLimits";
|
||||
|
||||
@@ -31,8 +32,9 @@ describe("publishLimits", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the ClawPack tarball limit separate from legacy file limits", () => {
|
||||
it("keeps ClawPack capacity above the multipart request budget", () => {
|
||||
expect(MAX_CLAWPACK_BYTES).toBe(120 * 1024 * 1024);
|
||||
expect(MAX_CLAWPACK_BYTES).toBeGreaterThan(MAX_PACKAGE_MULTIPART_BYTES);
|
||||
expect(MAX_CLAWPACK_BYTES).toBeGreaterThan(MAX_PUBLISH_FILE_BYTES);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,17 @@
|
||||
import { MAX_PACKAGE_CLAWPACK_BYTES } from "clawhub-schema";
|
||||
|
||||
export {
|
||||
estimatePackageMultipartUploadBytes,
|
||||
getPackageMultipartSizeError,
|
||||
isPackageMultipartUploadTooLarge,
|
||||
MAX_PACKAGE_MULTIPART_BYTES,
|
||||
type PackageMultipartUploadField,
|
||||
type PackageMultipartUploadPart,
|
||||
} from "clawhub-schema";
|
||||
|
||||
export const MAX_PUBLISH_TOTAL_BYTES = 50 * 1024 * 1024;
|
||||
export const MAX_PUBLISH_FILE_BYTES = 10 * 1024 * 1024;
|
||||
export const MAX_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
export const MAX_CLAWPACK_BYTES = MAX_PACKAGE_CLAWPACK_BYTES;
|
||||
|
||||
type SizedPathLike = {
|
||||
path: string;
|
||||
|
||||
@@ -95,6 +95,29 @@ describe("extractDigestFields", () => {
|
||||
expect(digest.updatedAt).toBe(2000);
|
||||
});
|
||||
|
||||
it("fills digest rank stats from legacy nested stats", () => {
|
||||
const skill = makeSkillDoc({
|
||||
statsDownloads: undefined,
|
||||
statsStars: undefined,
|
||||
statsInstallsCurrent: undefined,
|
||||
statsInstallsAllTime: undefined,
|
||||
stats: {
|
||||
downloads: 42,
|
||||
installsCurrent: 10,
|
||||
installsAllTime: 100,
|
||||
stars: 5,
|
||||
versions: 3,
|
||||
comments: 1,
|
||||
},
|
||||
});
|
||||
const digest = extractDigestFields(skill as never);
|
||||
|
||||
expect(digest.statsDownloads).toBe(42);
|
||||
expect(digest.statsStars).toBe(5);
|
||||
expect(digest.statsInstallsCurrent).toBe(10);
|
||||
expect(digest.statsInstallsAllTime).toBe(100);
|
||||
});
|
||||
|
||||
it("omits large fields not needed for search", () => {
|
||||
const skill = makeSkillDoc({
|
||||
moderationEvidence: [
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { Doc, Id } from "../_generated/dataModel";
|
||||
import type { MutationCtx } from "../_generated/server";
|
||||
import type { HydratableSkill, PublicPublisher } from "./public";
|
||||
import { tokenize } from "./searchText";
|
||||
import { readCanonicalStat } from "./skillStats";
|
||||
|
||||
function pick<T extends Record<string, unknown>, K extends keyof T>(obj: T, keys: K[]): Pick<T, K> {
|
||||
return Object.fromEntries(keys.map((k) => [k, obj[k]])) as Pick<T, K>;
|
||||
@@ -61,6 +62,10 @@ export type SkillSearchDigestFields = Pick<Doc<"skills">, (typeof SHARED_KEYS)[n
|
||||
export function extractDigestFields(skill: Doc<"skills">): SkillSearchDigestFields {
|
||||
return {
|
||||
...pick(skill, [...SHARED_KEYS]),
|
||||
statsDownloads: readCanonicalStat(skill, "downloads"),
|
||||
statsStars: readCanonicalStat(skill, "stars"),
|
||||
statsInstallsCurrent: readCanonicalStat(skill, "installsCurrent"),
|
||||
statsInstallsAllTime: readCanonicalStat(skill, "installsAllTime"),
|
||||
skillId: skill._id,
|
||||
normalizedSlug: normalizeSkillSearchText(skill.slug),
|
||||
normalizedSlugFirstToken: getFirstSearchToken(skill.slug),
|
||||
|
||||
@@ -44,6 +44,7 @@ const {
|
||||
applySkillCapabilityTagsInternal,
|
||||
backfillDigestVersionSummary,
|
||||
backfillLatestVersionSummaryInternal,
|
||||
backfillSkillSearchDigestInternal,
|
||||
backfillSkillFingerprintsInternalHandler,
|
||||
backfillSkillSummariesInternalHandler,
|
||||
backfillUserStatsInternalHandler,
|
||||
@@ -59,6 +60,113 @@ function makeBlob(text: string) {
|
||||
}
|
||||
|
||||
describe("maintenance backfill", () => {
|
||||
it("patches stale skill search digest rank stats from legacy skill stats", async () => {
|
||||
const existingDigest = {
|
||||
_id: "skillSearchDigest:1",
|
||||
skillId: "skills:1",
|
||||
slug: "demo",
|
||||
displayName: "Demo",
|
||||
summary: "Old summary",
|
||||
ownerUserId: "users:owner",
|
||||
tags: {},
|
||||
stats: {
|
||||
downloads: 3,
|
||||
stars: 2,
|
||||
installsCurrent: 4,
|
||||
installsAllTime: 5,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
softDeletedAt: undefined,
|
||||
createdAt: 100,
|
||||
updatedAt: 200,
|
||||
};
|
||||
const skill = {
|
||||
_id: "skills:1",
|
||||
slug: "demo",
|
||||
displayName: "Demo",
|
||||
summary: "New summary",
|
||||
ownerUserId: "users:owner",
|
||||
tags: {},
|
||||
stats: {
|
||||
downloads: 42,
|
||||
stars: 7,
|
||||
installsCurrent: 9,
|
||||
installsAllTime: 100,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
softDeletedAt: undefined,
|
||||
createdAt: 100,
|
||||
updatedAt: 300,
|
||||
};
|
||||
const paginate = vi.fn().mockResolvedValue({
|
||||
page: [skill],
|
||||
continueCursor: null,
|
||||
isDone: true,
|
||||
});
|
||||
const unique = vi.fn().mockResolvedValue(existingDigest);
|
||||
class TestEqBuilder {
|
||||
eq(_field: string, _value: unknown) {
|
||||
return this;
|
||||
}
|
||||
}
|
||||
const withIndex = vi.fn((_indexName: string, build: (q: TestEqBuilder) => unknown) => {
|
||||
build(new TestEqBuilder());
|
||||
return { unique };
|
||||
});
|
||||
const query = vi.fn((table: string) => {
|
||||
if (table === "skills") return { paginate };
|
||||
if (table === "skillSearchDigest") return { withIndex };
|
||||
throw new Error(`unexpected table ${table}`);
|
||||
});
|
||||
const patch = vi.fn().mockResolvedValue(undefined);
|
||||
const insert = vi.fn().mockResolvedValue("skillSearchDigest:inserted");
|
||||
const replace = vi.fn().mockResolvedValue(undefined);
|
||||
const deleteDoc = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
const result = await (
|
||||
backfillSkillSearchDigestInternal as unknown as { _handler: Function }
|
||||
)._handler(
|
||||
{
|
||||
db: {
|
||||
get: vi.fn(),
|
||||
query,
|
||||
patch,
|
||||
insert,
|
||||
replace,
|
||||
delete: deleteDoc,
|
||||
normalizeId: vi.fn(),
|
||||
},
|
||||
scheduler: {
|
||||
runAfter: vi.fn(),
|
||||
},
|
||||
} as never,
|
||||
{ batchSize: 10 },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ upserted: 1, isDone: true, scanned: 1 });
|
||||
expect(paginate).toHaveBeenCalledWith({ cursor: null, numItems: 10 });
|
||||
expect(withIndex).toHaveBeenCalledWith("by_skill", expect.any(Function));
|
||||
expect(insert).not.toHaveBeenCalled();
|
||||
expect(patch).toHaveBeenCalledWith(
|
||||
"skillSearchDigest:1",
|
||||
expect.objectContaining({
|
||||
summary: "New summary",
|
||||
statsDownloads: 42,
|
||||
statsStars: 7,
|
||||
statsInstallsCurrent: 9,
|
||||
statsInstallsAllTime: 100,
|
||||
stats: expect.objectContaining({
|
||||
downloads: 42,
|
||||
stars: 7,
|
||||
installsCurrent: 9,
|
||||
installsAllTime: 100,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("repairs summary + parsed by reparsing SKILL.md", async () => {
|
||||
const runQuery = vi.fn().mockResolvedValue({
|
||||
items: [
|
||||
@@ -290,6 +398,14 @@ describe("maintenance backfill", () => {
|
||||
_id: "skills:1",
|
||||
slug: "demo",
|
||||
displayName: "Demo",
|
||||
stats: {
|
||||
downloads: 0,
|
||||
stars: 0,
|
||||
installsCurrent: 0,
|
||||
installsAllTime: 0,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
latestVersionId: "skillVersions:1",
|
||||
latestVersionSummary: digest.latestVersionSummary,
|
||||
capabilityTags: ["read-files"],
|
||||
|
||||
+5
-10
@@ -20,6 +20,7 @@ import {
|
||||
extractValidatedDigestFields,
|
||||
getFirstSearchToken,
|
||||
normalizeSkillSearchText,
|
||||
upsertSkillSearchDigest,
|
||||
} from "./lib/skillSearchDigest";
|
||||
import { generateSkillSummary } from "./lib/skillSummary";
|
||||
|
||||
@@ -2083,16 +2084,10 @@ export const backfillSkillSearchDigestInternal = internalMutation({
|
||||
.query("skills")
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
|
||||
|
||||
let inserted = 0;
|
||||
let upserted = 0;
|
||||
for (const skill of page) {
|
||||
const existing = await ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_skill", (q) => q.eq("skillId", skill._id))
|
||||
.unique();
|
||||
if (!existing) {
|
||||
await ctx.db.insert("skillSearchDigest", await extractValidatedDigestFields(ctx, skill));
|
||||
inserted++;
|
||||
}
|
||||
await upsertSkillSearchDigest(ctx, await extractValidatedDigestFields(ctx, skill));
|
||||
upserted++;
|
||||
}
|
||||
|
||||
if (!isDone) {
|
||||
@@ -2102,7 +2097,7 @@ export const backfillSkillSearchDigestInternal = internalMutation({
|
||||
});
|
||||
}
|
||||
|
||||
return { inserted, isDone, scanned: page.length };
|
||||
return { upserted, isDone, scanned: page.length };
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
getPackageReleaseScanBackfillBatchInternal,
|
||||
getByName,
|
||||
list,
|
||||
publishPackage,
|
||||
publishPackageForTrustedPublisherInternal,
|
||||
publishPackageForUserInternal,
|
||||
listPackageReportsInternal,
|
||||
@@ -237,14 +236,6 @@ const searchForViewerInternalHandler = (
|
||||
Array<{ package: { name: string } }>
|
||||
>
|
||||
)._handler;
|
||||
const publishPackageHandler = (
|
||||
publishPackage as unknown as WrappedHandler<
|
||||
{
|
||||
payload: unknown;
|
||||
},
|
||||
unknown
|
||||
>
|
||||
)._handler;
|
||||
const publishPackageForUserInternalHandler = (
|
||||
publishPackageForUserInternal as unknown as WrappedHandler<
|
||||
{
|
||||
@@ -2826,6 +2817,62 @@ describe("packages public queries", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("derives missing public verification source paths from legacy release provenance", async () => {
|
||||
const verification = {
|
||||
tier: "source-linked",
|
||||
scope: "artifact-only",
|
||||
sourceRepo: "OpenViking/OpenViking",
|
||||
sourceCommit: "abcdef0123456789abcdef0123456789abcdef01",
|
||||
scanStatus: "clean",
|
||||
};
|
||||
const latestRelease = makeReleaseDoc({
|
||||
verification,
|
||||
source: {
|
||||
kind: "github",
|
||||
repo: "OpenViking/OpenViking",
|
||||
path: "openclaw-plugin",
|
||||
},
|
||||
});
|
||||
const { ctx } = makePackageCtx({
|
||||
pkg: makePackageDoc({
|
||||
name: "@openviking/openclaw-plugin",
|
||||
normalizedName: "@openviking/openclaw-plugin",
|
||||
verification,
|
||||
latestVersionSummary: {
|
||||
version: "1.0.0",
|
||||
verification,
|
||||
},
|
||||
}),
|
||||
latestRelease,
|
||||
});
|
||||
|
||||
await expect(
|
||||
getByNameHandler(ctx, {
|
||||
name: "@openviking/openclaw-plugin",
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
package: {
|
||||
verification: { sourcePath: "openclaw-plugin" },
|
||||
},
|
||||
latestRelease: {
|
||||
verification: { sourcePath: "openclaw-plugin" },
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
getVersionByNameHandler(ctx, {
|
||||
name: "@openviking/openclaw-plugin",
|
||||
version: "1.0.0",
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
package: {
|
||||
verification: { sourcePath: "openclaw-plugin" },
|
||||
},
|
||||
version: {
|
||||
verification: { sourcePath: "openclaw-plugin" },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not mark owner-readable blocked public packages as public download blocked", async () => {
|
||||
const { ctx } = makePackageCtx({
|
||||
pkg: makePackageDoc({
|
||||
@@ -6300,20 +6347,6 @@ describe("packages public queries", () => {
|
||||
expect(result).toEqual([expect.objectContaining({ name: "demo-plugin" })]);
|
||||
});
|
||||
|
||||
it("requires auth inside the public publish action", async () => {
|
||||
await expect(
|
||||
publishPackageHandler({ runQuery: vi.fn(), runMutation: vi.fn() } as never, {
|
||||
payload: {
|
||||
name: "demo-plugin",
|
||||
family: "bundle-plugin",
|
||||
version: "1.0.0",
|
||||
changelog: "init",
|
||||
files: [],
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("Unauthorized");
|
||||
});
|
||||
|
||||
it("records package reports for moderation", async () => {
|
||||
const insert = vi.fn(async (table: string) =>
|
||||
table === "packageReports" ? "packageReports:1" : "auditLogs:1",
|
||||
|
||||
+45
-33
@@ -1,5 +1,5 @@
|
||||
import {
|
||||
PackagePublishRequestSchema,
|
||||
ServerPackagePublishRequestSchema,
|
||||
getPackageScopeOwnerMismatch,
|
||||
isPluginCategorySlug,
|
||||
parseArk,
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
type PackageModerationQueueStatus,
|
||||
type PackageOfficialMigrationListPhase,
|
||||
type PackageOfficialMigrationPhase,
|
||||
type PackagePublishRequest,
|
||||
type ServerPackagePublishRequest,
|
||||
type PackageVerificationTier,
|
||||
} from "clawhub-schema";
|
||||
import { paginationOptsValidator } from "convex/server";
|
||||
@@ -33,7 +33,6 @@ import {
|
||||
assertModerator,
|
||||
getOptionalActiveAuthUserId,
|
||||
requireUser,
|
||||
requireUserFromAction,
|
||||
} from "./lib/access";
|
||||
import {
|
||||
assertArtifactAppealFinalAction,
|
||||
@@ -765,13 +764,29 @@ function resolvePublicPackageScanStatus(
|
||||
return pkg.scanStatus;
|
||||
}
|
||||
|
||||
function normalizePublicPackageSourcePath(sourcePath: unknown) {
|
||||
if (typeof sourcePath !== "string") return undefined;
|
||||
const trimmed = sourcePath.trim();
|
||||
if (!trimmed || trimmed === ".") return undefined;
|
||||
return trimmed.replace(/^\/+/, "").replace(/\/+$/, "") || undefined;
|
||||
}
|
||||
|
||||
function getReleaseSourcePath(release?: Pick<Doc<"packageReleases">, "source"> | null) {
|
||||
const source = release?.source;
|
||||
if (!source || typeof source !== "object" || Array.isArray(source)) return undefined;
|
||||
return normalizePublicPackageSourcePath((source as { path?: unknown }).path);
|
||||
}
|
||||
|
||||
function resolvePublicPackageVerification(
|
||||
pkg: Pick<Doc<"packages">, "verification" | "latestVersionSummary" | "scanStatus">,
|
||||
latestRelease?: Doc<"packageReleases"> | null,
|
||||
) {
|
||||
const scanStatus = resolvePublicPackageScanStatus(pkg, latestRelease);
|
||||
const source = pkg.verification ?? pkg.latestVersionSummary?.verification;
|
||||
return source && scanStatus ? { ...source, scanStatus } : source;
|
||||
if (!source) return source;
|
||||
const sourcePath = source.sourcePath ?? getReleaseSourcePath(latestRelease);
|
||||
const verification = sourcePath ? { ...source, sourcePath } : source;
|
||||
return scanStatus ? { ...verification, scanStatus } : verification;
|
||||
}
|
||||
|
||||
function toPublicPackage(
|
||||
@@ -823,6 +838,19 @@ function omitLegacyClawScanNoteFields(release: Doc<"packageReleases">) {
|
||||
return publicRelease;
|
||||
}
|
||||
|
||||
function toPublicPackageRelease(release: Doc<"packageReleases">) {
|
||||
const publicRelease = omitLegacyClawScanNoteFields(release);
|
||||
const sourcePath = release.verification?.sourcePath ?? getReleaseSourcePath(release);
|
||||
if (!release.verification || !sourcePath) return publicRelease;
|
||||
return {
|
||||
...publicRelease,
|
||||
verification: {
|
||||
...release.verification,
|
||||
sourcePath,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function packageArtifactSummary(
|
||||
release: Pick<
|
||||
Doc<"packageReleases">,
|
||||
@@ -1908,7 +1936,7 @@ export const getByName = query({
|
||||
package: publicPackage,
|
||||
latestRelease:
|
||||
latestRelease && !latestRelease.softDeletedAt
|
||||
? omitLegacyClawScanNoteFields(latestRelease)
|
||||
? toPublicPackageRelease(latestRelease)
|
||||
: null,
|
||||
owner,
|
||||
};
|
||||
@@ -1949,7 +1977,7 @@ export const getManageContext = query({
|
||||
|
||||
return {
|
||||
package: pkg,
|
||||
latestRelease: omitLegacyClawScanNoteFields(latestRelease),
|
||||
latestRelease: toPublicPackageRelease(latestRelease),
|
||||
};
|
||||
},
|
||||
});
|
||||
@@ -1979,7 +2007,7 @@ export const getByNameForStaff = query({
|
||||
package: pkg,
|
||||
latestRelease:
|
||||
latestRelease && !latestRelease.softDeletedAt
|
||||
? omitLegacyClawScanNoteFields(latestRelease)
|
||||
? toPublicPackageRelease(latestRelease)
|
||||
: null,
|
||||
owner,
|
||||
highlighted: highlighted
|
||||
@@ -2013,7 +2041,7 @@ export const getByNameForViewerInternal = internalQuery({
|
||||
package: publicPackage,
|
||||
latestRelease:
|
||||
latestRelease && !latestRelease.softDeletedAt
|
||||
? omitLegacyClawScanNoteFields(latestRelease)
|
||||
? toPublicPackageRelease(latestRelease)
|
||||
: null,
|
||||
owner,
|
||||
};
|
||||
@@ -2092,7 +2120,7 @@ export const getVersionByName = query({
|
||||
if (!publicPackage) return null;
|
||||
return {
|
||||
package: publicPackage,
|
||||
version: omitLegacyClawScanNoteFields(release),
|
||||
version: toPublicPackageRelease(release),
|
||||
};
|
||||
},
|
||||
});
|
||||
@@ -2123,7 +2151,7 @@ export const getVersionByNameForViewerInternal = internalQuery({
|
||||
if (!publicPackage) return null;
|
||||
return {
|
||||
package: publicPackage,
|
||||
version: omitLegacyClawScanNoteFields(release),
|
||||
version: toPublicPackageRelease(release),
|
||||
};
|
||||
},
|
||||
});
|
||||
@@ -2160,7 +2188,7 @@ export const getVersionSecurityByNameForViewerInternal = internalQuery({
|
||||
...publicPackage,
|
||||
publicDownloadBlocked,
|
||||
},
|
||||
version: omitLegacyClawScanNoteFields(release),
|
||||
version: toPublicPackageRelease(release),
|
||||
};
|
||||
},
|
||||
});
|
||||
@@ -5027,9 +5055,9 @@ function buildGitHubActionsPublishActor(
|
||||
}
|
||||
|
||||
function resolveTrustedPublishSource(
|
||||
payload: PackagePublishRequest,
|
||||
payload: ServerPackagePublishRequest,
|
||||
publishToken: Doc<"packagePublishTokens">,
|
||||
): PackagePublishRequest["source"] {
|
||||
): ServerPackagePublishRequest["source"] {
|
||||
const source = payload.source;
|
||||
if (source && source.kind !== "github") {
|
||||
throw new ConvexError("Trusted publishes only support GitHub source metadata");
|
||||
@@ -5081,11 +5109,11 @@ async function publishPackageImpl(
|
||||
auth: PackagePublishAuthContext,
|
||||
rawPayload: unknown,
|
||||
) {
|
||||
const payload = parseArk(
|
||||
PackagePublishRequestSchema,
|
||||
const payload = parseArk<ServerPackagePublishRequest>(
|
||||
ServerPackagePublishRequestSchema,
|
||||
rawPayload,
|
||||
"Package publish payload",
|
||||
) as PackagePublishRequest;
|
||||
);
|
||||
if (payload.family === "skill") {
|
||||
throw new ConvexError("Skill packages must use the skills publish flow");
|
||||
}
|
||||
@@ -5204,7 +5232,7 @@ async function publishPackageImpl(
|
||||
}
|
||||
|
||||
const displayName = payload.displayName?.trim() || name;
|
||||
const files = normalizePublishFiles(payload.files as never);
|
||||
const files = normalizePublishFiles(payload.files);
|
||||
if (payload.artifact?.kind !== "npm-pack") {
|
||||
const oversizedFile = findOversizedPublishFile(files);
|
||||
if (oversizedFile) {
|
||||
@@ -5459,14 +5487,6 @@ async function publishPackageImpl(
|
||||
return publishResult;
|
||||
}
|
||||
|
||||
export const publishPackage = action({
|
||||
args: { payload: v.any() },
|
||||
handler: async (ctx, args) => {
|
||||
const { userId } = await requireUserFromAction(ctx);
|
||||
return await publishPackageImpl(ctx, { kind: "user", actorUserId: userId }, args.payload);
|
||||
},
|
||||
});
|
||||
|
||||
export const publishPackageForUserInternal = internalAction({
|
||||
args: {
|
||||
actorUserId: v.id("users"),
|
||||
@@ -5509,14 +5529,6 @@ export const publishPackageForTrustedPublisherInternal = internalAction({
|
||||
},
|
||||
});
|
||||
|
||||
export const publishRelease = action({
|
||||
args: { payload: v.any() },
|
||||
handler: async (ctx, args) => {
|
||||
const { userId } = await requireUserFromAction(ctx);
|
||||
return await publishPackageImpl(ctx, { kind: "user", actorUserId: userId }, args.payload);
|
||||
},
|
||||
});
|
||||
|
||||
export const reservePackageNameInternal = internalMutation({
|
||||
args: {
|
||||
actorUserId: v.id("users"),
|
||||
|
||||
+121
-1
@@ -111,6 +111,53 @@ const llmRiskSummaryBucketValidator = v.object({
|
||||
highestSeverity: v.optional(v.string()),
|
||||
});
|
||||
|
||||
const llmAnalysisValidator = v.object({
|
||||
status: v.string(),
|
||||
verdict: v.optional(v.string()),
|
||||
confidence: v.optional(v.string()),
|
||||
summary: v.optional(v.string()),
|
||||
dimensions: v.optional(
|
||||
v.array(
|
||||
v.object({
|
||||
name: v.string(),
|
||||
label: v.string(),
|
||||
rating: v.string(),
|
||||
detail: v.string(),
|
||||
}),
|
||||
),
|
||||
),
|
||||
guidance: v.optional(v.string()),
|
||||
findings: v.optional(v.string()),
|
||||
agenticRiskFindings: v.optional(v.array(llmAgenticRiskFindingValidator)),
|
||||
riskSummary: v.optional(
|
||||
v.object({
|
||||
abnormal_behavior_control: llmRiskSummaryBucketValidator,
|
||||
permission_boundary: llmRiskSummaryBucketValidator,
|
||||
sensitive_data_protection: llmRiskSummaryBucketValidator,
|
||||
}),
|
||||
),
|
||||
model: v.optional(v.string()),
|
||||
checkedAt: v.number(),
|
||||
});
|
||||
|
||||
const staticScanValidator = v.object({
|
||||
status: v.union(v.literal("clean"), v.literal("suspicious"), v.literal("malicious")),
|
||||
reasonCodes: v.array(v.string()),
|
||||
findings: v.array(
|
||||
v.object({
|
||||
code: v.string(),
|
||||
severity: v.union(v.literal("info"), v.literal("warn"), v.literal("critical")),
|
||||
file: v.string(),
|
||||
line: v.number(),
|
||||
message: v.string(),
|
||||
evidence: v.string(),
|
||||
}),
|
||||
),
|
||||
summary: v.string(),
|
||||
engineVersion: v.string(),
|
||||
checkedAt: v.number(),
|
||||
});
|
||||
|
||||
const users = defineTable({
|
||||
name: v.optional(v.string()),
|
||||
image: v.optional(v.string()),
|
||||
@@ -361,6 +408,7 @@ const packageVerificationValidator = v.optional(
|
||||
sourceRepo: v.optional(v.string()),
|
||||
sourceCommit: v.optional(v.string()),
|
||||
sourceTag: v.optional(v.string()),
|
||||
sourcePath: v.optional(v.string()),
|
||||
hasProvenance: v.optional(v.boolean()),
|
||||
trustedOpenClawPlugin: v.optional(v.boolean()),
|
||||
scanStatus: v.optional(
|
||||
@@ -412,6 +460,7 @@ const packageReleaseModerationOverrideValidator = v.object({
|
||||
const securityScanTargetKindValidator = v.union(
|
||||
v.literal("skillVersion"),
|
||||
v.literal("packageRelease"),
|
||||
v.literal("skillScanRequest"),
|
||||
);
|
||||
const securityScanJobStatusValidator = v.union(
|
||||
v.literal("queued"),
|
||||
@@ -449,6 +498,8 @@ const packageFilesValidator = v.array(
|
||||
}),
|
||||
);
|
||||
|
||||
const skillScanRequestSourceKindValidator = v.union(v.literal("upload"), v.literal("published"));
|
||||
|
||||
const skills = defineTable({
|
||||
slug: v.string(),
|
||||
displayName: v.string(),
|
||||
@@ -913,6 +964,13 @@ const skillSearchDigest = defineTable({
|
||||
"statsInstallsAllTime",
|
||||
"updatedAt",
|
||||
])
|
||||
.index("by_active_recommended_rank", [
|
||||
"softDeletedAt",
|
||||
"statsStars",
|
||||
"statsInstallsAllTime",
|
||||
"statsDownloads",
|
||||
"updatedAt",
|
||||
])
|
||||
.index("by_nonsuspicious_updated", ["softDeletedAt", "isSuspicious", "updatedAt"])
|
||||
.index("by_nonsuspicious_created", ["softDeletedAt", "isSuspicious", "createdAt"])
|
||||
.index("by_nonsuspicious_name", ["softDeletedAt", "isSuspicious", "displayName"])
|
||||
@@ -945,6 +1003,14 @@ const skillSearchDigest = defineTable({
|
||||
"statsInstallsAllTime",
|
||||
"updatedAt",
|
||||
])
|
||||
.index("by_nonsuspicious_recommended_rank", [
|
||||
"softDeletedAt",
|
||||
"isSuspicious",
|
||||
"statsStars",
|
||||
"statsInstallsAllTime",
|
||||
"statsDownloads",
|
||||
"updatedAt",
|
||||
])
|
||||
.searchIndex("search_by_display_name", {
|
||||
searchField: "displayName",
|
||||
filterFields: ["softDeletedAt", "isSuspicious"],
|
||||
@@ -1109,6 +1175,7 @@ const securityScanJobs = defineTable({
|
||||
targetKind: securityScanTargetKindValidator,
|
||||
skillVersionId: v.optional(v.id("skillVersions")),
|
||||
packageReleaseId: v.optional(v.id("packageReleases")),
|
||||
skillScanRequestId: v.optional(v.id("skillScanRequests")),
|
||||
status: securityScanJobStatusValidator,
|
||||
source: securityScanJobSourceValidator,
|
||||
priority: v.number(),
|
||||
@@ -1132,7 +1199,48 @@ const securityScanJobs = defineTable({
|
||||
.index("by_status_and_lease_expires_at", ["status", "leaseExpiresAt"])
|
||||
.index("by_status_malicious_signal_next_run_at", ["status", "hasMaliciousSignal", "nextRunAt"])
|
||||
.index("by_skill_version", ["skillVersionId"])
|
||||
.index("by_package_release", ["packageReleaseId"]);
|
||||
.index("by_package_release", ["packageReleaseId"])
|
||||
.index("by_skill_scan_request", ["skillScanRequestId"]);
|
||||
|
||||
const skillScanRequests = defineTable({
|
||||
actorUserId: v.id("users"),
|
||||
sourceKind: skillScanRequestSourceKindValidator,
|
||||
update: v.boolean(),
|
||||
writtenBack: v.boolean(),
|
||||
status: securityScanJobStatusValidator,
|
||||
securityScanJobId: v.optional(v.id("securityScanJobs")),
|
||||
slug: v.optional(v.string()),
|
||||
displayName: v.optional(v.string()),
|
||||
version: v.optional(v.string()),
|
||||
skillId: v.optional(v.id("skills")),
|
||||
skillVersionId: v.optional(v.id("skillVersions")),
|
||||
files: packageFilesValidator,
|
||||
parsed: v.optional(
|
||||
v.object({
|
||||
frontmatter: v.record(v.string(), v.any()),
|
||||
metadata: v.optional(v.any()),
|
||||
clawdis: v.optional(v.any()),
|
||||
moltbot: v.optional(v.any()),
|
||||
license: v.optional(v.literal(PLATFORM_SKILL_LICENSE)),
|
||||
}),
|
||||
),
|
||||
sha256hash: v.optional(v.string()),
|
||||
vtAnalysis: v.optional(vtAnalysisValidator),
|
||||
skillSpectorAnalysis: v.optional(skillSpectorAnalysisValidator),
|
||||
llmAnalysis: v.optional(llmAnalysisValidator),
|
||||
capabilityTags: v.optional(v.array(v.string())),
|
||||
staticScan: v.optional(staticScanValidator),
|
||||
lastError: v.optional(v.string()),
|
||||
runId: v.optional(v.string()),
|
||||
completedAt: v.optional(v.number()),
|
||||
expiresAt: v.number(),
|
||||
createdAt: v.number(),
|
||||
updatedAt: v.number(),
|
||||
})
|
||||
.index("by_actor_user_id_and_created_at", ["actorUserId", "createdAt"])
|
||||
.index("by_security_scan_job_id", ["securityScanJobId"])
|
||||
.index("by_skill_version_id_and_created_at", ["skillVersionId", "createdAt"])
|
||||
.index("by_expires_at", ["expiresAt"]);
|
||||
|
||||
const skillCardGenerationJobs = defineTable({
|
||||
skillId: v.id("skills"),
|
||||
@@ -1209,6 +1317,16 @@ const packagePublishTokens = defineTable({
|
||||
.index("by_package", ["packageId", "version", "createdAt"])
|
||||
.index("by_package_revoked_created", ["packageId", "revokedAt", "createdAt"]);
|
||||
|
||||
const packagePublishUploadTickets = defineTable({
|
||||
kind: v.union(v.literal("user"), v.literal("github-actions")),
|
||||
userId: v.optional(v.id("users")),
|
||||
publishTokenId: v.optional(v.id("packagePublishTokens")),
|
||||
createdAt: v.number(),
|
||||
expiresAt: v.number(),
|
||||
usedAt: v.optional(v.number()),
|
||||
storageId: v.optional(v.id("_storage")),
|
||||
});
|
||||
|
||||
const packageSearchDigest = defineTable({
|
||||
packageId: v.id("packages"),
|
||||
name: v.string(),
|
||||
@@ -2125,10 +2243,12 @@ export default defineSchema({
|
||||
packages,
|
||||
packageReleases,
|
||||
securityScanJobs,
|
||||
skillScanRequests,
|
||||
skillCardGenerationJobs,
|
||||
packageStatEvents,
|
||||
packageTrustedPublishers,
|
||||
packagePublishTokens,
|
||||
packagePublishUploadTickets,
|
||||
packageBadges,
|
||||
packageSearchDigest,
|
||||
packageCapabilitySearchDigest,
|
||||
|
||||
+104
-13
@@ -544,6 +544,8 @@ describe("search helpers", () => {
|
||||
slug: "antigravity-image-generator",
|
||||
displayName: "Antigravity Image Generator",
|
||||
downloads: 1_000_000_000,
|
||||
installsAllTime: 1_000,
|
||||
stars: 100,
|
||||
}),
|
||||
version: null,
|
||||
ownerHandle: "owner",
|
||||
@@ -563,7 +565,7 @@ describe("search helpers", () => {
|
||||
vectorSearch: vi.fn().mockResolvedValue(
|
||||
vectorEntries.map((entry, index) => ({
|
||||
_id: entry.embeddingId,
|
||||
_score: 0.5 - index * 0.001,
|
||||
_score: 0.05 - index * 0.001,
|
||||
})),
|
||||
),
|
||||
runQuery,
|
||||
@@ -1142,8 +1144,16 @@ describe("search helpers", () => {
|
||||
|
||||
it("boosts exact slug/name matches over loose matches", () => {
|
||||
const queryTokens = tokenize("notion");
|
||||
const exactScore = __test.scoreSkillResult(queryTokens, 0.4, "Notion Sync", "notion-sync", 5);
|
||||
const looseScore = __test.scoreSkillResult(queryTokens, 0.6, "Notes Sync", "notes-sync", 500);
|
||||
const exactScore = __test.scoreSkillResult(queryTokens, 0.4, "Notion Sync", "notion-sync", {
|
||||
downloads: 5,
|
||||
installsAllTime: 0,
|
||||
stars: 0,
|
||||
});
|
||||
const looseScore = __test.scoreSkillResult(queryTokens, 0.6, "Notes Sync", "notes-sync", {
|
||||
downloads: 500,
|
||||
installsAllTime: 100,
|
||||
stars: 20,
|
||||
});
|
||||
expect(exactScore).toBeGreaterThan(looseScore);
|
||||
});
|
||||
|
||||
@@ -1154,35 +1164,114 @@ describe("search helpers", () => {
|
||||
0.5,
|
||||
"Self Improving Agent",
|
||||
"self-improving-agent",
|
||||
10,
|
||||
{ downloads: 10, installsAllTime: 0, stars: 0 },
|
||||
);
|
||||
const containingScore = __test.scoreSkillResult(
|
||||
queryTokens,
|
||||
0.6,
|
||||
"Self Improving Agent",
|
||||
"xiucheng-self-improving-agent",
|
||||
100,
|
||||
{ downloads: 100, installsAllTime: 50, stars: 10 },
|
||||
);
|
||||
expect(exactScore).toBeGreaterThan(containingScore);
|
||||
});
|
||||
|
||||
it("adds a popularity prior for equally relevant matches", () => {
|
||||
it("keeps extreme popularity below direct lexical relevance", () => {
|
||||
const queryTokens = tokenize("needle");
|
||||
const exactScore = __test.scoreSkillResult(queryTokens, 0, "Unrelated Name", "needle", {
|
||||
downloads: 0,
|
||||
installsAllTime: 0,
|
||||
stars: 0,
|
||||
});
|
||||
const popularLooseScore = __test.scoreSkillResult(
|
||||
queryTokens,
|
||||
0.9,
|
||||
"Different Tool",
|
||||
"different-tool",
|
||||
{ downloads: 1_000_000, installsAllTime: 25_000, stars: 25_000 },
|
||||
);
|
||||
expect(exactScore).toBeGreaterThan(popularLooseScore);
|
||||
});
|
||||
|
||||
it("keeps popularity from flipping a strong name match", () => {
|
||||
const queryTokens = tokenize("notion");
|
||||
const lowDownloads = __test.scoreSkillResult(
|
||||
const nameMatchScore = __test.scoreSkillResult(queryTokens, 0, "Notion Helper", "helper", {
|
||||
downloads: 0,
|
||||
installsAllTime: 0,
|
||||
stars: 0,
|
||||
});
|
||||
const popularVectorScore = __test.scoreSkillResult(
|
||||
queryTokens,
|
||||
1,
|
||||
"Different Tool",
|
||||
"different-tool",
|
||||
{ downloads: 1_000_000, installsAllTime: 25_000, stars: 25_000 },
|
||||
);
|
||||
expect(nameMatchScore).toBeGreaterThan(popularVectorScore);
|
||||
});
|
||||
|
||||
it("adds a stars and installs popularity prior for equally relevant matches", () => {
|
||||
const queryTokens = tokenize("notion");
|
||||
const highDownloadsOnly = __test.scoreSkillResult(
|
||||
queryTokens,
|
||||
0.5,
|
||||
"Notion Helper",
|
||||
"notion-helper",
|
||||
0,
|
||||
{ downloads: 1000, installsAllTime: 0, stars: 0 },
|
||||
);
|
||||
const highDownloads = __test.scoreSkillResult(
|
||||
const trustedUsage = __test.scoreSkillResult(
|
||||
queryTokens,
|
||||
0.5,
|
||||
"Notion Helper",
|
||||
"notion-helper",
|
||||
1000,
|
||||
{ downloads: 0, installsAllTime: 20, stars: 5 },
|
||||
);
|
||||
expect(highDownloads).toBeGreaterThan(lowDownloads);
|
||||
expect(trustedUsage).toBeGreaterThan(highDownloadsOnly);
|
||||
});
|
||||
|
||||
it("breaks capped popularity ties by stars and installs before downloads", async () => {
|
||||
generateEmbeddingMock.mockResolvedValueOnce([0, 1, 2]);
|
||||
const trustedUsage = {
|
||||
skill: makePublicSkill({
|
||||
id: "skills:trusted",
|
||||
slug: "tool-trusted",
|
||||
displayName: "Tool",
|
||||
downloads: 0,
|
||||
installsAllTime: 1_000,
|
||||
stars: 1_000,
|
||||
}),
|
||||
version: null,
|
||||
ownerHandle: "owner",
|
||||
owner: null,
|
||||
};
|
||||
const downloadedOnly = {
|
||||
skill: makePublicSkill({
|
||||
id: "skills:downloaded",
|
||||
slug: "tool-downloaded",
|
||||
displayName: "Tool",
|
||||
downloads: 1_000_000_000,
|
||||
installsAllTime: 0,
|
||||
stars: 0,
|
||||
}),
|
||||
version: null,
|
||||
ownerHandle: "owner",
|
||||
owner: null,
|
||||
};
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(null) // getExactSkillSlugMatch
|
||||
.mockResolvedValueOnce([]) // directPrefixSkillMatches
|
||||
.mockResolvedValueOnce([downloadedOnly, trustedUsage]); // lexicalFallbackSkills
|
||||
|
||||
const result = await searchSkillsHandler(
|
||||
{
|
||||
vectorSearch: vi.fn().mockResolvedValue([]),
|
||||
runQuery,
|
||||
},
|
||||
{ query: "tool", limit: 2 },
|
||||
);
|
||||
|
||||
expect(result.map((entry) => entry.skill.slug)).toEqual(["tool-trusted", "tool-downloaded"]);
|
||||
});
|
||||
|
||||
it("uses digest doc instead of full skill doc in hydrateResults but revalidates the owner", async () => {
|
||||
@@ -1532,6 +1621,8 @@ function makePublicSkill(params: {
|
||||
slug: string;
|
||||
displayName: string;
|
||||
downloads?: number;
|
||||
installsAllTime?: number;
|
||||
stars?: number;
|
||||
capabilityTags?: string[];
|
||||
}) {
|
||||
return {
|
||||
@@ -1550,8 +1641,8 @@ function makePublicSkill(params: {
|
||||
stats: {
|
||||
downloads: params.downloads ?? 0,
|
||||
installsCurrent: 0,
|
||||
installsAllTime: 0,
|
||||
stars: 0,
|
||||
installsAllTime: params.installsAllTime ?? 0,
|
||||
stars: params.stars ?? 0,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
|
||||
+45
-7
@@ -75,7 +75,10 @@ const SLUG_TOKEN_BOOST = 1.4;
|
||||
const SLUG_PREFIX_BOOST = 0.8;
|
||||
const NAME_EXACT_BOOST = 1.1;
|
||||
const NAME_PREFIX_BOOST = 0.6;
|
||||
const POPULARITY_WEIGHT = 0.08;
|
||||
const STAR_POPULARITY_WEIGHT = 0.12;
|
||||
const INSTALL_POPULARITY_WEIGHT = 0.04;
|
||||
const DOWNLOAD_POPULARITY_WEIGHT = 0.005;
|
||||
const MAX_POPULARITY_BOOST = 0.09;
|
||||
const FALLBACK_SCAN_LIMIT = 2000;
|
||||
const MIN_FALLBACK_SCAN_LIMIT = 100;
|
||||
const FALLBACK_RECALL_MULTIPLIER = 2;
|
||||
@@ -119,15 +122,29 @@ function getLexicalBoost(queryTokens: string[], displayName: string, slug: strin
|
||||
return boost;
|
||||
}
|
||||
|
||||
type PopularityStats = {
|
||||
downloads: number;
|
||||
installsAllTime?: number;
|
||||
stars: number;
|
||||
};
|
||||
|
||||
function getPopularityBoost(stats: PopularityStats) {
|
||||
const rawBoost =
|
||||
Math.log1p(Math.max(stats.stars, 0)) * STAR_POPULARITY_WEIGHT +
|
||||
Math.log1p(Math.max(stats.installsAllTime ?? 0, 0)) * INSTALL_POPULARITY_WEIGHT +
|
||||
Math.log1p(Math.max(stats.downloads, 0)) * DOWNLOAD_POPULARITY_WEIGHT;
|
||||
return Math.min(rawBoost, MAX_POPULARITY_BOOST);
|
||||
}
|
||||
|
||||
function scoreSkillResult(
|
||||
queryTokens: string[],
|
||||
vectorScore: number,
|
||||
displayName: string,
|
||||
slug: string,
|
||||
downloads: number,
|
||||
stats: PopularityStats,
|
||||
) {
|
||||
const lexicalBoost = getLexicalBoost(queryTokens, displayName, slug);
|
||||
const popularityBoost = Math.log1p(Math.max(downloads, 0)) * POPULARITY_WEIGHT;
|
||||
const popularityBoost = getPopularityBoost(stats);
|
||||
return vectorScore + lexicalBoost + popularityBoost;
|
||||
}
|
||||
|
||||
@@ -179,6 +196,14 @@ function classifySkillMatch(
|
||||
return null;
|
||||
}
|
||||
|
||||
function comparePopularityStats(a: PopularityStats, b: PopularityStats) {
|
||||
return (
|
||||
b.stars - a.stars ||
|
||||
(b.installsAllTime ?? 0) - (a.installsAllTime ?? 0) ||
|
||||
b.downloads - a.downloads
|
||||
);
|
||||
}
|
||||
|
||||
function mergeUniqueBySkillId(primary: SkillSearchEntry[], fallback: SkillSearchEntry[]) {
|
||||
if (fallback.length === 0) return primary;
|
||||
const out = [...primary];
|
||||
@@ -353,7 +378,11 @@ export const searchSkills: ReturnType<typeof action> = action({
|
||||
vectorScore,
|
||||
entry.skill.displayName,
|
||||
entry.skill.slug,
|
||||
entry.skill.stats.downloads,
|
||||
{
|
||||
downloads: entry.skill.stats.downloads,
|
||||
installsAllTime: entry.skill.stats.installsAllTime,
|
||||
stars: entry.skill.stats.stars,
|
||||
},
|
||||
),
|
||||
};
|
||||
})
|
||||
@@ -362,7 +391,8 @@ export const searchSkills: ReturnType<typeof action> = action({
|
||||
(a, b) =>
|
||||
a.rankTier - b.rankTier ||
|
||||
b.score - a.score ||
|
||||
b.skill.stats.downloads - a.skill.stats.downloads,
|
||||
comparePopularityStats(a.skill.stats, b.skill.stats) ||
|
||||
b.skill.updatedAt - a.skill.updatedAt,
|
||||
)
|
||||
.slice(0, limit);
|
||||
return rankedMatches.map(({ rankTier: _rankTier, ...entry }) => entry);
|
||||
@@ -879,12 +909,20 @@ export const searchSouls: ReturnType<typeof action> = action({
|
||||
vectorScore,
|
||||
entry.soul.displayName,
|
||||
entry.soul.slug,
|
||||
entry.soul.stats.downloads,
|
||||
{
|
||||
downloads: entry.soul.stats.downloads,
|
||||
stars: entry.soul.stats.stars,
|
||||
},
|
||||
),
|
||||
};
|
||||
})
|
||||
.filter((entry) => entry.soul)
|
||||
.sort((a, b) => b.score - a.score || b.soul.stats.downloads - a.soul.stats.downloads)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
b.score - a.score ||
|
||||
comparePopularityStats(a.soul.stats, b.soul.stats) ||
|
||||
b.soul.updatedAt - a.soul.updatedAt,
|
||||
)
|
||||
.slice(0, limit);
|
||||
},
|
||||
});
|
||||
|
||||
+90
-13
@@ -8,7 +8,10 @@ import { getOwnerPublisher } from "./lib/publishers";
|
||||
|
||||
const MAX_EXPORT_PAGE_SIZE = 50;
|
||||
const MAX_EXPORT_BATCH_PAGES = 20;
|
||||
const REDACTION_POLICY_VERSION = "public-signals-v1";
|
||||
const MAX_REDACTED_BUNDLE_FILE_BYTES = 192 * 1024;
|
||||
const MAX_REDACTED_BUNDLE_BYTES_PER_ARTIFACT = 256 * 1024;
|
||||
const MAX_REDACTED_BUNDLE_BYTES_PER_RESPONSE = 256 * 1024;
|
||||
const REDACTION_POLICY_VERSION = "public-signals-v2-bundle-files";
|
||||
const SOURCE_TABLES = ["skillVersions", "packageReleases"] as const;
|
||||
const SCANNER_SOURCES = [
|
||||
"static",
|
||||
@@ -297,17 +300,80 @@ function sanitizeFiles(files: Array<Doc<"skillVersions">["files"][number]>) {
|
||||
}
|
||||
|
||||
async function enrichAndSanitizeArtifactRows(ctx: ActionCtx, rows: ArtifactExportRow[]) {
|
||||
return await Promise.all(
|
||||
rows.map(async (row) => {
|
||||
const skillContent =
|
||||
row.sourceKind === "skill" ? await readRedactedSkillMdContent(ctx, row.files) : null;
|
||||
return {
|
||||
...row,
|
||||
...(skillContent ? { skillMdContentRedacted: skillContent } : {}),
|
||||
files: row.files.map(({ storageId: _storageId, ...file }) => file),
|
||||
};
|
||||
}),
|
||||
const enrichedRows = [];
|
||||
let remainingBundleBytes = MAX_REDACTED_BUNDLE_BYTES_PER_RESPONSE;
|
||||
for (const row of rows) {
|
||||
const skillContent =
|
||||
row.sourceKind === "skill" ? await readRedactedSkillMdContent(ctx, row.files) : null;
|
||||
const bundleFiles =
|
||||
row.sourceKind === "skill"
|
||||
? await readRedactedBundleFiles(ctx, row.files, remainingBundleBytes)
|
||||
: [];
|
||||
remainingBundleBytes -= totalBundleBytes(bundleFiles);
|
||||
enrichedRows.push({
|
||||
...row,
|
||||
...(skillContent ? { skillMdContentRedacted: skillContent } : {}),
|
||||
...(bundleFiles.length > 0 ? { bundleFilesRedacted: bundleFiles } : {}),
|
||||
files: row.files.map(({ storageId: _storageId, ...file }) => file),
|
||||
});
|
||||
}
|
||||
return enrichedRows;
|
||||
}
|
||||
|
||||
async function readRedactedBundleFiles(
|
||||
ctx: Pick<ActionCtx, "storage">,
|
||||
files: Array<{ path: string; size?: number; storageId?: unknown }>,
|
||||
remainingResponseBytes: number,
|
||||
) {
|
||||
const bundleFiles: Array<{ path: string; content: string }> = [];
|
||||
let remainingArtifactBytes = Math.min(
|
||||
remainingResponseBytes,
|
||||
MAX_REDACTED_BUNDLE_BYTES_PER_ARTIFACT,
|
||||
);
|
||||
for (const file of files) {
|
||||
if (isExcludedSkillBundlePath(file.path) || typeof file.storageId !== "string") continue;
|
||||
if (typeof file.size === "number" && file.size > MAX_REDACTED_BUNDLE_FILE_BYTES) continue;
|
||||
if (remainingArtifactBytes <= 0) break;
|
||||
const blob = await ctx.storage.get(file.storageId as never);
|
||||
if (!blob) continue;
|
||||
const content = redactBundleContent(await blob.text());
|
||||
const contentBytes = utf8Bytes(content);
|
||||
if (contentBytes > MAX_REDACTED_BUNDLE_FILE_BYTES || contentBytes > remainingArtifactBytes) {
|
||||
continue;
|
||||
}
|
||||
bundleFiles.push({ path: file.path, content });
|
||||
remainingArtifactBytes -= contentBytes;
|
||||
}
|
||||
return bundleFiles;
|
||||
}
|
||||
|
||||
function isExcludedSkillBundlePath(path: string) {
|
||||
return (
|
||||
isPrimarySkillReadmePath(path) || normalizeBundlePathForComparison(path) === "skill-card.md"
|
||||
);
|
||||
}
|
||||
|
||||
function isPrimarySkillReadmePath(path: string) {
|
||||
const normalized = normalizeBundlePathForComparison(path);
|
||||
return normalized === "skill.md" || normalized === "skills.md";
|
||||
}
|
||||
|
||||
function normalizeBundlePathForComparison(path: string) {
|
||||
return path
|
||||
.trim()
|
||||
.replace(/^\/+/, "")
|
||||
.split("/")
|
||||
.filter((segment) => segment && segment !== ".")
|
||||
.join("/")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function totalBundleBytes(files: Array<{ content: string }>) {
|
||||
return files.reduce((sum, file) => sum + utf8Bytes(file.content), 0);
|
||||
}
|
||||
|
||||
function utf8Bytes(value: string) {
|
||||
return new TextEncoder().encode(value).byteLength;
|
||||
}
|
||||
|
||||
async function readRedactedSkillMdContent(
|
||||
@@ -315,8 +381,7 @@ async function readRedactedSkillMdContent(
|
||||
files: Array<{ path: string; storageId?: unknown }>,
|
||||
) {
|
||||
const skillFile = files.find((file) => {
|
||||
const path = file.path.toLowerCase();
|
||||
return path === "skill.md" || path.endsWith("/skill.md");
|
||||
return isPrimarySkillReadmePath(file.path);
|
||||
});
|
||||
if (!skillFile || typeof skillFile.storageId !== "string") return null;
|
||||
const blob = await ctx.storage.get(skillFile.storageId as never);
|
||||
@@ -336,6 +401,18 @@ function redactSkillContent(value: string) {
|
||||
return redacted.trim();
|
||||
}
|
||||
|
||||
function redactBundleContent(value: string) {
|
||||
let redacted = "";
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const code = value.charCodeAt(index);
|
||||
redacted += code < 32 && code !== 9 && code !== 10 && code !== 13 ? " " : value.charAt(index);
|
||||
}
|
||||
for (const pattern of SECRET_PATTERNS) {
|
||||
redacted = redacted.replace(pattern, "[REDACTED_SECRET]");
|
||||
}
|
||||
return redacted;
|
||||
}
|
||||
|
||||
function normalizeVtAnalysis(analysis: StoredVtAnalysis) {
|
||||
if (!analysis) return null;
|
||||
return {
|
||||
|
||||
+105
-18
@@ -8,6 +8,9 @@ import type { ActionCtx } from "./_generated/server";
|
||||
import { internalAction } from "./functions";
|
||||
|
||||
const MAX_EXPORT_BATCH_PAGES = 20;
|
||||
const MAX_REDACTED_BUNDLE_FILE_BYTES = 192 * 1024;
|
||||
const MAX_REDACTED_BUNDLE_BYTES_PER_ARTIFACT = 256 * 1024;
|
||||
const MAX_REDACTED_BUNDLE_BYTES_PER_RESPONSE = 256 * 1024;
|
||||
|
||||
type ArtifactExportPage = {
|
||||
page: unknown[];
|
||||
@@ -75,30 +78,102 @@ export const listArtifactExportBatchCompressedInternal = internalAction({
|
||||
});
|
||||
|
||||
async function enrichAndSanitizeArtifactRows(ctx: ActionCtx, rows: unknown[]) {
|
||||
return await Promise.all(
|
||||
rows.map(async (row) => {
|
||||
if (!isRecord(row)) return row;
|
||||
const files = Array.isArray(row.files) ? row.files : [];
|
||||
const skillContent =
|
||||
row.sourceKind === "skill" ? await readRedactedSkillMdContent(ctx, files) : null;
|
||||
return {
|
||||
...row,
|
||||
...(skillContent ? { skillMdContentRedacted: skillContent } : {}),
|
||||
files: files.map((file) => {
|
||||
if (!isRecord(file)) return file;
|
||||
const { storageId: _storageId, ...rest } = file;
|
||||
return rest;
|
||||
}),
|
||||
};
|
||||
}),
|
||||
const enrichedRows = [];
|
||||
let remainingBundleBytes = MAX_REDACTED_BUNDLE_BYTES_PER_RESPONSE;
|
||||
for (const row of rows) {
|
||||
if (!isRecord(row)) {
|
||||
enrichedRows.push(row);
|
||||
continue;
|
||||
}
|
||||
const files = Array.isArray(row.files) ? row.files : [];
|
||||
const skillContent =
|
||||
row.sourceKind === "skill" ? await readRedactedSkillMdContent(ctx, files) : null;
|
||||
const bundleFiles =
|
||||
row.sourceKind === "skill"
|
||||
? await readRedactedBundleFiles(ctx, files, remainingBundleBytes)
|
||||
: [];
|
||||
remainingBundleBytes -= totalBundleBytes(bundleFiles);
|
||||
enrichedRows.push({
|
||||
...row,
|
||||
...(skillContent ? { skillMdContentRedacted: skillContent } : {}),
|
||||
...(bundleFiles.length > 0 ? { bundleFilesRedacted: bundleFiles } : {}),
|
||||
files: files.map((file) => {
|
||||
if (!isRecord(file)) return file;
|
||||
const { storageId: _storageId, ...rest } = file;
|
||||
return rest;
|
||||
}),
|
||||
});
|
||||
}
|
||||
return enrichedRows;
|
||||
}
|
||||
|
||||
async function readRedactedBundleFiles(
|
||||
ctx: Pick<ActionCtx, "storage">,
|
||||
files: unknown[],
|
||||
remainingResponseBytes: number,
|
||||
) {
|
||||
const bundleFiles: Array<{ path: string; content: string }> = [];
|
||||
let remainingArtifactBytes = Math.min(
|
||||
remainingResponseBytes,
|
||||
MAX_REDACTED_BUNDLE_BYTES_PER_ARTIFACT,
|
||||
);
|
||||
for (const file of files) {
|
||||
if (
|
||||
!isRecord(file) ||
|
||||
typeof file.path !== "string" ||
|
||||
typeof file.storageId !== "string" ||
|
||||
isExcludedSkillBundlePath(file.path)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (typeof file.size === "number" && file.size > MAX_REDACTED_BUNDLE_FILE_BYTES) continue;
|
||||
if (remainingArtifactBytes <= 0) break;
|
||||
const blob = await ctx.storage.get(file.storageId as never);
|
||||
if (!blob) continue;
|
||||
const content = redactBundleContent(await blob.text());
|
||||
const contentBytes = utf8Bytes(content);
|
||||
if (contentBytes > MAX_REDACTED_BUNDLE_FILE_BYTES || contentBytes > remainingArtifactBytes) {
|
||||
continue;
|
||||
}
|
||||
bundleFiles.push({ path: file.path, content });
|
||||
remainingArtifactBytes -= contentBytes;
|
||||
}
|
||||
return bundleFiles;
|
||||
}
|
||||
|
||||
function isExcludedSkillBundlePath(path: string) {
|
||||
return (
|
||||
isPrimarySkillReadmePath(path) || normalizeBundlePathForComparison(path) === "skill-card.md"
|
||||
);
|
||||
}
|
||||
|
||||
function isPrimarySkillReadmePath(path: string) {
|
||||
const normalized = normalizeBundlePathForComparison(path);
|
||||
return normalized === "skill.md" || normalized === "skills.md";
|
||||
}
|
||||
|
||||
function normalizeBundlePathForComparison(path: string) {
|
||||
return path
|
||||
.trim()
|
||||
.replace(/^\/+/, "")
|
||||
.split("/")
|
||||
.filter((segment) => segment && segment !== ".")
|
||||
.join("/")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function totalBundleBytes(files: Array<{ content: string }>) {
|
||||
return files.reduce((sum, file) => sum + utf8Bytes(file.content), 0);
|
||||
}
|
||||
|
||||
function utf8Bytes(value: string) {
|
||||
return new TextEncoder().encode(value).byteLength;
|
||||
}
|
||||
|
||||
async function readRedactedSkillMdContent(ctx: Pick<ActionCtx, "storage">, files: unknown[]) {
|
||||
const skillFile = files.find((file) => {
|
||||
if (!isRecord(file) || typeof file.path !== "string") return false;
|
||||
const path = file.path.toLowerCase();
|
||||
return path === "skill.md" || path.endsWith("/skill.md");
|
||||
return isPrimarySkillReadmePath(file.path);
|
||||
});
|
||||
if (!isRecord(skillFile) || typeof skillFile.storageId !== "string") return null;
|
||||
const blob = await ctx.storage.get(skillFile.storageId as never);
|
||||
@@ -118,6 +193,18 @@ function redactSkillContent(value: string) {
|
||||
return redacted.trim();
|
||||
}
|
||||
|
||||
function redactBundleContent(value: string) {
|
||||
let redacted = "";
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const code = value.charCodeAt(index);
|
||||
redacted += code < 32 && code !== 9 && code !== 10 && code !== 13 ? " " : value.charAt(index);
|
||||
}
|
||||
for (const pattern of SECRET_PATTERNS) {
|
||||
redacted = redacted.replace(pattern, "[REDACTED_SECRET]");
|
||||
}
|
||||
return redacted;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
enqueueBulkSkillRescanBatchForAdminInternal,
|
||||
failCodexScanJob,
|
||||
getBulkSkillRescanBatchStatusForAdminInternal,
|
||||
pruneExpiredSkillScanRequestsInternal,
|
||||
requestPackageRescanForUserInternal,
|
||||
requestPackageRescan,
|
||||
requestSkillRescanForUserInternal,
|
||||
@@ -123,6 +124,12 @@ const clearQueuedBackfillJobsForLocalDevHandler = (
|
||||
{ dryRun: boolean; matched: number; deleted: number; sampleDeletedJobIds: string[] }
|
||||
>
|
||||
)._handler;
|
||||
const pruneExpiredSkillScanRequestsInternalHandler = (
|
||||
pruneExpiredSkillScanRequestsInternal as unknown as WrappedHandler<
|
||||
{ batchSize?: number },
|
||||
{ ok: true; deletedRequests: number; deletedJobs: number; deletedFiles: number; done: boolean }
|
||||
>
|
||||
)._handler;
|
||||
|
||||
const requestSkillRescanHandler = (
|
||||
requestSkillRescan as unknown as WrappedHandler<
|
||||
@@ -1206,6 +1213,81 @@ describe("securityScan", () => {
|
||||
expect(deleted).toEqual(["securityScanJobs:backfill-1", "securityScanJobs:backfill-2"]);
|
||||
});
|
||||
|
||||
it("prunes expired uploaded scan request blobs without deleting published version files", async () => {
|
||||
const requests = [
|
||||
{
|
||||
_id: "skillScanRequests:upload",
|
||||
sourceKind: "upload",
|
||||
securityScanJobId: "securityScanJobs:upload",
|
||||
files: [{ storageId: "storage:upload-1" }, { storageId: "storage:upload-2" }],
|
||||
},
|
||||
{
|
||||
_id: "skillScanRequests:published",
|
||||
sourceKind: "published",
|
||||
securityScanJobId: "securityScanJobs:published",
|
||||
files: [{ storageId: "storage:published-version-file" }],
|
||||
},
|
||||
];
|
||||
const deletedDocs: string[] = [];
|
||||
const deletedStorage: string[] = [];
|
||||
const take = vi.fn(async () => requests);
|
||||
const indexBuilder = {
|
||||
lt: vi.fn(() => indexBuilder),
|
||||
};
|
||||
const withIndex = vi.fn(
|
||||
(indexName: string, buildRange: (q: typeof indexBuilder) => unknown) => {
|
||||
expect(indexName).toBe("by_expires_at");
|
||||
buildRange(indexBuilder);
|
||||
expect(indexBuilder.lt).toHaveBeenCalledWith("expiresAt", expect.any(Number));
|
||||
return { take };
|
||||
},
|
||||
);
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn((tableName: string) => {
|
||||
expect(tableName).toBe("skillScanRequests");
|
||||
return { withIndex };
|
||||
}),
|
||||
insert: vi.fn(async () => "noop"),
|
||||
patch: vi.fn(async () => undefined),
|
||||
replace: vi.fn(async () => undefined),
|
||||
get: vi.fn(async (id: string) => ({
|
||||
_id: id,
|
||||
targetKind: "skillScanRequest",
|
||||
})),
|
||||
delete: vi.fn(async (id: string) => {
|
||||
deletedDocs.push(id);
|
||||
}),
|
||||
normalizeId: vi.fn(() => null),
|
||||
system: {},
|
||||
},
|
||||
storage: {
|
||||
delete: vi.fn(async (id: string) => {
|
||||
deletedStorage.push(id);
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
const result = await pruneExpiredSkillScanRequestsInternalHandler(ctx as never, {
|
||||
batchSize: 10,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
ok: true,
|
||||
deletedRequests: 2,
|
||||
deletedJobs: 2,
|
||||
deletedFiles: 2,
|
||||
done: true,
|
||||
});
|
||||
expect(deletedStorage).toEqual(["storage:upload-1", "storage:upload-2"]);
|
||||
expect(deletedDocs).toEqual([
|
||||
"securityScanJobs:upload",
|
||||
"skillScanRequests:upload",
|
||||
"securityScanJobs:published",
|
||||
"skillScanRequests:published",
|
||||
]);
|
||||
});
|
||||
|
||||
it("fails claimed package jobs when the ClawPack URL is unavailable", async () => {
|
||||
vi.stubEnv("SECURITY_SCAN_WORKER_TOKEN", "worker-secret");
|
||||
|
||||
|
||||
+439
-1
@@ -18,6 +18,8 @@ const DEFAULT_CANCEL_SCAN_LIMIT = 1000;
|
||||
const DEFAULT_CANCEL_DELETE_LIMIT = 500;
|
||||
const MAX_CANCEL_SCAN_LIMIT = 5000;
|
||||
const CANCEL_SAMPLE_LIMIT = 20;
|
||||
const DEFAULT_PRUNE_SKILL_SCAN_REQUEST_LIMIT = 250;
|
||||
const MAX_PRUNE_SKILL_SCAN_REQUEST_LIMIT = 1000;
|
||||
const DEFAULT_BULK_RESCAN_BATCH_SIZE = 50;
|
||||
const MAX_BULK_RESCAN_BATCH_SIZE = 100;
|
||||
const MAX_BULK_RESCAN_STATUS_JOB_IDS = 200;
|
||||
@@ -25,6 +27,7 @@ const BULK_RESCAN_SAMPLE_LIMIT = 10;
|
||||
const MAX_STORED_SKILLSPECTOR_ISSUES = 25;
|
||||
const MAX_STORED_SKILLSPECTOR_TEXT_CHARS = 2_000;
|
||||
const MAX_STORED_SKILLSPECTOR_SHORT_TEXT_CHARS = 512;
|
||||
const DEFAULT_SKILL_SCAN_REQUEST_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
|
||||
const finalLlmAnalysisStatuses = new Set(["clean", "suspicious", "malicious"]);
|
||||
const artifactBackedLlmAnalysisStatuses = new Set(["clean", "benign", "suspicious", "malicious"]);
|
||||
@@ -42,8 +45,10 @@ type CancelSkipReason =
|
||||
|
||||
type JobTarget = {
|
||||
job: Doc<"securityScanJobs">;
|
||||
skill?: Doc<"skills"> | null;
|
||||
version?: Doc<"skillVersions">;
|
||||
release?: Doc<"packageReleases">;
|
||||
scanRequest?: Doc<"skillScanRequests">;
|
||||
missing?: true;
|
||||
};
|
||||
|
||||
@@ -206,6 +211,14 @@ const skillSpectorAnalysisValidator = v.object({
|
||||
checkedAt: v.number(),
|
||||
});
|
||||
|
||||
const scanRequestFileValidator = v.object({
|
||||
path: v.string(),
|
||||
size: v.number(),
|
||||
storageId: v.id("_storage"),
|
||||
sha256: v.string(),
|
||||
contentType: v.optional(v.string()),
|
||||
});
|
||||
|
||||
const internalRefs = internal as unknown as {
|
||||
packages: {
|
||||
getPackageByIdInternal: unknown;
|
||||
@@ -215,10 +228,15 @@ const internalRefs = internal as unknown as {
|
||||
};
|
||||
securityScan: {
|
||||
claimQueuedJobsInternal: unknown;
|
||||
createUploadedSkillScanRequestInternal: unknown;
|
||||
createPublishedSkillScanRequestInternal: unknown;
|
||||
enqueuePackageReleaseScanInternal: unknown;
|
||||
enqueueSkillVersionScanInternal: unknown;
|
||||
failJobInternal: unknown;
|
||||
getSkillScanRequestForUserInternal: unknown;
|
||||
getJobTargetInternal: unknown;
|
||||
recordSkillScanRequestFailedInternal: unknown;
|
||||
recordSkillScanRequestSucceededInternal: unknown;
|
||||
succeedJobInternal: unknown;
|
||||
};
|
||||
skills: {
|
||||
@@ -754,6 +772,362 @@ export const requestSkillRescan = mutation({
|
||||
},
|
||||
});
|
||||
|
||||
function skillScanRequestExpiresAt(now: number) {
|
||||
return now + DEFAULT_SKILL_SCAN_REQUEST_RETENTION_MS;
|
||||
}
|
||||
|
||||
function skillScanReportFromRequest(request: Doc<"skillScanRequests">) {
|
||||
return {
|
||||
clawscan: request.llmAnalysis ?? null,
|
||||
skillspector: request.skillSpectorAnalysis ?? null,
|
||||
staticAnalysis: request.staticScan ?? null,
|
||||
virustotal: request.vtAnalysis
|
||||
? {
|
||||
...request.vtAnalysis,
|
||||
...request.vtAnalysis.engineStats,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function skillScanArtifactFromRequest(request: Doc<"skillScanRequests">) {
|
||||
return {
|
||||
...(request.slug ? { slug: request.slug } : {}),
|
||||
...(request.displayName ? { displayName: request.displayName } : {}),
|
||||
...(request.version ? { version: request.version } : {}),
|
||||
...(request.sha256hash ? { sha256hash: request.sha256hash } : {}),
|
||||
fileCount: request.files.length,
|
||||
};
|
||||
}
|
||||
|
||||
function skillScanStatusResponse(
|
||||
request: Doc<"skillScanRequests">,
|
||||
job: Doc<"securityScanJobs"> | null,
|
||||
) {
|
||||
const status =
|
||||
request.status === "succeeded" || request.status === "failed"
|
||||
? request.status
|
||||
: (job?.status ?? request.status);
|
||||
return {
|
||||
ok: true as const,
|
||||
scanId: request._id,
|
||||
jobId: request.securityScanJobId,
|
||||
status,
|
||||
sourceKind: request.sourceKind,
|
||||
update: request.update,
|
||||
writtenBack: request.writtenBack,
|
||||
artifact: skillScanArtifactFromRequest(request),
|
||||
report: skillScanReportFromRequest(request),
|
||||
lastError: request.lastError ?? job?.lastError,
|
||||
createdAt: request.createdAt,
|
||||
updatedAt: Math.max(request.updatedAt, job?.updatedAt ?? request.updatedAt),
|
||||
completedAt: request.completedAt ?? job?.completedAt,
|
||||
};
|
||||
}
|
||||
|
||||
async function enqueueSkillScanRequestJob(ctx: MutationCtx, requestId: Id<"skillScanRequests">) {
|
||||
const request = await ctx.db.get(requestId);
|
||||
if (!request) throw new ConvexError("Scan request not found");
|
||||
const now = Date.now();
|
||||
const jobId = await ctx.db.insert("securityScanJobs", {
|
||||
targetKind: "skillScanRequest",
|
||||
skillScanRequestId: request._id,
|
||||
status: "queued",
|
||||
source: "manual",
|
||||
priority: 100,
|
||||
hasMaliciousSignal: false,
|
||||
waitForVtUntil: now,
|
||||
nextRunAt: now,
|
||||
attempts: 0,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
await ctx.db.patch(request._id, {
|
||||
securityScanJobId: jobId,
|
||||
updatedAt: now,
|
||||
});
|
||||
return jobId;
|
||||
}
|
||||
|
||||
export const createUploadedSkillScanRequestInternal = internalMutation({
|
||||
args: {
|
||||
actorUserId: v.id("users"),
|
||||
files: v.array(scanRequestFileValidator),
|
||||
displayName: v.optional(v.string()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const actor = await ctx.db.get(args.actorUserId);
|
||||
if (!actor) throw new ConvexError("Unauthorized");
|
||||
if (args.files.length === 0) throw new ConvexError("files required");
|
||||
if (
|
||||
!args.files.some((file) => {
|
||||
const lower = file.path.trim().toLowerCase();
|
||||
return lower === "skill.md";
|
||||
})
|
||||
) {
|
||||
throw new ConvexError("SKILL.md required");
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const scanId = await ctx.db.insert("skillScanRequests", {
|
||||
actorUserId: actor._id,
|
||||
sourceKind: "upload",
|
||||
update: false,
|
||||
writtenBack: false,
|
||||
status: "queued",
|
||||
displayName: args.displayName,
|
||||
version: "local",
|
||||
files: args.files,
|
||||
expiresAt: skillScanRequestExpiresAt(now),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
const jobId = await enqueueSkillScanRequestJob(ctx, scanId);
|
||||
|
||||
await ctx.db.insert("auditLogs", {
|
||||
actorUserId: actor._id,
|
||||
action: "skill.clawscan.scan_upload",
|
||||
targetType: "skillScanRequest",
|
||||
targetId: scanId,
|
||||
metadata: {
|
||||
jobId,
|
||||
fileCount: args.files.length,
|
||||
},
|
||||
createdAt: now,
|
||||
});
|
||||
|
||||
return {
|
||||
ok: true as const,
|
||||
scanId,
|
||||
jobId,
|
||||
status: "queued" as const,
|
||||
sourceKind: "upload" as const,
|
||||
update: false,
|
||||
alreadyQueued: false,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const createPublishedSkillScanRequestInternal = internalMutation({
|
||||
args: {
|
||||
actorUserId: v.id("users"),
|
||||
slug: v.string(),
|
||||
version: v.optional(v.string()),
|
||||
update: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const actor = await ctx.db.get(args.actorUserId);
|
||||
if (!actor) throw new ConvexError("Unauthorized");
|
||||
|
||||
const slug = args.slug.trim().toLowerCase();
|
||||
if (!slug) throw new ConvexError("Slug required");
|
||||
const skill = await ctx.db
|
||||
.query("skills")
|
||||
.withIndex("by_slug", (q) => q.eq("slug", slug))
|
||||
.unique();
|
||||
if (!skill || skill.softDeletedAt) throw new ConvexError("Skill not found");
|
||||
|
||||
await assertCanManageOwnedResource(ctx, {
|
||||
actor,
|
||||
ownerUserId: skill.ownerUserId,
|
||||
ownerPublisherId: skill.ownerPublisherId,
|
||||
allowPlatformModerator: true,
|
||||
});
|
||||
|
||||
const requestedVersion = args.version?.trim();
|
||||
const version = requestedVersion
|
||||
? await ctx.db
|
||||
.query("skillVersions")
|
||||
.withIndex("by_skill_version", (q) =>
|
||||
q.eq("skillId", skill._id).eq("version", requestedVersion),
|
||||
)
|
||||
.unique()
|
||||
: skill.latestVersionId
|
||||
? await ctx.db.get(skill.latestVersionId)
|
||||
: null;
|
||||
if (!version || version.softDeletedAt) throw new ConvexError("Skill version not found");
|
||||
|
||||
const fingerprintEntries = await ctx.db
|
||||
.query("skillVersionFingerprints")
|
||||
.withIndex("by_version", (q) => q.eq("versionId", version._id))
|
||||
.collect();
|
||||
const files = sourceSkillVersionFiles(version.files, {
|
||||
generatedBundleFingerprints: fingerprintEntries
|
||||
.filter((entry) => entry.kind === "generated-bundle")
|
||||
.map((entry) => entry.fingerprint),
|
||||
});
|
||||
|
||||
const now = Date.now();
|
||||
const update = args.update === true;
|
||||
const scanId = await ctx.db.insert("skillScanRequests", {
|
||||
actorUserId: actor._id,
|
||||
sourceKind: "published",
|
||||
update,
|
||||
writtenBack: false,
|
||||
status: "queued",
|
||||
slug: skill.slug,
|
||||
displayName: skill.displayName,
|
||||
version: version.version,
|
||||
skillId: skill._id,
|
||||
skillVersionId: version._id,
|
||||
files,
|
||||
parsed: version.parsed,
|
||||
sha256hash: version.sha256hash,
|
||||
vtAnalysis: version.vtAnalysis,
|
||||
capabilityTags: version.capabilityTags,
|
||||
staticScan: version.staticScan,
|
||||
expiresAt: skillScanRequestExpiresAt(now),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
const jobId = await enqueueSkillScanRequestJob(ctx, scanId);
|
||||
|
||||
await ctx.db.insert("auditLogs", {
|
||||
actorUserId: actor._id,
|
||||
action: update ? "skill.clawscan.scan_published_update" : "skill.clawscan.scan_published",
|
||||
targetType: "skillVersion",
|
||||
targetId: version._id,
|
||||
metadata: {
|
||||
skillId: skill._id,
|
||||
slug: skill.slug,
|
||||
version: version.version,
|
||||
scanId,
|
||||
jobId,
|
||||
update,
|
||||
},
|
||||
createdAt: now,
|
||||
});
|
||||
|
||||
return {
|
||||
ok: true as const,
|
||||
scanId,
|
||||
jobId,
|
||||
status: "queued" as const,
|
||||
sourceKind: "published" as const,
|
||||
update,
|
||||
alreadyQueued: false,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const getSkillScanRequestForUserInternal = internalQuery({
|
||||
args: {
|
||||
actorUserId: v.id("users"),
|
||||
scanId: v.id("skillScanRequests"),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const actor = await ctx.db.get(args.actorUserId);
|
||||
if (!actor) throw new ConvexError("Unauthorized");
|
||||
const request = await ctx.db.get(args.scanId);
|
||||
if (!request) throw new ConvexError("Scan not found");
|
||||
if (request.actorUserId !== actor._id && actor.role !== "admin" && actor.role !== "moderator") {
|
||||
throw new ConvexError("Forbidden");
|
||||
}
|
||||
const job = request.securityScanJobId ? await ctx.db.get(request.securityScanJobId) : null;
|
||||
return skillScanStatusResponse(request, job);
|
||||
},
|
||||
});
|
||||
|
||||
export const recordSkillScanRequestSucceededInternal = internalMutation({
|
||||
args: {
|
||||
scanId: v.id("skillScanRequests"),
|
||||
jobId: v.id("securityScanJobs"),
|
||||
runId: v.optional(v.string()),
|
||||
llmAnalysis: llmAnalysisValidator,
|
||||
skillSpectorAnalysis: v.optional(skillSpectorAnalysisValidator),
|
||||
writtenBack: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const request = await ctx.db.get(args.scanId);
|
||||
if (!request) throw new ConvexError("Scan request not found");
|
||||
const now = Date.now();
|
||||
await ctx.db.patch(request._id, {
|
||||
status: "succeeded",
|
||||
llmAnalysis: args.llmAnalysis,
|
||||
...(args.skillSpectorAnalysis
|
||||
? { skillSpectorAnalysis: capSkillSpectorAnalysisForStorage(args.skillSpectorAnalysis) }
|
||||
: {}),
|
||||
writtenBack: args.writtenBack === true || request.writtenBack,
|
||||
runId: args.runId,
|
||||
completedAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
return { ok: true as const };
|
||||
},
|
||||
});
|
||||
|
||||
export const recordSkillScanRequestFailedInternal = internalMutation({
|
||||
args: {
|
||||
scanId: v.id("skillScanRequests"),
|
||||
error: v.string(),
|
||||
llmAnalysis: v.optional(llmAnalysisValidator),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const request = await ctx.db.get(args.scanId);
|
||||
if (!request) throw new ConvexError("Scan request not found");
|
||||
const now = Date.now();
|
||||
await ctx.db.patch(request._id, {
|
||||
status: "failed",
|
||||
lastError: args.error.slice(0, 2000),
|
||||
...(args.llmAnalysis ? { llmAnalysis: args.llmAnalysis } : {}),
|
||||
completedAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
return { ok: true as const };
|
||||
},
|
||||
});
|
||||
|
||||
export const pruneExpiredSkillScanRequestsInternal = internalMutation({
|
||||
args: {
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const batchSize = Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
args.batchSize ?? DEFAULT_PRUNE_SKILL_SCAN_REQUEST_LIMIT,
|
||||
MAX_PRUNE_SKILL_SCAN_REQUEST_LIMIT,
|
||||
),
|
||||
);
|
||||
const now = Date.now();
|
||||
const requests = await ctx.db
|
||||
.query("skillScanRequests")
|
||||
.withIndex("by_expires_at", (q) => q.lt("expiresAt", now))
|
||||
.take(batchSize);
|
||||
|
||||
let deletedJobs = 0;
|
||||
let deletedFiles = 0;
|
||||
for (const request of requests) {
|
||||
if (request.securityScanJobId) {
|
||||
const job = await ctx.db.get(request.securityScanJobId);
|
||||
if (job?.targetKind === "skillScanRequest") {
|
||||
await ctx.db.delete(job._id);
|
||||
deletedJobs += 1;
|
||||
}
|
||||
}
|
||||
if (request.sourceKind === "upload") {
|
||||
for (const file of request.files) {
|
||||
try {
|
||||
await ctx.storage.delete(file.storageId);
|
||||
deletedFiles += 1;
|
||||
} catch {
|
||||
// Missing storage objects should not block expiry of the request row.
|
||||
}
|
||||
}
|
||||
}
|
||||
await ctx.db.delete(request._id);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true as const,
|
||||
deletedRequests: requests.length,
|
||||
deletedJobs,
|
||||
deletedFiles,
|
||||
done: requests.length < batchSize,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
async function requestPackageRescanForActor(
|
||||
ctx: MutationCtx,
|
||||
args: {
|
||||
@@ -1168,6 +1542,13 @@ export const claimQueuedJobsInternal = internalMutation({
|
||||
lastError: undefined,
|
||||
updatedAt: now,
|
||||
});
|
||||
if (job.targetKind === "skillScanRequest" && job.skillScanRequestId) {
|
||||
await ctx.db.patch(job.skillScanRequestId, {
|
||||
status: "running",
|
||||
lastError: undefined,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
claimed.push({
|
||||
...job,
|
||||
status: "running" as const,
|
||||
@@ -1206,6 +1587,15 @@ export const getJobTargetInternal = internalQuery({
|
||||
trustedOpenClawPlugin: isOpenClawPluginPackage(pkg, ownerPublisher),
|
||||
};
|
||||
}
|
||||
if (job.targetKind === "skillScanRequest" && job.skillScanRequestId) {
|
||||
const scanRequest = await ctx.db.get(job.skillScanRequestId);
|
||||
if (!scanRequest) return { job, missing: true as const };
|
||||
const version = scanRequest.skillVersionId
|
||||
? await ctx.db.get(scanRequest.skillVersionId)
|
||||
: null;
|
||||
const skill = scanRequest.skillId ? await ctx.db.get(scanRequest.skillId) : null;
|
||||
return { job, skill, version: version ?? undefined, scanRequest };
|
||||
}
|
||||
return { job, missing: true as const };
|
||||
},
|
||||
});
|
||||
@@ -1252,6 +1642,14 @@ export const failJobInternal = internalMutation({
|
||||
workerId: undefined,
|
||||
updatedAt: now,
|
||||
});
|
||||
if (job.targetKind === "skillScanRequest" && job.skillScanRequestId) {
|
||||
await ctx.db.patch(job.skillScanRequestId, {
|
||||
status: retry ? "queued" : "failed",
|
||||
lastError: args.error.slice(0, 2000),
|
||||
...(retry ? {} : { completedAt: now }),
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
return { ok: true as const, retry };
|
||||
},
|
||||
});
|
||||
@@ -1291,6 +1689,7 @@ export const claimCodexScanJobs = action({
|
||||
continue;
|
||||
}
|
||||
|
||||
const scanRequest = target.scanRequest as Doc<"skillScanRequests"> | undefined;
|
||||
const version = target.version as Doc<"skillVersions"> | undefined;
|
||||
const release = target.release as Doc<"packageReleases"> | undefined;
|
||||
let files: Array<{
|
||||
@@ -1300,7 +1699,9 @@ export const claimCodexScanJobs = action({
|
||||
storageId: Id<"_storage">;
|
||||
contentType?: string;
|
||||
}> = [];
|
||||
if (version) {
|
||||
if (scanRequest) {
|
||||
files = scanRequest.files;
|
||||
} else if (version) {
|
||||
const fingerprintEntries = await runQueryRef<
|
||||
Array<{ fingerprint: string; kind?: "source" | "generated-bundle" }>
|
||||
>(ctx, internalRefs.skills.listVersionFingerprintsInternal, {
|
||||
@@ -1406,6 +1807,33 @@ export const completeCodexScanJob = action({
|
||||
releaseId: target.release._id,
|
||||
llmAnalysis: args.llmAnalysis,
|
||||
});
|
||||
} else if (target.job.targetKind === "skillScanRequest" && target.scanRequest) {
|
||||
let writtenBack = false;
|
||||
if (
|
||||
target.scanRequest.sourceKind === "published" &&
|
||||
target.scanRequest.update &&
|
||||
target.version
|
||||
) {
|
||||
if (args.skillSpectorAnalysis) {
|
||||
await runMutationRef(ctx, internalRefs.skills.updateVersionSkillSpectorAnalysisInternal, {
|
||||
versionId: target.version._id,
|
||||
skillSpectorAnalysis: capSkillSpectorAnalysisForStorage(args.skillSpectorAnalysis),
|
||||
});
|
||||
}
|
||||
await runMutationRef(ctx, internalRefs.skills.updateVersionLlmAnalysisInternal, {
|
||||
versionId: target.version._id,
|
||||
llmAnalysis: args.llmAnalysis,
|
||||
});
|
||||
writtenBack = true;
|
||||
}
|
||||
await runMutationRef(ctx, internalRefs.securityScan.recordSkillScanRequestSucceededInternal, {
|
||||
scanId: target.scanRequest._id,
|
||||
jobId: args.jobId,
|
||||
runId: args.runId,
|
||||
llmAnalysis: args.llmAnalysis,
|
||||
skillSpectorAnalysis: args.skillSpectorAnalysis,
|
||||
writtenBack,
|
||||
});
|
||||
} else {
|
||||
throw new ConvexError("Unsupported security scan target");
|
||||
}
|
||||
@@ -1462,6 +1890,16 @@ export const failCodexScanJob = action({
|
||||
llmAnalysis,
|
||||
});
|
||||
}
|
||||
} else if (target.job.targetKind === "skillScanRequest" && target.scanRequest) {
|
||||
await runMutationRef(
|
||||
ctx,
|
||||
internalRefs.securityScan.recordSkillScanRequestFailedInternal,
|
||||
{
|
||||
scanId: target.scanRequest._id,
|
||||
error: args.error,
|
||||
llmAnalysis,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
/* @vitest-environment node */
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import schema from "./schema";
|
||||
|
||||
vi.mock("@convex-dev/auth/server", () => ({
|
||||
getAuthUserId: vi.fn(),
|
||||
authTables: {},
|
||||
}));
|
||||
|
||||
const { __test, listPublicPageV4 } = await import("./skills");
|
||||
|
||||
const listPublicPageV4Handler = (
|
||||
listPublicPageV4 as unknown as {
|
||||
_handler: (
|
||||
ctx: unknown,
|
||||
args: unknown,
|
||||
) => Promise<{ page: Array<{ skill: { slug: string } }> }>;
|
||||
}
|
||||
)._handler;
|
||||
|
||||
describe("skills.listPublicPageV4", () => {
|
||||
it("defines recommended rank indexes in contract order", () => {
|
||||
expect(getSkillSearchDigestIndexFields("by_active_recommended_rank")).toEqual([
|
||||
"softDeletedAt",
|
||||
"statsStars",
|
||||
"statsInstallsAllTime",
|
||||
"statsDownloads",
|
||||
"updatedAt",
|
||||
]);
|
||||
expect(getSkillSearchDigestIndexFields("by_nonsuspicious_recommended_rank")).toEqual([
|
||||
"softDeletedAt",
|
||||
"isSuspicious",
|
||||
"statsStars",
|
||||
"statsInstallsAllTime",
|
||||
"statsDownloads",
|
||||
"updatedAt",
|
||||
]);
|
||||
});
|
||||
|
||||
it("forces Recommended ranking to descending for stale URLs", () => {
|
||||
expect(__test.resolvePublicListDir("recommended", "asc")).toBe("desc");
|
||||
expect(__test.resolvePublicListDir("default", "asc")).toBe("desc");
|
||||
});
|
||||
|
||||
it("keeps explicit non-default sort directions", () => {
|
||||
expect(__test.resolvePublicListDir("name", undefined)).toBe("asc");
|
||||
expect(__test.resolvePublicListDir("downloads", "asc")).toBe("asc");
|
||||
});
|
||||
|
||||
it("keeps recommended-rank cursors on the index that created them", () => {
|
||||
expect(
|
||||
__test.resolveRecommendedPublicListSort({
|
||||
decodedCursor: null,
|
||||
hasMissingRankStats: false,
|
||||
}),
|
||||
).toBe("recommended");
|
||||
expect(
|
||||
__test.resolveRecommendedPublicListSort({
|
||||
decodedCursor: null,
|
||||
hasMissingRankStats: true,
|
||||
}),
|
||||
).toBe("updated");
|
||||
expect(
|
||||
__test.resolveRecommendedPublicListSort({
|
||||
decodedCursor: [undefined, 123, 456, "skillSearchDigest:updated"],
|
||||
hasMissingRankStats: false,
|
||||
}),
|
||||
).toBe("updated");
|
||||
expect(
|
||||
__test.resolveRecommendedPublicListSort({
|
||||
decodedCursor: [undefined, false, 123, 456, "skillSearchDigest:nonsuspicious-updated"],
|
||||
hasMissingRankStats: false,
|
||||
}),
|
||||
).toBe("updated");
|
||||
expect(
|
||||
__test.resolveRecommendedPublicListSort({
|
||||
decodedCursor: [undefined, 10, 20, 30, 123, 456, "skillSearchDigest:recommended"],
|
||||
hasMissingRankStats: true,
|
||||
}),
|
||||
).toBe("recommended");
|
||||
expect(
|
||||
__test.resolveRecommendedPublicListSort({
|
||||
decodedCursor: [
|
||||
undefined,
|
||||
false,
|
||||
10,
|
||||
20,
|
||||
30,
|
||||
123,
|
||||
456,
|
||||
"skillSearchDigest:nonsuspicious-recommended",
|
||||
],
|
||||
hasMissingRankStats: true,
|
||||
}),
|
||||
).toBe("recommended");
|
||||
});
|
||||
|
||||
it("sorts highlighted recommended results by stars, installs, downloads, then updatedAt", async () => {
|
||||
const result = await listPublicPageV4Handler(
|
||||
makeHighlightedCtx([
|
||||
makeDigest({
|
||||
id: "updated",
|
||||
slug: "updated-skill",
|
||||
stars: 2,
|
||||
installsAllTime: 10,
|
||||
downloads: 10,
|
||||
updatedAt: 400,
|
||||
}),
|
||||
makeDigest({
|
||||
id: "downloads",
|
||||
slug: "downloads-skill",
|
||||
stars: 2,
|
||||
installsAllTime: 10,
|
||||
downloads: 50,
|
||||
updatedAt: 100,
|
||||
}),
|
||||
makeDigest({
|
||||
id: "installs",
|
||||
slug: "installs-skill",
|
||||
stars: 2,
|
||||
installsAllTime: 20,
|
||||
downloads: 0,
|
||||
updatedAt: 100,
|
||||
}),
|
||||
makeDigest({
|
||||
id: "stars",
|
||||
slug: "stars-skill",
|
||||
stars: 3,
|
||||
installsAllTime: 0,
|
||||
downloads: 0,
|
||||
updatedAt: 100,
|
||||
}),
|
||||
]),
|
||||
{ highlightedOnly: true, numItems: 10 },
|
||||
);
|
||||
|
||||
expect(result.page.map((entry) => entry.skill.slug)).toEqual([
|
||||
"stars-skill",
|
||||
"installs-skill",
|
||||
"downloads-skill",
|
||||
"updated-skill",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
function getSkillSearchDigestIndexFields(indexDescriptor: string) {
|
||||
const index = schema.tables.skillSearchDigest[" indexes"]().find(
|
||||
(candidate) => candidate.indexDescriptor === indexDescriptor,
|
||||
);
|
||||
if (!index) throw new Error(`Missing skillSearchDigest index ${indexDescriptor}`);
|
||||
return index.fields;
|
||||
}
|
||||
|
||||
type EqBuilder = {
|
||||
eq: (field: string, value: unknown) => EqBuilder;
|
||||
getLastValue: () => unknown;
|
||||
};
|
||||
|
||||
function makeEqBuilder(): EqBuilder {
|
||||
let lastValue: unknown;
|
||||
const builder: EqBuilder = {
|
||||
eq: (_field, value) => {
|
||||
lastValue = value;
|
||||
return builder;
|
||||
},
|
||||
getLastValue: () => lastValue,
|
||||
};
|
||||
return builder;
|
||||
}
|
||||
|
||||
function makeHighlightedCtx(digests: Array<ReturnType<typeof makeDigest>>) {
|
||||
const digestBySkillId = new Map(digests.map((digest) => [digest.skillId, digest]));
|
||||
return {
|
||||
db: {
|
||||
query: vi.fn((table: string) => {
|
||||
if (table === "skillBadges") {
|
||||
return {
|
||||
withIndex: vi.fn((_indexName: string, build: (q: EqBuilder) => unknown) => {
|
||||
build(makeEqBuilder());
|
||||
return {
|
||||
order: vi.fn(() => ({
|
||||
take: vi.fn().mockResolvedValue(
|
||||
digests.map((digest) => ({
|
||||
_id: `skillBadges:${digest.skillId}`,
|
||||
skillId: digest.skillId,
|
||||
kind: "highlighted",
|
||||
awardedAt: digest.updatedAt,
|
||||
})),
|
||||
),
|
||||
})),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (table === "skillSearchDigest") {
|
||||
return {
|
||||
withIndex: vi.fn((_indexName: string, build: (q: EqBuilder) => unknown) => {
|
||||
const eqBuilder = makeEqBuilder();
|
||||
build(eqBuilder);
|
||||
const skillId = eqBuilder.getLastValue();
|
||||
return {
|
||||
unique: vi
|
||||
.fn()
|
||||
.mockResolvedValue(
|
||||
typeof skillId === "string" ? (digestBySkillId.get(skillId) ?? null) : null,
|
||||
),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
throw new Error(`Unexpected table ${table}`);
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeDigest(params: {
|
||||
id: string;
|
||||
slug: string;
|
||||
stars: number;
|
||||
installsAllTime: number;
|
||||
downloads: number;
|
||||
updatedAt: number;
|
||||
}) {
|
||||
return {
|
||||
_id: `skillSearchDigest:${params.id}`,
|
||||
_creationTime: params.updatedAt,
|
||||
skillId: `skills:${params.id}`,
|
||||
slug: params.slug,
|
||||
displayName: params.slug,
|
||||
summary: `${params.slug} summary`,
|
||||
ownerUserId: "users:owner",
|
||||
ownerPublisherId: undefined,
|
||||
ownerHandle: "owner",
|
||||
ownerKind: "user",
|
||||
ownerName: "owner",
|
||||
ownerDisplayName: "Owner",
|
||||
ownerImage: undefined,
|
||||
canonicalSkillId: undefined,
|
||||
forkOf: undefined,
|
||||
latestVersionId: undefined,
|
||||
latestVersionSummary: undefined,
|
||||
tags: {},
|
||||
capabilityTags: [],
|
||||
badges: undefined,
|
||||
stats: {
|
||||
downloads: params.downloads,
|
||||
installsCurrent: 0,
|
||||
installsAllTime: params.installsAllTime,
|
||||
stars: params.stars,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
},
|
||||
statsDownloads: params.downloads,
|
||||
statsStars: params.stars,
|
||||
statsInstallsCurrent: 0,
|
||||
statsInstallsAllTime: params.installsAllTime,
|
||||
softDeletedAt: undefined,
|
||||
moderationStatus: "active",
|
||||
moderationFlags: [],
|
||||
moderationReason: undefined,
|
||||
isSuspicious: false,
|
||||
createdAt: 1,
|
||||
updatedAt: params.updatedAt,
|
||||
};
|
||||
}
|
||||
@@ -63,6 +63,15 @@ function chainEq(constraints: Record<string, unknown>) {
|
||||
};
|
||||
}
|
||||
|
||||
const defaultSkillStats = {
|
||||
downloads: 0,
|
||||
stars: 0,
|
||||
installsCurrent: 0,
|
||||
installsAllTime: 0,
|
||||
versions: 1,
|
||||
comments: 0,
|
||||
};
|
||||
|
||||
describe("skills ownership", () => {
|
||||
it("resolves alias slugs to the live target skill", async () => {
|
||||
const result = await getSkillBySlugInternalHandler(
|
||||
@@ -430,6 +439,7 @@ describe("skills ownership", () => {
|
||||
softDeletedAt: undefined,
|
||||
moderationVerdict: "clean",
|
||||
moderationReasonCodes: ["suspicious.dynamic_code_execution"],
|
||||
stats: defaultSkillStats,
|
||||
};
|
||||
const aliases = [
|
||||
{
|
||||
@@ -688,6 +698,7 @@ describe("skills ownership", () => {
|
||||
ownerUserId: "users:actor",
|
||||
ownerPublisherId: "publishers:actor",
|
||||
softDeletedAt: undefined,
|
||||
stats: defaultSkillStats,
|
||||
};
|
||||
const aliases = [
|
||||
{
|
||||
|
||||
@@ -33,7 +33,15 @@ type WrappedHandler<TArgs, TResult> = {
|
||||
type PublicListArgs = {
|
||||
cursor?: string;
|
||||
numItems?: number;
|
||||
sort?: "newest" | "updated" | "downloads" | "installs" | "stars" | "name";
|
||||
sort?:
|
||||
| "default"
|
||||
| "recommended"
|
||||
| "newest"
|
||||
| "updated"
|
||||
| "downloads"
|
||||
| "installs"
|
||||
| "stars"
|
||||
| "name";
|
||||
dir?: "asc" | "desc";
|
||||
highlightedOnly?: boolean;
|
||||
nonSuspiciousOnly?: boolean;
|
||||
@@ -133,12 +141,81 @@ function cursorForIndex(index: string, key: unknown[]): string {
|
||||
return JSON.stringify({ v: 1, index, key });
|
||||
}
|
||||
|
||||
class TestEqBuilder {
|
||||
eq(_field: string, _value: unknown) {
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
function makeMissingRecommendedRankStatsCtx() {
|
||||
const first = vi.fn(async () => makeSearchDigest({ statsStars: undefined }));
|
||||
const withIndex = vi.fn((_indexName: string, build: (q: TestEqBuilder) => unknown) => {
|
||||
build(new TestEqBuilder());
|
||||
return { first };
|
||||
});
|
||||
const query = vi.fn((table: string) => {
|
||||
if (table !== "skillSearchDigest") throw new Error(`unexpected table ${table}`);
|
||||
return { withIndex };
|
||||
});
|
||||
|
||||
return {
|
||||
ctx: { db: { query } },
|
||||
first,
|
||||
query,
|
||||
withIndex,
|
||||
};
|
||||
}
|
||||
|
||||
describe("public skill list deterministic cursors", () => {
|
||||
beforeEach(() => {
|
||||
getPageMock.mockReset();
|
||||
getPageMock.mockResolvedValue({ page: [], hasMore: false, indexKeys: [] });
|
||||
});
|
||||
|
||||
it("falls back to the updated index while default rank stats are missing", async () => {
|
||||
const { ctx, withIndex } = makeMissingRecommendedRankStatsCtx();
|
||||
|
||||
await listPublicPageV4Handler(ctx, {
|
||||
numItems: 10,
|
||||
});
|
||||
|
||||
expect(withIndex.mock.calls.map(([indexName]) => indexName)).toEqual([
|
||||
"by_active_stats_stars",
|
||||
"by_active_stats_installs_all_time",
|
||||
"by_active_stats_downloads",
|
||||
]);
|
||||
expect(getPageMock).toHaveBeenCalledTimes(1);
|
||||
expect(getPageMock.mock.calls[0]?.[1]).toMatchObject({
|
||||
index: "by_active_updated",
|
||||
startIndexKey: [undefined],
|
||||
endIndexKey: [undefined],
|
||||
startInclusive: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to the non-suspicious updated index while default rank stats are missing", async () => {
|
||||
const { ctx, withIndex } = makeMissingRecommendedRankStatsCtx();
|
||||
|
||||
await listPublicApiPageV1Handler(ctx, {
|
||||
numItems: 10,
|
||||
sort: "recommended",
|
||||
nonSuspiciousOnly: true,
|
||||
});
|
||||
|
||||
expect(withIndex.mock.calls.map(([indexName]) => indexName)).toEqual([
|
||||
"by_nonsuspicious_stars",
|
||||
"by_nonsuspicious_installs",
|
||||
"by_nonsuspicious_downloads",
|
||||
]);
|
||||
expect(getPageMock).toHaveBeenCalledTimes(1);
|
||||
expect(getPageMock.mock.calls[0]?.[1]).toMatchObject({
|
||||
index: "by_nonsuspicious_updated",
|
||||
startIndexKey: [undefined, false],
|
||||
endIndexKey: [undefined, false],
|
||||
startInclusive: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores stale legacy cursors that are longer than the selected index", async () => {
|
||||
const staleDownloadsCursor = legacyCursor([{ __undef: 1 }, false, 100, 200]);
|
||||
|
||||
@@ -368,7 +445,10 @@ describe("public skill list deterministic cursors", () => {
|
||||
indexKeys: [],
|
||||
});
|
||||
|
||||
const result = await listPublicApiPageV1Handler({} as never, { numItems: 10 });
|
||||
const result = await listPublicApiPageV1Handler({} as never, {
|
||||
numItems: 10,
|
||||
sort: "updated",
|
||||
});
|
||||
|
||||
expect(result.items).toHaveLength(1);
|
||||
expect(result.items[0]).toMatchObject({ latestVersion: null });
|
||||
@@ -400,7 +480,7 @@ describe("public skill list deterministic cursors", () => {
|
||||
),
|
||||
},
|
||||
} as never,
|
||||
{ numItems: 10 },
|
||||
{ numItems: 10, sort: "updated" },
|
||||
);
|
||||
|
||||
expect(result.items).toHaveLength(1);
|
||||
|
||||
+194
-13
@@ -626,6 +626,7 @@ const NEW_SKILL_RATE_LIMITS = {
|
||||
} as const;
|
||||
|
||||
const SORT_INDEXES = {
|
||||
recommended: "by_active_recommended_rank",
|
||||
newest: "by_active_created",
|
||||
updated: "by_active_updated",
|
||||
name: "by_active_name",
|
||||
@@ -636,6 +637,7 @@ const SORT_INDEXES = {
|
||||
|
||||
// Compound indexes on skillSearchDigest that filter isSuspicious at the index level.
|
||||
const NONSUSPICIOUS_SORT_INDEXES = {
|
||||
recommended: "by_nonsuspicious_recommended_rank",
|
||||
newest: "by_nonsuspicious_created",
|
||||
updated: "by_nonsuspicious_updated",
|
||||
name: "by_nonsuspicious_name",
|
||||
@@ -4525,6 +4527,8 @@ export const listPublicPageV2 = query({
|
||||
paginationOpts: paginationOptsValidator,
|
||||
sort: v.optional(
|
||||
v.union(
|
||||
v.literal("default"),
|
||||
v.literal("recommended"),
|
||||
v.literal("newest"),
|
||||
v.literal("updated"),
|
||||
v.literal("downloads"),
|
||||
@@ -4621,6 +4625,7 @@ export const listPublicPageV3 = query({
|
||||
type PublicListSort = keyof typeof SORT_INDEXES;
|
||||
|
||||
const SORT_INDEX_FIELD_COUNTS: Record<PublicListSort, number> = {
|
||||
recommended: 5,
|
||||
newest: 2,
|
||||
updated: 2,
|
||||
name: 2,
|
||||
@@ -4630,6 +4635,7 @@ const SORT_INDEX_FIELD_COUNTS: Record<PublicListSort, number> = {
|
||||
};
|
||||
|
||||
const NONSUSPICIOUS_SORT_INDEX_FIELD_COUNTS: Record<PublicListSort, number> = {
|
||||
recommended: 6,
|
||||
newest: 3,
|
||||
updated: 3,
|
||||
name: 3,
|
||||
@@ -4739,6 +4745,8 @@ export const listPublicPageV4 = query({
|
||||
numItems: v.optional(v.number()),
|
||||
sort: v.optional(
|
||||
v.union(
|
||||
v.literal("default"),
|
||||
v.literal("recommended"),
|
||||
v.literal("newest"),
|
||||
v.literal("updated"),
|
||||
v.literal("downloads"),
|
||||
@@ -4764,16 +4772,37 @@ export const listPublicPageV4 = query({
|
||||
args.excludeCategoryKeywords ?? [],
|
||||
);
|
||||
const categorySlug = normalizeRelatedCategorySlug(args.categorySlug);
|
||||
const sort = args.sort ?? "newest";
|
||||
const dir = args.dir ?? (sort === "name" ? "asc" : "desc");
|
||||
const requestedSort = normalizePublicListSort(args.sort);
|
||||
const dir = resolvePublicListDir(requestedSort, args.dir);
|
||||
const numItems = clampInt(args.numItems ?? 25, 1, MAX_PUBLIC_LIST_LIMIT);
|
||||
const eqPrefix: IndexKey = args.nonSuspiciousOnly ? [undefined, false] : [undefined];
|
||||
const recommendedIndexName = args.nonSuspiciousOnly
|
||||
? NONSUSPICIOUS_SORT_INDEXES.recommended
|
||||
: SORT_INDEXES.recommended;
|
||||
const updatedIndexName = args.nonSuspiciousOnly
|
||||
? NONSUSPICIOUS_SORT_INDEXES.updated
|
||||
: SORT_INDEXES.updated;
|
||||
const recommendedCursor = getPublicListCursorKey({
|
||||
cursor: args.cursor,
|
||||
sort: "recommended",
|
||||
nonSuspiciousOnly: args.nonSuspiciousOnly ?? false,
|
||||
indexName: recommendedIndexName,
|
||||
eqPrefix,
|
||||
});
|
||||
const updatedCursor = getPublicListCursorKey({
|
||||
cursor: args.cursor,
|
||||
sort: "updated",
|
||||
nonSuspiciousOnly: args.nonSuspiciousOnly ?? false,
|
||||
indexName: updatedIndexName,
|
||||
eqPrefix,
|
||||
});
|
||||
|
||||
// Highlighted skills use a completely different path: query skillBadges
|
||||
// by kind to find highlighted skill IDs, then look up their digests.
|
||||
// This avoids scanning thousands of rows in the sort index.
|
||||
if (args.highlightedOnly) {
|
||||
return fetchHighlightedPage(ctx, {
|
||||
sort,
|
||||
sort: requestedSort,
|
||||
dir,
|
||||
numItems,
|
||||
capabilityTag: args.capabilityTag,
|
||||
@@ -4784,14 +4813,22 @@ export const listPublicPageV4 = query({
|
||||
});
|
||||
}
|
||||
|
||||
const sort =
|
||||
requestedSort === "recommended"
|
||||
? resolveRecommendedPublicListSort({
|
||||
decodedCursor: recommendedCursor ?? updatedCursor,
|
||||
hasMissingRankStats: await hasMissingRecommendedRankStats(
|
||||
ctx,
|
||||
args.nonSuspiciousOnly ?? false,
|
||||
recommendedCursor ?? updatedCursor,
|
||||
),
|
||||
})
|
||||
: requestedSort;
|
||||
|
||||
const indexName = args.nonSuspiciousOnly
|
||||
? NONSUSPICIOUS_SORT_INDEXES[sort]
|
||||
: SORT_INDEXES[sort];
|
||||
|
||||
// Equality prefix constrains getPage to active (non-deleted) rows.
|
||||
// Without this, getPage walks the entire index including soft-deleted items.
|
||||
const eqPrefix: IndexKey = args.nonSuspiciousOnly ? [undefined, false] : [undefined];
|
||||
|
||||
const decodedCursor = getPublicListCursorKey({
|
||||
cursor: args.cursor,
|
||||
sort,
|
||||
@@ -5274,6 +5311,8 @@ export const listPublicApiPageV1 = query({
|
||||
numItems: v.optional(v.number()),
|
||||
sort: v.optional(
|
||||
v.union(
|
||||
v.literal("default"),
|
||||
v.literal("recommended"),
|
||||
v.literal("newest"),
|
||||
v.literal("updated"),
|
||||
v.literal("downloads"),
|
||||
@@ -5286,13 +5325,44 @@ export const listPublicApiPageV1 = query({
|
||||
nonSuspiciousOnly: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const sort = args.sort ?? "newest";
|
||||
const dir = args.dir ?? (sort === "name" ? "asc" : "desc");
|
||||
const requestedSort = normalizePublicListSort(args.sort);
|
||||
const dir = resolvePublicListDir(requestedSort, args.dir);
|
||||
const numItems = clampInt(args.numItems ?? 25, 1, MAX_PUBLIC_LIST_LIMIT);
|
||||
const eqPrefix: IndexKey = args.nonSuspiciousOnly ? [undefined, false] : [undefined];
|
||||
const recommendedIndexName = args.nonSuspiciousOnly
|
||||
? NONSUSPICIOUS_SORT_INDEXES.recommended
|
||||
: SORT_INDEXES.recommended;
|
||||
const updatedIndexName = args.nonSuspiciousOnly
|
||||
? NONSUSPICIOUS_SORT_INDEXES.updated
|
||||
: SORT_INDEXES.updated;
|
||||
const recommendedCursor = getPublicListCursorKey({
|
||||
cursor: args.cursor,
|
||||
sort: "recommended",
|
||||
nonSuspiciousOnly: args.nonSuspiciousOnly ?? false,
|
||||
indexName: recommendedIndexName,
|
||||
eqPrefix,
|
||||
});
|
||||
const updatedCursor = getPublicListCursorKey({
|
||||
cursor: args.cursor,
|
||||
sort: "updated",
|
||||
nonSuspiciousOnly: args.nonSuspiciousOnly ?? false,
|
||||
indexName: updatedIndexName,
|
||||
eqPrefix,
|
||||
});
|
||||
const sort =
|
||||
requestedSort === "recommended"
|
||||
? resolveRecommendedPublicListSort({
|
||||
decodedCursor: recommendedCursor ?? updatedCursor,
|
||||
hasMissingRankStats: await hasMissingRecommendedRankStats(
|
||||
ctx,
|
||||
args.nonSuspiciousOnly ?? false,
|
||||
recommendedCursor ?? updatedCursor,
|
||||
),
|
||||
})
|
||||
: requestedSort;
|
||||
const indexName = args.nonSuspiciousOnly
|
||||
? NONSUSPICIOUS_SORT_INDEXES[sort]
|
||||
: SORT_INDEXES[sort];
|
||||
const eqPrefix: IndexKey = args.nonSuspiciousOnly ? [undefined, false] : [undefined];
|
||||
const decodedCursor = getPublicListCursorKey({
|
||||
cursor: args.cursor,
|
||||
sort,
|
||||
@@ -5713,6 +5783,98 @@ export const searchPackageCatalogForHttpInternal = internalQuery({
|
||||
});
|
||||
|
||||
type SortKey = keyof typeof SORT_INDEXES;
|
||||
type SortKeyInput = SortKey | "default" | undefined;
|
||||
|
||||
function normalizePublicListSort(sort: SortKeyInput): SortKey {
|
||||
return sort === undefined || sort === "default" ? "recommended" : sort;
|
||||
}
|
||||
|
||||
function resolvePublicListDir(sort: SortKeyInput, dir: "asc" | "desc" | undefined) {
|
||||
const normalizedSort = normalizePublicListSort(sort);
|
||||
if (normalizedSort === "recommended") return "desc";
|
||||
return dir ?? (normalizedSort === "name" ? "asc" : "desc");
|
||||
}
|
||||
|
||||
function resolveRecommendedPublicListSort({
|
||||
decodedCursor,
|
||||
hasMissingRankStats,
|
||||
}: {
|
||||
decodedCursor: readonly unknown[] | null;
|
||||
hasMissingRankStats: boolean;
|
||||
}): SortKey {
|
||||
if (decodedCursor) {
|
||||
return decodedCursor.length <= 5 ? "updated" : "recommended";
|
||||
}
|
||||
return hasMissingRankStats ? "updated" : "recommended";
|
||||
}
|
||||
|
||||
async function hasMissingRecommendedRankStats(
|
||||
ctx: Pick<QueryCtx, "db">,
|
||||
nonSuspiciousOnly: boolean,
|
||||
decodedCursor: IndexKey | null,
|
||||
) {
|
||||
if (decodedCursor) return false;
|
||||
if (nonSuspiciousOnly) {
|
||||
const [missingStars, missingInstalls, missingDownloads] = await Promise.all([
|
||||
ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_nonsuspicious_stars", (q) =>
|
||||
q.eq("softDeletedAt", undefined).eq("isSuspicious", false).eq("statsStars", undefined),
|
||||
)
|
||||
.first(),
|
||||
ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_nonsuspicious_installs", (q) =>
|
||||
q
|
||||
.eq("softDeletedAt", undefined)
|
||||
.eq("isSuspicious", false)
|
||||
.eq("statsInstallsAllTime", undefined),
|
||||
)
|
||||
.first(),
|
||||
ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_nonsuspicious_downloads", (q) =>
|
||||
q
|
||||
.eq("softDeletedAt", undefined)
|
||||
.eq("isSuspicious", false)
|
||||
.eq("statsDownloads", undefined),
|
||||
)
|
||||
.first(),
|
||||
]);
|
||||
return Boolean(missingStars || missingInstalls || missingDownloads);
|
||||
}
|
||||
|
||||
const [missingStars, missingInstalls, missingDownloads] = await Promise.all([
|
||||
ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_active_stats_stars", (q) =>
|
||||
q.eq("softDeletedAt", undefined).eq("statsStars", undefined),
|
||||
)
|
||||
.first(),
|
||||
ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_active_stats_installs_all_time", (q) =>
|
||||
q.eq("softDeletedAt", undefined).eq("statsInstallsAllTime", undefined),
|
||||
)
|
||||
.first(),
|
||||
ctx.db
|
||||
.query("skillSearchDigest")
|
||||
.withIndex("by_active_stats_downloads", (q) =>
|
||||
q.eq("softDeletedAt", undefined).eq("statsDownloads", undefined),
|
||||
)
|
||||
.first(),
|
||||
]);
|
||||
return Boolean(missingStars || missingInstalls || missingDownloads);
|
||||
}
|
||||
|
||||
function readDigestRankStat(
|
||||
digest: Doc<"skillSearchDigest">,
|
||||
field: "downloads" | "stars" | "installsAllTime",
|
||||
): number {
|
||||
if (field === "downloads") return digest.statsDownloads ?? digest.stats.downloads ?? 0;
|
||||
if (field === "stars") return digest.statsStars ?? digest.stats.stars ?? 0;
|
||||
return digest.statsInstallsAllTime ?? digest.stats.installsAllTime ?? 0;
|
||||
}
|
||||
|
||||
/** Fetch highlighted skills via the skillBadges index, then sort in JS. */
|
||||
async function fetchHighlightedPage(
|
||||
@@ -5762,11 +5924,24 @@ async function fetchHighlightedPage(
|
||||
digests.sort((a, b) => {
|
||||
switch (opts.sort) {
|
||||
case "downloads":
|
||||
return ((a.statsDownloads ?? 0) - (b.statsDownloads ?? 0)) * multiplier;
|
||||
return (
|
||||
(readDigestRankStat(a, "downloads") - readDigestRankStat(b, "downloads")) * multiplier
|
||||
);
|
||||
case "recommended":
|
||||
return (
|
||||
(readDigestRankStat(a, "stars") - readDigestRankStat(b, "stars")) * multiplier ||
|
||||
(readDigestRankStat(a, "installsAllTime") - readDigestRankStat(b, "installsAllTime")) *
|
||||
multiplier ||
|
||||
(readDigestRankStat(a, "downloads") - readDigestRankStat(b, "downloads")) * multiplier ||
|
||||
(a.updatedAt - b.updatedAt) * multiplier
|
||||
);
|
||||
case "stars":
|
||||
return ((a.statsStars ?? 0) - (b.statsStars ?? 0)) * multiplier;
|
||||
return (readDigestRankStat(a, "stars") - readDigestRankStat(b, "stars")) * multiplier;
|
||||
case "installs":
|
||||
return ((a.statsInstallsAllTime ?? 0) - (b.statsInstallsAllTime ?? 0)) * multiplier;
|
||||
return (
|
||||
(readDigestRankStat(a, "installsAllTime") - readDigestRankStat(b, "installsAllTime")) *
|
||||
multiplier
|
||||
);
|
||||
case "updated":
|
||||
return (a.updatedAt - b.updatedAt) * multiplier;
|
||||
case "name":
|
||||
@@ -10971,3 +11146,9 @@ export const backfillLatestVersionSummaryApiKeyRequiredInternal = internalMutati
|
||||
return { scanned, updated, rebuiltSummary, skippedNoLatest, skippedAlreadyMatches };
|
||||
},
|
||||
});
|
||||
|
||||
export const __test = {
|
||||
normalizePublicListSort,
|
||||
resolveRecommendedPublicListSort,
|
||||
resolvePublicListDir,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { consumePackagePublishUploadTicketInternal } from "./uploads";
|
||||
|
||||
type ConsumeArgs = {
|
||||
uploadTicket: string;
|
||||
storageId: string;
|
||||
auth: { kind: "user"; userId: string } | { kind: "github-actions"; publishTokenId: string };
|
||||
};
|
||||
|
||||
type WrappedHandler<TArgs> = {
|
||||
_handler: (ctx: unknown, args: TArgs) => Promise<void>;
|
||||
};
|
||||
|
||||
const consumeHandler = (
|
||||
consumePackagePublishUploadTicketInternal as unknown as WrappedHandler<ConsumeArgs>
|
||||
)._handler;
|
||||
|
||||
function makeCtx(ticket: Record<string, unknown> | null, storage: Record<string, unknown> | null) {
|
||||
return {
|
||||
db: {
|
||||
get: vi.fn(async () => ticket),
|
||||
insert: vi.fn(),
|
||||
normalizeId: vi.fn(),
|
||||
patch: vi.fn(),
|
||||
query: vi.fn(),
|
||||
replace: vi.fn(),
|
||||
delete: vi.fn(),
|
||||
system: {
|
||||
get: vi.fn(async () => storage),
|
||||
query: vi.fn(),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("package publish upload tickets", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("consumes a fresh upload ticket for the same user", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const ctx = makeCtx(
|
||||
{
|
||||
_id: "packagePublishUploadTickets:1",
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 10_000,
|
||||
},
|
||||
{ _id: "storage:1", _creationTime: 1_500 },
|
||||
);
|
||||
|
||||
await consumeHandler(ctx, {
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
auth: { kind: "user", userId: "users:1" },
|
||||
});
|
||||
|
||||
expect(ctx.db.system.get).toHaveBeenCalledWith("_storage", "storage:1");
|
||||
expect(ctx.db.patch).toHaveBeenCalledWith("packagePublishUploadTickets:1", {
|
||||
usedAt: 2_000,
|
||||
storageId: "storage:1",
|
||||
});
|
||||
});
|
||||
|
||||
it("allows retrying a used upload ticket for the same user and storage id", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(3_000);
|
||||
const ctx = makeCtx(
|
||||
{
|
||||
_id: "packagePublishUploadTickets:1",
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 10_000,
|
||||
usedAt: 2_000,
|
||||
storageId: "storage:1",
|
||||
},
|
||||
{ _id: "storage:1", _creationTime: 1_500 },
|
||||
);
|
||||
|
||||
await consumeHandler(ctx, {
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
auth: { kind: "user", userId: "users:1" },
|
||||
});
|
||||
|
||||
expect(ctx.db.patch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects upload tickets from another auth context", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const ctx = makeCtx(
|
||||
{
|
||||
_id: "packagePublishUploadTickets:1",
|
||||
kind: "user",
|
||||
userId: "users:1",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 10_000,
|
||||
},
|
||||
{ _id: "storage:1", _creationTime: 1_500 },
|
||||
);
|
||||
|
||||
await expect(
|
||||
consumeHandler(ctx, {
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
auth: { kind: "user", userId: "users:2" },
|
||||
}),
|
||||
).rejects.toThrow("Package tarball upload ticket does not match this publish token");
|
||||
|
||||
expect(ctx.db.patch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects storage created before the upload ticket", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000);
|
||||
const ctx = makeCtx(
|
||||
{
|
||||
_id: "packagePublishUploadTickets:1",
|
||||
kind: "github-actions",
|
||||
publishTokenId: "packagePublishTokens:1",
|
||||
createdAt: 1_000,
|
||||
expiresAt: 10_000,
|
||||
},
|
||||
{ _id: "storage:1", _creationTime: 999 },
|
||||
);
|
||||
|
||||
await expect(
|
||||
consumeHandler(ctx, {
|
||||
uploadTicket: "packagePublishUploadTickets:1",
|
||||
storageId: "storage:1",
|
||||
auth: { kind: "github-actions", publishTokenId: "packagePublishTokens:1" },
|
||||
}),
|
||||
).rejects.toThrow("Package tarball upload must be created after its upload ticket");
|
||||
|
||||
expect(ctx.db.patch).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
+74
-2
@@ -2,6 +2,8 @@ import { v } from "convex/values";
|
||||
import { internalMutation, mutation } from "./functions";
|
||||
import { requireUser } from "./lib/access";
|
||||
|
||||
const PACKAGE_PUBLISH_UPLOAD_TICKET_TTL_MS = 15 * 60_000;
|
||||
|
||||
export const generateUploadUrl = mutation({
|
||||
args: {},
|
||||
handler: async (ctx) => {
|
||||
@@ -10,11 +12,81 @@ export const generateUploadUrl = mutation({
|
||||
},
|
||||
});
|
||||
|
||||
export const generateUploadUrlForUserInternal = internalMutation({
|
||||
export const createPackagePublishUploadForUserInternal = internalMutation({
|
||||
args: { userId: v.id("users") },
|
||||
handler: async (ctx, args) => {
|
||||
const user = await ctx.db.get(args.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt) throw new Error("User not found");
|
||||
return ctx.storage.generateUploadUrl();
|
||||
const now = Date.now();
|
||||
const uploadTicket = await ctx.db.insert("packagePublishUploadTickets", {
|
||||
kind: "user",
|
||||
userId: args.userId,
|
||||
createdAt: now,
|
||||
expiresAt: now + PACKAGE_PUBLISH_UPLOAD_TICKET_TTL_MS,
|
||||
});
|
||||
const uploadUrl = await ctx.storage.generateUploadUrl();
|
||||
return { uploadUrl, uploadTicket };
|
||||
},
|
||||
});
|
||||
|
||||
export const createPackagePublishUploadForTokenInternal = internalMutation({
|
||||
args: { publishTokenId: v.id("packagePublishTokens") },
|
||||
handler: async (ctx, args) => {
|
||||
const publishToken = await ctx.db.get(args.publishTokenId);
|
||||
const now = Date.now();
|
||||
if (!publishToken || publishToken.revokedAt || publishToken.expiresAt <= now) {
|
||||
throw new Error("Trusted publish token is missing or expired");
|
||||
}
|
||||
const uploadTicket = await ctx.db.insert("packagePublishUploadTickets", {
|
||||
kind: "github-actions",
|
||||
publishTokenId: args.publishTokenId,
|
||||
createdAt: now,
|
||||
expiresAt: now + PACKAGE_PUBLISH_UPLOAD_TICKET_TTL_MS,
|
||||
});
|
||||
const uploadUrl = await ctx.storage.generateUploadUrl();
|
||||
return { uploadUrl, uploadTicket };
|
||||
},
|
||||
});
|
||||
|
||||
export const consumePackagePublishUploadTicketInternal = internalMutation({
|
||||
args: {
|
||||
uploadTicket: v.id("packagePublishUploadTickets"),
|
||||
storageId: v.id("_storage"),
|
||||
auth: v.union(
|
||||
v.object({ kind: v.literal("user"), userId: v.id("users") }),
|
||||
v.object({
|
||||
kind: v.literal("github-actions"),
|
||||
publishTokenId: v.id("packagePublishTokens"),
|
||||
}),
|
||||
),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const ticket = await ctx.db.get(args.uploadTicket);
|
||||
const now = Date.now();
|
||||
if (!ticket || ticket.expiresAt <= now) {
|
||||
throw new Error("Package tarball upload ticket is missing or expired");
|
||||
}
|
||||
if (
|
||||
args.auth.kind === "user"
|
||||
? ticket.kind !== "user" || ticket.userId !== args.auth.userId
|
||||
: ticket.kind !== "github-actions" || ticket.publishTokenId !== args.auth.publishTokenId
|
||||
) {
|
||||
throw new Error("Package tarball upload ticket does not match this publish token");
|
||||
}
|
||||
if (ticket.usedAt) {
|
||||
if (ticket.storageId === args.storageId) return;
|
||||
throw new Error("Package tarball upload ticket was already used");
|
||||
}
|
||||
|
||||
const metadata = await ctx.db.system.get("_storage", args.storageId);
|
||||
if (!metadata) throw new Error("Package tarball upload no longer exists");
|
||||
if (metadata._creationTime < ticket.createdAt) {
|
||||
throw new Error("Package tarball upload must be created after its upload ticket");
|
||||
}
|
||||
|
||||
await ctx.db.patch(ticket._id, {
|
||||
usedAt: now,
|
||||
storageId: args.storageId,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
+2
-1
@@ -83,12 +83,13 @@ Public read:
|
||||
- Optional filters: `highlightedOnly=true`, `nonSuspiciousOnly=true`
|
||||
- Legacy alias: `nonSuspicious=true`
|
||||
- `GET /api/v1/skills?limit=&cursor=&sort=`
|
||||
- `sort`: `updated` (default), `createdAt` (`newest`), `downloads`, `stars` (`rating`), `installsCurrent` (`installs`), `installsAllTime`, `trending`
|
||||
- `sort`: `updated` (default), `recommended` (`default`), `createdAt` (`newest`), `downloads`, `stars` (`rating`), `installsCurrent` (`installs`), `installsAllTime`, `trending`
|
||||
- Invalid `sort` values return `400`
|
||||
- `cursor` applies to non-`trending` sorts
|
||||
- Optional filter: `nonSuspiciousOnly=true`
|
||||
- Legacy alias: `nonSuspicious=true`
|
||||
- With `nonSuspiciousOnly=true`, cursor-based pages may contain fewer than `limit` items; use `nextCursor` to continue.
|
||||
- `recommended` ranks by stars, then all-time installs, then downloads, then `updatedAt`.
|
||||
- `GET /api/v1/skills/{slug}`
|
||||
- `GET /api/v1/skills/{slug}/moderation`
|
||||
- `GET /api/v1/skills/{slug}/versions?limit=&cursor=`
|
||||
|
||||
@@ -72,6 +72,12 @@ Override the path with:
|
||||
export CLAWHUB_CONFIG_PATH=/path/to/config.json
|
||||
```
|
||||
|
||||
Print the stored token for CI setup with:
|
||||
|
||||
```bash
|
||||
clawhub token
|
||||
```
|
||||
|
||||
## Revocation
|
||||
|
||||
You can revoke API tokens in the ClawHub web UI.
|
||||
|
||||
+66
@@ -89,6 +89,11 @@ Stores your API token + cached registry URL.
|
||||
|
||||
- Verifies the stored token via `/api/v1/whoami`.
|
||||
|
||||
### `token`
|
||||
|
||||
- Prints the stored API token to stdout.
|
||||
- Useful for piping a local login token into CI secret setup commands.
|
||||
|
||||
### `star <slug>` / `unstar <slug>`
|
||||
|
||||
- Adds/removes a skill from your highlights.
|
||||
@@ -184,6 +189,64 @@ Stores your API token + cached registry URL.
|
||||
clawhub skill publish ./my-skill --version 1.0.0
|
||||
```
|
||||
|
||||
### `scan [path]`
|
||||
|
||||
- Requires `clawhub login`.
|
||||
- Runs ClawHub ClawScan through `POST /api/v1/skills/-/scan`, then polls until the scan is terminal.
|
||||
- Local path scans are always ephemeral. They upload the local skill bundle for scanning, print the security report, and never create or update a published skill/version.
|
||||
- Published scans require ownership or publisher management access. Moderators/admins can use the same backend through `clawhub-mod`.
|
||||
- `--update` is valid only with `--slug`; it writes successful published scan results back to the selected version.
|
||||
- `--output <file.zip>` downloads the full report archive with `manifest.json`, `clawscan.json`, `skillspector.json`, `static-analysis.json`, `virustotal.json`, and `README.md`.
|
||||
- `--json` prints the full poll response for automation.
|
||||
|
||||
```bash
|
||||
clawhub scan ./my-skill
|
||||
clawhub scan ./my-skill --output report.zip
|
||||
clawhub scan --slug gifgrep
|
||||
clawhub scan --slug gifgrep --version 1.2.3
|
||||
clawhub scan --slug gifgrep --update --output report.zip
|
||||
```
|
||||
|
||||
#### GitHub Actions
|
||||
|
||||
ClawHub ships an official reusable workflow at
|
||||
[`/.github/workflows/skill-publish.yml`](../.github/workflows/skill-publish.yml)
|
||||
for skill repos and catalog repos.
|
||||
|
||||
Typical catalog setup:
|
||||
|
||||
```yaml
|
||||
name: Skill Publish
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
dry-run:
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: openclaw/clawhub/.github/workflows/skill-publish.yml@v1
|
||||
with:
|
||||
owner: nvidia
|
||||
dry_run: true
|
||||
|
||||
publish:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
uses: openclaw/clawhub/.github/workflows/skill-publish.yml@v1
|
||||
with:
|
||||
owner: nvidia
|
||||
dry_run: false
|
||||
secrets:
|
||||
clawhub_token: ${{ secrets.CLAWHUB_TOKEN }}
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `root` defaults to `skills` for catalog repos.
|
||||
- Pass `skill_path: skills/review-helper` to process one skill folder.
|
||||
- `owner` maps to the CLI `--owner` flag; omit it to publish as the authenticated user.
|
||||
- V1 skill publishing uses `clawhub_token`; GitHub OIDC trusted publishing is package-only for now.
|
||||
|
||||
### `delete <slug>`
|
||||
|
||||
- Soft-delete a skill (owner, moderator, or admin).
|
||||
@@ -609,10 +672,13 @@ Notes:
|
||||
- `--root <dir...>` extra scan roots
|
||||
- `--all` upload without prompting
|
||||
- `--dry-run` show plan only
|
||||
- `--json` machine-readable summary for CI
|
||||
- `--owner <handle>` publish under a user or org publisher
|
||||
- `--bump patch|minor|major` (default: patch)
|
||||
- `--changelog <text>` (non-interactive)
|
||||
- `--tags a,b,c` (default: latest)
|
||||
- `--concurrency <n>` (default: 4)
|
||||
- `--source-repo <repo>`, `--source-commit <sha>`, `--source-ref <ref>` for GitHub provenance
|
||||
|
||||
Telemetry:
|
||||
|
||||
|
||||
+68
-5
@@ -123,10 +123,10 @@ Response:
|
||||
|
||||
Notes:
|
||||
|
||||
- Results are returned in relevance order (embedding similarity + exact slug/name token boosts + popularity prior from downloads).
|
||||
- Results are returned in relevance order (embedding similarity + exact slug/name token boosts + a small popularity prior from stars, all-time installs, and downloads).
|
||||
- Relevance is stronger than popularity. A precise slug or display-name token match can outrank a looser match with many more downloads.
|
||||
- ASCII text is tokenized on word and punctuation boundaries. For example, `personal-map` contains a standalone `map` token, while `amap-jsapi-skill` contains `amap`, `jsapi`, and `skill`; searching for `map` therefore gives `personal-map` a stronger lexical match than `amap-jsapi-skill`.
|
||||
- Downloads are used as a small log-scaled prior and tie-breaker, not as the primary ranking signal. High-download skills can rank lower when the query text is a weaker match.
|
||||
- Popularity is log-scaled and capped. Stars carry the strongest weight, all-time installs carry a smaller weight, and downloads are only a tiny fallback signal. High-download skills can rank lower when the query text is a weaker match.
|
||||
- Suspicious or hidden moderation state can remove a skill from public search depending on caller filters and current moderation status.
|
||||
|
||||
Publisher discoverability guidance:
|
||||
@@ -142,7 +142,7 @@ Query params:
|
||||
|
||||
- `limit` (optional): integer (1–200)
|
||||
- `cursor` (optional): pagination cursor for any non-`trending` sort
|
||||
- `sort` (optional): `updated` (default), `createdAt` (alias: `newest`), `downloads`, `stars` (alias: `rating`), `installsCurrent` (alias: `installs`), `installsAllTime`, `trending`
|
||||
- `sort` (optional): `updated` (default), `recommended` (alias: `default`), `createdAt` (alias: `newest`), `downloads`, `stars` (alias: `rating`), `installsCurrent` (alias: `installs`), `installsAllTime`, `trending`
|
||||
- `nonSuspiciousOnly` (optional): `true` to hide suspicious (`flagged.suspicious`) skills
|
||||
- `nonSuspicious` (optional): legacy alias for `nonSuspiciousOnly`
|
||||
|
||||
@@ -150,6 +150,7 @@ Invalid `sort` values return `400`.
|
||||
|
||||
Notes:
|
||||
|
||||
- `recommended` ranks by stars, then all-time installs, then downloads, then `updatedAt`.
|
||||
- `trending` ranks by installs in the last 7 days (telemetry-based).
|
||||
- `createdAt` is stable for new-skill crawls; `updated` changes when existing skills are republished.
|
||||
- When `nonSuspiciousOnly=true`, cursor-based sorts may return fewer than `limit` items on a page because suspicious skills are filtered after page retrieval.
|
||||
@@ -368,6 +369,56 @@ Notes:
|
||||
- `moderation` is a current skill-level moderation snapshot derived from the latest version.
|
||||
- When querying a historical version, check `moderation.matchesRequestedVersion` and `moderation.sourceVersion` before treating `moderation` and `security` as the same version context.
|
||||
|
||||
### `POST /api/v1/skills/-/scan`
|
||||
|
||||
Authenticated submit endpoint for new ClawScan jobs.
|
||||
|
||||
Local upload scans use `multipart/form-data`:
|
||||
|
||||
- `payload`: JSON string, usually `{ "source": { "kind": "upload" }, "update": false }`
|
||||
- `files`: repeated local skill files
|
||||
|
||||
Published scans use JSON:
|
||||
|
||||
```json
|
||||
{
|
||||
"source": { "kind": "published", "slug": "gifgrep", "version": "1.2.3" },
|
||||
"update": false
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- Local upload scans require auth but are ephemeral. They never mutate public skill, version, moderation, or trust state.
|
||||
- Scan request payloads and downloadable reports expire from the scan-request store after the retention window.
|
||||
- Local upload scans reject `update: true`.
|
||||
- Published scans require owner/publisher management access, or platform moderator/admin authority.
|
||||
- Published scans write back only when `update: true` and the scan completes successfully.
|
||||
- Response is `202` with `{ "ok": true, "scanId": "...", "jobId": "...", "status": "queued", "sourceKind": "upload|published", "update": false }`.
|
||||
|
||||
### `GET /api/v1/skills/-/scan/{scanId}`
|
||||
|
||||
Authenticated poll endpoint for a submitted scan.
|
||||
|
||||
- Returns queued/running/succeeded/failed status.
|
||||
- When available, `report` contains `clawscan`, `skillspector`, `staticAnalysis`, and `virustotal` sections.
|
||||
- Failed scan jobs return `status: "failed"` with `lastError`.
|
||||
|
||||
### `GET /api/v1/skills/-/scan/{scanId}/download`
|
||||
|
||||
Authenticated report archive endpoint.
|
||||
|
||||
- Requires a succeeded scan; non-terminal scans return `409`.
|
||||
- Returns a ZIP with `manifest.json`, `clawscan.json`, `skillspector.json`, `static-analysis.json`, `virustotal.json`, and `README.md`.
|
||||
|
||||
### `POST /api/v1/skills/-/scan/batch`
|
||||
|
||||
Admin-only canonical batch rescan route. It accepts the same payload shape as legacy `POST /api/v1/skills/-/rescan-batch`.
|
||||
|
||||
### `POST /api/v1/skills/-/scan/batch/status`
|
||||
|
||||
Admin-only canonical batch status route. It accepts `{ "jobIds": ["..."] }` and returns the same aggregate counters as legacy `POST /api/v1/skills/-/rescan-batch/status`.
|
||||
|
||||
### `GET /api/v1/skills/{slug}/verify`
|
||||
|
||||
Returns the Skill Card verification envelope used by `clawhub skill verify`.
|
||||
@@ -1224,8 +1275,16 @@ Publishes a new version.
|
||||
Publishes a code-plugin or bundle-plugin release.
|
||||
|
||||
- Requires Bearer token auth.
|
||||
- Preferred: `multipart/form-data` with `payload` JSON + `files[]` blobs.
|
||||
- JSON body with `files` (storageId-based) is also accepted.
|
||||
- Requires `multipart/form-data`.
|
||||
- Allowed form fields are `payload`, repeated `files` blobs, or one `clawpack`
|
||||
tarball reference. `clawpack` may be a `.tgz` blob or a storage id returned by
|
||||
the upload-url flow. Staged storage-id publishes must also include the
|
||||
`clawpackUploadTicket` returned with that upload URL.
|
||||
- Use either `files` or `clawpack`, never both in the same request.
|
||||
- JSON bodies and caller-supplied `payload.files` / `payload.artifact`
|
||||
metadata are rejected.
|
||||
- Direct multipart publish requests are capped at 18MB. ClawPack tarballs may
|
||||
use the upload-url flow up to the 120MB tarball cap.
|
||||
- Optional payload field: `ownerHandle`. When present, only admins may publish on behalf of that owner.
|
||||
|
||||
Validation highlights:
|
||||
@@ -1478,6 +1537,10 @@ Still supported for older CLI versions:
|
||||
|
||||
See `DEPRECATIONS.md` for removal plan.
|
||||
|
||||
`POST /api/cli/upload-url` returns `uploadUrl` and `uploadTicket`. Package
|
||||
publishes that stage a ClawPack tarball must send the resulting storage id as
|
||||
`clawpack` and the returned ticket as `clawpackUploadTicket`.
|
||||
|
||||
## Registry discovery (`/.well-known/clawhub.json`)
|
||||
|
||||
The CLI can discover registry/auth settings from the site:
|
||||
|
||||
+116
-56
@@ -8,64 +8,136 @@ read_when:
|
||||
|
||||
# Publishing
|
||||
|
||||
ClawHub publishing is owner-scoped: every publish targets a publisher, and the
|
||||
server decides whether the signed-in user is allowed to publish there.
|
||||
Publishing sends a skill folder or plugin package to ClawHub under the owner you
|
||||
choose. ClawHub checks that your token can publish for that owner, validates the
|
||||
metadata, name, version, files, and source information, then stores the release
|
||||
and starts automated security checks.
|
||||
|
||||
## Owners
|
||||
|
||||
An owner is a ClawHub publisher handle, such as `@alice` or `@openclaw`.
|
||||
Personal owners are created for users. Org owners can have multiple members.
|
||||
|
||||
When you publish, you either use your personal owner or choose an org owner
|
||||
where you have publisher access.
|
||||
|
||||
## Official
|
||||
|
||||
Official is a ClawHub policy flag derived from the hard-coded `openclaw`
|
||||
organization. The `openclaw` org publisher is Official, and personal publishers
|
||||
for `openclaw` org members are Official while that membership exists.
|
||||
|
||||
Official does not come from uploaded skill or package metadata, and org
|
||||
membership outside the `openclaw` org does not make a personal publisher
|
||||
Official.
|
||||
|
||||
The same policy shows as an `Official` badge on publisher/profile UI. New
|
||||
public packages from an Official publisher use the `official` channel; private
|
||||
packages stay private.
|
||||
|
||||
`trustedPublisher` is an internal automated-publish permission. It does not make
|
||||
a publisher or package Official.
|
||||
If validation fails, nothing is published. New releases may also stay out of
|
||||
normal install and download surfaces until review finishes.
|
||||
|
||||
## Skills
|
||||
|
||||
Skills are published from a skill folder. The public page is:
|
||||
The simplest publishing path is the CLI. Sign in, preview the sync plan, then
|
||||
publish the new or changed skills:
|
||||
|
||||
```text
|
||||
https://clawhub.ai/<owner>/<slug>
|
||||
```bash
|
||||
clawhub login
|
||||
clawhub sync --dry-run --owner <owner>
|
||||
clawhub sync --all --owner <owner>
|
||||
```
|
||||
|
||||
Example:
|
||||
`sync` scans for folders containing `SKILL.md` and compares them with ClawHub.
|
||||
When you run it without `--dry-run`, it publishes anything new or changed.
|
||||
|
||||
```text
|
||||
https://clawhub.ai/alice/review-helper
|
||||
Use `--dry-run` first to see the plan without uploading.
|
||||
|
||||
Use `--owner <handle>` when publishing to an org owner. Omit it to publish as
|
||||
the authenticated user.
|
||||
|
||||
### GitHub Actions for Skills
|
||||
|
||||
If you want to run skill publishing from CI, call ClawHub's reusable
|
||||
[`skill-publish.yml` workflow](https://github.com/openclaw/clawhub/blob/main/.github/workflows/skill-publish.yml)
|
||||
from a small workflow in your repo.
|
||||
|
||||
The example below is shaped for a catalog repo: operators choose whether to
|
||||
preview the full catalog, publish one skill folder, or publish the whole
|
||||
catalog.
|
||||
|
||||
```yaml
|
||||
name: Publish Skills to ClawHub
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: What to run.
|
||||
type: choice
|
||||
required: true
|
||||
default: dry-run
|
||||
options:
|
||||
- dry-run
|
||||
- publish-single
|
||||
- publish-catalog
|
||||
skill_path:
|
||||
description: Skill folder for publish-single, for example skills/<slug>.
|
||||
type: string
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
validate-single:
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.mode == 'publish-single'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Validate single-skill input
|
||||
env:
|
||||
SKILL_PATH: ${{ inputs.skill_path }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ -z "${SKILL_PATH}" ]]; then
|
||||
echo "::error::skill_path is required when mode is publish-single."
|
||||
exit 1
|
||||
fi
|
||||
case "${SKILL_PATH}" in
|
||||
skills/*) ;;
|
||||
*)
|
||||
echo "::error::skill_path must point under skills/, for example skills/<slug>."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
dry-run:
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.mode == 'dry-run'
|
||||
uses: openclaw/clawhub/.github/workflows/skill-publish.yml@main
|
||||
with:
|
||||
owner: <owner>
|
||||
dry_run: true
|
||||
secrets:
|
||||
clawhub_token: ${{ secrets.CLAWHUB_TOKEN }}
|
||||
|
||||
publish-single:
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.mode == 'publish-single'
|
||||
needs: validate-single
|
||||
uses: openclaw/clawhub/.github/workflows/skill-publish.yml@main
|
||||
with:
|
||||
owner: <owner>
|
||||
skill_path: ${{ inputs.skill_path }}
|
||||
dry_run: false
|
||||
secrets:
|
||||
clawhub_token: ${{ secrets.CLAWHUB_TOKEN }}
|
||||
|
||||
publish-catalog:
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.mode == 'publish-catalog'
|
||||
uses: openclaw/clawhub/.github/workflows/skill-publish.yml@main
|
||||
with:
|
||||
owner: <owner>
|
||||
dry_run: false
|
||||
secrets:
|
||||
clawhub_token: ${{ secrets.CLAWHUB_TOKEN }}
|
||||
```
|
||||
|
||||
The publish request includes the selected owner, slug, version, changelog, and
|
||||
files. The server verifies that the actor can publish as that owner before it
|
||||
creates the release.
|
||||
Replace `<owner>` with your ClawHub owner handle. The called workflow defaults to
|
||||
scanning `skills/`; pass `skill_path` only when you want to process one folder.
|
||||
|
||||
To move an existing skill to another owner while publishing a new version, choose
|
||||
the new owner and explicitly confirm the ownership move. In the CLI/API, pass the
|
||||
target owner plus the migration opt-in:
|
||||
Before running a real publish, sign in as a ClawHub user that can publish to the
|
||||
selected owner, then store the current CLI token as a `CLAWHUB_TOKEN` repository
|
||||
secret:
|
||||
|
||||
```sh
|
||||
clawhub skill publish ./review-helper --owner openclaw --migrate-owner --version 1.2.0
|
||||
```bash
|
||||
clawhub login --label "Skills GitHub Actions"
|
||||
gh secret set CLAWHUB_TOKEN \
|
||||
--repo OWNER/REPO \
|
||||
--body "$(clawhub token)"
|
||||
```
|
||||
|
||||
Skill owner migration requires admin or owner access on both the current owner
|
||||
and the destination owner. It preserves the skill, version history, stats,
|
||||
comments, forks, aliases, and audit trail; old owner URLs continue through the
|
||||
alias/redirect path.
|
||||
Start with `dry-run`, then publish one skill with `publish-single`, and only then
|
||||
use `publish-catalog` for the full catalog.
|
||||
|
||||
## Plugins
|
||||
|
||||
@@ -94,18 +166,6 @@ not control.
|
||||
- Expect new releases to stay out of public install surfaces until automated
|
||||
security checks and verification finish.
|
||||
|
||||
## Release Flow
|
||||
|
||||
1. The UI, CLI, or GitHub workflow gathers package metadata and files.
|
||||
2. The publish request is sent to ClawHub with the selected owner.
|
||||
3. The server validates owner permissions, package scope, package name, version,
|
||||
file limits, and source metadata.
|
||||
4. ClawHub stores the release and starts automated security checks.
|
||||
5. New releases are hidden from normal install/download surfaces until review
|
||||
and verification finish.
|
||||
|
||||
If validation fails, the release is not created.
|
||||
|
||||
## FAQ
|
||||
|
||||
### Package scope must match selected owner
|
||||
|
||||
@@ -126,6 +126,18 @@ clawhub sync --all --dry-run
|
||||
clawhub sync --all
|
||||
```
|
||||
|
||||
For catalog repos, ClawHub also provides a reusable GitHub workflow. By
|
||||
default it scans `skills/`; pass `skill_path` to process one folder.
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
dry-run:
|
||||
uses: openclaw/clawhub/.github/workflows/skill-publish.yml@v1
|
||||
with:
|
||||
owner: nvidia
|
||||
dry_run: true
|
||||
```
|
||||
|
||||
When you are signed in, `sync` may also send a minimal install snapshot for
|
||||
aggregate install counts. See [Telemetry](./telemetry.md) for what is reported
|
||||
and how to opt out.
|
||||
|
||||
@@ -152,12 +152,75 @@ function extractLastJsonObject(output: string) {
|
||||
throw new Error(`No JSON object in convex run output:\n${output}`);
|
||||
}
|
||||
|
||||
export async function fetchWithTimeout(input: RequestInfo | URL, init?: RequestInit) {
|
||||
export async function fetchWithTimeout(
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
timeoutMs: number = REQUEST_TIMEOUT_MS,
|
||||
) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(new Error("Timeout")), REQUEST_TIMEOUT_MS);
|
||||
const timeout = setTimeout(() => controller.abort(new Error("Timeout")), timeoutMs);
|
||||
try {
|
||||
return await fetch(input, { ...init, signal: controller.signal });
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_RATE_LIMIT_WAIT_MS = 15_000;
|
||||
const TRANSIENT_RETRY_DELAY_MS = 1_000;
|
||||
|
||||
function parsePositiveNumber(value: string | null) {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
|
||||
}
|
||||
|
||||
function getRetryDelayMs(response: Response) {
|
||||
const retryAfterSeconds = parsePositiveNumber(response.headers.get("Retry-After"));
|
||||
if (retryAfterSeconds !== null) {
|
||||
return Math.min(retryAfterSeconds * 1000, MAX_RATE_LIMIT_WAIT_MS);
|
||||
}
|
||||
const relativeResetSeconds = parsePositiveNumber(response.headers.get("RateLimit-Reset"));
|
||||
if (relativeResetSeconds !== null) {
|
||||
return Math.min(relativeResetSeconds * 1000, MAX_RATE_LIMIT_WAIT_MS);
|
||||
}
|
||||
const absoluteResetSeconds = parsePositiveNumber(response.headers.get("X-RateLimit-Reset"));
|
||||
if (absoluteResetSeconds !== null) {
|
||||
return Math.min(Math.max(absoluteResetSeconds * 1000 - Date.now(), 0), MAX_RATE_LIMIT_WAIT_MS);
|
||||
}
|
||||
return TRANSIENT_RETRY_DELAY_MS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch with timeout, retrying on transient failures (network abort/timeout,
|
||||
* 429, and 5xx). Used for read-only verification calls against a real registry
|
||||
* where occasional cold starts or rate-limit hits would otherwise flake the
|
||||
* test. Non-transient HTTP responses (e.g. 401/403/404) are returned as-is.
|
||||
*/
|
||||
export async function fetchWithRetry(
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
options: { maxAttempts?: number; timeoutMs?: number } = {},
|
||||
) {
|
||||
const maxAttempts = options.maxAttempts ?? 3;
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
try {
|
||||
const response = await fetchWithTimeout(input, init, options.timeoutMs);
|
||||
if (attempt >= maxAttempts) return response;
|
||||
if (response.status === 429) {
|
||||
await new Promise((resolve) => setTimeout(resolve, getRetryDelayMs(response)));
|
||||
continue;
|
||||
}
|
||||
if (response.status >= 500) {
|
||||
await new Promise((resolve) => setTimeout(resolve, TRANSIENT_RETRY_DELAY_MS * attempt));
|
||||
continue;
|
||||
}
|
||||
return response;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (attempt >= maxAttempts) throw error;
|
||||
await new Promise((resolve) => setTimeout(resolve, TRANSIENT_RETRY_DELAY_MS * attempt));
|
||||
}
|
||||
}
|
||||
throw lastError ?? new Error("fetchWithRetry exhausted attempts");
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import { readGlobalConfig } from "../packages/clawhub/src/config";
|
||||
import {
|
||||
allowLiveMutations,
|
||||
buildE2ESkillMarkdown,
|
||||
fetchWithRetry,
|
||||
fetchWithTimeout,
|
||||
getRegistry,
|
||||
getSite,
|
||||
@@ -114,7 +115,7 @@ describe("permission boundary e2e", () => {
|
||||
] as const;
|
||||
|
||||
for (const testCase of cases) {
|
||||
const response = await fetchWithTimeout(new URL(testCase.path, registry), {
|
||||
const response = await fetchWithRetry(new URL(testCase.path, registry), {
|
||||
method: testCase.method,
|
||||
headers: { Accept: "application/json", "Content-Type": "application/json" },
|
||||
body: "body" in testCase ? JSON.stringify(testCase.body) : undefined,
|
||||
|
||||
@@ -170,12 +170,11 @@ describe("cmdRescanSkill", () => {
|
||||
it("posts a moderator skill rescan request", async () => {
|
||||
httpMocks.apiRequest.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
slug: "markdown2doc",
|
||||
version: "1.0.4",
|
||||
skillId: "skills:1",
|
||||
skillVersionId: "skillVersions:1",
|
||||
scanId: "skillScanRequests:1",
|
||||
jobId: "securityScanJobs:1",
|
||||
alreadyQueued: false,
|
||||
status: "queued",
|
||||
sourceKind: "published",
|
||||
update: true,
|
||||
});
|
||||
|
||||
const result = await cmdRescanSkill(
|
||||
@@ -185,14 +184,17 @@ describe("cmdRescanSkill", () => {
|
||||
false,
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ ok: true, slug: "markdown2doc", version: "1.0.4" });
|
||||
expect(result).toMatchObject({ ok: true, scanId: "skillScanRequests:1", update: true });
|
||||
expect(httpMocks.apiRequest).toHaveBeenCalledWith(
|
||||
"https://clawhub.ai",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
path: "/api/v1/skills/markdown2doc/rescan",
|
||||
path: "/api/v1/skills/-/scan",
|
||||
token: "tkn",
|
||||
body: { version: "1.0.4" },
|
||||
body: {
|
||||
source: { kind: "published", slug: "markdown2doc", version: "1.0.4" },
|
||||
update: true,
|
||||
},
|
||||
}),
|
||||
expect.anything(),
|
||||
);
|
||||
@@ -242,7 +244,7 @@ describe("cmdRescanAllSkills", () => {
|
||||
"https://clawhub.ai",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
path: "/api/v1/skills/-/rescan-batch",
|
||||
path: "/api/v1/skills/-/scan/batch",
|
||||
body: {
|
||||
mode: "all-active-latest",
|
||||
cursor: null,
|
||||
@@ -257,7 +259,7 @@ describe("cmdRescanAllSkills", () => {
|
||||
"https://clawhub.ai",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
path: "/api/v1/skills/-/rescan-batch",
|
||||
path: "/api/v1/skills/-/scan/batch",
|
||||
body: {
|
||||
mode: "all-active-latest",
|
||||
cursor: "cursor-2",
|
||||
@@ -307,7 +309,7 @@ describe("cmdRescanAllSkills", () => {
|
||||
"https://clawhub.ai",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
path: "/api/v1/skills/-/rescan-batch",
|
||||
path: "/api/v1/skills/-/scan/batch",
|
||||
token: "tkn",
|
||||
body: {
|
||||
mode: "all-active-latest",
|
||||
@@ -323,7 +325,7 @@ describe("cmdRescanAllSkills", () => {
|
||||
"https://clawhub.ai",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
path: "/api/v1/skills/-/rescan-batch/status",
|
||||
path: "/api/v1/skills/-/scan/batch/status",
|
||||
token: "tkn",
|
||||
body: { jobIds: ["securityScanJobs:1"] },
|
||||
}),
|
||||
|
||||
@@ -16,10 +16,10 @@ import {
|
||||
ApiV1ReclassifyBanResponseSchema,
|
||||
ApiV1RemediateAutobansResponseSchema,
|
||||
ApiV1SetRoleResponseSchema,
|
||||
ApiV1SkillBulkRescanBatchResponseSchema,
|
||||
ApiV1SkillBulkRescanStatusResponseSchema,
|
||||
ApiV1SkillScanBatchResponseSchema,
|
||||
ApiV1SkillScanBatchStatusResponseSchema,
|
||||
ApiV1SkillScanSubmitResponseSchema,
|
||||
ApiV1SkillRepairVtPendingResponseSchema,
|
||||
ApiV1SkillRescanResponseSchema,
|
||||
ApiV1UnbanUserResponseSchema,
|
||||
ApiV1UserSearchResponseSchema,
|
||||
parseArk,
|
||||
@@ -217,15 +217,22 @@ export async function cmdRescanSkill(
|
||||
registry,
|
||||
{
|
||||
method: "POST",
|
||||
path: `${ApiRoutes.skills}/${encodeURIComponent(slug)}/rescan`,
|
||||
path: ApiRoutes.skillScans,
|
||||
token,
|
||||
body: version ? { version } : {},
|
||||
body: {
|
||||
source: {
|
||||
kind: "published",
|
||||
slug,
|
||||
...(version ? { version } : {}),
|
||||
},
|
||||
update: true,
|
||||
},
|
||||
},
|
||||
ApiV1SkillRescanResponseSchema,
|
||||
ApiV1SkillScanSubmitResponseSchema,
|
||||
);
|
||||
const parsed = parseArk(ApiV1SkillRescanResponseSchema, result, "Skill rescan response");
|
||||
const parsed = parseArk(ApiV1SkillScanSubmitResponseSchema, result, "Skill rescan response");
|
||||
spinner?.succeed(
|
||||
`OK. Queued ClawScan for ${parsed.slug}@${parsed.version} (${parsed.alreadyQueued ? "existing job" : "new job"}).`,
|
||||
`OK. Queued ClawScan for ${slug}${version ? `@${version}` : ""} (${parsed.alreadyQueued ? "existing job" : "new job"}).`,
|
||||
);
|
||||
if (options.json) {
|
||||
process.stdout.write(`${JSON.stringify(parsed, null, 2)}\n`);
|
||||
@@ -283,7 +290,7 @@ export async function cmdRescanAllSkills(
|
||||
registry,
|
||||
{
|
||||
method: "POST",
|
||||
path: `${ApiRoutes.skills}/-/rescan-batch`,
|
||||
path: `${ApiRoutes.skillScans}/batch`,
|
||||
token,
|
||||
body: {
|
||||
mode: "all-active-latest",
|
||||
@@ -292,10 +299,10 @@ export async function cmdRescanAllSkills(
|
||||
dryRun: options.dryRun === true,
|
||||
},
|
||||
},
|
||||
ApiV1SkillBulkRescanBatchResponseSchema,
|
||||
ApiV1SkillScanBatchResponseSchema,
|
||||
);
|
||||
const batch = parseArk(
|
||||
ApiV1SkillBulkRescanBatchResponseSchema,
|
||||
ApiV1SkillScanBatchResponseSchema,
|
||||
result,
|
||||
"Bulk skill rescan batch response",
|
||||
);
|
||||
@@ -473,14 +480,14 @@ async function pollBulkRescanStatus(
|
||||
registry,
|
||||
{
|
||||
method: "POST",
|
||||
path: `${ApiRoutes.skills}/-/rescan-batch/status`,
|
||||
path: `${ApiRoutes.skillScans}/batch/status`,
|
||||
token,
|
||||
body: { jobIds },
|
||||
},
|
||||
ApiV1SkillBulkRescanStatusResponseSchema,
|
||||
ApiV1SkillScanBatchStatusResponseSchema,
|
||||
);
|
||||
const status = parseArk(
|
||||
ApiV1SkillBulkRescanStatusResponseSchema,
|
||||
ApiV1SkillScanBatchStatusResponseSchema,
|
||||
result,
|
||||
"Bulk skill rescan status response",
|
||||
);
|
||||
|
||||
@@ -24,6 +24,9 @@ clawhub login --device
|
||||
|
||||
# or (token paste / headless)
|
||||
clawhub login --token clh_...
|
||||
|
||||
# print the stored token for CI setup
|
||||
clawhub token
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "clawhub",
|
||||
"version": "0.18.0",
|
||||
"version": "0.19.0",
|
||||
"description": "ClawHub CLI \\u2014 install, update, search, and publish skills plus OpenClaw packages.",
|
||||
"homepage": "https://clawhub.ai",
|
||||
"bugs": {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { join, resolve } from "node:path";
|
||||
import { Command } from "commander";
|
||||
import { getCliBuildLabel, getCliVersion } from "./cli/buildInfo.js";
|
||||
import { resolveClawdbotDefaultWorkspace } from "./cli/clawdbotConfig.js";
|
||||
import { cmdLoginFlow, cmdLogout, cmdWhoami } from "./cli/commands/auth.js";
|
||||
import { cmdLoginFlow, cmdLogout, cmdToken, cmdWhoami } from "./cli/commands/auth.js";
|
||||
import {
|
||||
cmdDeleteSkill,
|
||||
cmdHideSkill,
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
} from "./cli/commands/packages.js";
|
||||
import { cmdPublish } from "./cli/commands/publish.js";
|
||||
import { cmdCreatePublisher } from "./cli/commands/publishers.js";
|
||||
import { cmdScan } from "./cli/commands/scan.js";
|
||||
import {
|
||||
cmdExplore,
|
||||
cmdInstall,
|
||||
@@ -219,6 +220,12 @@ registerCommand(program, ["whoami"])
|
||||
await cmdWhoami(opts);
|
||||
});
|
||||
|
||||
registerCommand(program, ["token"])
|
||||
.description("Print stored API token")
|
||||
.action(async () => {
|
||||
await cmdToken();
|
||||
});
|
||||
|
||||
const auth = registerCommandGroup(program, ["auth"])
|
||||
.description("Authentication commands")
|
||||
.showHelpAfterError()
|
||||
@@ -365,6 +372,19 @@ registerCommand(program, ["publish"])
|
||||
await cmdPublish(opts, folder, options);
|
||||
});
|
||||
|
||||
registerCommand(program, ["scan"])
|
||||
.description("Run ClawScan on a local skill bundle or one of your published skills")
|
||||
.argument("[path]", "Local skill folder path")
|
||||
.option("--slug <slug>", "Published skill slug to scan")
|
||||
.option("--version <version>", "Published skill version to scan")
|
||||
.option("--update", "Write published scan results back to the selected version")
|
||||
.option("-o, --output <path>", "Write the full report ZIP to a file")
|
||||
.option("--json", "Output scan report JSON")
|
||||
.action(async (folder, options) => {
|
||||
const opts = await resolveGlobalOpts();
|
||||
await cmdScan(opts, folder, options);
|
||||
});
|
||||
|
||||
registerCommand(program, ["delete"])
|
||||
.description("Soft-delete one of your skills")
|
||||
.argument("<slug>", "Skill slug")
|
||||
@@ -747,10 +767,16 @@ registerCommand(program, ["sync"])
|
||||
.option("--root <dir...>", "Extra scan roots (one or more)")
|
||||
.option("--all", "Upload all new/updated skills without prompting")
|
||||
.option("--dry-run", "Show what would be uploaded")
|
||||
.option("--json", "Output JSON")
|
||||
.option("--owner <handle>", "Publish under an org/user publisher handle")
|
||||
.option("--bump <type>", "Version bump for updates (patch|minor|major)", "patch")
|
||||
.option("--changelog <text>", "Changelog to use for updates (non-interactive)")
|
||||
.option("--tags <tags>", "Comma-separated tags", "latest")
|
||||
.option("--concurrency <n>", "Concurrent registry checks (default: 4)", "4")
|
||||
.option("--no-clawdbot-roots", "Only scan the configured workdir/dir and --root values")
|
||||
.option("--source-repo <repo>", "GitHub repo (owner/repo or URL)")
|
||||
.option("--source-commit <sha>", "Git commit SHA")
|
||||
.option("--source-ref <ref>", "Git ref/tag/branch")
|
||||
.action(async (options) => {
|
||||
const opts = await resolveGlobalOpts();
|
||||
const bump = String(options.bump ?? "patch") as "patch" | "minor" | "major";
|
||||
@@ -764,10 +790,16 @@ registerCommand(program, ["sync"])
|
||||
root: options.root,
|
||||
all: options.all,
|
||||
dryRun: options.dryRun,
|
||||
json: options.json,
|
||||
owner: options.owner,
|
||||
bump,
|
||||
changelog: options.changelog,
|
||||
tags: options.tags,
|
||||
concurrency,
|
||||
clawdbotRoots: options.clawdbotRoots,
|
||||
sourceRepo: options.sourceRepo,
|
||||
sourceCommit: options.sourceCommit,
|
||||
sourceRef: options.sourceRef,
|
||||
},
|
||||
isInputAllowed(),
|
||||
);
|
||||
|
||||
@@ -16,7 +16,7 @@ const registryMocks = createRegistryModuleMocks();
|
||||
const mockGetRegistry = registryMocks.getRegistry;
|
||||
vi.mock("../registry.js", () => registryMocks.moduleFactory());
|
||||
|
||||
const { cmdLogout } = await import("./auth");
|
||||
const { cmdLogout, cmdToken } = await import("./auth");
|
||||
|
||||
const mockLog = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
|
||||
@@ -54,3 +54,16 @@ describe("cmdLogout", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("cmdToken", () => {
|
||||
it("prints the stored token", async () => {
|
||||
mockReadGlobalConfig.mockResolvedValueOnce({
|
||||
registry: "https://clawhub.ai",
|
||||
token: "clh_test",
|
||||
});
|
||||
|
||||
await cmdToken();
|
||||
|
||||
expect(mockLog).toHaveBeenCalledWith("clh_test");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -98,6 +98,11 @@ export async function cmdWhoami(opts: GlobalOpts) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function cmdToken() {
|
||||
const token = await requireAuthToken();
|
||||
console.log(token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Device Flow login for headless environments.
|
||||
* Requests a device code, displays it to the user, then polls until authorized.
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { gzipSync, zipSync } from "fflate";
|
||||
@@ -67,6 +67,10 @@ async function makeTmpWorkdir() {
|
||||
return await mkdtemp(join(tmpdir(), "clawhub-package-"));
|
||||
}
|
||||
|
||||
async function listClawPackTempDirs() {
|
||||
return new Set((await readdir(tmpdir())).filter((name) => name.startsWith("clawhub-clawpack-")));
|
||||
}
|
||||
|
||||
function runGit(cwd: string, args: string[]) {
|
||||
const result = spawnSync("git", ["-C", cwd, ...args], {
|
||||
encoding: "utf8",
|
||||
@@ -143,8 +147,8 @@ function tarOctal(value: number, width: number) {
|
||||
return value.toString(8).padStart(width - 1, "0") + "\0";
|
||||
}
|
||||
|
||||
function tarFile(path: string, content: string) {
|
||||
const bytes = new TextEncoder().encode(content);
|
||||
function tarFile(path: string, content: string | Uint8Array) {
|
||||
const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content;
|
||||
const header = new Uint8Array(TAR_BLOCK_SIZE);
|
||||
writeTarString(header, 0, 100, path);
|
||||
writeTarString(header, 100, 8, tarOctal(0o644, 8));
|
||||
@@ -167,7 +171,7 @@ function tarFile(path: string, content: string) {
|
||||
return [header, body];
|
||||
}
|
||||
|
||||
function npmPackFixture(files: Record<string, string>) {
|
||||
function npmPackFixture(files: Record<string, string | Uint8Array>) {
|
||||
const parts: Uint8Array[] = [];
|
||||
for (const [path, content] of Object.entries(files)) {
|
||||
parts.push(...tarFile(path, content));
|
||||
@@ -1274,6 +1278,65 @@ describe("package commands", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("stages ClawPack tarballs over the multipart publish budget", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
const packName = "oversized-plugin-1.0.0.tgz";
|
||||
const packBytes = npmPackFixture({
|
||||
"package/package.json": makeCodePluginPackageJson({
|
||||
name: "@scope/oversized-plugin",
|
||||
displayName: "Oversized Plugin",
|
||||
version: "1.0.0",
|
||||
}),
|
||||
"package/openclaw.plugin.json": JSON.stringify({ id: "oversized.plugin" }),
|
||||
"package/dist/index.js": "export const demo = true;\n",
|
||||
"package/dist/model.bin": randomBytes(24 * 1024 * 1024),
|
||||
});
|
||||
expect(packBytes.byteLength).toBeGreaterThan(18 * 1024 * 1024);
|
||||
await writeFile(join(workdir, packName), packBytes);
|
||||
httpMocks.apiRequest.mockResolvedValueOnce({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "uploadTickets:clawpack",
|
||||
});
|
||||
httpMocks.uploadBinary.mockResolvedValueOnce({ storageId: "storage:clawpack" });
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
packageId: "pkg_1",
|
||||
releaseId: "rel_1",
|
||||
});
|
||||
|
||||
await cmdPublishPackage(makeOpts(workdir), packName, {
|
||||
sourceRepo: "openclaw/oversized-plugin",
|
||||
sourceCommit: "abc123",
|
||||
});
|
||||
|
||||
expect(httpMocks.apiRequest).toHaveBeenCalledWith(
|
||||
"https://clawhub.ai",
|
||||
{
|
||||
method: "POST",
|
||||
path: "/api/cli/upload-url",
|
||||
token: "tkn",
|
||||
},
|
||||
expect.anything(),
|
||||
);
|
||||
expect(httpMocks.uploadBinary).toHaveBeenCalledWith(
|
||||
{
|
||||
url: "https://upload.local",
|
||||
bytes: expect.any(Uint8Array),
|
||||
contentType: "application/octet-stream",
|
||||
retryCount: 5,
|
||||
},
|
||||
expect.anything(),
|
||||
);
|
||||
expect(getPublishForm().get("clawpack")).toBe("storage:clawpack");
|
||||
expect(getPublishForm().get("clawpackUploadTicket")).toBe("uploadTickets:clawpack");
|
||||
expect(getPublishPayload()).not.toHaveProperty("artifact");
|
||||
expect(getPublishPayload()).not.toHaveProperty("files");
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("packs a plugin folder through npm pack and validates the ClawPack", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
@@ -1315,6 +1378,88 @@ describe("package commands", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("packs local ClawPacks over the multipart publish upload budget", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
const folder = join(workdir, "demo-heavy-plugin");
|
||||
await mkdir(join(folder, "dist"), { recursive: true });
|
||||
await mkdir(join(workdir, "packs"), { recursive: true });
|
||||
await writeFile(
|
||||
join(folder, "package.json"),
|
||||
makeCodePluginPackageJson({
|
||||
name: "demo-heavy-plugin",
|
||||
displayName: "Demo Heavy Plugin",
|
||||
version: "1.0.0",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
join(folder, "openclaw.plugin.json"),
|
||||
JSON.stringify({ id: "demo.heavy.plugin" }),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(join(folder, "dist", "index.js"), "export const demo = true;\n", "utf8");
|
||||
await writeFile(join(folder, "dist", "model.bin"), randomBytes(24 * 1024 * 1024));
|
||||
|
||||
await cmdPackPackage(makeOpts(workdir), "demo-heavy-plugin", {
|
||||
packDestination: "packs",
|
||||
});
|
||||
|
||||
const packPath = join(workdir, "packs", "demo-heavy-plugin-1.0.0.tgz");
|
||||
const packed = await readFile(packPath);
|
||||
expect(packed.byteLength).toBeGreaterThan(18 * 1024 * 1024);
|
||||
expect(parseClawPack(new Uint8Array(packed)).packageName).toBe("demo-heavy-plugin");
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans generated ClawPack temp dirs after staged publish failure", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
const beforeTempDirs = await listClawPackTempDirs();
|
||||
try {
|
||||
const folder = join(workdir, "demo-heavy-plugin");
|
||||
await mkdir(join(folder, "dist"), { recursive: true });
|
||||
await writeFile(
|
||||
join(folder, "package.json"),
|
||||
makeCodePluginPackageJson({
|
||||
name: "demo-heavy-plugin",
|
||||
displayName: "Demo Heavy Plugin",
|
||||
version: "1.0.0",
|
||||
repository: "https://github.com/openclaw/demo-heavy-plugin.git",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
join(folder, "openclaw.plugin.json"),
|
||||
JSON.stringify({ id: "demo.heavy.plugin" }),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(join(folder, "dist", "index.js"), "export const demo = true;\n", "utf8");
|
||||
await writeFile(join(folder, "dist", "model.bin"), randomBytes(24 * 1024 * 1024));
|
||||
httpMocks.apiRequest.mockResolvedValueOnce({
|
||||
uploadUrl: "https://upload.local",
|
||||
uploadTicket: "uploadTickets:clawpack",
|
||||
});
|
||||
httpMocks.uploadBinary.mockResolvedValueOnce({ storageId: "storage:clawpack" });
|
||||
httpMocks.apiRequestForm.mockRejectedValueOnce(new Error("Registry rejected upload"));
|
||||
|
||||
await expect(
|
||||
cmdPublishPackage(makeOpts(workdir), "demo-heavy-plugin", {
|
||||
sourceRepo: "openclaw/demo-heavy-plugin",
|
||||
sourceCommit: "abc123",
|
||||
}),
|
||||
).rejects.toThrow("Registry rejected upload");
|
||||
expect(getPublishForm().get("clawpack")).toBe("storage:clawpack");
|
||||
expect(getPublishForm().get("clawpackUploadTicket")).toBe("uploadTickets:clawpack");
|
||||
|
||||
const afterTempDirs = await listClawPackTempDirs();
|
||||
expect([...afterTempDirs].filter((name) => !beforeTempDirs.has(name))).toEqual([]);
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a code plugin ClawPack with TypeScript entries and no compiled runtime", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
|
||||
@@ -7,10 +7,20 @@ import ignore from "ignore";
|
||||
import mime from "mime";
|
||||
import semver from "semver";
|
||||
import { parseClawPack } from "../../clawpack.js";
|
||||
import { apiRequest, apiRequestForm, fetchBinary, fetchText, registryUrl } from "../../http.js";
|
||||
import {
|
||||
apiRequest,
|
||||
apiRequestForm,
|
||||
fetchBinary,
|
||||
fetchText,
|
||||
registryUrl,
|
||||
uploadBinary,
|
||||
} from "../../http.js";
|
||||
import {
|
||||
ApiCliUploadUrlResponseSchema,
|
||||
ApiRoutes,
|
||||
LegacyApiRoutes,
|
||||
ApiV1DeleteResponseSchema,
|
||||
ApiUploadFileResponseSchema,
|
||||
ApiV1PackageArtifactResponseSchema,
|
||||
ApiV1PackageListResponseSchema,
|
||||
ApiV1PackageModerationStatusResponseSchema,
|
||||
@@ -24,6 +34,10 @@ import {
|
||||
ApiV1PackageVersionListResponseSchema,
|
||||
ApiV1PackageVersionResponseSchema,
|
||||
ApiV1PublishTokenMintResponseSchema,
|
||||
estimatePackageMultipartUploadBytes,
|
||||
getPackageMultipartSizeError,
|
||||
MAX_PACKAGE_CLAWPACK_BYTES,
|
||||
MAX_PACKAGE_MULTIPART_BYTES,
|
||||
normalizeOpenClawExternalPluginCompatibility,
|
||||
type PackageArtifactSummary,
|
||||
type PackageCapabilitySummary,
|
||||
@@ -50,7 +64,6 @@ const DOT_DIR = ".clawhub";
|
||||
const LEGACY_DOT_DIR = ".clawdhub";
|
||||
const DOT_IGNORE = ".clawhubignore";
|
||||
const LEGACY_DOT_IGNORE = ".clawdhubignore";
|
||||
const MAX_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
const PACKAGE_PUBLISH_RETRY_COUNT = 5;
|
||||
|
||||
type PackageInspectOptions = {
|
||||
@@ -598,7 +611,6 @@ async function createClawPackFromFolder(options: {
|
||||
|
||||
const packPath = resolve(options.packDestination, filename);
|
||||
const bytes = new Uint8Array(await readFile(packPath));
|
||||
assertClawPackSize(bytes.byteLength, basename(packPath));
|
||||
const parsed = parseClawPack(bytes);
|
||||
return {
|
||||
path: packPath,
|
||||
@@ -665,14 +677,26 @@ export async function cmdPublishPackage(
|
||||
spinner,
|
||||
});
|
||||
const form = new FormData();
|
||||
form.set("payload", JSON.stringify(plan.payload));
|
||||
const payloadJson = JSON.stringify(plan.payload);
|
||||
form.set("payload", payloadJson);
|
||||
|
||||
if (plan.clawpackOnDisk) {
|
||||
if (spinner) spinner.text = `Uploading ${plan.clawpackOnDisk.relPath}`;
|
||||
const blob = new Blob([Buffer.from(plan.clawpackOnDisk.bytes)], {
|
||||
type: "application/octet-stream",
|
||||
});
|
||||
form.append("clawpack", blob, plan.clawpackOnDisk.relPath);
|
||||
if (isPackageMultipartTooLarge(payloadJson, "clawpack", [plan.clawpackOnDisk])) {
|
||||
const staged = await uploadClawPackToStorage(
|
||||
registry,
|
||||
publishToken,
|
||||
plan.clawpackOnDisk,
|
||||
spinner,
|
||||
);
|
||||
form.set("clawpack", staged.storageId);
|
||||
form.set("clawpackUploadTicket", staged.uploadTicket);
|
||||
} else {
|
||||
if (spinner) spinner.text = `Uploading ${plan.clawpackOnDisk.relPath}`;
|
||||
const blob = new Blob([Buffer.from(plan.clawpackOnDisk.bytes)], {
|
||||
type: "application/octet-stream",
|
||||
});
|
||||
form.append("clawpack", blob, plan.clawpackOnDisk.relPath);
|
||||
}
|
||||
} else {
|
||||
let index = 0;
|
||||
for (const file of plan.filesOnDisk) {
|
||||
@@ -1551,12 +1575,66 @@ function packageJsonString(value: Record<string, unknown> | null, key: string):
|
||||
return typeof candidate === "string" && candidate.trim() ? candidate.trim() : undefined;
|
||||
}
|
||||
|
||||
function assertClawPackSize(size: number, label: string) {
|
||||
if (size > MAX_CLAWPACK_BYTES) {
|
||||
fail(`ClawPack "${label}" exceeds 120MB limit`);
|
||||
function assertPackageMultipartSize(
|
||||
payloadJson: string,
|
||||
fileFieldName: "files" | "clawpack",
|
||||
files: PackageFile[],
|
||||
) {
|
||||
if (isPackageMultipartTooLarge(payloadJson, fileFieldName, files)) {
|
||||
fail(getPackageMultipartSizeError());
|
||||
}
|
||||
}
|
||||
|
||||
function getClawPackSizeError(path: string) {
|
||||
return `ClawPack "${path}" exceeds 120MB limit`;
|
||||
}
|
||||
|
||||
function isPackageMultipartTooLarge(
|
||||
payloadJson: string,
|
||||
fileFieldName: "files" | "clawpack",
|
||||
files: PackageFile[],
|
||||
) {
|
||||
return (
|
||||
estimatePackageMultipartUploadBytes({
|
||||
payloadJson,
|
||||
fileFieldName,
|
||||
files: files.map((file) => ({
|
||||
name: file.relPath,
|
||||
size: file.bytes.byteLength,
|
||||
type: file.contentType,
|
||||
})),
|
||||
}) > MAX_PACKAGE_MULTIPART_BYTES
|
||||
);
|
||||
}
|
||||
|
||||
async function uploadClawPackToStorage(
|
||||
registry: string,
|
||||
publishToken: string,
|
||||
file: PackageFile,
|
||||
spinner: ReturnType<typeof createSpinner> | null,
|
||||
) {
|
||||
if (spinner) spinner.text = `Uploading ${file.relPath}`;
|
||||
const { uploadUrl, uploadTicket } = await apiRequest(
|
||||
registry,
|
||||
{
|
||||
method: "POST",
|
||||
path: LegacyApiRoutes.cliUploadUrl,
|
||||
token: publishToken,
|
||||
},
|
||||
ApiCliUploadUrlResponseSchema,
|
||||
);
|
||||
const result = await uploadBinary(
|
||||
{
|
||||
url: uploadUrl,
|
||||
bytes: file.bytes,
|
||||
contentType: file.contentType ?? "application/octet-stream",
|
||||
retryCount: PACKAGE_PUBLISH_RETRY_COUNT,
|
||||
},
|
||||
ApiUploadFileResponseSchema,
|
||||
);
|
||||
return { storageId: result.storageId, uploadTicket };
|
||||
}
|
||||
|
||||
const REAL_BUNDLE_MANIFESTS = [
|
||||
{ path: ".codex-plugin/plugin.json", format: "codex" },
|
||||
{ path: ".claude-plugin/plugin.json", format: "claude" },
|
||||
@@ -1666,11 +1744,13 @@ async function preparePackagePublishPlan(
|
||||
}
|
||||
} else {
|
||||
const folderStat = await stat(folder).catch(() => null);
|
||||
if (!folderStat) fail("Path must be a folder or ClawPack .tgz");
|
||||
if (!folderStat) fail("Path must be a folder or package tarball .tgz");
|
||||
if (folderStat.isFile()) {
|
||||
if (!folder.endsWith(".tgz")) fail("ClawPack publish files must end in .tgz");
|
||||
if (!folder.endsWith(".tgz")) fail("Package publish files must end in .tgz");
|
||||
const bytes = new Uint8Array(await readFile(folder));
|
||||
assertClawPackSize(bytes.byteLength, basename(folder));
|
||||
if (bytes.byteLength > MAX_PACKAGE_CLAWPACK_BYTES) {
|
||||
fail(getClawPackSizeError(basename(folder)));
|
||||
}
|
||||
parsedClawpack = parseClawPack(bytes);
|
||||
clawpackOnDisk = {
|
||||
relPath: basename(folder),
|
||||
@@ -1678,7 +1758,7 @@ async function preparePackagePublishPlan(
|
||||
contentType: "application/octet-stream",
|
||||
};
|
||||
} else if (!folderStat.isDirectory()) {
|
||||
fail("Path must be a folder or ClawPack .tgz");
|
||||
fail("Path must be a folder or package tarball .tgz");
|
||||
}
|
||||
|
||||
const localGitInfo = folderStat.isDirectory() ? resolveLocalGitInfo(folder) : null;
|
||||
@@ -1782,7 +1862,9 @@ async function preparePackagePublishPlan(
|
||||
contentType: mime.getType(entry.path) ?? "application/octet-stream",
|
||||
}));
|
||||
}
|
||||
|
||||
const totalBytes = clawpackOnDisk
|
||||
? clawpackOnDisk.bytes.byteLength
|
||||
: filesOnDisk.reduce((sum, file) => sum + file.bytes.byteLength, 0);
|
||||
const payload: PackagePublishPayload = {
|
||||
name,
|
||||
displayName,
|
||||
@@ -1804,6 +1886,18 @@ async function preparePackagePublishPlan(
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
try {
|
||||
if (clawpackOnDisk) {
|
||||
if (clawpackOnDisk.bytes.byteLength > MAX_PACKAGE_CLAWPACK_BYTES) {
|
||||
fail(getClawPackSizeError(clawpackOnDisk.relPath));
|
||||
}
|
||||
} else {
|
||||
assertPackageMultipartSize(JSON.stringify(payload), "files", filesOnDisk);
|
||||
}
|
||||
} catch (error) {
|
||||
await cleanup?.();
|
||||
throw error;
|
||||
}
|
||||
const sourceLabel = describePublishSource(sourceForFetch, source, folder);
|
||||
|
||||
return {
|
||||
@@ -1826,9 +1920,7 @@ async function preparePackagePublishPlan(
|
||||
version,
|
||||
...(source?.commit ? { commit: source.commit } : {}),
|
||||
files: filesOnDisk.length,
|
||||
totalBytes: clawpackOnDisk
|
||||
? clawpackOnDisk.bytes.byteLength
|
||||
: filesOnDisk.reduce((sum, file) => sum + file.bytes.byteLength, 0),
|
||||
totalBytes,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -227,6 +227,54 @@ describe("cmdPublish", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("includes GitHub source provenance for CI publishes", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
const dateSpy = vi.spyOn(Date, "now").mockReturnValue(123_456_789);
|
||||
try {
|
||||
const folder = join(workdir, "source-skill");
|
||||
await mkdir(folder, { recursive: true });
|
||||
await writeFile(join(folder, "SKILL.md"), "# Skill\n", "utf8");
|
||||
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
skillId: "skill_1",
|
||||
versionId: "ver_1",
|
||||
});
|
||||
|
||||
await cmdPublish(makeOpts(workdir), "source-skill", {
|
||||
slug: "source-skill",
|
||||
name: "Source Skill",
|
||||
version: "1.0.0",
|
||||
sourceRepo: "https://github.com/NVIDIA/skills",
|
||||
sourceCommit: "abc123",
|
||||
sourceRef: "refs/heads/main",
|
||||
sourcePath: "skills/source-skill",
|
||||
});
|
||||
|
||||
const publishCall = httpMocks.apiRequestForm.mock.calls.find((call) => {
|
||||
const req = call[1] as { path?: string } | undefined;
|
||||
return req?.path === "/api/v1/skills";
|
||||
});
|
||||
if (!publishCall) throw new Error("Missing publish call");
|
||||
const publishForm = (publishCall[1] as { form?: FormData }).form as FormData;
|
||||
const payloadEntry = publishForm.get("payload");
|
||||
if (typeof payloadEntry !== "string") throw new Error("Missing publish payload");
|
||||
const payload = JSON.parse(payloadEntry);
|
||||
expect(payload.source).toEqual({
|
||||
kind: "github",
|
||||
url: "https://github.com/NVIDIA/skills",
|
||||
repo: "NVIDIA/skills",
|
||||
ref: "refs/heads/main",
|
||||
commit: "abc123",
|
||||
path: "skills/source-skill",
|
||||
importedAt: 123_456_789,
|
||||
});
|
||||
dateSpy.mockRestore();
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects plugin folders with guidance to use "clawhub package publish"', async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
|
||||
@@ -9,6 +9,7 @@ import { getRegistry } from "../registry.js";
|
||||
import { sanitizeSlug, titleCase } from "../slug.js";
|
||||
import type { GlobalOpts } from "../types.js";
|
||||
import { createSpinner, fail, formatError } from "../ui.js";
|
||||
import { normalizeGitHubRepo } from "./github.js";
|
||||
|
||||
export async function cmdPublish(
|
||||
opts: GlobalOpts,
|
||||
@@ -22,6 +23,10 @@ export async function cmdPublish(
|
||||
tags?: string;
|
||||
forkOf?: string;
|
||||
migrateOwner?: boolean;
|
||||
sourceRepo?: string;
|
||||
sourceCommit?: string;
|
||||
sourceRef?: string;
|
||||
sourcePath?: string;
|
||||
},
|
||||
) {
|
||||
const folder = folderArg ? resolve(opts.workdir, folderArg) : null;
|
||||
@@ -48,6 +53,7 @@ export async function cmdPublish(
|
||||
|
||||
const forkOfRaw = options.forkOf?.trim();
|
||||
const forkOf = forkOfRaw ? parseForkOf(forkOfRaw) : undefined;
|
||||
const source = buildPublishSource(options);
|
||||
|
||||
if (!slug) fail("--slug required");
|
||||
if (!displayName) fail("--name required");
|
||||
@@ -80,6 +86,7 @@ export async function cmdPublish(
|
||||
changelog,
|
||||
acceptLicenseTerms: true,
|
||||
tags,
|
||||
...(source ? { source } : {}),
|
||||
...(forkOf ? { forkOf } : {}),
|
||||
}),
|
||||
);
|
||||
@@ -174,3 +181,34 @@ function parseForkOf(value: string) {
|
||||
if (version && !semver.valid(version)) fail("--fork-of version must be valid semver");
|
||||
return { slug, version: version || undefined };
|
||||
}
|
||||
|
||||
function buildPublishSource(options: {
|
||||
sourceRepo?: string;
|
||||
sourceCommit?: string;
|
||||
sourceRef?: string;
|
||||
sourcePath?: string;
|
||||
}) {
|
||||
const rawRepo = options.sourceRepo?.trim();
|
||||
const commit = options.sourceCommit?.trim();
|
||||
const ref = options.sourceRef?.trim();
|
||||
const path = normalizeSourcePath(options.sourcePath);
|
||||
if (!rawRepo && !commit && !ref && !options.sourcePath?.trim()) return undefined;
|
||||
if (!rawRepo || !commit) fail("--source-repo and --source-commit must be provided together");
|
||||
const repo = normalizeGitHubRepo(rawRepo);
|
||||
if (!repo) fail("--source-repo must be a GitHub repo or URL");
|
||||
return {
|
||||
kind: "github" as const,
|
||||
url: `https://github.com/${repo}`,
|
||||
repo,
|
||||
ref: ref || commit,
|
||||
commit,
|
||||
path,
|
||||
importedAt: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeSourcePath(value: string | undefined) {
|
||||
const normalized = (value?.trim() || ".").replaceAll("\\", "/").replace(/^\.\/+/, "");
|
||||
if (!normalized || normalized === ".") return ".";
|
||||
return normalized.replace(/\/+$/, "") || ".";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
/* @vitest-environment node */
|
||||
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createAuthTokenModuleMocks,
|
||||
createHttpModuleMocks,
|
||||
createRegistryModuleMocks,
|
||||
createUiModuleMocks,
|
||||
makeGlobalOpts,
|
||||
} from "../../../test/cliCommandTestKit.js";
|
||||
import { ApiRoutes } from "../../schema/index.js";
|
||||
|
||||
const authTokenMocks = createAuthTokenModuleMocks();
|
||||
const registryMocks = createRegistryModuleMocks();
|
||||
const httpMocks = createHttpModuleMocks();
|
||||
const uiMocks = createUiModuleMocks();
|
||||
|
||||
vi.mock("../authToken.js", () => authTokenMocks.moduleFactory());
|
||||
vi.mock("../registry.js", () => registryMocks.moduleFactory());
|
||||
vi.mock("../../http.js", () => httpMocks.moduleFactory());
|
||||
vi.mock("../ui.js", () => uiMocks.moduleFactory());
|
||||
|
||||
const { cmdScan } = await import("./scan");
|
||||
|
||||
const mockLog = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
const mockWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
|
||||
|
||||
async function makeTmpWorkdir() {
|
||||
return await mkdtemp(join(tmpdir(), "clawhub-scan-"));
|
||||
}
|
||||
|
||||
function completedScan(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
ok: true,
|
||||
scanId: "scan_123",
|
||||
jobId: "job_123",
|
||||
status: "succeeded",
|
||||
sourceKind: "published",
|
||||
update: false,
|
||||
writtenBack: false,
|
||||
artifact: {
|
||||
slug: "demo",
|
||||
displayName: "Demo",
|
||||
version: "1.2.3",
|
||||
},
|
||||
report: {
|
||||
clawscan: {
|
||||
status: "clean",
|
||||
verdict: "clean",
|
||||
confidence: "high",
|
||||
summary: "No suspicious behavior found.",
|
||||
guidance: "OK to publish.",
|
||||
findings: "No findings.",
|
||||
checkedAt: 1_700_000_000_000,
|
||||
},
|
||||
skillspector: {
|
||||
status: "clean",
|
||||
score: 100,
|
||||
severity: "none",
|
||||
issueCount: 0,
|
||||
issues: [],
|
||||
checkedAt: 1_700_000_000_000,
|
||||
},
|
||||
staticAnalysis: {
|
||||
status: "clean",
|
||||
reasonCodes: [],
|
||||
findings: [],
|
||||
summary: "Static checks passed.",
|
||||
checkedAt: 1_700_000_000_000,
|
||||
},
|
||||
virustotal: null,
|
||||
},
|
||||
createdAt: 1_700_000_000_000,
|
||||
updatedAt: 1_700_000_100_000,
|
||||
completedAt: 1_700_000_100_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockLog.mockClear();
|
||||
mockWrite.mockClear();
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
|
||||
describe("cmdScan", () => {
|
||||
it("uploads a local skill bundle and polls until complete", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
const folder = join(workdir, "local-skill");
|
||||
await mkdir(folder, { recursive: true });
|
||||
await writeFile(join(folder, "SKILL.md"), "# Local Skill\n", "utf8");
|
||||
await writeFile(join(folder, "notes.md"), "notes\n", "utf8");
|
||||
|
||||
httpMocks.apiRequestForm.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
scanId: "scan_123",
|
||||
jobId: "job_123",
|
||||
status: "queued",
|
||||
sourceKind: "upload",
|
||||
update: false,
|
||||
});
|
||||
httpMocks.apiRequest.mockResolvedValueOnce(completedScan({ sourceKind: "upload" }));
|
||||
|
||||
await cmdScan(makeGlobalOpts(workdir), "local-skill", {});
|
||||
|
||||
const submitCall = httpMocks.apiRequestForm.mock.calls[0];
|
||||
expect(submitCall?.[1]).toMatchObject({
|
||||
method: "POST",
|
||||
path: ApiRoutes.skillScans,
|
||||
token: "tkn",
|
||||
});
|
||||
if (!submitCall) throw new Error("missing scan submit call");
|
||||
const form = (submitCall[1] as { form: FormData }).form;
|
||||
const payloadRaw = form.get("payload");
|
||||
expect(typeof payloadRaw).toBe("string");
|
||||
expect(JSON.parse(payloadRaw as string)).toEqual({
|
||||
source: { kind: "upload" },
|
||||
update: false,
|
||||
});
|
||||
const files = form?.getAll("files") as Array<Blob & { name?: string }>;
|
||||
expect(files.map((file) => file.name ?? "").sort()).toEqual(["SKILL.md", "notes.md"]);
|
||||
expect(httpMocks.apiRequest).toHaveBeenCalledWith(
|
||||
"https://clawhub.ai",
|
||||
expect.objectContaining({
|
||||
method: "GET",
|
||||
path: `${ApiRoutes.skillScans}/scan_123`,
|
||||
token: "tkn",
|
||||
}),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(mockLog).toHaveBeenCalledWith(expect.stringContaining("ClawScan"));
|
||||
expect(mockLog).toHaveBeenCalledWith(
|
||||
expect.stringContaining("No suspicious behavior found."),
|
||||
);
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("submits a published scan with update mode", async () => {
|
||||
httpMocks.apiRequest
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
scanId: "scan_123",
|
||||
jobId: "job_123",
|
||||
status: "queued",
|
||||
sourceKind: "published",
|
||||
update: true,
|
||||
})
|
||||
.mockResolvedValueOnce(completedScan({ update: true, writtenBack: true }));
|
||||
|
||||
await cmdScan(makeGlobalOpts(), undefined, {
|
||||
slug: "demo",
|
||||
version: "1.2.3",
|
||||
update: true,
|
||||
});
|
||||
|
||||
expect(httpMocks.apiRequest.mock.calls[0]?.[1]).toMatchObject({
|
||||
method: "POST",
|
||||
path: ApiRoutes.skillScans,
|
||||
token: "tkn",
|
||||
body: {
|
||||
source: { kind: "published", slug: "demo", version: "1.2.3" },
|
||||
update: true,
|
||||
},
|
||||
});
|
||||
expect(mockLog).toHaveBeenCalledWith(expect.stringContaining("Written back: yes"));
|
||||
});
|
||||
|
||||
it("downloads the canonical report zip when --output is set", async () => {
|
||||
const workdir = await makeTmpWorkdir();
|
||||
try {
|
||||
const output = join(workdir, "report.zip");
|
||||
httpMocks.apiRequest
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
scanId: "scan_123",
|
||||
jobId: "job_123",
|
||||
status: "queued",
|
||||
sourceKind: "published",
|
||||
update: false,
|
||||
})
|
||||
.mockResolvedValueOnce(completedScan());
|
||||
httpMocks.fetchBinary.mockResolvedValueOnce(new Uint8Array([80, 75, 3, 4]));
|
||||
|
||||
await cmdScan(makeGlobalOpts(workdir), undefined, { slug: "demo", output });
|
||||
|
||||
expect(httpMocks.fetchBinary).toHaveBeenCalledWith("https://clawhub.ai", {
|
||||
path: `${ApiRoutes.skillScans}/scan_123/download`,
|
||||
token: "tkn",
|
||||
});
|
||||
expect(await readFile(output)).toEqual(Buffer.from([80, 75, 3, 4]));
|
||||
} finally {
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects ambiguous or invalid source options", async () => {
|
||||
await expect(cmdScan(makeGlobalOpts(), "local-skill", { slug: "demo" })).rejects.toThrow(
|
||||
"Choose either a local path or --slug, not both",
|
||||
);
|
||||
await expect(cmdScan(makeGlobalOpts(), "local-skill", { update: true })).rejects.toThrow(
|
||||
"--update is only valid with --slug",
|
||||
);
|
||||
await expect(cmdScan(makeGlobalOpts(), undefined, {})).rejects.toThrow(
|
||||
"Provide a local path or --slug",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,295 @@
|
||||
import { mkdir, stat, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { apiRequest, apiRequestForm, fetchBinary } from "../../http.js";
|
||||
import {
|
||||
ApiRoutes,
|
||||
ApiV1SkillScanStatusResponseSchema,
|
||||
ApiV1SkillScanSubmitResponseSchema,
|
||||
type ApiV1SkillScanStatusResponse,
|
||||
} from "../../schema/index.js";
|
||||
import { listTextFiles } from "../../skills.js";
|
||||
import { requireAuthToken } from "../authToken.js";
|
||||
import { getRegistry } from "../registry.js";
|
||||
import type { GlobalOpts } from "../types.js";
|
||||
import { createSpinner, fail, formatError } from "../ui.js";
|
||||
|
||||
const DEFAULT_POLL_INTERVAL_MS = 2_000;
|
||||
const MAX_POLL_ATTEMPTS = 900;
|
||||
|
||||
type ScanOptions = {
|
||||
slug?: string;
|
||||
version?: string;
|
||||
update?: boolean;
|
||||
output?: string;
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type ReportRecord = Record<string, unknown>;
|
||||
|
||||
export async function cmdScan(opts: GlobalOpts, pathArg: string | undefined, options: ScanOptions) {
|
||||
validateScanOptions(pathArg, options);
|
||||
|
||||
const token = await requireAuthToken();
|
||||
const registry = await getRegistry(opts, { cache: true });
|
||||
const spinner = createSpinner("Submitting scan");
|
||||
|
||||
try {
|
||||
const submitted = pathArg
|
||||
? await submitLocalScan(opts, registry, token, pathArg)
|
||||
: await submitPublishedScan(registry, token, options);
|
||||
|
||||
spinner.text = `Scan queued (${submitted.scanId})`;
|
||||
const status = await pollScan(registry, token, submitted.scanId, spinner);
|
||||
|
||||
if (status.status === "failed") {
|
||||
spinner.fail(`Scan failed (${status.scanId})`);
|
||||
if (options.json) printJson(status);
|
||||
else printScanReport(status);
|
||||
throw new Error(status.lastError ?? "Scan failed");
|
||||
}
|
||||
|
||||
spinner.succeed(`Scan complete (${status.scanId})`);
|
||||
|
||||
if (options.json) printJson(status);
|
||||
else printScanReport(status);
|
||||
|
||||
if (options.output) {
|
||||
const bytes = await fetchBinary(registry, {
|
||||
path: `${ApiRoutes.skillScans}/${encodeURIComponent(status.scanId)}/download`,
|
||||
token,
|
||||
});
|
||||
await mkdir(dirname(resolve(opts.workdir, options.output)), { recursive: true });
|
||||
await writeFile(resolve(opts.workdir, options.output), bytes);
|
||||
if (!options.json) console.log(`Report ZIP: ${resolve(opts.workdir, options.output)}`);
|
||||
}
|
||||
} catch (error) {
|
||||
spinner.fail(formatError(error));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function validateScanOptions(pathArg: string | undefined, options: ScanOptions) {
|
||||
const hasPath = Boolean(pathArg?.trim());
|
||||
const hasSlug = Boolean(options.slug?.trim());
|
||||
if (hasPath && hasSlug) fail("Choose either a local path or --slug, not both");
|
||||
if (!hasPath && !hasSlug) fail("Provide a local path or --slug");
|
||||
if (hasPath && options.update) fail("--update is only valid with --slug");
|
||||
}
|
||||
|
||||
async function submitLocalScan(opts: GlobalOpts, registry: string, token: string, pathArg: string) {
|
||||
const folder = resolve(opts.workdir, pathArg);
|
||||
const folderStat = await stat(folder).catch(() => null);
|
||||
if (!folderStat?.isDirectory()) fail("Path must be a folder");
|
||||
|
||||
const files = await listTextFiles(folder);
|
||||
if (
|
||||
!files.some((file) => {
|
||||
const lower = file.relPath.toLowerCase();
|
||||
return lower === "skill.md";
|
||||
})
|
||||
) {
|
||||
fail("SKILL.md required");
|
||||
}
|
||||
if (files.length === 0) fail("No files found");
|
||||
|
||||
const form = new FormData();
|
||||
form.set("payload", JSON.stringify({ source: { kind: "upload" }, update: false }));
|
||||
for (const file of files) {
|
||||
const blob = new Blob([Buffer.from(file.bytes)], { type: file.contentType ?? "text/plain" });
|
||||
form.append("files", blob, file.relPath);
|
||||
}
|
||||
|
||||
return await apiRequestForm(
|
||||
registry,
|
||||
{ method: "POST", path: ApiRoutes.skillScans, token, form },
|
||||
ApiV1SkillScanSubmitResponseSchema,
|
||||
);
|
||||
}
|
||||
|
||||
async function submitPublishedScan(registry: string, token: string, options: ScanOptions) {
|
||||
const slug = options.slug?.trim();
|
||||
if (!slug) fail("--slug required");
|
||||
const version = options.version?.trim();
|
||||
return await apiRequest(
|
||||
registry,
|
||||
{
|
||||
method: "POST",
|
||||
path: ApiRoutes.skillScans,
|
||||
token,
|
||||
body: {
|
||||
source: {
|
||||
kind: "published",
|
||||
slug,
|
||||
...(version ? { version } : {}),
|
||||
},
|
||||
update: options.update === true,
|
||||
},
|
||||
},
|
||||
ApiV1SkillScanSubmitResponseSchema,
|
||||
);
|
||||
}
|
||||
|
||||
async function pollScan(
|
||||
registry: string,
|
||||
token: string,
|
||||
scanId: string,
|
||||
spinner: ReturnType<typeof createSpinner>,
|
||||
) {
|
||||
for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) {
|
||||
const status = await apiRequest(
|
||||
registry,
|
||||
{
|
||||
method: "GET",
|
||||
path: `${ApiRoutes.skillScans}/${encodeURIComponent(scanId)}`,
|
||||
token,
|
||||
},
|
||||
ApiV1SkillScanStatusResponseSchema,
|
||||
);
|
||||
spinner.text = `Scan ${status.status} (${scanId})`;
|
||||
if (status.status === "succeeded" || status.status === "failed") return status;
|
||||
await sleep(DEFAULT_POLL_INTERVAL_MS);
|
||||
}
|
||||
throw new Error(`Timed out waiting for scan ${scanId}`);
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
|
||||
}
|
||||
|
||||
function printJson(status: ApiV1SkillScanStatusResponse) {
|
||||
console.log(JSON.stringify(status, null, 2));
|
||||
}
|
||||
|
||||
function printScanReport(status: ApiV1SkillScanStatusResponse) {
|
||||
const artifact = asRecord(status.artifact) ?? {};
|
||||
const report = asRecord(status.report) ?? {};
|
||||
const clawscan = asRecord(report.clawscan) ?? {};
|
||||
const skillspector = asRecord(report.skillspector) ?? {};
|
||||
const staticAnalysis = asRecord(report.staticAnalysis) ?? {};
|
||||
const virustotal = asRecord(report.virustotal);
|
||||
|
||||
console.log("");
|
||||
console.log("ClawHub Scan Report");
|
||||
console.log(`Scan ID: ${status.scanId}`);
|
||||
console.log(`Status: ${status.status.toUpperCase()}`);
|
||||
console.log(`Source: ${status.sourceKind}`);
|
||||
console.log(`Update requested: ${status.update ? "yes" : "no"}`);
|
||||
console.log(`Written back: ${status.writtenBack ? "yes" : "no"}`);
|
||||
printOptional("Slug", stringValue(artifact.slug));
|
||||
printOptional("Name", stringValue(artifact.displayName));
|
||||
printOptional("Version", stringValue(artifact.version));
|
||||
printOptional("Created", dateValue(status.createdAt));
|
||||
printOptional("Completed", dateValue(status.completedAt));
|
||||
|
||||
console.log("");
|
||||
console.log("ClawScan");
|
||||
printOptional("Verdict", upperValue(clawscan.verdict ?? clawscan.status));
|
||||
printOptional("Confidence", stringValue(clawscan.confidence));
|
||||
printOptional("Summary", stringValue(clawscan.summary));
|
||||
printOptional("Guidance", stringValue(clawscan.guidance));
|
||||
printFindings(clawscan.findings);
|
||||
printAgenticRisks(clawscan.agenticRiskFindings);
|
||||
|
||||
console.log("");
|
||||
console.log("SkillSpector");
|
||||
printOptional("Status", upperValue(skillspector.status));
|
||||
printOptional("Score", numberValue(skillspector.score));
|
||||
printOptional("Severity", stringValue(skillspector.severity));
|
||||
printOptional("Issue count", numberValue(skillspector.issueCount));
|
||||
printIssueList(skillspector.issues);
|
||||
|
||||
console.log("");
|
||||
console.log("Static Analysis");
|
||||
printOptional("Status", upperValue(staticAnalysis.status));
|
||||
printOptional("Reason codes", arrayValue(staticAnalysis.reasonCodes));
|
||||
printOptional("Summary", stringValue(staticAnalysis.summary));
|
||||
printIssueList(staticAnalysis.findings);
|
||||
|
||||
console.log("");
|
||||
console.log("VirusTotal");
|
||||
if (!virustotal) {
|
||||
console.log("Status: not available");
|
||||
} else {
|
||||
printOptional("Status", stringValue(virustotal.status));
|
||||
printOptional("Malicious", numberValue(virustotal.malicious));
|
||||
printOptional("Suspicious", numberValue(virustotal.suspicious));
|
||||
printOptional("Harmless", numberValue(virustotal.harmless));
|
||||
printOptional("Undetected", numberValue(virustotal.undetected));
|
||||
}
|
||||
}
|
||||
|
||||
function printOptional(label: string, value: string | undefined) {
|
||||
if (!value) return;
|
||||
console.log(`${label}: ${value}`);
|
||||
}
|
||||
|
||||
function printFindings(value: unknown) {
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
console.log(`Findings: ${value.trim()}`);
|
||||
}
|
||||
}
|
||||
|
||||
function printAgenticRisks(value: unknown) {
|
||||
if (!Array.isArray(value) || value.length === 0) return;
|
||||
console.log("Agentic risk findings:");
|
||||
for (const item of value.slice(0, 20)) {
|
||||
const finding = asRecord(item);
|
||||
if (!finding) continue;
|
||||
const label = stringValue(finding.categoryLabel) ?? stringValue(finding.categoryId) ?? "risk";
|
||||
const status = stringValue(finding.status) ?? "unknown";
|
||||
const severity = stringValue(finding.severity) ?? "unknown";
|
||||
console.log(`- ${label}: ${status} (${severity})`);
|
||||
printIndented("Impact", stringValue(finding.userImpact));
|
||||
printIndented("Recommendation", stringValue(finding.recommendation));
|
||||
}
|
||||
}
|
||||
|
||||
function printIssueList(value: unknown) {
|
||||
if (!Array.isArray(value) || value.length === 0) return;
|
||||
for (const item of value.slice(0, 25)) {
|
||||
const issue = asRecord(item);
|
||||
if (!issue) continue;
|
||||
const code = stringValue(issue.code ?? issue.issueId ?? issue.pattern) ?? "issue";
|
||||
const severity = stringValue(issue.severity) ?? "unknown";
|
||||
const file = stringValue(issue.file);
|
||||
const message = stringValue(issue.message ?? issue.explanation ?? issue.finding);
|
||||
console.log(`- ${code}: ${severity}${file ? ` in ${file}` : ""}`);
|
||||
printIndented("Detail", message);
|
||||
printIndented("Remediation", stringValue(issue.remediation));
|
||||
}
|
||||
}
|
||||
|
||||
function printIndented(label: string, value: string | undefined) {
|
||||
if (!value) return;
|
||||
console.log(` ${label}: ${value}`);
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): ReportRecord | null {
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as ReportRecord)
|
||||
: null;
|
||||
}
|
||||
|
||||
function stringValue(value: unknown) {
|
||||
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function upperValue(value: unknown) {
|
||||
return stringValue(value)?.toUpperCase();
|
||||
}
|
||||
|
||||
function numberValue(value: unknown) {
|
||||
return typeof value === "number" && Number.isFinite(value) ? String(value) : undefined;
|
||||
}
|
||||
|
||||
function arrayValue(value: unknown) {
|
||||
return Array.isArray(value) && value.length > 0
|
||||
? value.map((item) => String(item)).join(", ")
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function dateValue(value: unknown) {
|
||||
return typeof value === "number" && Number.isFinite(value)
|
||||
? new Date(value).toISOString()
|
||||
: undefined;
|
||||
}
|
||||
@@ -116,7 +116,6 @@ afterEach(async () => {
|
||||
vi.spyOn(console, "log").mockImplementation((...args) => {
|
||||
mockLog(args.map(String).join(" "));
|
||||
});
|
||||
|
||||
describe("cmdSync", () => {
|
||||
it("classifies skills as new/update/synced (dry-run, mocked HTTP)", async () => {
|
||||
interactive = false;
|
||||
@@ -151,6 +150,84 @@ describe("cmdSync", () => {
|
||||
expect(String(dryRunOutro)).toMatch(/Dry run: would upload 2 skill/);
|
||||
});
|
||||
|
||||
it("emits CI JSON dry-run without requiring auth", async () => {
|
||||
interactive = false;
|
||||
const stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
|
||||
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
|
||||
if (args.path.startsWith("/api/v1/resolve?")) {
|
||||
const u = new URL(`https://x.test${args.path}`);
|
||||
const slug = u.searchParams.get("slug");
|
||||
if (slug === "new-skill") {
|
||||
throw new Error("Skill not found");
|
||||
}
|
||||
if (slug === "synced-skill") {
|
||||
return { match: { version: "1.2.3" }, latestVersion: { version: "1.2.3" } };
|
||||
}
|
||||
if (slug === "update-skill") {
|
||||
return { match: null, latestVersion: { version: "1.0.0" } };
|
||||
}
|
||||
}
|
||||
throw new Error(`Unexpected apiRequest: ${args.path}`);
|
||||
});
|
||||
|
||||
let output = "";
|
||||
try {
|
||||
await cmdSync(
|
||||
makeOpts(),
|
||||
{
|
||||
root: ["/scan"],
|
||||
all: true,
|
||||
dryRun: true,
|
||||
json: true,
|
||||
owner: "nvidia",
|
||||
sourceRepo: "NVIDIA/skills",
|
||||
sourceCommit: "abc123",
|
||||
sourceRef: "refs/heads/main",
|
||||
},
|
||||
false,
|
||||
);
|
||||
output = String(stdoutWrite.mock.calls.at(-1)?.[0] ?? "").trim();
|
||||
} finally {
|
||||
stdoutWrite.mockRestore();
|
||||
}
|
||||
|
||||
expect(authTokenMocks.requireAuthToken).not.toHaveBeenCalled();
|
||||
expect(mockCmdPublish).not.toHaveBeenCalled();
|
||||
expect(mockLog).not.toHaveBeenCalled();
|
||||
expect(mockIntro).not.toHaveBeenCalled();
|
||||
expect(mockOutro).not.toHaveBeenCalled();
|
||||
|
||||
const parsed = JSON.parse(output) as {
|
||||
ok: boolean;
|
||||
dryRun: boolean;
|
||||
owner?: string;
|
||||
summary: { wouldPublish: number; alreadySynced: number; failed: number };
|
||||
wouldPublish: Array<{
|
||||
slug: string;
|
||||
version: string;
|
||||
status: string;
|
||||
source?: { repo: string };
|
||||
}>;
|
||||
alreadySynced: Array<{ slug: string; version: string }>;
|
||||
published: unknown[];
|
||||
failed: unknown[];
|
||||
};
|
||||
expect(parsed.ok).toBe(true);
|
||||
expect(parsed.dryRun).toBe(true);
|
||||
expect(parsed.owner).toBe("nvidia");
|
||||
expect(parsed.summary).toMatchObject({ wouldPublish: 2, alreadySynced: 1, failed: 0 });
|
||||
expect(parsed.wouldPublish.map((entry) => [entry.slug, entry.version, entry.status])).toEqual([
|
||||
["new-skill", "1.0.0", "new"],
|
||||
["update-skill", "1.0.1", "update"],
|
||||
]);
|
||||
expect(parsed.wouldPublish[0]?.source?.repo).toBe("NVIDIA/skills");
|
||||
expect(parsed.alreadySynced).toEqual([
|
||||
expect.objectContaining({ slug: "synced-skill", version: "1.2.3" }),
|
||||
]);
|
||||
expect(parsed.published).toEqual([]);
|
||||
expect(parsed.failed).toEqual([]);
|
||||
});
|
||||
|
||||
it("prints bullet lists and selects all actionable by default", async () => {
|
||||
interactive = true;
|
||||
mockMultiselect.mockImplementation(async (args?: unknown) => {
|
||||
@@ -191,6 +268,108 @@ describe("cmdSync", () => {
|
||||
expect(mockCmdPublish).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("passes owner and source provenance into real bulk publishes", async () => {
|
||||
interactive = false;
|
||||
const opts = makeGlobalOpts("/repo");
|
||||
const { findSkillFolders } = await import("../scanSkills.js");
|
||||
mocked(findSkillFolders).mockImplementation(async (root: string) => {
|
||||
if (root !== "/repo/skills") return [];
|
||||
return [
|
||||
{ folder: "/repo/skills/new-skill", slug: "new-skill", displayName: "New Skill" },
|
||||
{ folder: "/repo/skills/update-skill", slug: "update-skill", displayName: "Update Skill" },
|
||||
];
|
||||
});
|
||||
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
|
||||
if (args.path === "/api/v1/whoami") return { user: { handle: "mikehollinger" } };
|
||||
if (args.path === "/api/cli/telemetry/sync") return { ok: true };
|
||||
if (args.path.startsWith("/api/v1/resolve?")) {
|
||||
const u = new URL(`https://x.test${args.path}`);
|
||||
const slug = u.searchParams.get("slug");
|
||||
if (slug === "new-skill") throw new Error("Skill not found");
|
||||
if (slug === "update-skill") {
|
||||
return { match: null, latestVersion: { version: "1.0.0" } };
|
||||
}
|
||||
}
|
||||
throw new Error(`Unexpected apiRequest: ${args.path}`);
|
||||
});
|
||||
|
||||
await cmdSync(
|
||||
opts,
|
||||
{
|
||||
root: ["/repo/skills"],
|
||||
all: true,
|
||||
dryRun: false,
|
||||
owner: "nvidia",
|
||||
tags: "latest,catalog",
|
||||
sourceRepo: "https://github.com/NVIDIA/skills",
|
||||
sourceCommit: "abc123",
|
||||
sourceRef: "refs/heads/main",
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
expect(mockCmdPublish).toHaveBeenCalledTimes(2);
|
||||
expect(mockCmdPublish.mock.calls.map((call) => call[2])).toEqual([
|
||||
expect.objectContaining({
|
||||
slug: "new-skill",
|
||||
owner: "nvidia",
|
||||
version: "1.0.0",
|
||||
tags: "latest,catalog",
|
||||
sourceRepo: "https://github.com/NVIDIA/skills",
|
||||
sourceCommit: "abc123",
|
||||
sourceRef: "refs/heads/main",
|
||||
sourcePath: "skills/new-skill",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
slug: "update-skill",
|
||||
owner: "nvidia",
|
||||
version: "1.0.1",
|
||||
tags: "latest,catalog",
|
||||
sourceRepo: "https://github.com/NVIDIA/skills",
|
||||
sourceCommit: "abc123",
|
||||
sourceRef: "refs/heads/main",
|
||||
sourcePath: "skills/update-skill",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses dot source path for a root skill publish", async () => {
|
||||
interactive = false;
|
||||
const opts = makeGlobalOpts("/repo");
|
||||
const { findSkillFolders } = await import("../scanSkills.js");
|
||||
mocked(findSkillFolders).mockImplementation(async (root: string) => {
|
||||
if (root !== "/repo") return [];
|
||||
return [{ folder: "/repo", slug: "root-skill", displayName: "Root Skill" }];
|
||||
});
|
||||
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
|
||||
if (args.path === "/api/v1/whoami") return { user: { handle: "mikehollinger" } };
|
||||
if (args.path === "/api/cli/telemetry/sync") return { ok: true };
|
||||
if (args.path.startsWith("/api/v1/resolve?")) {
|
||||
throw new Error("Skill not found");
|
||||
}
|
||||
throw new Error(`Unexpected apiRequest: ${args.path}`);
|
||||
});
|
||||
|
||||
await cmdSync(
|
||||
opts,
|
||||
{
|
||||
all: true,
|
||||
dryRun: false,
|
||||
sourceRepo: "NVIDIA/root-skill",
|
||||
sourceCommit: "abc123",
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
expect(mockCmdPublish).toHaveBeenCalledTimes(1);
|
||||
expect(mockCmdPublish.mock.calls[0]?.[2]).toEqual(
|
||||
expect.objectContaining({
|
||||
slug: "root-skill",
|
||||
sourcePath: ".",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("labels unmatched local content as proposed publish versions, not registry updates", async () => {
|
||||
interactive = false;
|
||||
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
|
||||
@@ -300,6 +479,71 @@ describe("cmdSync", () => {
|
||||
expect(output).toMatch(/Agent: Work/);
|
||||
});
|
||||
|
||||
it("can disable auto-discovered clawdbot roots for CI exact scans", async () => {
|
||||
interactive = false;
|
||||
const stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
|
||||
const scannedRoots: string[] = [];
|
||||
const { findSkillFolders } = await import("../scanSkills.js");
|
||||
mocked(findSkillFolders).mockImplementation(async (root: string) => {
|
||||
scannedRoots.push(root);
|
||||
if (root === "/scan") {
|
||||
return [{ folder: "/scan/ci-skill", slug: "ci-skill", displayName: "CI Skill" }];
|
||||
}
|
||||
if (root === "/auto") {
|
||||
return [{ folder: "/auto/auto-skill", slug: "auto-skill", displayName: "Auto Skill" }];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
mockResolveClawdbotSkillRoots.mockResolvedValueOnce({
|
||||
roots: ["/auto"],
|
||||
labels: { "/auto": "Agent: Work" },
|
||||
});
|
||||
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
|
||||
if (args.path.startsWith("/api/v1/resolve?")) {
|
||||
throw new Error("Skill not found");
|
||||
}
|
||||
throw new Error(`Unexpected apiRequest: ${args.path}`);
|
||||
});
|
||||
|
||||
let output = "";
|
||||
try {
|
||||
await cmdSync(
|
||||
makeOpts(),
|
||||
{ root: ["/scan"], all: true, dryRun: true, json: true, clawdbotRoots: false },
|
||||
false,
|
||||
);
|
||||
output = String(stdoutWrite.mock.calls.at(-1)?.[0] ?? "").trim();
|
||||
} finally {
|
||||
stdoutWrite.mockRestore();
|
||||
}
|
||||
|
||||
expect(mockResolveClawdbotSkillRoots).not.toHaveBeenCalled();
|
||||
expect(scannedRoots).not.toContain("/auto");
|
||||
const parsed = JSON.parse(output) as {
|
||||
roots: string[];
|
||||
wouldPublish: Array<{ slug: string }>;
|
||||
};
|
||||
expect(parsed.roots).toEqual(["/work", "/work/skills", "/scan"]);
|
||||
expect(parsed.wouldPublish.map((entry) => entry.slug)).toEqual(["ci-skill"]);
|
||||
});
|
||||
|
||||
it("does not fall back to ambient roots when exact CI scans find no skills", async () => {
|
||||
interactive = false;
|
||||
const { findSkillFolders, getFallbackSkillRoots } = await import("../scanSkills.js");
|
||||
mocked(findSkillFolders).mockImplementation(async () => []);
|
||||
|
||||
await expect(
|
||||
cmdSync(
|
||||
makeOpts(),
|
||||
{ root: ["/scan"], all: true, dryRun: true, json: true, clawdbotRoots: false },
|
||||
false,
|
||||
),
|
||||
).rejects.toThrow("No skills found (checked configured roots)");
|
||||
|
||||
expect(mockResolveClawdbotSkillRoots).not.toHaveBeenCalled();
|
||||
expect(getFallbackSkillRoots).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows empty changelog for updates (interactive)", async () => {
|
||||
interactive = true;
|
||||
mockApiRequest.mockImplementation(async (_registry: string, args: { path: string }) => {
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { isAbsolute, relative } from "node:path";
|
||||
import { intro, outro } from "@clack/prompts";
|
||||
import { hashSkillFiles, listTextFiles, readSkillOrigin } from "../../skills.js";
|
||||
import { requireAuthToken } from "../authToken.js";
|
||||
import { getOptionalAuthToken, requireAuthToken } from "../authToken.js";
|
||||
import { resolveClawdbotSkillRoots } from "../clawdbotConfig.js";
|
||||
import { getRegistry } from "../registry.js";
|
||||
import { getFallbackSkillRoots } from "../scanSkills.js";
|
||||
import type { GlobalOpts } from "../types.js";
|
||||
import { createSpinner, fail, formatError, isInteractive } from "../ui.js";
|
||||
import { normalizeGitHubRepo } from "./github.js";
|
||||
import { cmdPublish } from "./publish.js";
|
||||
import {
|
||||
buildScanRoots,
|
||||
@@ -29,53 +32,64 @@ import {
|
||||
import type { Candidate, LocalSkill, SyncOptions } from "./syncTypes.js";
|
||||
|
||||
export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllowed: boolean) {
|
||||
const allowPrompt = isInteractive() && inputAllowed !== false;
|
||||
intro("ClawHub sync");
|
||||
const jsonMode = options.json === true;
|
||||
const allowPrompt = !jsonMode && isInteractive() && inputAllowed !== false;
|
||||
if (!jsonMode) intro("ClawHub sync");
|
||||
|
||||
const token = await requireAuthToken();
|
||||
const token = options.dryRun ? await getOptionalAuthToken() : await requireAuthToken();
|
||||
|
||||
const registry = await getRegistryWithAuth(opts, token);
|
||||
const registry = token
|
||||
? await getRegistryWithAuth(opts, token)
|
||||
: await getRegistry(opts, { cache: true });
|
||||
const selectedRoots = buildScanRoots(opts, options.root);
|
||||
const clawdbotRoots = await resolveClawdbotSkillRoots();
|
||||
const includeClawdbotRoots = options.clawdbotRoots !== false;
|
||||
const clawdbotRoots = includeClawdbotRoots
|
||||
? await resolveClawdbotSkillRoots()
|
||||
: { roots: [], labels: {} };
|
||||
const combinedRoots = Array.from(
|
||||
new Set([...selectedRoots, ...clawdbotRoots.roots].map((root) => root.trim()).filter(Boolean)),
|
||||
);
|
||||
const concurrency = normalizeConcurrency(options.concurrency);
|
||||
|
||||
const spinner = createSpinner("Scanning for local skills");
|
||||
const spinner = jsonMode ? null : createSpinner("Scanning for local skills");
|
||||
const primaryScan = await scanRootsWithLabels(combinedRoots, clawdbotRoots.labels);
|
||||
let scan = primaryScan;
|
||||
let telemetryScan = primaryScan;
|
||||
if (primaryScan.skills.length === 0) {
|
||||
if (!includeClawdbotRoots) {
|
||||
fail("No skills found (checked configured roots)");
|
||||
}
|
||||
const fallback = getFallbackSkillRoots(opts.workdir);
|
||||
const fallbackScan = await scanRootsWithLabels(fallback);
|
||||
spinner.stop();
|
||||
spinner?.stop();
|
||||
telemetryScan = mergeScan(primaryScan, fallbackScan);
|
||||
scan = fallbackScan;
|
||||
if (fallbackScan.skills.length === 0)
|
||||
fail("No skills found (checked workdir and known Clawdis/Clawd locations)");
|
||||
printSection(
|
||||
`No skills in workdir. Found ${fallbackScan.skills.length} in fallback locations.`,
|
||||
formatList(fallbackScan.rootsWithSkills, 10),
|
||||
);
|
||||
if (!jsonMode) {
|
||||
printSection(
|
||||
`No skills in workdir. Found ${fallbackScan.skills.length} in fallback locations.`,
|
||||
formatList(fallbackScan.rootsWithSkills, 10),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
spinner.stop();
|
||||
spinner?.stop();
|
||||
const labeledRoots = primaryScan.rootsWithSkills
|
||||
.map((root) => {
|
||||
const label = primaryScan.rootLabels?.[root];
|
||||
return label ? `${label} (${root})` : root;
|
||||
})
|
||||
.filter(Boolean);
|
||||
if (labeledRoots.length > 0) {
|
||||
if (!jsonMode && labeledRoots.length > 0) {
|
||||
printSection("Roots with skills", formatList(labeledRoots, 10));
|
||||
}
|
||||
}
|
||||
const deduped = dedupeSkillsBySlug(scan.skills);
|
||||
const skills = deduped.skills;
|
||||
if (deduped.duplicates.length > 0) {
|
||||
if (!jsonMode && deduped.duplicates.length > 0) {
|
||||
printSection("Skipped duplicate slugs", formatCommaList(deduped.duplicates, 16));
|
||||
}
|
||||
const parsingSpinner = createSpinner("Parsing local skills");
|
||||
const parsingSpinner = jsonMode ? null : createSpinner("Parsing local skills");
|
||||
const locals: LocalSkill[] = [];
|
||||
try {
|
||||
let done = 0;
|
||||
@@ -84,7 +98,7 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
|
||||
const hashed = hashSkillFiles(filesOnDisk);
|
||||
const origin = await readSkillOrigin(skill.folder);
|
||||
done += 1;
|
||||
parsingSpinner.text = `Parsing local skills ${done}/${skills.length}`;
|
||||
if (parsingSpinner) parsingSpinner.text = `Parsing local skills ${done}/${skills.length}`;
|
||||
return {
|
||||
...skill,
|
||||
fingerprint: hashed.fingerprint,
|
||||
@@ -94,13 +108,13 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
|
||||
});
|
||||
locals.push(...parsed);
|
||||
} catch (error) {
|
||||
parsingSpinner.fail(formatError(error));
|
||||
parsingSpinner?.fail(formatError(error));
|
||||
throw error;
|
||||
} finally {
|
||||
parsingSpinner.stop();
|
||||
parsingSpinner?.stop();
|
||||
}
|
||||
|
||||
const candidatesSpinner = createSpinner("Checking registry sync state");
|
||||
const candidatesSpinner = jsonMode ? null : createSpinner("Checking registry sync state");
|
||||
const candidates: Candidate[] = [];
|
||||
const resolveSupport: { value: boolean | null } = { value: null };
|
||||
try {
|
||||
@@ -110,29 +124,50 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
|
||||
return await checkRegistrySyncState(registry, skill, resolveSupport, token);
|
||||
} finally {
|
||||
done += 1;
|
||||
candidatesSpinner.text = `Checking registry sync state ${done}/${locals.length}`;
|
||||
if (candidatesSpinner) {
|
||||
candidatesSpinner.text = `Checking registry sync state ${done}/${locals.length}`;
|
||||
}
|
||||
}
|
||||
});
|
||||
candidates.push(...resolved);
|
||||
} catch (error) {
|
||||
candidatesSpinner.fail(formatError(error));
|
||||
candidatesSpinner?.fail(formatError(error));
|
||||
throw error;
|
||||
} finally {
|
||||
candidatesSpinner.stop();
|
||||
candidatesSpinner?.stop();
|
||||
}
|
||||
|
||||
await reportTelemetryIfEnabled({
|
||||
token,
|
||||
registry,
|
||||
scan: telemetryScan,
|
||||
candidates,
|
||||
});
|
||||
if (token) {
|
||||
await reportTelemetryIfEnabled({
|
||||
token,
|
||||
registry,
|
||||
scan: telemetryScan,
|
||||
candidates,
|
||||
});
|
||||
}
|
||||
|
||||
const synced = candidates.filter((candidate) => candidate.status === "synced");
|
||||
const actionable = candidates.filter((candidate) => candidate.status !== "synced");
|
||||
const bump = options.bump ?? "patch";
|
||||
|
||||
if (actionable.length === 0) {
|
||||
if (jsonMode) {
|
||||
writeSyncJson(
|
||||
buildSyncJsonOutput({
|
||||
ok: true,
|
||||
dryRun: Boolean(options.dryRun),
|
||||
registry,
|
||||
roots: combinedRoots,
|
||||
owner: normalizeOwner(options.owner),
|
||||
duplicates: deduped.duplicates,
|
||||
alreadySynced: synced.map(formatSyncedJson),
|
||||
wouldPublish: [],
|
||||
published: [],
|
||||
failed: [],
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (synced.length > 0) {
|
||||
printSection("Already synced", formatCommaList(synced.map(formatSyncedSummary), 16));
|
||||
}
|
||||
@@ -140,14 +175,16 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
|
||||
return;
|
||||
}
|
||||
|
||||
printSection(
|
||||
"To sync",
|
||||
formatBulletList(
|
||||
actionable.map((candidate) => formatActionableLine(candidate, bump)),
|
||||
20,
|
||||
),
|
||||
);
|
||||
if (synced.length > 0) {
|
||||
if (!jsonMode) {
|
||||
printSection(
|
||||
"To sync",
|
||||
formatBulletList(
|
||||
actionable.map((candidate) => formatActionableLine(candidate, bump)),
|
||||
20,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (!jsonMode && synced.length > 0) {
|
||||
printSection("Already synced", formatSyncedDisplay(synced));
|
||||
}
|
||||
|
||||
@@ -157,11 +194,60 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
|
||||
bump,
|
||||
});
|
||||
if (selected.length === 0) {
|
||||
if (jsonMode) {
|
||||
writeSyncJson(
|
||||
buildSyncJsonOutput({
|
||||
ok: true,
|
||||
dryRun: Boolean(options.dryRun),
|
||||
registry,
|
||||
roots: combinedRoots,
|
||||
owner: normalizeOwner(options.owner),
|
||||
duplicates: deduped.duplicates,
|
||||
alreadySynced: synced.map(formatSyncedJson),
|
||||
wouldPublish: [],
|
||||
published: [],
|
||||
failed: [],
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
outro("Nothing selected.");
|
||||
return;
|
||||
}
|
||||
|
||||
const plannedPublishes = selected.map((skill) => {
|
||||
const source = buildSourceProvenance(opts, skill, options);
|
||||
return { skill, source };
|
||||
});
|
||||
|
||||
if (options.dryRun) {
|
||||
const wouldPublish = await Promise.all(
|
||||
plannedPublishes.map(async ({ skill, source }) => {
|
||||
const { publishVersion } = await resolvePublishMeta(skill, {
|
||||
bump,
|
||||
allowPrompt,
|
||||
changelogFlag: options.changelog,
|
||||
});
|
||||
return formatPublishJson(skill, publishVersion, source);
|
||||
}),
|
||||
);
|
||||
if (jsonMode) {
|
||||
writeSyncJson(
|
||||
buildSyncJsonOutput({
|
||||
ok: true,
|
||||
dryRun: true,
|
||||
registry,
|
||||
roots: combinedRoots,
|
||||
owner: normalizeOwner(options.owner),
|
||||
duplicates: deduped.duplicates,
|
||||
alreadySynced: synced.map(formatSyncedJson),
|
||||
wouldPublish,
|
||||
published: [],
|
||||
failed: [],
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
outro(`Dry run: would upload ${selected.length} skill(s).`);
|
||||
return;
|
||||
}
|
||||
@@ -170,7 +256,8 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
|
||||
const failedUploads: Array<{ slug: string; message: string }> = [];
|
||||
let uploaded = 0;
|
||||
|
||||
for (const skill of selected) {
|
||||
const published: Array<{ slug: string; folder: string; version: string }> = [];
|
||||
for (const { skill, source } of plannedPublishes) {
|
||||
const { publishVersion, changelog } = await resolvePublishMeta(skill, {
|
||||
bump,
|
||||
allowPrompt,
|
||||
@@ -186,18 +273,46 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
|
||||
await cmdPublish(opts, skill.folder, {
|
||||
slug: skill.slug,
|
||||
name: skill.displayName,
|
||||
owner: normalizeOwner(options.owner),
|
||||
version: publishVersion,
|
||||
changelog,
|
||||
tags,
|
||||
forkOf,
|
||||
...(source
|
||||
? {
|
||||
sourceRepo: options.sourceRepo,
|
||||
sourceCommit: options.sourceCommit,
|
||||
sourceRef: options.sourceRef,
|
||||
sourcePath: source.path,
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
uploaded += 1;
|
||||
published.push({ slug: skill.slug, folder: skill.folder, version: publishVersion });
|
||||
} catch (error) {
|
||||
failedUploads.push({ slug: skill.slug, message: formatError(error) });
|
||||
}
|
||||
}
|
||||
|
||||
if (failedUploads.length > 0) {
|
||||
if (jsonMode) {
|
||||
writeSyncJson(
|
||||
buildSyncJsonOutput({
|
||||
ok: false,
|
||||
dryRun: false,
|
||||
registry,
|
||||
roots: combinedRoots,
|
||||
owner: normalizeOwner(options.owner),
|
||||
duplicates: deduped.duplicates,
|
||||
alreadySynced: synced.map(formatSyncedJson),
|
||||
wouldPublish: [],
|
||||
published,
|
||||
failed: failedUploads,
|
||||
}),
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
printSection(
|
||||
"Failed to upload",
|
||||
formatBulletList(
|
||||
@@ -210,9 +325,140 @@ export async function cmdSync(opts: GlobalOpts, options: SyncOptions, inputAllow
|
||||
return;
|
||||
}
|
||||
|
||||
if (jsonMode) {
|
||||
writeSyncJson(
|
||||
buildSyncJsonOutput({
|
||||
ok: true,
|
||||
dryRun: false,
|
||||
registry,
|
||||
roots: combinedRoots,
|
||||
owner: normalizeOwner(options.owner),
|
||||
duplicates: deduped.duplicates,
|
||||
alreadySynced: synced.map(formatSyncedJson),
|
||||
wouldPublish: [],
|
||||
published,
|
||||
failed: [],
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
outro(`Uploaded ${selected.length} skill(s).`);
|
||||
}
|
||||
|
||||
function normalizeRegistry(value: string) {
|
||||
return value.trim().replace(/\/+$/, "").toLowerCase();
|
||||
}
|
||||
|
||||
function normalizeOwner(value: string | undefined) {
|
||||
return value?.trim().replace(/^@+/, "") || undefined;
|
||||
}
|
||||
|
||||
function formatSyncedJson(candidate: Candidate) {
|
||||
return {
|
||||
slug: candidate.slug,
|
||||
folder: candidate.folder,
|
||||
version: candidate.matchVersion ?? candidate.latestVersion ?? "unknown",
|
||||
};
|
||||
}
|
||||
|
||||
function formatPublishJson(
|
||||
candidate: Candidate,
|
||||
version: string,
|
||||
source: ReturnType<typeof buildSourceProvenance>,
|
||||
) {
|
||||
return {
|
||||
slug: candidate.slug,
|
||||
displayName: candidate.displayName,
|
||||
folder: candidate.folder,
|
||||
status: candidate.status,
|
||||
version,
|
||||
latestVersion: candidate.latestVersion,
|
||||
fileCount: candidate.fileCount,
|
||||
fingerprint: candidate.fingerprint,
|
||||
...(source ? { source } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function buildSourceProvenance(opts: GlobalOpts, skill: Candidate, options: SyncOptions) {
|
||||
const rawRepo = options.sourceRepo?.trim();
|
||||
const commit = options.sourceCommit?.trim();
|
||||
if (!rawRepo && !commit && !options.sourceRef?.trim()) return undefined;
|
||||
if (!rawRepo || !commit) fail("--source-repo and --source-commit must be provided together");
|
||||
const repo = normalizeGitHubRepo(rawRepo);
|
||||
if (!repo) fail("--source-repo must be a GitHub repo or URL");
|
||||
return {
|
||||
kind: "github" as const,
|
||||
url: `https://github.com/${repo}`,
|
||||
repo,
|
||||
ref: options.sourceRef?.trim() || commit,
|
||||
commit,
|
||||
path: sourcePathForSkill(opts, skill.folder),
|
||||
};
|
||||
}
|
||||
|
||||
function sourcePathForSkill(opts: GlobalOpts, folder: string) {
|
||||
return (
|
||||
relativeInside(process.cwd(), folder) ??
|
||||
relativeInside(opts.workdir, folder) ??
|
||||
relativeInside(opts.dir, folder) ??
|
||||
normalizeSourcePath(folder)
|
||||
);
|
||||
}
|
||||
|
||||
function relativeInside(base: string, target: string) {
|
||||
const rel = relative(base, target);
|
||||
if (!rel) return ".";
|
||||
if (rel.startsWith("..") || isAbsolute(rel)) return null;
|
||||
return normalizeSourcePath(rel);
|
||||
}
|
||||
|
||||
function normalizeSourcePath(value: string) {
|
||||
const normalized = value
|
||||
.replaceAll("\\", "/")
|
||||
.replace(/^\.\/+/, "")
|
||||
.replace(/\/+$/, "");
|
||||
return normalized || ".";
|
||||
}
|
||||
|
||||
function buildSyncJsonOutput(params: {
|
||||
ok: boolean;
|
||||
dryRun: boolean;
|
||||
registry: string;
|
||||
roots: string[];
|
||||
owner?: string;
|
||||
duplicates: string[];
|
||||
alreadySynced: Array<{ slug: string; folder: string; version: string }>;
|
||||
wouldPublish: Array<ReturnType<typeof formatPublishJson>>;
|
||||
published: Array<{ slug: string; folder: string; version: string }>;
|
||||
failed: Array<{ slug: string; message: string }>;
|
||||
}) {
|
||||
const skipped = params.duplicates.map((duplicate) => ({
|
||||
slug: duplicate.replace(/\s+\(\d+\)$/, ""),
|
||||
reason: "duplicate-slug",
|
||||
detail: duplicate,
|
||||
}));
|
||||
return {
|
||||
ok: params.ok,
|
||||
dryRun: params.dryRun,
|
||||
registry: params.registry,
|
||||
roots: params.roots,
|
||||
...(params.owner ? { owner: params.owner } : {}),
|
||||
summary: {
|
||||
wouldPublish: params.wouldPublish.length,
|
||||
published: params.published.length,
|
||||
alreadySynced: params.alreadySynced.length,
|
||||
skipped: skipped.length,
|
||||
failed: params.failed.length,
|
||||
},
|
||||
wouldPublish: params.wouldPublish,
|
||||
published: params.published,
|
||||
alreadySynced: params.alreadySynced,
|
||||
skipped,
|
||||
failed: params.failed,
|
||||
};
|
||||
}
|
||||
|
||||
function writeSyncJson(value: unknown) {
|
||||
process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
@@ -5,10 +5,16 @@ export type SyncOptions = {
|
||||
root?: string[];
|
||||
all?: boolean;
|
||||
dryRun?: boolean;
|
||||
json?: boolean;
|
||||
owner?: string;
|
||||
bump?: "patch" | "minor" | "major";
|
||||
changelog?: string;
|
||||
tags?: string;
|
||||
concurrency?: number;
|
||||
clawdbotRoots?: boolean;
|
||||
sourceRepo?: string;
|
||||
sourceCommit?: string;
|
||||
sourceRef?: string;
|
||||
};
|
||||
|
||||
export type Candidate = SkillFolder & {
|
||||
|
||||
@@ -52,6 +52,12 @@ type FormRequestArgs =
|
||||
| { method: "POST"; url: string; token?: string; form: FormData; retryCount?: number };
|
||||
|
||||
type TextRequestArgs = { path: string; token?: string } | { url: string; token?: string };
|
||||
type BinaryUploadArgs = {
|
||||
url: string;
|
||||
bytes: Uint8Array;
|
||||
contentType?: string;
|
||||
retryCount?: number;
|
||||
};
|
||||
|
||||
type HeaderSource = Headers | Record<string, string> | null | undefined;
|
||||
|
||||
@@ -91,6 +97,7 @@ type HttpClient = {
|
||||
apiRequestForm<T>(registry: string, args: FormRequestArgs, schema: ArkValidator<T>): Promise<T>;
|
||||
fetchText(registry: string, args: TextRequestArgs): Promise<string>;
|
||||
fetchBinary(registry: string, args: TextRequestArgs): Promise<Uint8Array>;
|
||||
uploadBinary<T>(args: BinaryUploadArgs, schema?: ArkValidator<T>): Promise<T>;
|
||||
downloadZip(
|
||||
registry: string,
|
||||
args: { slug: string; version?: string; token?: string },
|
||||
@@ -264,6 +271,41 @@ export function createHttpClient(options: HttpClientOptions = {}): HttpClient {
|
||||
});
|
||||
}
|
||||
|
||||
async function uploadBinaryRequest<T>(
|
||||
args: BinaryUploadArgs,
|
||||
schema?: ArkValidator<T>,
|
||||
): Promise<T> {
|
||||
const json = await runWithRetries(async () => {
|
||||
if (deps.runtime === "bun") {
|
||||
return await uploadBinaryViaCurl(deps, args);
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (args.contentType) headers["Content-Type"] = args.contentType;
|
||||
const response = await fetchWithTimeout(
|
||||
deps,
|
||||
args.url,
|
||||
{
|
||||
method: "POST",
|
||||
headers,
|
||||
body: bytesToArrayBuffer(args.bytes),
|
||||
},
|
||||
UPLOAD_TIMEOUT_MS,
|
||||
);
|
||||
if (!response.ok) {
|
||||
throwHttpStatusError(
|
||||
response.status,
|
||||
await readResponseTextSafe(response),
|
||||
response.headers,
|
||||
deps.now,
|
||||
);
|
||||
}
|
||||
return (await response.json()) as unknown;
|
||||
}, args.retryCount);
|
||||
if (schema) return parseArk(schema, json, "API response");
|
||||
return json as T;
|
||||
}
|
||||
|
||||
async function downloadZipRequest(
|
||||
registry: string,
|
||||
args: { slug: string; version?: string; token?: string },
|
||||
@@ -296,6 +338,7 @@ export function createHttpClient(options: HttpClientOptions = {}): HttpClient {
|
||||
apiRequestForm,
|
||||
fetchText: fetchTextRequest,
|
||||
fetchBinary: fetchBinaryRequest,
|
||||
uploadBinary: uploadBinaryRequest,
|
||||
downloadZip: downloadZipRequest,
|
||||
};
|
||||
}
|
||||
@@ -361,6 +404,15 @@ export async function fetchBinary(registry: string, args: TextRequestArgs): Prom
|
||||
return await defaultHttpClient.fetchBinary(registry, args);
|
||||
}
|
||||
|
||||
export async function uploadBinary<T>(args: BinaryUploadArgs): Promise<T>;
|
||||
export async function uploadBinary<T>(args: BinaryUploadArgs, schema: ArkValidator<T>): Promise<T>;
|
||||
export async function uploadBinary<T>(
|
||||
args: BinaryUploadArgs,
|
||||
schema?: ArkValidator<T>,
|
||||
): Promise<T> {
|
||||
return await defaultHttpClient.uploadBinary<T>(args, schema);
|
||||
}
|
||||
|
||||
export async function downloadZip(
|
||||
registry: string,
|
||||
args: { slug: string; version?: string; token?: string },
|
||||
@@ -388,6 +440,12 @@ function createRetryRunner(deps: Pick<HttpClientDeps, "setTimeoutImpl" | "random
|
||||
};
|
||||
}
|
||||
|
||||
function bytesToArrayBuffer(bytes: Uint8Array) {
|
||||
const copy = new Uint8Array(bytes.byteLength);
|
||||
copy.set(bytes);
|
||||
return copy.buffer;
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(
|
||||
deps: Pick<HttpClientDeps, "fetchImpl" | "setTimeoutImpl" | "clearTimeoutImpl">,
|
||||
url: string,
|
||||
@@ -699,6 +757,45 @@ async function fetchJsonFormViaCurl(
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadBinaryViaCurl(
|
||||
deps: Pick<
|
||||
HttpClientDeps,
|
||||
"spawnSyncImpl" | "mkdtempImpl" | "writeFileImpl" | "rmImpl" | "tmpdirPath" | "now"
|
||||
>,
|
||||
args: BinaryUploadArgs,
|
||||
) {
|
||||
const tempDir = await deps.mkdtempImpl(join(deps.tmpdirPath, "clawhub-upload-"));
|
||||
try {
|
||||
const filePath = join(tempDir, "upload.bin");
|
||||
await deps.writeFileImpl(filePath, args.bytes);
|
||||
const curlArgs = [
|
||||
"--silent",
|
||||
"--show-error",
|
||||
"--location",
|
||||
"--max-time",
|
||||
String(UPLOAD_TIMEOUT_SECONDS),
|
||||
"--write-out",
|
||||
CURL_WRITE_OUT_FORMAT,
|
||||
"-X",
|
||||
"POST",
|
||||
];
|
||||
if (args.contentType) curlArgs.push("-H", `Content-Type: ${args.contentType}`);
|
||||
curlArgs.push("--data-binary", `@${filePath}`, args.url);
|
||||
|
||||
const result = deps.spawnSyncImpl("curl", curlArgs, { encoding: "utf8" });
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr || "curl failed");
|
||||
}
|
||||
const { body, status, headers: responseHeaders } = parseCurlBodyAndMeta(result.stdout ?? "");
|
||||
if (status < 200 || status >= 300) {
|
||||
throwHttpStatusError(status, body, responseHeaders, deps.now);
|
||||
}
|
||||
return JSON.parse(body || "null") as unknown;
|
||||
} finally {
|
||||
await deps.rmImpl(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchTextViaCurl(
|
||||
deps: Pick<HttpClientDeps, "spawnSyncImpl" | "now">,
|
||||
url: string,
|
||||
|
||||
@@ -52,6 +52,12 @@ export const PackageVerificationSummarySchema = type({
|
||||
sourceRepo: "string?",
|
||||
sourceCommit: "string?",
|
||||
sourceTag: "string?",
|
||||
// Path of the package directory inside the source repo (e.g.
|
||||
// "examples/openclaw-plugin"). Forward slash separated, no leading or
|
||||
// trailing slash. Used when resolving relative README asset URLs against
|
||||
// raw.githubusercontent.com so that subdirectory packages render correctly.
|
||||
// Absent or "." means the package lives at the repo root.
|
||||
sourcePath: "string?",
|
||||
hasProvenance: "boolean?",
|
||||
scanStatus: '"clean"|"suspicious"|"malicious"|"pending"|"not-run"?',
|
||||
});
|
||||
@@ -229,7 +235,81 @@ export const PackageTrustedPublisherSchema = type({
|
||||
});
|
||||
export type PackageTrustedPublisher = (typeof PackageTrustedPublisherSchema)[inferred];
|
||||
|
||||
export const PackagePublishRequestSchema = type({
|
||||
export const MAX_PACKAGE_MULTIPART_BYTES = 18 * 1024 * 1024;
|
||||
export const MAX_PACKAGE_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
const PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES = 4096;
|
||||
const PACKAGE_MULTIPART_PART_OVERHEAD_BYTES = 1024;
|
||||
|
||||
export type PackageMultipartUploadField = "files" | "clawpack";
|
||||
export type PackageMultipartUploadPart = {
|
||||
name: string;
|
||||
size: number;
|
||||
type?: string;
|
||||
};
|
||||
export type PackageMultipartUploadSizeInput = {
|
||||
payloadJson: string;
|
||||
fileFieldName: PackageMultipartUploadField;
|
||||
files: readonly PackageMultipartUploadPart[];
|
||||
};
|
||||
|
||||
export function estimatePackageMultipartUploadBytes(
|
||||
input: PackageMultipartUploadSizeInput,
|
||||
): number {
|
||||
return (
|
||||
PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES +
|
||||
estimateMultipartStringPartBytes("payload", input.payloadJson) +
|
||||
input.files.reduce(
|
||||
(sum, file) => sum + estimateMultipartFilePartBytes(input.fileFieldName, file),
|
||||
0,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function isPackageMultipartUploadTooLarge(input: PackageMultipartUploadSizeInput): boolean {
|
||||
return estimatePackageMultipartUploadBytes(input) > MAX_PACKAGE_MULTIPART_BYTES;
|
||||
}
|
||||
|
||||
export function getPackageMultipartSizeError(): string {
|
||||
return "Package upload exceeds 18MB multipart upload limit";
|
||||
}
|
||||
|
||||
function estimateMultipartStringPartBytes(fieldName: string, value: string): number {
|
||||
return PACKAGE_MULTIPART_PART_OVERHEAD_BYTES + utf8ByteLength(fieldName) + utf8ByteLength(value);
|
||||
}
|
||||
|
||||
function estimateMultipartFilePartBytes(
|
||||
fieldName: PackageMultipartUploadField,
|
||||
file: PackageMultipartUploadPart,
|
||||
): number {
|
||||
return (
|
||||
file.size +
|
||||
PACKAGE_MULTIPART_PART_OVERHEAD_BYTES +
|
||||
utf8ByteLength(fieldName) +
|
||||
utf8ByteLength(file.name) +
|
||||
utf8ByteLength(file.type ?? "")
|
||||
);
|
||||
}
|
||||
|
||||
function utf8ByteLength(value: string): number {
|
||||
let bytes = 0;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const codePoint = value.codePointAt(index);
|
||||
if (codePoint === undefined) continue;
|
||||
if (codePoint > 0xffff) index += 1;
|
||||
if (codePoint <= 0x7f) {
|
||||
bytes += 1;
|
||||
} else if (codePoint <= 0x7ff) {
|
||||
bytes += 2;
|
||||
} else if (codePoint <= 0xffff) {
|
||||
bytes += 3;
|
||||
} else {
|
||||
bytes += 4;
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
const PackagePublishMetadataFields = {
|
||||
name: "string",
|
||||
displayName: "string?",
|
||||
ownerHandle: "string?",
|
||||
@@ -241,10 +321,21 @@ export const PackagePublishRequestSchema = type({
|
||||
tags: "string[]?",
|
||||
source: PublishSourceSchema.optional(),
|
||||
bundle: BundlePublishMetadataSchema.optional(),
|
||||
} as const;
|
||||
|
||||
export const PackagePublishMetadataSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
});
|
||||
export type PackagePublishMetadata = (typeof PackagePublishMetadataSchema)[inferred];
|
||||
|
||||
export const ServerPackagePublishRequestSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
artifact: PackagePublishArtifactSchema.optional(),
|
||||
files: CliPublishFileSchema.array(),
|
||||
});
|
||||
export type PackagePublishRequest = (typeof PackagePublishRequestSchema)[inferred];
|
||||
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
|
||||
|
||||
export const PackageListItemSchema = type({
|
||||
name: "string",
|
||||
|
||||
@@ -17,6 +17,7 @@ export const ApiRoutes = {
|
||||
download: "/api/v1/download",
|
||||
publishTokenMint: "/api/v1/publish/token/mint",
|
||||
skills: "/api/v1/skills",
|
||||
skillScans: "/api/v1/skills/-/scan",
|
||||
packages: "/api/v1/packages",
|
||||
codePlugins: "/api/v1/code-plugins",
|
||||
bundlePlugins: "/api/v1/bundle-plugins",
|
||||
|
||||
@@ -54,6 +54,7 @@ export const ApiSkillMetaResponseSchema = type({
|
||||
|
||||
export const ApiCliUploadUrlResponseSchema = type({
|
||||
uploadUrl: "string",
|
||||
uploadTicket: "string",
|
||||
});
|
||||
|
||||
export const ApiUploadFileResponseSchema = type({
|
||||
@@ -426,6 +427,66 @@ export const ApiV1SkillRescanResponseSchema = type({
|
||||
});
|
||||
export type ApiV1SkillRescanResponse = (typeof ApiV1SkillRescanResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanStatusSchema = type('"queued"|"running"|"succeeded"|"failed"');
|
||||
export type ApiV1SkillScanStatus = (typeof ApiV1SkillScanStatusSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanSourceSchema = type({
|
||||
kind: '"upload"',
|
||||
}).or({
|
||||
kind: '"published"',
|
||||
slug: "string",
|
||||
version: "string?",
|
||||
});
|
||||
export type ApiV1SkillScanSource = (typeof ApiV1SkillScanSourceSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanSubmitRequestSchema = type({
|
||||
source: ApiV1SkillScanSourceSchema,
|
||||
update: "boolean?",
|
||||
});
|
||||
export type ApiV1SkillScanSubmitRequest = (typeof ApiV1SkillScanSubmitRequestSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanSubmitResponseSchema = type({
|
||||
ok: "true",
|
||||
scanId: "string",
|
||||
jobId: "string?",
|
||||
status: ApiV1SkillScanStatusSchema,
|
||||
sourceKind: '"upload"|"published"',
|
||||
update: "boolean",
|
||||
alreadyQueued: "boolean?",
|
||||
});
|
||||
export type ApiV1SkillScanSubmitResponse = (typeof ApiV1SkillScanSubmitResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanStatusResponseSchema = type({
|
||||
ok: "true",
|
||||
scanId: "string",
|
||||
jobId: "string?",
|
||||
status: ApiV1SkillScanStatusSchema,
|
||||
sourceKind: '"upload"|"published"',
|
||||
update: "boolean",
|
||||
writtenBack: "boolean?",
|
||||
artifact: "unknown?",
|
||||
report: "unknown?",
|
||||
lastError: "string?",
|
||||
createdAt: "number",
|
||||
updatedAt: "number",
|
||||
completedAt: "number?",
|
||||
});
|
||||
export type ApiV1SkillScanStatusResponse = (typeof ApiV1SkillScanStatusResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanDownloadManifestSchema = type({
|
||||
scanId: "string",
|
||||
sourceKind: '"upload"|"published"',
|
||||
update: "boolean",
|
||||
status: ApiV1SkillScanStatusSchema,
|
||||
artifact: "unknown?",
|
||||
createdAt: "number",
|
||||
updatedAt: "number",
|
||||
completedAt: "number?",
|
||||
writtenBack: "boolean?",
|
||||
});
|
||||
export type ApiV1SkillScanDownloadManifest =
|
||||
(typeof ApiV1SkillScanDownloadManifestSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillBulkRescanBatchRequestSchema = type({
|
||||
mode: '"all-active-latest"?',
|
||||
cursor: "string|null?",
|
||||
@@ -470,6 +531,48 @@ export const ApiV1SkillBulkRescanStatusResponseSchema = type({
|
||||
export type ApiV1SkillBulkRescanStatusResponse =
|
||||
(typeof ApiV1SkillBulkRescanStatusResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanBatchRequestSchema = type({
|
||||
mode: '"all-active-latest"?',
|
||||
cursor: "string|null?",
|
||||
batchSize: "number?",
|
||||
dryRun: "boolean?",
|
||||
});
|
||||
export type ApiV1SkillScanBatchRequest = (typeof ApiV1SkillScanBatchRequestSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanBatchResponseSchema = type({
|
||||
ok: "true",
|
||||
mode: '"all-active-latest"',
|
||||
queued: "number",
|
||||
alreadyQueued: "number",
|
||||
skipped: "number",
|
||||
jobIds: "string[]",
|
||||
nextCursor: "string|null",
|
||||
done: "boolean",
|
||||
sampleSlugs: "string[]",
|
||||
});
|
||||
export type ApiV1SkillScanBatchResponse = (typeof ApiV1SkillScanBatchResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanBatchStatusRequestSchema = type({
|
||||
jobIds: "string[]",
|
||||
});
|
||||
export type ApiV1SkillScanBatchStatusRequest =
|
||||
(typeof ApiV1SkillScanBatchStatusRequestSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanBatchStatusResponseSchema = type({
|
||||
ok: "true",
|
||||
total: "number",
|
||||
queued: "number",
|
||||
running: "number",
|
||||
succeeded: "number",
|
||||
failed: "number",
|
||||
missing: "number",
|
||||
terminal: "number",
|
||||
done: "boolean",
|
||||
failedJobIds: "string[]",
|
||||
});
|
||||
export type ApiV1SkillScanBatchStatusResponse =
|
||||
(typeof ApiV1SkillScanBatchStatusResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillRepairVtPendingRequestSchema = type({
|
||||
cursor: "string|null?",
|
||||
batchSize: "number?",
|
||||
|
||||
@@ -24,6 +24,7 @@ export function createHttpModuleMocks() {
|
||||
const downloadZip = vi.fn();
|
||||
const fetchBinary = vi.fn();
|
||||
const fetchText = vi.fn();
|
||||
const uploadBinary = vi.fn();
|
||||
const registryUrl = vi.fn(buildRegistryUrl);
|
||||
|
||||
return {
|
||||
@@ -32,6 +33,7 @@ export function createHttpModuleMocks() {
|
||||
downloadZip,
|
||||
fetchBinary,
|
||||
fetchText,
|
||||
uploadBinary,
|
||||
registryUrl,
|
||||
moduleFactory: () => ({
|
||||
apiRequest: (registry: unknown, args: unknown, schema?: unknown) =>
|
||||
@@ -41,6 +43,7 @@ export function createHttpModuleMocks() {
|
||||
downloadZip: (registry: unknown, args: unknown) => downloadZip(registry, args),
|
||||
fetchBinary: (registry: unknown, args: unknown) => fetchBinary(registry, args),
|
||||
fetchText: (registry: unknown, args: unknown) => fetchText(registry, args),
|
||||
uploadBinary: (args: unknown, schema?: unknown) => uploadBinary(args, schema),
|
||||
registryUrl: (...args: [string, string]) => registryUrl(...args),
|
||||
}),
|
||||
};
|
||||
|
||||
Vendored
+54
-9
@@ -50,6 +50,7 @@ export declare const PackageVerificationSummarySchema: import("arktype/internal/
|
||||
sourceRepo?: string | undefined;
|
||||
sourceCommit?: string | undefined;
|
||||
sourceTag?: string | undefined;
|
||||
sourcePath?: string | undefined;
|
||||
hasProvenance?: boolean | undefined;
|
||||
trustedOpenClawPlugin?: boolean | undefined;
|
||||
scanStatus?: "clean" | "suspicious" | "malicious" | "pending" | "not-run" | undefined;
|
||||
@@ -228,18 +229,27 @@ export declare const PackageTrustedPublisherSchema: import("arktype/internal/var
|
||||
environment?: string | undefined;
|
||||
}, {}>;
|
||||
export type PackageTrustedPublisher = (typeof PackageTrustedPublisherSchema)[inferred];
|
||||
export declare const PackagePublishRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
export declare const MAX_PACKAGE_MULTIPART_BYTES: number;
|
||||
export declare const MAX_PACKAGE_CLAWPACK_BYTES: number;
|
||||
export type PackageMultipartUploadField = "files" | "clawpack";
|
||||
export type PackageMultipartUploadPart = {
|
||||
name: string;
|
||||
size: number;
|
||||
type?: string;
|
||||
};
|
||||
export type PackageMultipartUploadSizeInput = {
|
||||
payloadJson: string;
|
||||
fileFieldName: PackageMultipartUploadField;
|
||||
files: readonly PackageMultipartUploadPart[];
|
||||
};
|
||||
export declare function estimatePackageMultipartUploadBytes(input: PackageMultipartUploadSizeInput): number;
|
||||
export declare function isPackageMultipartUploadTooLarge(input: PackageMultipartUploadSizeInput): boolean;
|
||||
export declare function getPackageMultipartSizeError(): string;
|
||||
export declare const PackagePublishMetadataSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
name: string;
|
||||
family: "skill" | "code-plugin" | "bundle-plugin";
|
||||
version: string;
|
||||
changelog: string;
|
||||
files: {
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string | undefined;
|
||||
}[];
|
||||
displayName?: string | undefined;
|
||||
ownerHandle?: string | undefined;
|
||||
manualOverrideReason?: string | undefined;
|
||||
@@ -259,6 +269,20 @@ export declare const PackagePublishRequestSchema: import("arktype/internal/varia
|
||||
format?: string | undefined;
|
||||
hostTargets?: string[] | undefined;
|
||||
} | undefined;
|
||||
}, {}>;
|
||||
export type PackagePublishMetadata = (typeof PackagePublishMetadataSchema)[inferred];
|
||||
export declare const ServerPackagePublishRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
files: {
|
||||
path: string;
|
||||
size: number;
|
||||
storageId: string;
|
||||
sha256: string;
|
||||
contentType?: string | undefined;
|
||||
}[];
|
||||
name: string;
|
||||
family: "skill" | "code-plugin" | "bundle-plugin";
|
||||
version: string;
|
||||
changelog: string;
|
||||
artifact?: {
|
||||
kind: "npm-pack";
|
||||
storageId: string;
|
||||
@@ -271,8 +295,27 @@ export declare const PackagePublishRequestSchema: import("arktype/internal/varia
|
||||
npmUnpackedSize: number;
|
||||
npmFileCount: number;
|
||||
} | undefined;
|
||||
displayName?: string | undefined;
|
||||
ownerHandle?: string | undefined;
|
||||
manualOverrideReason?: string | undefined;
|
||||
channel?: "official" | "community" | "private" | undefined;
|
||||
tags?: string[] | undefined;
|
||||
source?: {
|
||||
kind: "github";
|
||||
url: string;
|
||||
repo: string;
|
||||
ref: string;
|
||||
commit: string;
|
||||
path: string;
|
||||
importedAt: number;
|
||||
} | undefined;
|
||||
bundle?: {
|
||||
id?: string | undefined;
|
||||
format?: string | undefined;
|
||||
hostTargets?: string[] | undefined;
|
||||
} | undefined;
|
||||
}, {}>;
|
||||
export type PackagePublishRequest = (typeof PackagePublishRequestSchema)[inferred];
|
||||
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
|
||||
export declare const PackageListItemSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
name: string;
|
||||
displayName: string;
|
||||
@@ -379,6 +422,7 @@ export declare const ApiV1PackageResponseSchema: import("arktype/internal/varian
|
||||
sourceRepo?: string | undefined;
|
||||
sourceCommit?: string | undefined;
|
||||
sourceTag?: string | undefined;
|
||||
sourcePath?: string | undefined;
|
||||
hasProvenance?: boolean | undefined;
|
||||
trustedOpenClawPlugin?: boolean | undefined;
|
||||
scanStatus?: "clean" | "suspicious" | "malicious" | "pending" | "not-run" | undefined;
|
||||
@@ -469,6 +513,7 @@ export declare const ApiV1PackageVersionResponseSchema: import("arktype/internal
|
||||
sourceRepo?: string | undefined;
|
||||
sourceCommit?: string | undefined;
|
||||
sourceTag?: string | undefined;
|
||||
sourcePath?: string | undefined;
|
||||
hasProvenance?: boolean | undefined;
|
||||
trustedOpenClawPlugin?: boolean | undefined;
|
||||
scanStatus?: "clean" | "suspicious" | "malicious" | "pending" | "not-run" | undefined;
|
||||
|
||||
Vendored
+63
-1
@@ -58,6 +58,12 @@ export const PackageVerificationSummarySchema = type({
|
||||
sourceRepo: "string?",
|
||||
sourceCommit: "string?",
|
||||
sourceTag: "string?",
|
||||
// Path of the package directory inside the source repo (e.g.
|
||||
// "examples/openclaw-plugin"). Forward slash separated, no leading or
|
||||
// trailing slash. Used when resolving relative README asset URLs against
|
||||
// raw.githubusercontent.com so that subdirectory packages render correctly.
|
||||
// Absent or "." means the package lives at the repo root.
|
||||
sourcePath: "string?",
|
||||
hasProvenance: "boolean?",
|
||||
trustedOpenClawPlugin: "boolean?",
|
||||
scanStatus: '"clean"|"suspicious"|"malicious"|"pending"|"not-run"?',
|
||||
@@ -188,7 +194,55 @@ export const PackageTrustedPublisherSchema = type({
|
||||
workflowFilename: "string",
|
||||
environment: "string?",
|
||||
});
|
||||
export const PackagePublishRequestSchema = type({
|
||||
export const MAX_PACKAGE_MULTIPART_BYTES = 18 * 1024 * 1024;
|
||||
export const MAX_PACKAGE_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
const PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES = 4096;
|
||||
const PACKAGE_MULTIPART_PART_OVERHEAD_BYTES = 1024;
|
||||
export function estimatePackageMultipartUploadBytes(input) {
|
||||
return (PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES +
|
||||
estimateMultipartStringPartBytes("payload", input.payloadJson) +
|
||||
input.files.reduce((sum, file) => sum + estimateMultipartFilePartBytes(input.fileFieldName, file), 0));
|
||||
}
|
||||
export function isPackageMultipartUploadTooLarge(input) {
|
||||
return estimatePackageMultipartUploadBytes(input) > MAX_PACKAGE_MULTIPART_BYTES;
|
||||
}
|
||||
export function getPackageMultipartSizeError() {
|
||||
return "Package upload exceeds 18MB multipart upload limit";
|
||||
}
|
||||
function estimateMultipartStringPartBytes(fieldName, value) {
|
||||
return PACKAGE_MULTIPART_PART_OVERHEAD_BYTES + utf8ByteLength(fieldName) + utf8ByteLength(value);
|
||||
}
|
||||
function estimateMultipartFilePartBytes(fieldName, file) {
|
||||
return (file.size +
|
||||
PACKAGE_MULTIPART_PART_OVERHEAD_BYTES +
|
||||
utf8ByteLength(fieldName) +
|
||||
utf8ByteLength(file.name) +
|
||||
utf8ByteLength(file.type ?? ""));
|
||||
}
|
||||
function utf8ByteLength(value) {
|
||||
let bytes = 0;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const codePoint = value.codePointAt(index);
|
||||
if (codePoint === undefined)
|
||||
continue;
|
||||
if (codePoint > 0xffff)
|
||||
index += 1;
|
||||
if (codePoint <= 0x7f) {
|
||||
bytes += 1;
|
||||
}
|
||||
else if (codePoint <= 0x7ff) {
|
||||
bytes += 2;
|
||||
}
|
||||
else if (codePoint <= 0xffff) {
|
||||
bytes += 3;
|
||||
}
|
||||
else {
|
||||
bytes += 4;
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
const PackagePublishMetadataFields = {
|
||||
name: "string",
|
||||
displayName: "string?",
|
||||
ownerHandle: "string?",
|
||||
@@ -200,6 +254,14 @@ export const PackagePublishRequestSchema = type({
|
||||
tags: "string[]?",
|
||||
source: PublishSourceSchema.optional(),
|
||||
bundle: BundlePublishMetadataSchema.optional(),
|
||||
};
|
||||
export const PackagePublishMetadataSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
});
|
||||
export const ServerPackagePublishRequestSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
artifact: PackagePublishArtifactSchema.optional(),
|
||||
files: CliPublishFileSchema.array(),
|
||||
});
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
Vendored
+1
@@ -16,6 +16,7 @@ export declare const ApiRoutes: {
|
||||
readonly download: "/api/v1/download";
|
||||
readonly publishTokenMint: "/api/v1/publish/token/mint";
|
||||
readonly skills: "/api/v1/skills";
|
||||
readonly skillScans: "/api/v1/skills/-/scan";
|
||||
readonly plugins: "/api/v1/plugins";
|
||||
readonly packages: "/api/v1/packages";
|
||||
readonly codePlugins: "/api/v1/code-plugins";
|
||||
|
||||
Vendored
+1
@@ -16,6 +16,7 @@ export const ApiRoutes = {
|
||||
download: "/api/v1/download",
|
||||
publishTokenMint: "/api/v1/publish/token/mint",
|
||||
skills: "/api/v1/skills",
|
||||
skillScans: "/api/v1/skills/-/scan",
|
||||
plugins: "/api/v1/plugins",
|
||||
packages: "/api/v1/packages",
|
||||
codePlugins: "/api/v1/code-plugins",
|
||||
|
||||
Vendored
+1
-1
@@ -1 +1 @@
|
||||
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,gBAAgB,EAAE,yBAAyB;IAC3C,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,KAAK,EAAE,eAAe;IACtB,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"}
|
||||
{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,gBAAgB,EAAE,yBAAyB;IAC3C,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,MAAM,EAAE,gBAAgB;IACxB,UAAU,EAAE,uBAAuB;IACnC,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,KAAK,EAAE,eAAe;IACtB,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"}
|
||||
Vendored
+97
-1
@@ -47,6 +47,7 @@ export declare const ApiSkillMetaResponseSchema: import("arktype/internal/varian
|
||||
}, {}>;
|
||||
export declare const ApiCliUploadUrlResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
uploadUrl: string;
|
||||
uploadTicket: string;
|
||||
}, {}>;
|
||||
export declare const ApiUploadFileResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
storageId: string;
|
||||
@@ -370,6 +371,65 @@ export declare const ApiV1SkillRescanResponseSchema: import("arktype/internal/va
|
||||
alreadyQueued: boolean;
|
||||
}, {}>;
|
||||
export type ApiV1SkillRescanResponse = (typeof ApiV1SkillRescanResponseSchema)[inferred];
|
||||
export declare const ApiV1SkillScanStatusSchema: import("arktype/internal/variants/string.ts").StringType<"queued" | "running" | "succeeded" | "failed", {}>;
|
||||
export type ApiV1SkillScanStatus = (typeof ApiV1SkillScanStatusSchema)[inferred];
|
||||
export declare const ApiV1SkillScanSourceSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
kind: "upload";
|
||||
} | {
|
||||
kind: "published";
|
||||
slug: string;
|
||||
version?: string | undefined;
|
||||
}, {}>;
|
||||
export type ApiV1SkillScanSource = (typeof ApiV1SkillScanSourceSchema)[inferred];
|
||||
export declare const ApiV1SkillScanSubmitRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
source: {
|
||||
kind: "upload";
|
||||
} | {
|
||||
kind: "published";
|
||||
slug: string;
|
||||
version?: string | undefined;
|
||||
};
|
||||
update?: boolean | undefined;
|
||||
}, {}>;
|
||||
export type ApiV1SkillScanSubmitRequest = (typeof ApiV1SkillScanSubmitRequestSchema)[inferred];
|
||||
export declare const ApiV1SkillScanSubmitResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
ok: true;
|
||||
scanId: string;
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
sourceKind: "upload" | "published";
|
||||
update: boolean;
|
||||
jobId?: string | undefined;
|
||||
alreadyQueued?: boolean | undefined;
|
||||
}, {}>;
|
||||
export type ApiV1SkillScanSubmitResponse = (typeof ApiV1SkillScanSubmitResponseSchema)[inferred];
|
||||
export declare const ApiV1SkillScanStatusResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
ok: true;
|
||||
scanId: string;
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
sourceKind: "upload" | "published";
|
||||
update: boolean;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
jobId?: string | undefined;
|
||||
writtenBack?: boolean | undefined;
|
||||
artifact?: unknown;
|
||||
report?: unknown;
|
||||
lastError?: string | undefined;
|
||||
completedAt?: number | undefined;
|
||||
}, {}>;
|
||||
export type ApiV1SkillScanStatusResponse = (typeof ApiV1SkillScanStatusResponseSchema)[inferred];
|
||||
export declare const ApiV1SkillScanDownloadManifestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
scanId: string;
|
||||
sourceKind: "upload" | "published";
|
||||
update: boolean;
|
||||
status: "queued" | "running" | "succeeded" | "failed";
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
artifact?: unknown;
|
||||
completedAt?: number | undefined;
|
||||
writtenBack?: boolean | undefined;
|
||||
}, {}>;
|
||||
export type ApiV1SkillScanDownloadManifest = (typeof ApiV1SkillScanDownloadManifestSchema)[inferred];
|
||||
export declare const ApiV1SkillBulkRescanBatchRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
mode?: "all-active-latest" | undefined;
|
||||
cursor?: string | null | undefined;
|
||||
@@ -406,6 +466,42 @@ export declare const ApiV1SkillBulkRescanStatusResponseSchema: import("arktype/i
|
||||
failedJobIds: string[];
|
||||
}, {}>;
|
||||
export type ApiV1SkillBulkRescanStatusResponse = (typeof ApiV1SkillBulkRescanStatusResponseSchema)[inferred];
|
||||
export declare const ApiV1SkillScanBatchRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
mode?: "all-active-latest" | undefined;
|
||||
cursor?: string | null | undefined;
|
||||
batchSize?: number | undefined;
|
||||
dryRun?: boolean | undefined;
|
||||
}, {}>;
|
||||
export type ApiV1SkillScanBatchRequest = (typeof ApiV1SkillScanBatchRequestSchema)[inferred];
|
||||
export declare const ApiV1SkillScanBatchResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
ok: true;
|
||||
mode: "all-active-latest";
|
||||
queued: number;
|
||||
alreadyQueued: number;
|
||||
skipped: number;
|
||||
jobIds: string[];
|
||||
nextCursor: string | null;
|
||||
done: boolean;
|
||||
sampleSlugs: string[];
|
||||
}, {}>;
|
||||
export type ApiV1SkillScanBatchResponse = (typeof ApiV1SkillScanBatchResponseSchema)[inferred];
|
||||
export declare const ApiV1SkillScanBatchStatusRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
jobIds: string[];
|
||||
}, {}>;
|
||||
export type ApiV1SkillScanBatchStatusRequest = (typeof ApiV1SkillScanBatchStatusRequestSchema)[inferred];
|
||||
export declare const ApiV1SkillScanBatchStatusResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
ok: true;
|
||||
total: number;
|
||||
queued: number;
|
||||
running: number;
|
||||
succeeded: number;
|
||||
failed: number;
|
||||
missing: number;
|
||||
terminal: number;
|
||||
done: boolean;
|
||||
failedJobIds: string[];
|
||||
}, {}>;
|
||||
export type ApiV1SkillScanBatchStatusResponse = (typeof ApiV1SkillScanBatchStatusResponseSchema)[inferred];
|
||||
export declare const ApiV1SkillRepairVtPendingRequestSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
cursor?: string | null | undefined;
|
||||
batchSize?: number | undefined;
|
||||
@@ -479,7 +575,7 @@ export declare const ApiV1SkillResolveResponseSchema: import("arktype/internal/v
|
||||
export declare const ApiV1SkillVerifyResponseSchema: import("arktype/internal/variants/object.ts").ObjectType<{
|
||||
schema: "clawhub.skill.verify.v1";
|
||||
ok: boolean;
|
||||
decision: "pass" | "fail";
|
||||
decision: "fail" | "pass";
|
||||
reasons: string[];
|
||||
slug: string;
|
||||
displayName: string;
|
||||
|
||||
Vendored
+80
@@ -45,6 +45,7 @@ export const ApiSkillMetaResponseSchema = type({
|
||||
});
|
||||
export const ApiCliUploadUrlResponseSchema = type({
|
||||
uploadUrl: "string",
|
||||
uploadTicket: "string",
|
||||
});
|
||||
export const ApiUploadFileResponseSchema = type({
|
||||
storageId: "string",
|
||||
@@ -333,6 +334,53 @@ export const ApiV1SkillRescanResponseSchema = type({
|
||||
jobId: "string",
|
||||
alreadyQueued: "boolean",
|
||||
});
|
||||
export const ApiV1SkillScanStatusSchema = type('"queued"|"running"|"succeeded"|"failed"');
|
||||
export const ApiV1SkillScanSourceSchema = type({
|
||||
kind: '"upload"',
|
||||
}).or({
|
||||
kind: '"published"',
|
||||
slug: "string",
|
||||
version: "string?",
|
||||
});
|
||||
export const ApiV1SkillScanSubmitRequestSchema = type({
|
||||
source: ApiV1SkillScanSourceSchema,
|
||||
update: "boolean?",
|
||||
});
|
||||
export const ApiV1SkillScanSubmitResponseSchema = type({
|
||||
ok: "true",
|
||||
scanId: "string",
|
||||
jobId: "string?",
|
||||
status: ApiV1SkillScanStatusSchema,
|
||||
sourceKind: '"upload"|"published"',
|
||||
update: "boolean",
|
||||
alreadyQueued: "boolean?",
|
||||
});
|
||||
export const ApiV1SkillScanStatusResponseSchema = type({
|
||||
ok: "true",
|
||||
scanId: "string",
|
||||
jobId: "string?",
|
||||
status: ApiV1SkillScanStatusSchema,
|
||||
sourceKind: '"upload"|"published"',
|
||||
update: "boolean",
|
||||
writtenBack: "boolean?",
|
||||
artifact: "unknown?",
|
||||
report: "unknown?",
|
||||
lastError: "string?",
|
||||
createdAt: "number",
|
||||
updatedAt: "number",
|
||||
completedAt: "number?",
|
||||
});
|
||||
export const ApiV1SkillScanDownloadManifestSchema = type({
|
||||
scanId: "string",
|
||||
sourceKind: '"upload"|"published"',
|
||||
update: "boolean",
|
||||
status: ApiV1SkillScanStatusSchema,
|
||||
artifact: "unknown?",
|
||||
createdAt: "number",
|
||||
updatedAt: "number",
|
||||
completedAt: "number?",
|
||||
writtenBack: "boolean?",
|
||||
});
|
||||
export const ApiV1SkillBulkRescanBatchRequestSchema = type({
|
||||
mode: '"all-active-latest"?',
|
||||
cursor: "string|null?",
|
||||
@@ -365,6 +413,38 @@ export const ApiV1SkillBulkRescanStatusResponseSchema = type({
|
||||
done: "boolean",
|
||||
failedJobIds: "string[]",
|
||||
});
|
||||
export const ApiV1SkillScanBatchRequestSchema = type({
|
||||
mode: '"all-active-latest"?',
|
||||
cursor: "string|null?",
|
||||
batchSize: "number?",
|
||||
dryRun: "boolean?",
|
||||
});
|
||||
export const ApiV1SkillScanBatchResponseSchema = type({
|
||||
ok: "true",
|
||||
mode: '"all-active-latest"',
|
||||
queued: "number",
|
||||
alreadyQueued: "number",
|
||||
skipped: "number",
|
||||
jobIds: "string[]",
|
||||
nextCursor: "string|null",
|
||||
done: "boolean",
|
||||
sampleSlugs: "string[]",
|
||||
});
|
||||
export const ApiV1SkillScanBatchStatusRequestSchema = type({
|
||||
jobIds: "string[]",
|
||||
});
|
||||
export const ApiV1SkillScanBatchStatusResponseSchema = type({
|
||||
ok: "true",
|
||||
total: "number",
|
||||
queued: "number",
|
||||
running: "number",
|
||||
succeeded: "number",
|
||||
failed: "number",
|
||||
missing: "number",
|
||||
terminal: "number",
|
||||
done: "boolean",
|
||||
failedJobIds: "string[]",
|
||||
});
|
||||
export const ApiV1SkillRepairVtPendingRequestSchema = type({
|
||||
cursor: "string|null?",
|
||||
batchSize: "number?",
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
@@ -75,6 +75,12 @@ export const PackageVerificationSummarySchema = type({
|
||||
sourceRepo: "string?",
|
||||
sourceCommit: "string?",
|
||||
sourceTag: "string?",
|
||||
// Path of the package directory inside the source repo (e.g.
|
||||
// "examples/openclaw-plugin"). Forward slash separated, no leading or
|
||||
// trailing slash. Used when resolving relative README asset URLs against
|
||||
// raw.githubusercontent.com so that subdirectory packages render correctly.
|
||||
// Absent or "." means the package lives at the repo root.
|
||||
sourcePath: "string?",
|
||||
hasProvenance: "boolean?",
|
||||
trustedOpenClawPlugin: "boolean?",
|
||||
scanStatus: '"clean"|"suspicious"|"malicious"|"pending"|"not-run"?',
|
||||
@@ -253,7 +259,81 @@ export const PackageTrustedPublisherSchema = type({
|
||||
});
|
||||
export type PackageTrustedPublisher = (typeof PackageTrustedPublisherSchema)[inferred];
|
||||
|
||||
export const PackagePublishRequestSchema = type({
|
||||
export const MAX_PACKAGE_MULTIPART_BYTES = 18 * 1024 * 1024;
|
||||
export const MAX_PACKAGE_CLAWPACK_BYTES = 120 * 1024 * 1024;
|
||||
const PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES = 4096;
|
||||
const PACKAGE_MULTIPART_PART_OVERHEAD_BYTES = 1024;
|
||||
|
||||
export type PackageMultipartUploadField = "files" | "clawpack";
|
||||
export type PackageMultipartUploadPart = {
|
||||
name: string;
|
||||
size: number;
|
||||
type?: string;
|
||||
};
|
||||
export type PackageMultipartUploadSizeInput = {
|
||||
payloadJson: string;
|
||||
fileFieldName: PackageMultipartUploadField;
|
||||
files: readonly PackageMultipartUploadPart[];
|
||||
};
|
||||
|
||||
export function estimatePackageMultipartUploadBytes(
|
||||
input: PackageMultipartUploadSizeInput,
|
||||
): number {
|
||||
return (
|
||||
PACKAGE_MULTIPART_FIXED_OVERHEAD_BYTES +
|
||||
estimateMultipartStringPartBytes("payload", input.payloadJson) +
|
||||
input.files.reduce(
|
||||
(sum, file) => sum + estimateMultipartFilePartBytes(input.fileFieldName, file),
|
||||
0,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function isPackageMultipartUploadTooLarge(input: PackageMultipartUploadSizeInput): boolean {
|
||||
return estimatePackageMultipartUploadBytes(input) > MAX_PACKAGE_MULTIPART_BYTES;
|
||||
}
|
||||
|
||||
export function getPackageMultipartSizeError(): string {
|
||||
return "Package upload exceeds 18MB multipart upload limit";
|
||||
}
|
||||
|
||||
function estimateMultipartStringPartBytes(fieldName: string, value: string): number {
|
||||
return PACKAGE_MULTIPART_PART_OVERHEAD_BYTES + utf8ByteLength(fieldName) + utf8ByteLength(value);
|
||||
}
|
||||
|
||||
function estimateMultipartFilePartBytes(
|
||||
fieldName: PackageMultipartUploadField,
|
||||
file: PackageMultipartUploadPart,
|
||||
): number {
|
||||
return (
|
||||
file.size +
|
||||
PACKAGE_MULTIPART_PART_OVERHEAD_BYTES +
|
||||
utf8ByteLength(fieldName) +
|
||||
utf8ByteLength(file.name) +
|
||||
utf8ByteLength(file.type ?? "")
|
||||
);
|
||||
}
|
||||
|
||||
function utf8ByteLength(value: string): number {
|
||||
let bytes = 0;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const codePoint = value.codePointAt(index);
|
||||
if (codePoint === undefined) continue;
|
||||
if (codePoint > 0xffff) index += 1;
|
||||
if (codePoint <= 0x7f) {
|
||||
bytes += 1;
|
||||
} else if (codePoint <= 0x7ff) {
|
||||
bytes += 2;
|
||||
} else if (codePoint <= 0xffff) {
|
||||
bytes += 3;
|
||||
} else {
|
||||
bytes += 4;
|
||||
}
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
const PackagePublishMetadataFields = {
|
||||
name: "string",
|
||||
displayName: "string?",
|
||||
ownerHandle: "string?",
|
||||
@@ -265,10 +345,21 @@ export const PackagePublishRequestSchema = type({
|
||||
tags: "string[]?",
|
||||
source: PublishSourceSchema.optional(),
|
||||
bundle: BundlePublishMetadataSchema.optional(),
|
||||
} as const;
|
||||
|
||||
export const PackagePublishMetadataSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
});
|
||||
export type PackagePublishMetadata = (typeof PackagePublishMetadataSchema)[inferred];
|
||||
|
||||
export const ServerPackagePublishRequestSchema = type({
|
||||
"+": "reject",
|
||||
...PackagePublishMetadataFields,
|
||||
artifact: PackagePublishArtifactSchema.optional(),
|
||||
files: CliPublishFileSchema.array(),
|
||||
});
|
||||
export type PackagePublishRequest = (typeof PackagePublishRequestSchema)[inferred];
|
||||
export type ServerPackagePublishRequest = (typeof ServerPackagePublishRequestSchema)[inferred];
|
||||
|
||||
export const PackageListItemSchema = type({
|
||||
name: "string",
|
||||
|
||||
@@ -17,6 +17,7 @@ export const ApiRoutes = {
|
||||
download: "/api/v1/download",
|
||||
publishTokenMint: "/api/v1/publish/token/mint",
|
||||
skills: "/api/v1/skills",
|
||||
skillScans: "/api/v1/skills/-/scan",
|
||||
plugins: "/api/v1/plugins",
|
||||
packages: "/api/v1/packages",
|
||||
codePlugins: "/api/v1/code-plugins",
|
||||
|
||||
@@ -55,6 +55,7 @@ export const ApiSkillMetaResponseSchema = type({
|
||||
|
||||
export const ApiCliUploadUrlResponseSchema = type({
|
||||
uploadUrl: "string",
|
||||
uploadTicket: "string",
|
||||
});
|
||||
|
||||
export const ApiUploadFileResponseSchema = type({
|
||||
@@ -398,6 +399,66 @@ export const ApiV1SkillRescanResponseSchema = type({
|
||||
});
|
||||
export type ApiV1SkillRescanResponse = (typeof ApiV1SkillRescanResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanStatusSchema = type('"queued"|"running"|"succeeded"|"failed"');
|
||||
export type ApiV1SkillScanStatus = (typeof ApiV1SkillScanStatusSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanSourceSchema = type({
|
||||
kind: '"upload"',
|
||||
}).or({
|
||||
kind: '"published"',
|
||||
slug: "string",
|
||||
version: "string?",
|
||||
});
|
||||
export type ApiV1SkillScanSource = (typeof ApiV1SkillScanSourceSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanSubmitRequestSchema = type({
|
||||
source: ApiV1SkillScanSourceSchema,
|
||||
update: "boolean?",
|
||||
});
|
||||
export type ApiV1SkillScanSubmitRequest = (typeof ApiV1SkillScanSubmitRequestSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanSubmitResponseSchema = type({
|
||||
ok: "true",
|
||||
scanId: "string",
|
||||
jobId: "string?",
|
||||
status: ApiV1SkillScanStatusSchema,
|
||||
sourceKind: '"upload"|"published"',
|
||||
update: "boolean",
|
||||
alreadyQueued: "boolean?",
|
||||
});
|
||||
export type ApiV1SkillScanSubmitResponse = (typeof ApiV1SkillScanSubmitResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanStatusResponseSchema = type({
|
||||
ok: "true",
|
||||
scanId: "string",
|
||||
jobId: "string?",
|
||||
status: ApiV1SkillScanStatusSchema,
|
||||
sourceKind: '"upload"|"published"',
|
||||
update: "boolean",
|
||||
writtenBack: "boolean?",
|
||||
artifact: "unknown?",
|
||||
report: "unknown?",
|
||||
lastError: "string?",
|
||||
createdAt: "number",
|
||||
updatedAt: "number",
|
||||
completedAt: "number?",
|
||||
});
|
||||
export type ApiV1SkillScanStatusResponse = (typeof ApiV1SkillScanStatusResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanDownloadManifestSchema = type({
|
||||
scanId: "string",
|
||||
sourceKind: '"upload"|"published"',
|
||||
update: "boolean",
|
||||
status: ApiV1SkillScanStatusSchema,
|
||||
artifact: "unknown?",
|
||||
createdAt: "number",
|
||||
updatedAt: "number",
|
||||
completedAt: "number?",
|
||||
writtenBack: "boolean?",
|
||||
});
|
||||
export type ApiV1SkillScanDownloadManifest =
|
||||
(typeof ApiV1SkillScanDownloadManifestSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillBulkRescanBatchRequestSchema = type({
|
||||
mode: '"all-active-latest"?',
|
||||
cursor: "string|null?",
|
||||
@@ -442,6 +503,48 @@ export const ApiV1SkillBulkRescanStatusResponseSchema = type({
|
||||
export type ApiV1SkillBulkRescanStatusResponse =
|
||||
(typeof ApiV1SkillBulkRescanStatusResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanBatchRequestSchema = type({
|
||||
mode: '"all-active-latest"?',
|
||||
cursor: "string|null?",
|
||||
batchSize: "number?",
|
||||
dryRun: "boolean?",
|
||||
});
|
||||
export type ApiV1SkillScanBatchRequest = (typeof ApiV1SkillScanBatchRequestSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanBatchResponseSchema = type({
|
||||
ok: "true",
|
||||
mode: '"all-active-latest"',
|
||||
queued: "number",
|
||||
alreadyQueued: "number",
|
||||
skipped: "number",
|
||||
jobIds: "string[]",
|
||||
nextCursor: "string|null",
|
||||
done: "boolean",
|
||||
sampleSlugs: "string[]",
|
||||
});
|
||||
export type ApiV1SkillScanBatchResponse = (typeof ApiV1SkillScanBatchResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanBatchStatusRequestSchema = type({
|
||||
jobIds: "string[]",
|
||||
});
|
||||
export type ApiV1SkillScanBatchStatusRequest =
|
||||
(typeof ApiV1SkillScanBatchStatusRequestSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillScanBatchStatusResponseSchema = type({
|
||||
ok: "true",
|
||||
total: "number",
|
||||
queued: "number",
|
||||
running: "number",
|
||||
succeeded: "number",
|
||||
failed: "number",
|
||||
missing: "number",
|
||||
terminal: "number",
|
||||
done: "boolean",
|
||||
failedJobIds: "string[]",
|
||||
});
|
||||
export type ApiV1SkillScanBatchStatusResponse =
|
||||
(typeof ApiV1SkillScanBatchStatusResponseSchema)[inferred];
|
||||
|
||||
export const ApiV1SkillRepairVtPendingRequestSchema = type({
|
||||
cursor: "string|null?",
|
||||
batchSize: "number?",
|
||||
|
||||
@@ -3064,6 +3064,8 @@
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"recommended",
|
||||
"default",
|
||||
"updated",
|
||||
"createdAt",
|
||||
"newest",
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
export const LOCAL_CODEX_WORKER_OPT_IN = "CLAWHUB_ALLOW_LOCAL_CODEX_SCAN";
|
||||
|
||||
export function isGitHubActionsRunner(env: NodeJS.ProcessEnv) {
|
||||
return (
|
||||
env.GITHUB_ACTIONS === "true" &&
|
||||
env.CI === "true" &&
|
||||
Boolean(env.GITHUB_RUN_ID?.trim()) &&
|
||||
Boolean(env.GITHUB_REPOSITORY?.trim())
|
||||
);
|
||||
}
|
||||
|
||||
export function isCodexWorkerExecutionAllowed(env: NodeJS.ProcessEnv) {
|
||||
return env[LOCAL_CODEX_WORKER_OPT_IN] === "1" || isGitHubActionsRunner(env);
|
||||
}
|
||||
|
||||
export function localCodexWorkerOptInReason() {
|
||||
return `set ${LOCAL_CODEX_WORKER_OPT_IN}=1 to run Codex workers locally`;
|
||||
}
|
||||
|
||||
export function assertCodexWorkerExecutionAllowed(env: NodeJS.ProcessEnv) {
|
||||
if (isCodexWorkerExecutionAllowed(env)) return;
|
||||
throw new Error(`Refusing to run local Codex workers without ${LOCAL_CODEX_WORKER_OPT_IN}=1`);
|
||||
}
|
||||
|
||||
export function resolveCodexWorkerHome(env: NodeJS.ProcessEnv, fallbackLocalHome: string) {
|
||||
const explicitHome = env.CODEX_HOME?.trim();
|
||||
if (explicitHome) return explicitHome;
|
||||
if (isGitHubActionsRunner(env)) return undefined;
|
||||
if (env[LOCAL_CODEX_WORKER_OPT_IN] === "1") return fallbackLocalHome;
|
||||
return undefined;
|
||||
}
|
||||
@@ -110,15 +110,42 @@ describe("dev-workers", () => {
|
||||
env: {},
|
||||
});
|
||||
|
||||
expect(resolved.workers.map((worker) => worker.id)).toEqual(["security-scan"]);
|
||||
expect(resolved.workers.map((worker) => worker.id)).toEqual([]);
|
||||
expect(resolved.skipped).toEqual([
|
||||
expect.objectContaining({
|
||||
workerId: "security-scan",
|
||||
reason: expect.stringContaining("CLAWHUB_ALLOW_LOCAL_CODEX_SCAN=1"),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
workerId: "skill-card",
|
||||
reason: expect.stringContaining("NVIDIA Skill Card automation checkout"),
|
||||
reason: expect.stringContaining("CLAWHUB_ALLOW_LOCAL_CODEX_SCAN=1"),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps Codex workers with explicit local opt-in", async () => {
|
||||
const root = await tempDir();
|
||||
const toolDir = join(root, "nvidia-tooling");
|
||||
await mkdir(join(toolDir, "AI Transparency Card Automation", "scripts"), { recursive: true });
|
||||
await writeFile(
|
||||
join(toolDir, "AI Transparency Card Automation", "scripts", "render_card.py"),
|
||||
"print('render')\n",
|
||||
"utf8",
|
||||
);
|
||||
const selected = resolveEnabledWorkers({
|
||||
workers: [],
|
||||
skip: [],
|
||||
});
|
||||
|
||||
const resolved = resolveRunnableWorkers(selected, parseArgs(["--nvidia-tool-dir", toolDir]), {
|
||||
cwd: root,
|
||||
env: { CLAWHUB_ALLOW_LOCAL_CODEX_SCAN: "1" },
|
||||
});
|
||||
|
||||
expect(resolved.workers.map((worker) => worker.id)).toEqual(["security-scan", "skill-card"]);
|
||||
expect(resolved.skipped).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps the Skill Card worker when an explicit NVIDIA tool checkout exists", async () => {
|
||||
const root = await tempDir();
|
||||
const toolDir = join(root, "nvidia-tooling");
|
||||
@@ -136,7 +163,7 @@ describe("dev-workers", () => {
|
||||
const resolved = resolveRunnableWorkers(
|
||||
selected,
|
||||
parseArgs(["--workers", "skill-card", "--nvidia-tool-dir", toolDir]),
|
||||
{ cwd: root, env: {} },
|
||||
{ cwd: root, env: { CLAWHUB_ALLOW_LOCAL_CODEX_SCAN: "1" } },
|
||||
);
|
||||
|
||||
expect(resolved.workers.map((worker) => worker.id)).toEqual(["skill-card"]);
|
||||
|
||||
@@ -4,6 +4,7 @@ import { existsSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { basename, join, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { isCodexWorkerExecutionAllowed, localCodexWorkerOptInReason } from "./codex-worker-guard";
|
||||
|
||||
type DevWorkerId = "security-scan" | "skill-card";
|
||||
|
||||
@@ -13,6 +14,7 @@ type WorkerDefinition = {
|
||||
script: string;
|
||||
requiredEnv: string[];
|
||||
requiredAnyEnv?: string[];
|
||||
requiresCodexCli: boolean;
|
||||
productionWorkflow: string;
|
||||
};
|
||||
|
||||
@@ -36,6 +38,7 @@ export const WORKERS: WorkerDefinition[] = [
|
||||
script: "scripts/security/run-codex-scan-worker.ts",
|
||||
// Shared Convex worker credential used by security and Skill Card workers.
|
||||
requiredEnv: ["CONVEX_URL", "SECURITY_SCAN_WORKER_TOKEN"],
|
||||
requiresCodexCli: true,
|
||||
productionWorkflow: ".github/workflows/security-scan-codex.yml",
|
||||
},
|
||||
{
|
||||
@@ -44,6 +47,7 @@ export const WORKERS: WorkerDefinition[] = [
|
||||
script: "scripts/skill-cards/run-skill-card-worker.ts",
|
||||
// Shared Convex worker credential used by security and Skill Card workers.
|
||||
requiredEnv: ["CONVEX_URL", "SECURITY_SCAN_WORKER_TOKEN"],
|
||||
requiresCodexCli: true,
|
||||
productionWorkflow: ".github/workflows/skill-card-worker.yml",
|
||||
},
|
||||
];
|
||||
@@ -204,6 +208,13 @@ export function resolveRunnableWorkers(
|
||||
const runnable: WorkerDefinition[] = [];
|
||||
const skipped: Array<{ workerId: DevWorkerId; reason: string }> = [];
|
||||
for (const worker of workers) {
|
||||
if (worker.requiresCodexCli && !isCodexWorkerExecutionAllowed(context.env)) {
|
||||
skipped.push({
|
||||
workerId: worker.id,
|
||||
reason: localCodexWorkerOptInReason(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (worker.id !== "skill-card" || hasNvidiaSkillCardTooling(options, context)) {
|
||||
runnable.push(worker);
|
||||
continue;
|
||||
|
||||
@@ -42,6 +42,34 @@ describe("Convex export dataset ingestion", () => {
|
||||
sha256: "file-sha",
|
||||
content: "Use this skill safely. password=supersecret123",
|
||||
},
|
||||
{
|
||||
path: "scripts/export.py",
|
||||
size: 48,
|
||||
sha256: "script-sha",
|
||||
content: "import json\npassword=supersecret123\n",
|
||||
contentType: "text/x-python",
|
||||
},
|
||||
{
|
||||
path: "skill-card.md",
|
||||
size: 16,
|
||||
sha256: "card-sha",
|
||||
content: "Generated card",
|
||||
contentType: "text/markdown",
|
||||
},
|
||||
{
|
||||
path: "references/skill-card.md",
|
||||
size: 32,
|
||||
sha256: "nested-card-sha",
|
||||
content: "Author-authored card note",
|
||||
contentType: "text/markdown",
|
||||
},
|
||||
{
|
||||
path: "docs/SKILL.md",
|
||||
size: 36,
|
||||
sha256: "nested-skill-sha",
|
||||
content: "Nested authored skill reference",
|
||||
contentType: "text/markdown",
|
||||
},
|
||||
],
|
||||
llmAnalysis: {
|
||||
status: "suspicious",
|
||||
@@ -167,6 +195,20 @@ describe("Convex export dataset ingestion", () => {
|
||||
issues: [{ issueId: "SDI-1" }],
|
||||
},
|
||||
skillMdContentRedacted: "Use this skill safely. [REDACTED_SECRET]",
|
||||
bundleFilesRedacted: [
|
||||
{
|
||||
path: "scripts/export.py",
|
||||
content: "import json\n[REDACTED_SECRET]\n",
|
||||
},
|
||||
{
|
||||
path: "references/skill-card.md",
|
||||
content: "Author-authored card note",
|
||||
},
|
||||
{
|
||||
path: "docs/SKILL.md",
|
||||
content: "Nested authored skill reference",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(rows[1]?.llmAnalysis?.agenticRiskFindings).toMatchObject([
|
||||
{
|
||||
@@ -217,6 +259,54 @@ describe("Convex export dataset ingestion", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("treats root skills.md as primary skill content", () => {
|
||||
const rows = artifactInputsFromConvexExportTables({
|
||||
skills: [
|
||||
{
|
||||
_id: "skills:1",
|
||||
displayName: "Plural Readme Skill",
|
||||
slug: "plural-readme-skill",
|
||||
ownerUserId: "users:owner",
|
||||
},
|
||||
],
|
||||
skillVersions: [
|
||||
{
|
||||
_id: "skillVersions:1",
|
||||
skillId: "skills:1",
|
||||
version: "1.0.0",
|
||||
createdAt: 1,
|
||||
files: [
|
||||
{
|
||||
path: "docs/skills.md",
|
||||
size: 24,
|
||||
sha256: "nested-skills-sha",
|
||||
content: "Nested authored plural readme",
|
||||
},
|
||||
{
|
||||
path: "skills.md",
|
||||
size: 12,
|
||||
sha256: "skills-sha",
|
||||
content: "Primary readme token=supersecret123",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
packages: [],
|
||||
packageReleases: [],
|
||||
users: [{ _id: "users:owner", handle: "owner" }],
|
||||
});
|
||||
|
||||
expect(rows[0]).toMatchObject({
|
||||
skillMdContentRedacted: "Primary readme [REDACTED_SECRET]",
|
||||
bundleFilesRedacted: [
|
||||
{
|
||||
path: "docs/skills.md",
|
||||
content: "Nested authored plural readme",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores inactive owner handles from Convex export tables", () => {
|
||||
const rows = artifactInputsFromConvexExportTables({
|
||||
skills: [
|
||||
|
||||
@@ -10,7 +10,7 @@ import type {
|
||||
StaticScanInput,
|
||||
VtAnalysisInput,
|
||||
} from "./normalize";
|
||||
import { redactSkillContent } from "./normalize";
|
||||
import { redactBundleContent, redactSkillContent } from "./normalize";
|
||||
|
||||
type ConvexDoc = Record<string, unknown> & { _id?: unknown };
|
||||
|
||||
@@ -186,6 +186,7 @@ function skillVersionToExportRow(
|
||||
version: requiredString(version.version, "skillVersions.version"),
|
||||
artifactSha256: stringOrNull(version.sha256hash),
|
||||
skillMdContentRedacted: skillMdContentFromExport(version.files),
|
||||
bundleFilesRedacted: bundleFilesFromExport(version.files),
|
||||
createdAt: numberValue(version.createdAt, "skillVersions.createdAt"),
|
||||
softDeletedAt: numberOrNull(version.softDeletedAt),
|
||||
files: filesFromExport(version.files),
|
||||
@@ -295,8 +296,8 @@ function skillMdContentFromExport(value: unknown): string | null {
|
||||
if (!Array.isArray(value)) return null;
|
||||
for (const file of value) {
|
||||
if (!isRecord(file)) continue;
|
||||
const path = stringValue(file.path).toLowerCase();
|
||||
if (path !== "skill.md" && !path.endsWith("/skill.md")) continue;
|
||||
const path = stringValue(file.path);
|
||||
if (!isPrimarySkillReadmePath(path)) continue;
|
||||
const content =
|
||||
stringOrNull(file.contentRedacted) ??
|
||||
stringOrNull(file.content_redacted) ??
|
||||
@@ -309,6 +310,43 @@ function skillMdContentFromExport(value: unknown): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function bundleFilesFromExport(value: unknown) {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.flatMap((file) => {
|
||||
if (!isRecord(file)) return [];
|
||||
const path = stringValue(file.path);
|
||||
if (!path || isExcludedSkillBundlePath(path)) return [];
|
||||
const content =
|
||||
stringOrNull(file.contentRedacted) ??
|
||||
stringOrNull(file.content_redacted) ??
|
||||
stringOrNull(file.content) ??
|
||||
stringOrNull(file.text);
|
||||
if (!content) return [];
|
||||
return [{ path, content: redactBundleContent(content) }];
|
||||
});
|
||||
}
|
||||
|
||||
function isExcludedSkillBundlePath(path: string) {
|
||||
return (
|
||||
isPrimarySkillReadmePath(path) || normalizeBundlePathForComparison(path) === "skill-card.md"
|
||||
);
|
||||
}
|
||||
|
||||
function isPrimarySkillReadmePath(path: string) {
|
||||
const normalized = normalizeBundlePathForComparison(path);
|
||||
return normalized === "skill.md" || normalized === "skills.md";
|
||||
}
|
||||
|
||||
function normalizeBundlePathForComparison(path: string) {
|
||||
return path
|
||||
.trim()
|
||||
.replace(/^\/+/, "")
|
||||
.split("/")
|
||||
.filter((segment) => segment && segment !== ".")
|
||||
.join("/")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function vtAnalysisFromExport(value: unknown): VtAnalysisInput | null {
|
||||
if (!isRecord(value)) return null;
|
||||
return {
|
||||
|
||||
@@ -376,7 +376,7 @@ function buildManifest(input: {
|
||||
},
|
||||
scannerVersions: Array.from(state.scannerVersions).sort(),
|
||||
modelNames: Array.from(state.modelNames).sort(),
|
||||
redactionPolicyVersion: "public-signals-v2",
|
||||
redactionPolicyVersion: "public-signals-v2-bundle-files",
|
||||
sourceTables: ["skillVersions", "packageReleases"],
|
||||
timeWindow: options.timeWindow,
|
||||
});
|
||||
|
||||
@@ -248,4 +248,43 @@ describe("security dataset normalizer", () => {
|
||||
);
|
||||
expect(rows.artifacts[0]?.skill_md_content_redacted).not.toContain("PRIVATE KEY");
|
||||
});
|
||||
|
||||
it("emits redacted authored bundle files with content hashes and sizes", () => {
|
||||
const rows = normalizeArtifactExport([
|
||||
{
|
||||
...baseArtifact,
|
||||
bundleFilesRedacted: [
|
||||
{
|
||||
path: "scripts/export.py",
|
||||
content: "import json\npassword=[REDACTED_SECRET]\n",
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(rows.artifacts[0]?.bundle_files_redacted).toEqual([
|
||||
{
|
||||
path: "scripts/export.py",
|
||||
content: "import json\n[REDACTED_SECRET]\n",
|
||||
sha256: hashString("import json\n[REDACTED_SECRET]\n"),
|
||||
size_bytes: Buffer.byteLength("import json\n[REDACTED_SECRET]\n", "utf8"),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits oversized redacted bundle files", () => {
|
||||
const rows = normalizeArtifactExport([
|
||||
{
|
||||
...baseArtifact,
|
||||
bundleFilesRedacted: [
|
||||
{
|
||||
path: "scripts/large.py",
|
||||
content: "x".repeat(600 * 1024),
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(rows.artifacts[0]?.bundle_files_redacted).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,6 +19,11 @@ export type ExportFileInput = {
|
||||
contentType: string | null;
|
||||
};
|
||||
|
||||
export type BundleFileInput = {
|
||||
path: string;
|
||||
content: string;
|
||||
};
|
||||
|
||||
export type VtAnalysisInput = {
|
||||
status: string;
|
||||
verdict: string | null;
|
||||
@@ -119,6 +124,7 @@ export type ArtifactExportInput = {
|
||||
version: string;
|
||||
artifactSha256: string | null;
|
||||
skillMdContentRedacted?: string | null;
|
||||
bundleFilesRedacted?: BundleFileInput[] | null;
|
||||
createdAt: number;
|
||||
softDeletedAt: number | null;
|
||||
files: ExportFileInput[];
|
||||
@@ -147,6 +153,12 @@ export type ArtifactRow = {
|
||||
version: string;
|
||||
artifact_sha256: string | null;
|
||||
skill_md_content_redacted?: string | null;
|
||||
bundle_files_redacted?: Array<{
|
||||
path: string;
|
||||
content: string;
|
||||
sha256: string;
|
||||
size_bytes: number;
|
||||
}>;
|
||||
created_at: number;
|
||||
created_month: string;
|
||||
soft_deleted: boolean;
|
||||
@@ -247,6 +259,7 @@ export type NormalizedDatasetRows = {
|
||||
const SPLIT_VERSION = "sha256-v1";
|
||||
const MAX_REDACTED_TEXT_LENGTH = 240;
|
||||
const MAX_REDACTED_SKILL_CONTENT_LENGTH = 120_000;
|
||||
const MAX_REDACTED_BUNDLE_FILE_BYTES = 192 * 1024;
|
||||
const CLAWSCAN_SEVERITIES = new Set<ClawScanSeverity>([
|
||||
"none",
|
||||
"info",
|
||||
@@ -332,6 +345,7 @@ export function assignSplit(splitKey: string): DatasetSplit {
|
||||
}
|
||||
|
||||
function buildArtifactRow(input: ArtifactExportInput, artifactId: string): ArtifactRow {
|
||||
const bundleFiles = buildBundleFileRows(input);
|
||||
return {
|
||||
artifact_id: artifactId,
|
||||
source_kind: input.sourceKind,
|
||||
@@ -347,6 +361,7 @@ function buildArtifactRow(input: ArtifactExportInput, artifactId: string): Artif
|
||||
...(input.sourceKind === "skill" && input.skillMdContentRedacted
|
||||
? { skill_md_content_redacted: redactSkillContent(input.skillMdContentRedacted) }
|
||||
: {}),
|
||||
...(bundleFiles.length > 0 ? { bundle_files_redacted: bundleFiles } : {}),
|
||||
created_at: input.createdAt,
|
||||
created_month: createdMonth(input.createdAt),
|
||||
soft_deleted: input.softDeletedAt !== null,
|
||||
@@ -366,6 +381,38 @@ function buildArtifactRow(input: ArtifactExportInput, artifactId: string): Artif
|
||||
};
|
||||
}
|
||||
|
||||
function buildBundleFileRows(
|
||||
input: ArtifactExportInput,
|
||||
): NonNullable<ArtifactRow["bundle_files_redacted"]> {
|
||||
if (input.sourceKind !== "skill" || !Array.isArray(input.bundleFilesRedacted)) return [];
|
||||
return input.bundleFilesRedacted.flatMap((file) => {
|
||||
const path = file.path.trim();
|
||||
if (!path || !file.content) return [];
|
||||
const content = redactBundleContent(file.content);
|
||||
if (Buffer.byteLength(content, "utf8") > MAX_REDACTED_BUNDLE_FILE_BYTES) return [];
|
||||
return [
|
||||
{
|
||||
path,
|
||||
content,
|
||||
sha256: hashString(content),
|
||||
size_bytes: Buffer.byteLength(content, "utf8"),
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
export function redactBundleContent(value: string) {
|
||||
let redacted = "";
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const code = value.charCodeAt(index);
|
||||
redacted += code < 32 && code !== 9 && code !== 10 && code !== 13 ? " " : value.charAt(index);
|
||||
}
|
||||
for (const pattern of SECRET_PATTERNS) {
|
||||
redacted = redacted.replace(pattern, "[REDACTED_SECRET]");
|
||||
}
|
||||
return redacted;
|
||||
}
|
||||
|
||||
function qualifiedPublicSlug(input: ArtifactExportInput) {
|
||||
if (input.sourceKind !== "skill") return null;
|
||||
if (!input.publicOwnerHandle || !input.publicSlug) return null;
|
||||
|
||||
@@ -3,6 +3,12 @@ import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promis
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertCodexWorkerExecutionAllowed,
|
||||
isCodexWorkerExecutionAllowed,
|
||||
LOCAL_CODEX_WORKER_OPT_IN,
|
||||
resolveCodexWorkerHome,
|
||||
} from "../codex-worker-guard";
|
||||
import {
|
||||
buildPrompt,
|
||||
normalizeSkillSpectorAnalysis,
|
||||
@@ -24,6 +30,54 @@ async function tempDir() {
|
||||
}
|
||||
|
||||
describe("run-codex-scan-worker diagnostics", () => {
|
||||
it("blocks direct local Codex security worker runs without opt-in", () => {
|
||||
expect(isCodexWorkerExecutionAllowed({})).toBe(false);
|
||||
expect(() => assertCodexWorkerExecutionAllowed({})).toThrow(
|
||||
`Refusing to run local Codex workers without ${LOCAL_CODEX_WORKER_OPT_IN}=1`,
|
||||
);
|
||||
});
|
||||
|
||||
it("does not treat a bare GITHUB_ACTIONS flag as CI authorization", () => {
|
||||
expect(isCodexWorkerExecutionAllowed({ GITHUB_ACTIONS: "true" })).toBe(false);
|
||||
expect(() => assertCodexWorkerExecutionAllowed({ GITHUB_ACTIONS: "true" })).toThrow(
|
||||
`Refusing to run local Codex workers without ${LOCAL_CODEX_WORKER_OPT_IN}=1`,
|
||||
);
|
||||
});
|
||||
|
||||
it("allows direct Codex security worker runs in GitHub Actions", () => {
|
||||
const env = {
|
||||
CI: "true",
|
||||
GITHUB_ACTIONS: "true",
|
||||
GITHUB_REPOSITORY: "openclaw/clawhub",
|
||||
GITHUB_RUN_ID: "123",
|
||||
};
|
||||
|
||||
expect(isCodexWorkerExecutionAllowed(env)).toBe(true);
|
||||
expect(() => assertCodexWorkerExecutionAllowed(env)).not.toThrow();
|
||||
});
|
||||
|
||||
it("allows direct local Codex security worker runs with explicit opt-in", () => {
|
||||
expect(isCodexWorkerExecutionAllowed({ [LOCAL_CODEX_WORKER_OPT_IN]: "1" })).toBe(true);
|
||||
expect(() =>
|
||||
assertCodexWorkerExecutionAllowed({ [LOCAL_CODEX_WORKER_OPT_IN]: "1" }),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("uses an isolated local Codex home for opted-in local workers by default", () => {
|
||||
expect(
|
||||
resolveCodexWorkerHome(
|
||||
{ [LOCAL_CODEX_WORKER_OPT_IN]: "1" },
|
||||
"/repo/.codex/runtime/codex-workers/security-scan",
|
||||
),
|
||||
).toBe("/repo/.codex/runtime/codex-workers/security-scan");
|
||||
expect(
|
||||
resolveCodexWorkerHome(
|
||||
{ [LOCAL_CODEX_WORKER_OPT_IN]: "1", CODEX_HOME: "/tmp/custom-codex-home" },
|
||||
"/repo/.codex/runtime/codex-workers/security-scan",
|
||||
),
|
||||
).toBe("/tmp/custom-codex-home");
|
||||
});
|
||||
|
||||
it("frames workspace inspection as discretionary Codex research", () => {
|
||||
const prompt = buildPrompt(
|
||||
{
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user