From 7accfb71c771045533af715c6b5509c9cc0d236a Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Fri, 19 Jun 2026 13:35:44 -0700 Subject: [PATCH] chore: remove registry artifact backup jobs Stop custom registry artifact backup/backfill/restore behavior now that Convex backups with file storage are the recovery source of truth. Legacy backup schema tables remain inert until a separate verified cleanup removes stored rows. --- CONTRIBUTING.md | 11 +- convex/_generated/api.d.ts | 10 - convex/crons.test.ts | 16 - convex/crons.ts | 7 - convex/httpApiV1.handlers.test.ts | 94 - convex/httpApiV1/usersV1.ts | 70 - convex/lib/registryArtifactBackup.test.ts | 486 ----- convex/lib/registryArtifactBackup.ts | 525 ------ convex/lib/retentionPolicy.ts | 4 +- convex/lib/skillPublish.test.ts | 3 - convex/lib/skillPublish.ts | 46 - convex/packages.public.test.ts | 40 +- convex/packages.ts | 75 - convex/registryArtifactBackups.test.ts | 1963 -------------------- convex/registryArtifactBackups.ts | 816 -------- convex/registryArtifactBackupsNode.ts | 1033 ---------- convex/registryArtifactRestore.test.ts | 283 --- convex/registryArtifactRestore.ts | 350 ---- convex/registryArtifactRestoreMutations.ts | 168 -- convex/skills.ownership.test.ts | 15 +- convex/skills.ts | 138 -- e2e/permissions.e2e.test.ts | 1 - specs/spec.md | 6 +- 23 files changed, 12 insertions(+), 6148 deletions(-) delete mode 100644 convex/lib/registryArtifactBackup.test.ts delete mode 100644 convex/lib/registryArtifactBackup.ts delete mode 100644 convex/registryArtifactBackups.test.ts delete mode 100644 convex/registryArtifactBackups.ts delete mode 100644 convex/registryArtifactBackupsNode.ts delete mode 100644 convex/registryArtifactRestore.test.ts delete mode 100644 convex/registryArtifactRestore.ts delete mode 100644 convex/registryArtifactRestoreMutations.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbd7b9da..13454dc3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -175,12 +175,11 @@ Without `OPENAI_API_KEY`, public corpus import still works, but semantic search These features degrade gracefully without their keys: -| Variable | Purpose | -| ---------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | -| `OPENAI_API_KEY` | Embeddings and vector search (falls back to zero vectors) | -| `VT_API_KEY` | VirusTotal malware scanning | -| `DISCORD_WEBHOOK_URL` | Discord notifications | -| `REGISTRY_BACKUP_R2_ACCOUNT_ID` / `REGISTRY_BACKUP_BUCKET` / `REGISTRY_BACKUP_ACCESS_KEY_ID` / `REGISTRY_BACKUP_SECRET_ACCESS_KEY` | Registry artifact publish backup and seed/backfill | +| Variable | Purpose | +| --------------------- | --------------------------------------------------------- | +| `OPENAI_API_KEY` | Embeddings and vector search (falls back to zero vectors) | +| `VT_API_KEY` | VirusTotal malware scanning | +| `DISCORD_WEBHOOK_URL` | Discord notifications | ## CLI Development diff --git a/convex/_generated/api.d.ts b/convex/_generated/api.d.ts index 8689e081..31cdf1eb 100644 --- a/convex/_generated/api.d.ts +++ b/convex/_generated/api.d.ts @@ -93,7 +93,6 @@ import type * as lib_publisherStats from "../lib/publisherStats.js"; import type * as lib_publishers from "../lib/publishers.js"; import type * as lib_rateLimitConfig from "../lib/rateLimitConfig.js"; import type * as lib_recommendationScore from "../lib/recommendationScore.js"; -import type * as lib_registryArtifactBackup from "../lib/registryArtifactBackup.js"; import type * as lib_reporting from "../lib/reporting.js"; import type * as lib_reservedHandles from "../lib/reservedHandles.js"; import type * as lib_reservedSlugs from "../lib/reservedSlugs.js"; @@ -131,10 +130,6 @@ import type * as publisherAbuse from "../publisherAbuse.js"; import type * as publisherAbuseDevSeed from "../publisherAbuseDevSeed.js"; import type * as publishers from "../publishers.js"; import type * as rateLimits from "../rateLimits.js"; -import type * as registryArtifactBackups from "../registryArtifactBackups.js"; -import type * as registryArtifactBackupsNode from "../registryArtifactBackupsNode.js"; -import type * as registryArtifactRestore from "../registryArtifactRestore.js"; -import type * as registryArtifactRestoreMutations from "../registryArtifactRestoreMutations.js"; import type * as retention from "../retention.js"; import type * as search from "../search.js"; import type * as securityDataset from "../securityDataset.js"; @@ -245,7 +240,6 @@ declare const fullApi: ApiFromModules<{ "lib/publishers": typeof lib_publishers; "lib/rateLimitConfig": typeof lib_rateLimitConfig; "lib/recommendationScore": typeof lib_recommendationScore; - "lib/registryArtifactBackup": typeof lib_registryArtifactBackup; "lib/reporting": typeof lib_reporting; "lib/reservedHandles": typeof lib_reservedHandles; "lib/reservedSlugs": typeof lib_reservedSlugs; @@ -283,10 +277,6 @@ declare const fullApi: ApiFromModules<{ publisherAbuseDevSeed: typeof publisherAbuseDevSeed; publishers: typeof publishers; rateLimits: typeof rateLimits; - registryArtifactBackups: typeof registryArtifactBackups; - registryArtifactBackupsNode: typeof registryArtifactBackupsNode; - registryArtifactRestore: typeof registryArtifactRestore; - registryArtifactRestoreMutations: typeof registryArtifactRestoreMutations; retention: typeof retention; search: typeof search; securityDataset: typeof securityDataset; diff --git a/convex/crons.test.ts b/convex/crons.test.ts index 17a82d54..fd9b3035 100644 --- a/convex/crons.test.ts +++ b/convex/crons.test.ts @@ -4,7 +4,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => { const interval = vi.fn(); const githubSkillSyncRef = Symbol("github-skill-source-sync"); - const registryArtifactBackupRetryRef = Symbol("registry-artifact-backup-retry"); const installTelemetryDedupePruneRef = Symbol("install-telemetry-dedupe-prune"); const rateLimitCountersPruneRef = Symbol("rate-limit-counters-prune"); const skillStatEventPruneRef = Symbol("skill-stat-event-prune"); @@ -13,7 +12,6 @@ const mocks = vi.hoisted(() => { return { interval, githubSkillSyncRef, - registryArtifactBackupRetryRef, installTelemetryDedupePruneRef, rateLimitCountersPruneRef, skillStatEventPruneRef, @@ -30,9 +28,6 @@ vi.mock("convex/server", () => ({ vi.mock("./_generated/api", () => ({ internal: { - registryArtifactBackupsNode: { - processRegistryArtifactBackupRetriesInternal: mocks.registryArtifactBackupRetryRef, - }, githubSkillSyncNode: { syncGitHubSkillSourcesInternal: mocks.githubSkillSyncRef }, leaderboards: { rebuildTrendingLeaderboardAction: Symbol("trending-leaderboard") }, statsMaintenance: { @@ -93,17 +88,6 @@ describe("crons", () => { expect(mocks.interval).not.toHaveBeenCalled(); }); - it("drains registry artifact backup retries frequently enough for publish bursts", async () => { - await import("./crons"); - - expect(mocks.interval).toHaveBeenCalledWith( - "registry-artifact-backup-retries", - { minutes: 5 }, - mocks.registryArtifactBackupRetryRef, - {}, - ); - }); - it("runs GitHub skill source sync every 15 minutes", async () => { await import("./crons"); diff --git a/convex/crons.ts b/convex/crons.ts index 19e535e0..950f1c2d 100644 --- a/convex/crons.ts +++ b/convex/crons.ts @@ -5,13 +5,6 @@ import { RETENTION_STANDARD_BATCH_SIZE } from "./lib/retentionPolicy"; const crons = cronJobs(); if (process.env.CLAWHUB_DISABLE_CRONS !== "1") { - crons.interval( - "registry-artifact-backup-retries", - { minutes: 5 }, - internal.registryArtifactBackupsNode.processRegistryArtifactBackupRetriesInternal, - {}, - ); - crons.interval( "github-skill-source-sync", { minutes: 15 }, diff --git a/convex/httpApiV1.handlers.test.ts b/convex/httpApiV1.handlers.test.ts index 5c126a02..2c1144bb 100644 --- a/convex/httpApiV1.handlers.test.ts +++ b/convex/httpApiV1.handlers.test.ts @@ -366,100 +366,6 @@ describe("httpApiV1 handlers", () => { expect(runAction).not.toHaveBeenCalled(); }); - it("users/restore forbids non-admin api tokens", async () => { - const runQuery = vi.fn(); - const runAction = vi.fn(); - const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { - if (isRateLimitArgs(args)) return okRate(); - if (args.ownerHandle === "me") return { publisherId: "publishers:me" }; - return okRate(); - }); - vi.mocked(requireApiTokenUser).mockResolvedValue({ - userId: "users:actor", - user: { _id: "users:actor", role: "user" }, - } as never); - - const response = await __handlers.usersPostRouterV1Handler( - makeCtx({ runQuery, runAction, runMutation }), - new Request("https://example.com/api/v1/users/restore", { - method: "POST", - body: JSON.stringify({ handle: "target", slugs: ["a"] }), - }), - ); - expect(response.status).toBe(403); - expect(runQuery).not.toHaveBeenCalled(); - expect(runAction).not.toHaveBeenCalled(); - }); - - it("users/restore calls restore action for admin", async () => { - const runAction = vi.fn().mockResolvedValue({ ok: true, totalRestored: 1, results: [] }); - const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { - if (isRateLimitArgs(args)) return okRate(); - return { ok: true }; - }); - const runQuery = vi.fn(async (_query: unknown, args: Record) => { - if ("handle" in args) return { _id: "users:target" }; - return null; - }); - vi.mocked(requireApiTokenUser).mockResolvedValue({ - userId: "users:admin", - user: { _id: "users:admin", role: "admin" }, - } as never); - - const response = await __handlers.usersPostRouterV1Handler( - makeCtx({ runQuery, runAction, runMutation }), - new Request("https://example.com/api/v1/users/restore", { - method: "POST", - body: JSON.stringify({ - handle: "Target", - slugs: ["a", "b"], - versionsBySlug: { a: "1.0.0", b: "1.1.0" }, - forceOverwriteSquatter: true, - }), - }), - ); - if (response.status !== 200) throw new Error(await response.text()); - expect(runAction).toHaveBeenCalledWith(expect.anything(), { - actorUserId: "users:admin", - ownerHandle: "target", - ownerUserId: "users:target", - slugs: ["a", "b"], - versionsBySlug: { a: "1.0.0", b: "1.1.0" }, - forceOverwriteSquatter: true, - }); - }); - - it("users/restore requires a backup version for every slug", async () => { - const runAction = vi.fn(); - const runMutation = vi.fn(async (_mutation: unknown, args: Record) => { - if (isRateLimitArgs(args)) return okRate(); - return { ok: true }; - }); - const runQuery = vi.fn(); - vi.mocked(requireApiTokenUser).mockResolvedValue({ - userId: "users:admin", - user: { _id: "users:admin", role: "admin" }, - } as never); - - const response = await __handlers.usersPostRouterV1Handler( - makeCtx({ runQuery, runAction, runMutation }), - new Request("https://example.com/api/v1/users/restore", { - method: "POST", - body: JSON.stringify({ - handle: "Target", - slugs: ["a", "b"], - versionsBySlug: { a: "1.0.0" }, - forceOverwriteSquatter: true, - }), - }), - ); - - expect(response.status).toBe(400); - expect(await response.text()).toBe("Missing backup version for slug b"); - expect(runQuery).not.toHaveBeenCalled(); - expect(runAction).not.toHaveBeenCalled(); - }); - it("skills export allows authenticated non-admin users at the key rate limit", async () => { vi.mocked(requireApiTokenUser).mockResolvedValue({ userId: "users:actor", diff --git a/convex/httpApiV1/usersV1.ts b/convex/httpApiV1/usersV1.ts index 9ec41cb7..1806698c 100644 --- a/convex/httpApiV1/usersV1.ts +++ b/convex/httpApiV1/usersV1.ts @@ -86,7 +86,6 @@ export async function usersPostRouterV1Handler(ctx: ActionCtx, request: Request) action !== "ban" && action !== "unban" && action !== "role" && - action !== "restore" && action !== "reclassify-ban" && action !== "ban-appeal-unban" && action !== "reclaim" && @@ -114,13 +113,6 @@ export async function usersPostRouterV1Handler(ctx: ActionCtx, request: Request) const actorUserId = authResult.userId; const actorUser = authResult.user; - // Restore and reclaim have different parameter shapes, handle them separately - if (action === "restore") { - const admin = requireAdminOrResponse(actorUser, rate.headers); - if (!admin.ok) return admin.response; - return handleAdminRestore(ctx, request, payload, actorUserId, rate.headers); - } - if (action === "reclassify-ban") { const admin = requireAdminOrResponse(actorUser, rate.headers); if (!admin.ok) return admin.response; @@ -554,68 +546,6 @@ export async function usersGetRouterV1Handler(ctx: ActionCtx, request: Request) } } -/** - * POST /api/v1/users/restore - * Admin-only: restore skills from registry artifact backup for a user. - * Body: { handle: string, slugs: string[], versionsBySlug: Record, forceOverwriteSquatter?: boolean } - */ -async function handleAdminRestore( - ctx: ActionCtx, - _request: Request, - payload: Record, - actorUserId: Id<"users">, - headers: HeadersInit, -) { - const handle = typeof payload.handle === "string" ? payload.handle.trim().toLowerCase() : ""; - if (!handle) return text("Missing handle", 400, headers); - - const slugs = Array.isArray(payload.slugs) - ? payload.slugs.filter((s): s is string => typeof s === "string") - : []; - if (slugs.length === 0) return text("Missing slugs array", 400, headers); - if (slugs.length > 100) return text("Too many slugs (max 100)", 400, headers); - - const versionsBySlug = - payload.versionsBySlug && typeof payload.versionsBySlug === "object" - ? Object.fromEntries( - Object.entries(payload.versionsBySlug).filter( - (entry): entry is [string, string] => - typeof entry[0] === "string" && typeof entry[1] === "string", - ), - ) - : undefined; - if (!versionsBySlug) return text("Missing versionsBySlug", 400, headers); - const missingVersionSlug = slugs.find((slug) => !versionsBySlug[slug]?.trim()); - if (missingVersionSlug) { - return text(`Missing backup version for slug ${missingVersionSlug}`, 400, headers); - } - const forceOverwriteSquatter = Boolean(payload.forceOverwriteSquatter); - - const targetUser = await ctx.runQuery(api.users.getByHandle, { handle }); - if (!targetUser?._id) return text("User not found", 404, headers); - - try { - const result = await ctx.runAction( - internal.registryArtifactRestore.restoreUserSkillsFromBackup, - { - actorUserId, - ownerHandle: handle, - ownerUserId: targetUser._id, - slugs, - versionsBySlug, - forceOverwriteSquatter, - }, - ); - return json(result, 200, headers); - } catch (error) { - const message = error instanceof Error ? error.message : "Restore failed"; - if (message.toLowerCase().includes("forbidden")) { - return text("Forbidden", 403, headers); - } - return text(message, 400, headers); - } -} - /** * POST /api/v1/users/reclaim * Admin-only: reclaim root slugs for the rightful owner. diff --git a/convex/lib/registryArtifactBackup.test.ts b/convex/lib/registryArtifactBackup.test.ts deleted file mode 100644 index e018ae94..00000000 --- a/convex/lib/registryArtifactBackup.test.ts +++ /dev/null @@ -1,486 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; -import type { Id } from "../_generated/dataModel"; -import { - __registryArtifactBackupTestInternals, - backupPackageReleaseToObjectStorage, - backupSkillVersionToObjectStorage, - buildPackageReleaseBackupManifest, - buildSkillVersionBackupManifest, - getRegistryArtifactBackupSettings, - readRegistryArtifactBackupObject, -} from "./registryArtifactBackup"; - -describe("registry artifact backup settings", () => { - const originalEnv = { - endpoint: process.env.REGISTRY_BACKUP_S3_ENDPOINT, - accountId: process.env.REGISTRY_BACKUP_R2_ACCOUNT_ID, - bucket: process.env.REGISTRY_BACKUP_BUCKET, - accessKeyId: process.env.REGISTRY_BACKUP_ACCESS_KEY_ID, - secretAccessKey: process.env.REGISTRY_BACKUP_SECRET_ACCESS_KEY, - region: process.env.REGISTRY_BACKUP_S3_REGION, - skillsRoot: process.env.REGISTRY_BACKUP_SKILLS_ROOT, - packagesRoot: process.env.REGISTRY_BACKUP_PACKAGES_ROOT, - skillFileUploadConcurrency: process.env.REGISTRY_BACKUP_SKILL_FILE_UPLOAD_CONCURRENCY, - }; - - afterEach(() => { - setEnv("REGISTRY_BACKUP_S3_ENDPOINT", originalEnv.endpoint); - setEnv("REGISTRY_BACKUP_R2_ACCOUNT_ID", originalEnv.accountId); - setEnv("REGISTRY_BACKUP_BUCKET", originalEnv.bucket); - setEnv("REGISTRY_BACKUP_ACCESS_KEY_ID", originalEnv.accessKeyId); - setEnv("REGISTRY_BACKUP_SECRET_ACCESS_KEY", originalEnv.secretAccessKey); - setEnv("REGISTRY_BACKUP_S3_REGION", originalEnv.region); - setEnv("REGISTRY_BACKUP_SKILLS_ROOT", originalEnv.skillsRoot); - setEnv("REGISTRY_BACKUP_PACKAGES_ROOT", originalEnv.packagesRoot); - setEnv("REGISTRY_BACKUP_SKILL_FILE_UPLOAD_CONCURRENCY", originalEnv.skillFileUploadConcurrency); - }); - - it("defaults registry artifact backups to skills and packages object roots", () => { - delete process.env.REGISTRY_BACKUP_S3_ENDPOINT; - process.env.REGISTRY_BACKUP_R2_ACCOUNT_ID = "account-id"; - process.env.REGISTRY_BACKUP_BUCKET = "clawhub-registry-backup"; - process.env.REGISTRY_BACKUP_ACCESS_KEY_ID = "access-key"; - process.env.REGISTRY_BACKUP_SECRET_ACCESS_KEY = "secret-key"; - delete process.env.REGISTRY_BACKUP_S3_REGION; - delete process.env.REGISTRY_BACKUP_SKILLS_ROOT; - delete process.env.REGISTRY_BACKUP_PACKAGES_ROOT; - - expect(getRegistryArtifactBackupSettings()).toEqual({ - endpoint: "https://account-id.r2.cloudflarestorage.com", - bucket: "clawhub-registry-backup", - accessKeyId: "access-key", - secretAccessKey: "secret-key", - region: "auto", - skillsRoot: "skills", - packagesRoot: "packages", - }); - }); - - it("builds versioned skill backup paths and restore metadata", () => { - const manifest = buildSkillVersionBackupManifest({ - root: "skills", - ownerHandle: "OpenClaw Team", - skillId: "skills:demo" as Id<"skills">, - versionId: "skillVersions:demo-1" as Id<"skillVersions">, - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.2.3", - publishedAt: 1_700_000_000_000, - files: [ - { - path: "SKILL.md", - size: 42, - storageId: "storage:skill" as Id<"_storage">, - sha256: "sha256:skill", - contentType: "text/markdown", - }, - ], - }); - - expect(manifest).toMatchObject({ - skillRoot: "skills/openclaw-team/demo-skill", - versionRoot: "skills/openclaw-team/demo-skill/1%2E2%2E3", - metaPath: "skills/openclaw-team/demo-skill/1%2E2%2E3/_meta.json", - fileObjects: [ - { - key: "skills/openclaw-team/demo-skill/1%2E2%2E3/SKILL.md", - path: "SKILL.md", - sha256: "sha256:skill", - contentType: "text/markdown", - }, - ], - meta: { - kind: "skillVersion", - owner: "openclaw-team", - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.2.3", - restore: { - skillId: "skills:demo", - versionId: "skillVersions:demo-1", - }, - }, - }); - }); - - it("rejects unsafe skill file paths before writing backup object keys", () => { - expect(() => - buildSkillVersionBackupManifest({ - root: "skills", - ownerHandle: "OpenClaw Team", - versionId: "skillVersions:demo-1" as Id<"skillVersions">, - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.2.3", - publishedAt: 1_700_000_000_000, - files: [ - { - path: "../SKILL.md", - size: 42, - storageId: "storage:skill" as Id<"_storage">, - sha256: "sha256:skill", - }, - ], - }), - ).toThrow("Invalid skill backup file path"); - }); - - it("builds package release backup paths and restore metadata", () => { - const manifest = buildPackageReleaseBackupManifest({ - root: "packages", - ownerHandle: "OpenClaw Team", - packageId: "packages:demo" as Id<"packages">, - releaseId: "packageReleases:demo-1" as Id<"packageReleases">, - packageName: "@openclaw/demo-plugin", - normalizedName: "@openclaw/demo-plugin", - displayName: "Demo Plugin", - family: "code-plugin", - version: "1.2.3", - publishedAt: 1_700_000_000_000, - artifactKind: "npm-pack", - artifactFileName: "demo-plugin-1.2.3.tgz", - artifactSha256: "sha256:artifact", - artifactSize: 42, - artifactFormat: "tgz", - npmIntegrity: "sha512-demo", - npmShasum: "abc123", - files: [{ path: "package.json", size: 10, sha256: "sha256:package-json" }], - }); - - expect(manifest).toMatchObject({ - packageRoot: "packages/openclaw-team/%40openclaw%2Fdemo-plugin", - releaseRoot: "packages/openclaw-team/%40openclaw%2Fdemo-plugin/1%2E2%2E3", - artifactPath: - "packages/openclaw-team/%40openclaw%2Fdemo-plugin/1%2E2%2E3/demo-plugin-1.2.3.tgz", - metaPath: "packages/openclaw-team/%40openclaw%2Fdemo-plugin/1%2E2%2E3/_meta.json", - meta: { - kind: "packageRelease", - restore: { - packageId: "packages:demo", - releaseId: "packageReleases:demo-1", - }, - artifact: { - path: "demo-plugin-1.2.3.tgz", - sha256: "sha256:artifact", - size: 42, - format: "tgz", - npmIntegrity: "sha512-demo", - npmShasum: "abc123", - }, - }, - }); - }); - - it("rejects unsafe package artifact filenames before writing backup object keys", () => { - expect(() => - buildPackageReleaseBackupManifest({ - root: "packages", - ownerHandle: "OpenClaw Team", - packageId: "packages:demo" as Id<"packages">, - releaseId: "packageReleases:demo-1" as Id<"packageReleases">, - packageName: "@openclaw/demo-plugin", - normalizedName: "@openclaw/demo-plugin", - displayName: "Demo Plugin", - family: "code-plugin", - version: "1.2.3", - publishedAt: 1_700_000_000_000, - artifactKind: "npm-pack", - artifactFileName: "../evil.tgz", - artifactSha256: "sha256:artifact", - artifactSize: 42, - artifactFormat: "tgz", - files: [], - }), - ).toThrow("Invalid package backup artifact filename"); - }); - - it("uses lossless path encoding to avoid package and version collisions", () => { - expect(__registryArtifactBackupTestInternals.encodeBackupPathSegment("@openclaw/demo")).toBe( - "%40openclaw%2Fdemo", - ); - expect(__registryArtifactBackupTestInternals.encodeBackupPathSegment("foo.bar")).toBe( - "foo%2Ebar", - ); - expect(__registryArtifactBackupTestInternals.encodeBackupPathSegment("foo_bar")).toBe( - "foo_bar", - ); - }); - - it("uses strict AWS URI encoding for object keys before signing R2 requests", () => { - expect( - __registryArtifactBackupTestInternals.encodeObjectKey( - "skills/smartpeopleconnected/token-optimizer/1%2E0%2E0/infomaterial/4_github_publish_Alles ist fertig!.txt", - ), - ).toBe( - "skills/smartpeopleconnected/token-optimizer/1%252E0%252E0/infomaterial/4_github_publish_Alles%20ist%20fertig%21.txt", - ); - }); - - it("preserves valid owner handle punctuation in backup paths", () => { - const dotted = buildSkillVersionBackupManifest({ - root: "skills", - ownerHandle: "foo.bar", - versionId: "skillVersions:dotted" as Id<"skillVersions">, - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.0.0", - publishedAt: 1_700_000_000_000, - files: [], - }); - const underscored = buildSkillVersionBackupManifest({ - root: "skills", - ownerHandle: "foo_bar", - versionId: "skillVersions:underscored" as Id<"skillVersions">, - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.0.0", - publishedAt: 1_700_000_000_000, - files: [], - }); - const dashed = buildSkillVersionBackupManifest({ - root: "skills", - ownerHandle: "foo-bar", - versionId: "skillVersions:dashed" as Id<"skillVersions">, - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.0.0", - publishedAt: 1_700_000_000_000, - files: [], - }); - - expect([dotted.skillRoot, underscored.skillRoot, dashed.skillRoot]).toEqual([ - "skills/foo.bar/demo-skill", - "skills/foo_bar/demo-skill", - "skills/foo-bar/demo-skill", - ]); - }); - - it("reads object bytes from object storage", async () => { - vi.stubGlobal( - "fetch", - vi.fn(async (url: URL | string, init?: RequestInit) => { - const key = objectKey(String(url)); - if ( - init?.method === "GET" && - key === "skills/openclaw-team/demo-skill/1%2E2%2E3/SKILL.md" - ) { - return response(200, "hello skill"); - } - return response(404, ""); - }), - ); - - const bytes = await readRegistryArtifactBackupObject( - makeContext(), - "skills/openclaw-team/demo-skill/1%2E2%2E3/SKILL.md", - ); - - expect(Buffer.from(bytes!).toString("utf8")).toBe("hello skill"); - }); - - it("writes skill files and version metadata to object storage", async () => { - const calls: Array<{ method: string; url: string; body: string }> = []; - vi.stubGlobal( - "fetch", - vi.fn(async (url: URL | string, init?: RequestInit) => { - const method = init?.method ?? "GET"; - const body = await requestBodyText(init?.body); - calls.push({ method, url: String(url), body }); - return response(200, ""); - }), - ); - - await backupSkillVersionToObjectStorage( - makeStorageCtx({ "storage:skill": "hello skill" }) as never, - { - root: "skills", - ownerHandle: "OpenClaw Team", - versionId: "skillVersions:demo-1" as Id<"skillVersions">, - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.2.3", - publishedAt: 1_700_000_000_000, - files: [ - { - path: "SKILL.md", - size: 11, - storageId: "storage:skill" as Id<"_storage">, - sha256: "sha256:skill", - contentType: "text/markdown", - }, - ], - }, - makeContext(), - ); - - expect(calls.map((call) => [call.method, objectKey(call.url)])).toEqual([ - ["PUT", "skills/openclaw-team/demo-skill/1%2E2%2E3/SKILL.md"], - ["PUT", "skills/openclaw-team/demo-skill/1%2E2%2E3/_meta.json"], - ]); - expect(JSON.parse(calls[1].body)).toMatchObject({ - kind: "skillVersion", - version: "1.2.3", - metadata: { files: [{ path: "SKILL.md", sha256: "sha256:skill" }] }, - }); - }); - - it("uploads skill files with bounded parallelism before writing version metadata", async () => { - process.env.REGISTRY_BACKUP_SKILL_FILE_UPLOAD_CONCURRENCY = "3"; - - const calls: Array<{ method: string; key: string }> = []; - let activeFileUploads = 0; - let maxActiveFileUploads = 0; - let completedFileUploads = 0; - let completedWhenMetaStarted = 0; - - vi.stubGlobal( - "fetch", - vi.fn(async (url: URL | string, init?: RequestInit) => { - const key = objectKey(String(url)); - calls.push({ method: init?.method ?? "GET", key }); - if (!key.endsWith("/_meta.json")) { - activeFileUploads += 1; - maxActiveFileUploads = Math.max(maxActiveFileUploads, activeFileUploads); - await new Promise((resolve) => setTimeout(resolve, 20)); - activeFileUploads -= 1; - completedFileUploads += 1; - } else { - completedWhenMetaStarted = completedFileUploads; - } - return response(200, ""); - }), - ); - - const files = Array.from({ length: 6 }, (_, index) => ({ - path: `file-${index}.txt`, - size: 6, - storageId: `storage:file-${index}` as Id<"_storage">, - sha256: `sha256:file-${index}`, - contentType: "text/plain", - })); - - await backupSkillVersionToObjectStorage( - makeStorageCtx( - Object.fromEntries(files.map((file) => [file.storageId, `body-${file.path}`])), - ) as never, - { - root: "skills", - ownerHandle: "OpenClaw Team", - versionId: "skillVersions:demo-1" as Id<"skillVersions">, - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.2.3", - publishedAt: 1_700_000_000_000, - files, - }, - makeContext(), - ); - - expect(maxActiveFileUploads).toBe(3); - expect(completedWhenMetaStarted).toBe(6); - expect(calls.at(-1)).toEqual({ - method: "PUT", - key: "skills/openclaw-team/demo-skill/1%2E2%2E3/_meta.json", - }); - }); - - it("writes package artifacts and version metadata to object storage", async () => { - const calls: Array<{ method: string; url: string; body: string }> = []; - vi.stubGlobal( - "fetch", - vi.fn(async (url: URL | string, init?: RequestInit) => { - const method = init?.method ?? "GET"; - const body = await requestBodyText(init?.body); - calls.push({ method, url: String(url), body }); - return response(200, ""); - }), - ); - - await backupPackageReleaseToObjectStorage( - makeStorageCtx({ "storage:artifact": "tgz bytes" }) as never, - { - root: "packages", - ownerHandle: "OpenClaw Team", - packageId: "packages:demo" as Id<"packages">, - releaseId: "packageReleases:demo-1" as Id<"packageReleases">, - packageName: "@openclaw/demo-plugin", - normalizedName: "@openclaw/demo-plugin", - displayName: "Demo Plugin", - family: "code-plugin", - version: "1.2.3", - publishedAt: 1_700_000_000_000, - artifactStorageId: "storage:artifact" as Id<"_storage">, - artifactFileName: "demo-plugin-1.2.3.tgz", - artifactSha256: "sha256:artifact", - artifactSize: 9, - files: [], - }, - makeContext(), - ); - - expect(calls.map((call) => [call.method, objectKey(call.url)])).toEqual([ - ["PUT", "packages/openclaw-team/%40openclaw%2Fdemo-plugin/1%2E2%2E3/demo-plugin-1.2.3.tgz"], - ["PUT", "packages/openclaw-team/%40openclaw%2Fdemo-plugin/1%2E2%2E3/_meta.json"], - ]); - expect(JSON.parse(calls[1].body)).toMatchObject({ - kind: "packageRelease", - artifact: { path: "demo-plugin-1.2.3.tgz", sha256: "sha256:artifact" }, - }); - }); -}); - -function setEnv(name: keyof NodeJS.ProcessEnv, value: string | undefined) { - if (value === undefined) { - delete process.env[name]; - return; - } - process.env[name] = value; -} - -function makeContext() { - return { - endpoint: "https://account.r2.cloudflarestorage.com", - bucket: "clawhub-registry-backup", - accessKeyId: "access-key", - secretAccessKey: "secret-key", - region: "auto", - skillsRoot: "skills", - packagesRoot: "packages", - }; -} - -function makeStorageCtx(contents: Record) { - return { - storage: { - get: async (id: Id<"_storage">) => { - const value = contents[id]; - return value === undefined ? null : new Blob([value]); - }, - }, - }; -} - -function response(status: number, body: string, headers: Record = {}) { - return { - ok: status >= 200 && status < 300, - status, - headers: new Headers(headers), - text: async () => body, - json: async () => JSON.parse(body), - arrayBuffer: async () => { - const buffer = Buffer.from(body); - return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength); - }, - }; -} - -function objectKey(url: string) { - const parsed = new URL(url); - const prefix = "/clawhub-registry-backup/"; - return decodeURIComponent(parsed.pathname.slice(prefix.length)); -} - -async function requestBodyText(body: BodyInit | null | undefined) { - if (!body) return ""; - return Buffer.from(await new Response(body).arrayBuffer()).toString("utf8"); -} diff --git a/convex/lib/registryArtifactBackup.ts b/convex/lib/registryArtifactBackup.ts deleted file mode 100644 index 48aedfa4..00000000 --- a/convex/lib/registryArtifactBackup.ts +++ /dev/null @@ -1,525 +0,0 @@ -"use node"; - -import { createHash, createHmac } from "node:crypto"; -import type { Id } from "../_generated/dataModel"; -import type { ActionCtx } from "../_generated/server"; -import { validateFilePath } from "./skillZip"; - -const DEFAULT_SKILLS_ROOT = "skills"; -const DEFAULT_PACKAGES_ROOT = "packages"; -const DEFAULT_SKILL_FILE_UPLOAD_CONCURRENCY = 16; -const MAX_SKILL_FILE_UPLOAD_CONCURRENCY = 64; -const META_FILENAME = "_meta.json"; - -type BackupFile = { - path: string; - size: number; - storageId: Id<"_storage">; - sha256: string; - contentType?: string; -}; - -type SkillBackupParams = { - skillId?: Id<"skills">; - versionId?: Id<"skillVersions">; - slug: string; - version: string; - isLatest?: boolean; - displayName: string; - ownerHandle: string; - files: BackupFile[]; - publishedAt: number; -}; - -type PackageBackupParams = { - ownerHandle: string; - packageId: Id<"packages">; - releaseId: Id<"packageReleases">; - packageName: string; - normalizedName: string; - displayName: string; - family: "code-plugin" | "bundle-plugin"; - version: string; - isLatest?: boolean; - publishedAt: number; - artifactKind?: "legacy-zip" | "npm-pack"; - artifactFileName?: string; - artifactSha256?: string; - artifactSize?: number; - artifactFormat?: "tgz"; - npmIntegrity?: string; - npmShasum?: string; - npmUnpackedSize?: number; - npmFileCount?: number; - runtimeId?: string; - sourceRepo?: string; - compatibility?: unknown; - extractedPackageJson?: unknown; - extractedPluginManifest?: unknown; - normalizedBundleManifest?: unknown; - files: Array<{ path: string; size: number; sha256: string }>; -}; - -export type RegistryArtifactBackupContext = RegistryArtifactBackupSettings; - -export type RegistryArtifactBackupSettings = { - endpoint: string; - bucket: string; - accessKeyId: string; - secretAccessKey: string; - region: string; - skillsRoot: string; - packagesRoot: string; -}; - -export function isRegistryArtifactBackupConfigured() { - return Boolean( - (process.env.REGISTRY_BACKUP_S3_ENDPOINT || process.env.REGISTRY_BACKUP_R2_ACCOUNT_ID) && - process.env.REGISTRY_BACKUP_BUCKET && - process.env.REGISTRY_BACKUP_ACCESS_KEY_ID && - process.env.REGISTRY_BACKUP_SECRET_ACCESS_KEY, - ); -} - -export function getRegistryArtifactBackupSettings(): RegistryArtifactBackupSettings { - const endpoint = - process.env.REGISTRY_BACKUP_S3_ENDPOINT ?? - r2EndpointFromAccountId(process.env.REGISTRY_BACKUP_R2_ACCOUNT_ID); - if (!endpoint) { - throw new Error("REGISTRY_BACKUP_S3_ENDPOINT or REGISTRY_BACKUP_R2_ACCOUNT_ID is required"); - } - const bucket = requiredEnv("REGISTRY_BACKUP_BUCKET"); - const accessKeyId = requiredEnv("REGISTRY_BACKUP_ACCESS_KEY_ID"); - const secretAccessKey = requiredEnv("REGISTRY_BACKUP_SECRET_ACCESS_KEY"); - return { - endpoint, - bucket, - accessKeyId, - secretAccessKey, - region: process.env.REGISTRY_BACKUP_S3_REGION ?? "auto", - skillsRoot: process.env.REGISTRY_BACKUP_SKILLS_ROOT ?? DEFAULT_SKILLS_ROOT, - packagesRoot: process.env.REGISTRY_BACKUP_PACKAGES_ROOT ?? DEFAULT_PACKAGES_ROOT, - }; -} - -export function getRegistryArtifactBackupContext(): RegistryArtifactBackupContext { - return getRegistryArtifactBackupSettings(); -} - -export async function backupSkillVersionToObjectStorage( - ctx: Pick, - params: SkillBackupParams & { root?: string }, - context: RegistryArtifactBackupContext = getRegistryArtifactBackupContext(), -) { - const planned = buildSkillVersionBackupManifest({ - root: params.root ?? context.skillsRoot, - ...params, - }); - - await runWithConcurrency(planned.fileObjects, getSkillFileUploadConcurrency(), async (file) => { - const blob = await readStorageBlob(ctx, file.storageId); - await putObject(context, file.key, new Uint8Array(await blob.arrayBuffer()), { - contentType: file.contentType, - }); - }); - - await putJsonObject(context, planned.metaPath, planned.meta); -} - -export async function backupPackageReleaseToObjectStorage( - ctx: Pick, - params: PackageBackupParams & { artifactStorageId: Id<"_storage">; root?: string }, - context: RegistryArtifactBackupContext = getRegistryArtifactBackupContext(), -) { - const planned = buildPackageReleaseBackupManifest({ - root: params.root ?? context.packagesRoot, - ...params, - }); - const artifact = await readStorageBlob(ctx, params.artifactStorageId); - await putObject(context, planned.artifactPath, new Uint8Array(await artifact.arrayBuffer()), { - contentType: packageArtifactContentType(params.artifactFormat), - }); - - await putJsonObject(context, planned.metaPath, planned.meta); -} - -export async function fetchSkillVersionBackupMeta( - context: RegistryArtifactBackupContext, - ownerHandle: string, - slug: string, - version: string, -) { - const owner = normalizeOwner(ownerHandle); - const path = `${context.skillsRoot}/${owner}/${slug}/${encodeBackupPathSegment( - version, - )}/${META_FILENAME}`; - return getJsonObject["meta"]>(context, path); -} - -export async function fetchPackageReleaseBackupMeta( - context: RegistryArtifactBackupContext, - ownerHandle: string, - normalizedName: string, - version: string, -) { - const owner = normalizeOwner(ownerHandle); - const path = `${context.packagesRoot}/${owner}/${encodeBackupPathSegment( - normalizedName, - )}/${encodeBackupPathSegment(version)}/${META_FILENAME}`; - return getJsonObject["meta"]>(context, path); -} - -export async function readRegistryArtifactBackupObject( - context: RegistryArtifactBackupContext, - key: string, -) { - const response = await signedFetch(context, "GET", key); - if (response.status === 404) return null; - if (!response.ok) { - const body = await response.text(); - throw new Error(`Registry artifact backup GET ${key} failed: ${body}`); - } - return new Uint8Array(await response.arrayBuffer()); -} - -export function buildSkillVersionBackupManifest(params: SkillBackupParams & { root: string }) { - const owner = normalizeOwner(params.ownerHandle); - const versionSegment = encodeBackupPathSegment(params.version); - const skillRoot = `${params.root}/${owner}/${params.slug}`; - const versionRoot = `${skillRoot}/${versionSegment}`; - const metaPath = `${versionRoot}/${META_FILENAME}`; - const files = params.files.map((file) => { - if (!validateFilePath(file.path)) { - throw new Error(`Invalid skill backup file path: ${file.path}`); - } - return file; - }); - const fileObjects = files.map((file) => ({ - ...file, - key: `${versionRoot}/${file.path}`, - })); - const meta = { - kind: "skillVersion" as const, - owner, - slug: params.slug, - displayName: params.displayName, - version: params.version, - isLatest: params.isLatest, - publishedAt: params.publishedAt, - restore: { - skillId: params.skillId, - versionId: params.versionId, - }, - metadata: { - files: files.map(({ path, size, sha256, contentType }) => ({ - path, - size, - sha256, - contentType, - })), - }, - }; - - return { - skillRoot, - versionRoot, - metaPath, - fileObjects, - meta, - }; -} - -export function buildPackageReleaseBackupManifest(params: PackageBackupParams & { root: string }) { - const owner = normalizeOwner(params.ownerHandle); - const packageSegment = encodeBackupPathSegment(params.normalizedName || params.packageName); - const artifactFileName = validatePackageArtifactFileName( - params.artifactFileName ?? defaultPackageArtifactFileName(params), - ); - const packageRoot = `${params.root}/${owner}/${packageSegment}`; - const releaseRoot = `${packageRoot}/${encodeBackupPathSegment(params.version)}`; - const meta = { - kind: "packageRelease" as const, - owner, - packageName: params.packageName, - normalizedName: params.normalizedName, - displayName: params.displayName, - family: params.family, - version: params.version, - isLatest: params.isLatest, - publishedAt: params.publishedAt, - runtimeId: params.runtimeId, - sourceRepo: params.sourceRepo, - artifactKind: params.artifactKind, - artifact: { - path: artifactFileName, - sha256: params.artifactSha256, - size: params.artifactSize, - format: params.artifactFormat, - npmIntegrity: params.npmIntegrity, - npmShasum: params.npmShasum, - npmUnpackedSize: params.npmUnpackedSize, - npmFileCount: params.npmFileCount, - }, - restore: { - packageId: params.packageId, - releaseId: params.releaseId, - }, - metadata: { - compatibility: params.compatibility, - extractedPackageJson: params.extractedPackageJson, - extractedPluginManifest: params.extractedPluginManifest, - normalizedBundleManifest: params.normalizedBundleManifest, - files: params.files, - }, - }; - - return { - packageRoot, - releaseRoot, - artifactPath: `${releaseRoot}/${artifactFileName}`, - metaPath: `${releaseRoot}/${META_FILENAME}`, - meta, - }; -} - -export const __registryArtifactBackupTestInternals = { - encodeBackupPathSegment, - encodeObjectKey, - getSkillFileUploadConcurrency, -}; - -async function runWithConcurrency( - items: T[], - concurrency: number, - worker: (item: T, index: number) => Promise, -) { - if (items.length === 0) return; - let nextIndex = 0; - let firstError: unknown; - const workerCount = Math.min(concurrency, items.length); - - async function runWorker() { - while (firstError === undefined) { - const index = nextIndex; - nextIndex += 1; - if (index >= items.length) return; - try { - await worker(items[index]!, index); - } catch (error) { - firstError ??= error; - return; - } - } - } - - await Promise.allSettled(Array.from({ length: workerCount }, () => runWorker())); - if (firstError !== undefined) throw firstError; -} - -function getSkillFileUploadConcurrency() { - const raw = process.env.REGISTRY_BACKUP_SKILL_FILE_UPLOAD_CONCURRENCY; - const parsed = raw ? Number.parseInt(raw, 10) : DEFAULT_SKILL_FILE_UPLOAD_CONCURRENCY; - if (!Number.isFinite(parsed) || parsed < 1) return DEFAULT_SKILL_FILE_UPLOAD_CONCURRENCY; - return Math.min(parsed, MAX_SKILL_FILE_UPLOAD_CONCURRENCY); -} - -export function normalizeOwner(value: string) { - const normalized = value - .trim() - .toLowerCase() - .replace(/^@+/, "") - .replace(/[^a-z0-9._-]/g, "-") - .replace(/-+/g, "-") - .replace(/^[._-]+|[._-]+$/g, ""); - return normalized || "unknown"; -} - -function encodeBackupPathSegment(value: string) { - return encodeURIComponent(value.trim()).replace(/\./g, "%2E"); -} - -function normalizePackagePathSegment(value: string) { - return normalizeOwner(value.replace(/^@/, "").replace("/", "-")); -} - -function defaultPackageArtifactFileName( - params: Pick, -) { - return `${normalizePackagePathSegment(params.normalizedName)}-${encodeBackupPathSegment( - params.version, - )}.tgz`; -} - -function validatePackageArtifactFileName(value: string) { - const artifactFileName = value.trim(); - if ( - !artifactFileName || - artifactFileName === "." || - artifactFileName === ".." || - artifactFileName.includes("/") || - artifactFileName.includes("\\") || - artifactFileName.includes("\0") - ) { - throw new Error("Invalid package backup artifact filename"); - } - return artifactFileName; -} - -async function readStorageBlob(ctx: Pick, storageId: Id<"_storage">) { - const blob = await ctx.storage.get(storageId); - if (!blob) throw new Error("File missing in storage"); - return blob; -} - -async function putJsonObject(context: RegistryArtifactBackupContext, key: string, value: unknown) { - await putObject(context, key, `${JSON.stringify(value, null, 2)}\n`, { - contentType: "application/json; charset=utf-8", - }); -} - -async function getJsonObject(context: RegistryArtifactBackupContext, key: string) { - const response = await signedFetch(context, "GET", key); - if (response.status === 404) return null; - if (!response.ok) { - const body = await response.text(); - throw new Error(`Registry artifact backup GET ${key} failed: ${body}`); - } - return (await response.json()) as T; -} - -async function putObject( - context: RegistryArtifactBackupContext, - key: string, - body: string | Uint8Array, - options: { - contentType?: string; - } = {}, -) { - const response = await signedFetch(context, "PUT", key, body, options); - if (!response.ok) { - const responseBody = await response.text(); - throw new Error(`Registry artifact backup PUT ${key} failed: ${responseBody}`); - } -} - -async function signedFetch( - context: RegistryArtifactBackupContext, - method: "GET" | "PUT", - key: string, - body?: string | Uint8Array, - options: { contentType?: string } = {}, -) { - const now = new Date(); - const bodyBytes = body === undefined ? new Uint8Array() : toBytes(body); - const payloadHash = sha256Hex(bodyBytes); - const url = objectUrl(context, key); - const headers = new Headers(); - headers.set("host", url.host); - headers.set("x-amz-content-sha256", payloadHash); - headers.set("x-amz-date", amzDate(now)); - if (options.contentType) headers.set("content-type", options.contentType); - headers.set( - "authorization", - authorizationHeader(context, method, url, headers, payloadHash, now), - ); - - const init: RequestInit = { method, headers }; - if (method === "PUT") { - init.body = toArrayBuffer(bodyBytes); - } - return fetch(url, init); -} - -function toArrayBuffer(bytes: Uint8Array) { - return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer; -} - -function authorizationHeader( - context: RegistryArtifactBackupContext, - method: string, - url: URL, - headers: Headers, - payloadHash: string, - now: Date, -) { - const date = amzDate(now).slice(0, 8); - const credentialScope = `${date}/${context.region}/s3/aws4_request`; - const signedHeaders = Array.from(headers.keys()) - .map((name) => name.toLowerCase()) - .sort() - .join(";"); - const canonicalHeaders = signedHeaders - .split(";") - .map((name) => `${name}:${headers.get(name)?.trim() ?? ""}\n`) - .join(""); - const canonicalRequest = [ - method, - url.pathname, - url.search.slice(1), - canonicalHeaders, - signedHeaders, - payloadHash, - ].join("\n"); - const stringToSign = [ - "AWS4-HMAC-SHA256", - amzDate(now), - credentialScope, - sha256Hex(canonicalRequest), - ].join("\n"); - const signingKey = hmac( - hmac(hmac(hmac(`AWS4${context.secretAccessKey}`, date), context.region), "s3"), - "aws4_request", - ); - const signature = hmacHex(signingKey, stringToSign); - return `AWS4-HMAC-SHA256 Credential=${context.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`; -} - -function objectUrl(context: RegistryArtifactBackupContext, key: string) { - const endpoint = context.endpoint.replace(/\/+$/, ""); - return new URL(`${endpoint}/${encodePathSegment(context.bucket)}/${encodeObjectKey(key)}`); -} - -function encodeObjectKey(key: string) { - return key.split("/").map(encodePathSegment).join("/"); -} - -function encodePathSegment(value: string) { - return encodeURIComponent(value).replace( - /[!'()*]/g, - (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`, - ); -} - -function amzDate(date: Date) { - return date.toISOString().replace(/[:-]|\.\d{3}/g, ""); -} - -function toBytes(value: string | Uint8Array) { - return typeof value === "string" ? new TextEncoder().encode(value) : value; -} - -function sha256Hex(value: string | Uint8Array) { - return createHash("sha256").update(value).digest("hex"); -} - -function hmac(key: string | Buffer, value: string) { - return createHmac("sha256", key).update(value).digest(); -} - -function hmacHex(key: Buffer, value: string) { - return createHmac("sha256", key).update(value).digest("hex"); -} - -function packageArtifactContentType(format: PackageBackupParams["artifactFormat"]) { - return format === "tgz" ? "application/gzip" : "application/octet-stream"; -} - -function r2EndpointFromAccountId(accountId: string | undefined) { - return accountId ? `https://${accountId}.r2.cloudflarestorage.com` : undefined; -} - -function requiredEnv(name: string) { - const value = process.env[name]; - if (!value) throw new Error(`${name} is required`); - return value; -} diff --git a/convex/lib/retentionPolicy.ts b/convex/lib/retentionPolicy.ts index fa45b67e..a4bc14a9 100644 --- a/convex/lib/retentionPolicy.ts +++ b/convex/lib/retentionPolicy.ts @@ -230,8 +230,8 @@ export const RETENTION_POLICIES = { retention: "Slug reservation cooldown.", }), reservedHandles: permanent("Reserved handles are explicit policy records until released."), - registryArtifactBackupSyncState: permanent("Registry artifact backup cursor state."), - registryArtifactBackupJobs: permanent("Registry artifact backup job history and retry state."), + registryArtifactBackupSyncState: permanent("Legacy registry artifact backup cursor state."), + registryArtifactBackupJobs: permanent("Legacy registry artifact backup job history."), userSkillInstalls: permanent("Current user install records."), skillOwnershipTransfers: ephemeral("Ownership transfer invitations expire.", { expirationField: "expiresAt", diff --git a/convex/lib/skillPublish.test.ts b/convex/lib/skillPublish.test.ts index 1b12c20f..67c90489 100644 --- a/convex/lib/skillPublish.test.ts +++ b/convex/lib/skillPublish.test.ts @@ -78,7 +78,6 @@ description: Automation workflow for recurring reports. { bypassGitHubAccountAge: true, bypassQualityGate: true, - skipBackup: true, skipWebhook: true, }, ); @@ -157,7 +156,6 @@ description: Research helper for literature reviews. { bypassGitHubAccountAge: true, bypassQualityGate: true, - skipBackup: true, skipWebhook: true, }, ); @@ -236,7 +234,6 @@ description: Research helper for literature reviews. { bypassGitHubAccountAge: true, bypassQualityGate: true, - skipBackup: true, skipWebhook: true, }, ); diff --git a/convex/lib/skillPublish.ts b/convex/lib/skillPublish.ts index db12c667..f1bb475b 100644 --- a/convex/lib/skillPublish.ts +++ b/convex/lib/skillPublish.ts @@ -99,7 +99,6 @@ export type PublishOptions = { bypassGitHubAccountAge?: boolean; bypassNewSkillRateLimit?: boolean; bypassQualityGate?: boolean; - skipBackup?: boolean; skipWebhook?: boolean; ownerPublisherId?: Id<"publishers">; sourceOwnerPublisherId?: Id<"publishers">; @@ -143,10 +142,6 @@ export async function publishVersionForUser( migrateOwner: options.migrateOwner, })) as Doc<"skills"> | null; const isNewSkill = !existingSkill; - const publishedVersionIsLatest = shouldPublishVersionBecomeLatest( - version, - existingSkill?.latestVersionSummary?.version, - ); // For new skills, enforce the full write-path rules (length, pattern, // reserved-word blocklist). For existing skills the slug is already @@ -390,35 +385,6 @@ export async function publishVersionForUser( const ownerHandle = targetPublisher?.handle ?? owner?.handle ?? owner?.displayName ?? owner?.name ?? "unknown"; - if (!options.skipBackup) { - await ctx.scheduler - .runAfter(0, internal.registryArtifactBackupsNode.backupSkillForPublishInternal, { - skillId: publishResult.skillId, - versionId: publishResult.versionId, - slug, - version, - isLatest: publishedVersionIsLatest, - displayName, - ownerHandle, - files: publishFiles, - publishedAt: Date.now(), - }) - .catch((error) => { - const message = errorMessage(error); - console.error("registry artifact backup scheduling failed", error); - return ctx - .runMutation(internal.registryArtifactBackups.enqueueRegistryArtifactBackupJobInternal, { - targetKind: "skillVersion", - skillVersionId: publishResult.versionId, - reason: "publish", - error: message, - }) - .catch((enqueueError) => { - console.error("registry artifact backup retry enqueue failed", enqueueError); - }); - }); - } - if (!options.skipWebhook) { void schedulePublishWebhook(ctx, { slug, @@ -431,18 +397,6 @@ export async function publishVersionForUser( return publishResult; } -function errorMessage(error: unknown) { - return error instanceof Error ? error.message : String(error); -} - -function shouldPublishVersionBecomeLatest(version: string, previousLatestVersion?: string) { - return ( - !previousLatestVersion || - !semver.valid(previousLatestVersion) || - semver.gt(version, previousLatestVersion) - ); -} - function mergeSourceIntoMetadata( metadata: unknown, source: PublishVersionArgs["source"], diff --git a/convex/packages.public.test.ts b/convex/packages.public.test.ts index 213e1c4b..a89c03f1 100644 --- a/convex/packages.public.test.ts +++ b/convex/packages.public.test.ts @@ -7161,16 +7161,7 @@ describe("packages public queries", () => { runMutation, runAction: vi.fn(async () => makeCleanPackageInspectorResult()), scheduler: { - runAfter: vi.fn(async (_delayMs: number, _ref: unknown, args: unknown) => { - if ( - typeof args === "object" && - args !== null && - "artifactStorageId" in args && - args.artifactStorageId === "storage:clawpack" - ) { - throw new Error("scheduler unavailable"); - } - }), + runAfter: vi.fn(async () => {}), }, storage: { get: vi.fn(async (storageId: string) => { @@ -7274,35 +7265,6 @@ describe("packages public queries", () => { ]), }), ); - expect(ctx.scheduler.runAfter).toHaveBeenCalledWith( - 0, - expect.anything(), - expect.objectContaining({ - releaseId: "releases:demo-1", - packageName: "demo-plugin", - artifactStorageId: "storage:clawpack", - artifactSha256: "clawpack", - artifactFileName: "demo-plugin-1.0.0.tgz", - }), - ); - await vi.waitFor(() => { - const retryArgs = runMutation.mock.calls - .map(([, args]) => args) - .find( - (args): args is Record => - typeof args === "object" && - args !== null && - "targetKind" in args && - args.targetKind === "packageRelease", - ); - expect(retryArgs).toEqual( - expect.objectContaining({ - packageReleaseId: "releases:demo-1", - reason: "publish", - error: "scheduler unavailable", - }), - ); - }); }); it("rejects trusted publish tokens after trusted publisher rotation or deletion", async () => { diff --git a/convex/packages.ts b/convex/packages.ts index c92c3df5..6d22eb20 100644 --- a/convex/packages.ts +++ b/convex/packages.ts @@ -391,12 +391,6 @@ const internalRefs = internal as unknown as { packageInspectorNode: { runPackageInspectorForPublishInternal: unknown; }; - registryArtifactBackupsNode: { - backupPackageForPublishInternal: unknown; - }; - registryArtifactBackups: { - enqueueRegistryArtifactBackupJobInternal: unknown; - }; packagePublishTokens: { createInternal: unknown; getByIdInternal: unknown; @@ -6989,78 +6983,9 @@ async function publishPackageImpl( source: "publish", }); - if (payload.artifact?.storageId) { - const backupIsLatest = ( - payload.tags?.map((tag: string) => tag.trim()).filter(Boolean) ?? ["latest"] - ).includes("latest"); - const backupOwner = - ownerPublisher ?? - ((await runQueryRef | null>(ctx, internalRefs.users.getByIdInternal, { - userId: ownerUserId, - })) as Doc<"users"> | null); - const ownerHandle = backupOwner?.handle ?? String(ownerPublisherId ?? ownerUserId); - await runAfterRef( - ctx, - 0, - internalRefs.registryArtifactBackupsNode.backupPackageForPublishInternal, - { - ownerHandle, - packageId: publishResult.packageId, - releaseId: publishResult.releaseId, - packageName: name, - normalizedName: name, - displayName, - family, - version, - isLatest: backupIsLatest, - publishedAt: Date.now(), - artifactKind: payload.artifact.kind ?? "legacy-zip", - artifactStorageId: payload.artifact.storageId, - artifactFileName: payload.artifact.npmTarballName, - artifactSha256: payload.artifact.sha256, - artifactSize: payload.artifact.size, - artifactFormat: payload.artifact.format, - npmIntegrity: payload.artifact.npmIntegrity, - npmShasum: payload.artifact.npmShasum, - npmUnpackedSize: payload.artifact.npmUnpackedSize, - npmFileCount: payload.artifact.npmFileCount, - runtimeId: codeArtifacts?.runtimeId ?? bundleArtifacts?.runtimeId, - sourceRepo: effectiveSource?.repo || effectiveSource?.url, - compatibility: codeArtifacts?.compatibility ?? bundleArtifacts?.compatibility, - extractedPackageJson: storedPackageJson, - extractedPluginManifest: storedPluginManifest, - normalizedBundleManifest: family === "bundle-plugin" ? storedBundleManifest : undefined, - files: files.map((file) => ({ - path: file.path, - size: file.size, - sha256: file.sha256, - })), - }, - ).catch((error) => { - const message = errorMessage(error); - console.error("registry artifact package backup scheduling failed", error); - return runMutationRef( - ctx, - internalRefs.registryArtifactBackups.enqueueRegistryArtifactBackupJobInternal, - { - targetKind: "packageRelease", - packageReleaseId: publishResult.releaseId, - reason: "publish", - error: message, - }, - ).catch((enqueueError) => { - console.error("registry artifact package backup retry enqueue failed", enqueueError); - }); - }); - } - return inspectorFindings.length > 0 ? { ...publishResult, inspectorFindings } : publishResult; } -function errorMessage(error: unknown) { - return error instanceof Error ? error.message : String(error); -} - function toPackageInspectorPublishResponseFinding( finding: PackageInspectorFinding, metadata: PackageInspectorPublishResult["metadata"], diff --git a/convex/registryArtifactBackups.test.ts b/convex/registryArtifactBackups.test.ts deleted file mode 100644 index 1771a277..00000000 --- a/convex/registryArtifactBackups.test.ts +++ /dev/null @@ -1,1963 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { Id } from "./_generated/dataModel"; -import { - claimRegistryArtifactBackupJobsHandler, - enqueueRegistryArtifactBackupJobHandler, - getDueRegistryArtifactBackupJobsInternal, - getRegistryArtifactBackupHealthHandler, - getRegistryArtifactBackupPageInternal, - getPackageRegistryArtifactBackupPageInternal, - markRegistryArtifactBackupJobSucceededHandler, - releaseRegistryArtifactBackupRetryLeaseHandler, - tryAcquireRegistryArtifactBackupRetryLeaseHandler, -} from "./registryArtifactBackups"; -import { - backupPackageForPublishInternal, - backupSkillForPublishInternal, - processRegistryArtifactBackupQueueInternalHandler, - processRegistryArtifactBackupRetriesInternalHandler, - seedRegistryArtifactBackupsInternalHandler, -} from "./registryArtifactBackupsNode"; - -const registryBackupMocks = vi.hoisted(() => { - return { - backupPackageReleaseToObjectStorage: vi.fn(), - backupSkillVersionToObjectStorage: vi.fn(), - fetchPackageReleaseBackupMeta: vi.fn(), - fetchSkillVersionBackupMeta: vi.fn(), - getRegistryArtifactBackupContext: vi.fn(), - isRegistryArtifactBackupConfigured: vi.fn(), - }; -}); - -vi.mock("./lib/registryArtifactBackup", () => registryBackupMocks); - -const handler = (getRegistryArtifactBackupPageInternal as unknown as { _handler: Function }) - ._handler; -const packagePageHandler = ( - getPackageRegistryArtifactBackupPageInternal as unknown as { _handler: Function } -)._handler; -const dueJobsHandler = ( - getDueRegistryArtifactBackupJobsInternal as unknown as { _handler: Function } -)._handler; -const backupSkillForPublishHandler = ( - backupSkillForPublishInternal as unknown as { _handler: Function } -)._handler; -const backupPackageForPublishHandler = ( - backupPackageForPublishInternal as unknown as { _handler: Function } -)._handler; - -beforeEach(() => { - vi.clearAllMocks(); - const backupContext = { - endpoint: "https://account.r2.cloudflarestorage.com", - bucket: "clawhub-registry-backup", - accessKeyId: "access-key", - secretAccessKey: "secret-key", - region: "auto", - skillsRoot: "skills", - packagesRoot: "packages", - }; - registryBackupMocks.getRegistryArtifactBackupContext.mockReturnValue(backupContext); - registryBackupMocks.isRegistryArtifactBackupConfigured.mockReturnValue(true); - registryBackupMocks.backupSkillVersionToObjectStorage.mockResolvedValue(undefined); - registryBackupMocks.backupPackageReleaseToObjectStorage.mockResolvedValue(undefined); -}); - -function retryLeaseRunMutation(claimedJobs: Array = []) { - return vi.fn(async (_ref, args) => { - if (args && typeof args === "object" && "token" in args) { - return { acquired: true, released: true }; - } - if (args && typeof args === "object" && "leaseToken" in args && "limit" in args) { - return claimedJobs; - } - if (args && typeof args === "object" && "jobId" in args) { - return { missing: false, stale: false, exhausted: false, attempts: 1 }; - } - return undefined; - }); -} - -describe("publish-time registry artifact backups", () => { - it("rehydrates skill backup args from current Convex state before writing", async () => { - const runQuery = vi - .fn() - .mockResolvedValueOnce({ - _id: "skillVersions:demo-1", - skillId: "skills:demo", - version: "1.0.0", - files: [{ path: "SKILL.md", size: 5, storageId: "storage:skill", sha256: "sha" }], - createdAt: 1_700_000_000_000, - softDeletedAt: undefined, - }) - .mockResolvedValueOnce({ - _id: "skills:demo", - slug: "current-slug", - displayName: "Current Name", - ownerUserId: "users:owner", - ownerPublisherId: undefined, - latestVersionId: "skillVersions:newer", - softDeletedAt: undefined, - moderationStatus: "active", - }) - .mockResolvedValueOnce({ - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }); - - await backupSkillForPublishHandler( - { runQuery, runMutation: retryLeaseRunMutation() } as never, - { - skillId: "skills:demo", - versionId: "skillVersions:demo-1", - slug: "stale-slug", - version: "1.0.0", - isLatest: true, - displayName: "Stale Name", - ownerHandle: "stale-owner", - files: [], - publishedAt: 1, - }, - ); - - expect(registryBackupMocks.backupSkillVersionToObjectStorage).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ - slug: "current-slug", - displayName: "Current Name", - ownerHandle: "alice", - isLatest: false, - }), - ); - }); - - it("rehydrates package backup args from current Convex state before writing", async () => { - const runQuery = vi - .fn() - .mockResolvedValueOnce({ - _id: "packageReleases:demo-1", - packageId: "packages:demo", - version: "1.0.0", - createdAt: 1_700_000_000_000, - files: [], - clawpackStorageId: "storage:artifact", - clawpackSha256: "artifact-sha", - clawpackSize: 10, - clawpackFormat: "tgz", - softDeletedAt: undefined, - }) - .mockResolvedValueOnce({ - _id: "packages:demo", - ownerUserId: "users:owner", - ownerPublisherId: undefined, - name: "@openclaw/demo", - normalizedName: "@openclaw/demo", - displayName: "Current Package", - family: "code-plugin", - latestReleaseId: "packageReleases:newer", - softDeletedAt: undefined, - }) - .mockResolvedValueOnce({ - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }); - - await backupPackageForPublishHandler( - { runQuery, runMutation: retryLeaseRunMutation() } as never, - { - ownerHandle: "stale-owner", - packageId: "packages:demo", - releaseId: "packageReleases:demo-1", - packageName: "@openclaw/stale", - normalizedName: "@openclaw/stale", - displayName: "Stale Package", - family: "code-plugin", - version: "1.0.0", - isLatest: true, - publishedAt: 1, - artifactStorageId: "storage:artifact", - files: [], - }, - ); - - expect(registryBackupMocks.backupPackageReleaseToObjectStorage).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ - ownerHandle: "alice", - packageName: "@openclaw/demo", - normalizedName: "@openclaw/demo", - displayName: "Current Package", - isLatest: false, - }), - ); - }); -}); - -describe("registry artifact backup page filtering", () => { - it("scans all active skill versions so historical versions are present in the restore catalog", async () => { - const firstVersion = { - _id: "skillVersions:demo-1", - skillId: "skills:demo", - version: "1.0.0", - createdAt: 1_700_000_000_000, - softDeletedAt: undefined, - }; - const secondVersion = { - _id: "skillVersions:demo-2", - skillId: "skills:demo", - version: "1.1.0", - createdAt: 1_700_000_100_000, - softDeletedAt: undefined, - }; - const skill = { - _id: "skills:demo", - slug: "demo-skill", - displayName: "Demo Skill", - ownerUserId: "users:owner", - ownerPublisherId: "publishers:owner", - softDeletedAt: undefined, - moderationStatus: "active", - latestVersionId: "skillVersions:demo-2", - }; - const owner = { - _id: "publishers:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const paginate = vi.fn().mockResolvedValue({ - page: [firstVersion, secondVersion], - isDone: true, - continueCursor: null, - }); - const order = vi.fn().mockReturnValue({ paginate }); - const withIndex = vi.fn().mockReturnValue({ order }); - const query = vi.fn().mockReturnValue({ withIndex }); - const get = vi.fn(async (id: string) => { - if (id === "skills:demo") return skill; - if (id === "publishers:owner") return owner; - return null; - }); - - const result = await handler({ db: { query, get } } as never, { batchSize: 50 }); - - expect(query).toHaveBeenCalledWith("skillVersions"); - expect(result.items).toEqual([ - { - kind: "ok", - skillId: "skills:demo", - versionId: "skillVersions:demo-1", - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.0.0", - isLatest: false, - ownerHandle: "alice", - publishedAt: 1_700_000_000_000, - }, - { - kind: "ok", - skillId: "skills:demo", - versionId: "skillVersions:demo-2", - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.1.0", - isLatest: true, - ownerHandle: "alice", - publishedAt: 1_700_000_100_000, - }, - ]); - }); - - it("skips non-public skills and keeps legacy skills with undefined moderationStatus eligible", async () => { - const versions = [ - { - _id: "skillVersions:active", - skillId: "skills:active", - version: "1.0.0", - createdAt: 1_700_000_000_000, - softDeletedAt: undefined, - }, - { - _id: "skillVersions:legacy", - skillId: "skills:legacy", - version: "2.0.0", - createdAt: 1_700_000_000_100, - softDeletedAt: undefined, - }, - { - _id: "skillVersions:hidden", - skillId: "skills:hidden", - version: "1.0.0", - createdAt: 1_700_000_000_200, - softDeletedAt: undefined, - }, - { - _id: "skillVersions:removed", - skillId: "skills:removed", - version: "1.0.0", - createdAt: 1_700_000_000_300, - softDeletedAt: undefined, - }, - { - _id: "skillVersions:soft", - skillId: "skills:soft", - version: "1.0.0", - createdAt: 1_700_000_000_400, - softDeletedAt: undefined, - }, - ]; - const owner = { - _id: "publishers:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const skills = new Map([ - [ - "skills:active", - { - _id: "skills:active", - slug: "active-skill", - displayName: "Active Skill", - ownerUserId: "users:active", - ownerPublisherId: "publishers:owner", - softDeletedAt: undefined, - moderationStatus: "active", - }, - ], - [ - "skills:legacy", - { - _id: "skills:legacy", - slug: "legacy-skill", - displayName: "Legacy Skill", - ownerUserId: "users:legacy", - ownerPublisherId: "publishers:owner", - softDeletedAt: undefined, - moderationStatus: undefined, - }, - ], - [ - "skills:hidden", - { - _id: "skills:hidden", - slug: "hidden-skill", - displayName: "Hidden Skill", - ownerUserId: "users:hidden", - ownerPublisherId: "publishers:owner", - softDeletedAt: undefined, - moderationStatus: "hidden", - }, - ], - [ - "skills:removed", - { - _id: "skills:removed", - slug: "removed-skill", - displayName: "Removed Skill", - ownerUserId: "users:removed", - ownerPublisherId: "publishers:owner", - softDeletedAt: undefined, - moderationStatus: "removed", - }, - ], - [ - "skills:soft", - { - _id: "skills:soft", - slug: "soft-skill", - displayName: "Soft Skill", - ownerUserId: "users:soft", - ownerPublisherId: "publishers:owner", - softDeletedAt: 1, - moderationStatus: "active", - }, - ], - ]); - const paginate = vi.fn().mockResolvedValue({ - page: versions, - isDone: true, - continueCursor: null, - }); - const order = vi.fn().mockReturnValue({ paginate }); - const withIndex = vi.fn().mockReturnValue({ order }); - const query = vi.fn().mockReturnValue({ withIndex }); - const get = vi.fn(async (id: string) => { - if (id === "publishers:owner") return owner; - return skills.get(id) ?? null; - }); - - const result = await handler({ db: { query, get } } as never, { batchSize: 50 }); - - expect(query).toHaveBeenCalledWith("skillVersions"); - expect(result.items).toMatchObject([ - { - kind: "ok", - slug: "active-skill", - ownerHandle: "alice", - version: "1.0.0", - }, - { - kind: "ok", - slug: "legacy-skill", - ownerHandle: "alice", - version: "2.0.0", - }, - ]); - }); - - it("marks public skill versions with missing owners as skipped seed items", async () => { - const version = { - _id: "skillVersions:no-owner", - skillId: "skills:no-owner", - version: "1.0.0", - createdAt: 1, - softDeletedAt: undefined, - }; - const skill = { - _id: "skills:no-owner", - slug: "no-owner", - displayName: "No Owner", - ownerUserId: "users:no-owner", - ownerPublisherId: undefined, - softDeletedAt: undefined, - moderationStatus: "active", - }; - const paginate = vi.fn().mockResolvedValue({ - page: [version], - isDone: true, - continueCursor: null, - }); - const order = vi.fn().mockReturnValue({ paginate }); - const withIndex = vi.fn().mockReturnValue({ order }); - const query = vi.fn().mockReturnValue({ withIndex }); - const get = vi.fn(async (id: string) => (id === "skills:no-owner" ? skill : null)); - - const result = await handler({ db: { query, get } } as never, {}); - - expect(result.items).toEqual([ - { kind: "missingOwner", skillId: "skills:no-owner", ownerUserId: "users:no-owner" }, - ]); - }); - - it("resets stale cursors after switching the skill backup page query", async () => { - const paginate = vi - .fn() - .mockRejectedValueOnce(new Error("cursor is from a different query")) - .mockResolvedValueOnce({ page: [], isDone: true, continueCursor: null }); - const order = vi.fn().mockReturnValue({ paginate }); - const withIndex = vi.fn().mockReturnValue({ order }); - const query = vi.fn().mockReturnValue({ withIndex }); - - const result = await handler({ db: { query } } as never, { cursor: "stale-cursor" }); - - expect(result).toMatchObject({ items: [], isDone: true, cursor: null }); - expect(paginate).toHaveBeenNthCalledWith(1, { cursor: "stale-cursor", numItems: 50 }); - expect(paginate).toHaveBeenNthCalledWith(2, { cursor: null, numItems: 50 }); - }); -}); - -describe("package registry artifact backup page filtering", () => { - it("returns backup-ready package releases and marks missing artifact rows", async () => { - const backupableRelease = { - _id: "packageReleases:ready", - packageId: "packages:ready", - version: "1.0.0", - createdAt: 1_700_000_000_000, - files: [{ path: "package.json", size: 10, sha256: "sha256:package" }], - artifactKind: "npm-pack", - clawpackStorageId: "storage:clawpack", - clawpackSha256: "sha256:clawpack", - clawpackSize: 123, - clawpackFormat: "tgz", - npmTarballName: "ready-1.0.0.tgz", - compatibility: { openclaw: ">=2026.1.0" }, - capabilities: { executesCode: true }, - extractedPackageJson: { name: "ready" }, - extractedPluginManifest: { id: "ready" }, - softDeletedAt: undefined, - }; - const missingArtifactRelease = { - _id: "packageReleases:missing-artifact", - packageId: "packages:missing-artifact", - version: "1.0.0", - createdAt: 1_700_000_000_100, - files: [], - softDeletedAt: undefined, - }; - const readyPackage = { - _id: "packages:ready", - ownerUserId: "users:owner", - ownerPublisherId: "publishers:openclaw", - name: "@openclaw/ready", - normalizedName: "@openclaw/ready", - displayName: "Ready", - family: "code-plugin", - softDeletedAt: undefined, - latestReleaseId: "packageReleases:ready", - }; - const missingArtifactPackage = { - ...readyPackage, - _id: "packages:missing-artifact", - name: "@openclaw/missing-artifact", - normalizedName: "@openclaw/missing-artifact", - }; - const owner = { - _id: "publishers:openclaw", - handle: "openclaw", - deletedAt: undefined, - deactivatedAt: undefined, - }; - - const paginate = vi.fn().mockResolvedValue({ - page: [backupableRelease, missingArtifactRelease], - isDone: true, - continueCursor: null, - }); - const order = vi.fn().mockReturnValue({ paginate }); - const withIndex = vi.fn().mockReturnValue({ order }); - const query = vi.fn().mockReturnValue({ withIndex }); - const get = vi.fn(async (id: string) => { - if (id === "packages:ready") return readyPackage; - if (id === "packages:missing-artifact") return missingArtifactPackage; - if (id === "publishers:openclaw") return owner; - return null; - }); - - const result = await packagePageHandler({ db: { query, get } } as never, { batchSize: 50 }); - - expect(query).toHaveBeenCalledWith("packageReleases"); - expect(result).toMatchObject({ - isDone: true, - cursor: null, - items: [ - { - kind: "ok", - releaseId: "packageReleases:ready", - packageName: "@openclaw/ready", - ownerHandle: "openclaw", - isLatest: true, - artifactStorageId: "storage:clawpack", - artifactFileName: "ready-1.0.0.tgz", - }, - { - kind: "missingArtifact", - releaseId: "packageReleases:missing-artifact", - packageId: "packages:missing-artifact", - }, - ], - }); - }); -}); - -describe("seedRegistryArtifactBackupsInternalHandler", () => { - it("does not rescan backup dimensions whose sync state is already complete", async () => { - const runQuery = vi - .fn() - .mockResolvedValueOnce({ cursor: null, isDone: true }) - .mockResolvedValueOnce({ cursor: null, isDone: true }) - .mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - const runMutation = vi.fn(); - - const result = await seedRegistryArtifactBackupsInternalHandler( - { runQuery, runMutation } as never, - { queueOnly: true, batchSize: 2, maxBatches: 1 }, - ); - - expect(result).toMatchObject({ - cursor: null, - packageCursor: null, - skillsIsDone: true, - packageIsDone: true, - isDone: true, - }); - expect(result.stats.skillsEnqueued).toBe(0); - expect(result.stats.packagesEnqueued).toBe(0); - expect(runQuery).toHaveBeenCalledTimes(3); - expect(runMutation).not.toHaveBeenCalled(); - }); - - it("can seed the backup ledger without uploading artifacts inline", async () => { - const runQuery = vi - .fn() - .mockResolvedValueOnce({ cursor: null }) - .mockResolvedValueOnce({ - items: [ - { - kind: "ok", - skillId: "skills:demo", - versionId: "skillVersions:demo-1", - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.0.0", - isLatest: true, - ownerHandle: "alice", - publishedAt: 1, - }, - ], - cursor: null, - isDone: true, - }) - .mockResolvedValueOnce({ cursor: null }) - .mockResolvedValueOnce({ - items: [ - { - kind: "ok", - packageId: "packages:demo", - releaseId: "packageReleases:demo-1", - ownerHandle: "alice", - packageName: "@openclaw/demo", - normalizedName: "@openclaw/demo", - displayName: "Demo", - family: "code-plugin", - version: "1.0.0", - isLatest: true, - publishedAt: 1, - artifactStorageId: "storage:artifact", - files: [], - }, - { - kind: "missingArtifact", - releaseId: "packageReleases:missing", - packageId: "packages:missing", - }, - ], - cursor: null, - isDone: true, - }) - .mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - const runMutation = retryLeaseRunMutation(); - - const result = await seedRegistryArtifactBackupsInternalHandler( - { runQuery, runMutation } as never, - { queueOnly: true, batchSize: 2, maxBatches: 1 }, - ); - - expect(result.stats.skillsEnqueued).toBe(1); - expect(result.stats.packagesEnqueued).toBe(1); - expect(result.stats.packagesMissingArtifact).toBe(1); - expect(registryBackupMocks.fetchSkillVersionBackupMeta).not.toHaveBeenCalled(); - expect(registryBackupMocks.backupSkillVersionToObjectStorage).not.toHaveBeenCalled(); - expect(registryBackupMocks.backupPackageReleaseToObjectStorage).not.toHaveBeenCalled(); - expect(runMutation).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ - targetKind: "skillVersion", - skillVersionId: "skillVersions:demo-1", - reason: "seed", - preserveTerminal: true, - }), - ); - expect(runMutation).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ - targetKind: "packageRelease", - packageReleaseId: "packageReleases:demo-1", - reason: "seed", - preserveTerminal: true, - }), - ); - }); - - it("reports package cursor progress when skills are done but package releases remain", async () => { - const runQuery = vi - .fn() - .mockResolvedValueOnce({ items: [], cursor: null, isDone: true }) - .mockResolvedValueOnce({ items: [], cursor: "package-cursor", isDone: false }) - .mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - - const result = await seedRegistryArtifactBackupsInternalHandler( - { - runQuery, - runMutation: vi.fn(), - } as never, - { dryRun: true, batchSize: 1, maxBatches: 1 }, - ); - - expect(result).toMatchObject({ - cursor: null, - packageCursor: "package-cursor", - skillsIsDone: true, - packageIsDone: false, - isDone: false, - }); - }); - - it("queues failed skill seed attempts into the retry backlog", async () => { - registryBackupMocks.fetchSkillVersionBackupMeta.mockRejectedValueOnce(new Error("R2 500")); - const runQuery = vi - .fn() - .mockResolvedValueOnce({ cursor: null }) - .mockResolvedValueOnce({ - items: [ - { - kind: "ok", - skillId: "skills:demo", - versionId: "skillVersions:demo-1", - slug: "demo-skill", - displayName: "Demo Skill", - version: "1.0.0", - ownerHandle: "alice", - publishedAt: 1, - }, - ], - cursor: null, - isDone: true, - }) - .mockResolvedValueOnce({ cursor: null }) - .mockResolvedValueOnce({ items: [], cursor: null, isDone: true }) - .mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - const runMutation = retryLeaseRunMutation(); - - const result = await seedRegistryArtifactBackupsInternalHandler( - { runQuery, runMutation } as never, - { batchSize: 1, maxBatches: 1 }, - ); - - expect(result.stats.errors).toBe(1); - expect(runMutation).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ - targetKind: "skillVersion", - skillVersionId: "skillVersions:demo-1", - reason: "seed", - error: "R2 500", - }), - ); - }); -}); - -describe("processRegistryArtifactBackupRetriesInternalHandler", () => { - it("skips retry drain when another retry action holds the lease", async () => { - const runMutation = vi.fn().mockResolvedValueOnce({ acquired: false }); - const runQuery = vi.fn(); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation } as never, - {}, - ); - - expect(result.stats.retryJobsProcessed).toBe(0); - expect(runQuery).not.toHaveBeenCalled(); - expect(registryBackupMocks.backupPackageReleaseToObjectStorage).not.toHaveBeenCalled(); - }); - - it("releases the retry lease after draining jobs", async () => { - const runMutation = retryLeaseRunMutation(); - const runQuery = vi - .fn() - .mockResolvedValueOnce([]) - .mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - - await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation } as never, - {}, - ); - - const token = runMutation.mock.calls[0]?.[1]?.token; - expect(typeof token).toBe("string"); - expect(runMutation.mock.calls.at(-1)?.[1]).toMatchObject({ token }); - }); - - it("can force pending retry jobs regardless of nextRunAt for operator drains", async () => { - const runMutation = retryLeaseRunMutation(); - const runQuery = vi - .fn() - .mockResolvedValueOnce([]) - .mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - - await processRegistryArtifactBackupRetriesInternalHandler({ runQuery, runMutation } as never, { - forceDue: true, - }); - - expect(runMutation.mock.calls[1]?.[1]).toMatchObject({ forceDue: true }); - }); - - it("backs up retry artifacts without acquiring per-index leases", async () => { - const jobs = [makeSkillBackupJob("demo", "skillVersions:demo")]; - const skill = { - ...makeSkill("skills:demo", "demo-skill"), - latestVersionId: "skillVersions:demo", - }; - const owner = { - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const runQuery = vi.fn(async (_ref, args) => { - if ("limit" in args) return jobs; - if (args.versionId) return makeSkillVersion("skillVersions:demo", "skills:demo", "1.0.0"); - if (args.skillId === "skills:demo") return skill; - if (args.userId === "users:owner") return owner; - if ("staleAfterMs" in args) return { stale: 0, exhausted: 0 }; - throw new Error(`unexpected query ${JSON.stringify(args)}`); - }); - const runMutation = retryLeaseRunMutation(jobs); - registryBackupMocks.fetchSkillVersionBackupMeta.mockResolvedValue(null); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation } as never, - {}, - ); - - expect(result.stats.retryJobsSucceeded).toBe(1); - expect(registryBackupMocks.backupSkillVersionToObjectStorage).toHaveBeenCalledOnce(); - expect(runMutation.mock.calls.some((call) => "indexPath" in (call[1] ?? {}))).toBe(false); - }); - - it("drains retry jobs without scanning the historical registry", async () => { - const dueJob = { - _id: "registryArtifactBackupJobs:demo", - targetKind: "packageRelease", - packageReleaseId: "packageReleases:demo", - status: "pending", - attempts: 0, - nextRunAt: 1, - createdAt: 1, - updatedAt: 1, - }; - const runQuery = vi - .fn() - .mockResolvedValueOnce({ - _id: "packageReleases:demo", - packageId: "packages:demo", - version: "1.0.0", - createdAt: 1, - files: [], - clawpackStorageId: "storage:artifact", - softDeletedAt: undefined, - }) - .mockResolvedValueOnce({ - _id: "packages:demo", - ownerUserId: "users:owner", - ownerPublisherId: undefined, - name: "@openclaw/demo", - normalizedName: "@openclaw/demo", - displayName: "Demo", - family: "code-plugin", - softDeletedAt: undefined, - }) - .mockResolvedValueOnce({ - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }) - .mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - const runMutation = retryLeaseRunMutation([dueJob]); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation } as never, - {}, - ); - - expect(result.stats.retryJobsProcessed).toBe(1); - expect(registryBackupMocks.backupPackageReleaseToObjectStorage).toHaveBeenCalledOnce(); - expect(runQuery).not.toHaveBeenCalledWith( - expect.objectContaining({ - _name: "registryArtifactBackups:getRegistryArtifactBackupPageInternal", - }), - expect.anything(), - ); - expect(runQuery).not.toHaveBeenCalledWith( - expect.objectContaining({ - _name: "registryArtifactBackups:getPackageRegistryArtifactBackupPageInternal", - }), - expect.anything(), - ); - }); - - it("requests a larger retry batch so publish bursts drain promptly", async () => { - const runMutation = retryLeaseRunMutation(); - const runQuery = vi - .fn() - .mockResolvedValueOnce([]) - .mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - - await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation } as never, - {}, - ); - - expect(runMutation.mock.calls[1]?.[1]).toMatchObject({ - includeExhaustedRepair: true, - limit: 500, - maxRepairAttempts: 16, - }); - }); - - it("uses a unique queue lease token even with a stable worker id", async () => { - const runMutation = vi.fn().mockResolvedValueOnce([]); - const runQuery = vi.fn().mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - - await processRegistryArtifactBackupQueueInternalHandler({ runQuery, runMutation } as never, { - workerId: "worker-a", - limit: 1, - }); - - const leaseToken = runMutation.mock.calls[0]?.[1]?.leaseToken; - expect(leaseToken).not.toBe("worker-a"); - expect(leaseToken).toEqual(expect.stringMatching(/^worker-a-\d+-[a-z0-9]+$/)); - }); - - it("gives repaired exhausted jobs a finite second retry budget", async () => { - const dueJob = { - _id: "registryArtifactBackupJobs:demo", - targetKind: "skillVersion", - skillVersionId: "skillVersions:demo", - status: "exhausted", - attempts: 8, - nextRunAt: 1, - createdAt: 1, - updatedAt: 1, - }; - const version = { - _id: "skillVersions:demo", - skillId: "skills:demo", - version: "1.0.0", - createdAt: 1, - files: [{ path: "SKILL.md", size: 5, storageId: "storage:skill", sha256: "sha" }], - softDeletedAt: undefined, - }; - const skill = { - _id: "skills:demo", - ownerUserId: "users:owner", - ownerPublisherId: undefined, - slug: "demo-skill", - displayName: "Demo Skill", - latestVersionId: "skillVersions:demo", - softDeletedAt: undefined, - moderationStatus: "active", - }; - const owner = { - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const runQuery = vi.fn(async (_ref, args) => { - if ("limit" in args) return [dueJob]; - if (args.versionId === "skillVersions:demo") return version; - if (args.skillId === "skills:demo") return skill; - if (args.userId === "users:owner") return owner; - if ("staleAfterMs" in args) return { stale: 0, exhausted: 0 }; - throw new Error(`unexpected query ${JSON.stringify(args)}`); - }); - registryBackupMocks.fetchSkillVersionBackupMeta.mockResolvedValue(null); - registryBackupMocks.backupSkillVersionToObjectStorage.mockRejectedValueOnce( - new Error("R2 still down"), - ); - const runMutation = retryLeaseRunMutation([dueJob]); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation } as never, - {}, - ); - - expect(result.stats.retryJobsFailed).toBe(1); - expect(runMutation).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ - jobId: "registryArtifactBackupJobs:demo", - error: "R2 still down", - maxAttempts: 16, - }), - ); - }); - - it("marks skill retries succeeded without reuploading when version metadata already exists", async () => { - const dueJob = { - _id: "registryArtifactBackupJobs:demo", - targetKind: "skillVersion", - skillVersionId: "skillVersions:demo", - status: "pending", - attempts: 1, - nextRunAt: 1, - createdAt: 1, - updatedAt: 1, - }; - const version = { - _id: "skillVersions:demo", - skillId: "skills:demo", - version: "1.0.0", - createdAt: 1, - files: [{ path: "SKILL.md", size: 5, storageId: "storage:skill", sha256: "sha" }], - softDeletedAt: undefined, - }; - const skill = { - _id: "skills:demo", - ownerUserId: "users:owner", - ownerPublisherId: undefined, - slug: "demo-skill", - displayName: "Demo Skill", - latestVersionId: "skillVersions:demo", - softDeletedAt: undefined, - moderationStatus: "active", - }; - const owner = { - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const runQuery = vi.fn(async (_ref, args) => { - if ("limit" in args) return [dueJob]; - if (args.versionId === "skillVersions:demo") { - return version; - } - if (args.skillId === "skills:demo") return skill; - if (args.userId === "users:owner") return owner; - if ("staleAfterMs" in args) return { stale: 0, exhausted: 0 }; - throw new Error(`unexpected query ${JSON.stringify(args)}`); - }); - registryBackupMocks.fetchSkillVersionBackupMeta.mockResolvedValueOnce({ - version: "1.0.0", - restore: { versionId: "skillVersions:demo" }, - }); - const runMutation = retryLeaseRunMutation([dueJob]); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation } as never, - {}, - ); - - expect(result.stats.retryJobsSucceeded).toBe(1); - expect(registryBackupMocks.backupSkillVersionToObjectStorage).not.toHaveBeenCalled(); - expect(runMutation).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ jobId: "registryArtifactBackupJobs:demo" }), - ); - }); - - it("marks multiple skill retries succeeded from matching version metadata", async () => { - const jobs = [ - makeSkillBackupJob("demo-1", "skillVersions:demo-1"), - makeSkillBackupJob("demo-2", "skillVersions:demo-2"), - ]; - const versions = new Map([ - ["skillVersions:demo-1", makeSkillVersion("skillVersions:demo-1", "skills:demo", "1.0.0")], - ["skillVersions:demo-2", makeSkillVersion("skillVersions:demo-2", "skills:demo", "1.1.0")], - ]); - const skill = { - ...makeSkill("skills:demo", "demo-skill"), - latestVersionId: "skillVersions:demo-2", - }; - const owner = { - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const runQuery = vi.fn(async (_ref, args) => { - if ("limit" in args) return jobs; - if (args.versionId) return versions.get(args.versionId) ?? null; - if (args.skillId === "skills:demo") return skill; - if (args.userId === "users:owner") return owner; - if ("staleAfterMs" in args) return { stale: 0, exhausted: 0 }; - throw new Error(`unexpected query ${JSON.stringify(args)}`); - }); - const versionIdsByVersion = new Map([ - ["1.0.0", "skillVersions:demo-1"], - ["1.1.0", "skillVersions:demo-2"], - ]); - registryBackupMocks.fetchSkillVersionBackupMeta.mockImplementation( - async (_context, _ownerHandle, _slug, version) => ({ - version, - restore: { versionId: versionIdsByVersion.get(version) }, - }), - ); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation: retryLeaseRunMutation(jobs) } as never, - {}, - ); - - expect(result.stats.retryJobsSucceeded).toBe(2); - expect(result.stats.retryJobsFailed).toBe(0); - expect(registryBackupMocks.backupSkillVersionToObjectStorage).not.toHaveBeenCalled(); - expect(registryBackupMocks.fetchSkillVersionBackupMeta).toHaveBeenCalledTimes(2); - }); - - it("marks package retries succeeded from matching version metadata", async () => { - const jobs = [ - makePackageBackupJob("demo-1", "packageReleases:demo-1"), - makePackageBackupJob("demo-2", "packageReleases:demo-2"), - ]; - const releases = new Map([ - [ - "packageReleases:demo-1", - makePackageRelease("packageReleases:demo-1", "packages:demo", "1.0.0"), - ], - [ - "packageReleases:demo-2", - makePackageRelease("packageReleases:demo-2", "packages:demo", "1.1.0"), - ], - ]); - const pkg = makePackage("packages:demo", "@openclaw/demo"); - const owner = { - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const runQuery = vi.fn(async (_ref, args) => { - if ("limit" in args) return jobs; - if (args.releaseId) return releases.get(args.releaseId) ?? null; - if (args.packageId === "packages:demo") return pkg; - if (args.userId === "users:owner") return owner; - if ("staleAfterMs" in args) return { stale: 0, exhausted: 0 }; - throw new Error(`unexpected query ${JSON.stringify(args)}`); - }); - const releaseIdsByVersion = new Map([ - ["1.0.0", "packageReleases:demo-1"], - ["1.1.0", "packageReleases:demo-2"], - ]); - const shaByVersion = new Map([ - ["1.0.0", "sha:packageReleases:demo-1"], - ["1.1.0", "sha:packageReleases:demo-2"], - ]); - registryBackupMocks.fetchPackageReleaseBackupMeta.mockImplementation( - async (_context, _ownerHandle, _normalizedName, version) => ({ - restore: { releaseId: releaseIdsByVersion.get(version) }, - artifact: { sha256: shaByVersion.get(version) }, - }), - ); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation: retryLeaseRunMutation(jobs) } as never, - {}, - ); - - expect(result.stats.retryJobsSucceeded).toBe(2); - expect(result.stats.retryJobsFailed).toBe(0); - expect(registryBackupMocks.backupPackageReleaseToObjectStorage).not.toHaveBeenCalled(); - expect(registryBackupMocks.fetchPackageReleaseBackupMeta).toHaveBeenCalledTimes(2); - }); - - it("processes different retry roots in parallel while keeping one root sequential", async () => { - const jobs = [ - makeSkillBackupJob("same-1", "skillVersions:same-1"), - makeSkillBackupJob("same-2", "skillVersions:same-2"), - makeSkillBackupJob("other", "skillVersions:other"), - ]; - const versions = new Map([ - ["skillVersions:same-1", makeSkillVersion("skillVersions:same-1", "skills:same", "1.0.0")], - ["skillVersions:same-2", makeSkillVersion("skillVersions:same-2", "skills:same", "1.1.0")], - ["skillVersions:other", makeSkillVersion("skillVersions:other", "skills:other", "1.0.0")], - ]); - const skills = new Map([ - ["skills:same", makeSkill("skills:same", "same-root")], - ["skills:other", makeSkill("skills:other", "other-root")], - ]); - const owner = { - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const runQuery = vi.fn(async (_ref, args) => { - if ("limit" in args) return jobs; - if (args.versionId) return versions.get(args.versionId) ?? null; - if (args.skillId) return skills.get(args.skillId) ?? null; - if (args.userId) return owner; - if ("staleAfterMs" in args) return { stale: 0, exhausted: 0 }; - throw new Error(`unexpected query ${JSON.stringify(args)}`); - }); - registryBackupMocks.fetchSkillVersionBackupMeta.mockResolvedValue(null); - - const activeByRoot = new Map(); - let maxActiveTotal = 0; - let activeTotal = 0; - registryBackupMocks.backupSkillVersionToObjectStorage.mockImplementation(async (_ctx, item) => { - const root = `${item.ownerHandle}/${item.slug}`; - const active = activeByRoot.get(root) ?? 0; - if (active > 0) { - throw new Error(`same root overlapped: ${root}`); - } - activeByRoot.set(root, active + 1); - activeTotal += 1; - maxActiveTotal = Math.max(maxActiveTotal, activeTotal); - await new Promise((resolve) => setTimeout(resolve, 10)); - activeTotal -= 1; - activeByRoot.set(root, active); - }); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation: retryLeaseRunMutation(jobs) } as never, - {}, - ); - - expect(result.stats.retryJobsSucceeded).toBe(3); - expect(result.stats.retryJobsFailed).toBe(0); - expect(maxActiveTotal).toBeGreaterThan(1); - }); - - it("caps parallel full-artifact retry work by estimated bytes", async () => { - const jobs = [ - makePackageBackupJob("one", "packageReleases:one"), - makePackageBackupJob("two", "packageReleases:two"), - makePackageBackupJob("three", "packageReleases:three"), - ]; - const releases = new Map([ - ["packageReleases:one", makePackageRelease("packageReleases:one", "packages:one")], - ["packageReleases:two", makePackageRelease("packageReleases:two", "packages:two")], - ["packageReleases:three", makePackageRelease("packageReleases:three", "packages:three")], - ]); - const packages = new Map([ - ["packages:one", makePackage("packages:one", "@openclaw/one")], - ["packages:two", makePackage("packages:two", "@openclaw/two")], - ["packages:three", makePackage("packages:three", "@openclaw/three")], - ]); - const owner = { - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const runQuery = vi.fn(async (_ref, args) => { - if ("limit" in args) return jobs; - if (args.releaseId) return releases.get(args.releaseId) ?? null; - if (args.packageId) return packages.get(args.packageId) ?? null; - if (args.userId) return owner; - if ("staleAfterMs" in args) return { stale: 0, exhausted: 0 }; - throw new Error(`unexpected query ${JSON.stringify(args)}`); - }); - registryBackupMocks.fetchPackageReleaseBackupMeta.mockResolvedValue(null); - - let activeTotal = 0; - let maxActiveTotal = 0; - registryBackupMocks.backupPackageReleaseToObjectStorage.mockImplementation(async () => { - activeTotal += 1; - maxActiveTotal = Math.max(maxActiveTotal, activeTotal); - await new Promise((resolve) => setTimeout(resolve, 10)); - activeTotal -= 1; - }); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation: retryLeaseRunMutation(jobs) } as never, - {}, - ); - - expect(result.stats.retryJobsSucceeded).toBe(3); - expect(maxActiveTotal).toBe(1); - }); - - it("keeps retry source lookup failures isolated to their own jobs", async () => { - const jobs = [ - makeSkillBackupJob("bad", "skillVersions:bad"), - makeSkillBackupJob("good", "skillVersions:good"), - ]; - const goodVersion = makeSkillVersion("skillVersions:good", "skills:good", "1.0.0"); - const goodSkill = makeSkill("skills:good", "good-root"); - const owner = { - _id: "users:owner", - handle: "alice", - deletedAt: undefined, - deactivatedAt: undefined, - }; - const runQuery = vi.fn(async (_ref, args) => { - if ("limit" in args) return jobs; - if (args.versionId === "skillVersions:bad") throw new Error("lookup failed"); - if (args.versionId === "skillVersions:good") return goodVersion; - if (args.skillId === "skills:good") return goodSkill; - if (args.userId === "users:owner") return owner; - if ("staleAfterMs" in args) return { stale: 0, exhausted: 0 }; - throw new Error(`unexpected query ${JSON.stringify(args)}`); - }); - const runMutation = retryLeaseRunMutation(jobs); - registryBackupMocks.fetchSkillVersionBackupMeta.mockResolvedValue(null); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation } as never, - {}, - ); - - expect(result.stats.retryJobsProcessed).toBe(2); - expect(result.stats.retryJobsSucceeded).toBe(1); - expect(result.stats.retryJobsFailed).toBe(1); - expect(registryBackupMocks.backupSkillVersionToObjectStorage).toHaveBeenCalledOnce(); - expect(runMutation).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ - jobId: "registryArtifactBackupJobs:bad", - error: "lookup failed", - }), - ); - }); - - it("skips queued skill version retries after the skill is no longer public", async () => { - const dueJob = { - _id: "registryArtifactBackupJobs:hidden", - targetKind: "skillVersion", - skillVersionId: "skillVersions:hidden", - status: "pending", - attempts: 0, - nextRunAt: 1, - createdAt: 1, - updatedAt: 1, - }; - const runQuery = vi - .fn() - .mockResolvedValueOnce({ - _id: "skillVersions:hidden", - skillId: "skills:hidden", - version: "1.0.0", - createdAt: 1, - files: [], - softDeletedAt: undefined, - }) - .mockResolvedValueOnce({ - _id: "skills:hidden", - ownerUserId: "users:owner", - ownerPublisherId: undefined, - slug: "hidden-skill", - displayName: "Hidden Skill", - latestVersionId: "skillVersions:hidden", - softDeletedAt: undefined, - moderationStatus: "hidden", - }) - .mockResolvedValueOnce({ stale: 0, exhausted: 0 }); - const runMutation = retryLeaseRunMutation([dueJob]); - - const result = await processRegistryArtifactBackupRetriesInternalHandler( - { runQuery, runMutation } as never, - {}, - ); - - expect(result.stats.retryJobsProcessed).toBe(1); - expect(result.stats.retryJobsSucceeded).toBe(1); - expect(registryBackupMocks.backupSkillVersionToObjectStorage).not.toHaveBeenCalled(); - expect(runMutation).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ jobId: "registryArtifactBackupJobs:hidden" }), - ); - }); -}); - -function makeSkillBackupJob(suffix: string, skillVersionId: string) { - return { - _id: `registryArtifactBackupJobs:${suffix}`, - targetKind: "skillVersion", - skillVersionId, - status: "pending", - attempts: 0, - nextRunAt: 1, - createdAt: 1, - updatedAt: 1, - }; -} - -function makeSkillVersion(id: string, skillId: string, version: string) { - return { - _id: id, - skillId, - version, - createdAt: 1, - files: [{ path: "SKILL.md", size: 5, storageId: `storage:${id}`, sha256: `sha:${id}` }], - softDeletedAt: undefined, - }; -} - -function makeSkill(id: string, slug: string) { - return { - _id: id, - ownerUserId: "users:owner", - ownerPublisherId: undefined, - slug, - displayName: slug, - latestVersionId: "skillVersions:latest", - softDeletedAt: undefined, - moderationStatus: "active", - }; -} - -function makePackageBackupJob(suffix: string, packageReleaseId: string) { - return { - _id: `registryArtifactBackupJobs:${suffix}`, - targetKind: "packageRelease", - packageReleaseId, - status: "pending", - attempts: 0, - nextRunAt: 1, - createdAt: 1, - updatedAt: 1, - }; -} - -function makePackageRelease(id: string, packageId: string, version = "1.0.0") { - return { - _id: id, - packageId, - version, - createdAt: 1, - files: [], - clawpackStorageId: `storage:${id}`, - clawpackSha256: `sha:${id}`, - clawpackSize: 120 * 1024 * 1024, - clawpackFormat: "tgz", - softDeletedAt: undefined, - }; -} - -function makePackage(id: string, name: string) { - return { - _id: id, - ownerUserId: "users:owner", - ownerPublisherId: undefined, - name, - normalizedName: name, - displayName: name, - family: "code-plugin", - latestReleaseId: "packageReleases:latest", - softDeletedAt: undefined, - }; -} - -describe("registry artifact backup jobs", () => { - it("ignores stale worker completion after a job lease is reclaimed", async () => { - const now = 1_700_000_000_000; - const job = { - _id: "registryArtifactBackupJobs:demo", - status: "running", - leaseToken: "worker-b", - }; - const ctx = { - db: { - get: vi.fn().mockResolvedValue(job), - patch: vi.fn(), - }, - }; - - const result = await markRegistryArtifactBackupJobSucceededHandler(ctx as never, { - jobId: "registryArtifactBackupJobs:demo" as Id<"registryArtifactBackupJobs">, - leaseToken: "worker-a", - now, - }); - - expect(result).toEqual({ missing: false, stale: true }); - expect(ctx.db.patch).not.toHaveBeenCalled(); - }); - - it("ignores worker completion after a leased job is requeued", async () => { - const now = 1_700_000_000_000; - const job = { - _id: "registryArtifactBackupJobs:demo", - status: "pending", - leaseToken: "worker-a", - }; - const ctx = { - db: { - get: vi.fn().mockResolvedValue(job), - patch: vi.fn(), - }, - }; - - const result = await markRegistryArtifactBackupJobSucceededHandler(ctx as never, { - jobId: "registryArtifactBackupJobs:demo" as Id<"registryArtifactBackupJobs">, - leaseToken: "worker-a", - now, - }); - - expect(result).toEqual({ missing: false, stale: true }); - expect(ctx.db.patch).not.toHaveBeenCalled(); - }); - - it("claims pending and expired running jobs with a lease for parallel workers", async () => { - const now = 1_700_000_000_000; - const pendingJob = { - _id: "registryArtifactBackupJobs:pending", - status: "pending", - nextRunAt: now - 1, - }; - const expiredRunningJob = { - _id: "registryArtifactBackupJobs:expired", - status: "running", - leaseExpiresAt: now - 1, - }; - const pendingTake = vi.fn().mockResolvedValue([pendingJob]); - const runningTake = vi.fn().mockResolvedValue([expiredRunningJob]); - const patch = vi.fn(); - const withIndex = vi.fn((indexName: string) => ({ - take: indexName === "by_status_leaseExpiresAt" ? runningTake : pendingTake, - })); - const ctx = { - db: { - query: vi.fn(() => ({ withIndex })), - patch, - }, - }; - - const result = await claimRegistryArtifactBackupJobsHandler(ctx as never, { - now, - limit: 2, - leaseToken: "worker-1", - leaseTtlMs: 60_000, - }); - - expect(result.map((job: { _id: string }) => job._id)).toEqual([ - "registryArtifactBackupJobs:pending", - "registryArtifactBackupJobs:expired", - ]); - expect(patch).toHaveBeenCalledWith( - "registryArtifactBackupJobs:pending", - expect.objectContaining({ - status: "running", - leaseToken: "worker-1", - leaseExpiresAt: now + 60_000, - }), - ); - expect(patch).toHaveBeenCalledWith( - "registryArtifactBackupJobs:expired", - expect.objectContaining({ - status: "running", - leaseToken: "worker-1", - leaseExpiresAt: now + 60_000, - }), - ); - }); - - it("can force pending jobs without waiting for nextRunAt", async () => { - const now = 1_700_000_000_000; - const pendingJobs = [ - { - _id: "registryArtifactBackupJobs:future", - status: "pending", - attempts: 1, - nextRunAt: now + 60 * 60 * 1000, - }, - ]; - const take = vi.fn((limit: number) => Promise.resolve(pendingJobs.slice(0, limit))); - const lte = vi.fn(); - const withIndex = vi.fn( - ( - _indexName: string, - buildIndex: (q: { - eq: (field: string, value: unknown) => { lte: (field: string, value: number) => unknown }; - }) => unknown, - ) => { - buildIndex({ - eq: () => ({ - lte, - }), - }); - return { take }; - }, - ); - const ctx = { - db: { - query: vi.fn(() => ({ withIndex })), - }, - }; - - const result = await dueJobsHandler(ctx as never, { - ignoreNextRunAt: true, - limit: 3, - now, - }); - - expect(result).toEqual(pendingJobs); - expect(lte).not.toHaveBeenCalled(); - expect(take).toHaveBeenCalledWith(3); - }); - - it("can include exhausted jobs that still have repair attempts left", async () => { - const now = 1_700_000_000_000; - const pendingJobs = [ - { - _id: "registryArtifactBackupJobs:pending", - status: "pending", - attempts: 1, - nextRunAt: now - 1000, - }, - ]; - const exhaustedJobs = [ - { - _id: "registryArtifactBackupJobs:maxed", - status: "exhausted", - attempts: 16, - nextRunAt: now - 1000, - }, - { - _id: "registryArtifactBackupJobs:repairable", - status: "exhausted", - attempts: 8, - nextRunAt: now - 1000, - }, - ]; - let repairAttemptLimit = Number.POSITIVE_INFINITY; - const pendingTake = vi.fn((limit: number) => Promise.resolve(pendingJobs.slice(0, limit))); - const exhaustedTake = vi.fn((limit: number) => - Promise.resolve( - exhaustedJobs.filter((job) => job.attempts < repairAttemptLimit).slice(0, limit), - ), - ); - const withIndex = vi.fn( - ( - indexName: string, - buildIndex: - | ((q: { - eq: ( - field: string, - value: unknown, - ) => { - lt: (field: string, value: number) => unknown; - lte: (field: string, value: number) => unknown; - }; - }) => unknown) - | undefined, - ) => { - buildIndex?.({ - eq: () => ({ - lt: (_field: string, value: number) => { - repairAttemptLimit = value; - return {}; - }, - lte: () => ({}), - }), - }); - return { take: indexName === "by_status_attempts" ? exhaustedTake : pendingTake }; - }, - ); - const ctx = { - db: { - query: vi.fn(() => ({ withIndex })), - }, - }; - - const result = await dueJobsHandler(ctx as never, { - includeExhaustedRepair: true, - limit: 3, - maxRepairAttempts: 16, - now, - }); - - expect(result.map((job: { _id: string }) => job._id)).toEqual([ - "registryArtifactBackupJobs:pending", - "registryArtifactBackupJobs:repairable", - ]); - expect(withIndex).toHaveBeenNthCalledWith(1, "by_status_nextRunAt", expect.any(Function)); - expect(withIndex).toHaveBeenNthCalledWith(2, "by_status_attempts", expect.any(Function)); - expect(pendingTake).toHaveBeenCalledWith(3); - expect(exhaustedTake).toHaveBeenCalledWith(2); - }); - - it("acquires a retry lease when no active lease exists", async () => { - const now = 1_700_000_000_000; - const insert = vi.fn(); - const ctx = { - db: { - query: vi.fn(() => ({ - withIndex: vi.fn(() => ({ unique: vi.fn().mockResolvedValue(null) })), - })), - insert, - patch: vi.fn(), - }, - }; - - const result = await tryAcquireRegistryArtifactBackupRetryLeaseHandler(ctx as never, { - now, - token: "lease-token", - ttlMs: 60_000, - }); - - expect(result).toEqual({ acquired: true }); - expect(insert).toHaveBeenCalledWith("registryArtifactBackupSyncState", { - key: "retryLease", - cursor: "lease-token", - updatedAt: now, - }); - }); - - it("refuses a retry lease while a fresh lease exists", async () => { - const now = 1_700_000_000_000; - const existing = { - _id: "registryArtifactBackupSyncState:lease", - key: "retryLease", - cursor: "other-token", - updatedAt: now - 1_000, - }; - const patch = vi.fn(); - const ctx = { - db: { - query: vi.fn(() => ({ - withIndex: vi.fn(() => ({ unique: vi.fn().mockResolvedValue(existing) })), - })), - insert: vi.fn(), - patch, - }, - }; - - const result = await tryAcquireRegistryArtifactBackupRetryLeaseHandler(ctx as never, { - now, - token: "lease-token", - ttlMs: 60_000, - }); - - expect(result).toEqual({ acquired: false, holderUpdatedAt: existing.updatedAt }); - expect(patch).not.toHaveBeenCalled(); - }); - - it("acquires a retry lease after the previous lease was released", async () => { - const now = 1_700_000_000_000; - const existing = { - _id: "registryArtifactBackupSyncState:lease", - key: "retryLease", - cursor: undefined, - updatedAt: now - 1_000, - }; - const patch = vi.fn(); - const ctx = { - db: { - query: vi.fn(() => ({ - withIndex: vi.fn(() => ({ unique: vi.fn().mockResolvedValue(existing) })), - })), - insert: vi.fn(), - patch, - }, - }; - - const result = await tryAcquireRegistryArtifactBackupRetryLeaseHandler(ctx as never, { - now, - token: "new-token", - ttlMs: 60_000, - }); - - expect(result).toEqual({ acquired: true }); - expect(patch).toHaveBeenCalledWith("registryArtifactBackupSyncState:lease", { - cursor: "new-token", - updatedAt: now, - }); - }); - - it("reclaims a stale retry lease", async () => { - const now = 1_700_000_000_000; - const existing = { - _id: "registryArtifactBackupSyncState:lease", - key: "retryLease", - cursor: "old-token", - updatedAt: now - 120_000, - }; - const patch = vi.fn(); - const ctx = { - db: { - query: vi.fn(() => ({ - withIndex: vi.fn(() => ({ unique: vi.fn().mockResolvedValue(existing) })), - })), - insert: vi.fn(), - patch, - }, - }; - - const result = await tryAcquireRegistryArtifactBackupRetryLeaseHandler(ctx as never, { - now, - token: "lease-token", - ttlMs: 60_000, - }); - - expect(result).toEqual({ acquired: true }); - expect(patch).toHaveBeenCalledWith("registryArtifactBackupSyncState:lease", { - cursor: "lease-token", - updatedAt: now, - }); - }); - - it("releases only the matching retry lease token", async () => { - const now = 1_700_000_000_000; - const existing = { - _id: "registryArtifactBackupSyncState:lease", - key: "retryLease", - cursor: "lease-token", - updatedAt: now - 1_000, - }; - const patch = vi.fn(); - const ctx = { - db: { - query: vi.fn(() => ({ - withIndex: vi.fn(() => ({ unique: vi.fn().mockResolvedValue(existing) })), - })), - patch, - }, - }; - - const result = await releaseRegistryArtifactBackupRetryLeaseHandler(ctx as never, { - now, - token: "lease-token", - }); - - expect(result).toEqual({ released: true }); - expect(patch).toHaveBeenCalledWith("registryArtifactBackupSyncState:lease", { - cursor: undefined, - updatedAt: now, - }); - }); - - it("upserts package release backup failures into a retryable backlog", async () => { - const now = 1_700_000_000_000; - const existing = { - _id: "registryArtifactBackupJobs:existing", - targetKind: "packageRelease", - packageReleaseId: "packageReleases:demo" as Id<"packageReleases">, - status: "pending", - attempts: 1, - createdAt: now - 1000, - updatedAt: now - 1000, - nextRunAt: now - 1000, - }; - const patch = vi.fn(); - const ctx = { - db: { - query: vi.fn(() => ({ - withIndex: vi.fn(() => ({ unique: vi.fn().mockResolvedValue(existing) })), - })), - insert: vi.fn(), - patch, - }, - }; - - await enqueueRegistryArtifactBackupJobHandler(ctx as never, { - targetKind: "packageRelease", - packageReleaseId: "packageReleases:demo" as Id<"packageReleases">, - reason: "publish", - error: "R2 500", - now, - }); - - expect(ctx.db.insert).not.toHaveBeenCalled(); - expect(patch).toHaveBeenCalledWith( - "registryArtifactBackupJobs:existing", - expect.objectContaining({ - status: "pending", - reason: "publish", - attempts: 0, - lastError: "R2 500", - nextRunAt: now, - createdAt: now, - updatedAt: now, - exhaustedAt: undefined, - completedAt: undefined, - leaseToken: undefined, - leaseExpiresAt: undefined, - claimedAt: undefined, - }), - ); - }); - - it("reports stale, expired running, and exhausted backup jobs for alerting", async () => { - const now = 1_700_000_000_000; - const pendingJobs = [ - { - _id: "registryArtifactBackupJobs:stale", - targetKind: "packageRelease", - packageReleaseId: "packageReleases:stale", - status: "pending", - attempts: 2, - createdAt: now - 49 * 60 * 60 * 1000, - updatedAt: now - 60 * 60 * 1000, - nextRunAt: now - 1000, - }, - { - _id: "registryArtifactBackupJobs:extra", - targetKind: "packageRelease", - packageReleaseId: "packageReleases:extra", - status: "pending", - attempts: 1, - createdAt: now - 60 * 60 * 1000, - updatedAt: now - 1000, - nextRunAt: now - 1000, - }, - ]; - const exhaustedJobs = [ - { - _id: "registryArtifactBackupJobs:exhausted", - targetKind: "skillVersion", - skillVersionId: "skillVersions:exhausted", - status: "exhausted", - attempts: 8, - createdAt: now - 10 * 60 * 60 * 1000, - updatedAt: now - 1000, - nextRunAt: now - 1000, - }, - ]; - const runningJobs = [ - { - _id: "registryArtifactBackupJobs:running", - targetKind: "skillVersion", - skillVersionId: "skillVersions:running", - status: "running", - attempts: 1, - createdAt: now - 60 * 60 * 1000, - updatedAt: now - 1000, - nextRunAt: now - 1000, - leaseExpiresAt: now + 60 * 1000, - }, - { - _id: "registryArtifactBackupJobs:running-extra", - targetKind: "skillVersion", - skillVersionId: "skillVersions:running-extra", - status: "running", - attempts: 1, - createdAt: now - 60 * 60 * 1000, - updatedAt: now - 1000, - nextRunAt: now - 1000, - leaseExpiresAt: now + 60 * 1000, - }, - ]; - const expiredRunningJobs = [ - { - _id: "registryArtifactBackupJobs:expired-running", - targetKind: "skillVersion", - skillVersionId: "skillVersions:expired-running", - status: "running", - attempts: 1, - createdAt: now - 60 * 60 * 1000, - updatedAt: now - 1000, - nextRunAt: now - 1000, - leaseExpiresAt: now - 1000, - }, - ]; - const pendingTake = vi.fn((limit: number) => Promise.resolve(pendingJobs.slice(0, limit))); - const exhaustedTake = vi.fn((limit: number) => Promise.resolve(exhaustedJobs.slice(0, limit))); - const runningTake = vi.fn((limit: number) => Promise.resolve(runningJobs.slice(0, limit))); - const expiredRunningTake = vi.fn((limit: number) => - Promise.resolve(expiredRunningJobs.slice(0, limit)), - ); - const takeByCall = [pendingTake, exhaustedTake, runningTake, expiredRunningTake]; - const withIndex = vi.fn(() => ({ take: takeByCall[withIndex.mock.calls.length - 1] })); - const ctx = { - db: { - query: vi.fn(() => ({ - withIndex, - })), - }, - }; - - const result = await getRegistryArtifactBackupHealthHandler(ctx as never, { - now, - staleAfterMs: 24 * 60 * 60 * 1000, - sampleLimit: 1, - }); - - expect(pendingTake).toHaveBeenCalledWith(2); - expect(exhaustedTake).toHaveBeenCalledWith(2); - expect(runningTake).toHaveBeenCalledWith(2); - expect(expiredRunningTake).toHaveBeenCalledWith(2); - expect(result).toMatchObject({ - pending: 1, - running: 1, - expiredRunning: 1, - stale: 2, - exhausted: 1, - oldestPendingAgeMs: 49 * 60 * 60 * 1000, - pendingCapped: true, - runningCapped: true, - expiredRunningCapped: false, - exhaustedCapped: false, - }); - }); -}); diff --git a/convex/registryArtifactBackups.ts b/convex/registryArtifactBackups.ts deleted file mode 100644 index c52def29..00000000 --- a/convex/registryArtifactBackups.ts +++ /dev/null @@ -1,816 +0,0 @@ -import { v } from "convex/values"; -import { internal } from "./_generated/api"; -import type { Doc, Id } from "./_generated/dataModel"; -import type { MutationCtx, QueryCtx } from "./_generated/server"; -import { action, internalMutation, internalQuery } from "./functions"; -import { assertRole, requireUserFromAction } from "./lib/access"; -import { isPublicSkillDoc } from "./lib/globalStats"; -import { getOwnerPublisher } from "./lib/publishers"; - -const DEFAULT_BATCH_SIZE = 50; -const MAX_BATCH_SIZE = 200; -const SYNC_STATE_KEY = "default"; -const PACKAGE_SYNC_STATE_KEY = "packageReleases"; -const RETRY_LEASE_KEY = "retryLease"; -const MAX_BACKUP_JOB_ERROR_LENGTH = 4000; -const DEFAULT_BACKUP_HEALTH_SAMPLE_LIMIT = 500; -const MAX_BACKUP_HEALTH_SAMPLE_LIMIT = 1000; -const DEFAULT_BACKUP_JOB_LIMIT = 25; -const MAX_BACKUP_JOB_LIMIT = 500; -const DEFAULT_BACKUP_JOB_REPAIR_ATTEMPTS = 16; -const DEFAULT_RETRY_LEASE_TTL_MS = 20 * 60 * 1000; -const MAX_RETRY_LEASE_TTL_MS = 60 * 60 * 1000; -const DEFAULT_BACKUP_JOB_LEASE_TTL_MS = 20 * 60 * 1000; -const MAX_BACKUP_JOB_LEASE_TTL_MS = 60 * 60 * 1000; - -type BackupPageItem = - | { - kind: "ok"; - skillId: Id<"skills">; - versionId: Id<"skillVersions">; - slug: string; - displayName: string; - version: string; - isLatest: boolean; - ownerHandle: string; - publishedAt: number; - } - | { kind: "missingOwner"; skillId: Id<"skills">; ownerUserId: Id<"users"> }; - -type BackupPageResult = { - items: BackupPageItem[]; - cursor: string | null; - isDone: boolean; -}; - -type PackageBackupPageItem = - | { - kind: "ok"; - packageId: Id<"packages">; - releaseId: Id<"packageReleases">; - ownerHandle: string; - packageName: string; - normalizedName: string; - displayName: string; - family: "code-plugin" | "bundle-plugin"; - version: string; - isLatest: boolean; - publishedAt: number; - artifactKind?: "legacy-zip" | "npm-pack"; - artifactStorageId: Id<"_storage">; - artifactFileName?: string; - artifactSha256?: string; - artifactSize?: number; - artifactFormat?: "tgz"; - npmIntegrity?: string; - npmShasum?: string; - npmUnpackedSize?: number; - npmFileCount?: number; - runtimeId?: string; - sourceRepo?: string; - compatibility?: unknown; - extractedPackageJson?: unknown; - extractedPluginManifest?: unknown; - normalizedBundleManifest?: unknown; - files: Array<{ path: string; size: number; sha256: string }>; - } - | { kind: "missingPackage"; releaseId: Id<"packageReleases">; packageId: Id<"packages"> } - | { kind: "missingOwner"; releaseId: Id<"packageReleases">; packageId: Id<"packages"> } - | { kind: "missingArtifact"; releaseId: Id<"packageReleases">; packageId: Id<"packages"> }; - -type PackageBackupPageResult = { - items: PackageBackupPageItem[]; - cursor: string | null; - isDone: boolean; -}; - -type BackupSyncState = { - cursor: string | null; - isDone: boolean; -}; - -export type SeedRegistryArtifactBackupsResult = { - stats: { - skillsScanned: number; - skillsSkipped: number; - skillsBackedUp: number; - skillsMissingVersion: number; - skillsMissingOwner: number; - packagesScanned: number; - packagesSkipped: number; - packagesBackedUp: number; - packagesMissingArtifact: number; - packagesMissingPackage: number; - packagesMissingOwner: number; - skillsEnqueued: number; - packagesEnqueued: number; - retryJobsProcessed: number; - retryJobsSucceeded: number; - retryJobsFailed: number; - staleJobs: number; - exhaustedJobs: number; - errors: number; - }; - cursor: string | null; - packageCursor: string | null; - skillsIsDone: boolean; - packageIsDone: boolean; - isDone: boolean; -}; - -export const getRegistryArtifactBackupPageInternal = internalQuery({ - args: { - cursor: v.optional(v.string()), - batchSize: v.optional(v.number()), - }, - handler: async (ctx, args): Promise => { - const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE); - let pageResult; - try { - pageResult = await ctx.db - .query("skillVersions") - .withIndex("by_active_created", (q) => q.eq("softDeletedAt", undefined)) - .order("asc") - .paginate({ cursor: args.cursor ?? null, numItems: batchSize }); - } catch (error) { - if (!args.cursor || !isStaleCursorError(error)) throw error; - pageResult = await ctx.db - .query("skillVersions") - .withIndex("by_active_created", (q) => q.eq("softDeletedAt", undefined)) - .order("asc") - .paginate({ cursor: null, numItems: batchSize }); - } - - const items: BackupPageItem[] = []; - for (const version of pageResult.page) { - const item = await toSkillVersionBackupPageItem(ctx, version); - if (item) items.push(item); - } - - return { items, cursor: pageResult.continueCursor, isDone: pageResult.isDone }; - }, -}); - -export const getPackageRegistryArtifactBackupPageInternal = internalQuery({ - args: { - cursor: v.optional(v.string()), - batchSize: v.optional(v.number()), - }, - handler: async (ctx, args): Promise => { - const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE); - const pageResult = await ctx.db - .query("packageReleases") - .withIndex("by_active_created", (q) => q.eq("softDeletedAt", undefined)) - .order("asc") - .paginate({ cursor: args.cursor ?? null, numItems: batchSize }); - - const items: PackageBackupPageItem[] = []; - for (const release of pageResult.page) { - const item = await toPackageBackupPageItem(ctx, release); - if (item) items.push(item); - } - - return { items, cursor: pageResult.continueCursor, isDone: pageResult.isDone }; - }, -}); - -async function toSkillVersionBackupPageItem( - ctx: Parameters[0], - version: Doc<"skillVersions">, -): Promise { - const skill = await ctx.db.get(version.skillId); - if (!skill || !isPublicSkillDoc(skill)) return null; - const owner = await getOwnerPublisher(ctx, { - ownerPublisherId: skill.ownerPublisherId, - ownerUserId: skill.ownerUserId, - }); - if (!owner || owner.deletedAt || owner.deactivatedAt) { - return { kind: "missingOwner", skillId: skill._id, ownerUserId: skill.ownerUserId }; - } - return { - kind: "ok", - skillId: skill._id, - versionId: version._id, - slug: skill.slug, - displayName: skill.displayName, - version: version.version, - isLatest: skill.latestVersionId === version._id, - ownerHandle: owner.handle ?? String(skill.ownerPublisherId ?? skill.ownerUserId), - publishedAt: version.createdAt, - }; -} - -async function toPackageBackupPageItem( - ctx: Parameters[0], - release: Doc<"packageReleases">, -): Promise { - const pkg = await ctx.db.get(release.packageId); - if (!pkg || pkg.softDeletedAt) { - return { kind: "missingPackage", releaseId: release._id, packageId: release.packageId }; - } - if (pkg.family !== "code-plugin" && pkg.family !== "bundle-plugin") return null; - if (!release.clawpackStorageId) { - return { kind: "missingArtifact", releaseId: release._id, packageId: release.packageId }; - } - const owner = await getOwnerPublisher(ctx, { - ownerPublisherId: pkg.ownerPublisherId, - ownerUserId: pkg.ownerUserId, - }); - if (!owner || owner.deletedAt || owner.deactivatedAt) { - return { kind: "missingOwner", releaseId: release._id, packageId: release.packageId }; - } - return { - kind: "ok", - packageId: pkg._id, - releaseId: release._id, - ownerHandle: owner.handle, - packageName: pkg.name, - normalizedName: pkg.normalizedName, - displayName: pkg.displayName, - family: pkg.family, - version: release.version, - isLatest: pkg.latestReleaseId === release._id, - publishedAt: release.createdAt, - artifactKind: release.artifactKind, - artifactStorageId: release.clawpackStorageId, - artifactFileName: release.npmTarballName, - artifactSha256: release.clawpackSha256, - artifactSize: release.clawpackSize, - artifactFormat: release.clawpackFormat, - npmIntegrity: release.npmIntegrity, - npmShasum: release.npmShasum, - npmUnpackedSize: release.npmUnpackedSize, - npmFileCount: release.npmFileCount, - runtimeId: release.runtimeId, - sourceRepo: release.sourceRepo, - compatibility: release.compatibility, - extractedPackageJson: release.extractedPackageJson, - extractedPluginManifest: release.extractedPluginManifest, - normalizedBundleManifest: release.normalizedBundleManifest, - files: release.files.map((file) => ({ - path: file.path, - size: file.size, - sha256: file.sha256, - })), - }; -} - -function isStaleCursorError(error: unknown) { - const message = - typeof error === "string" - ? error - : error && typeof error === "object" && "message" in error - ? String((error as { message?: unknown }).message) - : ""; - return ( - message.includes("Failed to parse cursor") || - message.includes("cursor is from a different query") - ); -} - -export const getRegistryArtifactBackupSyncStateInternal = internalQuery({ - args: {}, - handler: async (ctx): Promise => { - const state = await ctx.db - .query("registryArtifactBackupSyncState") - .withIndex("by_key", (q) => q.eq("key", SYNC_STATE_KEY)) - .unique(); - return { cursor: state?.cursor ?? null, isDone: state?.isDone === true }; - }, -}); - -export const setRegistryArtifactBackupSyncStateInternal = internalMutation({ - args: { - cursor: v.optional(v.string()), - isDone: v.optional(v.boolean()), - }, - handler: async (ctx, args) => { - const now = Date.now(); - const state = await ctx.db - .query("registryArtifactBackupSyncState") - .withIndex("by_key", (q) => q.eq("key", SYNC_STATE_KEY)) - .unique(); - - if (!state) { - await ctx.db.insert("registryArtifactBackupSyncState", { - key: SYNC_STATE_KEY, - cursor: args.cursor, - isDone: args.isDone ?? false, - updatedAt: now, - }); - return { ok: true as const }; - } - - await ctx.db.patch(state._id, { - cursor: args.cursor, - isDone: args.isDone ?? false, - updatedAt: now, - }); - - return { ok: true as const }; - }, -}); - -export const getPackageRegistryArtifactBackupSyncStateInternal = internalQuery({ - args: {}, - handler: async (ctx): Promise => { - const state = await ctx.db - .query("registryArtifactBackupSyncState") - .withIndex("by_key", (q) => q.eq("key", PACKAGE_SYNC_STATE_KEY)) - .unique(); - return { cursor: state?.cursor ?? null, isDone: state?.isDone === true }; - }, -}); - -export const setPackageRegistryArtifactBackupSyncStateInternal = internalMutation({ - args: { - cursor: v.optional(v.string()), - isDone: v.optional(v.boolean()), - }, - handler: async (ctx, args) => { - const now = Date.now(); - const state = await ctx.db - .query("registryArtifactBackupSyncState") - .withIndex("by_key", (q) => q.eq("key", PACKAGE_SYNC_STATE_KEY)) - .unique(); - - if (!state) { - await ctx.db.insert("registryArtifactBackupSyncState", { - key: PACKAGE_SYNC_STATE_KEY, - cursor: args.cursor, - isDone: args.isDone ?? false, - updatedAt: now, - }); - return { ok: true as const }; - } - - await ctx.db.patch(state._id, { - cursor: args.cursor, - isDone: args.isDone ?? false, - updatedAt: now, - }); - - return { ok: true as const }; - }, -}); - -export async function tryAcquireRegistryArtifactBackupRetryLeaseHandler( - ctx: Pick, - args: { now?: number; token: string; ttlMs?: number }, -) { - const now = args.now ?? Date.now(); - const ttlMs = clampInt(args.ttlMs ?? DEFAULT_RETRY_LEASE_TTL_MS, 1_000, MAX_RETRY_LEASE_TTL_MS); - const state = await ctx.db - .query("registryArtifactBackupSyncState") - .withIndex("by_key", (q) => q.eq("key", RETRY_LEASE_KEY)) - .unique(); - if (state?.cursor && state.updatedAt + ttlMs > now) { - return { acquired: false as const, holderUpdatedAt: state.updatedAt }; - } - - if (!state) { - await ctx.db.insert("registryArtifactBackupSyncState", { - key: RETRY_LEASE_KEY, - cursor: args.token, - updatedAt: now, - }); - return { acquired: true as const }; - } - - await ctx.db.patch(state._id, { - cursor: args.token, - updatedAt: now, - }); - return { acquired: true as const }; -} - -export const tryAcquireRegistryArtifactBackupRetryLeaseInternal = internalMutation({ - args: { - now: v.optional(v.number()), - token: v.string(), - ttlMs: v.optional(v.number()), - }, - handler: tryAcquireRegistryArtifactBackupRetryLeaseHandler, -}); - -export async function releaseRegistryArtifactBackupRetryLeaseHandler( - ctx: Pick, - args: { now?: number; token: string }, -) { - const now = args.now ?? Date.now(); - const state = await ctx.db - .query("registryArtifactBackupSyncState") - .withIndex("by_key", (q) => q.eq("key", RETRY_LEASE_KEY)) - .unique(); - if (!state || state.cursor !== args.token) return { released: false as const }; - - await ctx.db.patch(state._id, { - cursor: undefined, - updatedAt: now, - }); - return { released: true as const }; -} - -export const releaseRegistryArtifactBackupRetryLeaseInternal = internalMutation({ - args: { - now: v.optional(v.number()), - token: v.string(), - }, - handler: releaseRegistryArtifactBackupRetryLeaseHandler, -}); - -const registryArtifactBackupTargetKindValidator = v.union( - v.literal("skillVersion"), - v.literal("packageRelease"), -); -const registryArtifactBackupReasonValidator = v.union( - v.literal("publish"), - v.literal("seed"), - v.literal("retry"), - v.literal("sync"), -); -const registryArtifactBackupStatusValidator = v.union( - v.literal("pending"), - v.literal("running"), - v.literal("succeeded"), - v.literal("exhausted"), - v.literal("missingArtifact"), -); - -export const enqueueRegistryArtifactBackupJobInternal = internalMutation({ - args: { - targetKind: registryArtifactBackupTargetKindValidator, - skillVersionId: v.optional(v.id("skillVersions")), - packageReleaseId: v.optional(v.id("packageReleases")), - reason: registryArtifactBackupReasonValidator, - status: v.optional(registryArtifactBackupStatusValidator), - preserveTerminal: v.optional(v.boolean()), - error: v.optional(v.string()), - now: v.optional(v.number()), - }, - handler: enqueueRegistryArtifactBackupJobHandler, -}); - -export async function claimRegistryArtifactBackupJobsHandler( - ctx: Pick, - args: { - now?: number; - limit?: number; - leaseToken: string; - leaseTtlMs?: number; - forceDue?: boolean; - includeExhaustedRepair?: boolean; - maxRepairAttempts?: number; - }, -) { - const now = args.now ?? Date.now(); - const limit = clampInt(args.limit ?? DEFAULT_BACKUP_JOB_LIMIT, 1, MAX_BACKUP_JOB_LIMIT); - const leaseTtlMs = clampInt( - args.leaseTtlMs ?? DEFAULT_BACKUP_JOB_LEASE_TTL_MS, - 1_000, - MAX_BACKUP_JOB_LEASE_TTL_MS, - ); - const pending = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_status_nextRunAt", (q) => { - const byStatus = q.eq("status", "pending"); - return args.forceDue ? byStatus : byStatus.lte("nextRunAt", now); - }) - .take(limit); - - let claimed = pending; - if (claimed.length < limit) { - const expiredRunning = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_status_leaseExpiresAt", (q) => - q.eq("status", "running").lte("leaseExpiresAt", now), - ) - .take(limit - claimed.length); - claimed = [...claimed, ...expiredRunning]; - } - if (args.includeExhaustedRepair && claimed.length < limit) { - const maxRepairAttempts = Math.max( - 1, - Math.floor(args.maxRepairAttempts ?? DEFAULT_BACKUP_JOB_REPAIR_ATTEMPTS), - ); - const exhausted = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_status_attempts", (q) => - q.eq("status", "exhausted").lt("attempts", maxRepairAttempts), - ) - .take(limit - claimed.length); - claimed = [...claimed, ...exhausted]; - } - - const leaseExpiresAt = now + leaseTtlMs; - for (const job of claimed) { - await ctx.db.patch(job._id, { - status: "running", - leaseToken: args.leaseToken, - leaseExpiresAt, - claimedAt: now, - lastAttemptAt: now, - updatedAt: now, - }); - } - return claimed; -} - -export const claimRegistryArtifactBackupJobsInternal = internalMutation({ - args: { - now: v.optional(v.number()), - limit: v.optional(v.number()), - leaseToken: v.string(), - leaseTtlMs: v.optional(v.number()), - forceDue: v.optional(v.boolean()), - includeExhaustedRepair: v.optional(v.boolean()), - maxRepairAttempts: v.optional(v.number()), - }, - handler: claimRegistryArtifactBackupJobsHandler, -}); - -export async function markRegistryArtifactBackupJobSucceededHandler( - ctx: Pick, - args: { jobId: Id<"registryArtifactBackupJobs">; leaseToken?: string; now?: number }, -) { - const now = args.now ?? Date.now(); - const job = await ctx.db.get(args.jobId); - if (!job) return { missing: true as const, stale: false as const }; - if (args.leaseToken && (job.status !== "running" || job.leaseToken !== args.leaseToken)) { - return { missing: false as const, stale: true as const }; - } - await ctx.db.patch(args.jobId, { - status: "succeeded", - completedAt: now, - lastError: undefined, - leaseToken: undefined, - leaseExpiresAt: undefined, - claimedAt: undefined, - updatedAt: now, - }); - return { missing: false as const, stale: false as const }; -} - -export const markRegistryArtifactBackupJobSucceededInternal = internalMutation({ - args: { - jobId: v.id("registryArtifactBackupJobs"), - leaseToken: v.optional(v.string()), - now: v.optional(v.number()), - }, - handler: markRegistryArtifactBackupJobSucceededHandler, -}); - -export const markRegistryArtifactBackupJobFailedInternal = internalMutation({ - args: { - jobId: v.id("registryArtifactBackupJobs"), - error: v.string(), - leaseToken: v.optional(v.string()), - now: v.optional(v.number()), - maxAttempts: v.optional(v.number()), - }, - handler: async (ctx, args) => { - const now = args.now ?? Date.now(); - const maxAttempts = Math.max(1, Math.floor(args.maxAttempts ?? 8)); - const job = await ctx.db.get(args.jobId); - if (!job) return { missing: true as const }; - if (args.leaseToken && (job.status !== "running" || job.leaseToken !== args.leaseToken)) { - return { missing: false as const, stale: true as const }; - } - const attempts = job.attempts + 1; - const exhausted = attempts >= maxAttempts; - await ctx.db.patch(args.jobId, { - status: exhausted ? "exhausted" : "pending", - attempts, - lastAttemptAt: now, - lastError: truncateBackupJobError(args.error), - nextRunAt: exhausted ? now : now + retryDelayMs(attempts), - exhaustedAt: exhausted ? now : undefined, - leaseToken: undefined, - leaseExpiresAt: undefined, - claimedAt: undefined, - updatedAt: now, - }); - return { missing: false as const, stale: false as const, exhausted, attempts }; - }, -}); - -export const getDueRegistryArtifactBackupJobsInternal = internalQuery({ - args: { - includeExhaustedRepair: v.optional(v.boolean()), - ignoreNextRunAt: v.optional(v.boolean()), - maxRepairAttempts: v.optional(v.number()), - now: v.optional(v.number()), - limit: v.optional(v.number()), - }, - handler: async (ctx, args) => { - const now = args.now ?? Date.now(); - const limit = clampInt(args.limit ?? DEFAULT_BACKUP_JOB_LIMIT, 1, MAX_BACKUP_JOB_LIMIT); - const pending = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_status_nextRunAt", (q) => { - const byStatus = q.eq("status", "pending"); - return args.ignoreNextRunAt ? byStatus : byStatus.lte("nextRunAt", now); - }) - .take(limit); - if (!args.includeExhaustedRepair || pending.length >= limit) return pending; - - const maxRepairAttempts = Math.max( - 1, - Math.floor(args.maxRepairAttempts ?? DEFAULT_BACKUP_JOB_REPAIR_ATTEMPTS), - ); - const remaining = limit - pending.length; - const exhausted = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_status_attempts", (q) => - q.eq("status", "exhausted").lt("attempts", maxRepairAttempts), - ) - .take(remaining); - - return [...pending, ...exhausted]; - }, -}); - -export const getRegistryArtifactBackupHealthInternal = internalQuery({ - args: { - now: v.optional(v.number()), - staleAfterMs: v.optional(v.number()), - sampleLimit: v.optional(v.number()), - }, - handler: getRegistryArtifactBackupHealthHandler, -}); - -export const seedRegistryArtifactBackups: ReturnType = action({ - args: { - dryRun: v.optional(v.boolean()), - batchSize: v.optional(v.number()), - maxBatches: v.optional(v.number()), - queueOnly: v.optional(v.boolean()), - resetCursor: v.optional(v.boolean()), - }, - handler: async (ctx, args): Promise => { - const { user } = await requireUserFromAction(ctx); - assertRole(user, ["admin"]); - - if (args.resetCursor && !args.dryRun) { - await ctx.runMutation( - internal.registryArtifactBackups.setRegistryArtifactBackupSyncStateInternal, - { - cursor: undefined, - isDone: false, - }, - ); - await ctx.runMutation( - internal.registryArtifactBackups.setPackageRegistryArtifactBackupSyncStateInternal, - { - cursor: undefined, - isDone: false, - }, - ); - } - - return ctx.runAction(internal.registryArtifactBackupsNode.seedRegistryArtifactBackupsInternal, { - dryRun: args.dryRun, - batchSize: args.batchSize, - maxBatches: args.maxBatches, - queueOnly: args.queueOnly, - }) as Promise; - }, -}); - -function clampInt(value: number, min: number, max: number) { - return Math.max(min, Math.min(max, Math.floor(value))); -} - -export async function enqueueRegistryArtifactBackupJobHandler( - ctx: Pick, - args: { - targetKind: "skillVersion" | "packageRelease"; - skillVersionId?: Id<"skillVersions">; - packageReleaseId?: Id<"packageReleases">; - reason: "publish" | "seed" | "retry" | "sync"; - status?: "pending" | "running" | "succeeded" | "exhausted" | "missingArtifact"; - preserveTerminal?: boolean; - error?: string; - now?: number; - }, -) { - const now = args.now ?? Date.now(); - const status = args.status ?? "pending"; - const existing = - args.targetKind === "skillVersion" && args.skillVersionId - ? await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_skill_version", (q) => q.eq("skillVersionId", args.skillVersionId)) - .unique() - : args.targetKind === "packageRelease" && args.packageReleaseId - ? await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_package_release", (q) => q.eq("packageReleaseId", args.packageReleaseId)) - .unique() - : null; - - const lastError = truncateBackupJobError(args.error); - if (existing) { - if ( - args.preserveTerminal && - (existing.status === "succeeded" || existing.status === "missingArtifact") - ) { - return { jobId: existing._id, created: false as const, preserved: true as const }; - } - await ctx.db.patch(existing._id, { - status, - reason: args.reason, - attempts: 0, - lastError, - nextRunAt: now, - leaseToken: undefined, - leaseExpiresAt: undefined, - claimedAt: undefined, - createdAt: now, - updatedAt: now, - exhaustedAt: undefined, - completedAt: status === "missingArtifact" || status === "succeeded" ? now : undefined, - }); - return { jobId: existing._id, created: false as const }; - } - - const jobId = await ctx.db.insert("registryArtifactBackupJobs", { - targetKind: args.targetKind, - skillVersionId: args.skillVersionId, - packageReleaseId: args.packageReleaseId, - status, - reason: args.reason, - attempts: 0, - nextRunAt: now, - lastError, - completedAt: status === "missingArtifact" || status === "succeeded" ? now : undefined, - createdAt: now, - updatedAt: now, - }); - return { jobId, created: true as const }; -} - -export async function getRegistryArtifactBackupHealthHandler( - ctx: Pick, - args: { now?: number; staleAfterMs?: number; sampleLimit?: number }, -) { - const now = args.now ?? Date.now(); - const staleAfterMs = args.staleAfterMs ?? 24 * 60 * 60 * 1000; - const sampleLimit = clampInt( - args.sampleLimit ?? DEFAULT_BACKUP_HEALTH_SAMPLE_LIMIT, - 1, - MAX_BACKUP_HEALTH_SAMPLE_LIMIT, - ); - const pending = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_status_nextRunAt", (q) => q.eq("status", "pending").lte("nextRunAt", now)) - .take(sampleLimit + 1); - const exhausted = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_status_nextRunAt", (q) => q.eq("status", "exhausted")) - .take(sampleLimit + 1); - const running = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_status_leaseExpiresAt", (q) => q.eq("status", "running")) - .take(sampleLimit + 1); - const expiredRunning = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_status_leaseExpiresAt", (q) => - q.eq("status", "running").lte("leaseExpiresAt", now), - ) - .take(sampleLimit + 1); - const pendingSample = pending.slice(0, sampleLimit); - const exhaustedSample = exhausted.slice(0, sampleLimit); - const runningSample = running.slice(0, sampleLimit); - const expiredRunningSample = expiredRunning.slice(0, sampleLimit); - const oldestPendingAgeMs = pendingSample.reduce( - (max: number, job: { createdAt: number }) => Math.max(max, now - job.createdAt), - 0, - ); - const stalePending = pendingSample.filter( - (job: { createdAt: number }) => now - job.createdAt >= staleAfterMs, - ).length; - const stale = stalePending + expiredRunningSample.length; - return { - pending: pendingSample.length, - running: runningSample.length, - expiredRunning: expiredRunningSample.length, - stale, - exhausted: exhaustedSample.length, - oldestPendingAgeMs, - pendingCapped: pending.length > sampleLimit, - runningCapped: running.length > sampleLimit, - expiredRunningCapped: expiredRunning.length > sampleLimit, - exhaustedCapped: exhausted.length > sampleLimit, - }; -} - -function truncateBackupJobError(error: string | undefined) { - if (!error) return undefined; - return error.slice(0, MAX_BACKUP_JOB_ERROR_LENGTH); -} - -function retryDelayMs(attempts: number) { - const minutes = Math.min(60, 2 ** Math.min(attempts, 6)); - return minutes * 60 * 1000; -} diff --git a/convex/registryArtifactBackupsNode.ts b/convex/registryArtifactBackupsNode.ts deleted file mode 100644 index af3a96e9..00000000 --- a/convex/registryArtifactBackupsNode.ts +++ /dev/null @@ -1,1033 +0,0 @@ -"use node"; - -import { v } from "convex/values"; -import { internal } from "./_generated/api"; -import type { Doc, Id } from "./_generated/dataModel"; -import type { ActionCtx } from "./_generated/server"; -import { internalAction } from "./functions"; -import { isPublicSkillDoc } from "./lib/globalStats"; -import { - backupPackageReleaseToObjectStorage, - backupSkillVersionToObjectStorage, - fetchPackageReleaseBackupMeta, - fetchSkillVersionBackupMeta, - getRegistryArtifactBackupContext, - isRegistryArtifactBackupConfigured, - type RegistryArtifactBackupContext, -} from "./lib/registryArtifactBackup"; - -const DEFAULT_BATCH_SIZE = 50; -const MAX_BATCH_SIZE = 200; -const DEFAULT_MAX_BATCHES = 5; -const MAX_MAX_BATCHES = 200; -const DEFAULT_JOB_BATCH_SIZE = 500; -const MAX_RETRY_REPAIR_ATTEMPTS = 16; -const MAX_PARALLEL_RETRY_ROOTS = 50; -const UNKNOWN_PACKAGE_ARTIFACT_BYTES = 120 * 1024 * 1024; -const UNKNOWN_SKILL_ARTIFACT_BYTES = 50 * 1024 * 1024; -const MAX_PARALLEL_RETRY_ARTIFACT_BYTES = UNKNOWN_PACKAGE_ARTIFACT_BYTES; -const STALE_BACKUP_JOB_MS = 24 * 60 * 60 * 1000; -const RETRY_LEASE_TTL_MS = 20 * 60 * 1000; - -type BackupPageItem = - | { - kind: "ok"; - skillId: Doc<"skills">["_id"]; - versionId: Doc<"skillVersions">["_id"]; - slug: string; - version: string; - isLatest: boolean; - displayName: string; - ownerHandle: string; - publishedAt: number; - } - | { kind: "missingOwner" }; - -type PackageBackupPageItem = - | { - kind: "ok"; - packageId: Doc<"packages">["_id"]; - releaseId: Doc<"packageReleases">["_id"]; - ownerHandle: string; - packageName: string; - normalizedName: string; - displayName: string; - family: "code-plugin" | "bundle-plugin"; - version: string; - isLatest: boolean; - publishedAt: number; - artifactKind?: "legacy-zip" | "npm-pack"; - artifactStorageId: Id<"_storage">; - artifactFileName?: string; - artifactSha256?: string; - artifactSize?: number; - artifactFormat?: "tgz"; - npmIntegrity?: string; - npmShasum?: string; - npmUnpackedSize?: number; - npmFileCount?: number; - runtimeId?: string; - sourceRepo?: string; - compatibility?: unknown; - extractedPackageJson?: unknown; - extractedPluginManifest?: unknown; - normalizedBundleManifest?: unknown; - files: Array<{ path: string; size: number; sha256: string }>; - } - | { kind: "missingPackage" } - | { kind: "missingOwner" } - | { kind: "missingArtifact"; releaseId: Id<"packageReleases"> }; - -export type RegistryArtifactBackupSyncStats = { - skillsScanned: number; - skillsSkipped: number; - skillsBackedUp: number; - skillsMissingVersion: number; - skillsMissingOwner: number; - packagesScanned: number; - packagesSkipped: number; - packagesBackedUp: number; - packagesMissingArtifact: number; - packagesMissingPackage: number; - packagesMissingOwner: number; - skillsEnqueued: number; - packagesEnqueued: number; - retryJobsProcessed: number; - retryJobsSucceeded: number; - retryJobsFailed: number; - staleJobs: number; - exhaustedJobs: number; - errors: number; -}; - -export type SeedRegistryArtifactBackupsInternalArgs = { - dryRun?: boolean; - batchSize?: number; - maxBatches?: number; - queueOnly?: boolean; -}; - -export type SeedRegistryArtifactBackupsInternalResult = { - stats: RegistryArtifactBackupSyncStats; - cursor: string | null; - packageCursor: string | null; - skillsIsDone: boolean; - packageIsDone: boolean; - isDone: boolean; -}; - -export type ProcessRegistryArtifactBackupRetriesInternalResult = { - stats: RegistryArtifactBackupSyncStats; -}; - -export type ProcessRegistryArtifactBackupRetriesInternalArgs = { - dryRun?: boolean; - forceDue?: boolean; -}; - -export type ProcessRegistryArtifactBackupQueueInternalArgs = { - dryRun?: boolean; - forceDue?: boolean; - limit?: number; - leaseTtlMs?: number; - workerId?: string; -}; - -export const backupSkillForPublishInternal = internalAction({ - args: { - skillId: v.optional(v.id("skills")), - versionId: v.optional(v.id("skillVersions")), - slug: v.string(), - version: v.string(), - isLatest: v.optional(v.boolean()), - displayName: v.string(), - ownerHandle: v.string(), - files: v.array( - v.object({ - path: v.string(), - size: v.number(), - storageId: v.id("_storage"), - sha256: v.string(), - contentType: v.optional(v.string()), - }), - ), - publishedAt: v.number(), - }, - handler: async (ctx, args) => { - if (!isRegistryArtifactBackupConfigured()) { - return { skipped: true as const }; - } - try { - const item = args.versionId ? await getSkillBackupItemForVersion(ctx, args.versionId) : null; - if (args.versionId && !item) { - return { skipped: true as const }; - } - await backupSkillVersionToObjectStorage(ctx, item ?? args); - return { skipped: false as const }; - } catch (error) { - if (args.versionId) { - await ctx.runMutation( - internal.registryArtifactBackups.enqueueRegistryArtifactBackupJobInternal, - { - targetKind: "skillVersion", - skillVersionId: args.versionId, - reason: "publish", - error: errorMessage(error), - }, - ); - } - console.error("Registry skill artifact backup failed", error); - return { skipped: false as const, queuedRetry: Boolean(args.versionId) }; - } - }, -}); - -export const backupPackageForPublishInternal = internalAction({ - args: { - ownerHandle: v.string(), - packageId: v.id("packages"), - releaseId: v.id("packageReleases"), - packageName: v.string(), - normalizedName: v.string(), - displayName: v.string(), - family: v.union(v.literal("code-plugin"), v.literal("bundle-plugin")), - version: v.string(), - isLatest: v.optional(v.boolean()), - publishedAt: v.number(), - artifactKind: v.optional(v.union(v.literal("legacy-zip"), v.literal("npm-pack"))), - artifactStorageId: v.id("_storage"), - artifactFileName: v.optional(v.string()), - artifactSha256: v.optional(v.string()), - artifactSize: v.optional(v.number()), - artifactFormat: v.optional(v.literal("tgz")), - npmIntegrity: v.optional(v.string()), - npmShasum: v.optional(v.string()), - npmUnpackedSize: v.optional(v.number()), - npmFileCount: v.optional(v.number()), - runtimeId: v.optional(v.string()), - sourceRepo: v.optional(v.string()), - compatibility: v.optional(v.any()), - extractedPackageJson: v.optional(v.any()), - extractedPluginManifest: v.optional(v.any()), - normalizedBundleManifest: v.optional(v.any()), - files: v.array(v.object({ path: v.string(), size: v.number(), sha256: v.string() })), - }, - handler: async (ctx, args) => { - if (!isRegistryArtifactBackupConfigured()) { - return { skipped: true as const }; - } - try { - const item = await getPackageBackupItemForRelease(ctx, args.releaseId); - if (!item) { - return { skipped: true as const }; - } - await backupPackageReleaseToObjectStorage(ctx, item); - return { skipped: false as const }; - } catch (error) { - await ctx.runMutation( - internal.registryArtifactBackups.enqueueRegistryArtifactBackupJobInternal, - { - targetKind: "packageRelease", - packageReleaseId: args.releaseId, - reason: "publish", - error: errorMessage(error), - }, - ); - console.error("Registry package artifact backup failed", error); - return { skipped: false as const, queuedRetry: true as const }; - } - }, -}); - -export async function seedRegistryArtifactBackupsInternalHandler( - ctx: ActionCtx, - args: SeedRegistryArtifactBackupsInternalArgs, -): Promise { - const dryRun = Boolean(args.dryRun); - const queueOnly = Boolean(args.queueOnly); - const stats = initialRegistryArtifactBackupSyncStats(); - - if (!isRegistryArtifactBackupConfigured()) { - return { - stats, - cursor: null, - packageCursor: null, - skillsIsDone: true, - packageIsDone: true, - isDone: true, - }; - } - - const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE); - const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES); - const context = getRegistryArtifactBackupContext(); - if (!queueOnly) { - await processDueRegistryArtifactBackupJobs(ctx, context, dryRun, stats); - } - - const state = dryRun - ? { cursor: null as string | null, isDone: false } - : ((await ctx.runQuery( - internal.registryArtifactBackups.getRegistryArtifactBackupSyncStateInternal, - {}, - )) as { - cursor: string | null; - isDone: boolean; - }); - - let cursor: string | null = state.cursor; - let isDone = state.isDone; - - for (let batch = 0; !isDone && batch < maxBatches; batch++) { - const page = (await ctx.runQuery( - internal.registryArtifactBackups.getRegistryArtifactBackupPageInternal, - { - cursor: cursor ?? undefined, - batchSize, - }, - )) as { items: BackupPageItem[]; cursor: string | null; isDone: boolean }; - - cursor = page.cursor; - isDone = page.isDone; - - for (const item of page.items) { - if (item.kind !== "ok") { - if (item.kind === "missingOwner") { - stats.skillsMissingOwner += 1; - } - continue; - } - - stats.skillsScanned += 1; - if (queueOnly) { - if (!dryRun) { - await ctx.runMutation( - internal.registryArtifactBackups.enqueueRegistryArtifactBackupJobInternal, - { - targetKind: "skillVersion", - skillVersionId: item.versionId, - reason: "seed", - preserveTerminal: true, - }, - ); - } - stats.skillsEnqueued += 1; - continue; - } - try { - const meta = await fetchSkillVersionBackupMeta( - context, - item.ownerHandle, - item.slug, - item.version, - ); - if (meta?.version === item.version && meta.restore.versionId === item.versionId) { - stats.skillsSkipped += 1; - continue; - } - - const version = (await ctx.runQuery(internal.skills.getVersionByIdInternal, { - versionId: item.versionId, - })) as Doc<"skillVersions"> | null; - if (!version) { - stats.skillsMissingVersion += 1; - continue; - } - - if (!dryRun) { - await backupSkillVersionToObjectStorage( - ctx, - { - skillId: item.skillId, - versionId: item.versionId, - slug: item.slug, - version: item.version, - isLatest: item.isLatest, - displayName: item.displayName, - ownerHandle: item.ownerHandle, - files: version.files, - publishedAt: item.publishedAt, - }, - context, - ); - stats.skillsBackedUp += 1; - } - } catch (error) { - console.error("Registry skill artifact backup seed failed", error); - stats.errors += 1; - if (!dryRun) { - await ctx.runMutation( - internal.registryArtifactBackups.enqueueRegistryArtifactBackupJobInternal, - { - targetKind: "skillVersion", - skillVersionId: item.versionId, - reason: "seed", - error: errorMessage(error), - }, - ); - } - } - } - - if (!dryRun) { - await ctx.runMutation( - internal.registryArtifactBackups.setRegistryArtifactBackupSyncStateInternal, - { - cursor: isDone ? undefined : (cursor ?? undefined), - isDone, - }, - ); - } - - if (isDone) break; - } - - const packageSync = await syncPackageReleaseBackups(ctx, context, args, dryRun, queueOnly, stats); - await alertOnUnhealthyBackupBacklog(ctx, stats); - - if (!dryRun && !state.isDone) { - await ctx.runMutation( - internal.registryArtifactBackups.setRegistryArtifactBackupSyncStateInternal, - { - cursor: isDone ? undefined : (cursor ?? undefined), - isDone, - }, - ); - } - - return { - stats, - cursor, - packageCursor: packageSync.cursor, - skillsIsDone: isDone, - packageIsDone: packageSync.isDone, - isDone: isDone && packageSync.isDone, - }; -} - -async function syncPackageReleaseBackups( - ctx: ActionCtx, - context: RegistryArtifactBackupContext, - args: SeedRegistryArtifactBackupsInternalArgs, - dryRun: boolean, - queueOnly: boolean, - stats: RegistryArtifactBackupSyncStats, -): Promise<{ cursor: string | null; isDone: boolean }> { - const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE); - const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES); - - const state = dryRun - ? { cursor: null as string | null, isDone: false } - : ((await ctx.runQuery( - internal.registryArtifactBackups.getPackageRegistryArtifactBackupSyncStateInternal, - {}, - )) as { - cursor: string | null; - isDone: boolean; - }); - let cursor: string | null = state.cursor; - let isDone = state.isDone; - - for (let batch = 0; !isDone && batch < maxBatches; batch++) { - const page = (await ctx.runQuery( - internal.registryArtifactBackups.getPackageRegistryArtifactBackupPageInternal, - { - cursor: cursor ?? undefined, - batchSize, - }, - )) as { items: PackageBackupPageItem[]; cursor: string | null; isDone: boolean }; - cursor = page.cursor; - isDone = page.isDone; - - for (const item of page.items) { - if (item.kind !== "ok") { - if (item.kind === "missingArtifact") { - stats.packagesMissingArtifact += 1; - if (queueOnly && !dryRun) { - await ctx.runMutation( - internal.registryArtifactBackups.enqueueRegistryArtifactBackupJobInternal, - { - targetKind: "packageRelease", - packageReleaseId: item.releaseId, - reason: "seed", - status: "missingArtifact", - preserveTerminal: true, - }, - ); - } - } - if (item.kind === "missingPackage") stats.packagesMissingPackage += 1; - if (item.kind === "missingOwner") stats.packagesMissingOwner += 1; - continue; - } - stats.packagesScanned += 1; - if (queueOnly) { - if (!dryRun) { - await ctx.runMutation( - internal.registryArtifactBackups.enqueueRegistryArtifactBackupJobInternal, - { - targetKind: "packageRelease", - packageReleaseId: item.releaseId, - reason: "seed", - preserveTerminal: true, - }, - ); - } - stats.packagesEnqueued += 1; - continue; - } - try { - const meta = await fetchPackageReleaseBackupMeta( - context, - item.ownerHandle, - item.normalizedName, - item.version, - ); - if ( - meta?.restore?.releaseId === item.releaseId && - meta.artifact.sha256 === item.artifactSha256 - ) { - stats.packagesSkipped += 1; - continue; - } - if (!dryRun) { - await backupPackageReleaseToObjectStorage(ctx, item, context); - stats.packagesBackedUp += 1; - } - } catch (error) { - stats.errors += 1; - console.error("Registry package artifact backup seed failed", error); - if (!dryRun) { - await ctx.runMutation( - internal.registryArtifactBackups.enqueueRegistryArtifactBackupJobInternal, - { - targetKind: "packageRelease", - packageReleaseId: item.releaseId, - reason: "seed", - error: errorMessage(error), - }, - ); - } - } - } - - if (!dryRun) { - await ctx.runMutation( - internal.registryArtifactBackups.setPackageRegistryArtifactBackupSyncStateInternal, - { - cursor: isDone ? undefined : (cursor ?? undefined), - isDone, - }, - ); - } - if (isDone) break; - } - return { cursor, isDone }; -} - -async function processDueRegistryArtifactBackupJobs( - ctx: ActionCtx, - context: RegistryArtifactBackupContext, - dryRun: boolean, - stats: RegistryArtifactBackupSyncStats, - options: { forceDue?: boolean; leaseToken?: string } = {}, -) { - if (dryRun) return; - const leaseToken = - options.leaseToken ?? - `registry-backup-serial-${Date.now()}-${Math.random().toString(36).slice(2)}`; - const jobs = (await ctx.runMutation( - internal.registryArtifactBackups.claimRegistryArtifactBackupJobsInternal, - { - forceDue: options.forceDue, - limit: DEFAULT_JOB_BATCH_SIZE, - leaseToken, - leaseTtlMs: RETRY_LEASE_TTL_MS, - includeExhaustedRepair: true, - maxRepairAttempts: MAX_RETRY_REPAIR_ATTEMPTS, - }, - )) as Array>; - - const groups = await groupRetryJobsByRoot(ctx, jobs); - const chunks = chunkRetryJobGroups(groups); - for (const chunk of chunks) { - const results = await Promise.all( - chunk.map((group) => processRetryJobGroup(ctx, context, group, leaseToken)), - ); - for (const result of results) { - stats.retryJobsProcessed += result.processed; - stats.retryJobsSucceeded += result.succeeded; - stats.retryJobsFailed += result.failed; - } - } -} - -type RetryJobWorkItem = - | { - error: unknown; - kind: "lookupFailed"; - job: Doc<"registryArtifactBackupJobs">; - } - | { - kind: "missing"; - job: Doc<"registryArtifactBackupJobs">; - } - | { - kind: "packageRelease"; - job: Doc<"registryArtifactBackupJobs">; - item: Extract; - rootKey: string; - estimatedBytes: number; - } - | { - kind: "skillVersion"; - job: Doc<"registryArtifactBackupJobs">; - item: Parameters[1]; - rootKey: string; - estimatedBytes: number; - }; - -type RetryJobGroup = { - estimatedBytes: number; - items: RetryJobWorkItem[]; -}; - -async function groupRetryJobsByRoot( - ctx: ActionCtx, - jobs: Array>, -) { - const groups = new Map(); - for (const job of jobs) { - const workItem = await toRetryJobWorkItem(ctx, job).catch((error: unknown) => ({ - error, - kind: "lookupFailed" as const, - job, - })); - const rootKey = - workItem.kind === "missing" || workItem.kind === "lookupFailed" - ? `${workItem.kind}:${job._id}` - : workItem.rootKey; - const group = groups.get(rootKey); - const estimatedBytes = estimatedRetryJobBytes(workItem); - if (group) { - group.items.push(workItem); - group.estimatedBytes = Math.max(group.estimatedBytes, estimatedBytes); - } else { - groups.set(rootKey, { estimatedBytes, items: [workItem] }); - } - } - return Array.from(groups.values()); -} - -async function toRetryJobWorkItem( - ctx: ActionCtx, - job: Doc<"registryArtifactBackupJobs">, -): Promise { - if (job.targetKind === "packageRelease" && job.packageReleaseId) { - const item = await getPackageBackupItemForRelease(ctx, job.packageReleaseId); - if (!item) return { kind: "missing", job }; - return { - estimatedBytes: item.artifactSize ?? UNKNOWN_PACKAGE_ARTIFACT_BYTES, - kind: "packageRelease", - job, - item, - rootKey: `package:${item.ownerHandle}/${item.normalizedName}`, - }; - } - if (job.targetKind === "skillVersion" && job.skillVersionId) { - const item = await getSkillBackupItemForVersion(ctx, job.skillVersionId); - if (!item) return { kind: "missing", job }; - return { - estimatedBytes: estimateSkillBackupBytes(item), - kind: "skillVersion", - job, - item, - rootKey: `skill:${item.ownerHandle}/${item.slug}`, - }; - } - return { kind: "missing", job }; -} - -async function processRetryJobGroup( - ctx: ActionCtx, - context: RegistryArtifactBackupContext, - group: RetryJobGroup, - leaseToken?: string, -) { - const result = { processed: 0, succeeded: 0, failed: 0 }; - for (const workItem of group.items) { - result.processed += 1; - try { - if (workItem.kind === "lookupFailed") { - throw workItem.error; - } else if (workItem.kind === "missing") { - const mark = await markRetryJobSucceeded(ctx, workItem.job, leaseToken); - if (!mark?.stale) result.succeeded += 1; - } else if (workItem.kind === "packageRelease") { - if (await hasMatchingPackageReleaseMeta(context, workItem.item)) { - const mark = await markRetryJobSucceeded(ctx, workItem.job, leaseToken); - if (!mark?.stale) result.succeeded += 1; - } else { - await backupPackageReleaseToObjectStorage(ctx, workItem.item, context); - const mark = await markRetryJobSucceeded(ctx, workItem.job, leaseToken); - if (!mark?.stale) result.succeeded += 1; - } - } else { - if (await hasMatchingSkillVersionMeta(context, workItem.item)) { - const mark = await markRetryJobSucceeded(ctx, workItem.job, leaseToken); - if (!mark?.stale) result.succeeded += 1; - } else { - await backupSkillVersionToObjectStorage(ctx, workItem.item, context); - const mark = await markRetryJobSucceeded(ctx, workItem.job, leaseToken); - if (!mark?.stale) result.succeeded += 1; - } - } - } catch (error) { - const mark = (await ctx.runMutation( - internal.registryArtifactBackups.markRegistryArtifactBackupJobFailedInternal, - { - jobId: workItem.job._id, - error: errorMessage(error), - leaseToken, - maxAttempts: MAX_RETRY_REPAIR_ATTEMPTS, - }, - )) as { stale?: boolean }; - if (!mark?.stale) result.failed += 1; - } - } - return result; -} - -async function markRetryJobSucceeded( - ctx: ActionCtx, - job: Doc<"registryArtifactBackupJobs">, - leaseToken?: string, -) { - return (await ctx.runMutation( - internal.registryArtifactBackups.markRegistryArtifactBackupJobSucceededInternal, - { - jobId: job._id, - leaseToken, - }, - )) as { stale?: boolean }; -} - -async function hasMatchingSkillVersionMeta( - context: RegistryArtifactBackupContext, - item: Parameters[1], -) { - const meta = await fetchSkillVersionBackupMeta( - context, - item.ownerHandle, - item.slug, - item.version, - ); - return meta?.version === item.version && meta.restore.versionId === item.versionId; -} - -async function hasMatchingPackageReleaseMeta( - context: RegistryArtifactBackupContext, - item: Extract, -) { - const meta = await fetchPackageReleaseBackupMeta( - context, - item.ownerHandle, - item.normalizedName, - item.version, - ); - return ( - meta?.restore?.releaseId === item.releaseId && meta.artifact.sha256 === item.artifactSha256 - ); -} - -function estimatedRetryJobBytes(workItem: RetryJobWorkItem) { - if (workItem.kind === "packageRelease" || workItem.kind === "skillVersion") { - return workItem.estimatedBytes; - } - return 0; -} - -function estimateSkillBackupBytes(item: Parameters[1]) { - const total = item.files.reduce((sum, file) => sum + file.size, 0); - return total || UNKNOWN_SKILL_ARTIFACT_BYTES; -} - -function chunkRetryJobGroups(groups: RetryJobGroup[]) { - const chunks: RetryJobGroup[][] = []; - let current: RetryJobGroup[] = []; - let currentBytes = 0; - - for (const group of groups) { - const groupBytes = group.estimatedBytes; - const wouldExceedRootLimit = current.length >= MAX_PARALLEL_RETRY_ROOTS; - const wouldExceedByteLimit = - current.length > 0 && currentBytes + groupBytes > MAX_PARALLEL_RETRY_ARTIFACT_BYTES; - if (wouldExceedRootLimit || wouldExceedByteLimit) { - chunks.push(current); - current = []; - currentBytes = 0; - } - current.push(group); - currentBytes += groupBytes; - } - - if (current.length > 0) { - chunks.push(current); - } - return chunks; -} - -async function getPackageBackupItemForRelease( - ctx: ActionCtx, - releaseId: Id<"packageReleases">, -): Promise | null> { - const release = (await ctx.runQuery(internal.packages.getReleaseByIdInternal, { - releaseId, - })) as Doc<"packageReleases"> | null; - if (!release || release.softDeletedAt) return null; - const pkg = (await ctx.runQuery(internal.packages.getPackageByIdInternal, { - packageId: release.packageId, - })) as Doc<"packages"> | null; - if (!pkg || pkg.softDeletedAt || !release.clawpackStorageId) return null; - if (pkg.family !== "code-plugin" && pkg.family !== "bundle-plugin") return null; - const owner = pkg.ownerPublisherId - ? ((await ctx.runQuery(internal.publishers.getByIdInternal, { - publisherId: pkg.ownerPublisherId, - })) as Doc<"publishers"> | null) - : ((await ctx.runQuery(internal.users.getByIdInternal, { - userId: pkg.ownerUserId, - })) as Doc<"users"> | null); - if (!owner || owner.deletedAt || owner.deactivatedAt) return null; - return { - kind: "ok", - packageId: pkg._id, - releaseId: release._id, - ownerHandle: owner.handle ?? String(pkg.ownerPublisherId ?? pkg.ownerUserId), - packageName: pkg.name, - normalizedName: pkg.normalizedName, - displayName: pkg.displayName, - family: pkg.family, - version: release.version, - isLatest: pkg.latestReleaseId === release._id, - publishedAt: release.createdAt, - artifactKind: release.artifactKind, - artifactStorageId: release.clawpackStorageId, - artifactFileName: release.npmTarballName, - artifactSha256: release.clawpackSha256, - artifactSize: release.clawpackSize, - artifactFormat: release.clawpackFormat, - npmIntegrity: release.npmIntegrity, - npmShasum: release.npmShasum, - npmUnpackedSize: release.npmUnpackedSize, - npmFileCount: release.npmFileCount, - runtimeId: release.runtimeId, - sourceRepo: release.sourceRepo, - compatibility: release.compatibility, - extractedPackageJson: release.extractedPackageJson, - extractedPluginManifest: release.extractedPluginManifest, - normalizedBundleManifest: release.normalizedBundleManifest, - files: release.files.map((file) => ({ path: file.path, size: file.size, sha256: file.sha256 })), - }; -} - -async function getSkillBackupItemForVersion( - ctx: ActionCtx, - versionId: Id<"skillVersions">, -): Promise[1] | null> { - const version = (await ctx.runQuery(internal.skills.getVersionByIdInternal, { - versionId, - })) as Doc<"skillVersions"> | null; - if (!version || version.softDeletedAt) return null; - const skill = (await ctx.runQuery(internal.skills.getSkillByIdInternal, { - skillId: version.skillId, - })) as Doc<"skills"> | null; - if (!isPublicSkillDoc(skill)) return null; - const owner = skill.ownerPublisherId - ? ((await ctx.runQuery(internal.publishers.getByIdInternal, { - publisherId: skill.ownerPublisherId, - })) as Doc<"publishers"> | null) - : ((await ctx.runQuery(internal.users.getByIdInternal, { - userId: skill.ownerUserId, - })) as Doc<"users"> | null); - if (!owner || owner.deletedAt || owner.deactivatedAt) return null; - return { - skillId: skill._id, - versionId: version._id, - slug: skill.slug, - version: version.version, - isLatest: skill.latestVersionId === version._id, - displayName: skill.displayName, - ownerHandle: owner.handle ?? String(skill.ownerPublisherId ?? skill.ownerUserId), - files: version.files, - publishedAt: version.createdAt, - }; -} - -async function alertOnUnhealthyBackupBacklog( - ctx: ActionCtx, - stats: RegistryArtifactBackupSyncStats, -) { - const health = (await ctx.runQuery( - internal.registryArtifactBackups.getRegistryArtifactBackupHealthInternal, - { - staleAfterMs: STALE_BACKUP_JOB_MS, - }, - )) as { stale: number; exhausted: number }; - stats.staleJobs = health.stale; - stats.exhaustedJobs = health.exhausted; - if (health.stale > 0 || health.exhausted > 0) { - console.error("Registry artifact backup backlog unhealthy", health); - } -} - -export async function processRegistryArtifactBackupRetriesInternalHandler( - ctx: ActionCtx, - args: ProcessRegistryArtifactBackupRetriesInternalArgs, -): Promise { - const stats = initialRegistryArtifactBackupSyncStats(); - - if (!isRegistryArtifactBackupConfigured()) { - return { stats }; - } - - const context = getRegistryArtifactBackupContext(); - if (args.dryRun) { - await processDueRegistryArtifactBackupJobs(ctx, context, true, stats); - await alertOnUnhealthyBackupBacklog(ctx, stats); - return { stats }; - } - - const token = `retry-${Date.now()}-${Math.random().toString(36).slice(2)}`; - const lease = (await ctx.runMutation( - internal.registryArtifactBackups.tryAcquireRegistryArtifactBackupRetryLeaseInternal, - { - token, - ttlMs: RETRY_LEASE_TTL_MS, - }, - )) as { acquired: boolean }; - if (!lease.acquired) return { stats }; - - try { - await processDueRegistryArtifactBackupJobs(ctx, context, false, stats, { - forceDue: args.forceDue, - leaseToken: token, - }); - await alertOnUnhealthyBackupBacklog(ctx, stats); - } finally { - await ctx.runMutation( - internal.registryArtifactBackups.releaseRegistryArtifactBackupRetryLeaseInternal, - { - token, - }, - ); - } - return { stats }; -} - -export async function processRegistryArtifactBackupQueueInternalHandler( - ctx: ActionCtx, - args: ProcessRegistryArtifactBackupQueueInternalArgs, -): Promise { - const stats = initialRegistryArtifactBackupSyncStats(); - - if (!isRegistryArtifactBackupConfigured()) { - return { stats }; - } - - const context = getRegistryArtifactBackupContext(); - if (args.dryRun) { - await processDueRegistryArtifactBackupJobs(ctx, context, true, stats); - await alertOnUnhealthyBackupBacklog(ctx, stats); - return { stats }; - } - - const workerLabel = args.workerId ?? "registry-backup-worker"; - const leaseToken = `${workerLabel}-${Date.now()}-${Math.random().toString(36).slice(2)}`; - const jobs = (await ctx.runMutation( - internal.registryArtifactBackups.claimRegistryArtifactBackupJobsInternal, - { - forceDue: args.forceDue, - leaseToken, - leaseTtlMs: args.leaseTtlMs, - limit: args.limit, - includeExhaustedRepair: true, - maxRepairAttempts: MAX_RETRY_REPAIR_ATTEMPTS, - }, - )) as Array>; - - const groups = await groupRetryJobsByRoot(ctx, jobs); - const chunks = chunkRetryJobGroups(groups); - for (const chunk of chunks) { - const results = await Promise.all( - chunk.map((group) => processRetryJobGroup(ctx, context, group, leaseToken)), - ); - for (const result of results) { - stats.retryJobsProcessed += result.processed; - stats.retryJobsSucceeded += result.succeeded; - stats.retryJobsFailed += result.failed; - } - } - await alertOnUnhealthyBackupBacklog(ctx, stats); - return { stats }; -} - -export const seedRegistryArtifactBackupsInternal = internalAction({ - args: { - dryRun: v.optional(v.boolean()), - batchSize: v.optional(v.number()), - maxBatches: v.optional(v.number()), - queueOnly: v.optional(v.boolean()), - }, - handler: seedRegistryArtifactBackupsInternalHandler, -}); - -export const processRegistryArtifactBackupRetriesInternal = internalAction({ - args: { - dryRun: v.optional(v.boolean()), - forceDue: v.optional(v.boolean()), - }, - handler: processRegistryArtifactBackupRetriesInternalHandler, -}); - -export const processRegistryArtifactBackupQueueInternal = internalAction({ - args: { - dryRun: v.optional(v.boolean()), - forceDue: v.optional(v.boolean()), - limit: v.optional(v.number()), - leaseTtlMs: v.optional(v.number()), - workerId: v.optional(v.string()), - }, - handler: processRegistryArtifactBackupQueueInternalHandler, -}); - -function clampInt(value: number, min: number, max: number) { - return Math.max(min, Math.min(max, Math.floor(value))); -} - -function errorMessage(error: unknown) { - return error instanceof Error ? error.message : String(error); -} - -function initialRegistryArtifactBackupSyncStats(): RegistryArtifactBackupSyncStats { - return { - skillsScanned: 0, - skillsSkipped: 0, - skillsBackedUp: 0, - skillsMissingVersion: 0, - skillsMissingOwner: 0, - packagesScanned: 0, - packagesSkipped: 0, - packagesBackedUp: 0, - packagesMissingArtifact: 0, - packagesMissingPackage: 0, - packagesMissingOwner: 0, - skillsEnqueued: 0, - packagesEnqueued: 0, - retryJobsProcessed: 0, - retryJobsSucceeded: 0, - retryJobsFailed: 0, - staleJobs: 0, - exhaustedJobs: 0, - errors: 0, - }; -} diff --git a/convex/registryArtifactRestore.test.ts b/convex/registryArtifactRestore.test.ts deleted file mode 100644 index c48444db..00000000 --- a/convex/registryArtifactRestore.test.ts +++ /dev/null @@ -1,283 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; -import { restoreSkillFromBackup } from "./registryArtifactRestore"; - -const registryBackupMocks = vi.hoisted(() => ({ - fetchSkillVersionBackupMeta: vi.fn(), - getRegistryArtifactBackupContext: vi.fn(), - isRegistryArtifactBackupConfigured: vi.fn(), - normalizeOwner: vi.fn((value: string) => value.toLowerCase()), - readRegistryArtifactBackupObject: vi.fn(), -})); - -const skillPublishMocks = vi.hoisted(() => ({ - publishVersionForUser: vi.fn(), -})); - -vi.mock("./lib/registryArtifactBackup", () => registryBackupMocks); -vi.mock("./lib/skillPublish", () => skillPublishMocks); - -const restoreHandler = (restoreSkillFromBackup as unknown as { _handler: Function })._handler; - -describe("restoreSkillFromBackup", () => { - beforeEach(() => { - vi.resetAllMocks(); - registryBackupMocks.normalizeOwner.mockImplementation((value: string) => value.toLowerCase()); - registryBackupMocks.getRegistryArtifactBackupContext.mockReturnValue({ - endpoint: "https://account.r2.cloudflarestorage.com", - bucket: "clawhub-registry-backup", - accessKeyId: "access-key", - secretAccessKey: "secret-key", - region: "auto", - skillsRoot: "skills", - packagesRoot: "packages", - }); - registryBackupMocks.isRegistryArtifactBackupConfigured.mockReturnValue(true); - }); - - it("blocks restore when the current slug row is not public", async () => { - const result = await restoreHandler( - { - runQuery: vi - .fn() - .mockResolvedValueOnce({ _id: "users:admin", role: "admin" }) - .mockResolvedValueOnce({ - _id: "skills:hidden", - ownerUserId: "users:owner", - slug: "demo-skill", - softDeletedAt: undefined, - moderationStatus: "hidden", - }), - } as never, - { - actorUserId: "users:admin", - ownerHandle: "alice", - ownerUserId: "users:owner", - slug: "demo-skill", - version: "1.0.0", - }, - ); - - expect(result).toEqual({ - slug: "demo-skill", - status: "error", - detail: "Existing skill is not public; restore blocked", - }); - expect(registryBackupMocks.fetchSkillVersionBackupMeta).not.toHaveBeenCalled(); - }); - - it("requires an explicit version before any forced slug eviction", async () => { - const runMutation = vi.fn(); - const result = await restoreHandler( - { - runQuery: vi.fn().mockResolvedValueOnce({ _id: "users:admin", role: "admin" }), - runMutation, - } as never, - { - actorUserId: "users:admin", - ownerHandle: "alice", - ownerUserId: "users:owner", - slug: "demo-skill", - forceOverwriteSquatter: true, - }, - ); - - expect(result).toEqual({ - slug: "demo-skill", - status: "no_backup", - detail: "Restore requires an explicit backup version", - }); - expect(runMutation).not.toHaveBeenCalled(); - expect(registryBackupMocks.fetchSkillVersionBackupMeta).not.toHaveBeenCalled(); - }); - - it("validates the requested version before forced slug eviction", async () => { - registryBackupMocks.fetchSkillVersionBackupMeta.mockResolvedValueOnce(null); - const runMutation = vi.fn(); - const result = await restoreHandler( - { - runQuery: vi - .fn() - .mockResolvedValueOnce({ _id: "users:admin", role: "admin" }) - .mockResolvedValueOnce({ - _id: "skills:squatter", - ownerUserId: "users:other", - slug: "demo-skill", - softDeletedAt: undefined, - moderationStatus: "active", - }), - runMutation, - } as never, - { - actorUserId: "users:admin", - ownerHandle: "alice", - ownerUserId: "users:owner", - slug: "demo-skill", - version: "typo-version", - forceOverwriteSquatter: true, - }, - ); - - expect(result).toEqual({ - slug: "demo-skill", - status: "no_backup", - detail: "No version backup found", - }); - expect(runMutation).not.toHaveBeenCalled(); - }); - - it("reactivates the same owner's soft-deleted skill row without republishing a duplicate version", async () => { - registryBackupMocks.fetchSkillVersionBackupMeta.mockResolvedValueOnce({ - version: "1.0.0", - displayName: "Demo Skill", - metadata: { - files: [ - { - path: "SKILL.md", - size: 5, - sha256: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", - }, - ], - }, - }); - registryBackupMocks.readRegistryArtifactBackupObject.mockResolvedValueOnce( - new TextEncoder().encode("hello"), - ); - const storage = { store: vi.fn().mockResolvedValue("storage:restored") }; - const runMutation = vi.fn(); - - const result = await restoreHandler( - { - runQuery: vi - .fn() - .mockResolvedValueOnce({ _id: "users:admin", role: "admin" }) - .mockResolvedValueOnce({ - _id: "skills:deleted", - ownerUserId: "users:owner", - slug: "demo-skill", - softDeletedAt: 123, - moderationStatus: "active", - }) - .mockResolvedValueOnce({ - _id: "skillVersions:existing", - skillId: "skills:deleted", - version: "1.0.0", - softDeletedAt: undefined, - }), - runMutation, - storage, - } as never, - { - actorUserId: "users:admin", - ownerHandle: "alice", - ownerUserId: "users:owner", - slug: "demo-skill", - version: "1.0.0", - }, - ); - - expect(result).toEqual({ slug: "demo-skill", status: "restored" }); - expect(skillPublishMocks.publishVersionForUser).not.toHaveBeenCalled(); - expect(runMutation).toHaveBeenNthCalledWith( - 1, - expect.anything(), - expect.objectContaining({ - userId: "users:admin", - slug: "demo-skill", - deleted: false, - }), - ); - expect(runMutation).toHaveBeenNthCalledWith( - 2, - expect.anything(), - expect.objectContaining({ - actorUserId: "users:admin", - skillId: "skills:deleted", - versionId: "skillVersions:existing", - files: [ - expect.objectContaining({ - path: "SKILL.md", - size: 5, - sha256: "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", - storageId: "storage:restored", - }), - ], - }), - ); - }); - - it("fails restore when a manifest file is missing from backup storage", async () => { - registryBackupMocks.fetchSkillVersionBackupMeta.mockResolvedValueOnce({ - version: "1.0.0", - displayName: "Demo Skill", - metadata: { - files: [{ path: "SKILL.md", size: 5, sha256: "2cf24dba5fb0a30e26e83b2ac5b9e29e" }], - }, - }); - registryBackupMocks.readRegistryArtifactBackupObject.mockResolvedValueOnce(null); - const storage = { store: vi.fn() }; - - const result = await restoreHandler( - { - runQuery: vi - .fn() - .mockResolvedValueOnce({ _id: "users:admin", role: "admin" }) - .mockResolvedValueOnce(null), - storage, - } as never, - { - actorUserId: "users:admin", - ownerHandle: "alice", - ownerUserId: "users:owner", - slug: "demo-skill", - version: "1.0.0", - }, - ); - - expect(result).toEqual({ - slug: "demo-skill", - status: "error", - detail: "Backup missing file SKILL.md", - }); - expect(storage.store).not.toHaveBeenCalled(); - expect(skillPublishMocks.publishVersionForUser).not.toHaveBeenCalled(); - }); - - it("fails restore when a manifest file checksum does not match backup storage", async () => { - registryBackupMocks.fetchSkillVersionBackupMeta.mockResolvedValueOnce({ - version: "1.0.0", - displayName: "Demo Skill", - metadata: { - files: [{ path: "SKILL.md", size: 5, sha256: "wrong-sha256" }], - }, - }); - registryBackupMocks.readRegistryArtifactBackupObject.mockResolvedValueOnce( - new TextEncoder().encode("hello"), - ); - const storage = { store: vi.fn() }; - - const result = await restoreHandler( - { - runQuery: vi - .fn() - .mockResolvedValueOnce({ _id: "users:admin", role: "admin" }) - .mockResolvedValueOnce(null), - storage, - } as never, - { - actorUserId: "users:admin", - ownerHandle: "alice", - ownerUserId: "users:owner", - slug: "demo-skill", - version: "1.0.0", - }, - ); - - expect(result).toEqual({ - slug: "demo-skill", - status: "error", - detail: "Backup file checksum mismatch for SKILL.md", - }); - expect(storage.store).not.toHaveBeenCalled(); - expect(skillPublishMocks.publishVersionForUser).not.toHaveBeenCalled(); - }); -}); diff --git a/convex/registryArtifactRestore.ts b/convex/registryArtifactRestore.ts deleted file mode 100644 index bfe9f6ee..00000000 --- a/convex/registryArtifactRestore.ts +++ /dev/null @@ -1,350 +0,0 @@ -"use node"; - -import { v } from "convex/values"; -import { internal } from "./_generated/api"; -import type { Doc, Id } from "./_generated/dataModel"; -import { internalAction } from "./functions"; -import { assertAdmin } from "./lib/access"; -import { guessContentTypeForPath } from "./lib/contentTypes"; -import { isPublicSkillDoc } from "./lib/globalStats"; -import { - fetchSkillVersionBackupMeta, - getRegistryArtifactBackupContext, - isRegistryArtifactBackupConfigured, - normalizeOwner, - readRegistryArtifactBackupObject, -} from "./lib/registryArtifactBackup"; -import { publishVersionForUser } from "./lib/skillPublish"; -import { validateFilePath } from "./lib/skillZip"; - -type RestoreResult = { - slug: string; - status: "restored" | "slug_conflict" | "already_exists" | "no_backup" | "error"; - detail?: string; -}; - -type BulkRestoreResult = { - results: RestoreResult[]; - totalRestored: number; - totalConflicts: number; - totalSkipped: number; - totalErrors: number; -}; - -type SkillBackupMeta = NonNullable>>; - -type VerifiedSkillBackup = { - meta: SkillBackupMeta; - files: Array<{ - path: string; - size: number; - sha256: string; - contentType: string; - content: Uint8Array; - }>; -}; - -/** - * Admin-only: restore a single skill from registry artifact backup. - * Reads backed-up objects and re-creates the skill in the database. - */ -export const restoreSkillFromBackup = internalAction({ - args: { - actorUserId: v.id("users"), - ownerHandle: v.string(), - ownerUserId: v.id("users"), - slug: v.string(), - version: v.optional(v.string()), - forceOverwriteSquatter: v.optional(v.boolean()), - }, - handler: async (ctx, args): Promise => { - try { - const actor = await ctx.runQuery(internal.users.getByIdInternal, { - userId: args.actorUserId, - }); - if (!actor || actor.deletedAt || actor.deactivatedAt) { - return { slug: args.slug, status: "error", detail: "Actor not found" }; - } - assertAdmin(actor as Doc<"users">); - - if (!isRegistryArtifactBackupConfigured()) { - return { slug: args.slug, status: "error", detail: "Registry backup not configured" }; - } - - const backupContext = getRegistryArtifactBackupContext(); - if (!args.version) { - return { - slug: args.slug, - status: "no_backup", - detail: "Restore requires an explicit backup version", - }; - } - - let verifiedBackup: VerifiedSkillBackup | null = null; - const loadVerifiedBackup = async (): Promise => { - if (verifiedBackup) return verifiedBackup; - const meta = await fetchSkillVersionBackupMeta( - backupContext, - args.ownerHandle, - args.slug, - args.version!, - ); - if (!meta) { - return { slug: args.slug, status: "no_backup", detail: "No version backup found" }; - } - - const backupFiles = meta.metadata.files; - if (backupFiles.length === 0) { - return { slug: args.slug, status: "no_backup", detail: "Backup has no files" }; - } - - const owner = normalizeOwner(args.ownerHandle); - const files: VerifiedSkillBackup["files"] = []; - for (const file of backupFiles) { - if (!validateFilePath(file.path)) { - return { slug: args.slug, status: "error", detail: "Backup contains unsafe file path" }; - } - const fileContent = await readRegistryArtifactBackupObject( - backupContext, - `${backupContext.skillsRoot}/${owner}/${args.slug}/${encodeBackupPathSegment( - meta.version, - )}/${file.path}`, - ); - if (!fileContent) { - return { - slug: args.slug, - status: "error", - detail: `Backup missing file ${file.path}`, - }; - } - if (fileContent.byteLength !== file.size) { - return { - slug: args.slug, - status: "error", - detail: `Backup file size mismatch for ${file.path}`, - }; - } - - const sha256 = await sha256Hex(fileContent); - if (sha256 !== file.sha256) { - return { - slug: args.slug, - status: "error", - detail: `Backup file checksum mismatch for ${file.path}`, - }; - } - - files.push({ - path: file.path, - size: fileContent.byteLength, - sha256, - contentType: file.contentType ?? guessContentTypeForPath(file.path), - content: fileContent, - }); - } - - verifiedBackup = { meta, files }; - return verifiedBackup; - }; - - // Check if skill already exists in the DB - const existingSkill = (await ctx.runQuery( - internal.skills.getSkillBySlugIncludingSoftDeletedInternal, - { - slug: args.slug, - }, - )) as Doc<"skills"> | null; - - const sameOwnerSoftDeletedSkill = - existingSkill?.ownerUserId === args.ownerUserId && existingSkill.softDeletedAt - ? existingSkill - : null; - - if (existingSkill) { - const sameOwner = existingSkill.ownerUserId === args.ownerUserId; - if (sameOwner && existingSkill.softDeletedAt) { - // Continue: if the backed-up version already exists, restore by - // reactivating that row instead of republishing a duplicate version. - } else if (!isPublicSkillDoc(existingSkill)) { - return { - slug: args.slug, - status: "error", - detail: "Existing skill is not public; restore blocked", - }; - } else if (sameOwner) { - return { - slug: args.slug, - status: "already_exists", - detail: "Skill already owned by user", - }; - } else if (!args.forceOverwriteSquatter) { - return { - slug: args.slug, - status: "slug_conflict", - detail: `Slug occupied by another user. Set forceOverwriteSquatter=true to reclaim.`, - }; - } else { - const backup = await loadVerifiedBackup(); - if ("status" in backup) return backup; - // Free the slug in-transaction by renaming the squatter, then enqueue cleanup. - await ctx.runMutation( - internal.registryArtifactRestoreMutations.evictSquatterSkillForRestoreInternal, - { - actorUserId: args.actorUserId, - slug: args.slug, - rightfulOwnerUserId: args.ownerUserId, - }, - ); - } - } - - const backup = verifiedBackup ?? (await loadVerifiedBackup()); - if ("status" in backup) return backup; - const { meta } = backup; - - // Download and store each file in Convex storage - const storedFiles: Array<{ - path: string; - size: number; - storageId: Id<"_storage">; - sha256: string; - contentType: string; - }> = []; - - for (const file of backup.files) { - const blob = new Blob([Buffer.from(file.content)], { type: file.contentType }); - const storageId = await ctx.storage.store(blob); - - storedFiles.push({ - path: file.path, - size: file.size, - storageId, - sha256: file.sha256, - contentType: file.contentType, - }); - } - - if (storedFiles.length === 0) { - return { slug: args.slug, status: "error", detail: "Could not download any backup files" }; - } - - if (sameOwnerSoftDeletedSkill) { - const existingVersion = (await ctx.runQuery( - internal.skills.getVersionBySkillAndVersionInternal, - { - skillId: sameOwnerSoftDeletedSkill._id, - version: meta.version, - }, - )) as Doc<"skillVersions"> | null; - if (existingVersion && !existingVersion.softDeletedAt) { - await ctx.runMutation(internal.skills.setSkillSoftDeletedInternal, { - userId: args.actorUserId, - slug: args.slug, - deleted: false, - reason: "Restored from registry artifact backup", - }); - await ctx.runMutation( - internal.registryArtifactRestoreMutations.refreshRestoredSkillVersionInternal, - { - actorUserId: args.actorUserId, - skillId: sameOwnerSoftDeletedSkill._id, - versionId: existingVersion._id, - files: storedFiles, - }, - ); - return { slug: args.slug, status: "restored" }; - } - } - - await publishVersionForUser( - ctx, - args.ownerUserId, - { - slug: args.slug, - displayName: meta.displayName, - version: meta.version, - changelog: "Restored from registry artifact backup", - files: storedFiles, - }, - { - bypassGitHubAccountAge: true, - bypassNewSkillRateLimit: true, - bypassQualityGate: true, - skipBackup: true, - skipWebhook: true, - }, - ); - - return { slug: args.slug, status: "restored" }; - } catch (error) { - const message = error instanceof Error ? error.message : "Unknown error"; - console.error(`[restore] Failed to restore ${args.slug}:`, message); - return { slug: args.slug, status: "error", detail: message }; - } - }, -}); - -/** - * Admin-only: bulk restore all skills for a user from registry artifact backup. - */ -export const restoreUserSkillsFromBackup = internalAction({ - args: { - actorUserId: v.id("users"), - ownerHandle: v.string(), - ownerUserId: v.id("users"), - slugs: v.array(v.string()), - versionsBySlug: v.optional(v.record(v.string(), v.string())), - forceOverwriteSquatter: v.optional(v.boolean()), - }, - handler: async (ctx, args): Promise => { - const results: RestoreResult[] = []; - let totalRestored = 0; - let totalConflicts = 0; - let totalSkipped = 0; - let totalErrors = 0; - - for (const slug of args.slugs) { - const result = (await ctx.runAction(internal.registryArtifactRestore.restoreSkillFromBackup, { - actorUserId: args.actorUserId, - ownerHandle: args.ownerHandle, - ownerUserId: args.ownerUserId, - slug, - version: args.versionsBySlug?.[slug], - forceOverwriteSquatter: args.forceOverwriteSquatter, - })) as RestoreResult; - - results.push(result); - - switch (result.status) { - case "restored": - totalRestored += 1; - break; - case "slug_conflict": - totalConflicts += 1; - break; - case "already_exists": - case "no_backup": - totalSkipped += 1; - break; - case "error": - totalErrors += 1; - break; - } - } - - return { results, totalRestored, totalConflicts, totalSkipped, totalErrors }; - }, -}); - -async function sha256Hex(bytes: Uint8Array) { - const { createHash } = await import("node:crypto"); - const hash = createHash("sha256"); - hash.update(bytes); - return hash.digest("hex"); -} - -function encodeBackupPathSegment(value: string) { - return encodeURIComponent(value.trim()).replace(/\./g, "%2E"); -} - -// guessContentTypeForPath in lib/contentTypes.ts diff --git a/convex/registryArtifactRestoreMutations.ts b/convex/registryArtifactRestoreMutations.ts deleted file mode 100644 index 5b6e1efc..00000000 --- a/convex/registryArtifactRestoreMutations.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { v } from "convex/values"; -import { internal } from "./_generated/api"; -import type { Doc } from "./_generated/dataModel"; -import { internalMutation } from "./functions"; -import { assertAdmin } from "./lib/access"; - -const restoredSkillFileValidator = v.object({ - path: v.string(), - size: v.number(), - storageId: v.id("_storage"), - sha256: v.string(), - contentType: v.optional(v.string()), -}); - -export const evictSquatterSkillForRestoreInternal = internalMutation({ - args: { - actorUserId: v.id("users"), - slug: v.string(), - rightfulOwnerUserId: v.id("users"), - }, - handler: async (ctx, args) => { - const actor = await ctx.db.get(args.actorUserId); - if (!actor || actor.deletedAt || actor.deactivatedAt) throw new Error("Actor not found"); - assertAdmin(actor); - - const slug = args.slug.trim().toLowerCase(); - if (!slug) throw new Error("Slug required"); - - const now = Date.now(); - - const existingSkill = await ctx.db - .query("skills") - .withIndex("by_slug", (q) => q.eq("slug", slug)) - .unique(); - if (!existingSkill) return { ok: true as const, action: "noop" as const }; - if (existingSkill.ownerUserId === args.rightfulOwnerUserId) { - return { ok: true as const, action: "already_owned" as const }; - } - - const evictedSlug = buildEvictedSlug(slug, now); - - // Free the slug immediately (same transaction) by renaming the squatter's skill. - await ctx.db.patch(existingSkill._id, { - slug: evictedSlug, - softDeletedAt: now, - hiddenAt: existingSkill.hiddenAt ?? now, - hiddenBy: existingSkill.hiddenBy ?? actor._id, - updatedAt: now, - }); - - // Remove from vector search ASAP. - const embeddings = await ctx.db - .query("skillEmbeddings") - .withIndex("by_skill", (q) => q.eq("skillId", existingSkill._id)) - .collect(); - for (const embedding of embeddings) { - await ctx.db.patch(embedding._id, { - visibility: "deleted", - updatedAt: now, - }); - } - - // Cleanup the rest asynchronously (versions, fingerprints, installs, etc.) - await ctx.scheduler.runAfter(0, internal.skills.hardDeleteInternal, { - skillId: existingSkill._id, - actorUserId: actor._id, - phase: "versions", - }); - - await ctx.db.insert("auditLogs", { - actorUserId: actor._id, - action: "slug.reclaim.sync", - targetType: "skill", - targetId: existingSkill._id, - metadata: { - slug, - evictedSlug, - squatterUserId: existingSkill.ownerUserId, - rightfulOwnerUserId: args.rightfulOwnerUserId, - reason: "Synchronous eviction during registry artifact restore", - }, - createdAt: now, - }); - - return { ok: true as const, action: "evicted" as const, evictedSlug }; - }, -}); - -export const refreshRestoredSkillVersionInternal = internalMutation({ - args: { - actorUserId: v.id("users"), - skillId: v.id("skills"), - versionId: v.id("skillVersions"), - files: v.array(restoredSkillFileValidator), - }, - handler: async (ctx, args) => { - const actor = await ctx.db.get(args.actorUserId); - if (!actor || actor.deletedAt || actor.deactivatedAt) throw new Error("Actor not found"); - assertAdmin(actor); - - const [skill, version] = await Promise.all([ - ctx.db.get(args.skillId), - ctx.db.get(args.versionId), - ]); - if (!skill) throw new Error("Skill not found"); - if (!version || version.skillId !== skill._id || version.softDeletedAt) { - throw new Error("Skill version not found"); - } - - const now = Date.now(); - await ctx.db.patch(version._id, { - files: args.files, - }); - await ctx.db.patch(skill._id, { - latestVersionId: version._id, - latestVersionSummary: latestVersionSummaryFromVersion(version), - tags: { ...skill.tags, latest: version._id }, - softDeletedAt: undefined, - moderationStatus: "active", - hiddenAt: undefined, - hiddenBy: undefined, - unpublishedSlugReservedUntil: undefined, - unpublishedSlugReleasedAt: undefined, - unpublishedOriginalSlug: undefined, - updatedAt: now, - }); - - await ctx.db.insert("auditLogs", { - actorUserId: actor._id, - action: "skill.restore.registry_artifact", - targetType: "skill", - targetId: skill._id, - metadata: { - slug: skill.slug, - version: version.version, - versionId: version._id, - }, - createdAt: now, - }); - - return { ok: true as const, skillId: skill._id, versionId: version._id }; - }, -}); - -function buildEvictedSlug(slug: string, now: number) { - const suffix = now.toString(36); - return `${slug}-evicted-${suffix}`; -} - -function latestVersionSummaryFromVersion( - version: Pick< - Doc<"skillVersions">, - "version" | "createdAt" | "changelog" | "changelogSource" | "parsed" - >, -): NonNullable["latestVersionSummary"]> { - return { - version: version.version, - createdAt: version.createdAt, - changelog: version.changelog, - changelogSource: version.changelogSource, - description: frontmatterString(version.parsed?.frontmatter?.description), - clawdis: version.parsed?.clawdis, - }; -} - -function frontmatterString(value: unknown) { - return typeof value === "string" ? value.trim() || undefined : undefined; -} diff --git a/convex/skills.ownership.test.ts b/convex/skills.ownership.test.ts index e5780724..f2bfa2c5 100644 --- a/convex/skills.ownership.test.ts +++ b/convex/skills.ownership.test.ts @@ -1205,11 +1205,10 @@ describe("skills ownership", () => { createdAt: 1_700_000_000_000, softDeletedAt: undefined, }; - const runAfter = vi.fn(async () => {}); const result = await renameOwnedSkillInternalHandler( { - scheduler: { runAfter }, + scheduler: { runAfter: vi.fn(async () => {}) }, db: { normalizeId: vi.fn(() => null), get: vi.fn(async (id: string) => { @@ -1301,18 +1300,6 @@ describe("skills ownership", () => { ownerPublisherId: "publishers:org", }), ); - expect(runAfter).toHaveBeenCalledWith( - 0, - expect.anything(), - expect.objectContaining({ - skillId: "skills:source", - versionId: "skillVersions:latest", - slug: "new-name", - version: "1.0.0", - isLatest: true, - ownerHandle: "org", - }), - ); }); it("allows publisher admins to move a skill into an org they administer", async () => { diff --git a/convex/skills.ts b/convex/skills.ts index c9f799c9..780d5686 100644 --- a/convex/skills.ts +++ b/convex/skills.ts @@ -9883,113 +9883,9 @@ export const updateTags = mutation({ if (latestEntry) { await setSkillEmbeddingsLatestVersion(ctx, skill._id, latestEntry.versionId, now); } - - if (latestEntry && latestEntry.versionId !== skill.latestVersionId) { - const version = versionsById.get(latestEntry.versionId); - const owner = await getOwnerPublisher(ctx, { - ownerPublisherId: skill.ownerPublisherId, - ownerUserId: skill.ownerUserId, - }); - if (version && owner) { - await scheduleRegistryArtifactSkillVersionBackupRefresh(ctx, { - skill, - version, - isLatest: true, - ownerHandle: owner.handle ?? String(skill.ownerPublisherId ?? skill.ownerUserId), - logContext: "latest refresh", - }); - } - } }, }); -async function scheduleRegistryArtifactSkillVersionBackupRefresh( - ctx: MutationCtx, - params: { - skill: Pick, "_id" | "slug" | "displayName" | "ownerUserId" | "ownerPublisherId">; - version: Doc<"skillVersions">; - ownerHandle: string; - isLatest: boolean; - logContext: string; - }, -) { - try { - await ctx.scheduler.runAfter( - 0, - internal.registryArtifactBackupsNode.backupSkillForPublishInternal, - { - skillId: params.skill._id, - versionId: params.version._id, - slug: params.skill.slug, - version: params.version.version, - isLatest: params.isLatest, - displayName: params.skill.displayName, - ownerHandle: params.ownerHandle, - files: params.version.files, - publishedAt: params.version.createdAt, - }, - ); - } catch (error) { - console.error(`registry artifact backup ${params.logContext} scheduling failed`, error); - await enqueueSkillVersionBackupRetryJob(ctx, { - versionId: params.version._id, - error: errorMessageForBackupJob(error), - }).catch((enqueueError) => { - console.error( - `registry artifact backup ${params.logContext} retry enqueue failed`, - enqueueError, - ); - }); - } -} - -async function enqueueSkillVersionBackupRetryJob( - ctx: Pick, - args: { versionId: Id<"skillVersions">; error?: string }, -) { - const now = Date.now(); - const lastError = truncateBackupJobError(args.error); - const existing = await ctx.db - .query("registryArtifactBackupJobs") - .withIndex("by_skill_version", (q) => q.eq("skillVersionId", args.versionId)) - .unique(); - if (existing) { - await ctx.db.patch(existing._id, { - status: "pending", - reason: "retry", - attempts: 0, - lastError, - nextRunAt: now, - createdAt: now, - updatedAt: now, - exhaustedAt: undefined, - completedAt: undefined, - }); - return; - } - await ctx.db.insert("registryArtifactBackupJobs", { - targetKind: "skillVersion", - skillVersionId: args.versionId, - packageReleaseId: undefined, - status: "pending", - reason: "retry", - attempts: 0, - nextRunAt: now, - lastError, - createdAt: now, - updatedAt: now, - }); -} - -function errorMessageForBackupJob(error: unknown) { - return error instanceof Error ? error.message : String(error); -} - -function truncateBackupJobError(error: string | undefined) { - if (!error) return undefined; - return error.length > 4000 ? `${error.slice(0, 3997)}...` : error; -} - export const deleteTags = mutation({ args: { skillId: v.id("skills"), @@ -10613,23 +10509,6 @@ async function renameOwnedSkillByActor( createdAt: now, }); - if (skill.latestVersionId) { - const latestVersion = await ctx.db.get(skill.latestVersionId); - const owner = await getOwnerPublisher(ctx, { - ownerPublisherId: skill.ownerPublisherId, - ownerUserId: skill.ownerUserId, - }); - if (latestVersion && !latestVersion.softDeletedAt && owner) { - await scheduleRegistryArtifactSkillVersionBackupRefresh(ctx, { - skill: { ...skill, slug: newSlug }, - version: latestVersion, - isLatest: true, - ownerHandle: owner.handle ?? String(skill.ownerPublisherId ?? skill.ownerUserId), - logContext: "rename refresh", - }); - } - } - return { ok: true as const, slug: newSlug, previousSlug: skill.slug }; } @@ -11063,23 +10942,6 @@ export const transferSkillOwnerForUserInternal = internalMutation({ createdAt: now, }); - if (skill.latestVersionId) { - const latestVersion = await ctx.db.get(skill.latestVersionId); - if (latestVersion && !latestVersion.softDeletedAt) { - await scheduleRegistryArtifactSkillVersionBackupRefresh(ctx, { - skill: { - ...skill, - ownerUserId: nextOwner._id, - ownerPublisherId: destinationPublisher._id, - }, - version: latestVersion, - isLatest: true, - ownerHandle: destinationPublisher.handle, - logContext: "owner transfer refresh", - }); - } - } - return { ok: true as const, transferred: true as const, diff --git a/e2e/permissions.e2e.test.ts b/e2e/permissions.e2e.test.ts index 99448fa0..7a721ed3 100644 --- a/e2e/permissions.e2e.test.ts +++ b/e2e/permissions.e2e.test.ts @@ -108,7 +108,6 @@ describe("permission boundary e2e", () => { method: "DELETE", path: `${ApiRoutes.packages}/e2e-nonexistent-permission/trusted-publisher`, }, - { method: "POST", path: `${ApiRoutes.users}/restore`, body: { handle: "nobody" } }, { method: "POST", path: `${ApiRoutes.users}/reclaim`, body: { handle: "nobody" } }, { method: "POST", path: `${ApiRoutes.users}/reserve`, body: { handle: "nobody" } }, { method: "POST", path: `${ApiRoutes.users}/publisher`, body: { handle: "nobody" } }, diff --git a/specs/spec.md b/specs/spec.md index 3445a4fb..5e34b2ea 100644 --- a/specs/spec.md +++ b/specs/spec.md @@ -12,8 +12,8 @@ read_when: - Minimal, fast SPA for browsing and publishing agent skills. - Skills stored in Convex (files + metadata + versions + stats). -- GitHub OAuth login; R2/object storage backs up hosted registry artifacts for - disaster recovery. +- GitHub OAuth login; Convex backups with file storage are the source of truth + for hosted registry artifact disaster recovery. - Vector-based search over skill text + metadata. - Versioning, tags (`latest` + user tags), changelog, rollback (tag movement). - Public read access; upload requires auth. @@ -149,7 +149,7 @@ Local fixture data lives in `convex/devSeed.ts` and `fixtures/public-corpus/`. ## Vercel -- Env vars: Convex deployment URLs + GitHub OAuth client + OpenAI key (if used) + registry artifact backup R2 credentials. +- Env vars: Convex deployment URLs + GitHub OAuth client + OpenAI key (if used). - SPA feel: client-side transitions, prefetching, optimistic UI. ## Open questions (carry forward)