diff --git a/convex.json b/convex.json index 41cea9d0..33e77d1b 100644 --- a/convex.json +++ b/convex.json @@ -1,3 +1,6 @@ { - "functions": "convex" + "functions": "convex", + "node": { + "externalPackages": ["sharp"] + } } diff --git a/convex/_generated/api.d.ts b/convex/_generated/api.d.ts index 579baad0..0ddfbc2b 100644 --- a/convex/_generated/api.d.ts +++ b/convex/_generated/api.d.ts @@ -124,6 +124,8 @@ import type * as lib_skillCards from "../lib/skillCards.js"; import type * as lib_skillDownloadBackfill from "../lib/skillDownloadBackfill.js"; import type * as lib_skillFileAccess from "../lib/skillFileAccess.js"; import type * as lib_skillInstallBackfill from "../lib/skillInstallBackfill.js"; +import type * as lib_skillPresentation from "../lib/skillPresentation.js"; +import type * as lib_skillPresentationBackfill from "../lib/skillPresentationBackfill.js"; import type * as lib_skillPublish from "../lib/skillPublish.js"; import type * as lib_skillQuality from "../lib/skillQuality.js"; import type * as lib_skillSafety from "../lib/skillSafety.js"; @@ -171,6 +173,10 @@ import type * as securityDatasetNode from "../securityDatasetNode.js"; import type * as securityScan from "../securityScan.js"; import type * as securityScanDispatch from "../securityScanDispatch.js"; import type * as skillCards from "../skillCards.js"; +import type * as skillPresentationAssets from "../skillPresentationAssets.js"; +import type * as skillPresentationAssetsHttp from "../skillPresentationAssetsHttp.js"; +import type * as skillPresentationBackfill from "../skillPresentationBackfill.js"; +import type * as skillPresentationImageNode from "../skillPresentationImageNode.js"; import type * as skillStatEvents from "../skillStatEvents.js"; import type * as skillTransfers from "../skillTransfers.js"; import type * as skills from "../skills.js"; @@ -308,6 +314,8 @@ declare const fullApi: ApiFromModules<{ "lib/skillDownloadBackfill": typeof lib_skillDownloadBackfill; "lib/skillFileAccess": typeof lib_skillFileAccess; "lib/skillInstallBackfill": typeof lib_skillInstallBackfill; + "lib/skillPresentation": typeof lib_skillPresentation; + "lib/skillPresentationBackfill": typeof lib_skillPresentationBackfill; "lib/skillPublish": typeof lib_skillPublish; "lib/skillQuality": typeof lib_skillQuality; "lib/skillSafety": typeof lib_skillSafety; @@ -355,6 +363,10 @@ declare const fullApi: ApiFromModules<{ securityScan: typeof securityScan; securityScanDispatch: typeof securityScanDispatch; skillCards: typeof skillCards; + skillPresentationAssets: typeof skillPresentationAssets; + skillPresentationAssetsHttp: typeof skillPresentationAssetsHttp; + skillPresentationBackfill: typeof skillPresentationBackfill; + skillPresentationImageNode: typeof skillPresentationImageNode; skillStatEvents: typeof skillStatEvents; skillTransfers: typeof skillTransfers; skills: typeof skills; diff --git a/convex/catalogFeed.test.ts b/convex/catalogFeed.test.ts index bd3b5e86..280ba8dd 100644 --- a/convex/catalogFeed.test.ts +++ b/convex/catalogFeed.test.ts @@ -307,10 +307,19 @@ describe("catalog feed projection", () => { it("projects only published skills from verified organization publishers", async () => { const result = (await listOfficialSkillEntriesHandler( - makeCtx([makeSkill({ summary: "Deploy AIQ services.", icon: "lucide:rocket" })], { - "publishers:1": { _id: "publishers:1", kind: "org", handle: "openclaw" }, - "skillVersions:1": makeSkillVersion(), - }), + makeCtx( + [ + makeSkill({ + displayName: "🚀 Demo skill", + summary: "Deploy AIQ services.", + icon: `/api/v1/skill-icons/${"a".repeat(64)}`, + }), + ], + { + "publishers:1": { _id: "publishers:1", kind: "org", handle: "openclaw" }, + "skillVersions:1": makeSkillVersion(), + }, + ), { publisherId: "publishers:1", cursor: null }, )) as { entries: unknown[]; isDone: boolean }; @@ -321,7 +330,7 @@ describe("catalog feed projection", () => { id: "@openclaw/demo", title: "Demo skill", description: "Deploy AIQ services.", - icon: "lucide:rocket", + icon: `https://clawhub.ai/api/v1/skill-icons/${"a".repeat(64)}`, version: "1.2.3", state: "available", featured: false, diff --git a/convex/catalogFeed.ts b/convex/catalogFeed.ts index 4ec72fdb..a5be692d 100644 --- a/convex/catalogFeed.ts +++ b/convex/catalogFeed.ts @@ -28,6 +28,7 @@ import { getSkillFileModerationInfoFromSkill, isPublicSkillVersionAvailableForSkill, } from "./lib/skillFileAccess"; +import { isHostedSkillPresentationIconPath, stripPresentationEmoji } from "./lib/skillPresentation"; const CATALOG_FEED_DESCRIPTION = "Official OpenClaw plugins published on ClawHub."; const CATALOG_FEED_PAGE_SIZE = 100; @@ -119,7 +120,7 @@ async function buildEntry( const packageName = pkg.name.trim(); const id = pkg.normalizedName.trim(); - const title = pkg.displayName.trim() || packageName; + const title = stripPresentationEmoji(pkg.displayName.trim()) || packageName; const description = pkg.summary?.trim(); const icon = pkg.icon?.trim(); const version = release.version.trim(); @@ -209,9 +210,9 @@ async function buildSkillEntry( const publisherId = owner.handle?.trim(); const slug = skill.slug.trim(); - const title = skill.displayName.trim() || slug; + const title = stripPresentationEmoji(skill.displayName.trim()) || slug; const description = skill.summary?.trim(); - const icon = skill.icon?.trim(); + const icon = catalogFeedIconUrl(skill.icon); const highlightedAt = skill.badges?.highlighted?.at; const packageName = `@${publisherId}/${slug}`; if (!publisherId || !slug || !title) return null; @@ -312,6 +313,15 @@ async function buildSkillEntry( }; } +function catalogFeedIconUrl(value: string | undefined) { + const icon = value?.trim(); + if (!icon) return undefined; + if (isHostedSkillPresentationIconPath(icon)) { + return `https://clawhub.ai${icon}`; + } + return icon.startsWith("https://") ? icon : undefined; +} + export const listOfficialPublisherPage = internalQuery({ args: { cursor: v.union(v.string(), v.null()), diff --git a/convex/githubSkillSync.test.ts b/convex/githubSkillSync.test.ts index 8a849098..50135aec 100644 --- a/convex/githubSkillSync.test.ts +++ b/convex/githubSkillSync.test.ts @@ -2937,6 +2937,9 @@ describe("verifyGitHubSkillHandler", () => { const commit = "3".repeat(40); const zip = zipSync({ "skills-main/skills/aiq-deploy/SKILL.md": new TextEncoder().encode("# AIQ Deploy\n"), + "skills-main/skills/aiq-deploy/agents/openai.yaml": new TextEncoder().encode( + "interface:\n display_name: AIQ Deploy Console\n short_description: OpenAI-specific summary.\n", + ), "skills-main/skills/aiq-deploy/scripts/deploy.sh": new TextEncoder().encode( "#!/bin/sh\necho deploy\n", ), @@ -3026,10 +3029,10 @@ describe("verifyGitHubSkillHandler", () => { ); expect(result).toMatchObject({ ok: true, queued: true }); - expect(store).toHaveBeenCalledTimes(2); + expect(store).toHaveBeenCalledTimes(3); expect((store.mock.calls[0]?.[0] as Blob | undefined)?.type).toBe("application/octet-stream"); expect(runMutation).toHaveBeenCalledTimes(3); - expect(events).toEqual(["prepare", "store", "store", "append", "finalize"]); + expect(events).toEqual(["prepare", "store", "store", "store", "append", "finalize"]); const [prepareMutation, prepareArgs] = runMutation.mock.calls[0] ?? []; expect(getFunctionName(prepareMutation as Parameters[0])).toBe( "securityScan:prepareGitHubSkillScanRequestInternal", @@ -3039,6 +3042,13 @@ describe("verifyGitHubSkillHandler", () => { skillId: "skills:aiq-deploy", contentHash, commit, + parsed: { + frontmatter: {}, + presentation: { + displayName: "AIQ Deploy Console", + summary: "OpenAI-specific summary.", + }, + }, staticScan: expect.objectContaining({ status: "clean" }), }), ); @@ -3054,6 +3064,7 @@ describe("verifyGitHubSkillHandler", () => { chunkIndex: 0, files: expect.arrayContaining([ expect.objectContaining({ path: "SKILL.md" }), + expect.objectContaining({ path: "agents/openai.yaml" }), expect.objectContaining({ path: "scripts/deploy.sh" }), ]), }), diff --git a/convex/githubSkillSync.ts b/convex/githubSkillSync.ts index df07c9a3..8022172b 100644 --- a/convex/githubSkillSync.ts +++ b/convex/githubSkillSync.ts @@ -35,6 +35,7 @@ import { getRuntimeRolloutCapabilities, isLegacyNvidiaSkillSource, } from "./lib/rolloutCapabilities"; +import { buildSkillPresentationIconPath } from "./lib/skillPresentation"; import { isMacJunkPath, parseFrontmatter } from "./lib/skills"; import { getSkillBySlugForPublisher, @@ -43,6 +44,10 @@ import { import { chunkSkillScanRequestFiles } from "./lib/skillScanRequestFiles"; import { syncSkillSearchDigestForSkill } from "./lib/skillSearchDigest"; import { assertValidSkillSlug } from "./lib/skillSlugValidator"; +import { + isDecodableSkillPresentationRaster, + storeSkillPresentationAsset, +} from "./skillPresentationAssets"; const DEFAULT_BRANCH = "main"; const GITHUB_SKILL_SCAN_ACTION_LEASE_MS = 15 * 60 * 1000; @@ -197,6 +202,14 @@ const discoveredSkillMetadataValidator = v.object({ path: v.string(), skillMarkdownPath: v.string(), skillCardMarkdownPath: v.optional(v.string()), + iconAsset: v.optional( + v.object({ + path: v.string(), + sha256: v.string(), + contentType: v.string(), + size: v.number(), + }), + ), contentHash: v.string(), }); @@ -210,6 +223,14 @@ const discoveredSkillContentValidator = v.object({ skillMarkdown: v.string(), skillCardMarkdownPath: v.optional(v.string()), skillCardMarkdown: v.optional(v.string()), + iconAsset: v.optional( + v.object({ + path: v.string(), + sha256: v.string(), + contentType: v.string(), + size: v.number(), + }), + ), contentHash: v.string(), }); @@ -993,6 +1014,7 @@ async function applyGenericGitHubSkillSourceSyncHandler( slug: discovered.slug, displayName: discovered.displayName, summary: discovered.summary, + icon: iconForDiscoveredGitHubSkill(discovered), ownerUserId: args.ownerUserId, ownerPublisherId: args.ownerPublisherId, installKind: "github", @@ -1079,6 +1101,7 @@ async function applyGenericGitHubSkillSourceSyncHandler( const patch = { displayName: discovered.displayName, summary: discovered.summary, + icon: iconForDiscoveredGitHubSkill(discovered), ownerUserId: args.ownerUserId, ownerPublisherId: args.ownerPublisherId, githubPath: discovered.path, @@ -1117,6 +1140,7 @@ async function applyGenericGitHubSkillSourceSyncHandler( const patch = { displayName: discovered.displayName, summary: discovered.summary, + icon: iconForDiscoveredGitHubSkill(discovered), ownerUserId: args.ownerUserId, ownerPublisherId: args.ownerPublisherId, installKind: "github" as const, @@ -1236,6 +1260,7 @@ async function upsertGitHubSkillCandidate( githubContentHash: args.discovered.contentHash, displayName: args.discovered.displayName, summary: args.discovered.summary, + icon: iconForDiscoveredGitHubSkill(args.discovered), upstreamVersion: args.discovered.upstreamVersion, skillMarkdownPath: undefined, skillMarkdown: undefined, @@ -1611,6 +1636,7 @@ export async function applyGitHubSkillVerificationResultHandler( const patch = { displayName: candidate.displayName, summary: candidate.summary, + icon: candidate.icon, installKind: "github" as const, githubSourceId: candidate.githubSourceId, githubPath: candidate.githubPath, @@ -1713,6 +1739,7 @@ export async function verifyGitHubSkillHandler( } const { snapshot, entries } = await fetchGitHubSkillSourceSnapshotWithEntries( + ctx, { repo: target.source.repo, ref: target.skill.githubCurrentCommit, @@ -1744,6 +1771,7 @@ export async function verifyGitHubSkillHandler( files: listGitHubSkillFiles(entries, discovered.path), fileContents: listGitHubSkillTextContents(entries, discovered.path), }); + const presentationIcon = iconForDiscoveredGitHubSkill(discovered); const prepared = (await ctx.runMutation( internal.securityScan.prepareGitHubSkillScanRequestInternal, @@ -1752,7 +1780,14 @@ export async function verifyGitHubSkillHandler( contentHash: args.contentHash, commit: target.skill.githubCurrentCommit, ...(args.force ? { force: true } : {}), - parsed: { frontmatter: parseFrontmatter(discovered.skillMarkdown) }, + parsed: { + frontmatter: parseFrontmatter(discovered.skillMarkdown), + presentation: { + displayName: discovered.displayName, + ...(discovered.summary ? { summary: discovered.summary } : {}), + ...(presentationIcon ? { icon: presentationIcon } : {}), + }, + }, staticScan, }, )) as GitHubSkillVerificationResult | undefined; @@ -1806,6 +1841,7 @@ export async function configurePublicGitHubSkillSourceHandler( }, )) as GitHubSkillSourceSetupContext; const snapshot = await fetchGitHubSkillSourceSnapshot( + ctx, { repo: metadata.repo, defaultBranch: metadata.defaultBranch, @@ -1848,6 +1884,7 @@ async function applyFetchedGitHubSkillSourceSnapshot( snapshot: GitHubSkillSourceSnapshot; }, ) { + await persistGitHubSkillPresentationAssets(ctx, args.snapshot); const result = (await ctx.runMutation( internal.githubSkillSync.applyGitHubSkillSourceSyncInternal, { @@ -1870,11 +1907,42 @@ function toGitHubSkillSourceMetadataSnapshot( return { ...snapshot, skills: snapshot.skills.map( - ({ skillMarkdown: _skillMarkdown, skillCardMarkdown: _skillCardMarkdown, ...skill }) => skill, + ({ + skillMarkdown: _skillMarkdown, + skillCardMarkdown: _skillCardMarkdown, + iconBytes: _iconBytes, + ...skill + }) => skill, ), }; } +async function persistGitHubSkillPresentationAssets( + ctx: ActionCtx, + snapshot: GitHubSkillSourceSnapshot, +) { + for (const skill of snapshot.skills) { + if (!skill.iconAsset || !skill.iconBytes) continue; + await storeSkillPresentationAsset(ctx, { + bytes: skill.iconBytes, + sha256: skill.iconAsset.sha256, + contentType: skill.iconAsset.contentType as + | "image/png" + | "image/jpeg" + | "image/webp" + | "image/svg+xml", + }); + } +} + +function iconForDiscoveredGitHubSkill( + discovered: GitHubSkillSourceMetadataSnapshot["skills"][number], +) { + return discovered.iconAsset + ? buildSkillPresentationIconPath(discovered.iconAsset.sha256) + : undefined; +} + async function persistGitHubSkillContentsForSnapshot( ctx: ActionCtx, result: SyncOneResult, @@ -1929,7 +1997,7 @@ export const syncGitHubSkillSource: ReturnType = action({ { publisherId: ownerPublisherId }, )) as Id<"users">; const metadata = await fetchPublicGitHubRepoMetadata(repo, fetch); - const snapshot = await fetchGitHubSkillSourceSnapshot({ + const snapshot = await fetchGitHubSkillSourceSnapshot(ctx, { repo, defaultBranch: args.defaultBranch ?? source?.defaultBranch ?? metadata.defaultBranch, }); @@ -2012,6 +2080,7 @@ export async function syncGitHubSkillSourcesHandler( throw new ConvexError("GitHub repository authorization no longer matches."); } const snapshot = await fetchGitHubSkillSourceSnapshot( + ctx, { repo: metadata.repo, defaultBranch: source.defaultBranch ?? metadata.defaultBranch, @@ -2098,6 +2167,7 @@ export async function syncGitHubSkillSourcesHandler( } async function fetchGitHubSkillSourceSnapshot( + ctx: Pick, { repo, defaultBranch, @@ -2108,6 +2178,7 @@ async function fetchGitHubSkillSourceSnapshot( fetcher: typeof fetch = fetch, ) { const { snapshot } = await fetchGitHubSkillSourceSnapshotWithEntries( + ctx, { repo, ref: defaultBranch, @@ -2119,6 +2190,7 @@ async function fetchGitHubSkillSourceSnapshot( } async function fetchGitHubSkillSourceSnapshotWithEntries( + ctx: Pick, { repo, ref, @@ -2141,6 +2213,7 @@ async function fetchGitHubSkillSourceSnapshotWithEntries( defaultBranch, commit: resolved.commit, entries, + validateRasterIcon: (args) => isDecodableSkillPresentationRaster(ctx, args), }); return { snapshot, entries }; } diff --git a/convex/http.ts b/convex/http.ts index 25efdfdb..3248bd5d 100644 --- a/convex/http.ts +++ b/convex/http.ts @@ -70,11 +70,18 @@ import { packageInspectorClaimHttp, packageInspectorResultsHttp, } from "./packageInspectorHttp"; +import { skillPresentationAssetHttp } from "./skillPresentationAssetsHttp"; const http = installRateLimitedRoutes(httpRouter()); auth.addHttpRoutes(http); +http.route({ + pathPrefix: "/api/v1/skill-icons/", + method: "GET", + handler: skillPresentationAssetHttp, +}); + // Convex routes HEAD through the matching GET action and strips the body. http.route({ pathPrefix: "/api/v1/agent-skills/", diff --git a/convex/lib/githubSkillSync.test.ts b/convex/lib/githubSkillSync.test.ts index f5dd9b00..b1d2af7f 100644 --- a/convex/lib/githubSkillSync.test.ts +++ b/convex/lib/githubSkillSync.test.ts @@ -6,13 +6,24 @@ import { } from "./githubSkillSync"; const encoder = new TextEncoder(); +const validPng = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + "base64", + ), +); function bytes(text: string) { return encoder.encode(text); } -function repoEntries(entries: Record) { - return Object.fromEntries(Object.entries(entries).map(([path, text]) => [path, bytes(text)])); +function repoEntries(entries: Record) { + return Object.fromEntries( + Object.entries(entries).map(([path, value]) => [ + path, + value instanceof Uint8Array ? value : bytes(value), + ]), + ); } describe("parseSkillsShDisplayManifest", () => { @@ -69,11 +80,14 @@ describe("buildGitHubSkillSourceSnapshot", () => { "skills/aiq-deploy/SKILL.md": "---\nname: AIQ Deploy\nversion: 0.2.0\ndescription: Deploy AgentIQ workflows.\n---\n# AIQ Deploy\n", "skills/aiq-deploy/skill-card.md": "# Card\n", + "skills/aiq-deploy/agents/openai.yaml": + "interface:\n display_name: '🚀 AIQ Deploy Console'\n short_description: Deploy from the OpenAI console.\n icon_small: ../assets/icon.png\n icon_large: assets/icon.png\n", "skills/vision-helper/SKILL.md": "# Vision Helper\n", "skills.sh.json": JSON.stringify({ groupings: [{ title: "Agentic AI", skills: ["aiq-deploy"] }], }), }); + baseEntries["skills/aiq-deploy/assets/icon.png"] = validPng; const changedEntries = { ...baseEntries, "skills/aiq-deploy/skill-card.md": bytes("# Card changed\n"), @@ -100,8 +114,15 @@ describe("buildGitHubSkillSourceSnapshot", () => { expect.arrayContaining([ expect.objectContaining({ slug: "aiq-deploy", - displayName: "AIQ Deploy", - summary: "Deploy AgentIQ workflows.", + displayName: "AIQ Deploy Console", + summary: "Deploy from the OpenAI console.", + iconAsset: { + path: "assets/icon.png", + contentType: "image/png", + size: validPng.byteLength, + sha256: expect.stringMatching(/^[a-f\d]{64}$/), + }, + iconBytes: validPng, upstreamVersion: "0.2.0", path: "skills/aiq-deploy", skillMarkdownPath: "skills/aiq-deploy/SKILL.md", @@ -150,6 +171,26 @@ describe("buildGitHubSkillSourceSnapshot", () => { expect(changed.skills[0]?.contentHash).not.toBe(base.skills[0]?.contentHash); }); + it("falls back to icon_large when icon_small cannot be decoded", async () => { + const snapshot = await buildGitHubSkillSourceSnapshot({ + repo: "NVIDIA/skills", + defaultBranch: "main", + commit: "1".repeat(40), + entries: repoEntries({ + "skills/demo/SKILL.md": "# Demo\n", + "skills/demo/agents/openai.yaml": + "interface:\n icon_small: assets/icon.png\n icon_large: assets/icon.svg\n", + "skills/demo/assets/icon.png": validPng, + "skills/demo/assets/icon.svg": '', + }), + validateRasterIcon: async () => false, + }); + + expect(snapshot.skills[0]).toMatchObject({ + iconAsset: { path: "assets/icon.svg", contentType: "image/svg+xml" }, + }); + }); + it("rejects duplicate normalized skill slugs before syncing content", async () => { await expect( buildGitHubSkillSourceSnapshot({ @@ -218,6 +259,9 @@ describe("buildGitHubSkillSyncPlan", () => { commit: "2".repeat(40), entries: repoEntries({ "skills/aiq-deploy/SKILL.md": "# AIQ Deploy v2\n", + "skills/aiq-deploy/agents/openai.yaml": + "interface:\n display_name: AIQ Deploy Console\n icon_small: assets/icon.png\n", + "skills/aiq-deploy/assets/icon.png": validPng, "skills.sh.json": JSON.stringify({ groupings: [{ title: "Agentic AI", skills: ["aiq-deploy"] }], }), @@ -248,6 +292,8 @@ describe("buildGitHubSkillSyncPlan", () => { skillId: "skills:aiq-deploy", slug: "aiq-deploy", patch: expect.objectContaining({ + displayName: "AIQ Deploy Console", + icon: expect.stringMatching(/^\/api\/v1\/skill-icons\/[a-f\d]{64}$/), githubCurrentCommit: "2".repeat(40), githubCurrentContentHash: snapshot.skills[0]?.contentHash, githubScanStatus: "pending", diff --git a/convex/lib/githubSkillSync.ts b/convex/lib/githubSkillSync.ts index 79c11595..de4d07d7 100644 --- a/convex/lib/githubSkillSync.ts +++ b/convex/lib/githubSkillSync.ts @@ -3,6 +3,13 @@ import { shouldPreserveSecurityScanStateForUnchangedContent, type SourceBackedSkillScanStatus, } from "./securityScanPolicy"; +import { + buildSkillPresentationIconPath, + OPENAI_SKILL_PRESENTATION_PATH, + parseOpenAiSkillPresentation, + resolveSkillPresentation, + validateSkillPresentationIcon, +} from "./skillPresentation"; import { getFrontmatterValue, parseFrontmatter } from "./skills"; export type GitHubSkillScanStatus = SourceBackedSkillScanStatus; @@ -42,12 +49,19 @@ export type DiscoveredGitHubSkill = { skillMarkdown: string; skillCardMarkdownPath?: string; skillCardMarkdown?: string; + iconAsset?: { + path: string; + sha256: string; + contentType: string; + size: number; + }; + iconBytes?: Uint8Array; contentHash: string; }; export type DiscoveredGitHubSkillMetadata = Omit< DiscoveredGitHubSkill, - "skillMarkdown" | "skillCardMarkdown" + "skillMarkdown" | "skillCardMarkdown" | "iconBytes" >; export type ExistingGitHubSkillForSync = { @@ -55,6 +69,7 @@ export type ExistingGitHubSkillForSync = { slug: string; displayName: string; summary?: string; + icon?: string; latestVersionSummary?: { version: string; createdAt: number; @@ -166,11 +181,16 @@ export async function buildGitHubSkillSourceSnapshot({ defaultBranch, commit, entries, + validateRasterIcon, }: { repo: string; defaultBranch: string; commit: string; entries: Record; + validateRasterIcon?: (args: { + bytes: Uint8Array; + contentType: "image/png" | "image/jpeg" | "image/webp"; + }) => Promise; }): Promise { const normalizedEntries = normalizeEntryMap(entries); const manifestBytes = normalizedEntries["skills.sh.json"]; @@ -193,6 +213,57 @@ export async function buildGitHubSkillSourceSnapshot({ const frontmatterDescription = getFrontmatterValue(frontmatter, "description")?.trim(); const frontmatterVersion = getFrontmatterValue(frontmatter, "version")?.trim(); const heading = firstMarkdownHeading(markdown); + const skillDisplayName = frontmatterName || heading || titleizeSlug(slug); + const openAiPresentationPath = findSkillRelativeFilePath( + normalizedEntries, + path, + OPENAI_SKILL_PRESENTATION_PATH, + ); + const openAiPresentationBytes = openAiPresentationPath + ? normalizedEntries[openAiPresentationPath] + : undefined; + const openAiPresentation = openAiPresentationBytes + ? parseOpenAiSkillPresentation(decodeUtf8(openAiPresentationBytes)) + : null; + const presentation = resolveSkillPresentation({ + openAi: openAiPresentation, + skillDisplayName, + skillDescription: frontmatterDescription, + slug, + }); + let iconBytes: Uint8Array | undefined; + let iconAsset: DiscoveredGitHubSkill["iconAsset"]; + for (const iconPath of presentation.iconPaths ?? []) { + const iconAssetPath = findSkillRelativeFilePath(normalizedEntries, path, iconPath); + const candidateBytes = iconAssetPath ? normalizedEntries[iconAssetPath] : undefined; + if (!candidateBytes) continue; + try { + const validated = validateSkillPresentationIcon({ + path: iconPath, + bytes: candidateBytes, + }); + if ( + validated.contentType !== "image/svg+xml" && + validateRasterIcon && + !(await validateRasterIcon({ + bytes: candidateBytes, + contentType: validated.contentType, + })) + ) { + continue; + } + iconAsset = { + path: iconPath, + sha256: await sha256Hex(candidateBytes), + contentType: validated.contentType, + size: validated.size, + }; + iconBytes = candidateBytes; + break; + } catch { + continue; + } + } const skillCardMarkdownPath = findFolderFilePath( normalizedEntries, path, @@ -211,14 +282,15 @@ export async function buildGitHubSkillSourceSnapshot({ skills.push({ slug, - displayName: frontmatterName || heading || titleizeSlug(slug), - ...(frontmatterDescription ? { summary: frontmatterDescription } : {}), + displayName: presentation.displayName, + ...(presentation.summary ? { summary: presentation.summary } : {}), ...(frontmatterVersion ? { upstreamVersion: frontmatterVersion } : {}), path, skillMarkdownPath: skillMdPath, skillMarkdown: markdown, ...(skillCardMarkdownPath ? { skillCardMarkdownPath } : {}), ...(skillCardMarkdown !== undefined ? { skillCardMarkdown } : {}), + ...(iconAsset && iconBytes ? { iconAsset, iconBytes: new Uint8Array(iconBytes) } : {}), contentHash: await computeGitHubSkillFolderContentHash(normalizedEntries, path), }); } @@ -315,6 +387,9 @@ export function buildGitHubSkillSyncPlan({ slug: discovered.slug, displayName: discovered.displayName, summary: discovered.summary, + icon: discovered.iconAsset + ? buildSkillPresentationIconPath(discovered.iconAsset.sha256) + : undefined, ownerUserId, ownerPublisherId, installKind: "github", @@ -369,11 +444,18 @@ export function buildGitHubSkillSyncPlan({ !currentContentUnchanged || existing.displayName !== discovered.displayName || (existing.summary ?? undefined) !== (discovered.summary ?? undefined) || + (existing.icon ?? undefined) !== + (discovered.iconAsset + ? buildSkillPresentationIconPath(discovered.iconAsset.sha256) + : undefined) || (existing.githubPath ?? undefined) !== discovered.path || !sameLatestVersionSummary(existing.latestVersionSummary, nextLatestVersionSummary); const patch = { displayName: discovered.displayName, summary: discovered.summary, + icon: discovered.iconAsset + ? buildSkillPresentationIconPath(discovered.iconAsset.sha256) + : undefined, ownerUserId, ...(ownerPublisherId ? { ownerPublisherId } : {}), githubSourceId: sourceId, @@ -605,6 +687,15 @@ function findFolderFilePath( }); } +function findSkillRelativeFilePath( + entries: Record, + folderPath: string, + relativePath: string, +) { + const expected = folderPath ? `${folderPath}/${relativePath}` : relativePath; + return Object.hasOwn(entries, expected) ? expected : undefined; +} + function normalizeRepoPath(path: string) { if (path.includes("\u0000")) return ""; const normalized = path diff --git a/convex/lib/retentionPolicy.ts b/convex/lib/retentionPolicy.ts index 9a9f2789..ef640e24 100644 --- a/convex/lib/retentionPolicy.ts +++ b/convex/lib/retentionPolicy.ts @@ -179,6 +179,9 @@ export const RETENTION_POLICIES = { "packages", ), skillVersions: permanent("Canonical skill version records."), + skillPresentationAssets: permanent( + "Immutable content-addressed icon copies referenced by skill presentation metadata.", + ), skillVersionFingerprints: derived("Fingerprint projection of skill versions.", "skillVersions"), skillBadges: permanent("Curated skill badges."), skillEmbeddings: derived("Search embedding projection of skill versions.", "skillVersions"), diff --git a/convex/lib/skillPresentation.test.ts b/convex/lib/skillPresentation.test.ts new file mode 100644 index 00000000..dba5e11b --- /dev/null +++ b/convex/lib/skillPresentation.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it } from "vitest"; +import { + buildSkillPresentationIconPath, + MAX_SKILL_PRESENTATION_DISPLAY_NAME_LENGTH, + MAX_SKILL_PRESENTATION_SHORT_DESCRIPTION_LENGTH, + parseOpenAiSkillPresentation, + resolveSkillPresentation, + stripPresentationEmoji, + validateSkillPresentationIcon, +} from "./skillPresentation"; + +const encoder = new TextEncoder(); +const validPng = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + "base64", + ), +); +const validJpeg = Uint8Array.from( + Buffer.from( + "/9j/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAABgj/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCdABykX//Z", + "base64", + ), +); +const validWebp = Uint8Array.from( + Buffer.from( + "UklGRjwAAABXRUJQVlA4IDAAAADQAQCdASoBAAEAAUAmJaACdLoB+AADsAD+8ut//NgVzXPv9//S4P0uD9Lg/9KQAAA=", + "base64", + ), +); + +describe("parseOpenAiSkillPresentation", () => { + it("keeps supported interface metadata and resolves the preferred icon path", () => { + expect( + parseOpenAiSkillPresentation(` +interface: + display_name: "✨ Better Search" + short_description: Search across project knowledge. + icon_small: ./assets/icon-small.png + icon_large: assets/icon-large.webp + brand_color: "#112233" + default_prompt: Ignore this for catalog rendering. +policy: + allow_implicit_invocation: true +`), + ).toEqual({ + displayName: "✨ Better Search", + shortDescription: "Search across project knowledge.", + iconPaths: ["assets/icon-small.png", "assets/icon-large.webp"], + }); + }); + + it("drops traversal, absolute, URL, and malformed icon references", () => { + for (const icon of ["../secret.png", "/tmp/icon.png", "https://example.com/icon.png"]) { + expect( + parseOpenAiSkillPresentation(`interface:\n display_name: Demo\n icon_small: ${icon}\n`), + ).toEqual({ displayName: "Demo" }); + } + expect(parseOpenAiSkillPresentation("{not yaml")).toBeNull(); + }); + + it("drops presentation text that exceeds catalog field limits", () => { + expect( + parseOpenAiSkillPresentation(`interface: + display_name: ${"n".repeat(MAX_SKILL_PRESENTATION_DISPLAY_NAME_LENGTH + 1)} + short_description: ${"s".repeat(MAX_SKILL_PRESENTATION_SHORT_DESCRIPTION_LENGTH + 1)} + icon_small: assets/icon.png +`), + ).toEqual({ iconPaths: ["assets/icon.png"] }); + }); +}); + +describe("resolveSkillPresentation", () => { + it("uses publisher overrides before OpenAI metadata and SKILL.md metadata", () => { + expect( + resolveSkillPresentation({ + publisherDisplayName: "🚀 Publisher Name", + publisherSummary: "Publisher summary.", + openAi: { + displayName: "OpenAI Name", + shortDescription: "OpenAI summary.", + iconPaths: ["assets/icon.png"], + }, + skillDisplayName: "Skill Name", + skillDescription: "Skill summary.", + slug: "skill-name", + }), + ).toEqual({ + displayName: "Publisher Name", + displayNameSource: "publisher", + summary: "Publisher summary.", + summarySource: "publisher", + iconPaths: ["assets/icon.png"], + }); + }); + + it("falls back through OpenAI metadata, SKILL.md, and slug while removing emoji", () => { + expect( + resolveSkillPresentation({ + openAi: { displayName: "🧭 OpenAI Name", shortDescription: "OpenAI summary." }, + skillDisplayName: "Skill Name", + skillDescription: "Skill summary.", + slug: "skill-name", + }), + ).toEqual({ + displayName: "OpenAI Name", + displayNameSource: "openai", + summary: "OpenAI summary.", + summarySource: "openai", + }); + expect(resolveSkillPresentation({ slug: "skill-name" })).toEqual({ + displayName: "Skill Name", + displayNameSource: "slug", + }); + }); +}); + +describe("skill presentation icons", () => { + it("accepts matching PNG, JPEG, WebP, and SVG assets", () => { + const fixtures = [ + { + path: "assets/icon.png", + bytes: validPng, + contentType: "image/png", + }, + { + path: "assets/icon.jpg", + bytes: validJpeg, + contentType: "image/jpeg", + }, + { + path: "assets/icon.webp", + bytes: validWebp, + contentType: "image/webp", + }, + { + path: "assets/icon.svg", + bytes: encoder.encode(''), + contentType: "image/svg+xml", + }, + ]; + + for (const fixture of fixtures) { + expect(validateSkillPresentationIcon(fixture)).toEqual({ + contentType: fixture.contentType, + size: fixture.bytes.byteLength, + }); + } + }); + + it("rejects mismatched, active SVG, and oversized assets", () => { + expect(() => + validateSkillPresentationIcon({ + path: "icon.gif", + bytes: encoder.encode("GIF89a"), + }), + ).toThrow(/unsupported/i); + expect(() => + validateSkillPresentationIcon({ + path: "icon.png", + bytes: encoder.encode(""), + }), + ).toThrow(/invalid png/i); + expect(() => + validateSkillPresentationIcon({ + path: "icon.png", + bytes: validPng.slice(0, 24), + }), + ).toThrow(/invalid png/i); + expect(() => + validateSkillPresentationIcon({ + path: "icon.svg", + bytes: encoder.encode(""), + }), + ).toThrow(/unsafe svg/i); + expect(() => + validateSkillPresentationIcon({ + path: "icon.png", + bytes: new Uint8Array(512 * 1024 + 1), + }), + ).toThrow(/512KB/i); + }); + + it("builds stable content-addressed paths", () => { + expect(buildSkillPresentationIconPath("A".repeat(64))).toBe( + `/api/v1/skill-icons/${"a".repeat(64)}`, + ); + expect(() => buildSkillPresentationIconPath("nope")).toThrow(/sha-256/i); + }); +}); + +describe("stripPresentationEmoji", () => { + it("removes emoji sequences and tidies the rendered title", () => { + expect(stripPresentationEmoji(" 🚀 Super ✨ Skill ")).toBe("Super Skill"); + expect(stripPresentationEmoji("🧑🏽‍💻 Dev Tools")).toBe("Dev Tools"); + }); +}); diff --git a/convex/lib/skillPresentation.ts b/convex/lib/skillPresentation.ts new file mode 100644 index 00000000..c523e771 --- /dev/null +++ b/convex/lib/skillPresentation.ts @@ -0,0 +1,328 @@ +import { parseDocument } from "yaml"; + +export const OPENAI_SKILL_PRESENTATION_PATH = "agents/openai.yaml"; +export const MAX_SKILL_PRESENTATION_YAML_BYTES = 64 * 1024; +export const MAX_SKILL_PRESENTATION_ICON_BYTES = 512 * 1024; +export const MAX_SKILL_PRESENTATION_DISPLAY_NAME_LENGTH = 120; +export const MAX_SKILL_PRESENTATION_SHORT_DESCRIPTION_LENGTH = 300; + +const ICON_CONTENT_TYPES = { + ".jpeg": "image/jpeg", + ".jpg": "image/jpeg", + ".png": "image/png", + ".svg": "image/svg+xml", + ".webp": "image/webp", +} as const; +const PRESENTATION_EMOJI_PATTERN = + /\p{Extended_Pictographic}|\p{Emoji_Presentation}|\p{Emoji_Modifier}|\p{Regional_Indicator}|\u200D|\uFE0F|\u20E3/gu; + +export type OpenAiSkillPresentation = { + displayName?: string; + shortDescription?: string; + iconPaths?: string[]; +}; + +export type ResolvedSkillPresentation = { + displayName: string; + displayNameSource: "publisher" | "openai" | "skill" | "slug"; + summary?: string; + summarySource?: "publisher" | "openai" | "skill" | "generated"; + iconPaths?: string[]; +}; + +export function parseOpenAiSkillPresentation( + raw: string | undefined | null, +): OpenAiSkillPresentation | null { + if (!raw?.trim()) return null; + if (new TextEncoder().encode(raw).byteLength > MAX_SKILL_PRESENTATION_YAML_BYTES) return null; + + try { + const document = parseDocument(raw); + if (document.errors.length > 0) return null; + const parsed = document.toJS({ maxAliasCount: 20 }) as unknown; + if (!isRecord(parsed) || !isRecord(parsed.interface)) return null; + + const interfaceMetadata = parsed.interface; + const displayName = cleanText( + interfaceMetadata.display_name, + MAX_SKILL_PRESENTATION_DISPLAY_NAME_LENGTH, + ); + const shortDescription = cleanText( + interfaceMetadata.short_description, + MAX_SKILL_PRESENTATION_SHORT_DESCRIPTION_LENGTH, + ); + const iconPaths = [ + normalizeSkillPresentationPath(interfaceMetadata.icon_small), + normalizeSkillPresentationPath(interfaceMetadata.icon_large), + ].filter( + (path, index, paths): path is string => Boolean(path) && paths.indexOf(path) === index, + ); + + if (!displayName && !shortDescription && iconPaths.length === 0) return null; + return { + ...(displayName ? { displayName } : {}), + ...(shortDescription ? { shortDescription } : {}), + ...(iconPaths.length > 0 ? { iconPaths } : {}), + }; + } catch { + return null; + } +} + +export function resolveSkillPresentation(args: { + publisherDisplayName?: string | null; + publisherSummary?: string | null; + openAi?: OpenAiSkillPresentation | null; + skillDisplayName?: string | null; + skillDescription?: string | null; + slug: string; +}): ResolvedSkillPresentation { + const publisherDisplayName = cleanText(args.publisherDisplayName); + const openAiDisplayName = cleanText(args.openAi?.displayName); + const skillDisplayName = cleanText(args.skillDisplayName); + const displayName = stripPresentationEmoji( + publisherDisplayName ?? openAiDisplayName ?? skillDisplayName ?? titleizeSlug(args.slug), + ); + const displayNameSource = displayName + ? publisherDisplayName + ? "publisher" + : openAiDisplayName + ? "openai" + : skillDisplayName + ? "skill" + : "slug" + : "slug"; + const publisherSummary = cleanText(args.publisherSummary); + const openAiSummary = cleanText(args.openAi?.shortDescription); + const skillSummary = cleanText(args.skillDescription); + const summary = publisherSummary ?? openAiSummary ?? skillSummary; + const summarySource = publisherSummary + ? "publisher" + : openAiSummary + ? "openai" + : skillSummary + ? "skill" + : undefined; + const iconPaths = (args.openAi?.iconPaths ?? []) + .map(normalizeSkillPresentationPath) + .filter((path, index, paths): path is string => Boolean(path) && paths.indexOf(path) === index); + + return { + displayName: displayName || titleizeSlug(args.slug), + displayNameSource, + ...(summary && summarySource ? { summary, summarySource } : {}), + ...(iconPaths.length > 0 ? { iconPaths } : {}), + }; +} + +export function normalizeSkillPresentationPath(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const path = value + .trim() + .replaceAll("\\", "/") + .replace(/^\.\/+/, ""); + if (!path || path.startsWith("/") || path.includes("\0")) return undefined; + if (/^[a-z][a-z\d+.-]*:/i.test(path) || path.startsWith("//")) return undefined; + if (path.split("/").some((segment) => !segment || segment === "." || segment === "..")) { + return undefined; + } + if (/(?:^|\/)%2e(?:%2e)?(?:\/|$)/i.test(path)) return undefined; + return path; +} + +export function validateSkillPresentationIcon(args: { + path: string; + bytes: Uint8Array; + contentType?: string | null; +}): { contentType: (typeof ICON_CONTENT_TYPES)[keyof typeof ICON_CONTENT_TYPES]; size: number } { + const extension = iconExtension(args.path); + if (!extension) throw new Error("Unsupported skill presentation icon type."); + const expectedContentType = ICON_CONTENT_TYPES[extension]; + if (args.bytes.byteLength > MAX_SKILL_PRESENTATION_ICON_BYTES) { + throw new Error("Skill presentation icon exceeds the 512KB limit."); + } + if (args.bytes.byteLength === 0) throw new Error("Skill presentation icon is empty."); + + const suppliedContentType = args.contentType?.split(";", 1)[0]?.trim().toLowerCase(); + if (suppliedContentType && suppliedContentType !== expectedContentType) { + throw new Error(`Skill presentation icon content type does not match ${extension}.`); + } + + if (expectedContentType === "image/png" && !isStructurallyValidPng(args.bytes)) { + throw new Error("Invalid PNG skill presentation icon."); + } + if (expectedContentType === "image/jpeg" && !isStructurallyValidJpeg(args.bytes)) { + throw new Error("Invalid JPEG skill presentation icon."); + } + if (expectedContentType === "image/webp" && !isStructurallyValidWebp(args.bytes)) { + throw new Error("Invalid WebP skill presentation icon."); + } + if (expectedContentType === "image/svg+xml") validateSafeSvg(args.bytes); + + return { contentType: expectedContentType, size: args.bytes.byteLength }; +} + +export function buildSkillPresentationIconPath(sha256: string) { + const normalized = sha256.trim().toLowerCase(); + if (!/^[a-f\d]{64}$/.test(normalized)) throw new Error("A SHA-256 digest is required."); + return `/api/v1/skill-icons/${normalized}`; +} + +export function isHostedSkillPresentationIconPath(value: string | null | undefined) { + return /^\/api\/v1\/skill-icons\/[a-f\d]{64}$/u.test(value?.trim() ?? ""); +} + +export function stripPresentationEmoji(value: string) { + return value.replace(PRESENTATION_EMOJI_PATTERN, " ").replace(/\s+/g, " ").trim(); +} + +function validateSafeSvg(bytes: Uint8Array) { + let svg: string; + try { + svg = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error("Invalid SVG skill presentation icon."); + } + if (!/^\s*(?:<\?xml[^>]*>\s*)?)/i.test(svg)) { + throw new Error("Invalid SVG skill presentation icon."); + } + if ( + /<\/?(?:script|foreignObject|iframe|object|embed|audio|video)\b/i.test(svg) || + /\son[a-z]+\s*=/i.test(svg) || + /(?:javascript|vbscript)\s*:/i.test(svg) || + / bytes[index] === value); +} + +function isStructurallyValidPng(bytes: Uint8Array) { + if (!hasPrefix(bytes, [137, 80, 78, 71, 13, 10, 26, 10])) return false; + let offset = 8; + let sawHeader = false; + let sawImageData = false; + while (offset + 12 <= bytes.byteLength) { + const length = readUint32Be(bytes, offset); + const type = ascii(bytes, offset + 4, offset + 8); + const nextOffset = offset + 12 + length; + if (nextOffset > bytes.byteLength) return false; + if (!sawHeader) { + if (type !== "IHDR" || length !== 13) return false; + if (readUint32Be(bytes, offset + 8) === 0 || readUint32Be(bytes, offset + 12) === 0) { + return false; + } + sawHeader = true; + } else if (type === "IDAT") { + sawImageData = true; + } else if (type === "IEND") { + return length === 0 && sawImageData && nextOffset === bytes.byteLength; + } + offset = nextOffset; + } + return false; +} + +function isStructurallyValidJpeg(bytes: Uint8Array) { + if ( + bytes.byteLength < 12 || + !hasPrefix(bytes, [255, 216, 255]) || + bytes.at(-2) !== 255 || + bytes.at(-1) !== 217 + ) { + return false; + } + let offset = 2; + while (offset + 4 <= bytes.byteLength - 2) { + if (bytes[offset] !== 255) return false; + while (bytes[offset] === 255) offset += 1; + const marker = bytes[offset]; + offset += 1; + if (marker === undefined || marker === 0 || marker === 255) return false; + if (marker === 217) return false; + if (marker === 218) break; + if (marker === 1 || (marker >= 208 && marker <= 215)) continue; + if (offset + 2 > bytes.byteLength - 2) return false; + const segmentLength = readUint16Be(bytes, offset); + if (segmentLength < 2 || offset + segmentLength > bytes.byteLength - 2) return false; + if (isJpegStartOfFrame(marker)) { + return ( + segmentLength >= 7 && + readUint16Be(bytes, offset + 3) > 0 && + readUint16Be(bytes, offset + 5) > 0 + ); + } + offset += segmentLength; + } + return false; +} + +function isStructurallyValidWebp(bytes: Uint8Array) { + if ( + bytes.byteLength < 30 || + ascii(bytes, 0, 4) !== "RIFF" || + ascii(bytes, 8, 12) !== "WEBP" || + readUint32Le(bytes, 4) + 8 !== bytes.byteLength + ) { + return false; + } + const chunkType = ascii(bytes, 12, 16); + const chunkLength = readUint32Le(bytes, 16); + const paddedLength = chunkLength + (chunkLength % 2); + if (20 + paddedLength > bytes.byteLength) return false; + if (chunkType === "VP8X") return chunkLength >= 10; + if (chunkType === "VP8L") return chunkLength >= 5 && bytes[20] === 47; + return chunkType === "VP8 " && chunkLength >= 10; +} + +function isJpegStartOfFrame(marker: number) { + return marker >= 192 && marker <= 207 && ![196, 200, 204].includes(marker); +} + +function readUint16Be(bytes: Uint8Array, offset: number) { + return ((bytes[offset] ?? 0) << 8) | (bytes[offset + 1] ?? 0); +} + +function readUint32Be(bytes: Uint8Array, offset: number) { + return new DataView(bytes.buffer, bytes.byteOffset + offset, 4).getUint32(0, false); +} + +function readUint32Le(bytes: Uint8Array, offset: number) { + return new DataView(bytes.buffer, bytes.byteOffset + offset, 4).getUint32(0, true); +} + +function ascii(bytes: Uint8Array, start: number, end: number) { + return String.fromCharCode(...bytes.slice(start, end)); +} + +function cleanText(value: unknown, maxLength?: number) { + if (typeof value !== "string") return undefined; + const normalized = value.replace(/\s+/g, " ").trim(); + if (!normalized || (maxLength !== undefined && normalized.length > maxLength)) return undefined; + return normalized; +} + +function titleizeSlug(slug: string) { + const title = slug + .trim() + .replace(/[-_]+/g, " ") + .replace(/\b\w/g, (character) => character.toUpperCase()) + .trim(); + return title || "Skill"; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} diff --git a/convex/lib/skillPresentationBackfill.test.ts b/convex/lib/skillPresentationBackfill.test.ts new file mode 100644 index 00000000..37a78253 --- /dev/null +++ b/convex/lib/skillPresentationBackfill.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { + preserveHistoricalHostedIcon, + resolveHistoricalSkillPresentation, +} from "./skillPresentationBackfill"; + +describe("resolveHistoricalSkillPresentation", () => { + it("adopts OpenAI presentation values when stored values came from frontmatter", () => { + expect( + resolveHistoricalSkillPresentation({ + slug: "demo-skill", + currentDisplayName: "Demo Skill", + currentSummary: "Frontmatter summary", + frontmatter: { name: "demo-skill", description: "Frontmatter summary" }, + openAi: { + displayName: "OpenAI Demo", + shortDescription: "OpenAI summary", + iconPaths: ["assets/icon.png"], + }, + }), + ).toEqual({ + displayName: "OpenAI Demo", + displayNameSource: "openai", + summary: "OpenAI summary", + summarySource: "openai", + iconPaths: ["assets/icon.png"], + }); + }); + + it("preserves historical publisher title and summary overrides", () => { + expect( + resolveHistoricalSkillPresentation({ + slug: "demo-skill", + currentDisplayName: "Publisher Title", + currentSummary: "Publisher summary", + frontmatter: { name: "demo-skill", description: "Frontmatter summary" }, + openAi: { + displayName: "OpenAI Demo", + shortDescription: "OpenAI summary", + iconPaths: ["assets/icon.png"], + }, + }), + ).toEqual({ + displayName: "Publisher Title", + displayNameSource: "publisher", + summary: "Publisher summary", + summarySource: "publisher", + iconPaths: ["assets/icon.png"], + }); + }); + + it("strips emoji while preserving an explicit publisher title", () => { + expect( + resolveHistoricalSkillPresentation({ + slug: "demo-skill", + currentDisplayName: "✨ Publisher Title", + frontmatter: {}, + openAi: { displayName: "OpenAI Demo" }, + }).displayName, + ).toBe("Publisher Title"); + }); +}); + +describe("preserveHistoricalHostedIcon", () => { + const hostedIcon = `/api/v1/skill-icons/${"a".repeat(64)}`; + + it("preserves an immutable hosted icon when source revalidation is unavailable", () => { + expect(preserveHistoricalHostedIcon(undefined, hostedIcon)).toBe(hostedIcon); + }); + + it("does not preserve arbitrary legacy icon values", () => { + expect( + preserveHistoricalHostedIcon("https://example.com/icon.png", "icon.png"), + ).toBeUndefined(); + }); +}); diff --git a/convex/lib/skillPresentationBackfill.ts b/convex/lib/skillPresentationBackfill.ts new file mode 100644 index 00000000..b4af655e --- /dev/null +++ b/convex/lib/skillPresentationBackfill.ts @@ -0,0 +1,55 @@ +import type { Doc } from "../_generated/dataModel"; +import { + isHostedSkillPresentationIconPath, + resolveSkillPresentation, + stripPresentationEmoji, + type OpenAiSkillPresentation, +} from "./skillPresentation"; +import { getFrontmatterMetadata, getFrontmatterValue } from "./skills"; + +export function resolveHistoricalSkillPresentation(args: { + slug: string; + currentDisplayName: string; + currentSummary?: string; + frontmatter: Doc<"skillVersions">["parsed"]["frontmatter"]; + openAi: OpenAiSkillPresentation; +}) { + const skillDisplayName = getFrontmatterValue(args.frontmatter, "name")?.trim(); + const defaultDisplayName = resolveSkillPresentation({ slug: args.slug }).displayName; + const publisherDisplayName = [skillDisplayName, defaultDisplayName].some( + (candidate) => + candidate && + stripPresentationEmoji(candidate) === stripPresentationEmoji(args.currentDisplayName), + ) + ? undefined + : args.currentDisplayName; + + const frontmatterMetadata = getFrontmatterMetadata(args.frontmatter); + const nestedDescription = + frontmatterMetadata && + typeof frontmatterMetadata === "object" && + !Array.isArray(frontmatterMetadata) && + typeof (frontmatterMetadata as Record).description === "string" + ? ((frontmatterMetadata as Record).description as string).trim() + : undefined; + const frontmatterDescription = + nestedDescription || getFrontmatterValue(args.frontmatter, "description")?.trim(); + const currentSummary = args.currentSummary?.trim(); + const publisherSummary = + currentSummary && currentSummary !== frontmatterDescription ? currentSummary : undefined; + + return resolveSkillPresentation({ + publisherDisplayName, + publisherSummary, + openAi: args.openAi, + skillDisplayName, + skillDescription: frontmatterDescription, + slug: args.slug, + }); +} + +export function preserveHistoricalHostedIcon( + ...icons: Array +): string | undefined { + return icons.find((icon): icon is string => isHostedSkillPresentationIconPath(icon)); +} diff --git a/convex/lib/skillPublish.test.ts b/convex/lib/skillPublish.test.ts index 88f9f3f8..57f120a5 100644 --- a/convex/lib/skillPublish.test.ts +++ b/convex/lib/skillPublish.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; import { MAX_PUBLISH_FILE_BYTES } from "./publishLimits"; import { @@ -12,6 +13,202 @@ vi.mock("./embeddings", () => ({ })); describe("skillPublish", () => { + it("normalizes agents/openai.yaml presentation metadata and hosts its icon", async () => { + const skillMarkdown = + "---\nname: Demo Skill\ndescription: SKILL.md summary.\n---\n# Demo Skill\n"; + const openAiYaml = + "interface:\n display_name: '✨ OpenAI Demo'\n short_description: OpenAI summary.\n icon_small: assets/missing.png\n icon_large: assets/icon.png\n"; + const iconBytes = validPng(); + const stored = new Map([ + ["_storage:skill", new Blob([skillMarkdown], { type: "text/markdown" })], + ["_storage:openai", new Blob([openAiYaml], { type: "application/yaml" })], + ["_storage:icon", new Blob([iconBytes], { type: "image/png" })], + ]); + const runMutation = vi.fn(async (_ref: unknown, args: Record) => { + if ("contentType" in args && "storageId" in args && !("version" in args)) { + return { ...args, _id: "skillPresentationAssets:1", createdAt: 1 }; + } + if ("version" in args && "embedding" in args) { + return { skillId: "skills:demo", versionId: "skillVersions:demo" }; + } + return null; + }); + const ctx = { + runAction: vi.fn(async () => true), + runQuery: vi + .fn() + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ _id: "users:1", handle: "demo", createdAt: 1 }) + .mockResolvedValueOnce(null), + runMutation, + scheduler: { runAfter: vi.fn() }, + storage: { + get: vi.fn(async (storageId: string) => stored.get(storageId) ?? null), + store: vi.fn(async () => "_storage:hosted-icon"), + delete: vi.fn(async () => undefined), + }, + }; + + await publishVersionForUser( + ctx as never, + "users:1" as never, + { + slug: "demo-skill", + displayName: "Demo Skill", + version: "1.0.0", + changelog: "Initial release", + files: [ + file("_storage:skill", "SKILL.md", skillMarkdown.length, "text/markdown"), + file("_storage:openai", "agents/openai.yaml", openAiYaml.length, "application/yaml"), + file("_storage:icon", "assets/icon.png", iconBytes.byteLength, "image/png"), + ], + }, + { + bypassGitHubAccountAge: true, + bypassQualityGate: true, + skipWebhook: true, + }, + ); + + const insertCall = runMutation.mock.calls.find( + ([, args]) => + "version" in (args as Record) && + "embedding" in (args as Record), + ); + expect(insertCall?.[1]).toMatchObject({ + displayName: "OpenAI Demo", + summary: "OpenAI summary.", + icon: expect.stringMatching(/^\/api\/v1\/skill-icons\/[a-f\d]{64}$/), + parsed: { + presentation: { + displayName: "OpenAI Demo", + summary: "OpenAI summary.", + icon: expect.stringMatching(/^\/api\/v1\/skill-icons\/[a-f\d]{64}$/), + }, + }, + }); + expect(ctx.storage.store).toHaveBeenCalledOnce(); + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + sha256: createHash("sha256").update(iconBytes).digest("hex"), + }), + ); + }); + + it("lets changed OpenAI metadata replace unchanged derived publish values", async () => { + const skillMarkdown = "---\nname: Demo Skill\ndescription: SKILL summary.\n---\n# Demo Skill\n"; + const openAiYaml = + "interface:\n display_name: OpenAI Demo v2\n short_description: OpenAI summary v2.\n"; + const stored = new Map([ + ["_storage:skill", new Blob([skillMarkdown], { type: "text/markdown" })], + ["_storage:openai", new Blob([openAiYaml], { type: "application/yaml" })], + ]); + const runMutation = vi.fn(async (_ref: unknown, args: Record) => + "version" in args && "embedding" in args + ? { skillId: "skills:demo", versionId: "skillVersions:v2" } + : null, + ); + const ctx = { + runQuery: vi + .fn() + .mockResolvedValueOnce({ + _id: "skills:demo", + slug: "demo-skill", + displayName: "OpenAI Demo v1", + summary: "OpenAI summary v1.", + latestVersionId: "skillVersions:v1", + }) + .mockResolvedValueOnce({ _id: "users:1", handle: "demo", createdAt: 1 }) + .mockResolvedValueOnce({ + _id: "skillVersions:v1", + parsed: { + frontmatter: {}, + presentation: { + displayName: "OpenAI Demo v1", + displayNameSource: "openai", + summary: "OpenAI summary v1.", + summarySource: "openai", + }, + }, + }), + runMutation, + scheduler: { runAfter: vi.fn() }, + storage: { + get: vi.fn(async (storageId: string) => stored.get(storageId) ?? null), + }, + }; + + await publishVersionForUser( + ctx as never, + "users:1" as never, + { + slug: "demo-skill", + displayName: "OpenAI Demo v1", + summary: "OpenAI summary v1.", + version: "2.0.0", + changelog: "Presentation refresh", + files: [ + file("_storage:skill", "SKILL.md", skillMarkdown.length, "text/markdown"), + file("_storage:openai", "agents/openai.yaml", openAiYaml.length, "application/yaml"), + ], + }, + { + bypassGitHubAccountAge: true, + bypassQualityGate: true, + skipWebhook: true, + }, + ); + + expect(runMutation).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + displayName: "OpenAI Demo v2", + summary: "OpenAI summary v2.", + parsed: { + frontmatter: expect.anything(), + metadata: undefined, + clawdis: undefined, + license: expect.anything(), + presentation: { + displayName: "OpenAI Demo v2", + displayNameSource: "openai", + summary: "OpenAI summary v2.", + summarySource: "openai", + }, + }, + }), + ); + }); + + it("rejects icon digest changes and propagates asset persistence failures", async () => { + const iconBytes = validPng(); + const iconFile = file("_storage:icon", "assets/icon.png", iconBytes.byteLength, "image/png"); + const storage = { + get: vi.fn(async () => new Blob([iconBytes], { type: "image/png" })), + store: vi.fn(async () => { + throw new Error("storage unavailable"); + }), + delete: vi.fn(async () => undefined), + }; + const ctx = { + runAction: vi.fn(async () => true), + runQuery: vi.fn(async () => null), + runMutation: vi.fn(), + storage, + }; + + await expect( + __test.hostDirectSkillPresentationIcon(ctx as never, [iconFile], [iconFile.path]), + ).rejects.toThrow(/changed during upload/i); + expect(storage.store).not.toHaveBeenCalled(); + + iconFile.sha256 = createHash("sha256").update(iconBytes).digest("hex"); + await expect( + __test.hostDirectSkillPresentationIcon(ctx as never, [iconFile], [iconFile.path]), + ).rejects.toThrow("storage unavailable"); + }); + it("publishes long display names without rewriting the stored label", async () => { const displayName = "A".repeat(120); const skillMarkdown = `---\ndescription: Long compatibility name.\n---\n# ${displayName}\n`; @@ -1302,3 +1499,22 @@ description: Expert guidance for sushi-rolls. expect(quality.decision).toBe("pass"); }); }); + +function file(storageId: string, path: string, size: number, contentType: string) { + return { + path, + size, + storageId: storageId as never, + sha256: "a".repeat(64), + contentType, + }; +} + +function validPng() { + return Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + "base64", + ), + ); +} diff --git a/convex/lib/skillPublish.ts b/convex/lib/skillPublish.ts index b1b261f1..2c3437d1 100644 --- a/convex/lib/skillPublish.ts +++ b/convex/lib/skillPublish.ts @@ -10,6 +10,10 @@ import semver from "semver"; import { api, internal } from "../_generated/api"; import type { Doc, Id } from "../_generated/dataModel"; import type { ActionCtx, MutationCtx } from "../_generated/server"; +import { + isDecodableSkillPresentationRaster, + storeSkillPresentationAsset, +} from "../skillPresentationAssets"; import { getSkillBadgeMap, isSkillHighlighted } from "./badges"; import { generateChangelogForPublish } from "./changelog"; import { generateEmbedding } from "./embeddings"; @@ -22,6 +26,14 @@ import { MAX_PUBLISH_TOTAL_BYTES, } from "./publishLimits"; import { isSkillCardPath } from "./skillCards"; +import { + MAX_SKILL_PRESENTATION_YAML_BYTES, + OPENAI_SKILL_PRESENTATION_PATH, + parseOpenAiSkillPresentation, + resolveSkillPresentation, + stripPresentationEmoji, + validateSkillPresentationIcon, +} from "./skillPresentation"; import { computeQualitySignals, evaluateQuality, @@ -190,7 +202,7 @@ async function publishVersionForUserInternal( const normalizedSlug = normalizeSkillSlug(args.slug); if (!normalizedSlug) throw new ConvexError("Slug is required."); - const displayName = args.displayName.trim(); + let displayName = stripPresentationEmoji(args.displayName.trim()); if (!displayName) throw new ConvexError("Display name required"); if (!semver.valid(version)) { throw new ConvexError("Version must be valid semver"); @@ -300,16 +312,64 @@ async function publishVersionForUserInternal( if (explicitSummary && explicitSummary.length > MAX_PUBLISH_SUMMARY_LENGTH) { throw new ConvexError(`Summary must be ${MAX_PUBLISH_SUMMARY_LENGTH} characters or less`); } + const openAiFile = publishFiles.find( + (file) => file.path.toLowerCase() === OPENAI_SKILL_PRESENTATION_PATH, + ); + const openAiPresentation = + openAiFile && openAiFile.size <= MAX_SKILL_PRESENTATION_YAML_BYTES + ? await fetchText(ctx, openAiFile.storageId) + .then(parseOpenAiSkillPresentation) + .catch(() => null) + : null; + const existingLatestVersion = existingSkill?.latestVersionId + ? ((await ctx.runQuery(internal.skills.getVersionByIdInternal, { + versionId: existingSkill.latestVersionId, + })) as Doc<"skillVersions"> | null) + : null; + const existingPresentation = existingLatestVersion?.parsed.presentation; + const skillDisplayName = getFrontmatterValue(frontmatter, "name")?.trim(); + const defaultDisplayName = resolveSkillPresentation({ slug }).displayName; + const reusesDerivedDisplayName = + existingPresentation?.displayNameSource !== undefined && + existingPresentation.displayNameSource !== "publisher" && + existingPresentation.displayName === displayName && + existingSkill?.displayName === displayName; + const publisherDisplayName = + reusesDerivedDisplayName || + [skillDisplayName, defaultDisplayName].some( + (candidate) => candidate && stripPresentationEmoji(candidate) === displayName, + ) + ? undefined + : displayName; + const reusesDerivedSummary = + explicitSummary !== undefined && + existingPresentation?.summarySource !== undefined && + existingPresentation.summarySource !== "publisher" && + existingPresentation.summary === explicitSummary && + existingSkill?.summary === explicitSummary; + const publisherSummary = + explicitSummary && explicitSummary !== summaryFromFrontmatter && !reusesDerivedSummary + ? explicitSummary + : undefined; + const presentation = resolveSkillPresentation({ + publisherDisplayName, + publisherSummary, + openAi: openAiPresentation, + skillDisplayName, + skillDescription: summaryFromFrontmatter, + slug, + }); + displayName = presentation.displayName; const shouldDeferAiEnrichment = options.stagePrePublicationChecks === true; const summary = - explicitSummary || + publisherSummary || (shouldDeferAiEnrichment - ? (summaryFromFrontmatter ?? existingSkill?.summary ?? "") + ? (presentation.summary ?? existingSkill?.summary ?? "") : await generateSkillSummary({ slug, displayName, readmeText, - currentSummary: summaryFromFrontmatter ?? existingSkill?.summary ?? undefined, + currentSummary: presentation.summary ?? existingSkill?.summary ?? undefined, })); let qualityAssessment: QualityAssessment | null = null; @@ -425,6 +485,7 @@ async function publishVersionForUserInternal( throw new ConvexError(formatEmbeddingError(error)); }), ]); + const icon = await hostDirectSkillPresentationIcon(ctx, publishFiles, presentation.iconPaths); const skillInsertArgs = { userId, @@ -458,18 +519,27 @@ async function publishVersionForUserInternal( metadata, clawdis, license: PLATFORM_SKILL_LICENSE, + presentation: { + displayName, + displayNameSource: presentation.displayNameSource, + ...(summary ? { summary } : {}), + ...(summary ? { summarySource: presentation.summarySource ?? ("generated" as const) } : {}), + ...(icon ? { icon } : {}), + }, }, summary, + icon, staticScan, embedding, deferredAiEnrichment: shouldDeferAiEnrichment ? ({ - summary: explicitSummary - ? { mode: "literal", literal: explicitSummary } - : { - mode: "generate", - currentSummary: summaryFromFrontmatter ?? existingSkill?.summary ?? undefined, - }, + summary: + publisherSummary || presentation.summary + ? { mode: "literal", literal: publisherSummary ?? presentation.summary } + : { + mode: "generate", + currentSummary: existingSkill?.summary ?? undefined, + }, changelog: { source: changelogSource, supplied: suppliedChangelog, @@ -923,6 +993,7 @@ export const __test = { toStructuralFingerprint, derivePublishFilesFromStorage, buildSkillPublishAttemptIdempotencyKey, + hostDirectSkillPresentationIcon, }; export async function queueHighlightedWebhook(ctx: MutationCtx, skillId: Id<"skills">) { @@ -959,6 +1030,51 @@ export async function fetchText( return text; } +async function hostDirectSkillPresentationIcon( + ctx: Pick, + files: SafePublishFile[], + iconPaths: string[] | undefined, +) { + for (const iconPath of iconPaths ?? []) { + const file = files.find((candidate) => candidate.path === iconPath); + if (!file) continue; + const blob = await ctx.storage.get(file.storageId); + if (!blob) { + throw new ConvexError("Skill presentation icon could not be read. Please retry."); + } + const bytes = new Uint8Array(await blob.arrayBuffer()); + let validated: ReturnType; + try { + validated = validateSkillPresentationIcon({ + path: file.path, + bytes, + contentType: file.contentType ?? blob.type, + }); + } catch { + continue; + } + if ( + validated.contentType !== "image/svg+xml" && + !(await isDecodableSkillPresentationRaster(ctx, { + bytes, + contentType: validated.contentType, + })) + ) { + continue; + } + const sha256 = await sha256Hex(bytes); + if (sha256 !== file.sha256.toLowerCase()) { + throw new ConvexError("Skill presentation icon changed during upload. Please retry."); + } + return await storeSkillPresentationAsset(ctx, { + bytes, + sha256, + contentType: validated.contentType, + }); + } + return undefined; +} + async function fetchPreviewText( ctx: { storage: { get: (id: Id<"_storage">) => Promise } }, storageId: Id<"_storage">, diff --git a/convex/schema.ts b/convex/schema.ts index f1e52e11..0eb42eae 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -505,6 +505,7 @@ const githubSkillCandidates = defineTable({ githubContentHash: v.string(), displayName: v.string(), summary: v.optional(v.string()), + icon: v.optional(v.string()), upstreamVersion: v.optional(v.string()), skillMarkdownPath: v.optional(v.string()), skillMarkdown: v.optional(v.string()), @@ -1089,6 +1090,29 @@ const skillVersions = defineTable({ clawdis: v.optional(v.any()), moltbot: v.optional(v.any()), license: v.optional(v.literal(PLATFORM_SKILL_LICENSE)), + presentation: v.optional( + v.object({ + displayName: v.string(), + displayNameSource: v.optional( + v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("slug"), + ), + ), + summary: v.optional(v.string()), + summarySource: v.optional( + v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("generated"), + ), + ), + icon: v.optional(v.string()), + }), + ), }), createdBy: v.id("users"), createdAt: v.number(), @@ -1176,6 +1200,19 @@ const publishAttemptStatusValidator = v.union( v.literal("expired"), ); +const skillPresentationAssets = defineTable({ + sha256: v.string(), + storageId: v.id("_storage"), + contentType: v.union( + v.literal("image/png"), + v.literal("image/jpeg"), + v.literal("image/webp"), + v.literal("image/svg+xml"), + ), + size: v.number(), + createdAt: v.number(), +}).index("by_sha256", ["sha256"]); + const publishAttemptCheckStateValidator = v.object({ status: v.union( v.literal("pending"), @@ -1978,6 +2015,29 @@ const skillScanRequests = defineTable({ clawdis: v.optional(v.any()), moltbot: v.optional(v.any()), license: v.optional(v.literal(PLATFORM_SKILL_LICENSE)), + presentation: v.optional( + v.object({ + displayName: v.string(), + displayNameSource: v.optional( + v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("slug"), + ), + ), + summary: v.optional(v.string()), + summarySource: v.optional( + v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("generated"), + ), + ), + icon: v.optional(v.string()), + }), + ), }), ), sha256hash: v.optional(v.string()), @@ -3961,6 +4021,7 @@ export default defineSchema({ packageTopicSearchDigest, packagePluginCategorySearchDigest, skillVersions, + skillPresentationAssets, publishAttempts, skillVersionFingerprints, skillBadges, diff --git a/convex/securityScan.ts b/convex/securityScan.ts index 15b93179..ef1f1887 100644 --- a/convex/securityScan.ts +++ b/convex/securityScan.ts @@ -1394,6 +1394,29 @@ export const prepareGitHubSkillScanRequestInternal = internalMutation({ force: v.optional(v.boolean()), parsed: v.object({ frontmatter: v.record(v.string(), v.any()), + presentation: v.optional( + v.object({ + displayName: v.string(), + displayNameSource: v.optional( + v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("slug"), + ), + ), + summary: v.optional(v.string()), + summarySource: v.optional( + v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("generated"), + ), + ), + icon: v.optional(v.string()), + }), + ), }), staticScan: staticScanResultValidator, }, diff --git a/convex/skillPresentationAssets.ts b/convex/skillPresentationAssets.ts new file mode 100644 index 00000000..f13dcb4c --- /dev/null +++ b/convex/skillPresentationAssets.ts @@ -0,0 +1,101 @@ +import { v } from "convex/values"; +import { internal } from "./_generated/api"; +import type { Doc } from "./_generated/dataModel"; +import type { ActionCtx, QueryCtx } from "./_generated/server"; +import { internalMutation, internalQuery } from "./functions"; +import { buildSkillPresentationIconPath } from "./lib/skillPresentation"; + +const skillPresentationContentTypeValidator = v.union( + v.literal("image/png"), + v.literal("image/jpeg"), + v.literal("image/webp"), + v.literal("image/svg+xml"), +); + +type SkillPresentationContentType = Doc<"skillPresentationAssets">["contentType"]; + +export const getBySha256Internal = internalQuery({ + args: { sha256: v.string() }, + handler: async (ctx, args) => getSkillPresentationAssetByHash(ctx, args.sha256), +}); + +export const registerInternal = internalMutation({ + args: { + sha256: v.string(), + storageId: v.id("_storage"), + contentType: skillPresentationContentTypeValidator, + size: v.number(), + }, + handler: async (ctx, args) => { + const sha256 = normalizeSha256(args.sha256); + const existing = await getSkillPresentationAssetByHash(ctx, sha256); + if (existing) return existing; + const assetId = await ctx.db.insert("skillPresentationAssets", { + ...args, + sha256, + createdAt: Date.now(), + }); + return (await ctx.db.get(assetId)) as Doc<"skillPresentationAssets">; + }, +}); + +export async function storeSkillPresentationAsset( + ctx: Pick, + args: { + bytes: Uint8Array; + sha256: string; + contentType: SkillPresentationContentType; + }, +) { + const sha256 = normalizeSha256(args.sha256); + const existing = (await ctx.runQuery(internal.skillPresentationAssets.getBySha256Internal, { + sha256, + })) as Doc<"skillPresentationAssets"> | null; + if (existing) return buildSkillPresentationIconPath(existing.sha256); + + const storageId = await ctx.storage.store( + new Blob([new Uint8Array(args.bytes)], { type: args.contentType }), + ); + try { + const asset = (await ctx.runMutation(internal.skillPresentationAssets.registerInternal, { + sha256, + storageId, + contentType: args.contentType, + size: args.bytes.byteLength, + })) as Doc<"skillPresentationAssets">; + if (asset.storageId !== storageId) await ctx.storage.delete(storageId); + return buildSkillPresentationIconPath(asset.sha256); + } catch (error) { + await ctx.storage.delete(storageId); + throw error; + } +} + +export async function isDecodableSkillPresentationRaster( + ctx: Pick, + args: { + bytes: Uint8Array; + contentType: Exclude; + }, +) { + return (await ctx.runAction(internal.skillPresentationImageNode.validateRasterInternal, { + bytes: new Uint8Array(args.bytes).buffer, + contentType: args.contentType, + })) as boolean; +} + +async function getSkillPresentationAssetByHash(ctx: Pick, rawSha256: string) { + const sha256 = normalizeSha256(rawSha256); + return await ctx.db + .query("skillPresentationAssets") + .withIndex("by_sha256", (query) => query.eq("sha256", sha256)) + .unique(); +} + +function normalizeSha256(value: string) { + const normalized = value.trim().toLowerCase(); + if (!/^[a-f\d]{64}$/.test(normalized)) throw new Error("Invalid skill icon SHA-256 digest."); + return normalized; +} + +export type { SkillPresentationContentType }; diff --git a/convex/skillPresentationAssetsHttp.test.ts b/convex/skillPresentationAssetsHttp.test.ts new file mode 100644 index 00000000..39764d72 --- /dev/null +++ b/convex/skillPresentationAssetsHttp.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, vi } from "vitest"; +import { skillPresentationAssetHandler } from "./skillPresentationAssetsHttp"; + +const sha256 = "a".repeat(64); +const bytes = Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10, 0]); +const asset = { + _id: "skillPresentationAssets:1", + _creationTime: 1, + sha256, + storageId: "_storage:icon", + contentType: "image/png", + size: bytes.byteLength, + createdAt: 1, +}; + +describe("skillPresentationAssetHandler", () => { + it("serves immutable exact bytes with image security headers", async () => { + const ctx = { + runQuery: vi.fn(async () => asset), + storage: { get: vi.fn(async () => new Blob([bytes], { type: "image/png" })) }, + }; + const response = await skillPresentationAssetHandler( + ctx as never, + new Request(`https://clawhub.ai/api/v1/skill-icons/${sha256}`), + ); + + expect(response.status).toBe(200); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(bytes); + expect(response.headers.get("content-type")).toBe("image/png"); + expect(response.headers.get("cache-control")).toContain("immutable"); + expect(response.headers.get("etag")).toBe(`"sha256:${sha256}"`); + expect(response.headers.get("x-content-type-options")).toBe("nosniff"); + }); + + it("returns 304 for the immutable content validator", async () => { + const ctx = { + runQuery: vi.fn(async () => asset), + storage: { get: vi.fn() }, + }; + const response = await skillPresentationAssetHandler( + ctx as never, + new Request(`https://clawhub.ai/api/v1/skill-icons/${sha256}`, { + headers: { "If-None-Match": `"sha256:${sha256}"` }, + }), + ); + + expect(response.status).toBe(304); + expect(ctx.storage.get).not.toHaveBeenCalled(); + }); +}); diff --git a/convex/skillPresentationAssetsHttp.ts b/convex/skillPresentationAssetsHttp.ts new file mode 100644 index 00000000..2a13040a --- /dev/null +++ b/convex/skillPresentationAssetsHttp.ts @@ -0,0 +1,59 @@ +import { internal } from "./_generated/api"; +import type { Doc } from "./_generated/dataModel"; +import type { ActionCtx } from "./_generated/server"; +import { httpAction } from "./functions"; +import { corsHeaders, mergeHeaders } from "./lib/httpHeaders"; + +const ICON_PATH_PREFIX = "/api/v1/skill-icons/"; + +export async function skillPresentationAssetHandler(ctx: ActionCtx, request: Request) { + const url = new URL(request.url); + const sha256 = url.pathname.startsWith(ICON_PATH_PREFIX) + ? url.pathname.slice(ICON_PATH_PREFIX.length).toLowerCase() + : ""; + if (!/^[a-f\d]{64}$/.test(sha256)) return iconText("Not found", 404); + + const asset = (await ctx.runQuery(internal.skillPresentationAssets.getBySha256Internal, { + sha256, + })) as Doc<"skillPresentationAssets"> | null; + if (!asset) return iconText("Not found", 404); + + const etag = `"sha256:${asset.sha256}"`; + if (request.headers.get("if-none-match") === etag) { + return new Response(null, { status: 304, headers: iconHeaders(asset, etag) }); + } + const blob = await ctx.storage.get(asset.storageId); + if (!blob) return iconText("Not found", 404); + return new Response(new Uint8Array(await blob.arrayBuffer()), { + status: 200, + headers: iconHeaders(asset, etag), + }); +} + +function iconHeaders(asset: Doc<"skillPresentationAssets">, etag: string) { + return mergeHeaders( + { + "Cache-Control": "public, max-age=31536000, immutable", + "Content-Type": asset.contentType, + "Content-Length": String(asset.size), + "Content-Security-Policy": "default-src 'none'; style-src 'unsafe-inline'; sandbox", + ETag: etag, + "X-Content-SHA256": asset.sha256, + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", + }, + corsHeaders(), + ); +} + +function iconText(value: string, status: number) { + return new Response(value, { + status, + headers: mergeHeaders( + { "Cache-Control": "no-store", "Content-Type": "text/plain; charset=utf-8" }, + corsHeaders(), + ), + }); +} + +export const skillPresentationAssetHttp = httpAction(skillPresentationAssetHandler); diff --git a/convex/skillPresentationBackfill.ts b/convex/skillPresentationBackfill.ts new file mode 100644 index 00000000..6daa2616 --- /dev/null +++ b/convex/skillPresentationBackfill.ts @@ -0,0 +1,461 @@ +import { ConvexError, v } from "convex/values"; +import { internal } from "./_generated/api"; +import type { Doc, Id } from "./_generated/dataModel"; +import type { ActionCtx } from "./_generated/server"; +import { internalAction, internalMutation, internalQuery } from "./_generated/server"; +import { sha256Hex } from "./lib/clawpack"; +import { + MAX_SKILL_PRESENTATION_YAML_BYTES, + OPENAI_SKILL_PRESENTATION_PATH, + buildSkillPresentationIconPath, + parseOpenAiSkillPresentation, + validateSkillPresentationIcon, +} from "./lib/skillPresentation"; +import { + preserveHistoricalHostedIcon, + resolveHistoricalSkillPresentation, +} from "./lib/skillPresentationBackfill"; +import { syncSkillSearchDigestForSkill } from "./lib/skillSearchDigest"; +import { + isDecodableSkillPresentationRaster, + storeSkillPresentationAsset, + type SkillPresentationContentType, +} from "./skillPresentationAssets"; + +const DEFAULT_BATCH_SIZE = 20; +const MAX_BATCH_SIZE = 50; +const DEFAULT_MAX_BATCHES = 50; +const MAX_MAX_BATCHES = 500; +const APPLY_CONFIRM = "backfill-skill-presentation-metadata"; +const MAX_SAMPLES = 25; + +type BackfillCandidate = { + skillId: Id<"skills">; + slug: string; + displayName: string; + summary?: string; + skillIcon?: string; + versionId: Id<"skillVersions">; + versionIcon?: string; + parsed: Doc<"skillVersions">["parsed"]; + files: Doc<"skillVersions">["files"]; +}; + +type BackfillPage = { + candidates: BackfillCandidate[]; + cursor: string | null; + isDone: boolean; + stats: { + skillsScanned: number; + skippedDeleted: number; + skippedGitHub: number; + missingLatestVersion: number; + unavailableLatestVersion: number; + }; +}; + +type BackfillStats = BackfillPage["stats"] & { + metadataFilesFound: number; + eligibleSkills: number; + eligibleSkillsWithIcon: number; + missingMetadataBlob: number; + invalidMetadata: number; + missingIconFile: number; + invalidIcon: number; + alreadyCurrent: number; + wouldPatchSkills: number; + patchedSkills: number; + changedBeforeApply: number; +}; + +type BackfillSample = { + skillId: Id<"skills">; + versionId: Id<"skillVersions">; + slug: string; + displayName: string; + nextDisplayName: string; + hasIcon: boolean; +}; + +type BackfillResult = { + ok: true; + dryRun: boolean; + confirmRequired?: typeof APPLY_CONFIRM; + cursor: string | null; + isDone: boolean; + stats: BackfillStats; + samples: BackfillSample[]; +}; + +type PreparedIcon = { + bytes: Uint8Array; + contentType: SkillPresentationContentType; + sha256: string; +}; + +export const getBackfillPageInternal = internalQuery({ + args: { + cursor: v.optional(v.string()), + batchSize: v.optional(v.number()), + }, + handler: async (ctx, args): Promise => { + const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE); + const page = await ctx.db + .query("skills") + .order("asc") + .paginate({ cursor: args.cursor ?? null, numItems: batchSize }); + const candidates: BackfillCandidate[] = []; + const stats = { + skillsScanned: page.page.length, + skippedDeleted: 0, + skippedGitHub: 0, + missingLatestVersion: 0, + unavailableLatestVersion: 0, + }; + + for (const skill of page.page) { + if (skill.softDeletedAt !== undefined) { + stats.skippedDeleted += 1; + continue; + } + if (skill.installKind === "github") { + stats.skippedGitHub += 1; + continue; + } + if (!skill.latestVersionId) { + stats.missingLatestVersion += 1; + continue; + } + const version = await ctx.db.get(skill.latestVersionId); + if ( + !version || + version.skillId !== skill._id || + version.softDeletedAt !== undefined || + version.ownerDeletedAt !== undefined || + version.publicationStatus === "pending" || + version.publicationStatus === "blocked" + ) { + stats.unavailableLatestVersion += 1; + continue; + } + candidates.push({ + skillId: skill._id, + slug: skill.slug, + displayName: skill.displayName, + summary: skill.summary, + skillIcon: skill.icon, + versionId: version._id, + versionIcon: version.icon, + parsed: version.parsed, + files: version.files, + }); + } + + return { + candidates, + cursor: page.continueCursor, + isDone: page.isDone, + stats, + }; + }, +}); + +export const applyBackfillPatchInternal = internalMutation({ + args: { + confirm: v.string(), + skillId: v.id("skills"), + versionId: v.id("skillVersions"), + displayName: v.string(), + displayNameSource: v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("slug"), + ), + summary: v.optional(v.string()), + summarySource: v.optional( + v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("generated"), + ), + ), + icon: v.optional(v.string()), + }, + handler: async (ctx, args) => { + if (args.confirm !== APPLY_CONFIRM) { + throw new ConvexError(`Pass confirm="${APPLY_CONFIRM}" to apply.`); + } + const skill = await ctx.db.get(args.skillId); + const version = await ctx.db.get(args.versionId); + if ( + !skill || + !version || + skill.latestVersionId !== version._id || + version.skillId !== skill._id || + skill.softDeletedAt !== undefined || + version.softDeletedAt !== undefined || + version.ownerDeletedAt !== undefined || + version.publicationStatus === "pending" || + version.publicationStatus === "blocked" + ) { + return { patched: false as const, reason: "changed_before_apply" as const }; + } + + const presentation = { + displayName: args.displayName, + displayNameSource: args.displayNameSource, + ...(args.summary && args.summarySource + ? { summary: args.summary, summarySource: args.summarySource } + : {}), + ...(args.icon ? { icon: args.icon } : {}), + }; + await ctx.db.patch(version._id, { + parsed: { ...version.parsed, presentation }, + // Legacy publisher icons are independent of agents/openai.yaml metadata. + ...(args.icon ? { icon: args.icon } : {}), + }); + await ctx.db.patch(skill._id, { + displayName: args.displayName, + ...(args.summary ? { summary: args.summary } : {}), + ...(args.icon ? { icon: args.icon } : {}), + ...(skill.latestVersionSummary && args.summary + ? { + latestVersionSummary: { + ...skill.latestVersionSummary, + description: args.summary, + }, + } + : {}), + }); + await syncSkillSearchDigestForSkill(ctx, await ctx.db.get(skill._id)); + return { patched: true as const }; + }, +}); + +export const runInternal = internalAction({ + args: { + dryRun: v.optional(v.boolean()), + confirm: v.optional(v.string()), + cursor: v.optional(v.string()), + batchSize: v.optional(v.number()), + maxBatches: v.optional(v.number()), + }, + handler: async (ctx, args): Promise => { + // This stays action-driven instead of using @convex-dev/migrations because + // every candidate needs storage reads and raster validation before a write. + const dryRun = args.dryRun !== false; + if (!dryRun && args.confirm !== APPLY_CONFIRM) { + throw new ConvexError(`Pass confirm="${APPLY_CONFIRM}" to apply.`); + } + const maxBatches = clampInt(args.maxBatches ?? DEFAULT_MAX_BATCHES, 1, MAX_MAX_BATCHES); + const stats = emptyStats(); + const samples: BackfillSample[] = []; + let cursor: string | null = args.cursor ?? null; + let isDone = false; + + for (let batchIndex = 0; batchIndex < maxBatches; batchIndex += 1) { + const page = (await ctx.runQuery(internal.skillPresentationBackfill.getBackfillPageInternal, { + cursor: cursor ?? undefined, + batchSize: args.batchSize, + })) as BackfillPage; + addPageStats(stats, page.stats); + cursor = page.cursor; + isDone = page.isDone; + + for (const candidate of page.candidates) { + const metadataFile = candidate.files.find( + (file) => file.path.toLowerCase() === OPENAI_SKILL_PRESENTATION_PATH, + ); + if (!metadataFile) continue; + stats.metadataFilesFound += 1; + const metadataBlob = await ctx.storage.get(metadataFile.storageId); + if (!metadataBlob) { + stats.missingMetadataBlob += 1; + continue; + } + if (metadataBlob.size > MAX_SKILL_PRESENTATION_YAML_BYTES) { + stats.invalidMetadata += 1; + continue; + } + const openAi = parseOpenAiSkillPresentation(await metadataBlob.text()); + if (!openAi) { + stats.invalidMetadata += 1; + continue; + } + + stats.eligibleSkills += 1; + const presentation = resolveHistoricalSkillPresentation({ + slug: candidate.slug, + currentDisplayName: candidate.displayName, + currentSummary: candidate.summary, + frontmatter: candidate.parsed.frontmatter, + openAi, + }); + const preparedIcon = await prepareIcon(ctx, candidate.files, presentation.iconPaths, stats); + const iconPath = preparedIcon + ? buildSkillPresentationIconPath(preparedIcon.sha256) + : preserveHistoricalHostedIcon( + candidate.parsed.presentation?.icon, + candidate.versionIcon, + candidate.skillIcon, + ); + if (preparedIcon) stats.eligibleSkillsWithIcon += 1; + const nextPresentation = { + displayName: presentation.displayName, + displayNameSource: presentation.displayNameSource, + ...(presentation.summary && presentation.summarySource + ? { summary: presentation.summary, summarySource: presentation.summarySource } + : {}), + ...(iconPath ? { icon: iconPath } : {}), + }; + const alreadyCurrent = + samePresentation(candidate.parsed.presentation, nextPresentation) && + candidate.displayName === presentation.displayName && + (candidate.summary ?? undefined) === (presentation.summary ?? undefined) && + // An absent presentation icon must not clear an unrelated publisher icon. + (!iconPath || (candidate.versionIcon === iconPath && candidate.skillIcon === iconPath)); + if (alreadyCurrent) { + stats.alreadyCurrent += 1; + continue; + } + + stats.wouldPatchSkills += 1; + if (samples.length < MAX_SAMPLES) { + samples.push({ + skillId: candidate.skillId, + versionId: candidate.versionId, + slug: candidate.slug, + displayName: candidate.displayName, + nextDisplayName: presentation.displayName, + hasIcon: Boolean(preparedIcon), + }); + } + if (dryRun) continue; + const storedIcon = preparedIcon + ? await storeSkillPresentationAsset(ctx, preparedIcon) + : iconPath; + const result = (await ctx.runMutation( + internal.skillPresentationBackfill.applyBackfillPatchInternal, + { + confirm: args.confirm as string, + skillId: candidate.skillId, + versionId: candidate.versionId, + displayName: presentation.displayName, + displayNameSource: presentation.displayNameSource, + summary: presentation.summary, + summarySource: presentation.summarySource, + icon: storedIcon, + }, + )) as { patched: boolean; reason?: "changed_before_apply" }; + if (result.patched) stats.patchedSkills += 1; + else stats.changedBeforeApply += 1; + } + + if (isDone) break; + } + + return { + ok: true, + dryRun, + ...(dryRun ? { confirmRequired: APPLY_CONFIRM } : {}), + cursor, + isDone, + stats, + samples, + }; + }, +}); + +async function prepareIcon( + ctx: ActionCtx, + files: Doc<"skillVersions">["files"], + iconPaths: string[] | undefined, + stats: BackfillStats, +): Promise { + for (const iconPath of iconPaths ?? []) { + const file = files.find((candidate) => candidate.path === iconPath); + if (!file) { + stats.missingIconFile += 1; + continue; + } + const blob = await ctx.storage.get(file.storageId); + if (!blob) { + stats.missingIconFile += 1; + continue; + } + const bytes = new Uint8Array(await blob.arrayBuffer()); + try { + const validated = validateSkillPresentationIcon({ + path: iconPath, + bytes, + contentType: file.contentType, + }); + if ( + validated.contentType !== "image/svg+xml" && + !(await isDecodableSkillPresentationRaster(ctx, { + bytes, + contentType: validated.contentType, + })) + ) { + stats.invalidIcon += 1; + continue; + } + return { + bytes, + contentType: validated.contentType, + sha256: await sha256Hex(bytes), + }; + } catch { + stats.invalidIcon += 1; + } + } + return undefined; +} + +function emptyStats(): BackfillStats { + return { + skillsScanned: 0, + skippedDeleted: 0, + skippedGitHub: 0, + missingLatestVersion: 0, + unavailableLatestVersion: 0, + metadataFilesFound: 0, + eligibleSkills: 0, + eligibleSkillsWithIcon: 0, + missingMetadataBlob: 0, + invalidMetadata: 0, + missingIconFile: 0, + invalidIcon: 0, + alreadyCurrent: 0, + wouldPatchSkills: 0, + patchedSkills: 0, + changedBeforeApply: 0, + }; +} + +function addPageStats(target: BackfillStats, page: BackfillPage["stats"]) { + target.skillsScanned += page.skillsScanned; + target.skippedDeleted += page.skippedDeleted; + target.skippedGitHub += page.skippedGitHub; + target.missingLatestVersion += page.missingLatestVersion; + target.unavailableLatestVersion += page.unavailableLatestVersion; +} + +function samePresentation( + current: Doc<"skillVersions">["parsed"]["presentation"], + next: NonNullable["parsed"]["presentation"]>, +) { + return ( + current?.displayName === next.displayName && + current.displayNameSource === next.displayNameSource && + (current.summary ?? undefined) === (next.summary ?? undefined) && + (current.summarySource ?? undefined) === (next.summarySource ?? undefined) && + (current.icon ?? undefined) === (next.icon ?? undefined) + ); +} + +function clampInt(value: number, min: number, max: number) { + return Math.max(min, Math.min(max, Math.floor(value))); +} diff --git a/convex/skillPresentationImageNode.test.ts b/convex/skillPresentationImageNode.test.ts new file mode 100644 index 00000000..d89317c5 --- /dev/null +++ b/convex/skillPresentationImageNode.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { validateRasterInternal } from "./skillPresentationImageNode"; + +type WrappedHandler = { + _handler: ( + ctx: unknown, + args: { bytes: ArrayBuffer; contentType: "image/png" }, + ) => Promise; +}; + +const validateRaster = (validateRasterInternal as unknown as WrappedHandler)._handler; +const png = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + "base64", + ), +); + +describe("validateRasterInternal", () => { + it("fully decodes valid raster bytes and rejects truncated images", async () => { + await expect( + validateRaster({}, { bytes: new Uint8Array(png).buffer, contentType: "image/png" }), + ).resolves.toBe(true); + await expect( + validateRaster( + {}, + { bytes: new Uint8Array(png.slice(0, 40)).buffer, contentType: "image/png" }, + ), + ).resolves.toBe(false); + }); +}); diff --git a/convex/skillPresentationImageNode.ts b/convex/skillPresentationImageNode.ts new file mode 100644 index 00000000..559bac57 --- /dev/null +++ b/convex/skillPresentationImageNode.ts @@ -0,0 +1,34 @@ +"use node"; + +import { v } from "convex/values"; +import sharp from "sharp"; +import { internalAction } from "./_generated/server"; + +const MAX_ICON_PIXELS = 2048 * 2048; + +export const validateRasterInternal = internalAction({ + args: { + bytes: v.bytes(), + contentType: v.union(v.literal("image/png"), v.literal("image/jpeg"), v.literal("image/webp")), + }, + handler: async (_ctx, args) => { + try { + const input = Buffer.from(args.bytes); + const image = sharp(input, { failOn: "warning", limitInputPixels: MAX_ICON_PIXELS }); + const metadata = await image.metadata(); + const expectedFormat = args.contentType === "image/jpeg" ? "jpeg" : args.contentType.slice(6); + if ( + metadata.format !== expectedFormat || + !metadata.width || + !metadata.height || + metadata.width * metadata.height > MAX_ICON_PIXELS + ) { + return false; + } + await image.raw().toBuffer(); + return true; + } catch { + return false; + } + }, +}); diff --git a/convex/skills.backportLatest.test.ts b/convex/skills.backportLatest.test.ts index 9d092f89..e13077ef 100644 --- a/convex/skills.backportLatest.test.ts +++ b/convex/skills.backportLatest.test.ts @@ -526,6 +526,32 @@ describe("skills.insertVersion latest-tag protection", () => { }); }); + it("clears a stale hosted presentation icon when the new latest omits it", async () => { + const skill = buildExistingSkill({ + icon: `/api/v1/skill-icons/${"a".repeat(64)}`, + }); + const { ctx, captured } = buildCtx(skill); + + await insertVersionHandler( + ctx as never, + buildPublishArgs({ version: "2.1.0", icon: undefined }) as never, + ); + + expect(captured.skillPatches.at(-1)).toHaveProperty("icon", undefined); + }); + + it("preserves a legacy publisher icon when the new latest omits presentation metadata", async () => { + const skill = buildExistingSkill({ icon: "lucide:Plug" }); + const { ctx, captured } = buildCtx(skill); + + await insertVersionHandler( + ctx as never, + buildPublishArgs({ version: "2.1.0", icon: undefined }) as never, + ); + + expect(captured.skillPatches.at(-1)).toMatchObject({ icon: "lucide:Plug" }); + }); + it("publishes suspicious prepublication results as active and flagged", async () => { const skill = buildExistingSkill(); const { ctx, captured } = buildCtx(skill); diff --git a/convex/skills.ts b/convex/skills.ts index dbb268ce..6f0f5b9f 100644 --- a/convex/skills.ts +++ b/convex/skills.ts @@ -137,6 +137,7 @@ import { sourceSkillVersionFiles, } from "./lib/skillCards"; import { isPublicSkillVersionAvailableForSkill } from "./lib/skillFileAccess"; +import { isHostedSkillPresentationIconPath } from "./lib/skillPresentation"; import { fetchText, queueHighlightedWebhook, @@ -575,6 +576,8 @@ function latestVersionSummaryFromSkillVersion( function skillSummaryFromSkillVersion( version: Pick, "parsed"> | null | undefined, ) { + const presentationSummary = version?.parsed?.presentation?.summary?.trim(); + if (presentationSummary) return presentationSummary; return version?.parsed?.frontmatter ? getFrontmatterValue(version.parsed.frontmatter, "description")?.trim() || undefined : undefined; @@ -583,6 +586,8 @@ function skillSummaryFromSkillVersion( function skillDisplayNameFromSkillVersion( version: Pick, "parsed"> | null | undefined, ) { + const presentationDisplayName = version?.parsed?.presentation?.displayName?.trim(); + if (presentationDisplayName) return presentationDisplayName; return version?.parsed?.frontmatter ? getFrontmatterValue(version.parsed.frontmatter, "name")?.trim() || undefined : undefined; @@ -2171,9 +2176,7 @@ function toPublicSkillVersion( version: Doc<"skillVersions"> | null | undefined, ): PublicSkillVersion | null { if (!version) return null; - const description = version.parsed?.frontmatter - ? getFrontmatterValue(version.parsed.frontmatter, "description")?.trim() - : undefined; + const description = skillSummaryFromSkillVersion(version); return { _id: version._id, _creationTime: version._creationTime, @@ -12137,6 +12140,7 @@ type SkillPendingPublishArgs = { userId: Id<"users">; ownerPublisherId?: Id<"publishers">; displayName: string; + icon?: string; version: string; changelog: string; changelogSource?: "auto" | "user"; @@ -12200,6 +12204,7 @@ export const insertVersion = internalMutation({ migrateOwner: v.optional(v.boolean()), slug: v.string(), displayName: v.string(), + icon: v.optional(v.string()), version: v.string(), changelog: v.string(), changelogSource: v.optional(v.union(v.literal("auto"), v.literal("user"))), @@ -12240,6 +12245,29 @@ export const insertVersion = internalMutation({ metadata: v.optional(v.any()), clawdis: v.optional(v.any()), license: v.optional(v.literal(PLATFORM_SKILL_LICENSE)), + presentation: v.optional( + v.object({ + displayName: v.string(), + displayNameSource: v.optional( + v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("slug"), + ), + ), + summary: v.optional(v.string()), + summarySource: v.optional( + v.union( + v.literal("publisher"), + v.literal("openai"), + v.literal("skill"), + v.literal("generated"), + ), + ), + icon: v.optional(v.string()), + }), + ), }), summary: v.optional(v.string()), qualityAssessment: v.optional( @@ -12706,7 +12734,7 @@ export const insertVersion = internalMutation({ slug, displayName: args.displayName, summary: summaryValue, - icon: undefined, + icon: args.icon, ownerUserId: userId, ownerPublisherId, canonicalSkillId, @@ -12793,7 +12821,7 @@ export const insertVersion = internalMutation({ sourceProvenance: args.sourceProvenance, changelog: args.changelog, changelogSource: args.changelogSource, - icon: undefined, + icon: args.icon, files: args.files, parsed: args.parsed, staticScan: args.staticScan, @@ -12893,7 +12921,9 @@ export const insertVersion = internalMutation({ const basePatch: SkillModerationPatch = { displayName: nextDisplayName, summary: nextSummary ?? undefined, - icon: skill.icon, + icon: isNewLatest + ? (args.icon ?? (isHostedSkillPresentationIconPath(skill.icon) ? undefined : skill.icon)) + : skill.icon, ownerPublisherId: skill.ownerPublisherId ?? ownerPublisherId, latestVersionId: isNewLatest ? versionId : skill.latestVersionId, latestVersionSummary: isNewLatest @@ -12902,7 +12932,8 @@ export const insertVersion = internalMutation({ createdAt: now, changelog: args.changelog, changelogSource: args.changelogSource, - description: getFrontmatterValue(args.parsed.frontmatter, "description")?.trim(), + description: + args.summary ?? getFrontmatterValue(args.parsed.frontmatter, "description")?.trim(), clawdis: args.parsed.clawdis, } : skill.latestVersionSummary, diff --git a/e2e/local-auth/helpers.ts b/e2e/local-auth/helpers.ts index 16bcf7b5..634505fa 100644 --- a/e2e/local-auth/helpers.ts +++ b/e2e/local-auth/helpers.ts @@ -544,6 +544,7 @@ export async function publishSkillVersion( ownerHandle: string; slug: string; displayName: string; + expectedDisplayName?: string; version: string; versionLabel: string; changelog: string; @@ -553,6 +554,7 @@ export async function publishSkillVersion( files?: Array<{ path: string; contents: string | Uint8Array }>; }, ) { + const expectedDisplayName = args.expectedDisplayName ?? args.displayName; const skillDir = testInfo.outputPath(`${args.slug}-${args.version}`); await mkdir(skillDir, { recursive: true }); await writeFile( @@ -584,7 +586,12 @@ export async function publishSkillVersion( // upload. Treat that navigation as success instead of waiting and retrying. if (pathname === "/dashboard") return "staged"; if (detailUrlPattern.test(pathname)) { - if (await isPublishedDetailCurrentVersionVisible(page, args)) { + if ( + await isPublishedDetailCurrentVersionVisible(page, { + ...args, + displayName: expectedDisplayName, + }) + ) { return "published"; } if (await versionExists()) return "staged"; @@ -615,7 +622,12 @@ export async function publishSkillVersion( await page.goto(skillDetailPath(args.ownerHandle, args.slug), { waitUntil: "domcontentloaded", }); - if (!(await isPublishedDetailCurrentVersionVisible(page, args))) { + if ( + !(await isPublishedDetailCurrentVersionVisible(page, { + ...args, + displayName: expectedDisplayName, + })) + ) { const completion = await completeMockPrePublicationChecks({ kind: "skill", slug: args.slug, @@ -628,14 +640,20 @@ export async function publishSkillVersion( await page.goto(skillDetailPath(args.ownerHandle, args.slug), { waitUntil: "domcontentloaded", }); - await expectPublishedDetailCurrentVersion(page, args); + await expectPublishedDetailCurrentVersion(page, { + ...args, + displayName: expectedDisplayName, + }); break; } catch (error) { await page.goto(skillDetailPath(args.ownerHandle, args.slug), { waitUntil: "domcontentloaded", }); try { - await expectPublishedDetailCurrentVersion(page, args); + await expectPublishedDetailCurrentVersion(page, { + ...args, + displayName: expectedDisplayName, + }); if (!args.versionExists || (await versionExists())) break; await page.goto(publishUrl, { waitUntil: "domcontentloaded" }); await waitForPublishSkillForm(page); @@ -656,7 +674,7 @@ export async function publishSkillVersion( expect(actualOwnerHandle?.toLowerCase()).toContain(args.ownerHandle.toLowerCase()); expect(actualSlug).toBe(args.slug); expect(new URL(page.url()).pathname).toBe(buildSkillDetailHref(actualOwnerHandle!, args.slug)); - await expectPublishedDetailPage(page, args.displayName); + await expectPublishedDetailPage(page, expectedDisplayName); const successDialog = page.getByRole("dialog", { name: /it's alive/i }); if (await successDialog.isVisible().catch(() => false)) { try { @@ -669,6 +687,9 @@ export async function publishSkillVersion( await waitForHydration(page); } } - await expectPublishedDetailCurrentVersion(page, args); + await expectPublishedDetailCurrentVersion(page, { + ...args, + displayName: expectedDisplayName, + }); return actualOwnerHandle!; } diff --git a/e2e/local-auth/skill-presentation-metadata.pw.test.ts b/e2e/local-auth/skill-presentation-metadata.pw.test.ts new file mode 100644 index 00000000..bf796f46 --- /dev/null +++ b/e2e/local-auth/skill-presentation-metadata.pw.test.ts @@ -0,0 +1,138 @@ +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { expect, test } from "@playwright/test"; +import { waitForHydration } from "../helpers/runtimeErrors"; +import { publishSkillVersion, signInAsLocalPublisher, skillMd } from "./helpers"; + +test.skip( + process.env.VITE_ENABLE_DEV_AUTH !== "1", + "skill presentation metadata tests require the local dev auth runner", +); + +test.setTimeout(600_000); + +const iconBytes = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + "base64", + ), +); + +test("agents/openai.yaml drives hosted skill presentation in UI and feed", async ({ + page, +}, testInfo) => { + await signInAsLocalPublisher(page, "officialOrgMember"); + const ownerHandle = "local-official-org"; + const slug = `pw-presentation-${Date.now().toString(36)}`; + const version = "1.0.0"; + + await publishSkillVersion(page, testInfo, { + ownerHandle, + slug, + displayName: "Presentation Fixture", + expectedDisplayName: "OpenAI Presentation Fixture", + version, + versionLabel: "presentation metadata release", + changelog: "Verify normalized presentation metadata and hosted icon delivery.", + skillMarkdown: skillMd({ + slug, + displayName: "Presentation Fixture", + versionLabel: "presentation metadata release", + }).replace(`name: ${slug}`, "name: Presentation Fixture"), + files: [ + { + path: "agents/openai.yaml", + contents: + "interface:\n display_name: '✨ OpenAI Presentation Fixture'\n short_description: Feed-ready OpenAI summary.\n icon_small: assets/icon.png\n", + }, + { path: "assets/icon.png", contents: iconBytes }, + ], + }); + + await waitForHydration(page); + await expect(page.locator("h1.skill-page-title")).toHaveText("OpenAI Presentation Fixture"); + await expect(page.getByText("Feed-ready OpenAI summary.")).toBeVisible(); + const icon = page.locator(".skill-hero-title-row img.marketplace-icon-image"); + await expect(icon).toBeVisible(); + await expect + .poll(() => + icon.evaluate((image: HTMLImageElement) => image.complete && image.naturalWidth > 0), + ) + .toBe(true); + const iconUrl = await icon.getAttribute("src"); + expect(iconUrl).toMatch(/^\/api\/v1\/skill-icons\/[a-f\d]{64}$/); + + const iconResponse = await page.request.get(new URL(iconUrl!, page.url()).toString()); + expect(iconResponse.status()).toBe(200); + expect(new Uint8Array(await iconResponse.body())).toEqual(iconBytes); + expect(iconResponse.headers()["cache-control"]).toContain("immutable"); + + markLatestVersionDownloadableForLocalFeed(ownerHandle, slug); + publishCatalogFeeds(); + const feedResponse = await page.request.get(`${convexSiteUrl()}/api/v1/feeds/skills`); + expect(feedResponse.status()).toBe(200); + const feed = (await feedResponse.json()) as { + entries: Array<{ id: string; title?: string; description?: string; icon?: string }>; + }; + expect(feed.entries).toContainEqual( + expect.objectContaining({ + id: `@${ownerHandle}/${slug}`, + title: "OpenAI Presentation Fixture", + description: "Feed-ready OpenAI summary.", + icon: expect.stringMatching(/^https:\/\/clawhub\.ai\/api\/v1\/skill-icons\/[a-f\d]{64}$/), + }), + ); +}); + +function convexSiteUrl() { + const url = process.env.VITE_CONVEX_SITE_URL; + if (!url) throw new Error("VITE_CONVEX_SITE_URL is required"); + return url.replace(/\/$/u, ""); +} + +function publishCatalogFeeds() { + runLocalConvex("catalogFeed:publish", { + expiresAt: new Date(Date.now() + 3_600_000).toISOString(), + }); +} + +function markLatestVersionDownloadableForLocalFeed(ownerHandle: string, slug: string) { + const result = runLocalConvex("skills:getBySlug", { ownerHandle, slug }) as { + latestVersion?: { _id?: string } | null; + }; + const versionId = result.latestVersion?._id; + if (!versionId) throw new Error("Published skill version was not available"); + runLocalConvex("skills:updateVersionScanResultsInternal", { + versionId, + sha256hash: "a".repeat(64), + }); +} + +function runLocalConvex(functionName: string, args: Record) { + const config = JSON.parse(readFileSync(".convex/local/default/config.json", "utf8")) as { + deploymentName?: string; + }; + if (!config.deploymentName) throw new Error("Local Convex deployment name was not available"); + const result = spawnSync( + "bunx", + [ + "convex", + "run", + "--typecheck", + "disable", + "--codegen", + "disable", + functionName, + JSON.stringify(args), + ], + { + cwd: process.cwd(), + env: { ...process.env, CONVEX_DEPLOYMENT: `local:${config.deploymentName}` }, + encoding: "utf8", + }, + ); + if (result.status !== 0) { + throw new Error(`Convex function ${functionName} failed:\n${result.stdout}\n${result.stderr}`); + } + return result.stdout.trim() ? (JSON.parse(result.stdout) as unknown) : null; +} diff --git a/src/components/Header.tsx b/src/components/Header.tsx index 2d394d34..d1efe784 100644 --- a/src/components/Header.tsx +++ b/src/components/Header.tsx @@ -31,6 +31,7 @@ import { gravatarUrl } from "../lib/gravatar"; import { PRIMARY_NAV_ITEMS, SECONDARY_NAV_ITEMS } from "../lib/nav-items"; import { buildPublisherProfileHref, buildSkillDetailHref } from "../lib/ownerRoute"; import { buildPluginDetailHref, displayPluginPackageName } from "../lib/pluginRoutes"; +import { presentationTitle } from "../lib/presentationTitle"; import { SITE_NAME } from "../lib/site"; import { applyTheme, useThemeMode } from "../lib/theme"; import { clearAuthError, setAuthError } from "../lib/useAuthError"; @@ -1019,16 +1020,22 @@ function getTypeaheadOptionId(item: TypeaheadItem) { function TypeaheadRowIcon({ item }: { item: TypeaheadItem }) { if (item.kind === "skill") { - const label = item.result.skill.displayName || item.result.skill.slug; + const label = presentationTitle(item.result.skill.displayName, item.result.skill.slug); return ( ); } if (item.kind === "plugin") { - const label = item.result.plugin.displayName || item.result.plugin.name; + const label = presentationTitle(item.result.plugin.displayName, item.result.plugin.name); return (