fix: export latest security dataset skills only

convex/securityDataset.test.ts

@@ -0,0 +1,115 @@
+/* @vitest-environment node */
+import { describe, expect, it, vi } from "vitest";
+import { skillVersionPageToLatestExportRows } from "./securityDataset";
+
+describe("security dataset export", () => {
+  it("exports only each skill's latest active version", async () => {
+    const get = vi.fn(async (id: string) => {
+      if (id === "skills:demo") {
+        return makeSkill({
+          id,
+          latestVersionId: "skillVersions:latest",
+          moderationSourceVersionId: "skillVersions:old",
+        });
+      }
+      if (id === "publishers:owner") {
+        return {
+          _id: id,
+          handle: "owner",
+          deletedAt: undefined,
+          deactivatedAt: undefined,
+        };
+      }
+      throw new Error(`unexpected db.get(${id})`);
+    });
+
+    const rows = await skillVersionPageToLatestExportRows(
+      { db: { get } } as never,
+      [
+        makeVersion({ id: "skillVersions:old", version: "1.0.0" }),
+        makeVersion({ id: "skillVersions:latest", version: "2.0.0" }),
+      ] as never,
+    );
+
+    expect(rows).toHaveLength(1);
+    expect(rows[0]).toMatchObject({
+      sourceDocId: "skillVersions:latest",
+      parentDocId: "skills:demo",
+      publicOwnerHandle: "owner",
+      publicSlug: "demo",
+      version: "2.0.0",
+      moderationConsensus: null,
+    });
+  });
+
+  it("skips skills without an active latest version", async () => {
+    const get = vi.fn(async (id: string) => {
+      if (id === "skills:missing-latest") {
+        return makeSkill({ id, latestVersionId: undefined });
+      }
+      if (id === "skills:other-latest") {
+        return makeSkill({ id, latestVersionId: "skillVersions:other" });
+      }
+      if (id === "skills:deleted") {
+        return makeSkill({ id, latestVersionId: "skillVersions:deleted", softDeletedAt: 123 });
+      }
+      return null;
+    });
+
+    const rows = await skillVersionPageToLatestExportRows(
+      { db: { get } } as never,
+      [
+        makeVersion({ id: "skillVersions:missing-latest", skillId: "skills:missing-latest" }),
+        makeVersion({ id: "skillVersions:not-latest", skillId: "skills:other-latest" }),
+        makeVersion({ id: "skillVersions:deleted", skillId: "skills:deleted" }),
+        makeVersion({ id: "skillVersions:missing-skill", skillId: "skills:missing" }),
+      ] as never,
+    );
+
+    expect(rows).toEqual([]);
+  });
+});
+
+function makeSkill(input: {
+  id: string;
+  latestVersionId: string | undefined;
+  moderationSourceVersionId?: string;
+  softDeletedAt?: number;
+}) {
+  return {
+    _id: input.id,
+    slug: "demo",
+    displayName: "Demo",
+    ownerPublisherId: "publishers:owner",
+    ownerUserId: "users:owner",
+    latestVersionId: input.latestVersionId,
+    moderationSourceVersionId: input.moderationSourceVersionId,
+    moderationVerdict: "clean",
+    moderationReasonCodes: [],
+    moderationSummary: "Clean",
+    moderationEngineVersion: "test",
+    moderationEvaluatedAt: 1,
+    softDeletedAt: input.softDeletedAt,
+  };
+}
+
+function makeVersion(input: {
+  id: string;
+  version?: string;
+  softDeletedAt?: number;
+  skillId?: string;
+}) {
+  return {
+    _id: input.id,
+    skillId: input.skillId ?? "skills:demo",
+    version: input.version ?? "1.0.0",
+    createdAt: 2,
+    softDeletedAt: input.softDeletedAt,
+    sha256hash: "a".repeat(64),
+    files: [],
+    vtAnalysis: undefined,
+    skillSpectorAnalysis: undefined,
+    staticScan: undefined,
+    llmAnalysis: undefined,
+  };
+}

convex/securityDataset.ts

@@ -25,7 +25,7 @@ type StoredVtAnalysis = Doc<"skillVersions">["vtAnalysis"];
 type StoredSkillSpectorAnalysis = Doc<"skillVersions">["skillSpectorAnalysis"];
 type StoredLlmAnalysis = Doc<"skillVersions">["llmAnalysis"];
 type ArtifactExportRow =
-  | Awaited<ReturnType<typeof skillVersionPageToExportRows>>[number]
+  | Awaited<ReturnType<typeof skillVersionPageToLatestExportRows>>[number]
   | Awaited<ReturnType<typeof packageReleasePageToExportRows>>[number];
 type ArtifactExportPage = {
   page: ArtifactExportRow[];
@@ -74,7 +74,7 @@ export const listArtifactExportPageInternal = internalQuery({
         .order("asc")
         .paginate(paginationOpts);
       return {
-        page: await skillVersionPageToExportRows(ctx, page.page),
+        page: await skillVersionPageToLatestExportRows(ctx, page.page),
         isDone: page.isDone,
         continueCursor: page.continueCursor,
         exportMode: args.mode ?? "public",
@@ -212,11 +212,14 @@ async function getActiveCreatedBounds(ctx: QueryCtx, sourceKind: "skill" | "pack
   };
 }
 
-async function skillVersionPageToExportRows(ctx: QueryCtx, versions: Array<Doc<"skillVersions">>) {
+export async function skillVersionPageToLatestExportRows(
+  ctx: Pick<QueryCtx, "db">,
+  versions: Array<Doc<"skillVersions">>,
+) {
   const rows = [];
   for (const version of versions) {
     const skill = await ctx.db.get(version.skillId);
-    if (!skill || skill.softDeletedAt) continue;
+    if (!skill || skill.softDeletedAt || skill.latestVersionId !== version._id) continue;
     const publicOwnerHandle = await getPublicOwnerHandle(ctx, skill);
     rows.push({
       sourceKind: "skill" as const,
@@ -462,7 +465,7 @@ function normalizeLlmAnalysis(analysis: StoredLlmAnalysis) {
 }
 
 async function getPublicOwnerHandle(
-  ctx: QueryCtx,
+  ctx: Pick<QueryCtx, "db">,
   source: Pick<Doc<"skills"> | Doc<"packages">, "ownerPublisherId" | "ownerUserId">,
 ) {
   const owner = await getOwnerPublisher(ctx, {