From b7bd53a697349fe60fd2fdb24c20cfa3f9e4a732 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Thu, 30 Jul 2026 19:15:16 -0700 Subject: [PATCH] fix: advance plugin scan pagination between claims (#3331) --- convex/packages.public.test.ts | 13 ++-- convex/packages.ts | 111 +++++++++++++++------------------ 2 files changed, 55 insertions(+), 69 deletions(-) diff --git a/convex/packages.public.test.ts b/convex/packages.public.test.ts index ed9dfc3d..0c31c49c 100644 --- a/convex/packages.public.test.ts +++ b/convex/packages.public.test.ts @@ -13416,7 +13416,7 @@ describe("packages public queries", () => { }); }); - it("continues through filtered source pages until the requested plugin batch is full", async () => { + it("returns a filtered source page without issuing a second paginated query", async () => { const paginate = vi .fn() .mockResolvedValueOnce({ @@ -13480,14 +13480,11 @@ describe("packages public queries", () => { await expect( previewPackageInspectorScanBatchInternalHandler(ctx as never, { batchSize: 2 }), ).resolves.toMatchObject({ - nextCursor: null, - items: [ - { releaseId: "packageReleases:current", packageName: "modern-plugin" }, - { releaseId: "packageReleases:other", packageName: "other-plugin" }, - ], + nextCursor: "page-2", + items: [{ releaseId: "packageReleases:current", packageName: "modern-plugin" }], }); - expect(paginate).toHaveBeenNthCalledWith(1, { cursor: null, numItems: 2 }); - expect(paginate).toHaveBeenNthCalledWith(2, { cursor: "page-2", numItems: 1 }); + expect(paginate).toHaveBeenCalledTimes(1); + expect(paginate).toHaveBeenCalledWith({ cursor: null, numItems: 2 }); }); it("rejects a non-advancing nightly scan pagination cursor", async () => { diff --git a/convex/packages.ts b/convex/packages.ts index 3c91dba6..f2f7c492 100644 --- a/convex/packages.ts +++ b/convex/packages.ts @@ -10213,72 +10213,61 @@ async function listPackageInspectorScanBatch( const batchSize = Math.max(1, Math.min(Math.round(args.batchSize ?? 25), 50)); const items: PackageInspectorScanBatchItem[] = []; let skippedUnchanged = 0; - let sourceCursor = args.cursor ?? null; - let nextCursor: string | null = sourceCursor; - let sourceReleasesRead = 0; - // Amortize filtered/unchanged releases without risking an unbounded Convex transaction. - const maxSourceReleases = 500; - while (items.length < batchSize && sourceReleasesRead < maxSourceReleases) { - const pageSize = Math.min(batchSize - items.length, maxSourceReleases - sourceReleasesRead); - const page = await ctx.db - .query("packageReleases") - .withIndex("by_active_created", (q) => q.eq("softDeletedAt", undefined)) - .order("asc") - .paginate({ - cursor: sourceCursor, - numItems: pageSize, - }); - if (!page.isDone && page.continueCursor === sourceCursor) { - throw new Error("Package Inspector scan pagination cursor did not advance"); + const sourceCursor = args.cursor ?? null; + // Convex permits only one native paginated query per function. The workflow + // acknowledges this source-page cursor and requests the next page separately. + const page = await ctx.db + .query("packageReleases") + .withIndex("by_active_created", (q) => q.eq("softDeletedAt", undefined)) + .order("asc") + .paginate({ + cursor: sourceCursor, + numItems: batchSize, + }); + if (!page.isDone && page.continueCursor === sourceCursor) { + throw new Error("Package Inspector scan pagination cursor did not advance"); + } + for (const release of page.page) { + if (!isPublishedPackageRelease(release)) continue; + const pkg = await ctx.db.get(release.packageId); + if ( + !pkg || + pkg.softDeletedAt || + (pkg.family !== "code-plugin" && pkg.family !== "bundle-plugin") || + pkg.channel === "private" || + isPackageBlockedFromPublic(resolvePublicPackageScanStatus(pkg, release)) + ) { + continue; } - // Count the requested page budget as consumed even when Convex returns an - // empty split page, so filtered releases remain transaction-bounded. - sourceReleasesRead += pageSize; - for (const release of page.page) { - if (items.length >= batchSize) break; - if (!isPublishedPackageRelease(release)) continue; - const pkg = await ctx.db.get(release.packageId); - if ( - !pkg || - pkg.softDeletedAt || - (pkg.family !== "code-plugin" && pkg.family !== "bundle-plugin") || - pkg.channel === "private" || - isPackageBlockedFromPublic(resolvePublicPackageScanStatus(pkg, release)) - ) { + const latestReleaseId = pkg.latestReleaseId ?? pkg.tags?.latest; + if (latestReleaseId !== release._id) continue; + const { inspectorVersion, targetOpenClawVersion } = args; + if (inspectorVersion && targetOpenClawVersion) { + const scanState = await ctx.db + .query("packageInspectorScanStates") + .withIndex("by_release_and_inspector_version_and_target_openclaw_version", (q) => + q + .eq("releaseId", release._id) + .eq("inspectorVersion", inspectorVersion) + .eq("targetOpenClawVersion", targetOpenClawVersion), + ) + .unique(); + if (scanState && (!args.notifyOwners || scanState.notificationCompletedAt)) { + skippedUnchanged += 1; continue; } - const latestReleaseId = pkg.latestReleaseId ?? pkg.tags?.latest; - if (latestReleaseId !== release._id) continue; - const { inspectorVersion, targetOpenClawVersion } = args; - if (inspectorVersion && targetOpenClawVersion) { - const scanState = await ctx.db - .query("packageInspectorScanStates") - .withIndex("by_release_and_inspector_version_and_target_openclaw_version", (q) => - q - .eq("releaseId", release._id) - .eq("inspectorVersion", inspectorVersion) - .eq("targetOpenClawVersion", targetOpenClawVersion), - ) - .unique(); - if (scanState && (!args.notifyOwners || scanState.notificationCompletedAt)) { - skippedUnchanged += 1; - continue; - } - } - items.push({ - packageId: pkg._id, - releaseId: release._id, - ownerUserId: pkg.ownerUserId, - ownerPublisherId: pkg.ownerPublisherId, - packageName: pkg.name, - version: release.version, - artifactKind: release.artifactKind ?? "legacy-zip", - }); } - nextCursor = page.isDone ? null : page.continueCursor; - if (page.isDone || items.length >= batchSize) break; - sourceCursor = page.continueCursor; + items.push({ + packageId: pkg._id, + releaseId: release._id, + ownerUserId: pkg.ownerUserId, + ownerPublisherId: pkg.ownerPublisherId, + packageName: pkg.name, + version: release.version, + artifactKind: release.artifactKind ?? "legacy-zip", + }); } + const nextCursor = page.isDone ? null : page.continueCursor; return { items, nextCursor,