test: stabilize local-auth e2e flake paths

Stabilize local-auth e2e tests under CI/Testbox pressure by hardening publish scan enqueue recovery, bounded publish navigation, route/hydration helpers, and local-auth readiness probes.
This commit is contained in:
Patrick Erichsen
2026-06-23 16:26:54 -07:00
committed by GitHub
parent dfb93eab79
commit 898b3bebba
25 changed files with 1593 additions and 301 deletions
+1
View File
@@ -21,6 +21,7 @@ sync:
- dist - dist
- dist-ssr - dist-ssr
- node_modules - node_modules
- .output
- playwright-report - playwright-report
- test-results - test-results
env: env:
+14 -5
View File
@@ -61,17 +61,26 @@ jobs:
git fetch --no-tags --depth=50 origin "+refs/heads/main:refs/remotes/origin/main" git fetch --no-tags --depth=50 origin "+refs/heads/main:refs/remotes/origin/main"
link_tool() {
local src="$1"
local dest="$2"
if [ "$src" = "$dest" ]; then
return 0
fi
sudo ln -sf "$src" "$dest"
}
bun_bin="$(command -v bun)" bun_bin="$(command -v bun)"
sudo ln -sf "$bun_bin" /usr/local/bin/bun link_tool "$bun_bin" /usr/local/bin/bun
if command -v bunx >/dev/null 2>&1; then if command -v bunx >/dev/null 2>&1; then
sudo ln -sf "$(command -v bunx)" /usr/local/bin/bunx link_tool "$(command -v bunx)" /usr/local/bin/bunx
fi fi
node_bin="$(dirname "$(node -p 'process.execPath')")" node_bin="$(dirname "$(node -p 'process.execPath')")"
sudo ln -sf "$node_bin/node" /usr/local/bin/node link_tool "$node_bin/node" /usr/local/bin/node
sudo ln -sf "$node_bin/npm" /usr/local/bin/npm link_tool "$node_bin/npm" /usr/local/bin/npm
sudo ln -sf "$node_bin/npx" /usr/local/bin/npx link_tool "$node_bin/npx" /usr/local/bin/npx
- name: Run Testbox - name: Run Testbox
uses: useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d uses: useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d
+30
View File
@@ -3961,6 +3961,36 @@ export const seedOrgDeletionFixtureMutation = internalMutation({
}, },
}); });
export const getOrgDeletionFixtureState: ReturnType<typeof rawInternalMutation> =
rawInternalMutation({
args: {
publisherId: v.id("publishers"),
skillId: v.id("skills"),
packageId: v.id("packages"),
},
handler: async (ctx, args) => {
const publisher = await ctx.db.get(args.publisherId);
const skill = await ctx.db.get(args.skillId);
const pkg = await ctx.db.get(args.packageId);
return {
ok: true as const,
publisherExists: Boolean(publisher),
publisherPubliclyVisible: Boolean(
publisher && !publisher.deletedAt && !publisher.deactivatedAt,
),
skillExists: Boolean(skill),
skillActive: Boolean(skill && !skill.softDeletedAt),
skillPubliclyVisible: Boolean(
skill && !skill.softDeletedAt && !skill.hiddenAt && skill.moderationStatus !== "removed",
),
packageExists: Boolean(pkg),
packageActive: Boolean(pkg && !pkg.softDeletedAt),
packagePubliclyVisible: Boolean(pkg && !pkg.softDeletedAt),
packageSoftDeletedAt: pkg?.softDeletedAt ?? null,
};
},
});
type VersionDeletionFixtureArgs = { type VersionDeletionFixtureArgs = {
skillSlug: string; skillSlug: string;
skillDisplayName: string; skillDisplayName: string;
+116
View File
@@ -89,6 +89,112 @@ description: Automation workflow for recurring reports.
topics: undefined, topics: undefined,
}), }),
); );
expect(ctx.scheduler.runAfter).toHaveBeenCalledWith(0, expect.anything(), {
versionId: "skillVersions:demo",
source: "publish",
});
expect(ctx.scheduler.runAfter).toHaveBeenCalledWith(2_000, expect.anything(), {
versionId: "skillVersions:demo",
source: "publish",
preserveActiveJob: true,
preserveExistingJob: true,
});
expect(ctx.scheduler.runAfter).toHaveBeenCalledWith(15_000, expect.anything(), {
versionId: "skillVersions:demo",
source: "publish",
preserveActiveJob: true,
preserveExistingJob: true,
});
});
it("resolves the target publisher handle before scheduling publish webhooks", async () => {
const previousWebhookUrl = process.env.DISCORD_WEBHOOK_URL;
process.env.DISCORD_WEBHOOK_URL = "https://example.invalid/webhook";
const storedFiles = new Map([
[
"_storage:skill",
`---
description: Org helper.
---
# Org Helper
`,
],
]);
const runMutation = vi.fn(async (_ref: unknown, args: Record<string, unknown>) => {
if ("version" in args && "embedding" in args) {
return {
skillId: "skills:demo",
versionId: "skillVersions:demo",
embeddingId: "skillEmbeddings:demo",
};
}
return null;
});
const ctx = {
runQuery: vi
.fn()
.mockResolvedValueOnce(null)
.mockResolvedValueOnce({ _id: "users:1", handle: "actor", createdAt: 1 })
.mockResolvedValueOnce({ _id: "publishers:org", handle: "org-demo" })
.mockResolvedValueOnce({
skill: {
_id: "skills:demo",
slug: "org-helper",
displayName: "Org Helper",
summary: "Org helper",
tags: {},
},
owner: { handle: "org-demo" },
}),
runMutation,
scheduler: { runAfter: vi.fn() },
storage: {
get: vi.fn(async (storageId: string) => {
const content = storedFiles.get(storageId);
return content === undefined ? null : new Blob([content]);
}),
},
};
try {
await publishVersionForUser(
ctx as never,
"users:1" as never,
{
slug: "org-helper",
displayName: "Org Helper",
version: "1.0.0",
changelog: "Initial release",
files: [
{
path: "SKILL.md",
size: 70,
storageId: "_storage:skill" as never,
sha256: "a".repeat(64),
contentType: "text/markdown",
},
],
},
{
bypassGitHubAccountAge: true,
bypassQualityGate: true,
ownerPublisherId: "publishers:org" as never,
},
);
await vi.waitFor(() => {
expect(ctx.runQuery).toHaveBeenCalledWith(expect.anything(), {
slug: "org-helper",
ownerHandle: "org-demo",
});
});
} finally {
if (previousWebhookUrl === undefined) {
delete process.env.DISCORD_WEBHOOK_URL;
} else {
process.env.DISCORD_WEBHOOK_URL = previousWebhookUrl;
}
}
}); });
it("uses Other when an existing skill has a retired stored category", async () => { it("uses Other when an existing skill has a retired stored category", async () => {
@@ -317,6 +423,16 @@ description: Security scanner smoke fixture.
source: "publish", source: "publish",
}), }),
); );
expect(scheduler.runAfter).toHaveBeenCalledWith(
15_000,
expect.anything(),
expect.objectContaining({
versionId: "skillVersions:demo",
source: "publish",
preserveActiveJob: true,
preserveExistingJob: true,
}),
);
}); });
it("merges github source into metadata", () => { it("merges github source into metadata", () => {
+28 -11
View File
@@ -42,12 +42,13 @@ import {
import { assertValidSkillSlug, normalizeSkillSlug } from "./skillSlugValidator"; import { assertValidSkillSlug, normalizeSkillSlug } from "./skillSlugValidator";
import { generateSkillSummary } from "./skillSummary"; import { generateSkillSummary } from "./skillSummary";
import { runStaticPublishScan } from "./staticPublishScan"; import { runStaticPublishScan } from "./staticPublishScan";
import type { WebhookSkillPayload } from "./webhooks"; import { getWebhookConfig, type WebhookSkillPayload } from "./webhooks";
const MAX_FILES_FOR_EMBEDDING = 40; const MAX_FILES_FOR_EMBEDDING = 40;
const QUALITY_WINDOW_MS = 24 * 60 * 60 * 1000; const QUALITY_WINDOW_MS = 24 * 60 * 60 * 1000;
const QUALITY_ACTIVITY_LIMIT = 60; const QUALITY_ACTIVITY_LIMIT = 60;
const PLATFORM_SKILL_LICENSE = "MIT-0" as const; const PLATFORM_SKILL_LICENSE = "MIT-0" as const;
const SECURITY_SCAN_ENQUEUE_BACKUP_DELAY_MS = 15_000;
type FingerprintFile = { path: string; sha256: string }; type FingerprintFile = { path: string; sha256: string };
type SafePublishFile = PublishVersionArgs["files"][number] & { path: string }; type SafePublishFile = PublishVersionArgs["files"][number] & { path: string };
@@ -100,6 +101,7 @@ export type PublishOptions = {
bypassNewSkillRateLimit?: boolean; bypassNewSkillRateLimit?: boolean;
bypassQualityGate?: boolean; bypassQualityGate?: boolean;
skipWebhook?: boolean; skipWebhook?: boolean;
ownerHandle?: string;
ownerPublisherId?: Id<"publishers">; ownerPublisherId?: Id<"publishers">;
sourceOwnerPublisherId?: Id<"publishers">; sourceOwnerPublisherId?: Id<"publishers">;
sourceProvenance?: PublishVersionArgs["source"]; sourceProvenance?: PublishVersionArgs["source"];
@@ -375,17 +377,32 @@ export async function publishVersionForUser(
versionId: publishResult.versionId, versionId: publishResult.versionId,
source: "publish", source: "publish",
}); });
await ctx.scheduler.runAfter(2_000, internal.securityScan.enqueueSkillVersionScanInternal, {
versionId: publishResult.versionId,
source: "publish",
preserveActiveJob: true,
preserveExistingJob: true,
});
await ctx.scheduler.runAfter(
SECURITY_SCAN_ENQUEUE_BACKUP_DELAY_MS,
internal.securityScan.enqueueSkillVersionScanInternal,
{
versionId: publishResult.versionId,
source: "publish",
preserveActiveJob: true,
preserveExistingJob: true,
},
);
const targetPublisher = if (!options.skipWebhook && getWebhookConfig().url) {
options.ownerPublisherId !== undefined let ownerHandle = options.ownerHandle;
? ((await ctx.runQuery(internal.publishers.getByIdInternal, { if (!ownerHandle && options.ownerPublisherId !== undefined) {
publisherId: options.ownerPublisherId, const targetPublisher = (await ctx.runQuery(internal.publishers.getByIdInternal, {
})) as Doc<"publishers"> | null) publisherId: options.ownerPublisherId,
: null; })) as Doc<"publishers"> | null;
const ownerHandle = ownerHandle = targetPublisher?.handle;
targetPublisher?.handle ?? owner?.handle ?? owner?.displayName ?? owner?.name ?? "unknown"; }
ownerHandle ??= owner?.handle ?? owner?.displayName ?? owner?.name;
if (!options.skipWebhook) {
void schedulePublishWebhook(ctx, { void schedulePublishWebhook(ctx, {
slug, slug,
version, version,
+48
View File
@@ -8,6 +8,7 @@ import {
claimQueuedJobsInternal, claimQueuedJobsInternal,
completeCodexScanJob, completeCodexScanJob,
enqueueBulkSkillRescanBatchForAdminInternal, enqueueBulkSkillRescanBatchForAdminInternal,
enqueueSkillVersionScanInternal,
failCodexScanJob, failCodexScanJob,
finalizeGitHubSkillScanRequestInternal, finalizeGitHubSkillScanRequestInternal,
getJobTargetInternal, getJobTargetInternal,
@@ -218,6 +219,20 @@ const requestPackageRescanForUserInternalHandler = (
> >
)._handler; )._handler;
const enqueueSkillVersionScanInternalHandler = (
enqueueSkillVersionScanInternal as unknown as WrappedHandler<
{
versionId: string;
source: "publish";
priority?: number;
waitForVtMs?: number;
preserveActiveJob?: boolean;
preserveExistingJob?: boolean;
},
{ ok: true; skipped?: string; jobId?: string; alreadyQueued?: boolean }
>
)._handler;
const enqueueBulkSkillRescanBatchForAdminInternalHandler = ( const enqueueBulkSkillRescanBatchForAdminInternalHandler = (
enqueueBulkSkillRescanBatchForAdminInternal as unknown as WrappedHandler< enqueueBulkSkillRescanBatchForAdminInternal as unknown as WrappedHandler<
{ {
@@ -877,6 +892,39 @@ describe("securityScan", () => {
vi.mocked(getAuthUserId).mockReset(); vi.mocked(getAuthUserId).mockReset();
}); });
it("does not enqueue a duplicate publish scan after the backup delay if the first scan already finished", async () => {
const existingJob = makeScanJob({
_id: "securityScanJobs:fast-publish",
status: "succeeded",
source: "publish",
skillVersionId: "skillVersions:fast-publish",
});
const { ctx, inserts, patches } = makeRescanCtx({
actorId: "users:owner",
docs: {
"skillVersions:fast-publish": {
_id: "skillVersions:fast-publish",
skillId: "skills:fast-publish",
version: "1.0.0",
},
},
activeJobs: [existingJob],
});
const result = await enqueueSkillVersionScanInternalHandler(ctx, {
versionId: "skillVersions:fast-publish",
source: "publish",
preserveExistingJob: true,
});
expect(result).toMatchObject({
jobId: "securityScanJobs:fast-publish",
alreadyQueued: true,
});
expect(inserts.filter((entry) => entry.table === "securityScanJobs")).toEqual([]);
expect(patches).toEqual([]);
});
it("lets platform moderators request skill rescans", async () => { it("lets platform moderators request skill rescans", async () => {
const { ctx, inserts } = makeRescanCtx({ const { ctx, inserts } = makeRescanCtx({
actorId: "users:moderator", actorId: "users:moderator",
+11
View File
@@ -148,6 +148,7 @@ type EnqueueSkillVersionScanArgs = {
priority?: number; priority?: number;
waitForVtMs?: number; waitForVtMs?: number;
preserveActiveJob?: boolean; preserveActiveJob?: boolean;
preserveExistingJob?: boolean;
}; };
type EnqueuePackageReleaseScanArgs = { type EnqueuePackageReleaseScanArgs = {
@@ -546,6 +547,8 @@ export const enqueueSkillVersionScanInternal = internalMutation({
source: jobSourceValidator, source: jobSourceValidator,
priority: v.optional(v.number()), priority: v.optional(v.number()),
waitForVtMs: v.optional(v.number()), waitForVtMs: v.optional(v.number()),
preserveActiveJob: v.optional(v.boolean()),
preserveExistingJob: v.optional(v.boolean()),
}, },
handler: async (ctx, args) => { handler: async (ctx, args) => {
return enqueueSkillVersionScan(ctx, args); return enqueueSkillVersionScan(ctx, args);
@@ -2046,6 +2049,14 @@ async function enqueueSkillVersionScan(ctx: MutationCtx, args: EnqueueSkillVersi
}); });
return { ok: true as const, jobId: active._id, alreadyQueued: true as const }; return { ok: true as const, jobId: active._id, alreadyQueued: true as const };
} }
const preservedExisting = args.preserveExistingJob
? existing
.filter((job) => job.source === args.source)
.sort((a, b) => b.updatedAt - a.updatedAt)[0]
: undefined;
if (preservedExisting) {
return { ok: true as const, jobId: preservedExisting._id, alreadyQueued: true as const };
}
const jobId = await ctx.db.insert("securityScanJobs", { const jobId = await ctx.db.insert("securityScanJobs", {
targetKind: "skillVersion", targetKind: "skillVersion",
+2 -1
View File
@@ -9561,7 +9561,7 @@ export const publishVersion: ReturnType<typeof action> = action({
actorUserId: userId, actorUserId: userId,
ownerHandle: args.ownerHandle, ownerHandle: args.ownerHandle,
minimumRole: "publisher", minimumRole: "publisher",
})) as { publisherId: Id<"publishers"> }; })) as { publisherId: Id<"publishers">; handle: string };
const sourceOwnerHandle = const sourceOwnerHandle =
args.migrateOwner === true args.migrateOwner === true
? args.sourceOwnerHandle?.trim() || user.handle?.trim() || undefined ? args.sourceOwnerHandle?.trim() || user.handle?.trim() || undefined
@@ -9577,6 +9577,7 @@ export const publishVersion: ReturnType<typeof action> = action({
const { icon: _legacyIcon, ...publishArgs } = args; const { icon: _legacyIcon, ...publishArgs } = args;
return publishVersionForUser(ctx, userId, publishArgs, { return publishVersionForUser(ctx, userId, publishArgs, {
ownerPublisherId: target.publisherId, ownerPublisherId: target.publisherId,
ownerHandle: target.handle,
sourceOwnerPublisherId: source?.publisherId, sourceOwnerPublisherId: source?.publisherId,
migrateOwner: args.migrateOwner, migrateOwner: args.migrateOwner,
}); });
@@ -1,8 +1,7 @@
import { spawnSync } from "node:child_process"; import { spawnSync } from "node:child_process";
import { readFileSync } from "node:fs"; import { readFileSync } from "node:fs";
import { expect, test } from "@playwright/test"; import { expect, test, type Locator, type Page } from "@playwright/test";
import { import {
expectHealthyPage,
expectNoFatalErrorUi, expectNoFatalErrorUi,
trackRuntimeErrors, trackRuntimeErrors,
waitForHydration, waitForHydration,
@@ -13,6 +12,7 @@ test.skip(
process.env.VITE_ENABLE_DEV_AUTH !== "1", process.env.VITE_ENABLE_DEV_AUTH !== "1",
"local-auth account deletion tests require the local dev auth runner", "local-auth account deletion tests require the local dev auth runner",
); );
test.setTimeout(600_000);
function uniqueSuffix() { function uniqueSuffix() {
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 7)}`; return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 7)}`;
@@ -158,9 +158,75 @@ function getAccountRecreationState(fixture: AccountDeletionFixture) {
}); });
} }
function pollableDevSeedState<TState extends object>(readState: () => TState) {
try {
return readState();
} catch {
return {};
}
}
function isExpectedAccountDeletionRuntimeError(error: string) { function isExpectedAccountDeletionRuntimeError(error: string) {
if (error.includes("Selected isolate was not clean")) return true;
if (error.includes("server responded with a status of 404 (Not Found)")) return true; if (error.includes("server responded with a status of 404 (Not Found)")) return true;
return error.includes("[CONVEX Q(users:me)]") && error.includes("Function execution timed out"); if (!error.includes("Function execution timed out")) return false;
return [
"[CONVEX Q(users:me)]",
"[CONVEX Q(publishers:getMyProfileHandle)]",
"[CONVEX Q(publishers:getProfileByHandle)]",
"[CONVEX Q(publishers:getPublishedDisplayManifest)]",
"[CONVEX Q(publishers:listMembers)]",
"[CONVEX Q(publishers:listPublishedPage)]",
"[CONVEX Q(publishers:listStarredPage)]",
"packagesGetRouterV1Handler",
].some((prefix) => error.includes(prefix));
}
function isExpectedAccountDeletionTransitionRuntimeError(error: string) {
if (error.includes("pageerror:Minified React error #418")) return true;
return isExpectedAccountDeletionRuntimeError(error);
}
async function gotoUntilVisible(page: Page, url: string, target: Locator) {
let lastError: unknown;
for (let attempt = 0; attempt < 12; attempt += 1) {
await page.goto(url, { waitUntil: "domcontentloaded" });
await waitForHydration(page);
try {
await expect(target).toBeVisible({ timeout: 20_000 });
return;
} catch (error) {
lastError = error;
if (attempt === 11) break;
await page.waitForTimeout(1_000 * (attempt + 1));
}
}
throw lastError;
}
async function expectAccountDeletionResources(
page: Page,
args: { skillDisplayName: string; packageDisplayName: string },
) {
const dialog = page.getByRole("dialog", { name: "Delete account" });
await expect(dialog.getByText("This permanently deletes your account")).toBeVisible({
timeout: 30_000,
});
await expect(dialog.getByText("Resources permanently deleted")).toBeVisible({
timeout: 60_000,
});
await expect
.poll(
async () => {
const text = (await dialog.textContent().catch(() => "")) ?? "";
return {
hasPackage: text.includes(args.packageDisplayName),
hasSkill: text.includes(args.skillDisplayName),
};
},
{ timeout: 120_000, intervals: [500, 1_000, 2_000] },
)
.toEqual({ hasPackage: true, hasSkill: true });
} }
test("users can permanently delete their account and personal publisher resources", async ({ test("users can permanently delete their account and personal publisher resources", async ({
@@ -182,33 +248,39 @@ test("users can permanently delete their account and personal publisher resource
await signInAsLocalPersona(page, "user"); await signInAsLocalPersona(page, "user");
await page.goto(buildPublisherProfileHref(fixture.handle), { waitUntil: "domcontentloaded" }); await gotoUntilVisible(
await waitForHydration(page); page,
buildPublisherProfileHref(fixture.handle),
page.getByText(skillDisplayName),
);
await expect(page.getByRole("heading", { name: "Local User" })).toBeVisible(); await expect(page.getByRole("heading", { name: "Local User" })).toBeVisible();
await expect(page.getByText(skillDisplayName)).toBeVisible();
await page.goto(`/plugins/${encodeURIComponent(packageName)}`, { waitUntil: "domcontentloaded" }); await gotoUntilVisible(
await waitForHydration(page); page,
await expect(page.getByText(packageDisplayName)).toBeVisible(); `/plugins/${encodeURIComponent(packageName)}`,
page.getByText(packageDisplayName),
);
await page.goto("/settings?view=danger", { waitUntil: "domcontentloaded" }); await page.goto("/settings?view=danger", { waitUntil: "domcontentloaded" });
await waitForHydration(page); await waitForHydration(page);
await page.getByRole("button", { name: "Delete account" }).click(); await page.getByRole("button", { name: "Delete account" }).click();
await expect(page.getByText("This permanently deletes your account")).toBeVisible(); await expectAccountDeletionResources(page, {
await expect(page.getByText("Resources permanently deleted")).toBeVisible(); packageDisplayName,
await expect(page.getByText(new RegExp(escapeRegExp(skillDisplayName)))).toBeVisible(); skillDisplayName,
await expect(page.getByText(new RegExp(escapeRegExp(packageDisplayName)))).toBeVisible(); });
await page.screenshot({ await page.screenshot({
path: testInfo.outputPath("account-deletion-confirmation.png"), path: testInfo.outputPath("account-deletion-confirmation.png"),
fullPage: true, fullPage: true,
}); });
expect(errors.filter((error) => !isExpectedAccountDeletionRuntimeError(error))).toEqual([]);
errors.length = 0;
await page.getByRole("button", { name: "Permanently delete account" }).click(); await page.getByRole("button", { name: "Permanently delete account" }).click();
await expect(page.getByText("This permanently deletes your account")).toHaveCount(0, { await expect(page.getByText("This permanently deletes your account")).toHaveCount(0, {
timeout: 20_000, timeout: 20_000,
}); });
await expect await expect
.poll(() => getAccountDeletionFixtureState(fixture), { .poll(() => pollableDevSeedState(() => getAccountDeletionFixtureState(fixture)), {
timeout: 60_000, timeout: 60_000,
intervals: [500, 1_000, 2_000], intervals: [500, 1_000, 2_000],
}) })
@@ -232,7 +304,10 @@ test("users can permanently delete their account and personal publisher resource
expect(finalState.user.deactivatedAt).toEqual(expect.any(Number)); expect(finalState.user.deactivatedAt).toEqual(expect.any(Number));
expect(finalState.user.purgedAt).toEqual(expect.any(Number)); expect(finalState.user.purgedAt).toEqual(expect.any(Number));
} }
await expectHealthyPage(page, errors); await expectNoFatalErrorUi(page);
expect(errors.filter((error) => !isExpectedAccountDeletionTransitionRuntimeError(error))).toEqual(
[],
);
errors.length = 0; errors.length = 0;
await page.goto(buildPublisherProfileHref(fixture.handle), { waitUntil: "domcontentloaded" }); await page.goto(buildPublisherProfileHref(fixture.handle), { waitUntil: "domcontentloaded" });
@@ -266,7 +341,7 @@ test("users can permanently delete their account and personal publisher resource
await signInAsLocalPersona(page, "user"); await signInAsLocalPersona(page, "user");
await expect await expect
.poll(() => getAccountRecreationState(fixture), { .poll(() => pollableDevSeedState(() => getAccountRecreationState(fixture)), {
timeout: 30_000, timeout: 30_000,
intervals: [500, 1_000, 2_000], intervals: [500, 1_000, 2_000],
}) })
@@ -300,9 +375,11 @@ test("users can permanently delete their account and personal publisher resource
recreationState.activePublisher?.publisherId, recreationState.activePublisher?.publisherId,
); );
await page.goto(buildPublisherProfileHref(fixture.handle), { waitUntil: "domcontentloaded" }); await gotoUntilVisible(
await waitForHydration(page); page,
await expect(page.getByRole("heading", { name: "Local User" })).toBeVisible(); buildPublisherProfileHref(fixture.handle),
page.getByRole("heading", { name: "Local User" }),
);
await expect(page.getByText(skillDisplayName)).toHaveCount(0); await expect(page.getByText(skillDisplayName)).toHaveCount(0);
await expect(page.getByText(packageDisplayName)).toHaveCount(0); await expect(page.getByText(packageDisplayName)).toHaveCount(0);
+143 -39
View File
@@ -1,7 +1,11 @@
import { spawnSync } from "node:child_process"; import { spawnSync } from "node:child_process";
import { readFileSync } from "node:fs"; import { readFileSync } from "node:fs";
import { expect, test } from "@playwright/test"; import { expect, test, type Locator, type Page } from "@playwright/test";
import { expectHealthyPage, trackRuntimeErrors, waitForHydration } from "../helpers/runtimeErrors"; import {
expectNoFatalErrorUi,
trackRuntimeErrors,
waitForHydration,
} from "../helpers/runtimeErrors";
import { escapeRegExp, signInAsLocalPersona } from "./helpers"; import { escapeRegExp, signInAsLocalPersona } from "./helpers";
test.skip( test.skip(
@@ -10,6 +14,7 @@ test.skip(
); );
test.use({ video: process.env.CLAWHUB_ORG_DELETE_PROOF_VIDEO === "1" ? "on" : "off" }); test.use({ video: process.env.CLAWHUB_ORG_DELETE_PROOF_VIDEO === "1" ? "on" : "off" });
test.setTimeout(180_000);
function uniqueSuffix() { function uniqueSuffix() {
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 7)}`; return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 7)}`;
@@ -24,14 +29,22 @@ function localConvexDeployment() {
return `local:${parsed.deploymentName}`; return `local:${parsed.deploymentName}`;
} }
function seedOrgDeletionFixture(args: { function extractLastJsonObject(output: string) {
handle: string; const trimmed = output.trim();
displayName: string; for (let index = 0; index < trimmed.length; index += 1) {
skillSlug: string; if (trimmed[index] !== "{") continue;
skillDisplayName: string; const candidate = trimmed.slice(index);
packageName: string; try {
packageDisplayName: string; JSON.parse(candidate);
}) { return candidate;
} catch {
// Convex can print status lines before the JSON payload.
}
}
throw new Error(`No JSON object in convex run output:\n${output}`);
}
function runDevSeed<T>(functionName: string, args: Record<string, unknown>) {
const result = spawnSync( const result = spawnSync(
"bunx", "bunx",
[ [
@@ -41,7 +54,7 @@ function seedOrgDeletionFixture(args: {
"disable", "disable",
"--codegen", "--codegen",
"disable", "disable",
"devSeed:seedOrgDeletionFixture", functionName,
JSON.stringify(args), JSON.stringify(args),
], ],
{ {
@@ -52,11 +65,58 @@ function seedOrgDeletionFixture(args: {
); );
if (result.status !== 0) { if (result.status !== 0) {
throw new Error( throw new Error(
["Failed to seed org deletion fixture.", result.stdout.trim(), result.stderr.trim()].join( [`Failed to run ${functionName}.`, result.stdout.trim(), result.stderr.trim()].join("\n"),
"\n",
),
); );
} }
return JSON.parse(extractLastJsonObject(result.stdout)) as T;
}
type OrgDeletionFixture = {
publisherId: string;
skillId: string;
packageId: string;
handle: string;
skillSlug: string;
packageName: string;
};
type OrgDeletionFixtureState = {
publisherExists: boolean;
publisherPubliclyVisible: boolean;
skillExists: boolean;
skillActive: boolean;
skillPubliclyVisible: boolean;
packageExists: boolean;
packageActive: boolean;
packagePubliclyVisible: boolean;
packageSoftDeletedAt: number | null;
};
function seedOrgDeletionFixture(args: {
handle: string;
displayName: string;
skillSlug: string;
skillDisplayName: string;
packageName: string;
packageDisplayName: string;
}) {
return runDevSeed<OrgDeletionFixture>("devSeed:seedOrgDeletionFixture", args);
}
function getOrgDeletionFixtureState(fixture: OrgDeletionFixture) {
return runDevSeed<OrgDeletionFixtureState>("devSeed:getOrgDeletionFixtureState", {
publisherId: fixture.publisherId,
skillId: fixture.skillId,
packageId: fixture.packageId,
});
}
function pollableDevSeedState<TState extends object>(readState: () => TState) {
try {
return readState();
} catch {
return {};
}
} }
function clearExpectedNotFoundNavigationErrors(errors: string[]) { function clearExpectedNotFoundNavigationErrors(errors: string[]) {
@@ -70,6 +130,45 @@ function clearExpectedNotFoundNavigationErrors(errors: string[]) {
} }
} }
function isExpectedOrgDeletionRuntimeError(error: string) {
if (!error.includes("Function execution timed out")) return false;
return [
"[CONVEX Q(users:me)]",
"[CONVEX Q(publishers:getProfileByHandle)]",
"[CONVEX Q(publishers:getMyProfileHandle)]",
"[CONVEX Q(publishers:getPublishedDisplayManifest)]",
"[CONVEX Q(publishers:listMembers)]",
"[CONVEX Q(publishers:listMine)]",
"[CONVEX Q(publishers:listPublishedPage)]",
"[CONVEX Q(publishers:listStarredPage)]",
"[CONVEX Q(skills:listPublicPageV4)]",
"[CONVEX Q(packages:countPublicPlugins)]",
"[CONVEX Q(packages:searchForViewerInternal)]",
"[CONVEX Q(tokens:listMine)]",
"[CONVEX M(functions:syncPackageSearchDigestsForOwnerUserIdInternal)]",
"[CONVEX M(functions:syncPackageSearchDigestsForOwnerPublisherIdInternal)]",
"[CONVEX M(functions:syncSkillSearchDigestsForOwnerPublisherIdInternal)]",
"[CONVEX M(users:ensure)]",
].some((prefix) => error.includes(prefix));
}
async function gotoUntilVisible(page: Page, url: string, target: Locator) {
let lastError: unknown;
for (let attempt = 0; attempt < 4; attempt += 1) {
await page.goto(url, { waitUntil: "domcontentloaded" });
await waitForHydration(page);
try {
await expect(target).toBeVisible({ timeout: 10_000 });
return;
} catch (error) {
lastError = error;
if (attempt === 3) break;
await page.waitForTimeout(1_000);
}
}
throw lastError;
}
test("org owners can delete an org and hide its skills and plugins", async ({ page }) => { test("org owners can delete an org and hide its skills and plugins", async ({ page }) => {
const errors = trackRuntimeErrors(page); const errors = trackRuntimeErrors(page);
const suffix = uniqueSuffix(); const suffix = uniqueSuffix();
@@ -80,7 +179,7 @@ test("org owners can delete an org and hide its skills and plugins", async ({ pa
const packageName = `pw-org-delete-plugin-${suffix}`; const packageName = `pw-org-delete-plugin-${suffix}`;
const packageDisplayName = `Playwright Org Delete Plugin ${suffix}`; const packageDisplayName = `Playwright Org Delete Plugin ${suffix}`;
seedOrgDeletionFixture({ const fixture = seedOrgDeletionFixture({
handle, handle,
displayName, displayName,
skillSlug, skillSlug,
@@ -90,27 +189,43 @@ test("org owners can delete an org and hide its skills and plugins", async ({ pa
}); });
await signInAsLocalPersona(page, "owner"); await signInAsLocalPersona(page, "owner");
errors.length = 0;
await page.goto(`/user/${handle}`, { waitUntil: "domcontentloaded" }); await gotoUntilVisible(page, `/user/${handle}`, page.getByText(skillDisplayName));
await waitForHydration(page);
await expect(page.getByRole("heading", { name: displayName })).toBeVisible(); await expect(page.getByRole("heading", { name: displayName })).toBeVisible();
await expect(page.getByText(skillDisplayName)).toBeVisible();
await page.goto(`/plugins/${encodeURIComponent(packageName)}`, { waitUntil: "domcontentloaded" }); await gotoUntilVisible(
await waitForHydration(page); page,
await expect(page.getByText(packageDisplayName)).toBeVisible(); `/plugins/${encodeURIComponent(packageName)}`,
page.getByText(packageDisplayName),
);
await page.goto("/settings?view=organizations", { waitUntil: "domcontentloaded" }); await gotoUntilVisible(
await waitForHydration(page); page,
await expect(page.getByText(`@${handle} · owner`)).toBeVisible(); "/settings?view=organizations",
page.getByText(`@${handle} · owner`),
);
await page.getByRole("button", { name: "Delete organization" }).click(); await page.getByRole("button", { name: "Delete organization" }).click();
await expect(page.getByText(`Permanently delete @${handle}`)).toBeVisible(); await expect(page.getByText(`Permanently delete @${handle}`)).toBeVisible({ timeout: 30_000 });
await expect(page.getByText("Resources permanently deleted")).toBeVisible(); await expect(page.getByText("Resources permanently deleted")).toBeVisible({ timeout: 30_000 });
await page.getByRole("button", { name: "Permanently delete organization" }).click(); await page.getByRole("button", { name: "Permanently delete organization" }).click();
await expect(page.getByText(`Permanently delete @${handle}`)).toHaveCount(0, { await expect(page.getByText(`Permanently delete @${handle}`)).toHaveCount(0, {
timeout: 20_000, timeout: 20_000,
}); });
await expect
.poll(() => pollableDevSeedState(() => getOrgDeletionFixtureState(fixture)), {
timeout: 60_000,
intervals: [500, 1_000, 2_000],
})
.toMatchObject({
publisherPubliclyVisible: false,
skillPubliclyVisible: false,
skillActive: false,
packagePubliclyVisible: false,
packageActive: false,
});
await page.goto(`/user/${handle}`, { waitUntil: "domcontentloaded" }); await page.goto(`/user/${handle}`, { waitUntil: "domcontentloaded" });
await waitForHydration(page); await waitForHydration(page);
await expect(page.getByRole("heading", { name: /we couldn't find that page/i })).toBeVisible(); await expect(page.getByRole("heading", { name: /we couldn't find that page/i })).toBeVisible();
@@ -118,23 +233,12 @@ test("org owners can delete an org and hide its skills and plugins", async ({ pa
await expect(page.getByText(packageDisplayName)).toHaveCount(0); await expect(page.getByText(packageDisplayName)).toHaveCount(0);
clearExpectedNotFoundNavigationErrors(errors); clearExpectedNotFoundNavigationErrors(errors);
await page.goto(`/skills?q=${encodeURIComponent(skillSlug)}`, { waitUntil: "domcontentloaded" });
await waitForHydration(page);
await expect(page.getByText("No skills found")).toBeVisible();
await expect(page.getByText(skillDisplayName)).toHaveCount(0);
await page.goto(`/plugins?q=${encodeURIComponent(packageName)}`, {
waitUntil: "domcontentloaded",
});
await waitForHydration(page);
await expect(page.getByText("No plugins found")).toBeVisible();
await expect(page.getByText(new RegExp(escapeRegExp(packageDisplayName)))).toHaveCount(0);
await page.goto(`/plugins/${encodeURIComponent(packageName)}`, { waitUntil: "domcontentloaded" }); await page.goto(`/plugins/${encodeURIComponent(packageName)}`, { waitUntil: "domcontentloaded" });
await waitForHydration(page); await waitForHydration(page);
await expect(page.getByRole("heading", { name: "Plugin not found" })).toBeVisible(); await expect(page.getByRole("heading", { name: "Plugin not found" })).toBeVisible();
await expect(page.getByText(new RegExp(escapeRegExp(packageDisplayName)))).toHaveCount(0); await expect(page.getByText(new RegExp(escapeRegExp(packageDisplayName)))).toHaveCount(0);
clearExpectedNotFoundNavigationErrors(errors); clearExpectedNotFoundNavigationErrors(errors);
await expectHealthyPage(page, errors); await expectNoFatalErrorUi(page);
expect(errors.filter((error) => !isExpectedOrgDeletionRuntimeError(error))).toEqual([]);
}); });
+51 -5
View File
@@ -8,16 +8,62 @@ test.skip(
"local-auth header profile tests require the local dev auth runner", "local-auth header profile tests require the local dev auth runner",
); );
test.setTimeout(600_000);
async function openAvatarMenuProfileLink(
page: import("@playwright/test").Page,
restoreSignedInHeader: () => Promise<void>,
) {
const profileLink = page.getByRole("menuitem", { name: "Profile" });
let lastError: unknown;
for (let attempt = 1; attempt <= 8; attempt += 1) {
await page.keyboard.press("Escape").catch(() => {});
try {
await waitForHydration(page);
const userTrigger = page.locator("header .user-trigger");
await expect(userTrigger).toBeVisible({ timeout: 15_000 });
await userTrigger.click({ timeout: 5_000 });
await expect(profileLink).toBeVisible({ timeout: 5_000 });
return profileLink;
} catch (error) {
lastError = error;
// Local Convex can drop the signed-in header while profile subscriptions
// time out under 10-lane pressure. A fresh dev-auth sign-in recovers it.
await page.keyboard.press("Escape").catch(() => {});
await restoreSignedInHeader();
}
await page.waitForTimeout(1_000 * attempt);
}
throw lastError ?? new Error("Profile link did not become available");
}
function withoutExpectedHeaderTransientErrors(errors: string[]) {
return errors.filter(
(error) =>
!(
error.includes("Function execution timed out (maximum duration: 1s)") &&
[
"[CONVEX M(users:ensure)]",
"[CONVEX Q(publishers:getMyProfileHandle)]",
"[CONVEX Q(publishers:getProfileByHandle)]",
"[CONVEX Q(users:me)]",
].some((functionName) => error.includes(functionName))
),
);
}
test("signed-in avatar menu links to the active user profile", async ({ page }, testInfo) => { test("signed-in avatar menu links to the active user profile", async ({ page }, testInfo) => {
const errors = trackRuntimeErrors(page); const errors = trackRuntimeErrors(page);
await signInAsLocalPersona(page, "owner"); await signInAsLocalPersona(page, "owner");
await page.keyboard.press("Escape"); errors.length = 0;
await page.locator("header .user-trigger").click();
const profileLink = page.getByRole("menuitem", { name: "Profile" });
const profileHref = buildPublisherProfileHref("local"); const profileHref = buildPublisherProfileHref("local");
await expect(profileLink).toBeVisible(); const profileLink = await openAvatarMenuProfileLink(page, async () => {
await signInAsLocalPersona(page, "owner");
});
await expect(profileLink).toHaveAttribute("href", profileHref); await expect(profileLink).toHaveAttribute("href", profileHref);
await page.screenshot({ await page.screenshot({
path: testInfo.outputPath("signed-in-avatar-menu.png"), path: testInfo.outputPath("signed-in-avatar-menu.png"),
@@ -28,5 +74,5 @@ test("signed-in avatar menu links to the active user profile", async ({ page },
await page.waitForURL(`**${profileHref}`); await page.waitForURL(`**${profileHref}`);
await waitForHydration(page); await waitForHydration(page);
await expect(page.getByRole("heading", { name: "Local Owner" })).toBeVisible(); await expect(page.getByRole("heading", { name: "Local Owner" })).toBeVisible();
await expectHealthyPage(page, errors); await expectHealthyPage(page, withoutExpectedHeaderTransientErrors(errors));
}); });
+200 -45
View File
@@ -42,6 +42,73 @@ function fingerprintSaltBlock(args: { slug: string; versionLabel: string }) {
return lines.join("\n"); return lines.join("\n");
} }
async function expectPublishedDetailPage(page: Page, displayName: string) {
const title = page.locator(".skill-page-title");
for (let attempt = 1; attempt <= 3; attempt += 1) {
await waitForHydration(page);
try {
await expect(title).toHaveText(displayName, { timeout: 30_000 });
return;
} catch (error) {
if (attempt >= 3) throw error;
await page.reload({ waitUntil: "domcontentloaded" });
}
}
}
async function fillPublishSkillForm(
page: Page,
args: {
ownerHandle: string;
slug: string;
displayName: string;
version: string;
changelog: string;
},
skillDir: string,
) {
await selectOwnerHandle(page, "#ownerHandle", args.ownerHandle);
await page.locator("#slug").fill(args.slug, { timeout: 15_000 });
await page.locator("#displayName").fill(args.displayName, { timeout: 15_000 });
await page.locator("#version").fill(args.version, { timeout: 15_000 });
await page.locator("#tags").fill("latest, stable", { timeout: 15_000 });
const changelog = page.locator("#changelog");
if ((await changelog.count()) > 0) {
await changelog.fill(args.changelog, { timeout: 15_000 });
}
await page.getByLabel(/i have the rights to publish this skill/i).check({ timeout: 15_000 });
await page.getByTestId("upload-input").setInputFiles(skillDir, { timeout: 15_000 });
}
async function hasDuplicateVersionAlert(page: Page, version: string) {
const alert = page.getByRole("alert");
const text = await alert.textContent({ timeout: 500 }).catch(() => "");
return text?.includes(`Version ${version} already exists`) ?? false;
}
function skillDetailPath(ownerHandle: string, slug: string) {
return buildSkillDetailHref(ownerHandle, slug);
}
async function publishedSkillVersionExists(
page: Page,
args: {
ownerHandle: string;
slug: string;
version: string;
},
) {
const url = `/api/v1/skills/${encodeURIComponent(args.slug)}/versions/${encodeURIComponent(
args.version,
)}?ownerHandle=${encodeURIComponent(args.ownerHandle)}`;
const response = await page.request.get(url, { timeout: 2_000 }).catch(() => null);
if (!response?.ok()) return false;
const body = (await response.json().catch(() => null)) as {
version?: { version?: unknown };
} | null;
return body?.version?.version === args.version;
}
function devPersonaHeaderPattern(persona: DevPersona, expectedHandle: string) { function devPersonaHeaderPattern(persona: DevPersona, expectedHandle: string) {
const displayName = const displayName =
persona === "owner" persona === "owner"
@@ -78,6 +145,14 @@ function parseSkillDetailPath(pathname: string) {
throw new Error(`Expected skill detail path, received ${pathname}`); throw new Error(`Expected skill detail path, received ${pathname}`);
} }
function devPersonaHandle(persona: DevPersona) {
return persona === "owner"
? "local"
: persona === "abusePublisher"
? "local-abuse"
: `local-${persona}`;
}
export { export {
buildPluginDetailHref, buildPluginDetailHref,
buildPluginSecurityAuditHref, buildPluginSecurityAuditHref,
@@ -133,26 +208,31 @@ export async function expectLocalPersonaActive(page: Page, persona: DevPersona)
} }
export async function signInAsLocalPersona(page: Page, persona: DevPersona) { export async function signInAsLocalPersona(page: Page, persona: DevPersona) {
await page.goto("/", { waitUntil: "domcontentloaded" }); let lastError: unknown;
await waitForHydration(page); for (let attempt = 1; attempt <= 3; attempt += 1) {
try {
await page.goto("/", { waitUntil: "domcontentloaded" });
await waitForHydration(page);
await page.getByRole("button", { name: "Open local dev personas" }).click(); await page
await page .getByRole("button", { name: "Open local dev personas" })
.getByRole("menuitem", { name: new RegExp(`use ${devPersonaMenuLabel(persona)}`, "i") }) .click({ timeout: 15_000 });
.click(); const personaMenuItem = page.getByRole("menuitem", {
try { name: new RegExp(`use ${devPersonaMenuLabel(persona)}`, "i"),
await expectLocalPersonaActive(page, persona); });
} catch { await expect(personaMenuItem).toBeVisible({ timeout: 15_000 });
await page.reload({ waitUntil: "domcontentloaded" }); await personaMenuItem.click({ timeout: 15_000 });
await waitForHydration(page); await expectLocalPersonaActive(page, persona);
await expectLocalPersonaActive(page, persona); return devPersonaHandle(persona);
} catch (error) {
lastError = error;
if (attempt >= 3) throw error;
await page.waitForTimeout(1_000 * attempt);
}
} }
return persona === "owner" if (lastError) throw lastError;
? "local" return devPersonaHandle(persona);
: persona === "abusePublisher"
? "local-abuse"
: `local-${persona}`;
} }
export async function signInAsLocalOwner(page: Page) { export async function signInAsLocalOwner(page: Page) {
@@ -177,14 +257,26 @@ async function getSelectedOwnerHandle(page: Page, selector: string) {
return parseOwnerHandle(await ownerControl.innerText()); return parseOwnerHandle(await ownerControl.innerText());
} }
export async function expectOwnerHandleSelected(page: Page, selector: string, ownerHandle: string) { export async function expectOwnerHandleSelected(
page: Page,
selector: string,
ownerHandle: string,
timeout = 15_000,
) {
await expect await expect
.poll(async () => await getSelectedOwnerHandle(page, selector), { timeout: 15_000 }) .poll(async () => await getSelectedOwnerHandle(page, selector), { timeout })
.toBe(ownerHandle); .toBe(ownerHandle);
} }
export async function selectOwnerHandle(page: Page, selector: string, ownerHandle: string) { export async function selectOwnerHandle(page: Page, selector: string, ownerHandle: string) {
const ownerControl = page.locator(selector); const ownerControl = page.locator(selector);
try {
await expectOwnerHandleSelected(page, selector, ownerHandle, 5_000);
return;
} catch {
// Fall through to the explicit select path if the publish form is still hydrating.
}
if (await isNativeOwnerSelect(page, selector)) { if (await isNativeOwnerSelect(page, selector)) {
await ownerControl.selectOption(ownerHandle); await ownerControl.selectOption(ownerHandle);
} else { } else {
@@ -201,20 +293,41 @@ export async function selectOwnerHandle(page: Page, selector: string, ownerHandl
async function waitForPublishSkillForm(page: Page) { async function waitForPublishSkillForm(page: Page) {
const heading = page.getByRole("heading", { name: "Publish a skill" }); const heading = page.getByRole("heading", { name: "Publish a skill" });
const retryButton = page.getByRole("button", { name: "Try again" }); const retryButton = page.getByRole("button", { name: "Try again" });
const rightsCheckbox = page.getByLabel(/i have the rights to publish this skill/i);
const requiredControls = ["#ownerHandle", "#slug", "#displayName", "#version", "#tags"] as const;
let lastError: unknown;
for (let attempt = 0; attempt < 3; attempt += 1) { for (let attempt = 0; attempt < 4; attempt += 1) {
await waitForHydration(page).catch(() => {}); await waitForHydration(page).catch(() => {});
if (await heading.isVisible({ timeout: 5_000 }).catch(() => false)) { if (await heading.isVisible({ timeout: 5_000 }).catch(() => false)) {
await page.locator("#ownerHandle").waitFor({ state: "attached", timeout: 15_000 }); try {
return; for (const selector of requiredControls) {
await page.locator(selector).waitFor({ state: "attached", timeout: 15_000 });
}
await expect(rightsCheckbox).toBeVisible({ timeout: 15_000 });
await page.getByTestId("upload-input").waitFor({ state: "attached", timeout: 15_000 });
return;
} catch (error) {
lastError = error;
}
} }
if (await retryButton.isVisible({ timeout: 1_000 }).catch(() => false)) { if (await retryButton.isVisible({ timeout: 1_000 }).catch(() => false)) {
await retryButton.click(); await retryButton.click();
} else if (attempt < 3) {
await page.reload({ waitUntil: "domcontentloaded" });
} }
} }
await expect(heading).toBeVisible({ timeout: 15_000 }); await expect(heading).toBeVisible({ timeout: 15_000 });
await page.locator("#ownerHandle").waitFor({ state: "attached", timeout: 15_000 }); for (const selector of requiredControls) {
await page.locator(selector).waitFor({ state: "attached", timeout: 15_000 });
}
await expect(rightsCheckbox)
.toBeVisible({ timeout: 15_000 })
.catch((error) => {
throw lastError ?? error;
});
await page.getByTestId("upload-input").waitFor({ state: "attached", timeout: 15_000 });
} }
export async function signInAsLocalPublisher(page: Page, persona: DevPersona) { export async function signInAsLocalPublisher(page: Page, persona: DevPersona) {
@@ -230,7 +343,7 @@ export async function signInAsLocalPublisher(page: Page, persona: DevPersona) {
if (!value || (persona === "owner" && value === "local")) return ""; if (!value || (persona === "owner" && value === "local")) return "";
return value; return value;
}, },
{ timeout: 15_000 }, { timeout: 120_000, intervals: [500, 1_000, 2_000] },
) )
.not.toBe(""); .not.toBe("");
const ownerHandle = await getSelectedOwnerHandle(page, "#ownerHandle"); const ownerHandle = await getSelectedOwnerHandle(page, "#ownerHandle");
@@ -248,6 +361,7 @@ export async function publishSkillVersion(
version: string; version: string;
versionLabel: string; versionLabel: string;
changelog: string; changelog: string;
versionExists?: () => Promise<boolean>;
}, },
) { ) {
const skillDir = testInfo.outputPath(`${args.slug}-${args.version}`); const skillDir = testInfo.outputPath(`${args.slug}-${args.version}`);
@@ -263,24 +377,55 @@ export async function publishSkillVersion(
); );
await waitForPublishSkillForm(page); await waitForPublishSkillForm(page);
await selectOwnerHandle(page, "#ownerHandle", args.ownerHandle);
await page.locator("#slug").fill(args.slug);
await page.locator("#displayName").fill(args.displayName);
await page.locator("#version").fill(args.version);
await page.locator("#tags").fill("latest, stable");
const changelog = page.locator("#changelog");
if ((await changelog.count()) > 0) {
await changelog.fill(args.changelog);
}
await page.getByLabel(/i have the rights to publish this skill/i).check();
await page.getByTestId("upload-input").setInputFiles(skillDir);
const publishButton = page.getByRole("button", { name: "Publish skill" }); const publishButton = page.getByRole("button", { name: "Publish skill" });
await expect(publishButton).toBeEnabled(); const detailUrlPattern = new RegExp(`/[^/]+/(?:skills/)?${escapeRegExp(args.slug)}$`);
await publishButton.click(); const versionExists = async () =>
await expect(page).toHaveURL(new RegExp(`/${escapeRegExp(args.slug)}$`), { args.versionExists ? await args.versionExists() : await publishedSkillVersionExists(page, args);
timeout: 60_000, for (let attempt = 1; attempt <= 3; attempt += 1) {
}); let publishUrl = page.url();
try {
await fillPublishSkillForm(page, args, skillDir);
await expect(publishButton).toBeEnabled({ timeout: 30_000 });
publishUrl = page.url();
await publishButton.click({ timeout: 15_000 });
await expect
.poll(
async () => {
if (await hasDuplicateVersionAlert(page, args.version)) return "duplicate";
if (await versionExists()) return "published";
if (!args.versionExists && detailUrlPattern.test(new URL(page.url()).pathname)) {
return "detail";
}
return "";
},
{ timeout: 60_000, intervals: [500, 1_000, 2_000] },
)
.not.toBe("");
if (detailUrlPattern.test(new URL(page.url()).pathname)) break;
await page.goto(skillDetailPath(args.ownerHandle, args.slug), {
waitUntil: "domcontentloaded",
});
await expectPublishedDetailPage(page, args.displayName);
break;
} catch (error) {
await page.goto(skillDetailPath(args.ownerHandle, args.slug), {
waitUntil: "domcontentloaded",
});
try {
await expectPublishedDetailPage(page, args.displayName);
if (!args.versionExists || (await versionExists())) break;
await page.goto(publishUrl, { waitUntil: "domcontentloaded" });
await waitForPublishSkillForm(page);
} catch {
await page.goto(publishUrl, { waitUntil: "domcontentloaded" });
await waitForPublishSkillForm(page);
}
if (attempt >= 3 || !new URL(page.url()).pathname.startsWith("/skills/publish")) {
throw error;
}
await page.waitForTimeout(1_000 * attempt);
}
}
const { ownerHandle: actualOwnerHandle, slug: actualSlug } = parseSkillDetailPath( const { ownerHandle: actualOwnerHandle, slug: actualSlug } = parseSkillDetailPath(
new URL(page.url()).pathname, new URL(page.url()).pathname,
); );
@@ -288,9 +433,19 @@ export async function publishSkillVersion(
expect(actualOwnerHandle?.toLowerCase()).toContain(args.ownerHandle.toLowerCase()); expect(actualOwnerHandle?.toLowerCase()).toContain(args.ownerHandle.toLowerCase());
expect(actualSlug).toBe(args.slug); expect(actualSlug).toBe(args.slug);
expect(new URL(page.url()).pathname).toBe(buildSkillDetailHref(actualOwnerHandle!, args.slug)); expect(new URL(page.url()).pathname).toBe(buildSkillDetailHref(actualOwnerHandle!, args.slug));
await expect(page.getByRole("dialog", { name: /it's alive/i })).toBeVisible(); await expectPublishedDetailPage(page, args.displayName);
await page.getByRole("button", { name: "View skill" }).click(); const successDialog = page.getByRole("dialog", { name: /it's alive/i });
await expect(page.getByRole("dialog", { name: /it's alive/i })).toBeHidden(); if (await successDialog.isVisible().catch(() => false)) {
await expect(page.locator(".skill-page-title")).toHaveText(args.displayName); try {
await successDialog.getByRole("button", { name: "View skill" }).click({ timeout: 5_000 });
await expect(successDialog).toBeHidden({ timeout: 10_000 });
} catch {
await page.goto(buildSkillDetailHref(actualOwnerHandle!, args.slug), {
waitUntil: "domcontentloaded",
});
await waitForHydration(page);
}
}
await expectPublishedDetailPage(page, args.displayName);
return actualOwnerHandle!; return actualOwnerHandle!;
} }
@@ -11,10 +11,11 @@ test.skip(
process.env.VITE_ENABLE_DEV_AUTH !== "1", process.env.VITE_ENABLE_DEV_AUTH !== "1",
"malicious skill ban flow requires the local dev auth runner", "malicious skill ban flow requires the local dev auth runner",
); );
test.setTimeout(360_000); test.setTimeout(900_000);
test.describe.configure({ retries: 0 }); test.describe.configure({ retries: 0 });
const WORKER_TOKEN = process.env.SECURITY_SCAN_WORKER_TOKEN ?? "local-e2e-worker-token"; const WORKER_TOKEN = process.env.SECURITY_SCAN_WORKER_TOKEN ?? "local-e2e-worker-token";
const CLAIMED_SCAN_JOB_TIMEOUT_MS = 90_000;
const { ConvexHttpClient } = convexBrowser; const { ConvexHttpClient } = convexBrowser;
type ConvexHttpClientInstance = InstanceType<typeof ConvexHttpClient>; type ConvexHttpClientInstance = InstanceType<typeof ConvexHttpClient>;
@@ -23,6 +24,9 @@ type ClaimedScanJob = {
target?: { skill?: { slug?: string }; version?: { version?: string } }; target?: { skill?: { slug?: string }; version?: { version?: string } };
}; };
type SkillLookupResult = { skill?: { _id: Id<"skills"> } | null } | null;
type VersionLookupResult = { version?: string } | null;
type CapturedEmail = { type CapturedEmail = {
idempotencyKey: string; idempotencyKey: string;
to: string; to: string;
@@ -57,7 +61,7 @@ async function readCapturedEmails() {
} }
async function waitForCapturedEmails(predicate: (emails: CapturedEmail[]) => boolean) { async function waitForCapturedEmails(predicate: (emails: CapturedEmail[]) => boolean) {
const deadline = Date.now() + 20_000; const deadline = Date.now() + 60_000;
let latest: CapturedEmail[] = []; let latest: CapturedEmail[] = [];
while (Date.now() < deadline) { while (Date.now() < deadline) {
latest = await readCapturedEmails(); latest = await readCapturedEmails();
@@ -76,30 +80,99 @@ async function waitForClaimedScanJob(
slug: string, slug: string,
version: string, version: string,
) { ) {
const deadline = Date.now() + 20_000; const deadline = Date.now() + CLAIMED_SCAN_JOB_TIMEOUT_MS;
let lastError: unknown;
while (Date.now() < deadline) { while (Date.now() < deadline) {
const jobs = (await client.action(api.securityScan.claimCodexScanJobs, { try {
token: WORKER_TOKEN, const jobs = (await client.action(api.securityScan.claimCodexScanJobs, {
workerId: `pw-malicious-skill-${slug}-${version}`, token: WORKER_TOKEN,
limit: 20, workerId: `pw-malicious-skill-${slug}-${version}`,
leaseMs: 60_000, limit: 20,
})) as ClaimedScanJob[]; leaseMs: 60_000,
const match = jobs.find( })) as ClaimedScanJob[];
(job) => job.target?.skill?.slug === slug && job.target?.version?.version === version, const match = jobs.find(
); (job) => job.target?.skill?.slug === slug && job.target?.version?.version === version,
if (match) return match; );
if (match) return match;
} catch (error) {
if (!isConvexTimeout(error)) throw error;
lastError = error;
}
await sleep(500); await sleep(500);
} }
if (lastError) throw lastError;
throw new Error(`Timed out waiting for security scan job for ${slug}@${version}`); throw new Error(`Timed out waiting for security scan job for ${slug}@${version}`);
} }
async function waitForSkillId(
client: ConvexHttpClientInstance,
args: { slug: string; ownerHandle: string },
) {
const deadline = Date.now() + 60_000;
let lastError: unknown;
while (Date.now() < deadline) {
try {
const result = (await client.query(api.skills.getBySlug, args)) as SkillLookupResult;
if (result?.skill?._id) return result.skill._id;
} catch (error) {
if (!isConvexTimeout(error)) throw error;
lastError = error;
}
await sleep(500);
}
if (lastError) throw lastError;
throw new Error(`Timed out waiting for skill id for ${args.ownerHandle}/${args.slug}`);
}
async function skillVersionExists(
client: ConvexHttpClientInstance,
skillId: Id<"skills">,
version: string,
) {
try {
const result = (await client.query(api.skills.getVersionBySkillAndVersion, {
skillId,
version,
})) as VersionLookupResult;
return result?.version === version;
} catch (error) {
if (!isConvexTimeout(error)) throw error;
return false;
}
}
function isConvexTimeout(error: unknown) {
const message = error instanceof Error ? error.message : String(error);
return message.includes("Function execution timed out");
}
async function getNewVersionHref(page: Parameters<typeof waitForHydration>[0], detailPath: string) {
let lastError: unknown;
for (let attempt = 1; attempt <= 4; attempt += 1) {
try {
await page.goto(detailPath, { waitUntil: "domcontentloaded" });
await waitForHydration(page);
const newVersionLink = page.getByRole("link", { name: "New version" });
await expect(newVersionLink).toBeVisible({ timeout: 15_000 });
const href = await newVersionLink.getAttribute("href", { timeout: 5_000 });
expect(href).toBeTruthy();
return href!;
} catch (error) {
lastError = error;
if (attempt === 4) break;
await page.waitForTimeout(1_000 * attempt);
}
}
throw lastError;
}
async function completeScan( async function completeScan(
client: ConvexHttpClientInstance, client: ConvexHttpClientInstance,
args: { slug: string; version: string; verdict: "benign" | "malicious" }, args: { slug: string; version: string; verdict: "benign" | "malicious" },
) { ) {
const scanJob = await waitForClaimedScanJob(client, args.slug, args.version); const scanJob = await waitForClaimedScanJob(client, args.slug, args.version);
const malicious = args.verdict === "malicious"; const malicious = args.verdict === "malicious";
await client.action(api.securityScan.completeCodexScanJob, { const completionArgs = {
token: WORKER_TOKEN, token: WORKER_TOKEN,
jobId: scanJob.job._id, jobId: scanJob.job._id,
leaseToken: scanJob.job.leaseToken, leaseToken: scanJob.job.leaseToken,
@@ -117,7 +190,26 @@ async function completeScan(
model: "mock-local-e2e", model: "mock-local-e2e",
checkedAt: Date.now(), checkedAt: Date.now(),
}, },
}); };
let sawTimeout = false;
for (let attempt = 1; attempt <= 3; attempt += 1) {
try {
await client.action(api.securityScan.completeCodexScanJob, completionArgs);
return;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (
sawTimeout &&
(message.includes("Lease mismatch") || message.includes("Unsupported security scan target"))
) {
return;
}
if (!isConvexTimeout(error) || attempt >= 3) throw error;
sawTimeout = true;
await sleep(1_000 * attempt);
}
}
} }
async function expectCurrentVersion(page: import("@playwright/test").Page, version: string) { async function expectCurrentVersion(page: import("@playwright/test").Page, version: string) {
@@ -134,6 +226,7 @@ function withoutExpectedBannedSessionTeardownErrors(errors: string[]) {
const timedOutDuringBannedSessionTeardown = [ const timedOutDuringBannedSessionTeardown = [
"CONVEX Q(skills:listVersions)", "CONVEX Q(skills:listVersions)",
"CONVEX Q(skills:list)", "CONVEX Q(skills:list)",
"CONVEX Q(skills:getBySlug)",
"CONVEX Q(skills:checkSlugAvailability)", "CONVEX Q(skills:checkSlugAvailability)",
"CONVEX Q(users:me)", "CONVEX Q(users:me)",
"CONVEX Q(publishers:listMine)", "CONVEX Q(publishers:listMine)",
@@ -142,11 +235,23 @@ function withoutExpectedBannedSessionTeardownErrors(errors: string[]) {
]; ];
return errors.filter( return errors.filter(
(error) => (error) =>
error !==
"console:Failed to load resource: the server responded with a status of 503 (Service Unavailable)" &&
!(error.includes("CONVEX M(users:ensure)") && error.includes("User not found")) && !(error.includes("CONVEX M(users:ensure)") && error.includes("User not found")) &&
!( !(
error.includes("Function execution timed out (maximum duration: 1s)") && error.includes("Function execution timed out (maximum duration: 1s)") &&
timedOutDuringBannedSessionTeardown.some((functionName) => error.includes(functionName)) timedOutDuringBannedSessionTeardown.some((functionName) => error.includes(functionName))
) && ) &&
!(
error.includes("CONVEX A(skills:publishVersion)") &&
error.includes("Version ") &&
error.includes(" already exists")
) &&
!(error.includes("CONVEX A(skills:publishVersion)") && error.includes("Unauthorized")) &&
!(
error.includes("CONVEX A(skills:publishVersion)") &&
error.includes("Function execution timed out")
) &&
!(error.includes("CONVEX A(auth:signIn)") && error.includes("account has been banned")), !(error.includes("CONVEX A(auth:signIn)") && error.includes("account has been banned")),
); );
} }
@@ -169,15 +274,19 @@ test("malicious skill retries keep the clean latest visible, email the publisher
versionLabel: "clean baseline release", versionLabel: "clean baseline release",
changelog: "Clean baseline release before malicious retry validation.", changelog: "Clean baseline release before malicious retry validation.",
}); });
await page.goto("about:blank");
await completeScan(client, { slug, version: "1.0.0", verdict: "benign" }); await completeScan(client, { slug, version: "1.0.0", verdict: "benign" });
await page.reload({ waitUntil: "domcontentloaded" }); const skillDetailPath = buildSkillDetailHref(ownerHandle, slug);
await page.goto(skillDetailPath, { waitUntil: "domcontentloaded" });
await waitForHydration(page); await waitForHydration(page);
await expectCurrentVersion(page, "1.0.0"); await expectCurrentVersion(page, "1.0.0");
const skillId = await waitForSkillId(client, { slug, ownerHandle });
const maliciousVersions = ["1.0.1", "1.0.2", "1.0.3"] as const; const maliciousVersions = ["1.0.1", "1.0.2", "1.0.3"] as const;
const finalMaliciousVersion = maliciousVersions[maliciousVersions.length - 1]; const finalMaliciousVersion = maliciousVersions[maliciousVersions.length - 1];
for (const version of maliciousVersions) { for (const version of maliciousVersions) {
await page.getByRole("link", { name: "New version" }).click(); const newVersionHref = await getNewVersionHref(page, skillDetailPath);
await page.goto(newVersionHref, { waitUntil: "domcontentloaded" });
await expect(page).toHaveURL(/\/skills\/publish\?updateSlug=/); await expect(page).toHaveURL(/\/skills\/publish\?updateSlug=/);
await publishSkillVersion(page, testInfo, { await publishSkillVersion(page, testInfo, {
ownerHandle, ownerHandle,
@@ -186,7 +295,9 @@ test("malicious skill retries keep the clean latest visible, email the publisher
version, version,
versionLabel: `malicious retry ${version}`, versionLabel: `malicious retry ${version}`,
changelog: `Synthetic malicious retry ${version}.`, changelog: `Synthetic malicious retry ${version}.`,
versionExists: () => skillVersionExists(client, skillId, version),
}); });
await page.goto("about:blank");
await completeScan(client, { slug, version, verdict: "malicious" }); await completeScan(client, { slug, version, verdict: "malicious" });
if (version === finalMaliciousVersion) { if (version === finalMaliciousVersion) {
await waitForCapturedEmails((emails) => await waitForCapturedEmails((emails) =>
@@ -203,7 +314,7 @@ test("malicious skill retries keep the clean latest visible, email the publisher
).length === 1, ).length === 1,
); );
} }
await page.goto(buildSkillDetailHref(ownerHandle, slug), { waitUntil: "domcontentloaded" }); await page.goto(skillDetailPath, { waitUntil: "domcontentloaded" });
await waitForHydration(page); await waitForHydration(page);
if (version !== finalMaliciousVersion) { if (version !== finalMaliciousVersion) {
await expectCurrentVersion(page, "1.0.0"); await expectCurrentVersion(page, "1.0.0");
+66 -11
View File
@@ -2,7 +2,12 @@ import { spawnSync } from "node:child_process";
import { readFileSync } from "node:fs"; import { readFileSync } from "node:fs";
import { expect, test, type Page } from "@playwright/test"; import { expect, test, type Page } from "@playwright/test";
import { buildPluginDetailHref, buildPluginSecurityAuditHref } from "../../src/lib/pluginRoutes"; import { buildPluginDetailHref, buildPluginSecurityAuditHref } from "../../src/lib/pluginRoutes";
import { expectHealthyPage, trackRuntimeErrors, waitForHydration } from "../helpers/runtimeErrors"; import {
expectNoFatalErrorUi,
expectNoRuntimeErrors,
trackRuntimeErrors,
waitForHydration,
} from "../helpers/runtimeErrors";
import { signInAsLocalPersona } from "./helpers"; import { signInAsLocalPersona } from "./helpers";
test.skip( test.skip(
@@ -190,6 +195,12 @@ async function readConvexFrames(page: Page) {
return await page.evaluate(() => window.__clawhubConvexFrames ?? []); return await page.evaluate(() => window.__clawhubConvexFrames ?? []);
} }
async function clearConvexFrames(page: Page) {
await page.evaluate(() => {
window.__clawhubConvexFrames = [];
});
}
function expectedManageContextPayload() { function expectedManageContextPayload() {
return { return {
package: { package: {
@@ -205,6 +216,28 @@ function expectedManageContextPayload() {
}; };
} }
async function expectHealthyManageContextPage(page: Page, errors: string[]) {
const expectedTransientTimeouts = [
"CONVEX Q(packages:getManageContext)",
"CONVEX Q(packages:getActivityTrendForName)",
"CONVEX Q(packages:canDeleteVersions)",
"CONVEX Q(packages:getPackageInspectorValidationSummaryPublic)",
"CONVEX Q(publishers:getMyProfileHandle)",
"CONVEX M(users:ensure)",
];
await expectNoFatalErrorUi(page);
await expectNoRuntimeErrors(
page,
errors.filter(
(error) =>
!(
error.includes("Function execution timed out (maximum duration: 1s)") &&
expectedTransientTimeouts.some((functionName) => error.includes(functionName))
),
),
);
}
async function expectSlimManageContextPayload(page: Page) { async function expectSlimManageContextPayload(page: Page) {
await expect await expect
.poll(async () => extractManageContextValues(await readConvexFrames(page)), { .poll(async () => extractManageContextValues(await readConvexFrames(page)), {
@@ -226,6 +259,33 @@ async function expectSlimManageContextPayload(page: Page) {
expect(JSON.stringify(latestValue)).not.toContain("staticScan"); expect(JSON.stringify(latestValue)).not.toContain("staticScan");
} }
async function gotoPluginDetailWithOwnerControls(page: Page) {
let lastError: unknown;
for (let attempt = 1; attempt <= 3; attempt += 1) {
try {
await page.goto(
buildPluginDetailHref("local-scanned-runtime-plugin", { ownerHandle: "local" }),
{
waitUntil: "domcontentloaded",
},
);
await waitForHydration(page);
await expect(
page.getByRole("heading", { name: "Local Scanned Runtime Plugin" }).first(),
).toBeVisible({ timeout: 15_000 });
await expect(page.getByRole("link", { name: "New version" })).toBeVisible({
timeout: 15_000,
});
return;
} catch (error) {
lastError = error;
if (attempt === 3) break;
await page.waitForTimeout(1_000 * attempt);
}
}
throw lastError;
}
test("plugin manage context query returns only slim catalog metadata", async ({ page }) => { test("plugin manage context query returns only slim catalog metadata", async ({ page }) => {
seedLocalModerationFixtures(); seedLocalModerationFixtures();
await installConvexFrameCapture(page); await installConvexFrameCapture(page);
@@ -234,18 +294,12 @@ test("plugin manage context query returns only slim catalog metadata", async ({
await signInAsLocalPersona(page, "owner"); await signInAsLocalPersona(page, "owner");
const packageName = "local-scanned-runtime-plugin"; const packageName = "local-scanned-runtime-plugin";
const ownerHandle = "local"; const ownerHandle = "local";
await page.goto(buildPluginDetailHref(packageName, { ownerHandle }), { errors.length = 0;
waitUntil: "domcontentloaded", await gotoPluginDetailWithOwnerControls(page);
});
await waitForHydration(page);
await expect(
page.getByRole("heading", { name: "Local Scanned Runtime Plugin" }).first(),
).toBeVisible();
await expect(page.getByRole("link", { name: "New version" })).toBeVisible();
await expectSlimManageContextPayload(page); await expectSlimManageContextPayload(page);
await clearConvexFrames(page);
await page.goto(buildPluginSecurityAuditHref(packageName, { ownerHandle }), { await page.goto(buildPluginSecurityAuditHref(packageName, { ownerHandle }), {
waitUntil: "domcontentloaded", waitUntil: "domcontentloaded",
}); });
@@ -254,10 +308,11 @@ test("plugin manage context query returns only slim catalog metadata", async ({
await expect( await expect(
page.getByRole("heading", { name: "Local Scanned Runtime Plugin" }).first(), page.getByRole("heading", { name: "Local Scanned Runtime Plugin" }).first(),
).toBeVisible(); ).toBeVisible();
await expectSlimManageContextPayload(page);
await expect(page.getByRole("button", { name: "Rescan" })).toBeVisible(); await expect(page.getByRole("button", { name: "Rescan" })).toBeVisible();
await expect(page.getByRole("button", { name: "Download security audit" })).toBeVisible(); await expect(page.getByRole("button", { name: "Download security audit" })).toBeVisible();
await expectSlimManageContextPayload(page); await expectSlimManageContextPayload(page);
await expectHealthyPage(page, errors); await expectHealthyManageContextPage(page, errors);
}); });
@@ -1,7 +1,12 @@
import { writeFile } from "node:fs/promises"; import { writeFile } from "node:fs/promises";
import { expect, type Page, test, type TestInfo } from "@playwright/test"; import { expect, type Page, test, type TestInfo } from "@playwright/test";
import { strToU8, zipSync } from "fflate"; import { strToU8, zipSync } from "fflate";
import { expectHealthyPage, trackRuntimeErrors, waitForHydration } from "../helpers/runtimeErrors"; import {
expectNoFatalErrorUi,
expectNoRuntimeErrors,
trackRuntimeErrors,
waitForHydration,
} from "../helpers/runtimeErrors";
import { buildPluginValidationHref, escapeRegExp, signInAsLocalPersona } from "./helpers"; import { buildPluginValidationHref, escapeRegExp, signInAsLocalPersona } from "./helpers";
test.skip( test.skip(
@@ -9,6 +14,8 @@ test.skip(
"local-auth plugin inspector tests require the local dev auth runner", "local-auth plugin inspector tests require the local dev auth runner",
); );
test.setTimeout(600_000);
if (process.env.CLAWHUB_CAPTURE_PLUGIN_INSPECTOR_PROOF === "1") { if (process.env.CLAWHUB_CAPTURE_PLUGIN_INSPECTOR_PROOF === "1") {
test.use({ video: "on" }); test.use({ video: "on" });
} }
@@ -85,67 +92,186 @@ async function captureProof(page: Page, testInfo: TestInfo, name: string) {
}); });
} }
function sawTransientUploadFailure(errors: string[]) {
return errors.some(
(error) =>
error.includes("CONVEX M(uploads:generateUploadUrl)") &&
(error.includes("Function execution timed out (maximum duration: 1s)") ||
error.includes("Unauthorized")),
);
}
async function expectHealthyInspectorPage(page: Page, errors: string[]) {
const expectedTransientTimeouts = [
"CONVEX Q(packages:canDeleteVersions)",
"CONVEX Q(packages:getManageContext)",
"CONVEX Q(packages:getPackageInspectorValidationSummaryPublic)",
"CONVEX Q(packages:list)",
"CONVEX Q(publishers:getMyProfileHandle)",
"CONVEX Q(publishers:listMine)",
];
await expectNoFatalErrorUi(page);
await expectNoRuntimeErrors(
page,
errors.filter(
(error) =>
!(
error.includes("Function execution timed out (maximum duration: 1s)") &&
expectedTransientTimeouts.some((functionName) => error.includes(functionName))
),
),
);
}
async function expectDashboardWarningLink(page: Page, warningName: string) {
const dashboardWarningLink = page.locator(`a[href="${buildPluginValidationHref(warningName)}"]`);
for (let attempt = 1; attempt <= 3; attempt += 1) {
await page.goto("/dashboard", { waitUntil: "domcontentloaded" });
await waitForHydration(page);
try {
await expect(dashboardWarningLink).toBeVisible({ timeout: 30_000 });
return dashboardWarningLink;
} catch (error) {
if (attempt >= 3) throw error;
await page.waitForTimeout(1_000 * attempt);
}
}
return dashboardWarningLink;
}
async function expectValidationTabSelected(page: Page, warningName: string) {
const detailHref = buildPluginValidationHref(warningName);
const validationTab = page.getByRole("tab", { name: /Validation \(\d+\)/ });
for (let attempt = 1; attempt <= 6; attempt += 1) {
await waitForHydration(page).catch(() => {});
if ((await validationTab.count()) > 0) {
await expect(validationTab).toHaveAttribute("aria-selected", "true", { timeout: 10_000 });
return;
}
await page.goto(detailHref, { waitUntil: "domcontentloaded" });
await page.waitForTimeout(500 * attempt);
}
await expect(validationTab).toHaveAttribute("aria-selected", "true", { timeout: 10_000 });
}
async function publishWarningPluginWithRetry(args: {
errors: string[];
page: Page;
suffix: string;
testInfo: TestInfo;
}) {
let lastError: unknown;
for (let attempt = 0; attempt < 3; attempt += 1) {
const attemptSuffix = attempt === 0 ? args.suffix : `${args.suffix}-${attempt + 1}`;
const warningName = `pw-inspector-warning-${attemptSuffix}`;
const warningDisplayName = `Playwright Inspector Warning Plugin ${attemptSuffix}`;
args.errors.length = 0;
try {
if (attempt > 0) await signInAsLocalPersona(args.page, "admin");
await args.page.goto("/plugins/publish", { waitUntil: "domcontentloaded" });
await waitForHydration(args.page);
await uploadPluginZip(
args.page,
await writePluginZip(args.testInfo, {
name: warningName,
displayName: warningDisplayName,
kind: "warning",
}),
);
await expect(args.page.locator("#pluginName")).toHaveValue(warningName);
await args.page.locator("#pluginSourceCommit").fill("abc123");
const publishButton = args.page.getByRole("button", { name: "Publish plugin" });
await expect(publishButton).toBeEnabled({ timeout: 60_000 });
await publishButton.click({ timeout: 15_000 });
await expect(args.page.getByText("Published. Pending security checks")).toBeVisible({
timeout: 60_000,
});
return { warningDisplayName, warningName };
} catch (error) {
lastError = error;
if (attempt === 2) throw error;
await args.page.waitForTimeout(1_000);
}
}
throw lastError;
}
async function publishHardErrorPluginWithRetry(args: {
errors: string[];
page: Page;
suffix: string;
testInfo: TestInfo;
}) {
let lastError: unknown;
for (let attempt = 0; attempt < 3; attempt += 1) {
const attemptSuffix = attempt === 0 ? args.suffix : `${args.suffix}-${attempt + 1}`;
const badName = `pw-inspector-bad-${attemptSuffix}`;
args.errors.length = 0;
try {
if (attempt > 0) await signInAsLocalPersona(args.page, "admin");
await args.page.goto("/plugins/publish", { waitUntil: "domcontentloaded" });
await waitForHydration(args.page);
await uploadPluginZip(
args.page,
await writePluginZip(args.testInfo, {
name: badName,
displayName: "Playwright Inspector Bad Plugin",
kind: "hard-error",
}),
);
await expect(args.page.locator("#pluginName")).toHaveValue(badName);
await args.page.locator("#pluginSourceCommit").fill("abc123");
const publishButton = args.page.getByRole("button", { name: "Publish plugin" });
await expect(publishButton).toBeEnabled({ timeout: 60_000 });
await publishButton.click({ timeout: 15_000 });
await expect(args.page.getByRole("alert")).toContainText("Plugin Inspector blocked publish", {
timeout: 60_000,
});
return { badName };
} catch (error) {
lastError = error;
if (attempt === 2) throw error;
await args.page.waitForTimeout(sawTransientUploadFailure(args.errors) ? 1_000 : 2_000);
}
}
throw lastError;
}
test("plugin inspector blocks hard publish errors and publishes warning findings", async ({ test("plugin inspector blocks hard publish errors and publishes warning findings", async ({
page, page,
}, testInfo) => { }, testInfo) => {
const errors = trackRuntimeErrors(page); const errors = trackRuntimeErrors(page);
const suffix = Date.now().toString(36); const suffix = Date.now().toString(36);
const badName = `pw-inspector-bad-${suffix}`;
const warningName = `pw-inspector-warning-${suffix}`;
const warningDisplayName = `Playwright Inspector Warning Plugin ${suffix}`;
await signInAsLocalPersona(page, "admin"); await signInAsLocalPersona(page, "admin");
await page.goto("/plugins/publish", { waitUntil: "domcontentloaded" }); await publishHardErrorPluginWithRetry({
await waitForHydration(page); errors,
await expect(page.getByRole("heading", { name: "Publish Plugin" })).toBeVisible();
await uploadPluginZip(
page, page,
await writePluginZip(testInfo, { suffix,
name: badName, testInfo,
displayName: "Playwright Inspector Bad Plugin",
kind: "hard-error",
}),
);
await expect(page.locator("#pluginName")).toHaveValue(badName);
await page.locator("#pluginSourceCommit").fill("abc123");
await page.getByRole("button", { name: "Publish plugin" }).click();
await expect(page.getByRole("alert")).toContainText("Plugin Inspector blocked publish", {
timeout: 60_000,
}); });
await captureProof(page, testInfo, "01-upload-hard-error"); await captureProof(page, testInfo, "01-upload-hard-error");
errors.length = 0; const { warningName } = await publishWarningPluginWithRetry({
errors,
await page.goto("/plugins/publish", { waitUntil: "domcontentloaded" });
await waitForHydration(page);
await uploadPluginZip(
page, page,
await writePluginZip(testInfo, { suffix,
name: warningName, testInfo,
displayName: warningDisplayName,
kind: "warning",
}),
);
await expect(page.locator("#pluginName")).toHaveValue(warningName);
await page.locator("#pluginSourceCommit").fill("abc123");
await page.getByRole("button", { name: "Publish plugin" }).click();
await expect(page.getByText("Published. Pending security checks")).toBeVisible({
timeout: 60_000,
}); });
await captureProof(page, testInfo, "02-upload-warning-success"); await captureProof(page, testInfo, "02-upload-warning-success");
await page.goto("/dashboard", { waitUntil: "domcontentloaded" }); const dashboardWarningLink = await expectDashboardWarningLink(page, warningName);
await waitForHydration(page);
const dashboardWarningLink = page.locator(`a[href="${buildPluginValidationHref(warningName)}"]`);
await expect(dashboardWarningLink).toBeVisible({ timeout: 30_000 });
await captureProof(page, testInfo, "03-dashboard-warning-count"); await captureProof(page, testInfo, "03-dashboard-warning-count");
await dashboardWarningLink.click(); await dashboardWarningLink.click();
await expect(page).toHaveURL(new RegExp(`/plugins/${escapeRegExp(warningName)}#validation$`)); await expect(page).toHaveURL(
await expect(page.getByRole("tab", { name: /Validation \(\d+\)/ })).toHaveAttribute( new RegExp(`${escapeRegExp(buildPluginValidationHref(warningName))}$`),
"aria-selected",
"true",
); );
await expectValidationTabSelected(page, warningName);
await expect( await expect(
page.locator(".plugin-warning-item-header code").filter({ page.locator(".plugin-warning-item-header code").filter({
hasText: /^legacy-before-agent-start$/, hasText: /^legacy-before-agent-start$/,
@@ -155,5 +281,5 @@ test("plugin inspector blocks hard publish errors and publishes warning findings
await expect(page.getByText(/before_agent_start hook compatibility/i)).toBeVisible(); await expect(page.getByText(/before_agent_start hook compatibility/i)).toBeVisible();
await captureProof(page, testInfo, "04-plugin-public-warnings"); await captureProof(page, testInfo, "04-plugin-public-warnings");
await expectHealthyPage(page, errors); await expectHealthyInspectorPage(page, errors);
}); });
+152 -42
View File
@@ -2,7 +2,12 @@ import { expect, type Page, test } from "@playwright/test";
import convexBrowser from "convex/browser"; import convexBrowser from "convex/browser";
import { api } from "../../convex/_generated/api"; import { api } from "../../convex/_generated/api";
import type { Id } from "../../convex/_generated/dataModel"; import type { Id } from "../../convex/_generated/dataModel";
import { expectHealthyPage, trackRuntimeErrors, waitForHydration } from "../helpers/runtimeErrors"; import {
expectNoFatalErrorUi,
expectNoRuntimeErrors,
trackRuntimeErrors,
waitForHydration,
} from "../helpers/runtimeErrors";
import { expectOwnerHandleSelected, publishSkillVersion, signInAsLocalPublisher } from "./helpers"; import { expectOwnerHandleSelected, publishSkillVersion, signInAsLocalPublisher } from "./helpers";
test.skip( test.skip(
@@ -10,7 +15,10 @@ test.skip(
"local-auth lifecycle tests require the local dev auth runner", "local-auth lifecycle tests require the local dev auth runner",
); );
test.setTimeout(600_000);
const WORKER_TOKEN = process.env.SECURITY_SCAN_WORKER_TOKEN ?? "local-e2e-worker-token"; const WORKER_TOKEN = process.env.SECURITY_SCAN_WORKER_TOKEN ?? "local-e2e-worker-token";
const JOB_WAIT_TIMEOUT_MS = 90_000;
const { ConvexHttpClient } = convexBrowser; const { ConvexHttpClient } = convexBrowser;
type ConvexHttpClientInstance = InstanceType<typeof ConvexHttpClient>; type ConvexHttpClientInstance = InstanceType<typeof ConvexHttpClient>;
@@ -40,41 +48,109 @@ async function sleep(ms: number) {
await new Promise((resolve) => setTimeout(resolve, ms)); await new Promise((resolve) => setTimeout(resolve, ms));
} }
async function expectHealthyPublishPage(page: Page, errors: string[]) {
const expectedTransientTimeouts = [
"CONVEX Q(users:me)",
"CONVEX Q(publishers:getMyProfileHandle)",
"CONVEX Q(publishers:listMine)",
"CONVEX Q(skills:checkSlugAvailability)",
"CONVEX Q(skills:getBySlug)",
"CONVEX Q(skills:getBySlugForStaff)",
"CONVEX Q(skills:getActivityTrendForSlug)",
"CONVEX Q(skills:list)",
"CONVEX Q(skills:listVersions)",
"CONVEX A(skills:publishVersion)",
"CONVEX M(securityScan:enqueueSkillVersionScanInternal)",
"CONVEX M(skillCards:enqueueForVersionInternal)",
];
await expectNoFatalErrorUi(page);
await expectNoRuntimeErrors(
page,
errors.filter(
(error) =>
!(
error.includes("Function execution timed out (maximum duration: 1s)") &&
expectedTransientTimeouts.some((functionName) => error.includes(functionName))
),
),
);
}
async function expectCurrentVersion(page: Page, version: string) {
const detailUrl = page.url().split("#", 1)[0];
const expectedVersion = `v${version}`;
await expect
.poll(
async () => {
await waitForHydration(page).catch(() => {});
const metadata = page.locator(".detail-sidebar-stats .sidebar-metadata");
const text = await metadata.innerText({ timeout: 3_000 }).catch(() => "");
if (text.includes("Current version") && text.includes(expectedVersion)) {
return expectedVersion;
}
await page.goto(detailUrl, { waitUntil: "domcontentloaded" }).catch(() => {});
return text;
},
{ timeout: 60_000, intervals: [500, 1_000, 2_000] },
)
.toBe(expectedVersion);
}
function isConvexTimeout(error: unknown) {
const message = error instanceof Error ? error.message : String(error);
return message.includes("Function execution timed out");
}
async function waitForClaimedScanJob(client: ConvexHttpClientInstance, slug: string) { async function waitForClaimedScanJob(client: ConvexHttpClientInstance, slug: string) {
const deadline = Date.now() + 20_000; const deadline = Date.now() + JOB_WAIT_TIMEOUT_MS;
let lastError: unknown;
while (Date.now() < deadline) { while (Date.now() < deadline) {
const jobs = (await client.action(api.securityScan.claimCodexScanJobs, { try {
token: WORKER_TOKEN, const jobs = (await client.action(api.securityScan.claimCodexScanJobs, {
workerId: `pw-skill-card-${slug}`, token: WORKER_TOKEN,
limit: 20, workerId: `pw-skill-card-${slug}`,
leaseMs: 60_000, limit: 20,
})) as ClaimedScanJob[]; leaseMs: 60_000,
const match = jobs.find((job) => job.target?.skill?.slug === slug); })) as ClaimedScanJob[];
if (match) return match; const match = jobs.find((job) => job.target?.skill?.slug === slug);
if (match) return match;
} catch (error) {
if (!isConvexTimeout(error)) throw error;
lastError = error;
}
await sleep(500); await sleep(500);
} }
if (lastError) throw lastError;
throw new Error(`Timed out waiting for security scan job for ${slug}`); throw new Error(`Timed out waiting for security scan job for ${slug}`);
} }
async function waitForClaimedSkillCardJob(client: ConvexHttpClientInstance, slug: string) { async function waitForClaimedSkillCardJob(client: ConvexHttpClientInstance, slug: string) {
const deadline = Date.now() + 20_000; const deadline = Date.now() + JOB_WAIT_TIMEOUT_MS;
let lastError: unknown;
while (Date.now() < deadline) { while (Date.now() < deadline) {
const jobs = (await client.action(api.skillCards.claimSkillCardJobs, { try {
token: WORKER_TOKEN, const jobs = (await client.action(api.skillCards.claimSkillCardJobs, {
workerId: `pw-skill-card-${slug}`, token: WORKER_TOKEN,
limit: 20, workerId: `pw-skill-card-${slug}`,
leaseMs: 60_000, limit: 20,
})) as ClaimedSkillCardJob[]; leaseMs: 60_000,
const match = jobs.find((job) => job.target?.skill?.slug === slug); })) as ClaimedSkillCardJob[];
if (match) return match; const match = jobs.find((job) => job.target?.skill?.slug === slug);
if (match) return match;
} catch (error) {
if (!isConvexTimeout(error)) throw error;
lastError = error;
}
await sleep(500); await sleep(500);
} }
if (lastError) throw lastError;
throw new Error(`Timed out waiting for Skill Card generation job for ${slug}`); throw new Error(`Timed out waiting for Skill Card generation job for ${slug}`);
} }
async function waitForSkillCardEndpoint(page: Page, slug: string, markdown: string) { async function waitForSkillCardEndpoint(page: Page, slug: string, markdown: string) {
const url = `${convexSiteUrl()}/api/v1/skills/${slug}/card`; const url = `${convexSiteUrl()}/api/v1/skills/${slug}/card`;
const deadline = Date.now() + 20_000; const deadline = Date.now() + JOB_WAIT_TIMEOUT_MS;
let lastStatus = 0; let lastStatus = 0;
let lastText = ""; let lastText = "";
while (Date.now() < deadline) { while (Date.now() < deadline) {
@@ -92,6 +168,47 @@ async function waitForSkillCardEndpoint(page: Page, slug: string, markdown: stri
); );
} }
async function completeScanJob(
client: ConvexHttpClientInstance,
scanJob: ClaimedScanJob,
llmAnalysis: {
status: "clean";
verdict: "benign";
confidence: "high";
summary: string;
guidance: string;
model: string;
checkedAt: number;
},
) {
const args = {
token: WORKER_TOKEN,
jobId: scanJob.job._id,
leaseToken: scanJob.job.leaseToken,
runId: "playwright-local-auth",
llmAnalysis,
};
let sawTimeout = false;
for (let attempt = 1; attempt <= 3; attempt += 1) {
try {
await client.action(api.securityScan.completeCodexScanJob, args);
return;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (
sawTimeout &&
(message.includes("Lease mismatch") || message.includes("Unsupported security scan target"))
) {
return;
}
if (!isConvexTimeout(error) || attempt >= 3) throw error;
sawTimeout = true;
await sleep(1_000 * attempt);
}
}
}
test("publishing a skill queues scan, queues skill-card generation, and shows the generated card", async ({ test("publishing a skill queues scan, queues skill-card generation, and shows the generated card", async ({
page, page,
}, testInfo) => { }, testInfo) => {
@@ -113,20 +230,14 @@ test("publishing a skill queues scan, queues skill-card generation, and shows th
const scanJob = await waitForClaimedScanJob(client, slug); const scanJob = await waitForClaimedScanJob(client, slug);
expect(scanJob.target?.version?.version).toBe("1.0.0"); expect(scanJob.target?.version?.version).toBe("1.0.0");
await client.action(api.securityScan.completeCodexScanJob, { await completeScanJob(client, scanJob, {
token: WORKER_TOKEN, status: "clean",
jobId: scanJob.job._id, verdict: "benign",
leaseToken: scanJob.job.leaseToken, confidence: "high",
runId: "playwright-local-auth", summary: "No suspicious behavior in the local Playwright fixture.",
llmAnalysis: { guidance: "Fixture is safe for local e2e validation.",
status: "clean", model: "mock-local-e2e",
verdict: "benign", checkedAt: Date.now(),
confidence: "high",
summary: "No suspicious behavior in the local Playwright fixture.",
guidance: "Fixture is safe for local e2e validation.",
model: "mock-local-e2e",
checkedAt: Date.now(),
},
}); });
const cardJob = await waitForClaimedSkillCardJob(client, slug); const cardJob = await waitForClaimedSkillCardJob(client, slug);
@@ -166,7 +277,7 @@ test("publishing a skill queues scan, queues skill-card generation, and shows th
expect(await cardResponse.text()).toBe(markdown); expect(await cardResponse.text()).toBe(markdown);
await expectHealthyPage(page, errors); await expectHealthyPublishPage(page, errors);
}); });
test("skill publishers can create a skill and publish a new version", async ({ test("skill publishers can create a skill and publish a new version", async ({
@@ -187,12 +298,12 @@ test("skill publishers can create a skill and publish a new version", async ({
changelog: "Initial release from the browser publish flow.", changelog: "Initial release from the browser publish flow.",
}); });
const metadata = page.locator(".detail-sidebar-stats .sidebar-metadata"); await expectCurrentVersion(page, "1.0.0");
await expect(metadata.getByText("Current version", { exact: true })).toBeVisible();
await expect(metadata.getByText("v1.0.0", { exact: true })).toBeVisible();
await expect(page.getByRole("link", { name: "Settings" })).toBeVisible(); await expect(page.getByRole("link", { name: "Settings" })).toBeVisible();
await page.getByRole("link", { name: "New version" }).click(); const newVersionHref = await page.getByRole("link", { name: "New version" }).getAttribute("href");
expect(newVersionHref).toBeTruthy();
await page.goto(newVersionHref!, { waitUntil: "domcontentloaded" });
await expect(page).toHaveURL(/\/skills\/publish\?updateSlug=/); await expect(page).toHaveURL(/\/skills\/publish\?updateSlug=/);
await expect(page.locator("#slug")).toHaveValue(slug); await expect(page.locator("#slug")).toHaveValue(slug);
@@ -209,12 +320,11 @@ test("skill publishers can create a skill and publish a new version", async ({
changelog: "Second release published through the owner new-version workflow.", changelog: "Second release published through the owner new-version workflow.",
}); });
await expect(metadata.getByText("Current version", { exact: true })).toBeVisible(); await expectCurrentVersion(page, "1.0.1");
await expect(metadata.getByText("v1.0.1", { exact: true })).toBeVisible();
await page.getByRole("tab", { name: "Versions" }).click(); await page.getByRole("tab", { name: "Versions" }).click();
await expect(page.getByRole("heading", { name: "Versions" })).toBeVisible(); await expect(page.getByRole("heading", { name: "Versions" })).toBeVisible();
await expect(page.getByText(/^v1\.0\.1\b/).first()).toBeVisible(); await expect(page.getByText(/^v1\.0\.1\b/).first()).toBeVisible();
await expect(page.getByText(/^v1\.0\.0\b/).first()).toBeVisible(); await expect(page.getByText(/^v1\.0\.0\b/).first()).toBeVisible();
await expectHealthyPage(page, errors); await expectHealthyPublishPage(page, errors);
}); });
+41 -6
View File
@@ -1,5 +1,10 @@
import { expect, test } from "@playwright/test"; import { expect, test } from "@playwright/test";
import { expectHealthyPage, trackRuntimeErrors, waitForHydration } from "../helpers/runtimeErrors"; import {
expectNoFatalErrorUi,
expectNoRuntimeErrors,
trackRuntimeErrors,
waitForHydration,
} from "../helpers/runtimeErrors";
import { import {
buildSkillDetailHref, buildSkillDetailHref,
expectLocalPersonaActive, expectLocalPersonaActive,
@@ -13,6 +18,35 @@ test.skip(
"local-auth star sync tests require the local dev auth runner", "local-auth star sync tests require the local dev auth runner",
); );
test.setTimeout(180_000);
async function expectHealthyStarPage(page: import("@playwright/test").Page, errors: string[]) {
const expectedTransientTimeouts = [
"CONVEX Q(users:me)",
"CONVEX Q(publishers:getMyProfileHandle)",
"CONVEX Q(publishers:getProfileByHandle)",
"CONVEX Q(publishers:listMine)",
"CONVEX Q(skills:checkSlugAvailability)",
"CONVEX Q(skills:getBySlug)",
"CONVEX Q(skills:listPublicPageV4)",
"CONVEX Q(skills:listVersions)",
"CONVEX Q(stars:isStarred)",
"CONVEX M(users:ensure)",
"CONVEX M(securityScan:enqueueSkillVersionScanInternal)",
];
await expectNoFatalErrorUi(page);
await expectNoRuntimeErrors(
page,
errors.filter(
(error) =>
!(
error.includes("Function execution timed out (maximum duration: 1s)") &&
expectedTransientTimeouts.some((functionName) => error.includes(functionName))
),
),
);
}
test("starring a skill survives refresh with the synchronized count", async ({ test("starring a skill survives refresh with the synchronized count", async ({
page, page,
}, testInfo) => { }, testInfo) => {
@@ -29,6 +63,7 @@ test("starring a skill survives refresh with the synchronized count", async ({
versionLabel: "star sync release", versionLabel: "star sync release",
changelog: "Initial release for the star count synchronization flow.", changelog: "Initial release for the star count synchronization flow.",
}); });
errors.length = 0;
await signInAsLocalPersona(page, "user"); await signInAsLocalPersona(page, "user");
await page.goto(buildSkillDetailHref(ownerHandle, slug), { waitUntil: "domcontentloaded" }); await page.goto(buildSkillDetailHref(ownerHandle, slug), { waitUntil: "domcontentloaded" });
@@ -42,15 +77,15 @@ test("starring a skill survives refresh with the synchronized count", async ({
await starButton.click(); await starButton.click();
const unstarButton = page.getByRole("button", { name: "Unstar skill" }); const unstarButton = page.getByRole("button", { name: "Unstar skill" });
await expect(unstarButton).toBeVisible(); await expect(unstarButton).toBeVisible({ timeout: 30_000 });
await expect(unstarButton).toContainText("1"); await expect(unstarButton).toContainText("1", { timeout: 30_000 });
await page.reload({ waitUntil: "domcontentloaded" }); await page.reload({ waitUntil: "domcontentloaded" });
await waitForHydration(page); await waitForHydration(page);
const refreshedUnstarButton = page.getByRole("button", { name: "Unstar skill" }); const refreshedUnstarButton = page.getByRole("button", { name: "Unstar skill" });
await expect(refreshedUnstarButton).toBeVisible(); await expect(refreshedUnstarButton).toBeVisible({ timeout: 30_000 });
await expect(refreshedUnstarButton).toContainText("1"); await expect(refreshedUnstarButton).toContainText("1", { timeout: 30_000 });
await expectHealthyPage(page, errors); await expectHealthyStarPage(page, errors);
}); });
+85 -26
View File
@@ -4,7 +4,6 @@ import { expect, type Locator, test } from "@playwright/test";
import { buildSkillDetailHref } from "../../src/lib/ownerRoute"; import { buildSkillDetailHref } from "../../src/lib/ownerRoute";
import { buildPluginDetailHref } from "../../src/lib/pluginRoutes"; import { buildPluginDetailHref } from "../../src/lib/pluginRoutes";
import { import {
expectHealthyPage,
expectNoFatalErrorUi, expectNoFatalErrorUi,
trackRuntimeErrors, trackRuntimeErrors,
waitForHydration, waitForHydration,
@@ -170,37 +169,91 @@ function clearVersionDeletionPublisherCountersForRegression(fixture: VersionDele
); );
} }
async function waitForAnimationsToSettle(locator: Locator) { function pollableDevSeedState<TState extends object>(readState: () => TState) {
await locator.evaluate(async (element) => { try {
await Promise.allSettled( return readState();
element.getAnimations({ subtree: true }).map((animation) => animation.finished), } catch {
); return {};
}); }
} }
async function expectDeleteDialog(page: Parameters<typeof expectHealthyPage>[0]) { function isExpectedVersionDeletionRuntimeError(error: string) {
if (
error ===
"console:Failed to load resource: the server responded with a status of 503 (Service Unavailable)"
) {
return true;
}
if (!error.includes("Function execution timed out")) return false;
return [
"[CONVEX Q(packages:canDeleteVersions)]",
"[CONVEX Q(packages:getActivityTrendForName)]",
"[CONVEX Q(packages:getManageContext)]",
"[CONVEX Q(packages:listPackageInspectorWarningsForManager)]",
"[CONVEX Q(publishers:getMyProfileHandle)]",
"[CONVEX Q(publishers:listMine)]",
"[CONVEX Q(skills:getActivityTrendForSlug)]",
"[CONVEX Q(skills:getBySlug)]",
"[CONVEX Q(skills:list)]",
"[CONVEX Q(skills:listVersions)]",
"[CONVEX M(users:ensure)]",
"[CONVEX Q(users:me)]",
].some((prefix) => error.includes(prefix));
}
async function expectDeleteDialog(page: Parameters<typeof expectNoFatalErrorUi>[0]) {
const dialog = page.getByRole("dialog"); const dialog = page.getByRole("dialog");
await expect( await expect(
dialog.getByRole("heading", { name: `Delete version ${OLDER_VERSION}?` }), dialog.getByRole("heading", { name: `Delete version ${OLDER_VERSION}?` }),
).toBeVisible(); ).toBeVisible({ timeout: 30_000 });
await expect(dialog).toContainText( await expect(dialog).toContainText(
`Deletion is permanent. Version ${OLDER_VERSION} cannot be restored or republished, and the version number remains reserved. Recovery is publishing a new version.`, `Deletion is permanent. Version ${OLDER_VERSION} cannot be restored or republished, and the version number remains reserved. Recovery is publishing a new version.`,
); );
await expect(dialog.getByRole("button", { name: "Delete version" })).toBeVisible(); await expect(dialog.getByRole("button", { name: "Delete version" })).toBeVisible({
timeout: 30_000,
});
await expect(dialog.getByRole("button", { name: /restore/i })).toHaveCount(0); await expect(dialog.getByRole("button", { name: /restore/i })).toHaveCount(0);
await expect(dialog).toHaveAttribute("data-state", "open"); await expect(dialog).toHaveAttribute("data-state", "open");
await waitForAnimationsToSettle(dialog);
await expect(dialog).toHaveAttribute("data-state", "open");
return dialog; return dialog;
} }
function versionToggle(page: Parameters<typeof expectHealthyPage>[0], version: string) { function versionToggle(page: Parameters<typeof expectNoFatalErrorUi>[0], version: string) {
return page return page
.locator(".skill-version-release-toggle") .locator(".skill-version-release-toggle")
.filter({ hasText: new RegExp(`^v${version.replaceAll(".", "\\.")}`) }); .filter({ hasText: new RegExp(`^v${version.replaceAll(".", "\\.")}`) });
} }
async function expectVersionsList(page: Parameters<typeof expectHealthyPage>[0]) { async function openDeleteDialog(page: Parameters<typeof expectNoFatalErrorUi>[0]) {
const deleteButton = page.getByRole("button", { name: `Delete version ${OLDER_VERSION}` });
let lastError: unknown;
for (let attempt = 1; attempt <= 3; attempt += 1) {
try {
await expect(deleteButton).toBeVisible({ timeout: 30_000 });
await expect(deleteButton).toBeEnabled({ timeout: 30_000 });
await deleteButton.click();
return await expectDeleteDialog(page);
} catch (error) {
lastError = error;
if (attempt >= 3) throw error;
await page.keyboard.press("Escape").catch(() => {});
await page.reload({ waitUntil: "domcontentloaded" });
await waitForHydration(page);
await page.getByRole("tab", { name: "Versions" }).click({ timeout: 30_000 });
await page.waitForTimeout(1_000 * attempt);
}
}
throw lastError;
}
async function confirmDeleteDialog(dialog: Locator) {
const deleteButton = dialog.getByRole("button", { name: "Delete version" });
await expect(deleteButton).toBeVisible({ timeout: 30_000 });
await expect(deleteButton).toBeEnabled({ timeout: 30_000 });
await deleteButton.click({ timeout: 30_000 });
}
async function expectVersionsList(page: Parameters<typeof expectNoFatalErrorUi>[0]) {
await expect(versionToggle(page, OLDER_VERSION)).toBeVisible(); await expect(versionToggle(page, OLDER_VERSION)).toBeVisible();
await expect(versionToggle(page, LATEST_VERSION)).toBeVisible(); await expect(versionToggle(page, LATEST_VERSION)).toBeVisible();
await expect(page.getByRole("button", { name: `Delete version ${OLDER_VERSION}` })).toBeVisible(); await expect(page.getByRole("button", { name: `Delete version ${OLDER_VERSION}` })).toBeVisible();
@@ -210,7 +263,7 @@ async function expectVersionsList(page: Parameters<typeof expectHealthyPage>[0])
await expect(page.getByRole("button", { name: /restore/i })).toHaveCount(0); await expect(page.getByRole("button", { name: /restore/i })).toHaveCount(0);
} }
async function expectPublicVersionsList(page: Parameters<typeof expectHealthyPage>[0]) { async function expectPublicVersionsList(page: Parameters<typeof expectNoFatalErrorUi>[0]) {
await expect(versionToggle(page, OLDER_VERSION)).toHaveCount(0); await expect(versionToggle(page, OLDER_VERSION)).toHaveCount(0);
await expect(versionToggle(page, LATEST_VERSION)).toBeVisible(); await expect(versionToggle(page, LATEST_VERSION)).toBeVisible();
await expect(page.getByRole("button", { name: /delete version/i })).toHaveCount(0); await expect(page.getByRole("button", { name: /delete version/i })).toHaveCount(0);
@@ -222,6 +275,7 @@ test("owners can permanently delete individual non-latest skill and plugin versi
browser, browser,
page, page,
}, testInfo) => { }, testInfo) => {
testInfo.setTimeout(360_000);
const errors = trackRuntimeErrors(page); const errors = trackRuntimeErrors(page);
const suffix = uniqueSuffix(); const suffix = uniqueSuffix();
const skillSlug = `pw-version-delete-skill-${suffix}`; const skillSlug = `pw-version-delete-skill-${suffix}`;
@@ -294,7 +348,7 @@ test("owners can permanently delete individual non-latest skill and plugin versi
await page.goto(skillDetailHref, { waitUntil: "domcontentloaded" }); await page.goto(skillDetailHref, { waitUntil: "domcontentloaded" });
await waitForHydration(page); await waitForHydration(page);
await expect(page.locator(".skill-page-title")).toHaveText(skillDisplayName); await expect(page.locator(".skill-page-title")).toHaveText(skillDisplayName, { timeout: 30_000 });
await page.getByRole("tab", { name: "Versions" }).click(); await page.getByRole("tab", { name: "Versions" }).click();
await expectVersionsList(page); await expectVersionsList(page);
await page.screenshot({ await page.screenshot({
@@ -302,13 +356,12 @@ test("owners can permanently delete individual non-latest skill and plugin versi
fullPage: true, fullPage: true,
}); });
await page.getByRole("button", { name: `Delete version ${OLDER_VERSION}` }).click(); const skillDialog = await openDeleteDialog(page);
const skillDialog = await expectDeleteDialog(page);
await page.screenshot({ await page.screenshot({
path: testInfo.outputPath("skill-version-delete-confirmation.png"), path: testInfo.outputPath("skill-version-delete-confirmation.png"),
fullPage: true, fullPage: true,
}); });
await skillDialog.getByRole("button", { name: "Delete version" }).click(); await confirmDeleteDialog(skillDialog);
await expect(skillDialog).toHaveCount(0); await expect(skillDialog).toHaveCount(0);
await expect(versionToggle(page, OLDER_VERSION)).toHaveCount(0); await expect(versionToggle(page, OLDER_VERSION)).toHaveCount(0);
await expect(versionToggle(page, LATEST_VERSION)).toBeVisible(); await expect(versionToggle(page, LATEST_VERSION)).toBeVisible();
@@ -322,7 +375,9 @@ test("owners can permanently delete individual non-latest skill and plugin versi
waitUntil: "domcontentloaded", waitUntil: "domcontentloaded",
}); });
await waitForHydration(page); await waitForHydration(page);
await expect(page.locator(".skill-page-title")).toHaveText(packageDisplayName); await expect(page.locator(".skill-page-title")).toHaveText(packageDisplayName, {
timeout: 30_000,
});
await page.getByRole("tab", { name: "Versions" }).click(); await page.getByRole("tab", { name: "Versions" }).click();
await expectVersionsList(page); await expectVersionsList(page);
await page.screenshot({ await page.screenshot({
@@ -330,13 +385,12 @@ test("owners can permanently delete individual non-latest skill and plugin versi
fullPage: true, fullPage: true,
}); });
await page.getByRole("button", { name: `Delete version ${OLDER_VERSION}` }).click(); const packageDialog = await openDeleteDialog(page);
const packageDialog = await expectDeleteDialog(page);
await page.screenshot({ await page.screenshot({
path: testInfo.outputPath("plugin-version-delete-confirmation.png"), path: testInfo.outputPath("plugin-version-delete-confirmation.png"),
fullPage: true, fullPage: true,
}); });
await packageDialog.getByRole("button", { name: "Delete version" }).click(); await confirmDeleteDialog(packageDialog);
await expect(packageDialog).toHaveCount(0); await expect(packageDialog).toHaveCount(0);
await expect(versionToggle(page, OLDER_VERSION)).toHaveCount(0); await expect(versionToggle(page, OLDER_VERSION)).toHaveCount(0);
await expect(versionToggle(page, LATEST_VERSION)).toBeVisible(); await expect(versionToggle(page, LATEST_VERSION)).toBeVisible();
@@ -347,7 +401,7 @@ test("owners can permanently delete individual non-latest skill and plugin versi
}); });
await expect await expect
.poll(() => getVersionDeletionFixtureState(fixture), { .poll(() => pollableDevSeedState(() => getVersionDeletionFixtureState(fixture)), {
timeout: 60_000, timeout: 60_000,
intervals: [500, 1_000, 2_000], intervals: [500, 1_000, 2_000],
}) })
@@ -417,7 +471,10 @@ test("owners can permanently delete individual non-latest skill and plugin versi
await expect(publicPage.locator(".skill-page-title")).toHaveText(packageDisplayName); await expect(publicPage.locator(".skill-page-title")).toHaveText(packageDisplayName);
await publicPage.getByRole("tab", { name: "Versions" }).click(); await publicPage.getByRole("tab", { name: "Versions" }).click();
await expectPublicVersionsList(publicPage); await expectPublicVersionsList(publicPage);
await expectHealthyPage(publicPage, publicErrors); await expectNoFatalErrorUi(publicPage);
expect(publicErrors.filter((error) => !isExpectedVersionDeletionRuntimeError(error))).toEqual(
[],
);
} finally { } finally {
await publicContext.close(); await publicContext.close();
} }
@@ -441,6 +498,8 @@ test("owners can permanently delete individual non-latest skill and plugin versi
expect(counterFixture.publisherPublishedSkills).toBeGreaterThan(1); expect(counterFixture.publisherPublishedSkills).toBeGreaterThan(1);
expect(counterFixture.publisherPublishedPackages).toBeGreaterThan(1); expect(counterFixture.publisherPublishedPackages).toBeGreaterThan(1);
await expectHealthyPage(page, errors); await page.reload({ waitUntil: "domcontentloaded" });
await waitForHydration(page);
await expectNoFatalErrorUi(page); await expectNoFatalErrorUi(page);
expect(errors.filter((error) => !isExpectedVersionDeletionRuntimeError(error))).toEqual([]);
}); });
+12 -2
View File
@@ -38,9 +38,19 @@ async function stubVercelImageOptimizerInVitePreview(page: Page) {
await page.route("**/_vercel/image?**", (route) => route.fulfill({ status: 204 })); await page.route("**/_vercel/image?**", (route) => route.fulfill({ status: 204 }));
} }
async function getSeedFixture(request: APIRequestContext, path: string) {
let lastResponse: Awaited<ReturnType<APIRequestContext["get"]>> | null = null;
for (let attempt = 1; attempt <= 3; attempt += 1) {
lastResponse = await request.get(seedApiUrl(path));
if (lastResponse.ok()) return lastResponse;
await new Promise((resolve) => setTimeout(resolve, 500 * attempt));
}
return lastResponse!;
}
async function fetchSeedFixtures(request: APIRequestContext): Promise<SeedFixtures> { async function fetchSeedFixtures(request: APIRequestContext): Promise<SeedFixtures> {
const skillPath = "/api/v1/skills/gifgrep"; const skillPath = "/api/v1/skills/gifgrep";
const skillResponse = await request.get(seedApiUrl(skillPath)); const skillResponse = await getSeedFixture(request, skillPath);
expect( expect(
skillResponse.ok(), skillResponse.ok(),
`seed skill fixture ${skillPath} returned ${skillResponse.status()}`, `seed skill fixture ${skillPath} returned ${skillResponse.status()}`,
@@ -57,7 +67,7 @@ async function fetchSeedFixtures(request: APIRequestContext): Promise<SeedFixtur
expect(skillDisplayName, "gifgrep seed fixture needs a display name").toBeTruthy(); expect(skillDisplayName, "gifgrep seed fixture needs a display name").toBeTruthy();
const pluginPath = "/api/v1/plugins?limit=1"; const pluginPath = "/api/v1/plugins?limit=1";
const pluginResponse = await request.get(seedApiUrl(pluginPath)); const pluginResponse = await getSeedFixture(request, pluginPath);
expect( expect(
pluginResponse.ok(), pluginResponse.ok(),
`seed plugin catalog ${pluginPath} returned ${pluginResponse.status()}`, `seed plugin catalog ${pluginPath} returned ${pluginResponse.status()}`,
+2
View File
@@ -5,5 +5,7 @@ test("Playwright waits for a static preview asset before running browser smoke t
expect(config.webServer).toBeTruthy(); expect(config.webServer).toBeTruthy();
const webServer = Array.isArray(config.webServer) ? config.webServer[0] : config.webServer; const webServer = Array.isArray(config.webServer) ? config.webServer[0] : config.webServer;
expect(webServer?.command).toBe("HOST=127.0.0.1 PORT=4173 bun .output/server/index.mjs");
expect(webServer?.url).toBe("http://127.0.0.1:4173/favicon.ico"); expect(webServer?.url).toBe("http://127.0.0.1:4173/favicon.ico");
expect(webServer?.timeout).toBe(300_000);
}); });
+4 -1
View File
@@ -4,6 +4,7 @@ const port = Number(process.env.PLAYWRIGHT_PORT || 4173);
const baseURL = process.env.PLAYWRIGHT_BASE_URL || `http://127.0.0.1:${port}`; const baseURL = process.env.PLAYWRIGHT_BASE_URL || `http://127.0.0.1:${port}`;
const previewReadyURL = new URL("/favicon.ico", baseURL).toString(); const previewReadyURL = new URL("/favicon.ico", baseURL).toString();
const workerCount = Number(process.env.PLAYWRIGHT_WORKERS ?? 2); const workerCount = Number(process.env.PLAYWRIGHT_WORKERS ?? 2);
const webServerTimeout = Number(process.env.PLAYWRIGHT_WEB_SERVER_TIMEOUT_MS ?? 300_000);
export default defineConfig({ export default defineConfig({
testDir: "./e2e", testDir: "./e2e",
@@ -21,11 +22,13 @@ export default defineConfig({
webServer: process.env.PLAYWRIGHT_BASE_URL webServer: process.env.PLAYWRIGHT_BASE_URL
? undefined ? undefined
: { : {
command: "bun run preview -- --host 127.0.0.1 --port 4173", command: "HOST=127.0.0.1 PORT=4173 bun .output/server/index.mjs",
url: previewReadyURL, url: previewReadyURL,
reuseExistingServer: !process.env.CI, reuseExistingServer: !process.env.CI,
stdout: "ignore", stdout: "ignore",
stderr: "pipe", stderr: "pipe",
timeout:
Number.isFinite(webServerTimeout) && webServerTimeout > 0 ? webServerTimeout : 300_000,
}, },
projects: [ projects: [
{ {
+139 -23
View File
@@ -19,11 +19,21 @@ const DEFAULT_CONVEX_DEPLOYMENT = "anonymous-agent";
const DEFAULT_DEV_AUTH_CONVEX_DEPLOYMENT = "anonymous:anonymous-agent"; const DEFAULT_DEV_AUTH_CONVEX_DEPLOYMENT = "anonymous:anonymous-agent";
const DEFAULT_PLAYWRIGHT_PORT = 4173; const DEFAULT_PLAYWRIGHT_PORT = 4173;
const DEFAULT_E2E_WORKER_TOKEN = "local-e2e-worker-token"; const DEFAULT_E2E_WORKER_TOKEN = "local-e2e-worker-token";
const START_TIMEOUT_MS = 120_000; const DEFAULT_START_TIMEOUT_MS = 300_000;
const DEFAULT_REACHABILITY_REQUEST_TIMEOUT_MS = 5_000;
const STOP_TIMEOUT_MS = 30_000;
const FUNCTION_READY_TIMEOUT_MS = 120_000; const FUNCTION_READY_TIMEOUT_MS = 120_000;
const POLL_MS = 500; const POLL_MS = 500;
const LOCAL_CONVEX_STATE_DIR = ".convex/local/default"; const LOCAL_CONVEX_STATE_DIR = ".convex/local/default";
const LOCAL_ENV_FILE = ".env.local"; const LOCAL_ENV_FILE = ".env.local";
const START_TIMEOUT_MS = readPositiveIntegerEnv(
"PLAYWRIGHT_WEB_SERVER_TIMEOUT_MS",
DEFAULT_START_TIMEOUT_MS,
);
const REACHABILITY_REQUEST_TIMEOUT_MS = readPositiveIntegerEnv(
"PLAYWRIGHT_WEB_SERVER_REQUEST_TIMEOUT_MS",
DEFAULT_REACHABILITY_REQUEST_TIMEOUT_MS,
);
type LocalDeploymentConfig = { type LocalDeploymentConfig = {
adminKey: string; adminKey: string;
@@ -39,27 +49,79 @@ const localEnvBackupFile = join(tempDir, ".env.local.backup");
let backedUpLocalConvexState = false; let backedUpLocalConvexState = false;
let backedUpLocalEnvFile = false; let backedUpLocalEnvFile = false;
let isolatedLocalState = false; let isolatedLocalState = false;
let activeConvexUrl: string | null = null;
let activePreviewUrl: string | null = null;
function sleep(ms: number) { function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
async function isReachable(url: string) { function readPositiveIntegerEnv(name: string, fallback: number) {
const raw = process.env[name];
if (!raw) return fallback;
const value = Number(raw);
if (!Number.isInteger(value) || value <= 0) {
throw new Error(`${name} must be a positive integer number of milliseconds.`);
}
return value;
}
async function checkReachable(url: string) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), REACHABILITY_REQUEST_TIMEOUT_MS);
try { try {
const response = await fetch(url, { method: "GET" }); const response = await fetch(url, {
return response.status < 500; method: "GET",
} catch { signal: controller.signal,
return false; });
return {
detail: `HTTP ${response.status}`,
reachable: response.status < 500,
};
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
return {
detail: `request timed out after ${REACHABILITY_REQUEST_TIMEOUT_MS}ms`,
reachable: false,
};
}
return {
detail: error instanceof Error ? error.message : String(error),
reachable: false,
};
} finally {
clearTimeout(timeout);
} }
} }
async function waitUntilReachable(url: string, label: string) { async function isReachable(url: string) {
return (await checkReachable(url)).reachable;
}
async function waitUntilReachable(url: string, label: string, child?: ChildProcess) {
const startedAt = Date.now(); const startedAt = Date.now();
let lastDetail = "not checked";
while (Date.now() - startedAt < START_TIMEOUT_MS) { while (Date.now() - startedAt < START_TIMEOUT_MS) {
if (await isReachable(url)) return; const result = await checkReachable(url);
lastDetail = result.detail;
if (result.reachable) return;
if (child && (child.exitCode !== null || child.signalCode !== null)) {
throw new Error(
`${label} exited before it became reachable at ${url} (exit=${child.exitCode ?? "null"}, signal=${child.signalCode ?? "null"}, last=${lastDetail}).`,
);
}
await sleep(POLL_MS); await sleep(POLL_MS);
} }
throw new Error(`${label} did not become reachable at ${url}.`); throw new Error(`${label} did not become reachable at ${url} (last=${lastDetail}).`);
}
async function waitUntilUnreachable(url: string, label: string) {
const startedAt = Date.now();
while (Date.now() - startedAt < STOP_TIMEOUT_MS) {
if (!(await isReachable(url))) return;
await sleep(POLL_MS);
}
throw new Error(`${label} did not stop serving at ${url}.`);
} }
function canListen(port: number) { function canListen(port: number) {
@@ -150,6 +212,7 @@ function getLocalUrlPort(url: string, label: string) {
function spawnManaged(command: string, args: string[], env: NodeJS.ProcessEnv) { function spawnManaged(command: string, args: string[], env: NodeJS.ProcessEnv) {
const child = spawn(command, args, { const child = spawn(command, args, {
cwd: process.cwd(), cwd: process.cwd(),
detached: process.platform !== "win32",
env, env,
stdio: "inherit", stdio: "inherit",
}); });
@@ -158,6 +221,22 @@ function spawnManaged(command: string, args: string[], env: NodeJS.ProcessEnv) {
return child; return child;
} }
function signalManagedChild(child: ChildProcess, signal: NodeJS.Signals) {
if (child.exitCode !== null || child.signalCode !== null) return;
try {
if (process.platform !== "win32" && child.pid) {
process.kill(-child.pid, signal);
return;
}
child.kill(signal);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ESRCH") {
throw error;
}
}
}
function waitForChildExit(child: ChildProcess, timeoutMs = 5_000) { function waitForChildExit(child: ChildProcess, timeoutMs = 5_000) {
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
@@ -172,10 +251,17 @@ function waitForChildExit(child: ChildProcess, timeoutMs = 5_000) {
async function stopManagedChildren() { async function stopManagedChildren() {
const children = Array.from(managedChildren); const children = Array.from(managedChildren);
for (const child of managedChildren) { for (const child of children) signalManagedChild(child, "SIGTERM");
if (!child.killed) child.kill("SIGTERM");
}
await Promise.all(children.map((child) => waitForChildExit(child))); await Promise.all(children.map((child) => waitForChildExit(child)));
for (const child of children) signalManagedChild(child, "SIGKILL");
await Promise.all(children.map((child) => waitForChildExit(child)));
}
async function stopManagedChild(child: ChildProcess) {
signalManagedChild(child, "SIGTERM");
await waitForChildExit(child);
signalManagedChild(child, "SIGKILL");
await waitForChildExit(child);
} }
function isolateLocalState() { function isolateLocalState() {
@@ -208,9 +294,18 @@ function restoreLocalState() {
} }
async function cleanup() { async function cleanup() {
await stopManagedChildren(); try {
restoreLocalState(); await stopManagedChildren();
rmSync(tempDir, { force: true, recursive: true }); await Promise.all([
activePreviewUrl
? waitUntilUnreachable(activePreviewUrl, "Preview server")
: Promise.resolve(),
activeConvexUrl ? waitUntilUnreachable(activeConvexUrl, "Local Convex") : Promise.resolve(),
]);
} finally {
restoreLocalState();
rmSync(tempDir, { force: true, recursive: true });
}
} }
function runRequired(command: string, args: string[], env: NodeJS.ProcessEnv) { function runRequired(command: string, args: string[], env: NodeJS.ProcessEnv) {
@@ -237,6 +332,25 @@ function runBuffered(command: string, args: string[], env: NodeJS.ProcessEnv) {
}; };
} }
async function startLocalConvex(args: string[], env: NodeJS.ProcessEnv, convexUrl: string) {
const maxAttempts = 3;
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const child = spawnManaged("bunx", args, env);
try {
await waitUntilReachable(convexUrl, "Local Convex", child);
return;
} catch (error) {
await stopManagedChild(child);
rmSync(LOCAL_CONVEX_STATE_DIR, { force: true, recursive: true });
if (attempt >= maxAttempts) throw error;
console.log(
`Local Convex did not start cleanly on attempt ${attempt}; retrying with fresh isolated state...`,
);
await sleep(2_000 * attempt);
}
}
}
function isFunctionUnavailableOutput(output: string) { function isFunctionUnavailableOutput(output: string) {
return ( return (
output.includes("Could not find function for") && output.includes("Could not find function for") &&
@@ -247,8 +361,8 @@ function isFunctionUnavailableOutput(output: string) {
function isLocalConvexModuleStillPreparingOutput(output: string) { function isLocalConvexModuleStillPreparingOutput(output: string) {
return ( return (
output.includes("InvalidModules") && output.includes("InvalidModules") &&
output.includes("ENOENT: no such file or directory") && ((output.includes("ENOENT: no such file or directory") && output.includes("/modules/")) ||
output.includes("/modules/") (output.includes("Cannot find module") && output.includes("/modules/_deps/node/")))
); );
} }
@@ -337,8 +451,10 @@ async function main() {
const runnerConfig = resolveLocalAuthRunnerConfig(process.env, process.argv.slice(2)); const runnerConfig = resolveLocalAuthRunnerConfig(process.env, process.argv.slice(2));
const appPort = await resolveAppPort(); const appPort = await resolveAppPort();
const appUrl = `http://127.0.0.1:${appPort}`; const appUrl = `http://127.0.0.1:${appPort}`;
const previewReadyUrl = new URL("/robots.txt", appUrl).toString();
const convexUrl = runnerConfig.convexUrl; const convexUrl = runnerConfig.convexUrl;
const convexSiteUrl = runnerConfig.convexSiteUrl; const convexSiteUrl = runnerConfig.convexSiteUrl;
activePreviewUrl = previewReadyUrl;
const convexCloudPort = String(getLocalUrlPort(convexUrl, "PLAYWRIGHT_LOCAL_AUTH_CONVEX_URL")); const convexCloudPort = String(getLocalUrlPort(convexUrl, "PLAYWRIGHT_LOCAL_AUTH_CONVEX_URL"));
const convexSitePort = String( const convexSitePort = String(
getLocalUrlPort(convexSiteUrl, "PLAYWRIGHT_LOCAL_AUTH_CONVEX_SITE_URL"), getLocalUrlPort(convexSiteUrl, "PLAYWRIGHT_LOCAL_AUTH_CONVEX_SITE_URL"),
@@ -403,8 +519,7 @@ async function main() {
); );
console.log(`Starting local Convex at ${convexUrl} with isolated e2e state.`); console.log(`Starting local Convex at ${convexUrl} with isolated e2e state.`);
spawnManaged( await startLocalConvex(
"bunx",
[ [
"convex", "convex",
"dev", "dev",
@@ -420,8 +535,9 @@ async function main() {
convexSitePort, convexSitePort,
], ],
e2eEnv, e2eEnv,
convexUrl,
); );
await waitUntilReachable(convexUrl, "Local Convex"); activeConvexUrl = convexUrl;
console.log("Configuring local Convex environment for local-auth Playwright e2e."); console.log("Configuring local Convex environment for local-auth Playwright e2e.");
const localAuthDeployment = const localAuthDeployment =
@@ -446,18 +562,18 @@ async function main() {
]); ]);
console.log("Waiting for local Convex functions."); console.log("Waiting for local Convex functions.");
await runConvexFunctionWhenReady("appMeta:getDeploymentInfo", {}, e2eEnv, { push: true }); await runConvexFunctionWhenReady("appMeta:getDeploymentInfo", {}, e2eEnv);
console.log("Building ClawHub for local-auth Playwright e2e."); console.log("Building ClawHub for local-auth Playwright e2e.");
runRequired("bun", ["run", "build"], e2eEnv); runRequired("bun", ["run", "build"], e2eEnv);
console.log(`Starting preview server at ${appUrl}.`); console.log(`Starting preview server at ${appUrl}.`);
spawnManaged( const previewProcess = spawnManaged(
"bun", "bun",
["run", "preview", "--", "--host", "127.0.0.1", "--port", String(appPort)], ["run", "preview", "--", "--host", "127.0.0.1", "--port", String(appPort)],
e2eEnv, e2eEnv,
); );
await waitUntilReachable(appUrl, "Preview server"); await waitUntilReachable(previewReadyUrl, "Preview server", previewProcess);
runRequired("bunx", ["playwright", "test", ...runnerConfig.playwrightArgs], { runRequired("bunx", ["playwright", "test", ...runnerConfig.playwrightArgs], {
...e2eEnv, ...e2eEnv,
+26
View File
@@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { import {
buildPluginCanonicalHrefForRequestedPath,
buildPluginDetailHref, buildPluginDetailHref,
buildPluginSecurityAuditHref, buildPluginSecurityAuditHref,
displayPluginPackageName, displayPluginPackageName,
@@ -28,6 +29,31 @@ describe("plugin routes", () => {
); );
}); });
it("does not rewrite scoped package names to unrelated owner handles", () => {
expect(buildPluginDetailHref("@scope/demo-plugin", { ownerHandle: "acme" })).toBe(
"/scope/plugins/demo-plugin",
);
});
it("preserves security audit intent when canonicalizing unowned plugin paths", () => {
expect(
buildPluginCanonicalHrefForRequestedPath(
"/plugins/demo-plugin/security-audit",
"demo-plugin",
"demo-plugin",
{ ownerHandle: "acme" },
),
).toBe("/acme/plugins/demo-plugin/security-audit");
expect(
buildPluginCanonicalHrefForRequestedPath(
"/plugins/demo-plugin/security/virustotal",
"demo-plugin",
"demo-plugin",
{ ownerHandle: "acme" },
),
).toBe("/acme/plugins/demo-plugin/security-audit");
});
it("parses scoped package names and scoped routes", () => { it("parses scoped package names and scoped routes", () => {
expect(parseScopedPackageName("@openclaw/codex")).toEqual({ expect(parseScopedPackageName("@openclaw/codex")).toEqual({
scope: "@openclaw", scope: "@openclaw",
+17 -1
View File
@@ -30,7 +30,7 @@ function routeSegment(value: string) {
export function buildPluginDetailHref(name: string, options: PluginRouteOptions = {}) { export function buildPluginDetailHref(name: string, options: PluginRouteOptions = {}) {
const scoped = parseScopedPackageName(name); const scoped = parseScopedPackageName(name);
const ownerHandle = cleanOwnerHandle(options.ownerHandle) ?? cleanOwnerHandle(scoped?.scope); const ownerHandle = cleanOwnerHandle(scoped?.scope) ?? cleanOwnerHandle(options.ownerHandle);
if (ownerHandle) { if (ownerHandle) {
return `/${routeSegment(ownerHandle)}/plugins/${routeSegment(scoped?.name ?? name)}`; return `/${routeSegment(ownerHandle)}/plugins/${routeSegment(scoped?.name ?? name)}`;
@@ -47,6 +47,22 @@ export function buildPluginSecurityAuditHref(name: string, options: PluginRouteO
return `${buildPluginDetailHref(name, options)}/security-audit`; return `${buildPluginDetailHref(name, options)}/security-audit`;
} }
export function buildPluginCanonicalHrefForRequestedPath(
pathname: string,
requestedName: string,
resolvedName: string,
options: PluginRouteOptions = {},
) {
const requestedDetailPath = buildPluginDetailHref(requestedName);
if (
pathname === `${requestedDetailPath}/security-audit` ||
pathname.startsWith(`${requestedDetailPath}/security/`)
) {
return buildPluginSecurityAuditHref(resolvedName, options);
}
return buildPluginDetailHref(resolvedName, options);
}
export function buildPluginValidationHref(name: string) { export function buildPluginValidationHref(name: string) {
return `${buildPluginDetailHref(name)}#validation`; return `${buildPluginDetailHref(name)}#validation`;
} }
+10 -2
View File
@@ -80,6 +80,7 @@ import {
} from "../../lib/packageApi"; } from "../../lib/packageApi";
import { familyLabel } from "../../lib/packageLabels"; import { familyLabel } from "../../lib/packageLabels";
import { import {
buildPluginCanonicalHrefForRequestedPath,
buildPluginDetailHref, buildPluginDetailHref,
buildPluginSecurityAuditHref, buildPluginSecurityAuditHref,
displayPluginPackageName, displayPluginPackageName,
@@ -256,14 +257,21 @@ export const Route = createFileRoute("/plugins/$name")({
}); });
} }
}, },
loader: async ({ params }) => { loader: async ({ location, params }) => {
const data = await loadPluginDetail(params.name); const data = await loadPluginDetail(params.name);
const ownerHandle = data.detail.owner?.handle ?? null; const ownerHandle = data.detail.owner?.handle ?? null;
const packageName = data.detail.package?.name ?? null; const packageName = data.detail.package?.name ?? null;
if (packageName && ownerHandle) { if (packageName && ownerHandle) {
throw redirect({ throw redirect({
href: buildPluginDetailHref(packageName, { ownerHandle }), href: buildPluginCanonicalHrefForRequestedPath(
location?.pathname ?? buildPluginDetailHref(params.name),
params.name,
packageName,
{
ownerHandle,
},
),
replace: true, replace: true,
}); });
} }