fix: alert when signal scans stop retrying (#3202)

convex/publisherAbuse.test.ts

@@ -50,6 +50,9 @@ vi.mock("./_generated/api", () => ({
       notifyPublisherAbuseSignalChangesInternal: Symbol(
         "notifyPublisherAbuseSignalChangesInternal",
       ),
+      notifyPublisherAbuseSignalScanFailureInternal: Symbol(
+        "notifyPublisherAbuseSignalScanFailureInternal",
+      ),
       persistTemporalPublisherAbuseCandidatesInternal: Symbol(
         "persistTemporalPublisherAbuseCandidatesInternal",
       ),
@@ -399,6 +402,19 @@ const notifyPublisherAbuseSignalChangesHandler = (
   >
 )._handler;
 
+const notifyPublisherAbuseSignalScanFailureHandler = (
+  publisherAbuse.notifyPublisherAbuseSignalScanFailureInternal as unknown as Wrapped<
+    {
+      runId: string;
+      failureCount: number;
+      errorMessage: string;
+      failedAt: number;
+      deliveryAttempt?: number;
+    },
+    { ok: boolean; sent: boolean; skipped?: boolean; error?: string }
+  >
+)._handler;
+
 const banPublisherAbuseOwnerHandler = (
   publisherAbuse.banPublisherAbuseOwner as unknown as Wrapped<
     {
@@ -1201,6 +1217,57 @@ describe("publisher abuse dry-run persistence", () => {
     }
   });
 
+  it("sends terminal signal scan failures through the Hermit publisher abuse endpoint", async () => {
+    const previousEnv = { ...process.env };
+    const previousFetch = globalThis.fetch;
+    process.env.CLAWHUB_HERMIT_TOKEN = "test-token-placeholder";
+    process.env.HERMIT_PUBLISHER_ABUSE_BASE_URL = "https://forms.example.test";
+    process.env.SITE_URL = "https://clawhub.example.test";
+    const fetchMock = vi.fn<typeof fetch>(async () => new Response("ok", { status: 200 }));
+    globalThis.fetch = fetchMock as typeof fetch;
+    const scheduler = { runAfter: vi.fn(async () => null) };
+
+    try {
+      await expect(
+        notifyPublisherAbuseSignalScanFailureHandler(
+          { scheduler },
+          {
+            runId: "publisherAbuseScoreRuns:failed-run",
+            failureCount: 5,
+            errorMessage: "Query exceeded the document read limit.",
+            failedAt: 1716000000000,
+          },
+        ),
+      ).resolves.toEqual({ ok: true, sent: true });
+
+      expect(fetchMock).toHaveBeenCalledWith(
+        "https://forms.example.test/api/clawhub-publisher-abuse/signals/digest",
+        expect.objectContaining({
+          method: "POST",
+          headers: expect.objectContaining({
+            Authorization: `Bearer ${process.env.CLAWHUB_HERMIT_TOKEN}`,
+            "Content-Type": "application/json",
+          }),
+        }),
+      );
+      const [, requestInit] = fetchMock.mock.calls[0] ?? [];
+      const requestBody = (requestInit as RequestInit | undefined)?.body;
+      if (typeof requestBody !== "string") throw new Error("Expected Hermit request body");
+      expect(JSON.parse(requestBody)).toEqual({
+        kind: "publisher_abuse_signal_scan_failed",
+        runId: "publisherAbuseScoreRuns:failed-run",
+        failureCount: 5,
+        errorMessage: "Query exceeded the document read limit.",
+        failedAt: 1716000000000,
+        dashboardUrl: "https://clawhub.example.test/management?view=abuse&tab=signals",
+      });
+      expect(scheduler.runAfter).not.toHaveBeenCalled();
+    } finally {
+      process.env = previousEnv;
+      globalThis.fetch = previousFetch;
+    }
+  });
+
   it("schedules the next Hermit signal digest immediately when more changed signals remain", async () => {
     const previousEnv = { ...process.env };
     const previousFetch = globalThis.fetch;

convex/publisherAbuse.ts

@@ -75,6 +75,8 @@ const MAX_PUBLISHER_ABUSE_SIGNAL_REVIEW_NOTE_LENGTH = 1000;
 const PUBLISHER_ABUSE_SIGNAL_NOTIFICATION_BATCH_SIZE = 10;
 const PUBLISHER_ABUSE_SIGNAL_NOTIFICATION_MAX_BATCH_SIZE = 25;
 const PUBLISHER_ABUSE_SIGNAL_NOTIFICATION_RETRY_MS = 60 * 60 * 1000;
+const PUBLISHER_ABUSE_SIGNAL_SCAN_FAILURE_NOTIFICATION_RETRY_MS = 5 * 60 * 1000;
+const MAX_PUBLISHER_ABUSE_SIGNAL_SCAN_FAILURE_NOTIFICATION_ATTEMPTS = 5;
 const MAX_STAFF_PUBLISHER_MANAGER_EXCLUSION_SCAN = 100;
 const MAX_STAFF_PUBLISHER_MANAGER_EXCLUSION_READS_PER_PAGE = 2_000;
 const STAFF_PUBLISHER_MANAGER_ROLES = ["owner", "admin"] as const;
@@ -1672,6 +1674,17 @@ export const notifyPublisherAbuseSignalChangesInternal = internalAction({
   handler: notifyPublisherAbuseSignalChangesInternalHandler,
 });
 
+export const notifyPublisherAbuseSignalScanFailureInternal = internalAction({
+  args: {
+    runId: v.id("publisherAbuseScoreRuns"),
+    failureCount: v.number(),
+    errorMessage: v.string(),
+    failedAt: v.number(),
+    deliveryAttempt: v.optional(v.number()),
+  },
+  handler: notifyPublisherAbuseSignalScanFailureInternalHandler,
+});
+
 export const processPublisherAbuseAutobansInternal = internalAction({
   args: {
     batchSize: v.optional(v.number()),
@@ -2751,6 +2764,65 @@ export async function notifyPublisherAbuseSignalChangesInternalHandler(
   }
 }
 
+export async function notifyPublisherAbuseSignalScanFailureInternalHandler(
+  ctx: Pick<ActionCtx, "scheduler">,
+  args: {
+    runId: Id<"publisherAbuseScoreRuns">;
+    failureCount: number;
+    errorMessage: string;
+    failedAt: number;
+    deliveryAttempt?: number;
+  },
+) {
+  const config = getHermitPublisherAbuseSignalConfig();
+  if (!config) {
+    console.error("[publisher-temporal-abuse-scan] Hermit failure alert skipped: missing config", {
+      runId: args.runId,
+    });
+    return { ok: false as const, sent: false as const, skipped: true as const };
+  }
+
+  const payload = {
+    kind: "publisher_abuse_signal_scan_failed" as const,
+    runId: args.runId,
+    failureCount: args.failureCount,
+    errorMessage: args.errorMessage,
+    failedAt: args.failedAt,
+    dashboardUrl: `${config.siteUrl}/management?view=abuse&tab=signals`,
+  };
+  try {
+    const response = await fetch(config.digestUrl, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${config.token}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(payload),
+    });
+    if (!response.ok) {
+      const body = await response.text();
+      throw new Error(`Hermit publisher abuse scan alert failed: ${response.status} ${body}`);
+    }
+    return { ok: true as const, sent: true as const };
+  } catch (error) {
+    const message = errorMessageFromUnknown(error);
+    const deliveryAttempt = (args.deliveryAttempt ?? 0) + 1;
+    console.error("[publisher-temporal-abuse-scan] Hermit failure alert failed", {
+      runId: args.runId,
+      deliveryAttempt,
+      message,
+    });
+    if (deliveryAttempt < MAX_PUBLISHER_ABUSE_SIGNAL_SCAN_FAILURE_NOTIFICATION_ATTEMPTS) {
+      await ctx.scheduler.runAfter(
+        PUBLISHER_ABUSE_SIGNAL_SCAN_FAILURE_NOTIFICATION_RETRY_MS,
+        internal.publisherAbuse.notifyPublisherAbuseSignalScanFailureInternal,
+        { ...args, deliveryAttempt },
+      );
+    }
+    return { ok: false as const, sent: false as const, error: message };
+  }
+}
+
 function getHermitPublisherAbuseSignalConfig() {
   const token =
     process.env.CLAWHUB_HERMIT_TOKEN?.trim() || process.env.CLAWHUB_BAN_APPEALS_TOKEN?.trim() || "";

convex/publisherAbuseTemporalScan.test.ts

@@ -127,7 +127,9 @@ describe("scheduled temporal publisher abuse scan", () => {
         isDone: false,
         scannedSkills: 0,
       });
-    const scheduler = { runAfter: vi.fn(async () => null) };
+    const scheduler = {
+      runAfter: vi.fn(async (_delay: number, _target: unknown, _args: unknown) => null),
+    };
 
     await expect(
       startPublisherAbuseSignalScanHandler({
@@ -567,7 +569,9 @@ describe("scheduled temporal publisher abuse scan", () => {
       lastTransientError: "fourth failure",
     });
     const patch = vi.fn(async () => null);
-    const scheduler = { runAfter: vi.fn(async () => null) };
+    const scheduler = {
+      runAfter: vi.fn(async (_delay: number, _target: unknown, _args: unknown) => null),
+    };
     const ctx = { db: { get: vi.fn(async () => run), patch }, scheduler };
 
     await expect(
@@ -588,7 +592,15 @@ describe("scheduled temporal publisher abuse scan", () => {
         errorMessage: "fifth failure",
       }),
     );
-    expect(scheduler.runAfter).not.toHaveBeenCalled();
+    expect(scheduler.runAfter).toHaveBeenCalledTimes(1);
+    const [delay, _target, alertArgs] = scheduler.runAfter.mock.calls[0] ?? [];
+    expect(delay).toBe(0);
+    expect(alertArgs).toEqual({
+      runId: run._id,
+      failureCount: 5,
+      errorMessage: "fifth failure",
+      failedAt: expect.any(Number),
+    });
   });
 
   it("schedules the next saved-page attempt after a non-terminal failure", async () => {

convex/publisherAbuseTemporalScan.ts

@@ -615,6 +615,16 @@ export async function recordScheduledTemporalScanFailureInternalHandler(
       errorMessage: args.errorMessage,
       nextTransientRetryAt: undefined,
     });
+    await ctx.scheduler.runAfter(
+      0,
+      internal.publisherAbuse.notifyPublisherAbuseSignalScanFailureInternal,
+      {
+        runId: run._id,
+        failureCount,
+        errorMessage: args.errorMessage,
+        failedAt: now,
+      },
+    );
     console.error("[publisher-temporal-abuse-scan] retry budget exhausted", {
       event: "publisher_temporal_abuse_scan_failed",
       runId: run._id,

specs/security-moderation.md

@@ -71,8 +71,9 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
   with bounded backoff; any successfully persisted page resets the consecutive
   failure count. A watchdog treats fifteen minutes without persisted progress as
   a failed attempt. After five consecutive failed attempts, the run becomes
-  terminal, retains the last error for the staff UI, and emits the structured
-  `publisher_temporal_abuse_scan_failed` operator-alert event.
+  terminal, retains the last error for the staff UI, emits the structured
+  `publisher_temporal_abuse_scan_failed` operator event, and sends a Hermit
+  alert to the configured ClawHub review channel.
   Moderators can start this same full signal pipeline from the staff Signals tab.
   New manual starts record the actor; requests made while a temporal scan is
   already active return that run without starting a competing worker. Stale

src/routes/-management.test.tsx

@@ -1091,6 +1091,39 @@ describe("Management", () => {
     expect(screen.queryByText("Retrying after 1 of 5 failed attempts")).toBeNull();
   });
 
+  it("shows the number of skills processed by a running signal scan", () => {
+    searchState = { view: "abuse", tab: "signals" };
+    useQueryMock.mockImplementation((query, args) => {
+      if (args === "skip") return undefined;
+      const name = getFunctionName(query);
+      if (name === "skills:listRecentVersions") return [];
+      if (name === "skills:listReportedSkills") return [];
+      if (name === "skills:listDuplicateCandidates") return [];
+      if (name === "publisherAbuse:listReviewDashboard") {
+        return {
+          latestRun: null,
+          latestSignalRun: {
+            status: "running",
+            scannedPublishers: 0,
+            scoredPublishers: 0,
+            temporalSampleSize: 4_600,
+            transientErrorCount: 0,
+          },
+          pendingItems: [],
+          pendingPotentialBanCandidateItems: [],
+          pendingReviewItems: [],
+          recentResolvedItems: [],
+        };
+      }
+      if (name === "users:list") return { items: [], total: 0 };
+      return undefined;
+    });
+
+    render(<Management />);
+
+    expect(screen.getByText("4,600")).toBeTruthy();
+  });
+
   it("shows users as a separate management view", () => {
     searchState = { view: "users" };
 

src/routes/-management/AbusePage.tsx

@@ -133,6 +133,10 @@ export function AbusePage({
   const latestRun = dashboard?.latestRun ?? null;
   const latestSignalRun = dashboard?.latestSignalRun ?? null;
   const displayedRun = tab === "signals" ? latestSignalRun : latestRun;
+  const displayedScannedCount =
+    tab === "signals"
+      ? (latestSignalRun?.temporalSampleSize ?? latestSignalRun?.scannedPublishers)
+      : displayedRun?.scannedPublishers;
   const signalFailureCount = latestSignalRun?.transientErrorCount ?? 0;
   const selectedScore = selectedItem?.latestScore ?? null;
   const selectedPublisher = selectedItem?.publisher ?? null;
@@ -286,7 +290,7 @@ export function AbusePage({
             </div>
             <div>
               <dt>Scanned</dt>
-              <dd>{formatWholeNumber(displayedRun?.scannedPublishers)}</dd>
+              <dd>{formatWholeNumber(displayedScannedCount)}</dd>
             </div>
             <div>
               <dt>Scored</dt>