mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 00:47:57 +00:00
fix: retire dependency registry scans
This commit is contained in:
Jesse Merhi
Vendored
-2
@@ -57,7 +57,6 @@ import type * as lib_changelog from "../lib/changelog.js";
|
||||
import type * as lib_clawpack from "../lib/clawpack.js";
|
||||
import type * as lib_commentScamPrompt from "../lib/commentScamPrompt.js";
|
||||
import type * as lib_contentTypes from "../lib/contentTypes.js";
|
||||
import type * as lib_depRegistryScan from "../lib/depRegistryScan.js";
|
||||
import type * as lib_devAuth from "../lib/devAuth.js";
|
||||
import type * as lib_devSeed from "../lib/devSeed.js";
|
||||
import type * as lib_emails from "../lib/emails.js";
|
||||
@@ -213,7 +212,6 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/clawpack": typeof lib_clawpack;
|
||||
"lib/commentScamPrompt": typeof lib_commentScamPrompt;
|
||||
"lib/contentTypes": typeof lib_contentTypes;
|
||||
"lib/depRegistryScan": typeof lib_depRegistryScan;
|
||||
"lib/devAuth": typeof lib_devAuth;
|
||||
"lib/devSeed": typeof lib_devSeed;
|
||||
"lib/emails": typeof lib_emails;
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
/* @vitest-environment node */
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const { checkDependencyRegistriesHandler } = await import("./depRegistryScan");
|
||||
|
||||
describe("dependency registry scan drain", () => {
|
||||
it("keeps legacy scheduled jobs harmless after the scanner is retired", async () => {
|
||||
await expect(checkDependencyRegistriesHandler()).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
+4
-263
@@ -1,270 +1,11 @@
|
||||
import { v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Doc, Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { internalAction, internalMutation, internalQuery } from "./functions";
|
||||
import {
|
||||
dedupeDeps,
|
||||
depRegistryUrl,
|
||||
parseDependencyFile,
|
||||
SUPPORTED_DEP_REGISTRIES,
|
||||
summarizeDepRegistryChecks,
|
||||
type DepEntry,
|
||||
type DepRegistryResult,
|
||||
type DepRegistryUnresolved,
|
||||
type SupportedDepRegistry,
|
||||
} from "./lib/depRegistryScan";
|
||||
import { readStorageText } from "./lib/packageRegistry";
|
||||
import { internalAction } from "./_generated/server";
|
||||
|
||||
const REQUEST_TIMEOUT_MS = 8_000;
|
||||
const MAX_RETRIES = 2;
|
||||
const BACKOFF_BASE_MS = 750;
|
||||
const INTER_REQUEST_DELAY_MS = 100;
|
||||
const MAX_DEPENDENCIES_PER_SCAN = 120;
|
||||
const CACHE_TTL_EXISTS_MS = 30 * 24 * 60 * 60 * 1_000;
|
||||
const CACHE_TTL_NOT_EXISTS_MS = 7 * 24 * 60 * 60 * 1_000;
|
||||
|
||||
const registryValidator = v.union(v.literal("pypi"), v.literal("npm"), v.literal("cargo"));
|
||||
|
||||
type RegistryCheck =
|
||||
| { kind: "found"; httpStatus: number }
|
||||
| { kind: "missing"; httpStatus: number }
|
||||
| { kind: "unresolved"; reason: string };
|
||||
|
||||
function isSupportedRegistry(value: string): value is SupportedDepRegistry {
|
||||
return (SUPPORTED_DEP_REGISTRIES as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
async function wait(ms: number) {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function checkRegistry(dep: DepEntry): Promise<RegistryCheck> {
|
||||
const headers: Record<string, string> = { Accept: "application/json" };
|
||||
if (dep.registry === "cargo") {
|
||||
headers["User-Agent"] = "ClawHub-DepRegistryScan/1.0 (https://clawhub.ai)";
|
||||
}
|
||||
|
||||
let lastStatus: number | undefined;
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt += 1) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetch(depRegistryUrl(dep.registry, dep.name), {
|
||||
method: "GET",
|
||||
headers,
|
||||
signal: controller.signal,
|
||||
});
|
||||
clearTimeout(timeout);
|
||||
lastStatus = response.status;
|
||||
if (response.status === 200) return { kind: "found", httpStatus: response.status };
|
||||
if (response.status === 404) return { kind: "missing", httpStatus: response.status };
|
||||
if (response.status !== 429 && response.status < 500) {
|
||||
return {
|
||||
kind: "unresolved",
|
||||
reason: `unexpected HTTP ${response.status}`,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
clearTimeout(timeout);
|
||||
if (attempt === MAX_RETRIES) {
|
||||
return {
|
||||
kind: "unresolved",
|
||||
reason: error instanceof Error ? error.message : "network error",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (attempt < MAX_RETRIES) {
|
||||
await wait(2 ** attempt * BACKOFF_BASE_MS);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
kind: "unresolved",
|
||||
reason: lastStatus ? `HTTP ${lastStatus}` : "network error",
|
||||
};
|
||||
}
|
||||
|
||||
async function extractDependencies(ctx: Pick<ActionCtx, "storage">, version: Doc<"skillVersions">) {
|
||||
const entries: DepEntry[] = [];
|
||||
for (const file of version.files) {
|
||||
const basename = file.path.split("/").pop()?.toLowerCase() ?? "";
|
||||
if (
|
||||
basename !== "requirements.txt" &&
|
||||
basename !== "requirements-dev.txt" &&
|
||||
basename !== "requirements_dev.txt" &&
|
||||
basename !== "requirements-test.txt" &&
|
||||
basename !== "requirements_test.txt" &&
|
||||
basename !== "package.json" &&
|
||||
basename !== "cargo.toml" &&
|
||||
basename !== "pyproject.toml"
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const content = await readStorageText(ctx, file.storageId);
|
||||
entries.push(...parseDependencyFile(file.path, content));
|
||||
}
|
||||
return dedupeDeps(entries);
|
||||
}
|
||||
|
||||
export const lookupCacheInternal = internalQuery({
|
||||
args: {
|
||||
registry: registryValidator,
|
||||
name: v.string(),
|
||||
},
|
||||
handler: async (ctx, args): Promise<Doc<"depRegistryCache"> | null> => {
|
||||
return ctx.db
|
||||
.query("depRegistryCache")
|
||||
.withIndex("by_registry_name", (q) => q.eq("registry", args.registry).eq("name", args.name))
|
||||
.unique();
|
||||
},
|
||||
});
|
||||
|
||||
export const upsertCacheInternal = internalMutation({
|
||||
args: {
|
||||
registry: registryValidator,
|
||||
name: v.string(),
|
||||
exists: v.boolean(),
|
||||
httpStatus: v.number(),
|
||||
checkedAt: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const existing = await ctx.db
|
||||
.query("depRegistryCache")
|
||||
.withIndex("by_registry_name", (q) => q.eq("registry", args.registry).eq("name", args.name))
|
||||
.unique();
|
||||
const patch = {
|
||||
registry: args.registry,
|
||||
name: args.name,
|
||||
exists: args.exists,
|
||||
httpStatus: args.httpStatus,
|
||||
checkedAt: args.checkedAt,
|
||||
};
|
||||
if (existing) {
|
||||
await ctx.db.patch(existing._id, patch);
|
||||
} else {
|
||||
await ctx.db.insert("depRegistryCache", patch);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
export const getRetryableVersionIdsInternal = internalQuery({
|
||||
args: {
|
||||
limit: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const limit = Math.min(Math.max(args.limit ?? 25, 1), 100);
|
||||
const versions = await ctx.db
|
||||
.query("skillVersions")
|
||||
.withIndex("by_dep_registry_scan_status_and_created", (q) =>
|
||||
q.eq("depRegistryScanStatus", "error"),
|
||||
)
|
||||
.order("desc")
|
||||
.take(limit);
|
||||
return versions.map((version) => version._id);
|
||||
},
|
||||
});
|
||||
|
||||
async function checkWithCache(ctx: ActionCtx, dep: DepEntry) {
|
||||
const now = Date.now();
|
||||
const cached = (await ctx.runQuery(internal.depRegistryScan.lookupCacheInternal, {
|
||||
registry: dep.registry,
|
||||
name: dep.name,
|
||||
})) as Doc<"depRegistryCache"> | null;
|
||||
if (cached) {
|
||||
const ttl = cached.exists ? CACHE_TTL_EXISTS_MS : CACHE_TTL_NOT_EXISTS_MS;
|
||||
if (now - cached.checkedAt < ttl) {
|
||||
return cached.exists
|
||||
? ({ kind: "found", httpStatus: cached.httpStatus } as const)
|
||||
: ({ kind: "missing", httpStatus: cached.httpStatus } as const);
|
||||
}
|
||||
}
|
||||
|
||||
const check = await checkRegistry(dep);
|
||||
if (check.kind !== "unresolved") {
|
||||
await ctx.runMutation(internal.depRegistryScan.upsertCacheInternal, {
|
||||
registry: dep.registry,
|
||||
name: dep.name,
|
||||
exists: check.kind === "found",
|
||||
httpStatus: check.httpStatus,
|
||||
checkedAt: now,
|
||||
});
|
||||
}
|
||||
return check;
|
||||
export async function checkDependencyRegistriesHandler(): Promise<null> {
|
||||
return null;
|
||||
}
|
||||
|
||||
export const checkDependencyRegistries = internalAction({
|
||||
args: { versionId: v.id("skillVersions") },
|
||||
handler: async (ctx, args) => {
|
||||
const version = (await ctx.runQuery(internal.skills.getVersionByIdInternal, {
|
||||
versionId: args.versionId,
|
||||
})) as Doc<"skillVersions"> | null;
|
||||
if (!version) return null;
|
||||
if (version.depRegistryAnalysis && version.depRegistryAnalysis.status !== "error") {
|
||||
return version.depRegistryAnalysis;
|
||||
}
|
||||
|
||||
const deps = await extractDependencies(ctx, version);
|
||||
const checkableDeps = deps.slice(0, MAX_DEPENDENCIES_PER_SCAN);
|
||||
const deferredDeps = deps.slice(MAX_DEPENDENCIES_PER_SCAN);
|
||||
const results: DepRegistryResult[] = [];
|
||||
const unresolved: DepRegistryUnresolved[] = deferredDeps.map((dep) => ({
|
||||
...dep,
|
||||
reason: "dependency scan limit reached",
|
||||
}));
|
||||
|
||||
for (const dep of checkableDeps) {
|
||||
if (!isSupportedRegistry(dep.registry)) continue;
|
||||
const check = await checkWithCache(ctx, dep);
|
||||
if (check.kind === "unresolved") {
|
||||
unresolved.push({ ...dep, reason: check.reason });
|
||||
} else {
|
||||
results.push({
|
||||
...dep,
|
||||
exists: check.kind === "found",
|
||||
httpStatus: check.httpStatus,
|
||||
});
|
||||
}
|
||||
await wait(INTER_REQUEST_DELAY_MS);
|
||||
}
|
||||
|
||||
const analysis = summarizeDepRegistryChecks({
|
||||
results,
|
||||
unresolved,
|
||||
checkedAt: Date.now(),
|
||||
});
|
||||
|
||||
await ctx.runMutation(internal.skills.updateVersionDepRegistryAnalysisInternal, {
|
||||
versionId: args.versionId,
|
||||
depRegistryAnalysis: analysis,
|
||||
});
|
||||
|
||||
return analysis;
|
||||
},
|
||||
});
|
||||
|
||||
export const rescanErrorDepRegistryVersions = internalAction({
|
||||
args: {
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const versionIds = (await ctx.runQuery(
|
||||
internal.depRegistryScan.getRetryableVersionIdsInternal,
|
||||
{ limit: args.batchSize ?? 25 },
|
||||
)) as Id<"skillVersions">[];
|
||||
|
||||
let scheduled = 0;
|
||||
for (const versionId of versionIds) {
|
||||
await ctx.scheduler.runAfter(
|
||||
scheduled * 2_000,
|
||||
internal.depRegistryScan.checkDependencyRegistries,
|
||||
{
|
||||
versionId,
|
||||
},
|
||||
);
|
||||
scheduled += 1;
|
||||
}
|
||||
return { scheduled };
|
||||
},
|
||||
handler: checkDependencyRegistriesHandler,
|
||||
});
|
||||
|
||||
@@ -4675,6 +4675,15 @@ describe("httpApiV1 handlers", () => {
|
||||
engineVersion: "static-v1",
|
||||
checkedAt: 2,
|
||||
},
|
||||
depRegistryAnalysis: {
|
||||
status: "suspicious",
|
||||
results: [],
|
||||
notFoundPackages: ["left-pad (npm)"],
|
||||
unresolvedPackages: [],
|
||||
summary: "Legacy dependency registry warning.",
|
||||
checkedAt: 9,
|
||||
},
|
||||
depRegistryScanStatus: "suspicious",
|
||||
llmAnalysis: {
|
||||
status: "clean",
|
||||
verdict: "clean",
|
||||
@@ -4683,14 +4692,6 @@ describe("httpApiV1 handlers", () => {
|
||||
checkedAt: 3,
|
||||
model: "gpt-test",
|
||||
},
|
||||
depRegistryAnalysis: {
|
||||
status: "clean",
|
||||
results: [],
|
||||
notFoundPackages: [],
|
||||
unresolvedPackages: [],
|
||||
summary: "No dependency issues.",
|
||||
checkedAt: 4,
|
||||
},
|
||||
capabilityTags: ["dev-tools"],
|
||||
softDeletedAt: undefined,
|
||||
};
|
||||
@@ -4743,7 +4744,7 @@ describe("httpApiV1 handlers", () => {
|
||||
requestedVersion: "1.0.0",
|
||||
version: "1.0.0",
|
||||
createdAt: 1,
|
||||
checkedAt: 4,
|
||||
checkedAt: 3,
|
||||
skillUrl: "https://example.com/acme/demo",
|
||||
securityAuditUrl: "https://example.com/acme/demo/security-audit?version=1.0.0",
|
||||
security: {
|
||||
@@ -4753,7 +4754,7 @@ describe("httpApiV1 handlers", () => {
|
||||
verdict: "clean",
|
||||
signals: {
|
||||
staticScan: { status: "clean", rawStatus: "clean" },
|
||||
dependencyRegistry: { status: "clean", rawStatus: "clean" },
|
||||
dependencyRegistry: null,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -4762,7 +4763,13 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(json.items[0].card).toBeUndefined();
|
||||
expect(json.items[0].artifact).toBeUndefined();
|
||||
expect(json.items[0].security.signals.staticScan.findings).toBeUndefined();
|
||||
expect(json.items[0].security.signals.dependencyRegistry.notFoundPackages).toBeUndefined();
|
||||
expect(Object.keys(json.items[0].security.signals)).toEqual([
|
||||
"staticScan",
|
||||
"virusTotal",
|
||||
"skillSpector",
|
||||
"dependencyRegistry",
|
||||
]);
|
||||
expect(json.items[0].security.signals.dependencyRegistry).toBeNull();
|
||||
expect(runQuery.mock.calls.map(([, args]) => args)).toContainEqual({
|
||||
slug: "demo",
|
||||
version: "1.0.0",
|
||||
@@ -5141,6 +5148,15 @@ describe("httpApiV1 handlers", () => {
|
||||
source: "engines",
|
||||
checkedAt: 4,
|
||||
},
|
||||
depRegistryAnalysis: {
|
||||
status: "suspicious",
|
||||
results: [],
|
||||
notFoundPackages: ["left-pad (npm)"],
|
||||
unresolvedPackages: [],
|
||||
summary: "Legacy dependency registry warning.",
|
||||
checkedAt: 9,
|
||||
},
|
||||
depRegistryScanStatus: "suspicious",
|
||||
skillSpectorAnalysis: {
|
||||
status: "clean",
|
||||
score: 0,
|
||||
@@ -5152,14 +5168,6 @@ describe("httpApiV1 handlers", () => {
|
||||
summary: "SkillSpector clean.",
|
||||
checkedAt: 5,
|
||||
},
|
||||
depRegistryAnalysis: {
|
||||
status: "clean",
|
||||
results: [],
|
||||
notFoundPackages: [],
|
||||
unresolvedPackages: [],
|
||||
summary: "No dependency issues.",
|
||||
checkedAt: 6,
|
||||
},
|
||||
capabilityTags: ["dev-tools"],
|
||||
softDeletedAt: undefined,
|
||||
};
|
||||
@@ -5256,7 +5264,7 @@ describe("httpApiV1 handlers", () => {
|
||||
recommendation: "INSTALL",
|
||||
issueCount: 0,
|
||||
},
|
||||
dependencyRegistry: { status: "clean" },
|
||||
dependencyRegistry: null,
|
||||
},
|
||||
},
|
||||
signature: { status: "unsigned" },
|
||||
@@ -5415,7 +5423,7 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(json.security).toMatchObject({ status: "clean", passed: true });
|
||||
});
|
||||
|
||||
it("passes verification when static and dependency findings are advisory but ClawScan is clean", async () => {
|
||||
it("passes verification when static findings are advisory but ClawScan is clean", async () => {
|
||||
const internalVersion = {
|
||||
_id: "skillVersions:1",
|
||||
skillId: "skills:1",
|
||||
@@ -5442,14 +5450,6 @@ describe("httpApiV1 handlers", () => {
|
||||
summary: "ClawScan clean.",
|
||||
checkedAt: 3,
|
||||
},
|
||||
depRegistryAnalysis: {
|
||||
status: "malicious",
|
||||
results: [],
|
||||
notFoundPackages: ["left-pad"],
|
||||
unresolvedPackages: [],
|
||||
summary: "Dependency advisory warning.",
|
||||
checkedAt: 4,
|
||||
},
|
||||
softDeletedAt: undefined,
|
||||
};
|
||||
const generatedBundleFingerprint = await buildBundleFingerprint(internalVersion.files);
|
||||
@@ -5498,7 +5498,7 @@ describe("httpApiV1 handlers", () => {
|
||||
verdict: "benign",
|
||||
signals: {
|
||||
staticScan: { status: "malicious", rawStatus: "malicious" },
|
||||
dependencyRegistry: { status: "malicious", rawStatus: "malicious" },
|
||||
dependencyRegistry: null,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -664,16 +664,6 @@ type VerifySecurityVersion = {
|
||||
| "checkedAt"
|
||||
> &
|
||||
Partial<Pick<NonNullable<Doc<"skillVersions">["skillSpectorAnalysis"]>, "summary" | "error">>;
|
||||
depRegistryAnalysis?: Pick<
|
||||
NonNullable<Doc<"skillVersions">["depRegistryAnalysis"]>,
|
||||
"status" | "summary" | "checkedAt"
|
||||
> &
|
||||
Partial<
|
||||
Pick<
|
||||
NonNullable<Doc<"skillVersions">["depRegistryAnalysis"]>,
|
||||
"notFoundPackages" | "unresolvedPackages"
|
||||
>
|
||||
>;
|
||||
};
|
||||
|
||||
type SecurityVerdictTargetResult = {
|
||||
@@ -723,9 +713,6 @@ function buildVerifySecurity(version: VerifySecurityVersion) {
|
||||
const skillSpectorStatus = version.skillSpectorAnalysis
|
||||
? normalizeVerificationStatus(version.skillSpectorAnalysis.status)
|
||||
: null;
|
||||
const depStatus = version.depRegistryAnalysis
|
||||
? normalizeVerificationStatus(version.depRegistryAnalysis.status)
|
||||
: null;
|
||||
const status = clawStatus;
|
||||
|
||||
return {
|
||||
@@ -781,16 +768,7 @@ function buildVerifySecurity(version: VerifySecurityVersion) {
|
||||
checkedAt: version.skillSpectorAnalysis.checkedAt ?? null,
|
||||
}
|
||||
: null,
|
||||
dependencyRegistry: version.depRegistryAnalysis
|
||||
? {
|
||||
status: depStatus ?? "pending",
|
||||
rawStatus: version.depRegistryAnalysis.status,
|
||||
summary: version.depRegistryAnalysis.summary ?? null,
|
||||
notFoundPackages: version.depRegistryAnalysis.notFoundPackages ?? [],
|
||||
unresolvedPackages: version.depRegistryAnalysis.unresolvedPackages ?? [],
|
||||
checkedAt: version.depRegistryAnalysis.checkedAt ?? null,
|
||||
}
|
||||
: null,
|
||||
dependencyRegistry: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -857,7 +835,6 @@ function getVerifySecurityCheckedAt(security: ReturnType<typeof buildVerifySecur
|
||||
security.signals.staticScan?.checkedAt,
|
||||
security.signals.virusTotal?.checkedAt,
|
||||
security.signals.skillSpector?.checkedAt,
|
||||
security.signals.dependencyRegistry?.checkedAt,
|
||||
].filter((value): value is number => typeof value === "number");
|
||||
return candidates.length > 0 ? Math.max(...candidates) : null;
|
||||
}
|
||||
@@ -904,14 +881,7 @@ function buildSecurityVerdictSummary(security: ReturnType<typeof buildVerifySecu
|
||||
checkedAt: security.signals.skillSpector.checkedAt,
|
||||
}
|
||||
: null,
|
||||
dependencyRegistry: security.signals.dependencyRegistry
|
||||
? {
|
||||
status: security.signals.dependencyRegistry.status,
|
||||
rawStatus: security.signals.dependencyRegistry.rawStatus,
|
||||
summary: security.signals.dependencyRegistry.summary,
|
||||
checkedAt: security.signals.dependencyRegistry.checkedAt,
|
||||
}
|
||||
: null,
|
||||
dependencyRegistry: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
depRegistryUrl,
|
||||
mergeDepRegistryFinding,
|
||||
parseDependencyFile,
|
||||
summarizeDepRegistryChecks,
|
||||
} from "./depRegistryScan";
|
||||
import { summarizeReasonCodes, verdictFromCodes } from "./moderationReasonCodes";
|
||||
|
||||
describe("depRegistryScan", () => {
|
||||
it("parses registry dependency manifests and skips vendored or non-registry specs", () => {
|
||||
expect(
|
||||
parseDependencyFile(
|
||||
"package.json",
|
||||
JSON.stringify({
|
||||
dependencies: {
|
||||
"@types/node": "^24.0.0",
|
||||
local: "file:../local",
|
||||
remote: "github:owner/repo",
|
||||
},
|
||||
optionalDependencies: {
|
||||
undici: "^7.0.0",
|
||||
},
|
||||
}),
|
||||
),
|
||||
).toEqual([
|
||||
{ name: "@types/node", registry: "npm", source: "package.json" },
|
||||
{ name: "undici", registry: "npm", source: "package.json" },
|
||||
]);
|
||||
|
||||
expect(
|
||||
parseDependencyFile("vendor/package.json", '{"dependencies":{"phantom":"1.0.0"}}'),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
parseDependencyFile(
|
||||
"requirements.txt",
|
||||
["requests>=2", "demo @ git+https://example.test/demo.git", "-r dev.txt"].join("\n"),
|
||||
),
|
||||
).toEqual([{ name: "requests", registry: "pypi", source: "requirements.txt" }]);
|
||||
});
|
||||
|
||||
it("keeps npm scope names compatible with registry URL lookup", () => {
|
||||
expect(depRegistryUrl("npm", "@types/node")).toBe("https://registry.npmjs.org/@types%2Fnode");
|
||||
});
|
||||
|
||||
it("does not produce clean status when registry lookups are unresolved", () => {
|
||||
const analysis = summarizeDepRegistryChecks({
|
||||
checkedAt: 123,
|
||||
results: [{ name: "requests", registry: "pypi", source: "requirements.txt", exists: true }],
|
||||
unresolved: [
|
||||
{
|
||||
name: "maybe-real",
|
||||
registry: "npm",
|
||||
source: "package.json",
|
||||
reason: "network error",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(analysis.status).toBe("error");
|
||||
expect(analysis.notFoundPackages).toEqual([]);
|
||||
expect(analysis.unresolvedPackages).toEqual(["maybe-real (npm)"]);
|
||||
});
|
||||
|
||||
it("injects a static finding only for confirmed missing packages", () => {
|
||||
const suspicious = summarizeDepRegistryChecks({
|
||||
checkedAt: 456,
|
||||
results: [
|
||||
{
|
||||
name: "phantom-package-xyz",
|
||||
registry: "npm",
|
||||
source: "package.json",
|
||||
exists: false,
|
||||
httpStatus: 404,
|
||||
},
|
||||
],
|
||||
unresolved: [],
|
||||
});
|
||||
|
||||
const merged = mergeDepRegistryFinding({
|
||||
staticScan: undefined,
|
||||
analysis: suspicious,
|
||||
statusFromCodes: verdictFromCodes,
|
||||
summarizeCodes: summarizeReasonCodes,
|
||||
});
|
||||
|
||||
expect(merged.status).toBe("suspicious");
|
||||
expect(merged.reasonCodes).toEqual(["suspicious.dep_not_found_on_registry"]);
|
||||
expect(merged.findings[0]?.file).toBe("Dependency manifests");
|
||||
|
||||
const cleanAgain = mergeDepRegistryFinding({
|
||||
staticScan: merged,
|
||||
analysis: summarizeDepRegistryChecks({ checkedAt: 789, results: [], unresolved: [] }),
|
||||
statusFromCodes: verdictFromCodes,
|
||||
summarizeCodes: summarizeReasonCodes,
|
||||
});
|
||||
expect(cleanAgain.status).toBe("clean");
|
||||
expect(cleanAgain.findings).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,321 +0,0 @@
|
||||
import {
|
||||
MODERATION_ENGINE_VERSION,
|
||||
REASON_CODES,
|
||||
type ModerationFinding,
|
||||
type ModerationVerdict,
|
||||
} from "./moderationReasonCodes";
|
||||
|
||||
export const SUPPORTED_DEP_REGISTRIES = ["pypi", "npm", "cargo"] as const;
|
||||
|
||||
export type SupportedDepRegistry = (typeof SUPPORTED_DEP_REGISTRIES)[number];
|
||||
|
||||
export type DepRegistryStatus = "clean" | "suspicious" | "error";
|
||||
|
||||
export type DepEntry = {
|
||||
name: string;
|
||||
registry: SupportedDepRegistry;
|
||||
source: string;
|
||||
};
|
||||
|
||||
export type DepRegistryResult = DepEntry & {
|
||||
exists: boolean;
|
||||
httpStatus?: number;
|
||||
};
|
||||
|
||||
export type DepRegistryUnresolved = DepEntry & {
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export type DepRegistryAnalysis = {
|
||||
status: DepRegistryStatus;
|
||||
results: DepRegistryResult[];
|
||||
notFoundPackages: string[];
|
||||
unresolvedPackages: string[];
|
||||
summary: string;
|
||||
checkedAt: number;
|
||||
};
|
||||
|
||||
const DEP_FILE_PARSERS: Record<string, (content: string, path: string) => DepEntry[]> = {
|
||||
"requirements.txt": parseRequirementsTxt,
|
||||
"requirements-dev.txt": parseRequirementsTxt,
|
||||
"requirements_dev.txt": parseRequirementsTxt,
|
||||
"requirements-test.txt": parseRequirementsTxt,
|
||||
"requirements_test.txt": parseRequirementsTxt,
|
||||
"package.json": parsePackageJson,
|
||||
"cargo.toml": parseCargoToml,
|
||||
"pyproject.toml": parsePyprojectToml,
|
||||
};
|
||||
|
||||
const NON_REGISTRY_NPM_SPEC_PREFIXES = [
|
||||
"file:",
|
||||
"link:",
|
||||
"git+",
|
||||
"git://",
|
||||
"github:",
|
||||
"bitbucket:",
|
||||
"gist:",
|
||||
"http:",
|
||||
"https:",
|
||||
"workspace:",
|
||||
"npm:",
|
||||
];
|
||||
|
||||
const VENDORED_PATH_PATTERNS = [
|
||||
/(^|\/)node_modules\//,
|
||||
/(^|\/)vendor\//,
|
||||
/(^|\/)__pycache__\//,
|
||||
/(^|\/)\.venv\//,
|
||||
/(^|\/)venv\//,
|
||||
/(^|\/)target\//,
|
||||
/(^|\/)\.cargo\//,
|
||||
/(^|\/)dist\//,
|
||||
/(^|\/)build\//,
|
||||
];
|
||||
|
||||
function normalizeName(name: string, registry: SupportedDepRegistry) {
|
||||
const normalized = name.trim().toLowerCase();
|
||||
return registry === "cargo" ? normalized.replaceAll("_", "-") : normalized;
|
||||
}
|
||||
|
||||
export function isVendoredDependencyPath(path: string) {
|
||||
return VENDORED_PATH_PATTERNS.some((pattern) => pattern.test(path));
|
||||
}
|
||||
|
||||
export function parseDependencyFile(path: string, content: string): DepEntry[] {
|
||||
if (isVendoredDependencyPath(path)) return [];
|
||||
const basename = path.split("/").pop()?.toLowerCase() ?? "";
|
||||
const parser = DEP_FILE_PARSERS[basename];
|
||||
return parser ? dedupeDeps(parser(content, path)) : [];
|
||||
}
|
||||
|
||||
export function dedupeDeps(entries: DepEntry[]) {
|
||||
const seen = new Set<string>();
|
||||
return entries.filter((entry) => {
|
||||
const key = `${entry.registry}:${entry.name}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function stripInlineComment(line: string) {
|
||||
return line.replace(/\s+#.*$/, "").trim();
|
||||
}
|
||||
|
||||
function parseRequirementsTxt(content: string, path: string): DepEntry[] {
|
||||
const entries: DepEntry[] = [];
|
||||
for (const rawLine of content.split("\n")) {
|
||||
const line = stripInlineComment(rawLine);
|
||||
if (!line || line.startsWith("-")) continue;
|
||||
if (/^(?:git\+|https?:|file:|\.{0,2}\/)/i.test(line)) continue;
|
||||
if (/\s@\s/.test(line)) continue;
|
||||
const match = line.match(/^([a-zA-Z0-9_][a-zA-Z0-9._-]*)/);
|
||||
if (!match) continue;
|
||||
entries.push({ name: normalizeName(match[1], "pypi"), registry: "pypi", source: path });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function parsePackageJson(content: string, path: string): DepEntry[] {
|
||||
const entries: DepEntry[] = [];
|
||||
let pkg: Record<string, unknown>;
|
||||
try {
|
||||
pkg = JSON.parse(content) as Record<string, unknown>;
|
||||
} catch {
|
||||
return entries;
|
||||
}
|
||||
|
||||
for (const field of ["dependencies", "devDependencies", "optionalDependencies"]) {
|
||||
const deps = pkg[field];
|
||||
if (!deps || typeof deps !== "object" || Array.isArray(deps)) continue;
|
||||
for (const [rawName, rawSpec] of Object.entries(deps as Record<string, unknown>)) {
|
||||
const spec = typeof rawSpec === "string" ? rawSpec.trim().toLowerCase() : "";
|
||||
if (NON_REGISTRY_NPM_SPEC_PREFIXES.some((prefix) => spec.startsWith(prefix))) continue;
|
||||
entries.push({ name: normalizeName(rawName, "npm"), registry: "npm", source: path });
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function parseCargoToml(content: string, path: string): DepEntry[] {
|
||||
const entries: DepEntry[] = [];
|
||||
let inDepSection = false;
|
||||
for (const rawLine of content.split("\n")) {
|
||||
const line = stripInlineComment(rawLine);
|
||||
if (/^\[.*\]$/.test(line)) {
|
||||
const section = line.replace(/[[\]\s]/g, "").toLowerCase();
|
||||
inDepSection =
|
||||
section === "dependencies" ||
|
||||
section === "dev-dependencies" ||
|
||||
section === "build-dependencies";
|
||||
continue;
|
||||
}
|
||||
if (!inDepSection || !line) continue;
|
||||
const match = line.match(/^([a-zA-Z0-9_][a-zA-Z0-9_-]*)\s*=/);
|
||||
if (!match) continue;
|
||||
entries.push({ name: normalizeName(match[1], "cargo"), registry: "cargo", source: path });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function parsePyprojectToml(content: string, path: string): DepEntry[] {
|
||||
const entries: DepEntry[] = [];
|
||||
let inDepArray = false;
|
||||
let inPoetryDepTable = false;
|
||||
for (const rawLine of content.split("\n")) {
|
||||
const line = stripInlineComment(rawLine);
|
||||
if (/^\[.*\]$/.test(line)) {
|
||||
inDepArray = false;
|
||||
const section = line.replace(/[[\]\s]/g, "").toLowerCase();
|
||||
inPoetryDepTable =
|
||||
section === "tool.poetry.dependencies" ||
|
||||
section === "tool.poetry.dev-dependencies" ||
|
||||
section === "tool.poetry.group.dev.dependencies";
|
||||
continue;
|
||||
}
|
||||
if (/^dependencies\s*=\s*\[/.test(line)) {
|
||||
inDepArray = true;
|
||||
const inline = line.match(/\[\s*(.*)\s*\]/);
|
||||
if (inline) {
|
||||
for (const item of extractQuotedStrings(inline[1])) addPyPiDependency(entries, item, path);
|
||||
inDepArray = false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (inDepArray) {
|
||||
if (line === "]") {
|
||||
inDepArray = false;
|
||||
continue;
|
||||
}
|
||||
const quoted = line.match(/^["']([^"']+)["']/);
|
||||
if (quoted) addPyPiDependency(entries, quoted[1], path);
|
||||
continue;
|
||||
}
|
||||
if (!inPoetryDepTable || !line) continue;
|
||||
const match = line.match(/^([a-zA-Z0-9_][a-zA-Z0-9._-]*)\s*=/);
|
||||
if (!match || match[1].toLowerCase() === "python") continue;
|
||||
entries.push({ name: normalizeName(match[1], "pypi"), registry: "pypi", source: path });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
function addPyPiDependency(entries: DepEntry[], spec: string, path: string) {
|
||||
if (/\s@\s/.test(spec)) return;
|
||||
const match = spec.match(/^([a-zA-Z0-9_][a-zA-Z0-9._-]*)/);
|
||||
if (!match) return;
|
||||
entries.push({ name: normalizeName(match[1], "pypi"), registry: "pypi", source: path });
|
||||
}
|
||||
|
||||
function extractQuotedStrings(s: string) {
|
||||
return [...s.matchAll(/["']([^"']+)["']/g)].map((match) => match[1]);
|
||||
}
|
||||
|
||||
export function depRegistryUrl(registry: SupportedDepRegistry, packageName: string) {
|
||||
const encoded =
|
||||
registry === "npm" && packageName.startsWith("@")
|
||||
? `@${encodeURIComponent(packageName.slice(1))}`
|
||||
: encodeURIComponent(packageName);
|
||||
if (registry === "pypi") return `https://pypi.org/pypi/${encoded}/json`;
|
||||
if (registry === "npm") return `https://registry.npmjs.org/${encoded}`;
|
||||
return `https://crates.io/api/v1/crates/${encoded}`;
|
||||
}
|
||||
|
||||
export function summarizeDepRegistryChecks(params: {
|
||||
results: DepRegistryResult[];
|
||||
unresolved: DepRegistryUnresolved[];
|
||||
checkedAt?: number;
|
||||
}): DepRegistryAnalysis {
|
||||
const notFound = params.results.filter((result) => !result.exists);
|
||||
const notFoundPackages = notFound.map((result) => `${result.name} (${result.registry})`);
|
||||
const unresolvedPackages = params.unresolved.map(
|
||||
(result) => `${result.name} (${result.registry})`,
|
||||
);
|
||||
const checkedAt = params.checkedAt ?? Date.now();
|
||||
|
||||
if (notFoundPackages.length > 0) {
|
||||
const partial =
|
||||
unresolvedPackages.length > 0
|
||||
? ` ${unresolvedPackages.length} package(s) could not be checked and will be retried.`
|
||||
: "";
|
||||
return {
|
||||
status: "suspicious",
|
||||
results: params.results,
|
||||
notFoundPackages,
|
||||
unresolvedPackages,
|
||||
summary: `${notFoundPackages.length} declared dependency package(s) were not found on their public registry: ${notFoundPackages.join(", ")}.${partial}`,
|
||||
checkedAt,
|
||||
};
|
||||
}
|
||||
|
||||
if (unresolvedPackages.length > 0) {
|
||||
return {
|
||||
status: "error",
|
||||
results: params.results,
|
||||
notFoundPackages: [],
|
||||
unresolvedPackages,
|
||||
summary: `${unresolvedPackages.length} dependency package(s) could not be verified due to registry lookup errors. The scan will be retried.`,
|
||||
checkedAt,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: "clean",
|
||||
results: params.results,
|
||||
notFoundPackages: [],
|
||||
unresolvedPackages: [],
|
||||
summary: `All ${params.results.length} declared dependency package(s) verified as present on their public registries.`,
|
||||
checkedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildDepRegistryFinding(analysis: DepRegistryAnalysis): ModerationFinding | null {
|
||||
if (analysis.status !== "suspicious" || analysis.notFoundPackages.length === 0) return null;
|
||||
return {
|
||||
code: REASON_CODES.DEP_NOT_FOUND,
|
||||
severity: "critical",
|
||||
file: "Dependency manifests",
|
||||
line: 1,
|
||||
message: `${analysis.notFoundPackages.length} package(s) referenced in dependency files do not exist on their public registries: ${analysis.notFoundPackages.join(", ")}`,
|
||||
evidence:
|
||||
"An attacker could register these phantom package names and inject malicious install-time code through dependency confusion.",
|
||||
};
|
||||
}
|
||||
|
||||
export function mergeDepRegistryFinding(params: {
|
||||
staticScan:
|
||||
| {
|
||||
status: ModerationVerdict;
|
||||
reasonCodes: string[];
|
||||
findings: ModerationFinding[];
|
||||
summary: string;
|
||||
engineVersion: string;
|
||||
checkedAt: number;
|
||||
}
|
||||
| undefined;
|
||||
analysis: DepRegistryAnalysis;
|
||||
statusFromCodes: (codes: string[]) => ModerationVerdict;
|
||||
summarizeCodes: (codes: string[]) => string;
|
||||
}) {
|
||||
const base = params.staticScan ?? {
|
||||
status: "clean" as ModerationVerdict,
|
||||
reasonCodes: [],
|
||||
findings: [],
|
||||
summary: "No suspicious patterns detected.",
|
||||
engineVersion: MODERATION_ENGINE_VERSION,
|
||||
checkedAt: params.analysis.checkedAt,
|
||||
};
|
||||
const findings = base.findings.filter((finding) => finding.code !== REASON_CODES.DEP_NOT_FOUND);
|
||||
const depFinding = buildDepRegistryFinding(params.analysis);
|
||||
if (depFinding) findings.push(depFinding);
|
||||
const reasonCodes = Array.from(new Set(findings.map((finding) => finding.code))).sort((a, b) =>
|
||||
a.localeCompare(b),
|
||||
);
|
||||
return {
|
||||
...base,
|
||||
status: params.statusFromCodes(reasonCodes),
|
||||
reasonCodes,
|
||||
findings,
|
||||
summary: params.summarizeCodes(reasonCodes),
|
||||
checkedAt: params.analysis.checkedAt,
|
||||
};
|
||||
}
|
||||
@@ -45,7 +45,6 @@ export const REASON_CODES = {
|
||||
MALICIOUS_INSTALL_PROMPT: "malicious.install_terminal_payload",
|
||||
KNOWN_BLOCKED_SIGNATURE: "malicious.known_blocked_signature",
|
||||
STEALTH_BROWSER_ABUSE: "malicious.stealth_browser_abuse",
|
||||
DEP_NOT_FOUND: "suspicious.dep_not_found_on_registry",
|
||||
} as const;
|
||||
|
||||
const MALICIOUS_CODES = new Set<string>([
|
||||
|
||||
@@ -365,10 +365,6 @@ export async function publishVersionForUser(
|
||||
source: "publish",
|
||||
});
|
||||
|
||||
await ctx.scheduler.runAfter(0, internal.depRegistryScan.checkDependencyRegistries, {
|
||||
versionId: publishResult.versionId,
|
||||
});
|
||||
|
||||
// Schedule the async "API key required?" analyser; non-fatal on failure
|
||||
// (UI treats `apiKeyRequired === undefined` as "no badge"). Mirrors the
|
||||
// `backupSkillForPublishInternal` pattern below: `void runAfter(...).catch(...)`
|
||||
|
||||
+489
-1
@@ -1,5 +1,9 @@
|
||||
/* @vitest-environment node */
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@convex-dev/auth/server", () => ({
|
||||
getAuthUserId: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./_generated/api", () => ({
|
||||
internal: {
|
||||
@@ -21,6 +25,12 @@ vi.mock("./_generated/api", () => ({
|
||||
backfillSkillFingerprintsInternal: Symbol("backfillSkillFingerprintsInternal"),
|
||||
applySkillCapabilityTagsInternal: Symbol("applySkillCapabilityTagsInternal"),
|
||||
backfillSkillCapabilityTagsInternal: Symbol("backfillSkillCapabilityTagsInternal"),
|
||||
getDependencyRegistryScanCleanupPageInternal: Symbol(
|
||||
"getDependencyRegistryScanCleanupPageInternal",
|
||||
),
|
||||
applyDependencyRegistryScanCleanupInternal: Symbol(
|
||||
"applyDependencyRegistryScanCleanupInternal",
|
||||
),
|
||||
backfillDigestVersionSummary: Symbol("backfillDigestVersionSummary"),
|
||||
getEmptySkillCleanupPageInternal: Symbol("getEmptySkillCleanupPageInternal"),
|
||||
applyEmptySkillCleanupInternal: Symbol("applyEmptySkillCleanupInternal"),
|
||||
@@ -34,6 +44,12 @@ vi.mock("./_generated/api", () => ({
|
||||
getVersionByIdInternal: Symbol("skills.getVersionByIdInternal"),
|
||||
getOwnerSkillActivityInternal: Symbol("skills.getOwnerSkillActivityInternal"),
|
||||
},
|
||||
skillCards: {
|
||||
enqueueForVersionInternal: Symbol("skillCards.enqueueForVersionInternal"),
|
||||
},
|
||||
securityScan: {
|
||||
enqueueSkillVersionScanInternal: Symbol("securityScan.enqueueSkillVersionScanInternal"),
|
||||
},
|
||||
users: {
|
||||
getByIdInternal: Symbol("users.getByIdInternal"),
|
||||
},
|
||||
@@ -45,6 +61,9 @@ vi.mock("./lib/skillSummary", () => ({
|
||||
}));
|
||||
|
||||
const {
|
||||
applyDependencyRegistryScanCleanupInternalHandler,
|
||||
cleanupDependencyRegistryScanDataHandler,
|
||||
cleanupDependencyRegistryScanDataInternalHandler,
|
||||
applySkillCapabilityTagsInternal,
|
||||
backfillDigestVersionSummary,
|
||||
backfillLatestVersionSummaryInternal,
|
||||
@@ -61,11 +80,480 @@ const {
|
||||
} = await import("./maintenance");
|
||||
const { internal } = await import("./_generated/api");
|
||||
const { generateSkillSummary } = await import("./lib/skillSummary");
|
||||
const { getAuthUserId } = await import("@convex-dev/auth/server");
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(getAuthUserId).mockReset();
|
||||
vi.mocked(getAuthUserId).mockResolvedValue(null);
|
||||
});
|
||||
|
||||
function makeBlob(text: string) {
|
||||
return { text: () => Promise.resolve(text) } as unknown as Blob;
|
||||
}
|
||||
|
||||
describe("maintenance dependency registry scan cleanup", () => {
|
||||
it("rejects non-admin public cleanup before scanning cleanup targets", async () => {
|
||||
vi.mocked(getAuthUserId).mockResolvedValue("users:caller" as never);
|
||||
const runQuery = vi.fn(async () => ({
|
||||
_id: "users:caller",
|
||||
role: "user",
|
||||
deletedAt: undefined,
|
||||
deactivatedAt: undefined,
|
||||
}));
|
||||
const runMutation = vi.fn();
|
||||
|
||||
await expect(
|
||||
cleanupDependencyRegistryScanDataHandler({ runQuery, runMutation } as never, {
|
||||
batchSize: 25,
|
||||
}),
|
||||
).rejects.toThrow("Forbidden");
|
||||
|
||||
expect(runQuery).toHaveBeenCalledOnce();
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows admin public cleanup to delegate to the internal cleanup runner", async () => {
|
||||
vi.mocked(getAuthUserId).mockResolvedValue("users:admin" as never);
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("userId" in args) {
|
||||
return {
|
||||
_id: "users:admin",
|
||||
role: "admin",
|
||||
deletedAt: undefined,
|
||||
deactivatedAt: undefined,
|
||||
};
|
||||
}
|
||||
return {
|
||||
versionItems: [],
|
||||
skillItems: [],
|
||||
cacheRowIds: [],
|
||||
versionCursor: null,
|
||||
skillCursor: null,
|
||||
cacheCursor: null,
|
||||
versionsDone: true,
|
||||
skillsDone: true,
|
||||
cacheDone: true,
|
||||
};
|
||||
});
|
||||
const runMutation = vi.fn();
|
||||
|
||||
const result = await cleanupDependencyRegistryScanDataHandler(
|
||||
{ runQuery, runMutation } as never,
|
||||
{ batchSize: 25 },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ ok: true, dryRun: true, isDone: true });
|
||||
expect(runQuery).toHaveBeenCalledTimes(2);
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("dry-runs by default and does not write", async () => {
|
||||
const runQuery = vi.fn(async () => ({
|
||||
versionItems: [
|
||||
{
|
||||
id: "skillVersions:legacy",
|
||||
hasDepRegistryAnalysis: true,
|
||||
hasDepRegistryScanStatus: true,
|
||||
hasLegacyStaticScan: true,
|
||||
},
|
||||
],
|
||||
skillItems: [{ id: "skills:legacy", hasLegacyModeration: true }],
|
||||
cacheRowIds: ["depRegistryCache:1"],
|
||||
versionCursor: null,
|
||||
skillCursor: null,
|
||||
cacheCursor: null,
|
||||
versionsDone: true,
|
||||
skillsDone: true,
|
||||
cacheDone: true,
|
||||
}));
|
||||
const runMutation = vi.fn();
|
||||
|
||||
const result = await cleanupDependencyRegistryScanDataInternalHandler(
|
||||
{ runQuery, runMutation } as never,
|
||||
{ batchSize: 25, maxBatches: 1 },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: true,
|
||||
dryRun: true,
|
||||
stats: {
|
||||
versionRowsScanned: 1,
|
||||
versionRowsMatched: 1,
|
||||
staticScansMatched: 1,
|
||||
skillRowsScanned: 1,
|
||||
skillRowsMatched: 1,
|
||||
cacheRowsScanned: 1,
|
||||
versionRowsPatched: 0,
|
||||
staticScansRewritten: 0,
|
||||
skillRowsPatched: 0,
|
||||
cacheRowsDeleted: 0,
|
||||
},
|
||||
samples: {
|
||||
versionIds: ["skillVersions:legacy"],
|
||||
skillIds: ["skills:legacy"],
|
||||
cacheRowIds: ["depRegistryCache:1"],
|
||||
},
|
||||
isDone: true,
|
||||
});
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("requires the confirmation token before applying", async () => {
|
||||
await expect(
|
||||
cleanupDependencyRegistryScanDataInternalHandler({ runQuery: vi.fn() } as never, {
|
||||
dryRun: false,
|
||||
}),
|
||||
).rejects.toThrow(/delete-dependency-registry-scan-data/);
|
||||
});
|
||||
|
||||
it("applies only matched dependency registry cleanup targets", async () => {
|
||||
const runQuery = vi.fn(async () => ({
|
||||
versionItems: [
|
||||
{
|
||||
id: "skillVersions:legacy",
|
||||
hasDepRegistryAnalysis: true,
|
||||
hasDepRegistryScanStatus: true,
|
||||
hasLegacyStaticScan: false,
|
||||
},
|
||||
{
|
||||
id: "skillVersions:clean",
|
||||
hasDepRegistryAnalysis: false,
|
||||
hasDepRegistryScanStatus: false,
|
||||
hasLegacyStaticScan: false,
|
||||
},
|
||||
],
|
||||
skillItems: [
|
||||
{ id: "skills:legacy", hasLegacyModeration: true },
|
||||
{ id: "skills:clean", hasLegacyModeration: false },
|
||||
],
|
||||
cacheRowIds: ["depRegistryCache:1"],
|
||||
versionCursor: "next-version",
|
||||
skillCursor: "next-skill",
|
||||
cacheCursor: null,
|
||||
versionsDone: false,
|
||||
skillsDone: false,
|
||||
cacheDone: true,
|
||||
}));
|
||||
const runMutation = vi.fn(async () => ({
|
||||
versionRowsPatched: 1,
|
||||
staticScansRewritten: 0,
|
||||
skillRowsPatched: 1,
|
||||
cacheRowsDeleted: 1,
|
||||
}));
|
||||
|
||||
const result = await cleanupDependencyRegistryScanDataInternalHandler(
|
||||
{ runQuery, runMutation } as never,
|
||||
{
|
||||
dryRun: false,
|
||||
confirm: "delete-dependency-registry-scan-data",
|
||||
batchSize: 25,
|
||||
maxBatches: 1,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
dryRun: false,
|
||||
stats: {
|
||||
versionRowsScanned: 2,
|
||||
versionRowsMatched: 1,
|
||||
skillRowsScanned: 2,
|
||||
skillRowsMatched: 1,
|
||||
cacheRowsScanned: 1,
|
||||
versionRowsPatched: 1,
|
||||
skillRowsPatched: 1,
|
||||
cacheRowsDeleted: 1,
|
||||
},
|
||||
cursors: { versionCursor: "next-version", skillCursor: "next-skill", cacheCursor: null },
|
||||
isDone: false,
|
||||
});
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
internal.maintenance.applyDependencyRegistryScanCleanupInternal,
|
||||
{
|
||||
confirm: "delete-dependency-registry-scan-data",
|
||||
versionIds: ["skillVersions:legacy"],
|
||||
skillIds: ["skills:legacy"],
|
||||
cacheRowIds: ["depRegistryCache:1"],
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("requires the confirmation token at the internal write boundary", async () => {
|
||||
const get = vi.fn();
|
||||
const patch = vi.fn();
|
||||
const deleteRow = vi.fn();
|
||||
|
||||
await expect(
|
||||
applyDependencyRegistryScanCleanupInternalHandler(
|
||||
{
|
||||
db: {
|
||||
get,
|
||||
patch,
|
||||
delete: deleteRow,
|
||||
},
|
||||
} as never,
|
||||
{
|
||||
versionIds: ["skillVersions:legacy" as never],
|
||||
skillIds: [],
|
||||
cacheRowIds: ["depRegistryCache:delete-me" as never],
|
||||
} as never,
|
||||
),
|
||||
).rejects.toThrow(/delete-dependency-registry-scan-data/);
|
||||
|
||||
expect(get).not.toHaveBeenCalled();
|
||||
expect(patch).not.toHaveBeenCalled();
|
||||
expect(deleteRow).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("removes only targeted version scan fields and cache rows", async () => {
|
||||
const skillVersions = new Map<string, Record<string, unknown>>([
|
||||
[
|
||||
"skillVersions:legacy",
|
||||
{
|
||||
_id: "skillVersions:legacy",
|
||||
depRegistryAnalysis: { status: "suspicious", missing: [] },
|
||||
depRegistryScanStatus: "suspicious",
|
||||
staticScan: {
|
||||
status: "suspicious",
|
||||
reasonCodes: ["suspicious.dep_not_found_on_registry", "review.too_short"],
|
||||
findings: [
|
||||
{ code: "suspicious.dep_not_found_on_registry", message: "missing package" },
|
||||
{ code: "review.too_short", message: "too short" },
|
||||
],
|
||||
summary: "Detected: missing package",
|
||||
},
|
||||
},
|
||||
],
|
||||
[
|
||||
"skillVersions:untouched",
|
||||
{
|
||||
_id: "skillVersions:untouched",
|
||||
depRegistryAnalysis: { status: "clean", missing: [] },
|
||||
depRegistryScanStatus: "clean",
|
||||
},
|
||||
],
|
||||
]);
|
||||
const depRegistryCache = new Map<string, Record<string, unknown>>([
|
||||
["depRegistryCache:delete-me", { _id: "depRegistryCache:delete-me" }],
|
||||
["depRegistryCache:keep-me", { _id: "depRegistryCache:keep-me" }],
|
||||
]);
|
||||
const tableMap: Record<string, Map<string, Record<string, unknown>>> = {
|
||||
skillVersions,
|
||||
depRegistryCache,
|
||||
};
|
||||
const getTableForId = (id: string) => id.split(":")[0];
|
||||
const patch = vi.fn(async (id: string, value: Record<string, unknown>) => {
|
||||
Object.assign(tableMap[getTableForId(id)].get(id) ?? {}, value);
|
||||
});
|
||||
const deleteRow = vi.fn(async (id: string) => {
|
||||
tableMap[getTableForId(id)].delete(id);
|
||||
});
|
||||
const runAfter = vi.fn();
|
||||
|
||||
const result = await applyDependencyRegistryScanCleanupInternalHandler(
|
||||
{
|
||||
db: {
|
||||
get: vi.fn(async (id: string) => tableMap[getTableForId(id)]?.get(id) ?? null),
|
||||
patch,
|
||||
delete: deleteRow,
|
||||
},
|
||||
scheduler: { runAfter },
|
||||
} as never,
|
||||
{
|
||||
confirm: "delete-dependency-registry-scan-data",
|
||||
versionIds: ["skillVersions:legacy" as never],
|
||||
skillIds: [],
|
||||
cacheRowIds: ["depRegistryCache:delete-me" as never],
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
versionRowsPatched: 1,
|
||||
staticScansRewritten: 1,
|
||||
skillRowsPatched: 0,
|
||||
cacheRowsDeleted: 1,
|
||||
});
|
||||
expect(patch).toHaveBeenCalledWith("skillVersions:legacy", {
|
||||
depRegistryAnalysis: undefined,
|
||||
depRegistryScanStatus: undefined,
|
||||
staticScan: {
|
||||
status: "clean",
|
||||
reasonCodes: ["review.too_short"],
|
||||
findings: [{ code: "review.too_short", message: "too short" }],
|
||||
summary: "Review: review.too_short",
|
||||
},
|
||||
});
|
||||
expect(deleteRow).toHaveBeenCalledWith("depRegistryCache:delete-me");
|
||||
expect(skillVersions.get("skillVersions:untouched")).toMatchObject({
|
||||
depRegistryAnalysis: { status: "clean", missing: [] },
|
||||
depRegistryScanStatus: "clean",
|
||||
});
|
||||
expect(depRegistryCache.has("depRegistryCache:keep-me")).toBe(true);
|
||||
expect(runAfter).toHaveBeenCalledTimes(2);
|
||||
expect(runAfter).toHaveBeenCalledWith(0, internal.skillCards.enqueueForVersionInternal, {
|
||||
versionId: "skillVersions:legacy",
|
||||
source: "scan",
|
||||
});
|
||||
expect(runAfter).toHaveBeenCalledWith(
|
||||
0,
|
||||
internal.securityScan.enqueueSkillVersionScanInternal,
|
||||
{
|
||||
versionId: "skillVersions:legacy",
|
||||
source: "backfill",
|
||||
waitForVtMs: 0,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("recomputes a hidden skill when retired dependency registry moderation was the only finding", async () => {
|
||||
const skill = {
|
||||
_id: "skills:legacy",
|
||||
_creationTime: 100,
|
||||
slug: "legacy",
|
||||
displayName: "Legacy",
|
||||
summary: "Legacy dep registry-only finding.",
|
||||
icon: undefined,
|
||||
ownerUserId: undefined,
|
||||
ownerPublisherId: undefined,
|
||||
canonicalSkillId: undefined,
|
||||
forkOf: undefined,
|
||||
latestVersionId: "skillVersions:legacy",
|
||||
installKind: undefined,
|
||||
githubHasSkillCard: undefined,
|
||||
githubCurrentStatus: undefined,
|
||||
githubScanStatus: undefined,
|
||||
latestVersionSummary: undefined,
|
||||
tags: {},
|
||||
capabilityTags: [],
|
||||
badges: undefined,
|
||||
stats: { downloads: 0, stars: 0, installsCurrent: 0, installsAllTime: 0 },
|
||||
statsDownloads: 0,
|
||||
statsStars: 0,
|
||||
statsInstallsCurrent: 0,
|
||||
statsInstallsAllTime: 0,
|
||||
softDeletedAt: undefined,
|
||||
moderationStatus: "hidden",
|
||||
moderationFlags: ["flagged.suspicious"],
|
||||
moderationReason: "scanner.aggregate.suspicious",
|
||||
moderationVerdict: "suspicious",
|
||||
moderationReasonCodes: ["suspicious.dep_not_found_on_registry"],
|
||||
moderationEvidence: [
|
||||
{
|
||||
code: "suspicious.dep_not_found_on_registry",
|
||||
severity: "warn",
|
||||
message: "Dependency was not found on its registry.",
|
||||
},
|
||||
],
|
||||
moderationSummary: "Suspicious: suspicious.dep_not_found_on_registry",
|
||||
isSuspicious: true,
|
||||
hiddenAt: 111,
|
||||
hiddenBy: "users:moderator",
|
||||
createdAt: 100,
|
||||
updatedAt: 110,
|
||||
};
|
||||
const skillVersions = new Map<string, Record<string, unknown>>([
|
||||
[
|
||||
"skillVersions:legacy",
|
||||
{
|
||||
_id: "skillVersions:legacy",
|
||||
skillId: "skills:legacy",
|
||||
softDeletedAt: undefined,
|
||||
},
|
||||
],
|
||||
]);
|
||||
const skills = new Map<string, Record<string, unknown>>([["skills:legacy", skill]]);
|
||||
const skillSearchDigest = new Map<string, Record<string, unknown>>([
|
||||
["skillSearchDigest:legacy", { _id: "skillSearchDigest:legacy", skillId: "skills:legacy" }],
|
||||
]);
|
||||
const globalStats = new Map<string, Record<string, unknown>>([
|
||||
["globalStats:default", { _id: "globalStats:default", key: "default", activeSkillsCount: 4 }],
|
||||
]);
|
||||
const tables: Record<string, Map<string, Record<string, unknown>>> = {
|
||||
skills,
|
||||
skillVersions,
|
||||
skillSearchDigest,
|
||||
globalStats,
|
||||
depRegistryCache: new Map(),
|
||||
};
|
||||
const getTableForId = (id: string) => id.split(":")[0];
|
||||
const uniqueFromTable = (tableName: string, filters: Record<string, unknown>) => {
|
||||
for (const row of tables[tableName]?.values() ?? []) {
|
||||
const matches = Object.entries(filters).every(([field, value]) => row[field] === value);
|
||||
if (matches) return row;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
const patch = vi.fn(async (id: string, value: Record<string, unknown>) => {
|
||||
Object.assign(tables[getTableForId(id)].get(id) ?? {}, value);
|
||||
});
|
||||
const runAfter = vi.fn();
|
||||
const query = vi.fn((tableName: string) => ({
|
||||
withIndex: (
|
||||
_indexName: string,
|
||||
build: (q: { eq: (field: string, value: unknown) => unknown }) => unknown,
|
||||
) => {
|
||||
const filters: Record<string, unknown> = {};
|
||||
build({
|
||||
eq: (field, value) => {
|
||||
filters[field] = value;
|
||||
return {
|
||||
eq: (nextField: string, nextValue: unknown) => {
|
||||
filters[nextField] = nextValue;
|
||||
return filters;
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
return { unique: async () => uniqueFromTable(tableName, filters) };
|
||||
},
|
||||
}));
|
||||
|
||||
const result = await applyDependencyRegistryScanCleanupInternalHandler(
|
||||
{
|
||||
db: {
|
||||
get: vi.fn(async (id: string) => tables[getTableForId(id)]?.get(id) ?? null),
|
||||
patch,
|
||||
delete: vi.fn(),
|
||||
query,
|
||||
},
|
||||
scheduler: { runAfter },
|
||||
} as never,
|
||||
{
|
||||
confirm: "delete-dependency-registry-scan-data",
|
||||
versionIds: [],
|
||||
skillIds: ["skills:legacy" as never],
|
||||
cacheRowIds: [],
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
versionRowsPatched: 0,
|
||||
staticScansRewritten: 0,
|
||||
skillRowsPatched: 1,
|
||||
cacheRowsDeleted: 0,
|
||||
});
|
||||
expect(skills.get("skills:legacy")).toMatchObject({
|
||||
moderationStatus: "active",
|
||||
moderationReason: "scanner.aggregate.clean",
|
||||
moderationVerdict: "clean",
|
||||
moderationReasonCodes: undefined,
|
||||
moderationEvidence: undefined,
|
||||
moderationFlags: undefined,
|
||||
moderationSummary: "No suspicious patterns detected.",
|
||||
isSuspicious: false,
|
||||
hiddenAt: undefined,
|
||||
hiddenBy: undefined,
|
||||
});
|
||||
expect(skillSearchDigest.get("skillSearchDigest:legacy")).toMatchObject({
|
||||
moderationStatus: "active",
|
||||
moderationReason: "scanner.aggregate.clean",
|
||||
isSuspicious: false,
|
||||
skillId: "skills:legacy",
|
||||
});
|
||||
expect(globalStats.get("globalStats:default")).toMatchObject({ activeSkillsCount: 5 });
|
||||
expect(runAfter).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
type QueryEq = {
|
||||
eq: (field: string, value: unknown) => QueryEq;
|
||||
};
|
||||
|
||||
@@ -4,6 +4,12 @@ import type { Doc, Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx, MutationCtx } from "./_generated/server";
|
||||
import { action, internalAction, internalMutation, internalQuery } from "./functions";
|
||||
import { assertRole, requireUserFromAction } from "./lib/access";
|
||||
import { adjustGlobalPublicSkillsCount, getPublicSkillVisibilityDelta } from "./lib/globalStats";
|
||||
import {
|
||||
legacyFlagsFromVerdict,
|
||||
summarizeReasonCodes,
|
||||
verdictFromCodes,
|
||||
} from "./lib/moderationReasonCodes";
|
||||
import { extractPackageDigestFields, upsertPackageSearchDigest } from "./lib/packageSearchDigest";
|
||||
import {
|
||||
derivePersonalPublisherHandle,
|
||||
@@ -30,6 +36,7 @@ import {
|
||||
extractValidatedDigestFields,
|
||||
getFirstSearchToken,
|
||||
normalizeSkillSearchText,
|
||||
syncSkillSearchDigestForSkill,
|
||||
upsertSkillSearchDigest,
|
||||
} from "./lib/skillSearchDigest";
|
||||
import { generateSkillSummary } from "./lib/skillSummary";
|
||||
@@ -42,6 +49,9 @@ const DEFAULT_EMPTY_SKILL_MAX_README_BYTES = 8000;
|
||||
const DEFAULT_EMPTY_SKILL_NOMINATION_THRESHOLD = 3;
|
||||
const DEFAULT_CAPABILITY_BACKFILL_DELAY_MS = 500;
|
||||
const PLATFORM_SKILL_LICENSE = "MIT-0" as const;
|
||||
const DEP_REGISTRY_CLEANUP_CONFIRMATION = "delete-dependency-registry-scan-data" as const;
|
||||
const RETIRED_DEP_REGISTRY_REASON_CODE = "suspicious.dep_not_found_on_registry";
|
||||
const CLEANUP_SAMPLE_LIMIT = 5;
|
||||
|
||||
type BackfillStats = {
|
||||
skillsScanned: number;
|
||||
@@ -128,6 +138,587 @@ type LegacyPublisherOwnershipForUserRepairResult = Omit<
|
||||
nextPhase?: LegacyPublisherOwnershipTargetPhase;
|
||||
};
|
||||
|
||||
type SkillVersionStaticScan = NonNullable<Doc<"skillVersions">["staticScan"]>;
|
||||
|
||||
type DependencyRegistryCleanupVersionItem = {
|
||||
id: Id<"skillVersions">;
|
||||
hasDepRegistryAnalysis: boolean;
|
||||
hasDepRegistryScanStatus: boolean;
|
||||
hasLegacyStaticScan: boolean;
|
||||
};
|
||||
|
||||
type DependencyRegistryCleanupSkillItem = {
|
||||
id: Id<"skills">;
|
||||
hasLegacyModeration: boolean;
|
||||
};
|
||||
|
||||
type DependencyRegistryCleanupPageResult = {
|
||||
versionItems: DependencyRegistryCleanupVersionItem[];
|
||||
skillItems: DependencyRegistryCleanupSkillItem[];
|
||||
cacheRowIds: Id<"depRegistryCache">[];
|
||||
versionCursor: string | null;
|
||||
skillCursor: string | null;
|
||||
cacheCursor: string | null;
|
||||
versionsDone: boolean;
|
||||
skillsDone: boolean;
|
||||
cacheDone: boolean;
|
||||
};
|
||||
|
||||
type DependencyRegistryCleanupStats = {
|
||||
versionRowsScanned: number;
|
||||
versionRowsMatched: number;
|
||||
staticScansMatched: number;
|
||||
skillRowsScanned: number;
|
||||
skillRowsMatched: number;
|
||||
versionRowsPatched: number;
|
||||
staticScansRewritten: number;
|
||||
skillRowsPatched: number;
|
||||
cacheRowsScanned: number;
|
||||
cacheRowsDeleted: number;
|
||||
};
|
||||
|
||||
type DependencyRegistryCleanupSamples = {
|
||||
versionIds: Id<"skillVersions">[];
|
||||
skillIds: Id<"skills">[];
|
||||
cacheRowIds: Id<"depRegistryCache">[];
|
||||
};
|
||||
|
||||
type DependencyRegistryCleanupActionArgs = {
|
||||
dryRun?: boolean;
|
||||
confirm?: typeof DEP_REGISTRY_CLEANUP_CONFIRMATION;
|
||||
batchSize?: number;
|
||||
maxBatches?: number;
|
||||
versionCursor?: string;
|
||||
skillCursor?: string;
|
||||
cacheCursor?: string;
|
||||
versionsDone?: boolean;
|
||||
skillsDone?: boolean;
|
||||
cacheDone?: boolean;
|
||||
};
|
||||
|
||||
type DependencyRegistryCleanupActionResult = {
|
||||
ok: true;
|
||||
dryRun: boolean;
|
||||
stats: DependencyRegistryCleanupStats;
|
||||
samples: DependencyRegistryCleanupSamples;
|
||||
cursors: {
|
||||
versionCursor: string | null;
|
||||
skillCursor: string | null;
|
||||
cacheCursor: string | null;
|
||||
};
|
||||
done: {
|
||||
versionsDone: boolean;
|
||||
skillsDone: boolean;
|
||||
cacheDone: boolean;
|
||||
};
|
||||
isDone: boolean;
|
||||
};
|
||||
|
||||
type DependencyRegistryCleanupApplyArgs = {
|
||||
confirm: typeof DEP_REGISTRY_CLEANUP_CONFIRMATION;
|
||||
versionIds: Id<"skillVersions">[];
|
||||
skillIds: Id<"skills">[];
|
||||
cacheRowIds: Id<"depRegistryCache">[];
|
||||
};
|
||||
|
||||
function emptyDependencyRegistryCleanupStats(): DependencyRegistryCleanupStats {
|
||||
return {
|
||||
versionRowsScanned: 0,
|
||||
versionRowsMatched: 0,
|
||||
staticScansMatched: 0,
|
||||
skillRowsScanned: 0,
|
||||
skillRowsMatched: 0,
|
||||
versionRowsPatched: 0,
|
||||
staticScansRewritten: 0,
|
||||
skillRowsPatched: 0,
|
||||
cacheRowsScanned: 0,
|
||||
cacheRowsDeleted: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function emptyDependencyRegistryCleanupSamples(): DependencyRegistryCleanupSamples {
|
||||
return {
|
||||
versionIds: [],
|
||||
skillIds: [],
|
||||
cacheRowIds: [],
|
||||
};
|
||||
}
|
||||
|
||||
function pushSample<T>(samples: T[], value: T) {
|
||||
if (samples.length < CLEANUP_SAMPLE_LIMIT) samples.push(value);
|
||||
}
|
||||
|
||||
function stripRetiredDependencyRegistryReasonCodes(codes: string[]) {
|
||||
return codes.filter((code) => code !== RETIRED_DEP_REGISTRY_REASON_CODE);
|
||||
}
|
||||
|
||||
function isScannerMaliciousReason(reason: string | undefined) {
|
||||
return Boolean(reason?.startsWith("scanner.") && reason.endsWith(".malicious"));
|
||||
}
|
||||
|
||||
function isScannerSuspiciousReason(reason: string | undefined) {
|
||||
return Boolean(reason?.startsWith("scanner.") && reason.endsWith(".suspicious"));
|
||||
}
|
||||
|
||||
function isRetainedScannerSuspiciousReason(reason: string | undefined) {
|
||||
return isScannerSuspiciousReason(reason) && reason !== "scanner.aggregate.suspicious";
|
||||
}
|
||||
|
||||
function verdictFromCodesWithScannerReason(params: {
|
||||
reasonCodes: string[];
|
||||
scannerReason?: string;
|
||||
isMalwareBlocked?: boolean;
|
||||
}) {
|
||||
if (params.isMalwareBlocked) return "malicious";
|
||||
const verdict = verdictFromCodes(params.reasonCodes);
|
||||
if (verdict !== "clean") return verdict;
|
||||
return isRetainedScannerSuspiciousReason(params.scannerReason) ? "suspicious" : "clean";
|
||||
}
|
||||
|
||||
function summarizeReasonCodesWithScannerReason(params: {
|
||||
reasonCodes: string[];
|
||||
scannerReason?: string;
|
||||
isMalwareBlocked?: boolean;
|
||||
}) {
|
||||
if (params.isMalwareBlocked && params.reasonCodes.length === 0) {
|
||||
return "Detected: malicious scanner verdict";
|
||||
}
|
||||
if (params.reasonCodes.length === 0 && isRetainedScannerSuspiciousReason(params.scannerReason)) {
|
||||
return `Detected: ${params.scannerReason}`;
|
||||
}
|
||||
return summarizeReasonCodes(params.reasonCodes);
|
||||
}
|
||||
|
||||
function hasLegacyDependencyRegistryStaticScan(
|
||||
staticScan: Doc<"skillVersions">["staticScan"],
|
||||
): staticScan is SkillVersionStaticScan {
|
||||
return Boolean(
|
||||
staticScan &&
|
||||
(staticScan.reasonCodes.includes(RETIRED_DEP_REGISTRY_REASON_CODE) ||
|
||||
staticScan.findings.some((finding) => finding.code === RETIRED_DEP_REGISTRY_REASON_CODE)),
|
||||
);
|
||||
}
|
||||
|
||||
function stripDependencyRegistryStaticScan(staticScan: SkillVersionStaticScan) {
|
||||
if (!hasLegacyDependencyRegistryStaticScan(staticScan)) return staticScan;
|
||||
|
||||
const reasonCodes = stripRetiredDependencyRegistryReasonCodes(staticScan.reasonCodes);
|
||||
const findings = staticScan.findings.filter(
|
||||
(finding) => finding.code !== RETIRED_DEP_REGISTRY_REASON_CODE,
|
||||
);
|
||||
|
||||
return {
|
||||
...staticScan,
|
||||
status: verdictFromCodes(reasonCodes),
|
||||
reasonCodes,
|
||||
findings,
|
||||
summary: summarizeReasonCodes(reasonCodes),
|
||||
};
|
||||
}
|
||||
|
||||
function hasLegacyDependencyRegistrySkillModeration(
|
||||
skill: Pick<Doc<"skills">, "moderationReasonCodes" | "moderationEvidence">,
|
||||
) {
|
||||
return (
|
||||
(skill.moderationReasonCodes ?? []).includes(RETIRED_DEP_REGISTRY_REASON_CODE) ||
|
||||
(skill.moderationEvidence ?? []).some(
|
||||
(finding) => finding.code === RETIRED_DEP_REGISTRY_REASON_CODE,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function isScannerModerationReason(reason: string | undefined) {
|
||||
return reason?.startsWith("scanner.") === true;
|
||||
}
|
||||
|
||||
function moderationReasonFromVerdict(
|
||||
verdict: Doc<"skills">["moderationVerdict"],
|
||||
reasonCodes: readonly string[],
|
||||
) {
|
||||
if (verdict === "clean" && reasonCodes.some((code) => code.startsWith("review."))) {
|
||||
return "scanner.llm.review";
|
||||
}
|
||||
if (verdict === "malicious") return "scanner.aggregate.malicious";
|
||||
if (verdict === "suspicious") return "scanner.aggregate.suspicious";
|
||||
return "scanner.aggregate.clean";
|
||||
}
|
||||
|
||||
function stripDependencyRegistrySkillModeration(
|
||||
skill: Doc<"skills">,
|
||||
now: number,
|
||||
): Partial<Doc<"skills">> | null {
|
||||
if (!hasLegacyDependencyRegistrySkillModeration(skill)) return null;
|
||||
|
||||
const reasonCodes = stripRetiredDependencyRegistryReasonCodes(skill.moderationReasonCodes ?? []);
|
||||
const moderationEvidence = (skill.moderationEvidence ?? []).filter(
|
||||
(finding) => finding.code !== RETIRED_DEP_REGISTRY_REASON_CODE,
|
||||
);
|
||||
if (skill.moderationStatus === "removed") {
|
||||
return {
|
||||
moderationReasonCodes: reasonCodes.length ? reasonCodes : undefined,
|
||||
moderationEvidence: moderationEvidence.length ? moderationEvidence : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const shouldRecomputeScannerState = isScannerModerationReason(skill.moderationReason);
|
||||
if (!shouldRecomputeScannerState) {
|
||||
return {
|
||||
moderationReasonCodes: reasonCodes.length ? reasonCodes : undefined,
|
||||
moderationEvidence: moderationEvidence.length ? moderationEvidence : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const rawIsMalwareBlocked =
|
||||
skill.moderationVerdict === "malicious" ||
|
||||
skill.moderationFlags?.includes("blocked.malware") === true ||
|
||||
isScannerMaliciousReason(skill.moderationReason);
|
||||
const moderationVerdict = verdictFromCodesWithScannerReason({
|
||||
reasonCodes,
|
||||
scannerReason: skill.moderationReason,
|
||||
isMalwareBlocked: rawIsMalwareBlocked,
|
||||
});
|
||||
const moderationFlags: Doc<"skills">["moderationFlags"] = rawIsMalwareBlocked
|
||||
? ["blocked.malware"]
|
||||
: moderationVerdict === "clean" && reasonCodes.some((code) => code.startsWith("review."))
|
||||
? ["flagged.review"]
|
||||
: legacyFlagsFromVerdict(moderationVerdict);
|
||||
const moderationReason =
|
||||
rawIsMalwareBlocked &&
|
||||
skill.moderationReason?.startsWith("scanner.") &&
|
||||
skill.moderationReason.endsWith(".malicious")
|
||||
? skill.moderationReason
|
||||
: moderationVerdict === "suspicious" &&
|
||||
reasonCodes.length === 0 &&
|
||||
isRetainedScannerSuspiciousReason(skill.moderationReason)
|
||||
? skill.moderationReason
|
||||
: moderationReasonFromVerdict(moderationVerdict, reasonCodes);
|
||||
const moderationStatus =
|
||||
rawIsMalwareBlocked || moderationVerdict === "malicious" ? "hidden" : "active";
|
||||
const moderationSummary = summarizeReasonCodesWithScannerReason({
|
||||
reasonCodes,
|
||||
scannerReason: skill.moderationReason,
|
||||
isMalwareBlocked: rawIsMalwareBlocked,
|
||||
});
|
||||
|
||||
const patch: Partial<Doc<"skills">> = {
|
||||
moderationVerdict,
|
||||
moderationReasonCodes: reasonCodes.length ? reasonCodes : undefined,
|
||||
moderationEvidence: moderationEvidence.length ? moderationEvidence : undefined,
|
||||
moderationFlags,
|
||||
moderationSummary,
|
||||
moderationReason,
|
||||
moderationStatus,
|
||||
isSuspicious: computeIsSuspicious({
|
||||
moderationFlags,
|
||||
moderationReason,
|
||||
}),
|
||||
};
|
||||
|
||||
if (moderationStatus === "hidden") {
|
||||
patch.hiddenAt = skill.hiddenAt ?? now;
|
||||
patch.hiddenBy = undefined;
|
||||
} else if (!skill.softDeletedAt) {
|
||||
patch.hiddenAt = undefined;
|
||||
patch.hiddenBy = undefined;
|
||||
}
|
||||
|
||||
return patch;
|
||||
}
|
||||
|
||||
export const getDependencyRegistryScanCleanupPageInternal = internalQuery({
|
||||
args: {
|
||||
versionCursor: v.optional(v.string()),
|
||||
skillCursor: v.optional(v.string()),
|
||||
cacheCursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
skipVersions: v.optional(v.boolean()),
|
||||
skipSkills: v.optional(v.boolean()),
|
||||
skipCache: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args): Promise<DependencyRegistryCleanupPageResult> => {
|
||||
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
|
||||
|
||||
const versionResult = args.skipVersions
|
||||
? { page: [], continueCursor: null, isDone: true }
|
||||
: await ctx.db
|
||||
.query("skillVersions")
|
||||
.order("asc")
|
||||
.paginate({ cursor: args.versionCursor ?? null, numItems: batchSize });
|
||||
const skillResult = args.skipSkills
|
||||
? { page: [], continueCursor: null, isDone: true }
|
||||
: await ctx.db
|
||||
.query("skills")
|
||||
.order("asc")
|
||||
.paginate({ cursor: args.skillCursor ?? null, numItems: batchSize });
|
||||
const cacheResult = args.skipCache
|
||||
? { page: [], continueCursor: null, isDone: true }
|
||||
: await ctx.db
|
||||
.query("depRegistryCache")
|
||||
.order("asc")
|
||||
.paginate({ cursor: args.cacheCursor ?? null, numItems: batchSize });
|
||||
|
||||
return {
|
||||
versionItems: versionResult.page.map((version) => ({
|
||||
id: version._id,
|
||||
hasDepRegistryAnalysis: version.depRegistryAnalysis !== undefined,
|
||||
hasDepRegistryScanStatus: version.depRegistryScanStatus !== undefined,
|
||||
hasLegacyStaticScan: hasLegacyDependencyRegistryStaticScan(version.staticScan),
|
||||
})),
|
||||
skillItems: skillResult.page.map((skill) => ({
|
||||
id: skill._id,
|
||||
hasLegacyModeration: hasLegacyDependencyRegistrySkillModeration(skill),
|
||||
})),
|
||||
cacheRowIds: cacheResult.page.map((row) => row._id),
|
||||
versionCursor: versionResult.continueCursor,
|
||||
skillCursor: skillResult.continueCursor,
|
||||
cacheCursor: cacheResult.continueCursor,
|
||||
versionsDone: versionResult.isDone,
|
||||
skillsDone: skillResult.isDone,
|
||||
cacheDone: cacheResult.isDone,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export async function applyDependencyRegistryScanCleanupInternalHandler(
|
||||
ctx: Pick<MutationCtx, "db"> & { scheduler?: Pick<MutationCtx["scheduler"], "runAfter"> },
|
||||
args: DependencyRegistryCleanupApplyArgs,
|
||||
) {
|
||||
if (args.confirm !== DEP_REGISTRY_CLEANUP_CONFIRMATION) {
|
||||
throw new ConvexError(
|
||||
'Set confirm to "delete-dependency-registry-scan-data" before applying cleanup.',
|
||||
);
|
||||
}
|
||||
|
||||
let versionRowsPatched = 0;
|
||||
let staticScansRewritten = 0;
|
||||
let skillRowsPatched = 0;
|
||||
let cacheRowsDeleted = 0;
|
||||
const now = Date.now();
|
||||
|
||||
for (const versionId of args.versionIds) {
|
||||
const version = await ctx.db.get(versionId);
|
||||
if (!version) continue;
|
||||
|
||||
const patch: Partial<Doc<"skillVersions">> = {};
|
||||
if (version.depRegistryAnalysis !== undefined) {
|
||||
patch.depRegistryAnalysis = undefined;
|
||||
}
|
||||
if (version.depRegistryScanStatus !== undefined) {
|
||||
patch.depRegistryScanStatus = undefined;
|
||||
}
|
||||
if (version.staticScan) {
|
||||
const staticScan = stripDependencyRegistryStaticScan(version.staticScan);
|
||||
if (staticScan !== version.staticScan) {
|
||||
patch.staticScan = staticScan;
|
||||
staticScansRewritten++;
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(patch).length > 0) {
|
||||
await ctx.db.patch(version._id, patch);
|
||||
if (patch.staticScan) {
|
||||
await ctx.scheduler?.runAfter(0, internal.skillCards.enqueueForVersionInternal, {
|
||||
versionId: version._id,
|
||||
source: "scan",
|
||||
});
|
||||
await ctx.scheduler?.runAfter(0, internal.securityScan.enqueueSkillVersionScanInternal, {
|
||||
versionId: version._id,
|
||||
source: "backfill",
|
||||
waitForVtMs: 0,
|
||||
});
|
||||
}
|
||||
versionRowsPatched++;
|
||||
}
|
||||
}
|
||||
|
||||
for (const skillId of args.skillIds) {
|
||||
const skill = await ctx.db.get(skillId);
|
||||
if (!skill) continue;
|
||||
|
||||
const patch = stripDependencyRegistrySkillModeration(skill, now);
|
||||
if (!patch) continue;
|
||||
|
||||
const nextSkill: Doc<"skills"> = { ...skill, ...patch };
|
||||
const visibilityDelta = getPublicSkillVisibilityDelta(skill, nextSkill);
|
||||
await ctx.db.patch(skill._id, patch);
|
||||
await syncSkillSearchDigestForSkill(ctx, nextSkill);
|
||||
await adjustGlobalPublicSkillsCount(ctx, visibilityDelta, now);
|
||||
skillRowsPatched++;
|
||||
}
|
||||
|
||||
for (const cacheRowId of args.cacheRowIds) {
|
||||
const row = await ctx.db.get(cacheRowId);
|
||||
if (!row) continue;
|
||||
await ctx.db.delete(row._id);
|
||||
cacheRowsDeleted++;
|
||||
}
|
||||
|
||||
return { versionRowsPatched, staticScansRewritten, skillRowsPatched, cacheRowsDeleted };
|
||||
}
|
||||
|
||||
export const applyDependencyRegistryScanCleanupInternal = internalMutation({
|
||||
args: {
|
||||
confirm: v.literal(DEP_REGISTRY_CLEANUP_CONFIRMATION),
|
||||
versionIds: v.array(v.id("skillVersions")),
|
||||
skillIds: v.array(v.id("skills")),
|
||||
cacheRowIds: v.array(v.id("depRegistryCache")),
|
||||
},
|
||||
handler: applyDependencyRegistryScanCleanupInternalHandler,
|
||||
});
|
||||
|
||||
export async function cleanupDependencyRegistryScanDataInternalHandler(
|
||||
ctx: ActionCtx,
|
||||
args: DependencyRegistryCleanupActionArgs,
|
||||
): Promise<DependencyRegistryCleanupActionResult> {
|
||||
const dryRun = args.dryRun !== false;
|
||||
if (!dryRun && args.confirm !== DEP_REGISTRY_CLEANUP_CONFIRMATION) {
|
||||
throw new ConvexError(
|
||||
'Set confirm to "delete-dependency-registry-scan-data" before applying cleanup.',
|
||||
);
|
||||
}
|
||||
|
||||
const batchSize = clampInt(args.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_BATCH_SIZE);
|
||||
const maxBatches = clampInt(
|
||||
args.maxBatches ?? (dryRun ? DEFAULT_MAX_BATCHES : 1),
|
||||
1,
|
||||
MAX_MAX_BATCHES,
|
||||
);
|
||||
const stats = emptyDependencyRegistryCleanupStats();
|
||||
const samples = emptyDependencyRegistryCleanupSamples();
|
||||
let versionsDone = args.versionsDone === true;
|
||||
let skillsDone = args.skillsDone === true;
|
||||
let cacheDone = args.cacheDone === true;
|
||||
let versionCursor = versionsDone ? null : (args.versionCursor ?? null);
|
||||
let skillCursor = skillsDone ? null : (args.skillCursor ?? null);
|
||||
let cacheCursor = cacheDone ? null : (args.cacheCursor ?? null);
|
||||
|
||||
for (let batchIndex = 0; batchIndex < maxBatches; batchIndex++) {
|
||||
const page = (await ctx.runQuery(
|
||||
internal.maintenance.getDependencyRegistryScanCleanupPageInternal,
|
||||
{
|
||||
versionCursor: versionCursor ?? undefined,
|
||||
skillCursor: skillCursor ?? undefined,
|
||||
cacheCursor: dryRun ? (cacheCursor ?? undefined) : undefined,
|
||||
batchSize,
|
||||
skipVersions: versionsDone,
|
||||
skipSkills: skillsDone,
|
||||
skipCache: cacheDone,
|
||||
},
|
||||
)) as DependencyRegistryCleanupPageResult;
|
||||
|
||||
stats.versionRowsScanned += page.versionItems.length;
|
||||
stats.skillRowsScanned += page.skillItems.length;
|
||||
stats.cacheRowsScanned += page.cacheRowIds.length;
|
||||
|
||||
const matchedVersionIds = page.versionItems
|
||||
.filter(
|
||||
(item) =>
|
||||
item.hasDepRegistryAnalysis || item.hasDepRegistryScanStatus || item.hasLegacyStaticScan,
|
||||
)
|
||||
.map((item) => item.id);
|
||||
const matchedSkillIds = page.skillItems
|
||||
.filter((item) => item.hasLegacyModeration)
|
||||
.map((item) => item.id);
|
||||
const staticScanMatches = page.versionItems.filter((item) => item.hasLegacyStaticScan).length;
|
||||
|
||||
stats.versionRowsMatched += matchedVersionIds.length;
|
||||
stats.staticScansMatched += staticScanMatches;
|
||||
stats.skillRowsMatched += matchedSkillIds.length;
|
||||
for (const id of matchedVersionIds) pushSample(samples.versionIds, id);
|
||||
for (const id of matchedSkillIds) pushSample(samples.skillIds, id);
|
||||
for (const id of page.cacheRowIds) pushSample(samples.cacheRowIds, id);
|
||||
|
||||
if (
|
||||
!dryRun &&
|
||||
(matchedVersionIds.length > 0 || matchedSkillIds.length > 0 || page.cacheRowIds.length > 0)
|
||||
) {
|
||||
const result = (await ctx.runMutation(
|
||||
internal.maintenance.applyDependencyRegistryScanCleanupInternal,
|
||||
{
|
||||
confirm: DEP_REGISTRY_CLEANUP_CONFIRMATION,
|
||||
versionIds: matchedVersionIds,
|
||||
skillIds: matchedSkillIds,
|
||||
cacheRowIds: page.cacheRowIds,
|
||||
},
|
||||
)) as {
|
||||
versionRowsPatched: number;
|
||||
staticScansRewritten: number;
|
||||
skillRowsPatched: number;
|
||||
cacheRowsDeleted: number;
|
||||
};
|
||||
stats.versionRowsPatched += result.versionRowsPatched;
|
||||
stats.staticScansRewritten += result.staticScansRewritten;
|
||||
stats.skillRowsPatched += result.skillRowsPatched;
|
||||
stats.cacheRowsDeleted += result.cacheRowsDeleted;
|
||||
}
|
||||
|
||||
versionCursor = page.versionCursor;
|
||||
skillCursor = page.skillCursor;
|
||||
versionsDone = page.versionsDone;
|
||||
skillsDone = page.skillsDone;
|
||||
if (dryRun) {
|
||||
cacheCursor = page.cacheCursor;
|
||||
cacheDone = page.cacheDone;
|
||||
} else {
|
||||
cacheCursor = null;
|
||||
cacheDone = page.cacheDone;
|
||||
}
|
||||
|
||||
if (versionsDone && skillsDone && cacheDone) break;
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
dryRun,
|
||||
stats,
|
||||
samples,
|
||||
cursors: { versionCursor, skillCursor, cacheCursor },
|
||||
done: { versionsDone, skillsDone, cacheDone },
|
||||
isDone: versionsDone && skillsDone && cacheDone,
|
||||
};
|
||||
}
|
||||
|
||||
export const cleanupDependencyRegistryScanDataInternal = internalAction({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
confirm: v.optional(v.literal(DEP_REGISTRY_CLEANUP_CONFIRMATION)),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
versionCursor: v.optional(v.string()),
|
||||
skillCursor: v.optional(v.string()),
|
||||
cacheCursor: v.optional(v.string()),
|
||||
versionsDone: v.optional(v.boolean()),
|
||||
skillsDone: v.optional(v.boolean()),
|
||||
cacheDone: v.optional(v.boolean()),
|
||||
},
|
||||
handler: cleanupDependencyRegistryScanDataInternalHandler,
|
||||
});
|
||||
|
||||
export async function cleanupDependencyRegistryScanDataHandler(
|
||||
ctx: ActionCtx,
|
||||
args: DependencyRegistryCleanupActionArgs,
|
||||
): Promise<DependencyRegistryCleanupActionResult> {
|
||||
const { user } = await requireUserFromAction(ctx);
|
||||
assertRole(user, ["admin"]);
|
||||
return cleanupDependencyRegistryScanDataInternalHandler(ctx, args);
|
||||
}
|
||||
|
||||
export const cleanupDependencyRegistryScanData: ReturnType<typeof action> = action({
|
||||
args: {
|
||||
dryRun: v.optional(v.boolean()),
|
||||
confirm: v.optional(v.literal(DEP_REGISTRY_CLEANUP_CONFIRMATION)),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxBatches: v.optional(v.number()),
|
||||
versionCursor: v.optional(v.string()),
|
||||
skillCursor: v.optional(v.string()),
|
||||
cacheCursor: v.optional(v.string()),
|
||||
versionsDone: v.optional(v.boolean()),
|
||||
skillsDone: v.optional(v.boolean()),
|
||||
cacheDone: v.optional(v.boolean()),
|
||||
},
|
||||
handler: cleanupDependencyRegistryScanDataHandler,
|
||||
});
|
||||
|
||||
export const getSkillBackfillPageInternal = internalQuery({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
|
||||
@@ -199,14 +199,6 @@ describe("skillCards queue", () => {
|
||||
engineVersion: "test",
|
||||
checkedAt: 1,
|
||||
},
|
||||
depRegistryAnalysis: {
|
||||
status: "suspicious",
|
||||
results: [],
|
||||
notFoundPackages: ["leftpad"],
|
||||
unresolvedPackages: [],
|
||||
summary: "raw dependency detail",
|
||||
checkedAt: 3,
|
||||
},
|
||||
vtAnalysis: {
|
||||
status: "suspicious",
|
||||
verdict: "suspicious",
|
||||
|
||||
@@ -34,7 +34,6 @@ import {
|
||||
import { getSkillBadgeMap, getSkillBadgeMaps, isSkillHighlighted } from "./lib/badges";
|
||||
import { scheduleNextBatchIfNeeded } from "./lib/batching";
|
||||
import { generateChangelogPreview as buildChangelogPreview } from "./lib/changelog";
|
||||
import { mergeDepRegistryFinding } from "./lib/depRegistryScan";
|
||||
import { embeddingVisibilityFor } from "./lib/embeddingVisibility";
|
||||
import {
|
||||
canHealSkillOwnershipByGitHubProviderAccountId,
|
||||
@@ -228,31 +227,6 @@ const skillSpectorAnalysisValidator = v.object({
|
||||
checkedAt: v.number(),
|
||||
});
|
||||
|
||||
const depRegistryStatusValidator = v.union(
|
||||
v.literal("clean"),
|
||||
v.literal("suspicious"),
|
||||
v.literal("error"),
|
||||
);
|
||||
|
||||
const depRegistryValidator = v.union(v.literal("pypi"), v.literal("npm"), v.literal("cargo"));
|
||||
|
||||
const depRegistryAnalysisValidator = v.object({
|
||||
status: depRegistryStatusValidator,
|
||||
results: v.array(
|
||||
v.object({
|
||||
name: v.string(),
|
||||
registry: depRegistryValidator,
|
||||
source: v.string(),
|
||||
exists: v.boolean(),
|
||||
httpStatus: v.optional(v.number()),
|
||||
}),
|
||||
),
|
||||
notFoundPackages: v.array(v.string()),
|
||||
unresolvedPackages: v.array(v.string()),
|
||||
summary: v.string(),
|
||||
checkedAt: v.number(),
|
||||
});
|
||||
|
||||
function buildStructuredModerationPatch(params: {
|
||||
staticScan?: Doc<"skillVersions">["staticScan"];
|
||||
vtAnalysis?: Doc<"skillVersions">["vtAnalysis"];
|
||||
@@ -3046,15 +3020,6 @@ function compactSecurityVerdictVersion(version: Doc<"skillVersions">) {
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(version.depRegistryAnalysis
|
||||
? {
|
||||
depRegistryAnalysis: {
|
||||
status: version.depRegistryAnalysis.status,
|
||||
summary: version.depRegistryAnalysis.summary,
|
||||
checkedAt: version.depRegistryAnalysis.checkedAt,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7138,36 +7103,6 @@ export const updateSkillVersionStaticScanInternal = internalMutation({
|
||||
},
|
||||
});
|
||||
|
||||
export const updateVersionDepRegistryAnalysisInternal = internalMutation({
|
||||
args: {
|
||||
versionId: v.id("skillVersions"),
|
||||
depRegistryAnalysis: depRegistryAnalysisValidator,
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const version = await ctx.db.get(args.versionId);
|
||||
if (!version) return { ok: true as const, skipped: "missing" as const };
|
||||
|
||||
const staticScan = mergeDepRegistryFinding({
|
||||
staticScan: version.staticScan,
|
||||
analysis: args.depRegistryAnalysis,
|
||||
statusFromCodes: verdictFromCodes,
|
||||
summarizeCodes: summarizeReasonCodes,
|
||||
});
|
||||
const versionPatch = {
|
||||
depRegistryAnalysis: args.depRegistryAnalysis,
|
||||
depRegistryScanStatus: args.depRegistryAnalysis.status,
|
||||
staticScan,
|
||||
};
|
||||
|
||||
await ctx.db.patch(version._id, versionPatch);
|
||||
await ctx.scheduler?.runAfter(0, internal.skillCards.enqueueForVersionInternal, {
|
||||
versionId: version._id,
|
||||
source: "scan",
|
||||
});
|
||||
return { ok: true as const, status: args.depRegistryAnalysis.status };
|
||||
},
|
||||
});
|
||||
|
||||
export const scanSkillVersionStaticallyInternal: ReturnType<typeof internalAction> = internalAction(
|
||||
{
|
||||
args: {
|
||||
|
||||
+3
-1
@@ -440,7 +440,8 @@ Notes:
|
||||
- `ok` is `true` only when the selected version has a generated Skill Card, is not malware-blocked by moderation, and ClawScan verification is clean.
|
||||
- Skill identity, publisher identity, and selected version metadata are top-level envelope fields (`slug`, `displayName`, `publisherHandle`, `version`, `resolvedFrom`, `tag`, `createdAt`) so shell automation can read them without unpacking nested wrappers.
|
||||
- `security` is the top-level ClawScan/security verdict. Automation should key off `ok`, `decision`, `reasons`, and `security.status`.
|
||||
- `security.signals` contains supporting scanner evidence such as `staticScan`, `virusTotal`, `skillSpector`, and `dependencyRegistry`.
|
||||
- `security.signals` contains supporting scanner evidence such as `staticScan`, `virusTotal`, and `skillSpector`.
|
||||
- `security.signals.dependencyRegistry` is retained for v1 response compatibility, but the dependency registry existence scanner is retired and this key is always `null`.
|
||||
- `provenance` is `server-resolved-github-import` only when ClawHub resolved and stored a GitHub repo/ref/commit/path during publish or import; otherwise it is `unavailable`.
|
||||
|
||||
### `POST /api/v1/skills/-/security-verdicts`
|
||||
@@ -463,6 +464,7 @@ Notes:
|
||||
- Results are per item; one missing skill or version does not fail the whole response.
|
||||
- The response is security-only. It does not include Skill Card data, generated card status, artifact file lists, or detailed scanner payloads.
|
||||
- `security.signals` contains status-level supporting evidence only; use `/scan` or the ClawHub security-audit page for full scanner details.
|
||||
- `security.signals.dependencyRegistry` is retained for v1 response compatibility, but the dependency registry existence scanner is retired and this key is always `null`.
|
||||
- Skill Card absence does not affect this endpoint's `ok`, `decision`, or `reasons`; clients should read installed `skill-card.md` locally when they need card content.
|
||||
- Use `/verify` when you need the single-skill Skill Card verification envelope, `/card` when you need generated card markdown, and `/scan` when you need detailed scanner data.
|
||||
|
||||
|
||||
@@ -1107,7 +1107,7 @@ describe("clawhub e2e", () => {
|
||||
recommendation: "SAFE",
|
||||
issueCount: 0,
|
||||
},
|
||||
dependencyRegistry: { status: "clean" },
|
||||
dependencyRegistry: null,
|
||||
},
|
||||
},
|
||||
signature: { status: "unsigned" },
|
||||
|
||||
Reference in New Issue
Block a user