mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 08:52:21 +00:00
fix: harden API rate limits (#2783)
This commit is contained in:
@@ -8,6 +8,7 @@
|
||||
"@auth/core": "0.41.2",
|
||||
"@convex-dev/auth": "0.0.94",
|
||||
"@convex-dev/migrations": "0.3.5",
|
||||
"@convex-dev/rate-limiter": "0.3.2",
|
||||
"@fontsource/bricolage-grotesque": "5.2.10",
|
||||
"@fontsource/ibm-plex-mono": "5.2.7",
|
||||
"@fontsource/manrope": "5.2.8",
|
||||
@@ -226,6 +227,8 @@
|
||||
|
||||
"@convex-dev/migrations": ["@convex-dev/migrations@0.3.5", "", { "peerDependencies": { "convex": "^1.35.0", "convex-helpers": "^0.1.115" } }, "sha512-vj5qjY5XB8laX9WvvxeIFZe8sNW8DGvJ5cPo1zvpjJzYnPLqs5C+VG9HP6sSkEW2QGhsfYOZv56FUNkHiLe1wA=="],
|
||||
|
||||
"@convex-dev/rate-limiter": ["@convex-dev/rate-limiter@0.3.2", "", { "peerDependencies": { "convex": "^1.24.8", "react": "^18.2.0 || ^19.0.0" }, "optionalPeers": ["react"] }, "sha512-+oBPsBfFbzdxiF/9XaaTQmVnvDlvEfg/c69/v8LxTbw4VLuiflIKlfnPQL8OS0azXQQ11hcPWHmU8ytFmHKDXA=="],
|
||||
|
||||
"@csstools/color-helpers": ["@csstools/color-helpers@6.0.2", "", {}, "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q=="],
|
||||
|
||||
"@csstools/css-calc": ["@csstools/css-calc@3.2.1", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-DtdHlgXh5ZkA43cwBcAm+huzgJiwx3ZTWVjBs94kwz2xKqSimDA3lBgCjphYgwgVUMWatSM0pDd8TILB1yrVVg=="],
|
||||
|
||||
Vendored
+5
-2
@@ -73,7 +73,9 @@ import type * as lib_githubSkillScans from "../lib/githubSkillScans.js";
|
||||
import type * as lib_githubSkillSync from "../lib/githubSkillSync.js";
|
||||
import type * as lib_globalStats from "../lib/globalStats.js";
|
||||
import type * as lib_httpHeaders from "../lib/httpHeaders.js";
|
||||
import type * as lib_httpPathSegments from "../lib/httpPathSegments.js";
|
||||
import type * as lib_httpRateLimit from "../lib/httpRateLimit.js";
|
||||
import type * as lib_httpRouteRateLimit from "../lib/httpRouteRateLimit.js";
|
||||
import type * as lib_httpUtils from "../lib/httpUtils.js";
|
||||
import type * as lib_installResolver from "../lib/installResolver.js";
|
||||
import type * as lib_leaderboards from "../lib/leaderboards.js";
|
||||
@@ -95,7 +97,6 @@ import type * as lib_publisherAbuseScoring from "../lib/publisherAbuseScoring.js
|
||||
import type * as lib_publisherCatalogDisplay from "../lib/publisherCatalogDisplay.js";
|
||||
import type * as lib_publisherStats from "../lib/publisherStats.js";
|
||||
import type * as lib_publishers from "../lib/publishers.js";
|
||||
import type * as lib_rateLimitConfig from "../lib/rateLimitConfig.js";
|
||||
import type * as lib_recommendationScore from "../lib/recommendationScore.js";
|
||||
import type * as lib_reporting from "../lib/reporting.js";
|
||||
import type * as lib_reservedHandles from "../lib/reservedHandles.js";
|
||||
@@ -226,7 +227,9 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/githubSkillSync": typeof lib_githubSkillSync;
|
||||
"lib/globalStats": typeof lib_globalStats;
|
||||
"lib/httpHeaders": typeof lib_httpHeaders;
|
||||
"lib/httpPathSegments": typeof lib_httpPathSegments;
|
||||
"lib/httpRateLimit": typeof lib_httpRateLimit;
|
||||
"lib/httpRouteRateLimit": typeof lib_httpRouteRateLimit;
|
||||
"lib/httpUtils": typeof lib_httpUtils;
|
||||
"lib/installResolver": typeof lib_installResolver;
|
||||
"lib/leaderboards": typeof lib_leaderboards;
|
||||
@@ -248,7 +251,6 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/publisherCatalogDisplay": typeof lib_publisherCatalogDisplay;
|
||||
"lib/publisherStats": typeof lib_publisherStats;
|
||||
"lib/publishers": typeof lib_publishers;
|
||||
"lib/rateLimitConfig": typeof lib_rateLimitConfig;
|
||||
"lib/recommendationScore": typeof lib_recommendationScore;
|
||||
"lib/reporting": typeof lib_reporting;
|
||||
"lib/reservedHandles": typeof lib_reservedHandles;
|
||||
@@ -336,4 +338,5 @@ export declare const internal: FilterApi<
|
||||
|
||||
export declare const components: {
|
||||
migrations: import("@convex-dev/migrations/_generated/component.js").ComponentApi<"migrations">;
|
||||
rateLimiter: import("@convex-dev/rate-limiter/_generated/component.js").ComponentApi<"rateLimiter">;
|
||||
};
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import migrations from "@convex-dev/migrations/convex.config.js";
|
||||
import rateLimiter from "@convex-dev/rate-limiter/convex.config.js";
|
||||
import { defineApp } from "convex/server";
|
||||
|
||||
const app = defineApp();
|
||||
app.use(migrations);
|
||||
app.use(rateLimiter);
|
||||
|
||||
export default app;
|
||||
|
||||
@@ -5,7 +5,7 @@ const mocks = vi.hoisted(() => {
|
||||
const interval = vi.fn();
|
||||
const githubSkillSyncRef = Symbol("github-skill-source-sync");
|
||||
const installTelemetryDedupePruneRef = Symbol("install-telemetry-dedupe-prune");
|
||||
const rateLimitCountersPruneRef = Symbol("rate-limit-counters-prune");
|
||||
const httpRateLimitKeysPruneRef = Symbol("http-rate-limit-keys-prune");
|
||||
const skillStatEventPruneRef = Symbol("skill-stat-event-prune");
|
||||
const packageStatEventPruneRef = Symbol("package-stat-event-prune");
|
||||
const authSessionsPruneRef = Symbol("auth-sessions-prune");
|
||||
@@ -14,7 +14,7 @@ const mocks = vi.hoisted(() => {
|
||||
interval,
|
||||
githubSkillSyncRef,
|
||||
installTelemetryDedupePruneRef,
|
||||
rateLimitCountersPruneRef,
|
||||
httpRateLimitKeysPruneRef,
|
||||
skillStatEventPruneRef,
|
||||
packageStatEventPruneRef,
|
||||
authSessionsPruneRef,
|
||||
@@ -63,7 +63,7 @@ vi.mock("./_generated/api", () => ({
|
||||
pruneInstallTelemetryDedupesInternal: mocks.installTelemetryDedupePruneRef,
|
||||
},
|
||||
rateLimits: {
|
||||
pruneRateLimitCountersInternal: mocks.rateLimitCountersPruneRef,
|
||||
pruneHttpRateLimitKeysInternal: mocks.httpRateLimitKeysPruneRef,
|
||||
},
|
||||
retention: {
|
||||
pruneExpiredAuthSessionsInternal: mocks.authSessionsPruneRef,
|
||||
@@ -124,13 +124,13 @@ describe("crons", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("prunes expired rate limit counters frequently", async () => {
|
||||
it("prunes stale component HTTP rate limit keys hourly", async () => {
|
||||
await import("./crons");
|
||||
|
||||
expect(mocks.interval).toHaveBeenCalledWith(
|
||||
"rate-limit-counters-prune",
|
||||
{ minutes: 15 },
|
||||
mocks.rateLimitCountersPruneRef,
|
||||
"http-rate-limit-keys-prune",
|
||||
{ hours: 1 },
|
||||
mocks.httpRateLimitKeysPruneRef,
|
||||
{ batchSize: 500 },
|
||||
);
|
||||
});
|
||||
|
||||
+3
-3
@@ -155,9 +155,9 @@ if (process.env.CLAWHUB_DISABLE_CRONS !== "1") {
|
||||
);
|
||||
|
||||
crons.interval(
|
||||
"rate-limit-counters-prune",
|
||||
{ minutes: 15 },
|
||||
internal.rateLimits.pruneRateLimitCountersInternal,
|
||||
"http-rate-limit-keys-prune",
|
||||
{ hours: 1 },
|
||||
internal.rateLimits.pruneHttpRateLimitKeysInternal,
|
||||
{ batchSize: RETENTION_STANDARD_BATCH_SIZE },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -89,6 +89,7 @@ function makeDb(
|
||||
describe("download metric helpers", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("uses a day bucket for download dedupe", () => {
|
||||
@@ -97,6 +98,7 @@ describe("download metric helpers", () => {
|
||||
});
|
||||
|
||||
it("prefers user identity and falls back to IP identity", () => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
const request = new Request("https://example.com", {
|
||||
headers: { "cf-connecting-ip": "203.0.113.10" },
|
||||
});
|
||||
|
||||
+16
-18
@@ -1,24 +1,25 @@
|
||||
import type { RateLimitArgs, RateLimitReturns } from "@convex-dev/rate-limiter";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { __test, downloadZipHandler } from "./downloads";
|
||||
|
||||
type RateLimitArgs = { key: string; limit: number; windowMs: number };
|
||||
|
||||
function isRateLimitArgs(args: unknown): args is RateLimitArgs {
|
||||
if (!args || typeof args !== "object") return false;
|
||||
const value = args as Record<string, unknown>;
|
||||
const config = value.config as Record<string, unknown> | undefined;
|
||||
return (
|
||||
typeof value.key === "string" &&
|
||||
typeof value.limit === "number" &&
|
||||
typeof value.windowMs === "number"
|
||||
typeof value.name === "string" &&
|
||||
(!("key" in value) || typeof value.key === "string") &&
|
||||
!!config &&
|
||||
typeof config === "object" &&
|
||||
(config.kind === "fixed window" || config.kind === "token bucket") &&
|
||||
typeof config.rate === "number" &&
|
||||
typeof config.period === "number"
|
||||
);
|
||||
}
|
||||
|
||||
const okRate = () => ({
|
||||
allowed: true,
|
||||
remaining: 10,
|
||||
limit: 100,
|
||||
resetAt: Date.now() + 60_000,
|
||||
const okRate = (): RateLimitReturns => ({
|
||||
ok: true,
|
||||
});
|
||||
|
||||
function stubZipResponse() {
|
||||
@@ -55,7 +56,8 @@ describe("downloads helpers", () => {
|
||||
expect(__test.getDownloadIdentityValue(request, "users_123")).toBe("user:users_123");
|
||||
});
|
||||
|
||||
it("uses cf-connecting-ip for anonymous identity", () => {
|
||||
it("uses cf-connecting-ip for anonymous identity when trusted headers are enabled", () => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
const request = new Request("https://example.com", {
|
||||
headers: { "cf-connecting-ip": "1.2.3.4" },
|
||||
});
|
||||
@@ -76,10 +78,10 @@ describe("downloads helpers", () => {
|
||||
});
|
||||
|
||||
it("schedules zip download stats outside the response path", async () => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
stubZipResponse();
|
||||
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if ("slug" in args) {
|
||||
return {
|
||||
skill: {
|
||||
@@ -152,7 +154,6 @@ describe("downloads helpers", () => {
|
||||
|
||||
it("threads owner handle through the skill lookup", async () => {
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if ("slug" in args) {
|
||||
return {
|
||||
skill: {
|
||||
@@ -202,7 +203,6 @@ describe("downloads helpers", () => {
|
||||
|
||||
it("does not serve a tag that points at another skill's version", async () => {
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if ("slug" in args) {
|
||||
return {
|
||||
skill: {
|
||||
@@ -262,7 +262,6 @@ describe("downloads helpers", () => {
|
||||
|
||||
it("returns ownerHandle guidance when a slug-only download is ambiguous", async () => {
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if ("slug" in args) return { skill: null, ambiguous: true };
|
||||
return null;
|
||||
});
|
||||
@@ -290,7 +289,6 @@ describe("downloads helpers", () => {
|
||||
|
||||
it("blocks the exact requested skill version when its ClawScan verdict is malicious", async () => {
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if ("slug" in args) {
|
||||
return {
|
||||
skill: {
|
||||
@@ -369,7 +367,6 @@ describe("downloads helpers", () => {
|
||||
stubZipResponse();
|
||||
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if ("tokenHash" in args) {
|
||||
return { _id: "apiTokens:1", revokedAt: undefined };
|
||||
}
|
||||
@@ -435,10 +432,10 @@ describe("downloads helpers", () => {
|
||||
});
|
||||
|
||||
it("returns zip downloads when download metering is scheduled", async () => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
stubZipResponse();
|
||||
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
if ("slug" in args) {
|
||||
return {
|
||||
skill: {
|
||||
@@ -497,6 +494,7 @@ describe("downloads helpers", () => {
|
||||
it.each(["clean", "suspicious"] as const)(
|
||||
"returns a metered public GitHub handoff descriptor for %s scan without scan metadata",
|
||||
async (scanStatus) => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
const commit = "1".repeat(40);
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
import { ApiRoutes, LegacyApiRoutes } from "clawhub-schema";
|
||||
/* @vitest-environment node */
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { vi } from "vitest";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import http from "./http";
|
||||
import { RATE_LIMITS } from "./lib/httpRateLimit";
|
||||
|
||||
type WrappedHttpAction = {
|
||||
_handler: (ctx: ActionCtx, request: Request) => Promise<Response>;
|
||||
};
|
||||
|
||||
type RateLimitBucketName = `${"read" | "write" | "trustedPublish" | "download" | "export"}Ip`;
|
||||
|
||||
function makeDeniedRateLimitCtx() {
|
||||
const runQuery = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
|
||||
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`);
|
||||
});
|
||||
const runMutation = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
|
||||
if ("name" in args && "config" in args) {
|
||||
const config = args.config as { period: number };
|
||||
return { ok: false, retryAfter: config.period };
|
||||
}
|
||||
throw new Error(`Unexpected runMutation args: ${JSON.stringify(args)}`);
|
||||
});
|
||||
return {
|
||||
ctx: {
|
||||
runQuery,
|
||||
runMutation,
|
||||
} as unknown as ActionCtx,
|
||||
runQuery,
|
||||
runMutation,
|
||||
};
|
||||
}
|
||||
|
||||
function makeAllowedRateLimitCtx() {
|
||||
const runQuery = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
|
||||
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`);
|
||||
});
|
||||
const runMutation = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
|
||||
if ("name" in args && "config" in args) {
|
||||
return { ok: true };
|
||||
}
|
||||
throw new Error(`Unexpected runMutation args: ${JSON.stringify(args)}`);
|
||||
});
|
||||
return {
|
||||
ctx: {
|
||||
runQuery,
|
||||
runMutation,
|
||||
runAction: vi.fn(async () => []),
|
||||
} as unknown as ActionCtx,
|
||||
runMutation,
|
||||
};
|
||||
}
|
||||
|
||||
async function expectRouteUsesIpBucket({
|
||||
path,
|
||||
method,
|
||||
requestUrl = `https://example.com${path}`,
|
||||
bucket,
|
||||
rate,
|
||||
}: {
|
||||
path: string;
|
||||
method: "GET" | "POST";
|
||||
requestUrl?: string;
|
||||
bucket: RateLimitBucketName;
|
||||
rate: number;
|
||||
}) {
|
||||
const route = http.lookup(path, method);
|
||||
if (!route) throw new Error(`Expected route for ${method} ${path}`);
|
||||
const [action] = route;
|
||||
const { ctx, runMutation } = makeDeniedRateLimitCtx();
|
||||
|
||||
const response = await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request(requestUrl, { method }),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(429);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
name: bucket,
|
||||
config: expect.objectContaining({ rate }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
describe("HTTP route rate limit defaults", () => {
|
||||
it("registers package version downloads behind the download limit", async () => {
|
||||
await expectRouteUsesIpBucket({
|
||||
path: "/api/v1/packages/demo/versions/1.0.0/download",
|
||||
method: "GET",
|
||||
bucket: "downloadIp",
|
||||
rate: RATE_LIMITS.download.ip,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["legacy download", LegacyApiRoutes.download, "downloadIp", RATE_LIMITS.download.ip],
|
||||
["plugins export", ApiRoutes.pluginsExport, "exportIp", RATE_LIMITS.export.ip],
|
||||
[
|
||||
"package inspector artifact",
|
||||
"/api/v1/package-inspector/artifact",
|
||||
"downloadIp",
|
||||
RATE_LIMITS.download.ip,
|
||||
],
|
||||
[
|
||||
"package artifact download",
|
||||
"/api/v1/packages/demo/versions/1.0.0/artifact/download",
|
||||
"downloadIp",
|
||||
RATE_LIMITS.download.ip,
|
||||
],
|
||||
] as const)(
|
||||
"registers %s behind the central special-case bucket",
|
||||
async (_name, path, bucket, rate) => {
|
||||
await expectRouteUsesIpBucket({
|
||||
path,
|
||||
method: "GET",
|
||||
bucket,
|
||||
rate,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("registers security verdict submission behind the read limit", async () => {
|
||||
await expectRouteUsesIpBucket({
|
||||
path: `${ApiRoutes.skills}/-/security-verdicts`,
|
||||
method: "POST",
|
||||
bucket: "readIp",
|
||||
rate: RATE_LIMITS.read.ip,
|
||||
});
|
||||
});
|
||||
|
||||
it("registers auth sign-in routes behind the router-level default limit", async () => {
|
||||
const route = http.lookup("/api/auth/signin/github", "GET");
|
||||
if (!route) throw new Error("Expected auth sign-in route");
|
||||
const [action] = route;
|
||||
const { ctx, runMutation } = makeDeniedRateLimitCtx();
|
||||
|
||||
const response = await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request("https://example.com/api/auth/signin/github"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(429);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
name: "readIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["search", LegacyApiRoutes.search, "GET"],
|
||||
["skill detail", LegacyApiRoutes.skill, "GET"],
|
||||
["skill resolve", LegacyApiRoutes.skillResolve, "GET"],
|
||||
["whoami", LegacyApiRoutes.cliWhoami, "GET"],
|
||||
["upload URL", LegacyApiRoutes.cliUploadUrl, "POST"],
|
||||
["publish", LegacyApiRoutes.cliPublish, "POST"],
|
||||
["install telemetry", LegacyApiRoutes.cliTelemetryInstall, "POST"],
|
||||
["skill delete", LegacyApiRoutes.cliSkillDelete, "POST"],
|
||||
["skill undelete", LegacyApiRoutes.cliSkillUndelete, "POST"],
|
||||
])("registers legacy %s behind the router-level default limit", async (_name, path, method) => {
|
||||
const route = http.lookup(path, method as "GET" | "POST");
|
||||
if (!route) throw new Error(`Expected legacy route for ${path}`);
|
||||
const [action] = route;
|
||||
const { ctx, runMutation } = makeDeniedRateLimitCtx();
|
||||
|
||||
const response = await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request(`https://example.com${path}`, { method }),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(429);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
name: method === "GET" ? "readIp" : "writeIp",
|
||||
config: expect.objectContaining({
|
||||
rate: method === "GET" ? RATE_LIMITS.read.ip : RATE_LIMITS.write.ip,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not double-consume routes that already rate limit inside their handler", async () => {
|
||||
const route = http.lookup("/api/v1/search", "GET");
|
||||
if (!route) throw new Error("Expected v1 search route");
|
||||
const [action] = route;
|
||||
const { ctx, runMutation } = makeAllowedRateLimitCtx();
|
||||
|
||||
const response = await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/search?q="),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(runMutation).toHaveBeenCalledTimes(1);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
name: "readIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves API preflights unmetered", async () => {
|
||||
const route = http.lookup("/api/v1/skills", "OPTIONS");
|
||||
if (!route) throw new Error("Expected API preflight route");
|
||||
const [action] = route;
|
||||
const { ctx, runQuery, runMutation } = makeDeniedRateLimitCtx();
|
||||
|
||||
const response = await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/skills", { method: "OPTIONS" }),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(204);
|
||||
expect(runQuery).not.toHaveBeenCalled();
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
+2
-1
@@ -56,13 +56,14 @@ import {
|
||||
contentRightsV1Http,
|
||||
} from "./httpApiV1";
|
||||
import { preflightHandler } from "./httpPreflight";
|
||||
import { installRateLimitedRoutes } from "./lib/httpRouteRateLimit";
|
||||
import {
|
||||
packageInspectorArtifactHttp,
|
||||
packageInspectorClaimHttp,
|
||||
packageInspectorResultsHttp,
|
||||
} from "./packageInspectorHttp";
|
||||
|
||||
const http = httpRouter();
|
||||
const http = installRateLimitedRoutes(httpRouter());
|
||||
|
||||
auth.addHttpRoutes(http);
|
||||
|
||||
|
||||
@@ -400,16 +400,8 @@ describe("httpApi handlers", () => {
|
||||
expires_in: 900,
|
||||
interval: 5,
|
||||
};
|
||||
const runQuery = vi.fn().mockResolvedValue({
|
||||
allowed: true,
|
||||
remaining: 300,
|
||||
limit: 300,
|
||||
resetAt: Date.now() + 60_000,
|
||||
});
|
||||
const runMutation = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ allowed: true, remaining: 299 })
|
||||
.mockResolvedValueOnce(result);
|
||||
const runQuery = vi.fn();
|
||||
const runMutation = vi.fn().mockResolvedValueOnce({ ok: true }).mockResolvedValueOnce(result);
|
||||
|
||||
const response = await __handlers.cliDeviceCodeHandler(
|
||||
makeCtx({ runQuery, runMutation }),
|
||||
@@ -436,13 +428,8 @@ describe("httpApi handlers", () => {
|
||||
|
||||
it("cliDeviceTokenHttp requires the device grant type", async () => {
|
||||
vi.mocked(getOptionalApiTokenUser).mockResolvedValueOnce(null);
|
||||
const runQuery = vi.fn().mockResolvedValue({
|
||||
allowed: true,
|
||||
remaining: 300,
|
||||
limit: 300,
|
||||
resetAt: Date.now() + 60_000,
|
||||
});
|
||||
const runMutation = vi.fn().mockResolvedValueOnce({ allowed: true, remaining: 299 });
|
||||
const runQuery = vi.fn();
|
||||
const runMutation = vi.fn().mockResolvedValueOnce({ ok: true });
|
||||
|
||||
const response = await __handlers.cliDeviceTokenHandler(
|
||||
makeCtx({ runQuery, runMutation }),
|
||||
@@ -460,15 +447,10 @@ describe("httpApi handlers", () => {
|
||||
|
||||
it("cliDeviceTokenHttp returns pending as retryable", async () => {
|
||||
vi.mocked(getOptionalApiTokenUser).mockResolvedValueOnce(null);
|
||||
const runQuery = vi.fn().mockResolvedValue({
|
||||
allowed: true,
|
||||
remaining: 300,
|
||||
limit: 300,
|
||||
resetAt: Date.now() + 60_000,
|
||||
});
|
||||
const runQuery = vi.fn();
|
||||
const runMutation = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ allowed: true, remaining: 299 })
|
||||
.mockResolvedValueOnce({ ok: true })
|
||||
.mockResolvedValueOnce({ error: "authorization_pending" });
|
||||
|
||||
const response = await __handlers.cliDeviceTokenHandler(
|
||||
|
||||
@@ -27,30 +27,14 @@ const { __handlers } = await import("./httpApiV1");
|
||||
|
||||
type ActionCtx = import("./_generated/server").ActionCtx;
|
||||
|
||||
type RateLimitArgs = { key: string; limit: number; windowMs: number };
|
||||
|
||||
function isRateLimitArgs(args: unknown): args is RateLimitArgs {
|
||||
if (!args || typeof args !== "object") return false;
|
||||
const value = args as Record<string, unknown>;
|
||||
return (
|
||||
typeof value.key === "string" &&
|
||||
typeof value.limit === "number" &&
|
||||
typeof value.windowMs === "number"
|
||||
);
|
||||
}
|
||||
|
||||
const okRate = () => ({
|
||||
allowed: true,
|
||||
remaining: 10,
|
||||
limit: 100,
|
||||
resetAt: Date.now() + 60_000,
|
||||
ok: true,
|
||||
});
|
||||
|
||||
function makeCtx(partial: {
|
||||
runQuery?: (query: unknown, args: Record<string, unknown>) => unknown;
|
||||
}) {
|
||||
const runQuery = vi.fn(async (query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return { ...okRate(), limit: args.limit };
|
||||
return partial.runQuery ? await partial.runQuery(query, args) : null;
|
||||
});
|
||||
return { runQuery, runMutation: vi.fn().mockResolvedValue(okRate()) } as unknown as ActionCtx;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
/* @vitest-environment node */
|
||||
import type { RateLimitArgs, RateLimitReturns } from "@convex-dev/rate-limiter";
|
||||
import { gzipSync, strFromU8, unzipSync } from "fflate";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { api, internal } from "./_generated/api";
|
||||
@@ -41,15 +42,18 @@ const { __handlers } = await import("./httpApiV1");
|
||||
|
||||
type ActionCtx = import("./_generated/server").ActionCtx;
|
||||
|
||||
type RateLimitArgs = { key: string; limit: number; windowMs: number };
|
||||
|
||||
function isRateLimitArgs(args: unknown): args is RateLimitArgs {
|
||||
if (!args || typeof args !== "object") return false;
|
||||
const value = args as Record<string, unknown>;
|
||||
const config = value.config as Record<string, unknown> | undefined;
|
||||
return (
|
||||
typeof value.key === "string" &&
|
||||
typeof value.limit === "number" &&
|
||||
typeof value.windowMs === "number"
|
||||
typeof value.name === "string" &&
|
||||
(!("key" in value) || typeof value.key === "string") &&
|
||||
!!config &&
|
||||
typeof config === "object" &&
|
||||
(config.kind === "fixed window" || config.kind === "token bucket") &&
|
||||
typeof config.rate === "number" &&
|
||||
typeof config.period === "number"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -229,31 +233,26 @@ function makeCtx(partial: Record<string, unknown>) {
|
||||
? (partial.runQuery as (query: unknown, args: Record<string, unknown>) => unknown)
|
||||
: null;
|
||||
const runQuery = vi.fn(async (query: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) {
|
||||
return rateLimitStatus?.(args) ?? { ...okRate(), limit: args.limit };
|
||||
}
|
||||
return partialRunQuery ? await partialRunQuery(query, args) : null;
|
||||
});
|
||||
const runMutation =
|
||||
typeof partial.runMutation === "function"
|
||||
? partial.runMutation
|
||||
: vi.fn().mockResolvedValue(okRate());
|
||||
: vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return rateLimitStatus?.(args) ?? okRate();
|
||||
return okRate();
|
||||
});
|
||||
|
||||
return { ...partial, runQuery, runMutation } as unknown as ActionCtx;
|
||||
}
|
||||
|
||||
const okRate = () => ({
|
||||
allowed: true,
|
||||
remaining: 10,
|
||||
limit: 100,
|
||||
resetAt: Date.now() + 60_000,
|
||||
const okRate = (): RateLimitReturns => ({
|
||||
ok: true,
|
||||
});
|
||||
|
||||
const blockedRate = () => ({
|
||||
allowed: false,
|
||||
remaining: 0,
|
||||
limit: 100,
|
||||
resetAt: Date.now() + 60_000,
|
||||
const blockedRate = (): RateLimitReturns => ({
|
||||
ok: false,
|
||||
retryAfter: 60_000,
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -378,7 +377,7 @@ describe("httpApiV1 handlers", () => {
|
||||
} as never);
|
||||
|
||||
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
|
||||
if (isRateLimitArgs(args)) return { ...okRate(), limit: args.limit };
|
||||
if (isRateLimitArgs(args)) return okRate();
|
||||
return { ok: true };
|
||||
});
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
@@ -399,7 +398,8 @@ describe("httpApiV1 handlers", () => {
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
key: "user:users:actor:export",
|
||||
limit: RATE_LIMITS.export.key,
|
||||
name: "exportKey",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.export.key }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
@@ -1453,7 +1453,9 @@ describe("httpApiV1 handlers", () => {
|
||||
if (response.status !== 200) throw new Error(await response.text());
|
||||
|
||||
const slugCalls = runMutation.mock.calls.filter(([, args]) => hasSlugArgs(args));
|
||||
const packageCalls = runMutation.mock.calls.filter(([, args]) => hasPackageNameArgs(args));
|
||||
const packageCalls = runMutation.mock.calls.filter(
|
||||
([, args]) => hasPackageNameArgs(args) && !isRateLimitArgs(args),
|
||||
);
|
||||
expect(slugCalls).toHaveLength(1);
|
||||
expect(slugCalls[0]?.[1]).toMatchObject({
|
||||
actorUserId: "users:admin",
|
||||
@@ -2005,18 +2007,16 @@ describe("httpApiV1 handlers", () => {
|
||||
});
|
||||
|
||||
it("search rate limits", async () => {
|
||||
const runMutation = vi.fn().mockResolvedValue(blockedRate());
|
||||
const response = await __handlers.searchSkillsV1Handler(
|
||||
makeCtx({ runAction: vi.fn(), runMutation }),
|
||||
makeCtx({ runAction: vi.fn(), rateLimitStatus: () => blockedRate() }),
|
||||
new Request("https://example.com/api/v1/search?q=test"),
|
||||
);
|
||||
expect(response.status).toBe(429);
|
||||
});
|
||||
|
||||
it("429 Retry-After is a relative delay, not an absolute epoch", async () => {
|
||||
const runMutation = vi.fn().mockResolvedValue(blockedRate());
|
||||
const response = await __handlers.searchSkillsV1Handler(
|
||||
makeCtx({ runAction: vi.fn(), runMutation }),
|
||||
makeCtx({ runAction: vi.fn(), rateLimitStatus: () => blockedRate() }),
|
||||
new Request("https://example.com/api/v1/search?q=test"),
|
||||
);
|
||||
expect(response.status).toBe(429);
|
||||
@@ -8548,7 +8548,8 @@ describe("httpApiV1 handlers", () => {
|
||||
);
|
||||
expect(findRateLimitCallArgs(runMutation)).toMatchObject({
|
||||
key: expect.stringMatching(/^ip:/),
|
||||
limit: RATE_LIMITS.read.ip,
|
||||
name: "readIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
|
||||
});
|
||||
expect(response.headers.get("RateLimit-Limit")).toBeTruthy();
|
||||
});
|
||||
@@ -12631,6 +12632,7 @@ describe("httpApiV1 handlers", () => {
|
||||
});
|
||||
|
||||
it("npm mirror tarball downloads record package installs and download metrics", async () => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("name" in args && !("paginationOpts" in args)) {
|
||||
return {
|
||||
@@ -12927,7 +12929,8 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(response.headers.get("RateLimit-Limit")).toBeTruthy();
|
||||
expect(findRateLimitCallArgs(runMutation)).toMatchObject({
|
||||
key: expect.stringMatching(/^ip:/),
|
||||
limit: RATE_LIMITS.download.ip,
|
||||
name: "downloadIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.download.ip }),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12987,7 +12990,8 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(response.headers.get("RateLimit-Limit")).toBeTruthy();
|
||||
expect(findRateLimitCallArgs(runMutation)).toMatchObject({
|
||||
key: expect.stringMatching(/^ip:/),
|
||||
limit: RATE_LIMITS.read.ip,
|
||||
name: "readIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13048,6 +13052,7 @@ describe("httpApiV1 handlers", () => {
|
||||
});
|
||||
|
||||
it("package download uses a package/ root without registry metadata", async () => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("name" in args) {
|
||||
@@ -13202,6 +13207,7 @@ describe("httpApiV1 handlers", () => {
|
||||
});
|
||||
|
||||
it("package downloads succeed and record download metrics", async () => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
const runMutation = vi.fn().mockResolvedValue(okRate());
|
||||
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
|
||||
if ("name" in args) {
|
||||
@@ -13590,7 +13596,8 @@ describe("httpApiV1 handlers", () => {
|
||||
expect(response.headers.get("RateLimit-Limit")).toBeTruthy();
|
||||
expect(findRateLimitCallArgs(runMutation)).toMatchObject({
|
||||
key: "user:users:1:write",
|
||||
limit: RATE_LIMITS.write.key,
|
||||
name: "writeKey",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.write.key }),
|
||||
});
|
||||
expect(runAction).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
@@ -14259,7 +14266,8 @@ describe("httpApiV1 handlers", () => {
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
key: "ip:unknown:trustedPublish",
|
||||
limit: RATE_LIMITS.trustedPublish.ip,
|
||||
name: "trustedPublishIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.trustedPublish.ip }),
|
||||
}),
|
||||
);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
|
||||
@@ -76,6 +76,7 @@ import {
|
||||
MAX_RAW_FILE_BYTES,
|
||||
getPathSegments,
|
||||
json,
|
||||
parsePackagePathSegments,
|
||||
publicApiOrigin,
|
||||
resolveTagsBatch,
|
||||
requireApiTokenUserOrResponse,
|
||||
@@ -4058,44 +4059,6 @@ function parseNpmMirrorPath(request: Request) {
|
||||
return parsePackagePathSegments(segments);
|
||||
}
|
||||
|
||||
function decodePackagePathSegment(segment: string) {
|
||||
let decoded = segment;
|
||||
for (let i = 0; i < 2 && decoded.includes("%"); i += 1) {
|
||||
try {
|
||||
const next = decodeURIComponent(decoded);
|
||||
if (next === decoded) break;
|
||||
decoded = next;
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
|
||||
function parsePackagePathSegments(segments: string[]) {
|
||||
if (segments.length === 0) return null;
|
||||
const firstSegment = decodePackagePathSegment(segments[0]!);
|
||||
if (firstSegment.startsWith("@")) {
|
||||
if (firstSegment.includes("/")) {
|
||||
const [scope, name, ...encodedRest] = firstSegment.split("/");
|
||||
if (!scope || !name) return null;
|
||||
return {
|
||||
packageName: `${scope}/${name}`,
|
||||
rest: [...encodedRest, ...segments.slice(1)],
|
||||
};
|
||||
}
|
||||
if (segments.length < 2) return null;
|
||||
return {
|
||||
packageName: `${firstSegment}/${decodePackagePathSegment(segments[1]!)}`,
|
||||
rest: segments.slice(2),
|
||||
};
|
||||
}
|
||||
return {
|
||||
packageName: firstSegment,
|
||||
rest: segments.slice(1),
|
||||
};
|
||||
}
|
||||
|
||||
type NpmPackReleasePage = {
|
||||
page: ReleaseLike[];
|
||||
isDone: boolean;
|
||||
|
||||
@@ -7,6 +7,7 @@ import { requireApiTokenUser, requirePackagePublishAuth } from "../lib/apiTokenA
|
||||
import { corsHeaders, mergeHeaders } from "../lib/httpHeaders";
|
||||
import { getPublishFileSizeError, MAX_PUBLISH_FILE_BYTES } from "../lib/publishLimits";
|
||||
import { isMacJunkPath } from "../lib/skills";
|
||||
export { getPathSegments, parsePackagePathSegments } from "../lib/httpPathSegments";
|
||||
|
||||
export const MAX_RAW_FILE_BYTES = 200 * 1024;
|
||||
const DEFAULT_PUBLIC_SITE_URL = "https://clawhub.ai";
|
||||
@@ -220,17 +221,6 @@ export function requireAdminOrResponse(user: Doc<"users">, headers: HeadersInit)
|
||||
}
|
||||
}
|
||||
|
||||
export function getPathSegments(request: Request, prefix: string) {
|
||||
const pathname = new URL(request.url).pathname;
|
||||
if (!pathname.startsWith(prefix)) return [];
|
||||
const rest = pathname.slice(prefix.length);
|
||||
return rest
|
||||
.split("/")
|
||||
.map((segment) => segment.trim())
|
||||
.filter(Boolean)
|
||||
.map((segment) => decodeURIComponent(segment));
|
||||
}
|
||||
|
||||
export function toOptionalNumber(value: string | null) {
|
||||
if (!value) return undefined;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
function toHeaderRecord(init?: HeadersInit): Record<string, string> {
|
||||
if (!init) return {};
|
||||
if (init instanceof Headers) return Object.fromEntries(init.entries());
|
||||
if (Array.isArray(init)) return Object.fromEntries(init);
|
||||
return { ...(init as Record<string, string>) };
|
||||
}
|
||||
|
||||
export function mergeHeaders(...inits: Array<HeadersInit | undefined>): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
const out = new Headers();
|
||||
for (const init of inits) {
|
||||
Object.assign(out, toHeaderRecord(init));
|
||||
if (!init) continue;
|
||||
for (const [key, value] of new Headers(init)) {
|
||||
out.set(key, value);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
return Object.fromEntries(out.entries());
|
||||
}
|
||||
|
||||
export function corsHeaders(origin: string = "*"): Record<string, string> {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
export type PackagePathRoute = {
|
||||
packageName: string;
|
||||
rest: string[];
|
||||
};
|
||||
|
||||
export function getPathSegments(request: Request, prefix: string) {
|
||||
const pathname = new URL(request.url).pathname;
|
||||
if (!pathname.startsWith(prefix)) return [];
|
||||
const rest = pathname.slice(prefix.length);
|
||||
return rest
|
||||
.split("/")
|
||||
.map((segment) => segment.trim())
|
||||
.filter(Boolean)
|
||||
.map((segment) => decodeURIComponent(segment));
|
||||
}
|
||||
|
||||
export function parsePackagePathSegments(segments: string[]): PackagePathRoute | null {
|
||||
if (segments.length === 0) return null;
|
||||
const firstSegment = decodePackagePathSegment(segments[0]!);
|
||||
if (firstSegment.startsWith("@")) {
|
||||
if (firstSegment.includes("/")) {
|
||||
const [scope, name, ...encodedRest] = firstSegment.split("/");
|
||||
if (!scope || !name) return null;
|
||||
return {
|
||||
packageName: `${scope}/${name}`,
|
||||
rest: [...encodedRest, ...segments.slice(1)],
|
||||
};
|
||||
}
|
||||
if (segments.length < 2) return null;
|
||||
return {
|
||||
packageName: `${firstSegment}/${decodePackagePathSegment(segments[1]!)}`,
|
||||
rest: segments.slice(2),
|
||||
};
|
||||
}
|
||||
return {
|
||||
packageName: firstSegment,
|
||||
rest: segments.slice(1),
|
||||
};
|
||||
}
|
||||
|
||||
function decodePackagePathSegment(segment: string) {
|
||||
let decoded = segment;
|
||||
for (let i = 0; i < 2 && decoded.includes("%"); i += 1) {
|
||||
try {
|
||||
const next = decodeURIComponent(decoded);
|
||||
if (next === decoded) break;
|
||||
decoded = next;
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
@@ -32,19 +32,24 @@ function makeRateLimitCtx(plan: MockRateLimitPlan) {
|
||||
role: plan.userRole ?? "user",
|
||||
};
|
||||
}
|
||||
if ("key" in args && "limit" in args && "windowMs" in args) {
|
||||
const key = String(args.key);
|
||||
if (key.startsWith("ip:")) return plan.ip;
|
||||
if (key.startsWith("user:")) return plan.user;
|
||||
}
|
||||
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`);
|
||||
});
|
||||
|
||||
const runMutation = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
|
||||
if ("name" in args && "key" in args && !("config" in args)) {
|
||||
return { action: "updated", expiresAt: Date.now() + 86_400_000 };
|
||||
}
|
||||
if (!("name" in args && "config" in args)) {
|
||||
throw new Error(`Unexpected runMutation args: ${JSON.stringify(args)}`);
|
||||
}
|
||||
const key = String(args.key);
|
||||
const source = key.startsWith("user:") ? plan.user : plan.ip;
|
||||
if (!source) throw new Error(`Missing rate limit source for ${key}`);
|
||||
return { allowed: source.allowed, remaining: source.remaining };
|
||||
if (source.allowed) {
|
||||
source.remaining = Math.max(0, source.remaining - 1);
|
||||
return { ok: true };
|
||||
}
|
||||
return { ok: false, retryAfter: Math.max(1, source.resetAt - Date.now()) };
|
||||
});
|
||||
|
||||
return {
|
||||
@@ -53,6 +58,12 @@ function makeRateLimitCtx(plan: MockRateLimitPlan) {
|
||||
} as unknown as Parameters<typeof applyRateLimit>[0];
|
||||
}
|
||||
|
||||
function componentRateLimitCalls(runMutation: ReturnType<typeof vi.fn>) {
|
||||
return runMutation.mock.calls.filter(([, args]) => {
|
||||
return Boolean(args && typeof args === "object" && "config" in args);
|
||||
}) as [unknown, Record<string, unknown>][];
|
||||
}
|
||||
|
||||
describe("getClientIp", () => {
|
||||
let prev: string | undefined;
|
||||
beforeEach(() => {
|
||||
@@ -86,12 +97,23 @@ describe("getClientIp", () => {
|
||||
expect(getClientIp(request)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns first ip from cf-connecting-ip", () => {
|
||||
it("ignores cf-connecting-ip unless client ip headers are explicitly trusted", () => {
|
||||
const request = new Request("https://example.com", {
|
||||
headers: {
|
||||
"cf-connecting-ip": "203.0.113.1, 198.51.100.2",
|
||||
},
|
||||
});
|
||||
delete process.env.TRUST_FORWARDED_IPS;
|
||||
expect(getClientIp(request)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns first ip from cf-connecting-ip when trusted mode is enabled", () => {
|
||||
const request = new Request("https://example.com", {
|
||||
headers: {
|
||||
"cf-connecting-ip": "203.0.113.1, 198.51.100.2",
|
||||
},
|
||||
});
|
||||
process.env.TRUST_FORWARDED_IPS = "true";
|
||||
expect(getClientIp(request)).toBe("203.0.113.1");
|
||||
});
|
||||
|
||||
@@ -142,20 +164,20 @@ describe("RATE_LIMITS", () => {
|
||||
describe("applyRateLimit headers", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("returns delay-seconds Retry-After on 429 (not epoch)", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
|
||||
const runMutation = vi.fn();
|
||||
const ctx = {
|
||||
runQuery: vi.fn().mockResolvedValue({
|
||||
const ctx = makeRateLimitCtx({
|
||||
ip: {
|
||||
allowed: false,
|
||||
remaining: 0,
|
||||
limit: 20,
|
||||
limit: RATE_LIMITS.download.ip,
|
||||
resetAt: 1_030_500,
|
||||
}),
|
||||
runMutation,
|
||||
} as unknown as Parameters<typeof applyRateLimit>[0];
|
||||
},
|
||||
});
|
||||
const request = new Request("https://example.com", {
|
||||
headers: { "cf-connecting-ip": "203.0.113.1" },
|
||||
});
|
||||
@@ -167,23 +189,18 @@ describe("applyRateLimit headers", () => {
|
||||
expect(result.response.headers.get("Retry-After")).toBe("31");
|
||||
expect(result.response.headers.get("X-RateLimit-Reset")).toBe("1031");
|
||||
expect(result.response.headers.get("RateLimit-Reset")).toBe("31");
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("includes rate-limit headers without Retry-After when allowed", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_000_000);
|
||||
const ctx = {
|
||||
runQuery: vi.fn().mockResolvedValue({
|
||||
const ctx = makeRateLimitCtx({
|
||||
ip: {
|
||||
allowed: true,
|
||||
remaining: 19,
|
||||
limit: 20,
|
||||
limit: RATE_LIMITS.download.ip,
|
||||
resetAt: 2_015_000,
|
||||
}),
|
||||
runMutation: vi.fn().mockResolvedValue({
|
||||
allowed: true,
|
||||
remaining: 18,
|
||||
}),
|
||||
} as unknown as Parameters<typeof applyRateLimit>[0];
|
||||
},
|
||||
});
|
||||
const request = new Request("https://example.com", {
|
||||
headers: { "cf-connecting-ip": "203.0.113.1" },
|
||||
});
|
||||
@@ -192,30 +209,22 @@ describe("applyRateLimit headers", () => {
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) return;
|
||||
const headers = new Headers(result.headers);
|
||||
expect(headers.get("X-RateLimit-Limit")).toBe("20");
|
||||
expect(headers.get("X-RateLimit-Remaining")).toBe("18");
|
||||
expect(headers.get("X-RateLimit-Reset")).toBe("2015");
|
||||
expect(headers.get("RateLimit-Limit")).toBe("20");
|
||||
expect(headers.get("RateLimit-Remaining")).toBe("18");
|
||||
expect(headers.get("RateLimit-Reset")).toBe("15");
|
||||
expect(headers.get("X-RateLimit-Limit")).toBe(String(RATE_LIMITS.download.ip));
|
||||
expect(headers.get("X-RateLimit-Remaining")).toBeNull();
|
||||
expect(headers.get("X-RateLimit-Reset")).toBe("2040");
|
||||
expect(headers.get("RateLimit-Limit")).toBe(String(RATE_LIMITS.download.ip));
|
||||
expect(headers.get("RateLimit-Remaining")).toBeNull();
|
||||
expect(headers.get("RateLimit-Reset")).toBe("40");
|
||||
expect(headers.get("Retry-After")).toBeNull();
|
||||
});
|
||||
|
||||
it("converts shard write conflicts into a rate-limit response", async () => {
|
||||
it("returns retryable unavailable response when component counter writes conflict", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_500_000);
|
||||
const ctx = {
|
||||
runQuery: vi.fn().mockResolvedValue({
|
||||
allowed: true,
|
||||
remaining: 19,
|
||||
limit: 20,
|
||||
resetAt: 2_530_000,
|
||||
}),
|
||||
runMutation: vi
|
||||
.fn()
|
||||
.mockRejectedValue(
|
||||
new Error(
|
||||
'Document in table "rateLimitCounters" changed while this mutation was being run',
|
||||
),
|
||||
new Error('Document in table "rateLimits" changed while this mutation was being run'),
|
||||
),
|
||||
} as unknown as Parameters<typeof applyRateLimit>[0];
|
||||
const request = new Request("https://example.com", {
|
||||
@@ -226,13 +235,38 @@ describe("applyRateLimit headers", () => {
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
expect(result.response.status).toBe(429);
|
||||
expect(result.response.headers.get("Retry-After")).toBe("30");
|
||||
expect(result.response.status).toBe(503);
|
||||
expect(result.response.headers.get("Retry-After")).toBe("1");
|
||||
await expect(result.response.text()).resolves.toBe("Rate limit temporarily unavailable");
|
||||
});
|
||||
|
||||
it("selects one of 16 active counter shards", async () => {
|
||||
it("returns retryable unavailable response when metadata key writes conflict", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_550_000);
|
||||
const ctx = {
|
||||
runMutation: vi
|
||||
.fn()
|
||||
.mockRejectedValue(
|
||||
new Error(
|
||||
'Document in table "httpRateLimitKeys" changed while this mutation was being run',
|
||||
),
|
||||
),
|
||||
} as unknown as Parameters<typeof applyRateLimit>[0];
|
||||
|
||||
const result = await applyRateLimit(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/packages/demo"),
|
||||
"read",
|
||||
);
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
expect(result.response.status).toBe(503);
|
||||
expect(result.response.headers.get("Retry-After")).toBe("1");
|
||||
await expect(result.response.text()).resolves.toBe("Rate limit temporarily unavailable");
|
||||
});
|
||||
|
||||
it("configures component-backed HTTP limits with 16 shards", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_600_000);
|
||||
vi.spyOn(Math, "random").mockReturnValue(0.999);
|
||||
const ctx = makeRateLimitCtx({
|
||||
ip: {
|
||||
allowed: true,
|
||||
@@ -249,8 +283,111 @@ describe("applyRateLimit headers", () => {
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
|
||||
const [, args] = runMutation.mock.calls[0] as [unknown, Record<string, unknown>];
|
||||
expect(args.shard).toBe(15);
|
||||
const [, args] = componentRateLimitCalls(runMutation)[0];
|
||||
expect(args).toMatchObject({
|
||||
name: "downloadIp",
|
||||
key: "ip:unknown:download",
|
||||
config: expect.objectContaining({
|
||||
kind: "fixed window",
|
||||
rate: RATE_LIMITS.download.ip,
|
||||
period: 60_000,
|
||||
start: 0,
|
||||
shards: 16,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it("passes component key metadata after an allowed limiter check", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_650_000);
|
||||
const ctx = makeRateLimitCtx({
|
||||
ip: {
|
||||
allowed: true,
|
||||
remaining: 19,
|
||||
limit: RATE_LIMITS.download.ip,
|
||||
resetAt: 2_700_000,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await applyRateLimit(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo"),
|
||||
"download",
|
||||
);
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
|
||||
expect(componentRateLimitCalls(runMutation).map(([, args]) => args)).toContainEqual(
|
||||
expect.objectContaining({
|
||||
config: expect.any(Object),
|
||||
name: "downloadIp",
|
||||
key: "ip:unknown:download",
|
||||
now: 2_650_000,
|
||||
ttlMs: 86_400_000,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("passes component key metadata after a denied limiter check", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_660_000);
|
||||
const ctx = makeRateLimitCtx({
|
||||
ip: {
|
||||
allowed: false,
|
||||
remaining: 0,
|
||||
limit: RATE_LIMITS.download.ip,
|
||||
resetAt: 2_690_000,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await applyRateLimit(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/download?slug=demo"),
|
||||
"download",
|
||||
);
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
|
||||
expect(componentRateLimitCalls(runMutation).map(([, args]) => args)).toContainEqual(
|
||||
expect.objectContaining({
|
||||
config: expect.any(Object),
|
||||
name: "downloadIp",
|
||||
key: "ip:unknown:download",
|
||||
now: 2_660_000,
|
||||
ttlMs: 86_400_000,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not shard low-rate export ip buckets", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(2_700_000);
|
||||
const ctx = makeRateLimitCtx({
|
||||
ip: {
|
||||
allowed: true,
|
||||
remaining: 9,
|
||||
limit: RATE_LIMITS.export.ip,
|
||||
resetAt: 2_740_000,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await applyRateLimit(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/skills/export"),
|
||||
"export",
|
||||
);
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
|
||||
const [, args] = componentRateLimitCalls(runMutation)[0];
|
||||
expect(args).toMatchObject({
|
||||
name: "exportIp",
|
||||
key: "ip:unknown:export",
|
||||
config: expect.objectContaining({
|
||||
kind: "fixed window",
|
||||
rate: RATE_LIMITS.export.ip,
|
||||
period: 60_000,
|
||||
start: 0,
|
||||
shards: 1,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it("allows authenticated users when user bucket is healthy and shared ip bucket is exhausted", async () => {
|
||||
@@ -280,8 +417,8 @@ describe("applyRateLimit headers", () => {
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) return;
|
||||
const headers = new Headers(result.headers);
|
||||
expect(headers.get("X-RateLimit-Limit")).toBe("120");
|
||||
expect(headers.get("X-RateLimit-Remaining")).toBe("41");
|
||||
expect(headers.get("X-RateLimit-Limit")).toBe(String(RATE_LIMITS.download.key));
|
||||
expect(headers.get("X-RateLimit-Remaining")).toBeNull();
|
||||
expect(headers.get("Retry-After")).toBeNull();
|
||||
});
|
||||
|
||||
@@ -311,7 +448,7 @@ describe("applyRateLimit headers", () => {
|
||||
const result = await applyRateLimit(ctx, request, "download");
|
||||
expect(result.ok).toBe(true);
|
||||
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
|
||||
const consumedKeys = runMutation.mock.calls.map(([, args]) => String(args.key));
|
||||
const consumedKeys = componentRateLimitCalls(runMutation).map(([, args]) => String(args.key));
|
||||
expect(consumedKeys.some((key) => key.startsWith("user:"))).toBe(true);
|
||||
expect(consumedKeys.some((key) => key.startsWith("ip:"))).toBe(false);
|
||||
});
|
||||
@@ -343,14 +480,12 @@ describe("applyRateLimit headers", () => {
|
||||
const result = await applyRateLimit(ctx, request, "write");
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
const runQuery = (ctx as unknown as { runQuery: ReturnType<typeof vi.fn> }).runQuery;
|
||||
const rateLimitStatusCalls = runQuery.mock.calls
|
||||
.map(([, args]) => args as Record<string, unknown>)
|
||||
.filter((args) => "key" in args && "limit" in args);
|
||||
expect(rateLimitStatusCalls).toContainEqual(
|
||||
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
|
||||
expect(componentRateLimitCalls(runMutation).map(([, args]) => args)).toContainEqual(
|
||||
expect.objectContaining({
|
||||
name: "writeAdminKey",
|
||||
key: "user:users_123:write",
|
||||
limit: RATE_LIMITS.write.adminKey,
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.write.adminKey }),
|
||||
}),
|
||||
);
|
||||
if (!result.ok) return;
|
||||
@@ -386,12 +521,12 @@ describe("applyRateLimit headers", () => {
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
expect(result.response.status).toBe(429);
|
||||
expect(result.response.headers.get("X-RateLimit-Limit")).toBe("120");
|
||||
expect(result.response.headers.get("X-RateLimit-Limit")).toBe(String(RATE_LIMITS.download.key));
|
||||
expect(result.response.headers.get("X-RateLimit-Remaining")).toBe("0");
|
||||
expect(result.response.headers.get("Retry-After")).toBe("30");
|
||||
});
|
||||
|
||||
it("scopes anonymous download fallback buckets when client ip is missing", async () => {
|
||||
it("uses one anonymous download fallback bucket when client ip is missing", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(4_500_000);
|
||||
const ctx = makeRateLimitCtx({
|
||||
ip: {
|
||||
@@ -401,21 +536,27 @@ describe("applyRateLimit headers", () => {
|
||||
resetAt: 4_530_000,
|
||||
},
|
||||
});
|
||||
const request = new Request(
|
||||
"https://example.com/api/v1/packages/tickflow-assist/download?version=0.2.10",
|
||||
|
||||
await applyRateLimit(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/download?slug=first&version=1.0.0"),
|
||||
"download",
|
||||
);
|
||||
await applyRateLimit(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/packages/second-plugin/download?version=0.2.0"),
|
||||
"download",
|
||||
);
|
||||
|
||||
const result = await applyRateLimit(ctx, request, "download");
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
|
||||
const consumedKeys = runMutation.mock.calls.map(([, args]) => String(args.key));
|
||||
expect(consumedKeys).toContain(
|
||||
"ip:unknown:download:/api/v1/packages/tickflow-assist/download?version=0.2.10",
|
||||
);
|
||||
expect(componentRateLimitCalls(runMutation).map(([, args]) => String(args.key))).toEqual([
|
||||
"ip:unknown:download",
|
||||
"ip:unknown:download",
|
||||
]);
|
||||
});
|
||||
|
||||
it("scopes known-ip anonymous buckets by rate limit kind", async () => {
|
||||
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
|
||||
vi.spyOn(Date, "now").mockReturnValue(4_550_000);
|
||||
const readCtx = makeRateLimitCtx({
|
||||
ip: {
|
||||
@@ -444,10 +585,10 @@ describe("applyRateLimit headers", () => {
|
||||
.runMutation;
|
||||
const downloadMutation = (downloadCtx as unknown as { runMutation: ReturnType<typeof vi.fn> })
|
||||
.runMutation;
|
||||
expect(readMutation.mock.calls.map(([, args]) => String(args.key))).toContain(
|
||||
expect(componentRateLimitCalls(readMutation).map(([, args]) => String(args.key))).toContain(
|
||||
"ip:203.0.113.1:read",
|
||||
);
|
||||
expect(downloadMutation.mock.calls.map(([, args]) => String(args.key))).toContain(
|
||||
expect(componentRateLimitCalls(downloadMutation).map(([, args]) => String(args.key))).toContain(
|
||||
"ip:203.0.113.1:download",
|
||||
);
|
||||
});
|
||||
@@ -479,12 +620,12 @@ describe("applyRateLimit headers", () => {
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
|
||||
expect(runMutation.mock.calls.map(([, args]) => String(args.key))).toContain(
|
||||
expect(componentRateLimitCalls(runMutation).map(([, args]) => String(args.key))).toContain(
|
||||
"user:users_123:download",
|
||||
);
|
||||
});
|
||||
|
||||
it("scopes non-download missing-ip anonymous requests by rate limit kind", async () => {
|
||||
it("uses one anonymous read fallback bucket when client ip is missing", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(4_600_000);
|
||||
const ctx = makeRateLimitCtx({
|
||||
ip: {
|
||||
@@ -494,14 +635,19 @@ describe("applyRateLimit headers", () => {
|
||||
resetAt: 4_630_000,
|
||||
},
|
||||
});
|
||||
const request = new Request("https://example.com/api/v1/search?q=demo");
|
||||
|
||||
const result = await applyRateLimit(ctx, request, "read");
|
||||
await applyRateLimit(ctx, new Request("https://example.com/api/v1/search?q=demo"), "read");
|
||||
await applyRateLimit(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/packages/second-plugin"),
|
||||
"read",
|
||||
);
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
|
||||
const consumedKeys = runMutation.mock.calls.map(([, args]) => String(args.key));
|
||||
expect(consumedKeys).toContain("ip:unknown:read");
|
||||
expect(componentRateLimitCalls(runMutation).map(([, args]) => String(args.key))).toEqual([
|
||||
"ip:unknown:read",
|
||||
"ip:unknown:read",
|
||||
]);
|
||||
});
|
||||
|
||||
it("falls back to ip enforcement when bearer token is invalid", async () => {
|
||||
@@ -526,7 +672,7 @@ describe("applyRateLimit headers", () => {
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
expect(result.response.status).toBe(429);
|
||||
expect(result.response.headers.get("X-RateLimit-Limit")).toBe("20");
|
||||
expect(result.response.headers.get("X-RateLimit-Limit")).toBe(String(RATE_LIMITS.download.ip));
|
||||
expect(result.response.headers.get("Retry-After")).toBe("30");
|
||||
});
|
||||
});
|
||||
|
||||
+146
-77
@@ -1,9 +1,9 @@
|
||||
import { MINUTE, type RateLimitConfig } from "@convex-dev/rate-limiter";
|
||||
import { internal } from "../_generated/api";
|
||||
import type { Doc } from "../_generated/dataModel";
|
||||
import type { ActionCtx } from "../_generated/server";
|
||||
import { getOptionalApiTokenUser } from "./apiTokenAuth";
|
||||
import { corsHeaders, mergeHeaders } from "./httpHeaders";
|
||||
import { RATE_LIMIT_COUNTER_SHARDS, RATE_LIMIT_WINDOW_MS } from "./rateLimitConfig";
|
||||
|
||||
export const RATE_LIMITS = {
|
||||
read: { ip: 3000, key: 12000, adminKey: 120000 },
|
||||
@@ -13,18 +13,76 @@ export const RATE_LIMITS = {
|
||||
export: { ip: 10, key: 60, adminKey: 60 },
|
||||
} as const;
|
||||
|
||||
const RATE_LIMIT_WINDOW_MS = 60_000;
|
||||
const HTTP_RATE_LIMIT_SHARDS = 16;
|
||||
const HTTP_RATE_LIMIT_MIN_SHARD_CAPACITY = 10;
|
||||
const HTTP_RATE_LIMIT_KEY_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
type RateLimitResult = {
|
||||
allowed: boolean;
|
||||
remaining: number;
|
||||
// The component does not expose an exact global remaining count for sharded
|
||||
// buckets, so successful responses omit this instead of guessing.
|
||||
remaining?: number;
|
||||
limit: number;
|
||||
resetAt: number;
|
||||
unavailable?: boolean;
|
||||
};
|
||||
|
||||
export type ApplyRateLimitResult =
|
||||
| { ok: true; headers: HeadersInit }
|
||||
| { ok: false; response: Response };
|
||||
|
||||
type RateLimitKind = keyof typeof RATE_LIMITS;
|
||||
type RateLimitSubject = "ip" | "key" | "adminKey";
|
||||
type HttpRateLimitName = `${RateLimitKind}${Capitalize<RateLimitSubject>}`;
|
||||
type FixedWindowRateLimitConfig = Extract<RateLimitConfig, { kind: "fixed window" }>;
|
||||
|
||||
const preappliedRateLimitHeaders = new WeakMap<Request, HeadersInit>();
|
||||
|
||||
function fixedWindowRateLimit(rate: number): FixedWindowRateLimitConfig {
|
||||
const shards = Math.max(
|
||||
1,
|
||||
Math.min(HTTP_RATE_LIMIT_SHARDS, Math.floor(rate / HTTP_RATE_LIMIT_MIN_SHARD_CAPACITY)),
|
||||
);
|
||||
return {
|
||||
kind: "fixed window",
|
||||
rate,
|
||||
period: MINUTE,
|
||||
start: 0,
|
||||
shards,
|
||||
};
|
||||
}
|
||||
|
||||
const HTTP_RATE_LIMIT_CONFIGS = {
|
||||
readIp: fixedWindowRateLimit(RATE_LIMITS.read.ip),
|
||||
readKey: fixedWindowRateLimit(RATE_LIMITS.read.key),
|
||||
readAdminKey: fixedWindowRateLimit(RATE_LIMITS.read.adminKey),
|
||||
writeIp: fixedWindowRateLimit(RATE_LIMITS.write.ip),
|
||||
writeKey: fixedWindowRateLimit(RATE_LIMITS.write.key),
|
||||
writeAdminKey: fixedWindowRateLimit(RATE_LIMITS.write.adminKey),
|
||||
trustedPublishIp: fixedWindowRateLimit(RATE_LIMITS.trustedPublish.ip),
|
||||
trustedPublishKey: fixedWindowRateLimit(RATE_LIMITS.trustedPublish.key),
|
||||
trustedPublishAdminKey: fixedWindowRateLimit(RATE_LIMITS.trustedPublish.adminKey),
|
||||
downloadIp: fixedWindowRateLimit(RATE_LIMITS.download.ip),
|
||||
downloadKey: fixedWindowRateLimit(RATE_LIMITS.download.key),
|
||||
downloadAdminKey: fixedWindowRateLimit(RATE_LIMITS.download.adminKey),
|
||||
exportIp: fixedWindowRateLimit(RATE_LIMITS.export.ip),
|
||||
exportKey: fixedWindowRateLimit(RATE_LIMITS.export.key),
|
||||
exportAdminKey: fixedWindowRateLimit(RATE_LIMITS.export.adminKey),
|
||||
} as const satisfies Record<HttpRateLimitName, RateLimitConfig>;
|
||||
|
||||
export function markRateLimitApplied(request: Request, headers: HeadersInit): void {
|
||||
preappliedRateLimitHeaders.set(request, headers);
|
||||
}
|
||||
|
||||
export async function applyRateLimit(
|
||||
ctx: ActionCtx,
|
||||
request: Request,
|
||||
kind: keyof typeof RATE_LIMITS,
|
||||
): Promise<{ ok: true; headers: HeadersInit } | { ok: false; response: Response }> {
|
||||
kind: RateLimitKind,
|
||||
): Promise<ApplyRateLimitResult> {
|
||||
const preappliedHeaders = preappliedRateLimitHeaders.get(request);
|
||||
if (preappliedHeaders) return { ok: true, headers: preappliedHeaders };
|
||||
|
||||
const auth = await getOptionalApiTokenUser(ctx, request);
|
||||
const ip = getClientIp(request) ?? "unknown";
|
||||
const ipSource = getClientIpSource(request);
|
||||
@@ -37,9 +95,11 @@ export async function applyRateLimit(
|
||||
const userResult = await checkRateLimit(
|
||||
ctx,
|
||||
getAuthenticatedRateLimitKey(auth.userId, kind),
|
||||
userLimit,
|
||||
userLimit.name,
|
||||
userLimit.limit,
|
||||
);
|
||||
const headers = rateHeaders(userResult);
|
||||
if (userResult.unavailable) return rateLimitUnavailable(headers);
|
||||
if (!userResult.allowed) {
|
||||
console.info("rate_limit_denied", {
|
||||
kind,
|
||||
@@ -71,10 +131,12 @@ export async function applyRateLimit(
|
||||
// Anonymous requests remain IP-enforced.
|
||||
const ipResult = await checkRateLimit(
|
||||
ctx,
|
||||
getAnonymousRateLimitKey(request, kind, ip),
|
||||
getAnonymousRateLimitKey(kind, ip),
|
||||
getHttpRateLimitName(kind, "ip"),
|
||||
RATE_LIMITS[kind].ip,
|
||||
);
|
||||
const headers = rateHeaders(ipResult);
|
||||
if (ipResult.unavailable) return rateLimitUnavailable(headers);
|
||||
|
||||
if (!ipResult.allowed) {
|
||||
console.info("rate_limit_denied", {
|
||||
@@ -104,29 +166,29 @@ export async function applyRateLimit(
|
||||
return { ok: true, headers };
|
||||
}
|
||||
|
||||
function getAnonymousRateLimitKey(request: Request, kind: keyof typeof RATE_LIMITS, ip: string) {
|
||||
function getAnonymousRateLimitKey(kind: RateLimitKind, ip: string) {
|
||||
if (ip !== "unknown") return `ip:${ip}:${kind}`;
|
||||
if (kind !== "download") return `ip:unknown:${kind}`;
|
||||
return `ip:unknown:download:${getDownloadRateLimitScope(request)}`;
|
||||
return `ip:unknown:${kind}`;
|
||||
}
|
||||
|
||||
function getAuthenticatedRateLimitKey(userId: string, kind: keyof typeof RATE_LIMITS) {
|
||||
function getAuthenticatedRateLimitKey(userId: string, kind: RateLimitKind) {
|
||||
return `user:${userId}:${kind}`;
|
||||
}
|
||||
|
||||
function getAuthenticatedRateLimit(
|
||||
kind: keyof typeof RATE_LIMITS,
|
||||
user: Pick<Doc<"users">, "role">,
|
||||
) {
|
||||
return user.role === "admin" ? RATE_LIMITS[kind].adminKey : RATE_LIMITS[kind].key;
|
||||
function getAuthenticatedRateLimit(kind: RateLimitKind, user: Pick<Doc<"users">, "role">) {
|
||||
const subject = user.role === "admin" ? "adminKey" : "key";
|
||||
return {
|
||||
name: getHttpRateLimitName(kind, subject),
|
||||
limit: RATE_LIMITS[kind][subject],
|
||||
};
|
||||
}
|
||||
|
||||
export function getClientIp(request: Request) {
|
||||
export function getClientIp(request: Request): string | null {
|
||||
if (!shouldTrustClientIpHeaders()) return null;
|
||||
|
||||
const cfHeader = request.headers.get("cf-connecting-ip");
|
||||
if (cfHeader) return splitFirstIp(cfHeader);
|
||||
|
||||
if (!shouldTrustForwardedIps()) return null;
|
||||
|
||||
const forwarded =
|
||||
request.headers.get("x-forwarded-for") ??
|
||||
request.headers.get("x-real-ip") ??
|
||||
@@ -136,8 +198,8 @@ export function getClientIp(request: Request) {
|
||||
}
|
||||
|
||||
function getClientIpSource(request: Request) {
|
||||
if (!shouldTrustClientIpHeaders()) return "none";
|
||||
if (request.headers.get("cf-connecting-ip")) return "cf-connecting-ip";
|
||||
if (!shouldTrustForwardedIps()) return "none";
|
||||
if (request.headers.get("x-forwarded-for")) return "x-forwarded-for";
|
||||
if (request.headers.get("x-real-ip")) return "x-real-ip";
|
||||
if (request.headers.get("fly-client-ip")) return "fly-client-ip";
|
||||
@@ -147,45 +209,62 @@ function getClientIpSource(request: Request) {
|
||||
async function checkRateLimit(
|
||||
ctx: ActionCtx,
|
||||
key: string,
|
||||
name: HttpRateLimitName,
|
||||
limit: number,
|
||||
): Promise<RateLimitResult> {
|
||||
// Step 1: Read-only check to avoid write conflicts on denied requests.
|
||||
const status = (await ctx.runQuery(internal.rateLimits.getRateLimitStatusInternal, {
|
||||
key,
|
||||
limit,
|
||||
windowMs: RATE_LIMIT_WINDOW_MS,
|
||||
})) as RateLimitResult;
|
||||
|
||||
if (!status.allowed) {
|
||||
return status;
|
||||
}
|
||||
|
||||
// Step 2: Consume with a mutation only when still allowed.
|
||||
let result: { allowed: boolean; remaining: number };
|
||||
const now = Date.now();
|
||||
try {
|
||||
result = (await ctx.runMutation(internal.rateLimits.consumeRateLimitInternal, {
|
||||
const status = await ctx.runMutation(internal.rateLimits.consumeHttpRateLimitKeyInternal, {
|
||||
name,
|
||||
key,
|
||||
limit,
|
||||
windowMs: RATE_LIMIT_WINDOW_MS,
|
||||
shard: Math.floor(Math.random() * RATE_LIMIT_COUNTER_SHARDS),
|
||||
})) as { allowed: boolean; remaining: number };
|
||||
} catch (error) {
|
||||
if (isRateLimitWriteConflict(error)) {
|
||||
config: HTTP_RATE_LIMIT_CONFIGS[name],
|
||||
now,
|
||||
ttlMs: HTTP_RATE_LIMIT_KEY_TTL_MS,
|
||||
});
|
||||
if (!status.ok) {
|
||||
return {
|
||||
allowed: false,
|
||||
remaining: 0,
|
||||
limit: status.limit,
|
||||
resetAt: status.resetAt,
|
||||
limit,
|
||||
resetAt: now + status.retryAfter,
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
allowed: true,
|
||||
limit,
|
||||
resetAt: getCurrentWindowResetAt(now),
|
||||
};
|
||||
} catch (error) {
|
||||
if (!isRateLimitWriteConflict(error)) throw error;
|
||||
return {
|
||||
allowed: false,
|
||||
remaining: 0,
|
||||
limit,
|
||||
resetAt: now + 1000,
|
||||
unavailable: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function rateLimitUnavailable(headers: HeadersInit): Extract<ApplyRateLimitResult, { ok: false }> {
|
||||
console.warn("rate_limit_unavailable", {
|
||||
reason: "counter_write_contention",
|
||||
});
|
||||
return {
|
||||
allowed: result.allowed,
|
||||
remaining: Math.max(0, status.remaining - 1),
|
||||
limit: status.limit,
|
||||
resetAt: status.resetAt,
|
||||
ok: false,
|
||||
response: new Response("Rate limit temporarily unavailable", {
|
||||
status: 503,
|
||||
headers: mergeHeaders(
|
||||
{
|
||||
"Content-Type": "text/plain; charset=utf-8",
|
||||
"Cache-Control": "no-store",
|
||||
"Retry-After": "1",
|
||||
},
|
||||
headers,
|
||||
corsHeaders(),
|
||||
),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -193,15 +272,27 @@ function rateHeaders(result: RateLimitResult): HeadersInit {
|
||||
const nowMs = Date.now();
|
||||
const resetSeconds = Math.ceil(result.resetAt / 1000);
|
||||
const resetDelaySeconds = Math.max(1, Math.ceil((result.resetAt - nowMs) / 1000));
|
||||
return {
|
||||
const headers: Record<string, string> = {
|
||||
"X-RateLimit-Limit": String(result.limit),
|
||||
"X-RateLimit-Remaining": String(result.remaining),
|
||||
"X-RateLimit-Reset": String(resetSeconds),
|
||||
"RateLimit-Limit": String(result.limit),
|
||||
"RateLimit-Remaining": String(result.remaining),
|
||||
"RateLimit-Reset": String(resetDelaySeconds),
|
||||
...(result.allowed ? {} : { "Retry-After": String(resetDelaySeconds) }),
|
||||
};
|
||||
if (result.remaining !== undefined) {
|
||||
headers["X-RateLimit-Remaining"] = String(result.remaining);
|
||||
headers["RateLimit-Remaining"] = String(result.remaining);
|
||||
}
|
||||
if (!result.allowed) headers["Retry-After"] = String(resetDelaySeconds);
|
||||
return headers;
|
||||
}
|
||||
|
||||
function getCurrentWindowResetAt(now: number) {
|
||||
return Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS + RATE_LIMIT_WINDOW_MS;
|
||||
}
|
||||
|
||||
function getHttpRateLimitName(kind: RateLimitKind, subject: RateLimitSubject): HttpRateLimitName {
|
||||
const suffix = subject === "ip" ? "Ip" : subject === "key" ? "Key" : "AdminKey";
|
||||
return `${kind}${suffix}` as HttpRateLimitName;
|
||||
}
|
||||
|
||||
export function parseBearerToken(request: Request) {
|
||||
@@ -220,32 +311,10 @@ function splitFirstIp(header: string | null) {
|
||||
return trimmed || null;
|
||||
}
|
||||
|
||||
function getDownloadRateLimitScope(request: Request) {
|
||||
try {
|
||||
const url = new URL(request.url);
|
||||
const path = normalizeRateLimitKeyPart(url.pathname.replace(/\/{2,}/g, "/") || "/");
|
||||
const params = new URLSearchParams();
|
||||
|
||||
for (const name of ["slug", "version", "tag"] as const) {
|
||||
const value = url.searchParams.get(name)?.trim();
|
||||
if (value) params.set(name, normalizeRateLimitKeyPart(value));
|
||||
}
|
||||
|
||||
const query = params.toString();
|
||||
return query ? `${path}?${query}` : path;
|
||||
} catch {
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeRateLimitKeyPart(value: string) {
|
||||
return value.slice(0, 500);
|
||||
}
|
||||
|
||||
function shouldTrustForwardedIps() {
|
||||
function shouldTrustClientIpHeaders() {
|
||||
const value = (process.env.TRUST_FORWARDED_IPS ?? "").trim().toLowerCase();
|
||||
// Hardening default: CF-only. Forwarded headers are trivial to spoof unless you
|
||||
// control the trusted proxy layer.
|
||||
// Direct Convex HTTP endpoints can be reached without ClawHub's edge. Trust
|
||||
// client IP headers only when the deployment is explicitly behind that edge.
|
||||
if (!value) return false;
|
||||
if (value === "1" || value === "true" || value === "yes") return true;
|
||||
return false;
|
||||
@@ -254,7 +323,7 @@ function shouldTrustForwardedIps() {
|
||||
function isRateLimitWriteConflict(error: unknown) {
|
||||
if (!(error instanceof Error)) return false;
|
||||
return (
|
||||
(error.message.includes("rateLimitCounters") || error.message.includes("rateLimits")) &&
|
||||
(error.message.includes("rateLimits") || error.message.includes("httpRateLimitKeys")) &&
|
||||
error.message.includes("changed while this mutation was being run")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
/* @vitest-environment node */
|
||||
import { httpRouter } from "convex/server";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { ActionCtx } from "../_generated/server";
|
||||
import { httpAction } from "../functions";
|
||||
import { applyRateLimit, RATE_LIMITS } from "./httpRateLimit";
|
||||
import { installRateLimitedRoutes, rateLimitedHttpAction } from "./httpRouteRateLimit";
|
||||
|
||||
type WrappedHttpAction = {
|
||||
_handler: (ctx: ActionCtx, request: Request) => Promise<Response>;
|
||||
};
|
||||
|
||||
function makeCtx({
|
||||
allowed = true,
|
||||
}: {
|
||||
allowed?: boolean;
|
||||
} = {}) {
|
||||
const runQuery = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
|
||||
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`);
|
||||
});
|
||||
const runMutation = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
|
||||
if ("name" in args && "config" in args) {
|
||||
const config = args.config as { period: number };
|
||||
return allowed ? { ok: true } : { ok: false, retryAfter: config.period };
|
||||
}
|
||||
throw new Error(`Unexpected runMutation args: ${JSON.stringify(args)}`);
|
||||
});
|
||||
return { ctx: { runQuery, runMutation } as unknown as ActionCtx, runQuery, runMutation };
|
||||
}
|
||||
|
||||
describe("rateLimitedHttpAction", () => {
|
||||
it("blocks before the wrapped HTTP handler runs", async () => {
|
||||
const handler = vi.fn(async () => new Response("ok"));
|
||||
const action = httpAction(handler) as unknown as Parameters<typeof rateLimitedHttpAction>[0];
|
||||
const wrapped = rateLimitedHttpAction(action, {
|
||||
resolveRateLimit: () => ({ kind: "read" }),
|
||||
}) as unknown as WrappedHttpAction;
|
||||
const { ctx, runMutation } = makeCtx({ allowed: false });
|
||||
|
||||
const response = await wrapped._handler(ctx, new Request("https://example.com/api/v1/skills"));
|
||||
|
||||
expect(response.status).toBe(429);
|
||||
expect(await response.text()).toBe("Rate limit exceeded");
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
name: "readIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("merges route rate-limit headers into successful responses", async () => {
|
||||
const action = httpAction(
|
||||
async () => new Response("ok", { headers: { "X-App": "1" } }),
|
||||
) as unknown as Parameters<typeof rateLimitedHttpAction>[0];
|
||||
const wrapped = rateLimitedHttpAction(action, {
|
||||
resolveRateLimit: () => ({ kind: "read" }),
|
||||
}) as unknown as WrappedHttpAction;
|
||||
const { ctx } = makeCtx();
|
||||
|
||||
const response = await wrapped._handler(ctx, new Request("https://example.com/api/v1/skills"));
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("X-App")).toBe("1");
|
||||
expect(response.headers.get("RateLimit-Limit")).toBe(String(RATE_LIMITS.read.ip));
|
||||
});
|
||||
|
||||
it("preserves multiple Set-Cookie headers on successful responses", async () => {
|
||||
const action = httpAction(async () => {
|
||||
const headers = new Headers({ "X-App": "1" });
|
||||
headers.append("Set-Cookie", "oauth_state=abc; HttpOnly; Path=/");
|
||||
headers.append("Set-Cookie", "redirect_to=%2Fdashboard; HttpOnly; Path=/");
|
||||
return new Response("ok", { headers });
|
||||
}) as unknown as Parameters<typeof rateLimitedHttpAction>[0];
|
||||
const wrapped = rateLimitedHttpAction(action, {
|
||||
resolveRateLimit: () => ({ kind: "read" }),
|
||||
}) as unknown as WrappedHttpAction;
|
||||
const { ctx } = makeCtx();
|
||||
|
||||
const response = await wrapped._handler(
|
||||
ctx,
|
||||
new Request("https://example.com/api/auth/signin"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("X-App")).toBe("1");
|
||||
expect(response.headers.get("RateLimit-Limit")).toBe(String(RATE_LIMITS.read.ip));
|
||||
expect([...response.headers.entries()].filter(([key]) => key === "set-cookie")).toEqual([
|
||||
["set-cookie", "oauth_state=abc; HttpOnly; Path=/"],
|
||||
["set-cookie", "redirect_to=%2Fdashboard; HttpOnly; Path=/"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("supports dynamic route-specific rate limit overrides", async () => {
|
||||
const action = httpAction(async () => new Response("ok")) as unknown as Parameters<
|
||||
typeof rateLimitedHttpAction
|
||||
>[0];
|
||||
const wrapped = rateLimitedHttpAction(action, {
|
||||
resolveRateLimit: (request) =>
|
||||
new URL(request.url).pathname.endsWith("/download")
|
||||
? { kind: "download" }
|
||||
: { kind: "read" },
|
||||
}) as unknown as WrappedHttpAction;
|
||||
const { ctx, runMutation } = makeCtx();
|
||||
|
||||
await wrapped._handler(ctx, new Request("https://example.com/api/v1/packages/demo/download"));
|
||||
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
name: "downloadIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.download.ip }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("makes wrapper-applied limits authoritative for nested handler checks", async () => {
|
||||
const handler = vi.fn(async (ctx: ActionCtx, request: Request) => {
|
||||
const nestedRate = await applyRateLimit(ctx, request, "write");
|
||||
if (!nestedRate.ok) return nestedRate.response;
|
||||
return new Response("ok", { headers: nestedRate.headers });
|
||||
});
|
||||
const action = httpAction(handler) as unknown as Parameters<typeof rateLimitedHttpAction>[0];
|
||||
const wrapped = rateLimitedHttpAction(action, {
|
||||
resolveRateLimit: () => ({ kind: "read" }),
|
||||
}) as unknown as WrappedHttpAction;
|
||||
const { ctx, runMutation } = makeCtx();
|
||||
|
||||
const response = await wrapped._handler(ctx, new Request("https://example.com/api/v1/search"));
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(runMutation).toHaveBeenCalledTimes(1);
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
name: "readIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
|
||||
}),
|
||||
);
|
||||
expect(response.headers.get("RateLimit-Limit")).toBe(String(RATE_LIMITS.read.ip));
|
||||
expect(response.headers.get("RateLimit-Limit")).not.toContain(",");
|
||||
});
|
||||
|
||||
it("allows explicit route opt-outs", async () => {
|
||||
const action = httpAction(async () => new Response("ok")) as unknown as Parameters<
|
||||
typeof rateLimitedHttpAction
|
||||
>[0];
|
||||
const wrapped = rateLimitedHttpAction(action, {
|
||||
resolveRateLimit: () => ({ kind: "none" }),
|
||||
}) as unknown as WrappedHttpAction;
|
||||
const { ctx, runQuery, runMutation } = makeCtx({ allowed: false });
|
||||
|
||||
const response = await wrapped._handler(ctx, new Request("https://example.com/api/v1/health"));
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(runQuery).not.toHaveBeenCalled();
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("installRateLimitedRoutes", () => {
|
||||
it.each([
|
||||
[
|
||||
"read default",
|
||||
"/api/v1/example",
|
||||
"GET",
|
||||
"https://example.com/api/v1/example",
|
||||
RATE_LIMITS.read.ip,
|
||||
],
|
||||
[
|
||||
"write default",
|
||||
"/api/v1/example",
|
||||
"POST",
|
||||
"https://example.com/api/v1/example",
|
||||
RATE_LIMITS.write.ip,
|
||||
],
|
||||
[
|
||||
"download exact",
|
||||
"/api/v1/download",
|
||||
"GET",
|
||||
"https://example.com/api/v1/download?slug=demo",
|
||||
RATE_LIMITS.download.ip,
|
||||
],
|
||||
[
|
||||
"export exact",
|
||||
"/api/v1/skills/export",
|
||||
"GET",
|
||||
"https://example.com/api/v1/skills/export",
|
||||
RATE_LIMITS.export.ip,
|
||||
],
|
||||
[
|
||||
"trusted publish",
|
||||
"/api/v1/publish/token/mint",
|
||||
"POST",
|
||||
"https://example.com/api/v1/publish/token/mint",
|
||||
RATE_LIMITS.trustedPublish.ip,
|
||||
],
|
||||
])("applies the central %s bucket", async (_name, path, method, requestUrl, expectedLimit) => {
|
||||
const router = installRateLimitedRoutes(httpRouter());
|
||||
router.route({
|
||||
path,
|
||||
method: method as "GET" | "POST",
|
||||
handler: httpAction(async () => new Response("ok")),
|
||||
});
|
||||
const route = router.lookup(path, method as "GET" | "POST");
|
||||
if (!route) throw new Error(`Expected route for ${path}`);
|
||||
const [action] = route;
|
||||
const { ctx, runMutation } = makeCtx();
|
||||
|
||||
await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request(requestUrl, { method }),
|
||||
);
|
||||
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
config: expect.objectContaining({ rate: expectedLimit }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["package detail", "/api/v1/packages/", "/api/v1/packages/demo", RATE_LIMITS.read.ip],
|
||||
[
|
||||
"package version download",
|
||||
"/api/v1/packages/",
|
||||
"/api/v1/packages/demo/versions/1.0.0/download",
|
||||
RATE_LIMITS.download.ip,
|
||||
],
|
||||
[
|
||||
"package artifact",
|
||||
"/api/v1/packages/",
|
||||
"/api/v1/packages/demo/versions/1.0.0/artifact",
|
||||
RATE_LIMITS.download.ip,
|
||||
],
|
||||
[
|
||||
"scoped package named download",
|
||||
"/api/v1/packages/",
|
||||
"/api/v1/packages/@scope/download",
|
||||
RATE_LIMITS.read.ip,
|
||||
],
|
||||
[
|
||||
"scoped package named artifact",
|
||||
"/api/v1/packages/",
|
||||
"/api/v1/packages/@scope/artifact",
|
||||
RATE_LIMITS.read.ip,
|
||||
],
|
||||
["npm metadata", "/api/npm/", "/api/npm/demo", RATE_LIMITS.read.ip],
|
||||
["npm tarball", "/api/npm/", "/api/npm/demo/-/demo-1.0.0.tgz", RATE_LIMITS.download.ip],
|
||||
[
|
||||
"npm tarball with encoded separator",
|
||||
"/api/npm/",
|
||||
"/api/npm/demo/%2D/demo-1.0.0.tgz",
|
||||
RATE_LIMITS.download.ip,
|
||||
],
|
||||
])(
|
||||
"classifies central prefix policy for %s",
|
||||
async (_name, pathPrefix, requestPath, expectedLimit) => {
|
||||
const router = installRateLimitedRoutes(httpRouter());
|
||||
router.route({
|
||||
pathPrefix,
|
||||
method: "GET",
|
||||
handler: httpAction(async () => new Response("ok")),
|
||||
});
|
||||
const route = router.lookup(requestPath, "GET");
|
||||
if (!route) throw new Error(`Expected route for ${requestPath}`);
|
||||
const [action] = route;
|
||||
const { ctx, runMutation } = makeCtx();
|
||||
|
||||
await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request(`https://example.com${requestPath}`),
|
||||
);
|
||||
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
config: expect.objectContaining({ rate: expectedLimit }),
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("wraps ordinary route registrations with the default method limit", async () => {
|
||||
const router = installRateLimitedRoutes(httpRouter());
|
||||
const handler = vi.fn(async () => new Response("ok"));
|
||||
router.route({
|
||||
path: "/api/v1/example",
|
||||
method: "GET",
|
||||
handler: httpAction(handler),
|
||||
});
|
||||
const route = router.lookup("/api/v1/example", "GET");
|
||||
if (!route) throw new Error("Expected wrapped route");
|
||||
const [action] = route;
|
||||
const { ctx, runMutation } = makeCtx({ allowed: false });
|
||||
|
||||
const response = await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request("https://example.com/api/v1/example"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(429);
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
name: "readIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves auth metadata route registrations explicitly unmetered", async () => {
|
||||
const router = installRateLimitedRoutes(httpRouter());
|
||||
router.route({
|
||||
path: "/.well-known/jwks.json",
|
||||
method: "GET",
|
||||
handler: httpAction(async () => new Response("jwks")),
|
||||
});
|
||||
const route = router.lookup("/.well-known/jwks.json", "GET");
|
||||
if (!route) throw new Error("Expected auth metadata route");
|
||||
const [action] = route;
|
||||
const { ctx, runQuery, runMutation } = makeCtx({ allowed: false });
|
||||
|
||||
const response = await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request("https://example.com/.well-known/jwks.json"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.text()).toBe("jwks");
|
||||
expect(runQuery).not.toHaveBeenCalled();
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("supports central route policy overrides", async () => {
|
||||
const router = installRateLimitedRoutes(httpRouter(), {
|
||||
resolveRateLimit: () => ({ kind: "export" }),
|
||||
});
|
||||
router.route({
|
||||
path: "/custom-export",
|
||||
method: "GET",
|
||||
handler: httpAction(async () => new Response("ok")),
|
||||
});
|
||||
const route = router.lookup("/custom-export", "GET");
|
||||
if (!route) throw new Error("Expected custom export route");
|
||||
const [action] = route;
|
||||
const { ctx, runMutation } = makeCtx();
|
||||
|
||||
await (action as unknown as WrappedHttpAction)._handler(
|
||||
ctx,
|
||||
new Request("https://example.com/custom-export"),
|
||||
);
|
||||
|
||||
expect(runMutation).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
name: "exportIp",
|
||||
config: expect.objectContaining({ rate: RATE_LIMITS.export.ip }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,162 @@
|
||||
import { ApiRoutes, LegacyApiRoutes } from "clawhub-schema";
|
||||
import type { HttpRouter, PublicHttpAction, RouteSpec } from "convex/server";
|
||||
import { httpAction } from "../functions";
|
||||
import { getPathSegments, parsePackagePathSegments } from "./httpPathSegments";
|
||||
import { applyRateLimit, markRateLimitApplied, RATE_LIMITS } from "./httpRateLimit";
|
||||
|
||||
type HttpHandler = Parameters<typeof httpAction>[0];
|
||||
|
||||
type ConvexHttpActionWithHandler = PublicHttpAction & {
|
||||
readonly _handler: HttpHandler;
|
||||
};
|
||||
|
||||
type RouteRateLimitKind = keyof typeof RATE_LIMITS;
|
||||
type RouteRateLimitDecision = { kind: "none" } | { kind: RouteRateLimitKind };
|
||||
|
||||
type RateLimitedHttpActionOptions = {
|
||||
resolveRateLimit: (request: Request) => RouteRateLimitDecision;
|
||||
};
|
||||
|
||||
type RouteRateLimitResolver = (spec: RouteSpec, request: Request) => RouteRateLimitDecision;
|
||||
|
||||
type InstallRateLimitedRoutesOptions = {
|
||||
resolveRateLimit?: RouteRateLimitResolver;
|
||||
};
|
||||
|
||||
const authMetadataPaths = new Set(["/.well-known/openid-configuration", "/.well-known/jwks.json"]);
|
||||
|
||||
export function installRateLimitedRoutes(
|
||||
http: HttpRouter,
|
||||
options: InstallRateLimitedRoutesOptions = {},
|
||||
): HttpRouter {
|
||||
const route = http.route.bind(http);
|
||||
|
||||
// Convex has no HTTP middleware hook, so install one wrapper at registration time.
|
||||
http.route = ((spec: RouteSpec) => {
|
||||
route({
|
||||
...spec,
|
||||
handler: rateLimitedHttpAction(spec.handler, {
|
||||
resolveRateLimit: (request) =>
|
||||
options.resolveRateLimit?.(spec, request) ?? resolveDefaultRouteRateLimit(spec, request),
|
||||
}),
|
||||
});
|
||||
}) as typeof http.route;
|
||||
|
||||
return http;
|
||||
}
|
||||
|
||||
function resolveDefaultRouteRateLimit(spec: RouteSpec, request: Request): RouteRateLimitDecision {
|
||||
if (spec.method === "OPTIONS") return { kind: "none" };
|
||||
|
||||
const routedPath = getRoutedPath(spec);
|
||||
if (authMetadataPaths.has(routedPath)) return { kind: "none" };
|
||||
|
||||
if (routedPath === ApiRoutes.publishTokenMint) return { kind: "trustedPublish" };
|
||||
if (routedPath === ApiRoutes.skillsExport || routedPath === ApiRoutes.pluginsExport) {
|
||||
return { kind: "export" };
|
||||
}
|
||||
|
||||
if (spec.method === "GET") {
|
||||
if (routedPath === ApiRoutes.download || routedPath === LegacyApiRoutes.download) {
|
||||
return { kind: "download" };
|
||||
}
|
||||
if (routedPath === "/api/v1/package-inspector/artifact") return { kind: "download" };
|
||||
if ("pathPrefix" in spec && spec.pathPrefix === `${ApiRoutes.packages}/`) {
|
||||
return packageReadRouteRateLimitKind(request);
|
||||
}
|
||||
if ("pathPrefix" in spec && spec.pathPrefix === "/api/npm/") {
|
||||
return npmMirrorRouteRateLimitKind(request);
|
||||
}
|
||||
}
|
||||
|
||||
if (spec.method === "POST" && routedPath === `${ApiRoutes.skills}/-/security-verdicts`) {
|
||||
return { kind: "read" };
|
||||
}
|
||||
|
||||
return defaultRouteRateLimitKind(spec.method);
|
||||
}
|
||||
|
||||
function defaultRouteRateLimitKind(method: RouteSpec["method"]): RouteRateLimitDecision {
|
||||
if (method === "GET") return { kind: "read" };
|
||||
if (method === "OPTIONS") return { kind: "none" };
|
||||
return { kind: "write" };
|
||||
}
|
||||
|
||||
export function rateLimitedHttpAction(
|
||||
action: PublicHttpAction,
|
||||
options: RateLimitedHttpActionOptions,
|
||||
): PublicHttpAction {
|
||||
const handler = getRegisteredHttpHandler(action);
|
||||
return httpAction(async (ctx, request) => {
|
||||
const decision = options.resolveRateLimit(request);
|
||||
|
||||
if (decision.kind === "none") {
|
||||
return await handler(ctx, request);
|
||||
}
|
||||
|
||||
const rate = await applyRateLimit(ctx, request, decision.kind);
|
||||
if (!rate.ok) return rate.response;
|
||||
markRateLimitApplied(request, rate.headers);
|
||||
|
||||
const response = await handler(ctx, request);
|
||||
return addRateLimitHeaders(response, rate.headers);
|
||||
});
|
||||
}
|
||||
|
||||
function addRateLimitHeaders(response: Response, headers: HeadersInit): Response {
|
||||
const rateHeaders = new Headers(headers);
|
||||
try {
|
||||
for (const [key, value] of rateHeaders) {
|
||||
response.headers.set(key, value);
|
||||
}
|
||||
return response;
|
||||
} catch {
|
||||
const mergedHeaders = new Headers(response.headers);
|
||||
for (const [key, value] of rateHeaders) {
|
||||
mergedHeaders.set(key, value);
|
||||
}
|
||||
return new Response(response.body, {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: mergedHeaders,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function getRegisteredHttpHandler(action: PublicHttpAction): HttpHandler {
|
||||
if (!hasRegisteredHttpHandler(action)) {
|
||||
throw new Error("HTTP action is missing its registered handler");
|
||||
}
|
||||
return action._handler;
|
||||
}
|
||||
|
||||
function hasRegisteredHttpHandler(action: PublicHttpAction): action is ConvexHttpActionWithHandler {
|
||||
return typeof (action as { _handler?: unknown })._handler === "function";
|
||||
}
|
||||
|
||||
function getRoutedPath(spec: RouteSpec): string {
|
||||
return "path" in spec ? spec.path : spec.pathPrefix;
|
||||
}
|
||||
|
||||
function packageReadRouteRateLimitKind(request: Request): RouteRateLimitDecision {
|
||||
const packageRoute = parsePackagePathSegments(getPathSegments(request, `${ApiRoutes.packages}/`));
|
||||
const packageSegments = packageRoute?.rest ?? [];
|
||||
if (
|
||||
packageSegments[0] === "download" ||
|
||||
(packageSegments[0] === "versions" &&
|
||||
packageSegments[1] &&
|
||||
(packageSegments[2] === "download" ||
|
||||
packageSegments[2] === "artifact" ||
|
||||
packageSegments[3] === "download"))
|
||||
) {
|
||||
return { kind: "download" };
|
||||
}
|
||||
return { kind: "read" };
|
||||
}
|
||||
|
||||
function npmMirrorRouteRateLimitKind(request: Request): RouteRateLimitDecision {
|
||||
const packageRoute = parsePackagePathSegments(getPathSegments(request, "/api/npm/"));
|
||||
return packageRoute?.rest[0] === "-" && packageRoute.rest[1]
|
||||
? { kind: "download" }
|
||||
: { kind: "read" };
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export const RATE_LIMIT_WINDOW_MS = 60_000;
|
||||
// Keep enough shards to spread bursty writes while limiting per-window row growth.
|
||||
export const RATE_LIMIT_COUNTER_SHARDS = 16;
|
||||
@@ -33,10 +33,11 @@ describe("retention policies", () => {
|
||||
});
|
||||
|
||||
it("documents active expiring operational tables", () => {
|
||||
expect(getRetentionPolicy("rateLimitCounters")).toMatchObject({
|
||||
expect(getRetentionPolicy("httpRateLimitKeys")).toMatchObject({
|
||||
classification: "ephemeral",
|
||||
expirationField: "expiresAt",
|
||||
expirationIndex: "by_expires_at",
|
||||
prune: "rateLimits.pruneHttpRateLimitKeysInternal",
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -200,12 +200,15 @@ export const RETENTION_POLICIES = {
|
||||
prune: "usage-time expiry plus pending retention cleanup",
|
||||
retention: "Device code TTL.",
|
||||
}),
|
||||
rateLimitCounters: ephemeral("Active rate-limit counters expire after their rate-limit window.", {
|
||||
expirationField: "expiresAt",
|
||||
expirationIndex: "by_expires_at",
|
||||
prune: "rateLimits.pruneRateLimitCountersInternal",
|
||||
retention: "Rate-limit window plus buffer.",
|
||||
}),
|
||||
httpRateLimitKeys: ephemeral(
|
||||
"Component-backed HTTP rate-limit key metadata is operational cleanup state.",
|
||||
{
|
||||
expirationField: "expiresAt",
|
||||
expirationIndex: "by_expires_at",
|
||||
prune: "rateLimits.pruneHttpRateLimitKeysInternal",
|
||||
retention: "Idle component key window plus buffer before component reset.",
|
||||
},
|
||||
),
|
||||
downloadMetricDedupes: ephemeral(
|
||||
"Download dedupe rows are only needed for recent metric windows.",
|
||||
{
|
||||
|
||||
+402
-144
@@ -2,191 +2,449 @@
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
consumeRateLimitInternal,
|
||||
getRateLimitStatusInternal,
|
||||
pruneRateLimitCountersInternal,
|
||||
consumeHttpRateLimitKeyInternal,
|
||||
pruneHttpRateLimitKeysInternal,
|
||||
touchHttpRateLimitKeyInternal,
|
||||
} from "./rateLimits";
|
||||
|
||||
type WrappedHandler<TArgs, TResult> = {
|
||||
_handler: (ctx: unknown, args: TArgs) => Promise<TResult>;
|
||||
};
|
||||
|
||||
const getStatusHandler = (
|
||||
getRateLimitStatusInternal as unknown as WrappedHandler<
|
||||
{ key: string; limit: number; windowMs: number },
|
||||
{ allowed: boolean; remaining: number; limit: number; resetAt: number }
|
||||
const touchHttpKeyHandler = (
|
||||
touchHttpRateLimitKeyInternal as unknown as WrappedHandler<
|
||||
{ name: string; key: string; shard?: number; now?: number; ttlMs?: number },
|
||||
{ action: "inserted" | "updated"; expiresAt: number; shard: number }
|
||||
>
|
||||
)._handler;
|
||||
|
||||
const consumeHandler = (
|
||||
consumeRateLimitInternal as unknown as WrappedHandler<
|
||||
{ key: string; limit: number; windowMs: number; shard?: number },
|
||||
{ allowed: boolean; remaining: number }
|
||||
const consumeHttpKeyHandler = (
|
||||
consumeHttpRateLimitKeyInternal as unknown as WrappedHandler<
|
||||
{
|
||||
name: string;
|
||||
key: string;
|
||||
config: {
|
||||
kind: "fixed window";
|
||||
rate: number;
|
||||
period: number;
|
||||
start?: number;
|
||||
shards?: number;
|
||||
};
|
||||
now?: number;
|
||||
ttlMs?: number;
|
||||
shard?: number;
|
||||
},
|
||||
{ ok: true; retryAfter?: number } | { ok: false; retryAfter: number }
|
||||
>
|
||||
)._handler;
|
||||
|
||||
const pruneHandler = (
|
||||
pruneRateLimitCountersInternal as unknown as WrappedHandler<
|
||||
const pruneHttpKeyHandler = (
|
||||
pruneHttpRateLimitKeysInternal as unknown as WrappedHandler<
|
||||
{ batchSize?: number },
|
||||
{ deleted: number; hasMore: boolean }
|
||||
>
|
||||
)._handler;
|
||||
|
||||
describe("rate limit sharding", () => {
|
||||
it("sums active counter rows without reading legacy rate limit tables", async () => {
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn(() => ({
|
||||
withIndex: vi.fn(() => ({
|
||||
collect: vi.fn(async () => [{ count: 4 }, { count: 5 }]),
|
||||
})),
|
||||
})),
|
||||
},
|
||||
};
|
||||
function makeDb(overrides: { query: ReturnType<typeof vi.fn>; delete: ReturnType<typeof vi.fn> }) {
|
||||
return {
|
||||
get: vi.fn(),
|
||||
insert: vi.fn(),
|
||||
patch: vi.fn(),
|
||||
replace: vi.fn(),
|
||||
delete: overrides.delete,
|
||||
query: overrides.query,
|
||||
normalizeId: vi.fn(() => null),
|
||||
system: {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const result = await getStatusHandler(ctx, {
|
||||
key: "ip:test",
|
||||
limit: 20,
|
||||
windowMs: 60_000,
|
||||
});
|
||||
|
||||
expect(result.allowed).toBe(true);
|
||||
expect(result.remaining).toBe(11);
|
||||
expect(ctx.db.query).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.db.query).toHaveBeenCalledWith("rateLimitCounters");
|
||||
expect(ctx.db.query).not.toHaveBeenCalledWith("rateLimits");
|
||||
});
|
||||
|
||||
it("writes only the selected active counter shard when consuming", async () => {
|
||||
describe("component HTTP rate limit key metadata", () => {
|
||||
it("consumes a component bucket and refreshes sharded metadata in one mutation", async () => {
|
||||
const take = vi.fn(async () => []);
|
||||
const insert = vi.fn();
|
||||
const runMutation = vi.fn(async () => ({ ok: true }));
|
||||
const eqShard = vi.fn();
|
||||
const eqKey = vi.fn(() => ({ eq: eqShard }));
|
||||
const eqName = vi.fn(() => ({ eq: eqKey }));
|
||||
const withIndex = vi.fn((_index, builder) => {
|
||||
builder({
|
||||
eq: vi.fn(() => ({
|
||||
eq: vi.fn(() => ({
|
||||
eq: vi.fn(),
|
||||
})),
|
||||
})),
|
||||
});
|
||||
return { first: vi.fn(async () => null) };
|
||||
builder({ eq: eqName });
|
||||
return { take };
|
||||
});
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn(() => ({ withIndex })),
|
||||
get: vi.fn(),
|
||||
normalizeId: vi.fn(),
|
||||
insert,
|
||||
patch: vi.fn(),
|
||||
replace: vi.fn(),
|
||||
runMutation,
|
||||
db: makeDb({
|
||||
query: vi.fn(() => ({
|
||||
withIndex,
|
||||
})),
|
||||
delete: vi.fn(),
|
||||
system: {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(),
|
||||
},
|
||||
}),
|
||||
scheduler: {
|
||||
runAfter: vi.fn(),
|
||||
},
|
||||
};
|
||||
ctx.db.insert = insert;
|
||||
|
||||
await consumeHandler(ctx, {
|
||||
key: "ip:test",
|
||||
limit: 20,
|
||||
windowMs: 60_000,
|
||||
const result = await consumeHttpKeyHandler(ctx, {
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
config: { kind: "fixed window", rate: 1200, period: 60_000, start: 0, shards: 16 },
|
||||
now: 10_000,
|
||||
ttlMs: 60_000,
|
||||
shard: 7,
|
||||
});
|
||||
|
||||
expect(withIndex).toHaveBeenCalledWith("by_key_window_shard", expect.any(Function));
|
||||
expect(insert).toHaveBeenCalledWith(
|
||||
"rateLimitCounters",
|
||||
expect.objectContaining({
|
||||
key: "ip:test",
|
||||
shard: 7,
|
||||
count: 1,
|
||||
expiresAt: expect.any(Number),
|
||||
}),
|
||||
);
|
||||
expect(result).toEqual({ ok: true });
|
||||
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
config: { kind: "fixed window", rate: 1200, period: 60_000, start: 0, shards: 16 },
|
||||
});
|
||||
expect(withIndex).toHaveBeenCalledWith("by_name_and_key_and_shard", expect.any(Function));
|
||||
expect(eqName).toHaveBeenCalledWith("name", "downloadIp");
|
||||
expect(eqKey).toHaveBeenCalledWith("key", "ip:203.0.113.1:download");
|
||||
expect(eqShard).toHaveBeenCalledWith("shard", 7);
|
||||
expect(insert).toHaveBeenCalledWith("httpRateLimitKeys", {
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
shard: 7,
|
||||
lastTouchedAt: 10_000,
|
||||
expiresAt: 70_000,
|
||||
});
|
||||
});
|
||||
|
||||
it("clamps out-of-range shard inputs to the active counter shard range", async () => {
|
||||
const insert = vi.fn();
|
||||
it("inserts sharded metadata for a newly observed component key", async () => {
|
||||
const take = vi.fn(async () => []);
|
||||
const eqShard = vi.fn();
|
||||
const eqKey = vi.fn(() => ({ eq: eqShard }));
|
||||
const eqName = vi.fn(() => ({ eq: eqKey }));
|
||||
const withIndex = vi.fn((_index, builder) => {
|
||||
builder({
|
||||
eq: vi.fn(() => ({
|
||||
eq: vi.fn(() => ({
|
||||
eq: vi.fn(),
|
||||
})),
|
||||
})),
|
||||
});
|
||||
return { first: vi.fn(async () => null) };
|
||||
});
|
||||
const ctx = {
|
||||
db: {
|
||||
query: vi.fn(() => ({ withIndex })),
|
||||
get: vi.fn(),
|
||||
normalizeId: vi.fn(),
|
||||
insert,
|
||||
patch: vi.fn(),
|
||||
replace: vi.fn(),
|
||||
delete: vi.fn(),
|
||||
system: {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await consumeHandler(ctx, {
|
||||
key: "ip:test",
|
||||
limit: 20,
|
||||
windowMs: 60_000,
|
||||
shard: 999,
|
||||
});
|
||||
|
||||
expect(insert).toHaveBeenCalledWith(
|
||||
"rateLimitCounters",
|
||||
expect.objectContaining({
|
||||
shard: 15,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("prunes only expired active counter rows in bounded batches", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
|
||||
const stale = [
|
||||
{ _id: "rateLimitCounters:a", expiresAt: 930_000 },
|
||||
{ _id: "rateLimitCounters:b", expiresAt: 940_000 },
|
||||
];
|
||||
const take = vi.fn(async () => stale);
|
||||
const withIndex = vi.fn((_index, builder) => {
|
||||
builder({ lt: vi.fn() });
|
||||
builder({ eq: eqName });
|
||||
return { take };
|
||||
});
|
||||
const deleteRow = vi.fn();
|
||||
const insert = vi.fn();
|
||||
const ctx = {
|
||||
db: {
|
||||
db: makeDb({
|
||||
query: vi.fn(() => ({ withIndex })),
|
||||
get: vi.fn(),
|
||||
normalizeId: vi.fn(),
|
||||
insert: vi.fn(),
|
||||
patch: vi.fn(),
|
||||
replace: vi.fn(),
|
||||
delete: deleteRow,
|
||||
system: {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(),
|
||||
},
|
||||
delete: vi.fn(),
|
||||
}),
|
||||
scheduler: {
|
||||
runAfter: vi.fn(),
|
||||
},
|
||||
};
|
||||
ctx.db.insert = insert;
|
||||
|
||||
const result = await touchHttpKeyHandler(ctx, {
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
shard: 3,
|
||||
now: 10_000,
|
||||
ttlMs: 60_000,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ action: "inserted", expiresAt: 70_000, shard: 3 });
|
||||
expect(ctx.db.query).toHaveBeenCalledWith("httpRateLimitKeys");
|
||||
expect(withIndex).toHaveBeenCalledWith("by_name_and_key_and_shard", expect.any(Function));
|
||||
expect(eqName).toHaveBeenCalledWith("name", "downloadIp");
|
||||
expect(eqKey).toHaveBeenCalledWith("key", "ip:203.0.113.1:download");
|
||||
expect(eqShard).toHaveBeenCalledWith("shard", 3);
|
||||
expect(insert).toHaveBeenCalledWith("httpRateLimitKeys", {
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
shard: 3,
|
||||
lastTouchedAt: 10_000,
|
||||
expiresAt: 70_000,
|
||||
});
|
||||
});
|
||||
|
||||
it("refreshes metadata for an existing component key shard", async () => {
|
||||
const existing = {
|
||||
_id: "httpRateLimitKeys:1",
|
||||
name: "readKey",
|
||||
key: "user:users_123:read",
|
||||
shard: 11,
|
||||
lastTouchedAt: 1_000,
|
||||
expiresAt: 61_000,
|
||||
};
|
||||
const ctx = {
|
||||
db: makeDb({
|
||||
query: vi.fn(() => ({
|
||||
withIndex: vi.fn(() => ({ take: vi.fn(async () => [existing]) })),
|
||||
})),
|
||||
delete: vi.fn(),
|
||||
}),
|
||||
scheduler: {
|
||||
runAfter: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
const result = await pruneHandler(ctx, { batchSize: 10 });
|
||||
const result = await touchHttpKeyHandler(ctx, {
|
||||
name: "readKey",
|
||||
key: "user:users_123:read",
|
||||
shard: 11,
|
||||
now: 20_000,
|
||||
ttlMs: 60_000,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ action: "updated", expiresAt: 80_000, shard: 11 });
|
||||
expect(ctx.db.patch).toHaveBeenCalledWith("httpRateLimitKeys:1", {
|
||||
lastTouchedAt: 20_000,
|
||||
expiresAt: 80_000,
|
||||
});
|
||||
expect(ctx.db.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("repairs duplicate metadata rows while refreshing a component key shard", async () => {
|
||||
const existing = {
|
||||
_id: "httpRateLimitKeys:1",
|
||||
name: "readKey",
|
||||
key: "user:users_123:read",
|
||||
shard: 11,
|
||||
lastTouchedAt: 1_000,
|
||||
expiresAt: 61_000,
|
||||
};
|
||||
const duplicate = {
|
||||
...existing,
|
||||
_id: "httpRateLimitKeys:2",
|
||||
};
|
||||
const ctx = {
|
||||
db: makeDb({
|
||||
query: vi.fn(() => ({
|
||||
withIndex: vi.fn(() => ({ take: vi.fn(async () => [existing, duplicate]) })),
|
||||
})),
|
||||
delete: vi.fn(),
|
||||
}),
|
||||
scheduler: {
|
||||
runAfter: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
const result = await touchHttpKeyHandler(ctx, {
|
||||
name: "readKey",
|
||||
key: "user:users_123:read",
|
||||
shard: 11,
|
||||
now: 20_000,
|
||||
ttlMs: 60_000,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ action: "updated", expiresAt: 80_000, shard: 11 });
|
||||
expect(ctx.db.patch).toHaveBeenCalledWith("httpRateLimitKeys:1", {
|
||||
lastTouchedAt: 20_000,
|
||||
expiresAt: 80_000,
|
||||
});
|
||||
expect(ctx.db.delete).toHaveBeenCalledWith("httpRateLimitKeys:2");
|
||||
});
|
||||
|
||||
it("keeps component buckets when another shard is still active", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
|
||||
const stale = {
|
||||
_id: "httpRateLimitKeys:a",
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
shard: 2,
|
||||
expiresAt: 900_000,
|
||||
};
|
||||
const active = {
|
||||
_id: "httpRateLimitKeys:b",
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
shard: 9,
|
||||
expiresAt: 1_050_000,
|
||||
};
|
||||
const staleTake = vi.fn(async () => [stale]);
|
||||
const activeTake = vi.fn(async () => [active]);
|
||||
const withIndex = vi
|
||||
.fn()
|
||||
.mockImplementationOnce((_index, builder) => {
|
||||
builder({ lt: vi.fn() });
|
||||
return { take: staleTake };
|
||||
})
|
||||
.mockImplementationOnce((_index, builder) => {
|
||||
builder({ eq: vi.fn(() => ({ eq: vi.fn(() => ({ gte: vi.fn() })) })) });
|
||||
return { take: activeTake };
|
||||
});
|
||||
const runMutation = vi.fn();
|
||||
const deleteRow = vi.fn();
|
||||
const ctx = {
|
||||
runMutation,
|
||||
db: makeDb({
|
||||
query: vi.fn(() => ({ withIndex })),
|
||||
delete: deleteRow,
|
||||
}),
|
||||
scheduler: {
|
||||
runAfter: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
const result = await pruneHttpKeyHandler(ctx, { batchSize: 10 });
|
||||
|
||||
expect(result).toEqual({ deleted: 1, hasMore: false });
|
||||
expect(withIndex).toHaveBeenNthCalledWith(1, "by_expires_at", expect.any(Function));
|
||||
expect(withIndex).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
"by_name_and_key_and_expires_at",
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(activeTake).toHaveBeenCalledWith(1);
|
||||
expect(runMutation).not.toHaveBeenCalled();
|
||||
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:a");
|
||||
});
|
||||
|
||||
it("resets component buckets once before deleting fully expired key metadata", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
|
||||
const stale = [
|
||||
{
|
||||
_id: "httpRateLimitKeys:a",
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
shard: 2,
|
||||
expiresAt: 900_000,
|
||||
},
|
||||
{
|
||||
_id: "httpRateLimitKeys:b",
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
shard: 9,
|
||||
expiresAt: 910_000,
|
||||
},
|
||||
];
|
||||
const staleTake = vi.fn(async () => stale);
|
||||
const activeTake = vi.fn(async () => []);
|
||||
const expiredTake = vi.fn(async () => stale);
|
||||
const withIndex = vi
|
||||
.fn()
|
||||
.mockImplementationOnce((_index, builder) => {
|
||||
builder({ lt: vi.fn() });
|
||||
return { take: staleTake };
|
||||
})
|
||||
.mockImplementationOnce((_index, builder) => {
|
||||
builder({ eq: vi.fn(() => ({ eq: vi.fn(() => ({ gte: vi.fn() })) })) });
|
||||
return { take: activeTake };
|
||||
})
|
||||
.mockImplementationOnce((_index, builder) => {
|
||||
builder({ eq: vi.fn(() => ({ eq: vi.fn(() => ({ lt: vi.fn() })) })) });
|
||||
return { take: expiredTake };
|
||||
});
|
||||
const runMutation = vi.fn();
|
||||
const deleteRow = vi.fn();
|
||||
const ctx = {
|
||||
runMutation,
|
||||
db: makeDb({
|
||||
query: vi.fn(() => ({ withIndex })),
|
||||
delete: deleteRow,
|
||||
}),
|
||||
scheduler: {
|
||||
runAfter: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
const result = await pruneHttpKeyHandler(ctx, { batchSize: 10 });
|
||||
|
||||
expect(ctx.db.query).toHaveBeenCalledWith("rateLimitCounters");
|
||||
expect(withIndex).toHaveBeenCalledWith("by_expires_at", expect.any(Function));
|
||||
expect(take).toHaveBeenCalledWith(10);
|
||||
expect(deleteRow).toHaveBeenCalledTimes(2);
|
||||
expect(deleteRow).toHaveBeenCalledWith("rateLimitCounters:a");
|
||||
expect(deleteRow).toHaveBeenCalledWith("rateLimitCounters:b");
|
||||
expect(ctx.scheduler.runAfter).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({ deleted: 2, hasMore: false });
|
||||
expect(staleTake).toHaveBeenCalledWith(10);
|
||||
expect(activeTake).toHaveBeenCalledWith(1);
|
||||
expect(expiredTake).toHaveBeenCalledWith(128);
|
||||
expect(runMutation).toHaveBeenCalledTimes(1);
|
||||
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
});
|
||||
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:a");
|
||||
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:b");
|
||||
expect(ctx.scheduler.runAfter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resets independent expired component keys before deleting metadata rows", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
|
||||
const first = {
|
||||
_id: "httpRateLimitKeys:a",
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
shard: 2,
|
||||
expiresAt: 900_000,
|
||||
};
|
||||
const second = {
|
||||
_id: "httpRateLimitKeys:b",
|
||||
name: "writeKey",
|
||||
key: "user:users_123:write",
|
||||
shard: 4,
|
||||
expiresAt: 910_000,
|
||||
};
|
||||
const staleTake = vi.fn(async () => [first, second]);
|
||||
const withIndex = vi.fn((_index, builder) => {
|
||||
builder({
|
||||
eq: vi.fn(() => ({ eq: vi.fn(() => ({ gte: vi.fn(), lt: vi.fn() })) })),
|
||||
lt: vi.fn(),
|
||||
});
|
||||
return { take };
|
||||
});
|
||||
const take = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce([first, second])
|
||||
.mockResolvedValueOnce([])
|
||||
.mockResolvedValueOnce([first])
|
||||
.mockResolvedValueOnce([])
|
||||
.mockResolvedValueOnce([second]);
|
||||
const runMutation = vi.fn();
|
||||
const deleteRow = vi.fn();
|
||||
const ctx = {
|
||||
runMutation,
|
||||
db: makeDb({
|
||||
query: vi.fn(() => ({ withIndex })),
|
||||
delete: deleteRow,
|
||||
}),
|
||||
scheduler: {
|
||||
runAfter: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
const result = await pruneHttpKeyHandler(ctx, { batchSize: 10 });
|
||||
|
||||
expect(result).toEqual({ deleted: 2, hasMore: false });
|
||||
expect(ctx.db.query).toHaveBeenCalledWith("httpRateLimitKeys");
|
||||
expect(withIndex).toHaveBeenCalledWith("by_expires_at", expect.any(Function));
|
||||
expect(staleTake).not.toHaveBeenCalled();
|
||||
expect(take).toHaveBeenCalledWith(10);
|
||||
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
|
||||
name: "downloadIp",
|
||||
key: "ip:203.0.113.1:download",
|
||||
});
|
||||
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
|
||||
name: "writeKey",
|
||||
key: "user:users_123:write",
|
||||
});
|
||||
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:a");
|
||||
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:b");
|
||||
expect(ctx.scheduler.runAfter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("continues pruning expired component keys when a full bounded batch is deleted", async () => {
|
||||
const stale = Array.from({ length: 3 }, (_, index) => ({
|
||||
_id: `httpRateLimitKeys:${index}`,
|
||||
name: "downloadIp",
|
||||
key: `ip:203.0.113.${index}:download`,
|
||||
expiresAt: 900_000 + index,
|
||||
}));
|
||||
const ctx = {
|
||||
runMutation: vi.fn(),
|
||||
db: makeDb({
|
||||
query: vi.fn(() => ({
|
||||
withIndex: vi.fn(() => ({ take: vi.fn(async () => stale) })),
|
||||
})),
|
||||
delete: vi.fn(),
|
||||
}),
|
||||
scheduler: {
|
||||
runAfter: vi.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
const result = await pruneHttpKeyHandler(ctx, { batchSize: 3 });
|
||||
|
||||
expect(result).toEqual({ deleted: 3, hasMore: true });
|
||||
expect(ctx.scheduler.runAfter).toHaveBeenCalledWith(
|
||||
0,
|
||||
expect.anything(),
|
||||
expect.objectContaining({ batchSize: 3 }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
+186
-94
@@ -1,127 +1,219 @@
|
||||
import { v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import { internalMutation, internalQuery } from "./functions";
|
||||
import { RATE_LIMIT_COUNTER_SHARDS } from "./lib/rateLimitConfig";
|
||||
import { components, internal } from "./_generated/api";
|
||||
import type { MutationCtx } from "./_generated/server";
|
||||
import { internalMutation } from "./functions";
|
||||
import { RETENTION_STANDARD_BATCH_SIZE } from "./lib/retentionPolicy";
|
||||
|
||||
const RATE_LIMIT_COUNTER_RETENTION_BUFFER_MS = 5 * 60_000;
|
||||
const DEFAULT_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE = RETENTION_STANDARD_BATCH_SIZE;
|
||||
const MAX_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE = 1_000;
|
||||
|
||||
/**
|
||||
* Read-only rate limit check. Returns current status without writing anything.
|
||||
* This eliminates write conflicts for denied requests entirely.
|
||||
*/
|
||||
export const getRateLimitStatusInternal = internalQuery({
|
||||
args: {
|
||||
key: v.string(),
|
||||
limit: v.number(),
|
||||
windowMs: v.number(),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now();
|
||||
const windowStart = Math.floor(now / args.windowMs) * args.windowMs;
|
||||
const resetAt = windowStart + args.windowMs;
|
||||
if (args.limit <= 0) {
|
||||
return { allowed: false, remaining: 0, limit: args.limit, resetAt };
|
||||
}
|
||||
|
||||
const shardRows = await ctx.db
|
||||
.query("rateLimitCounters")
|
||||
.withIndex("by_key_window", (q) => q.eq("key", args.key).eq("windowStart", windowStart))
|
||||
.collect();
|
||||
|
||||
const count = shardRows.reduce((sum, row) => sum + row.count, 0);
|
||||
const allowed = count < args.limit;
|
||||
return {
|
||||
allowed,
|
||||
remaining: Math.max(0, args.limit - count),
|
||||
limit: args.limit,
|
||||
resetAt,
|
||||
};
|
||||
},
|
||||
const DEFAULT_HTTP_RATE_LIMIT_KEY_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
const DEFAULT_PRUNE_HTTP_RATE_LIMIT_KEYS_BATCH_SIZE = RETENTION_STANDARD_BATCH_SIZE;
|
||||
const MAX_PRUNE_HTTP_RATE_LIMIT_KEYS_BATCH_SIZE = 1_000;
|
||||
const HTTP_RATE_LIMIT_KEY_METADATA_SHARDS = 64;
|
||||
const MAX_EXPIRED_HTTP_RATE_LIMIT_KEY_ROWS_PER_KEY = HTTP_RATE_LIMIT_KEY_METADATA_SHARDS * 2;
|
||||
const fixedWindowRateLimitConfigValidator = v.object({
|
||||
kind: v.literal("fixed window"),
|
||||
rate: v.number(),
|
||||
period: v.number(),
|
||||
capacity: v.optional(v.number()),
|
||||
maxReserved: v.optional(v.number()),
|
||||
shards: v.optional(v.number()),
|
||||
start: v.optional(v.number()),
|
||||
});
|
||||
const rateLimitStatusValidator = v.union(
|
||||
v.object({
|
||||
ok: v.literal(true),
|
||||
retryAfter: v.optional(v.number()),
|
||||
}),
|
||||
v.object({
|
||||
ok: v.literal(false),
|
||||
retryAfter: v.number(),
|
||||
}),
|
||||
);
|
||||
|
||||
/**
|
||||
* Consume one rate limit token. Only call this after getRateLimitStatusInternal
|
||||
* returns allowed=true. Includes a double-check to handle races between the
|
||||
* query and this mutation.
|
||||
*/
|
||||
export const consumeRateLimitInternal = internalMutation({
|
||||
type HttpRateLimitKeyWriteCtx = Pick<MutationCtx, "db">;
|
||||
|
||||
function clampBatchSize(
|
||||
requested: number | undefined,
|
||||
defaultBatchSize: number,
|
||||
maxBatchSize: number,
|
||||
) {
|
||||
const requestedBatchSize = Number.isFinite(requested)
|
||||
? Math.floor(requested ?? defaultBatchSize)
|
||||
: defaultBatchSize;
|
||||
return Math.max(1, Math.min(requestedBatchSize, maxBatchSize));
|
||||
}
|
||||
|
||||
function normalizeHttpRateLimitKeyShard(shard: number | undefined) {
|
||||
const candidate = Number.isFinite(shard)
|
||||
? Math.floor(shard ?? 0)
|
||||
: Math.floor(Math.random() * HTTP_RATE_LIMIT_KEY_METADATA_SHARDS);
|
||||
return (
|
||||
((candidate % HTTP_RATE_LIMIT_KEY_METADATA_SHARDS) + HTTP_RATE_LIMIT_KEY_METADATA_SHARDS) %
|
||||
HTTP_RATE_LIMIT_KEY_METADATA_SHARDS
|
||||
);
|
||||
}
|
||||
|
||||
async function touchHttpRateLimitKey(
|
||||
ctx: HttpRateLimitKeyWriteCtx,
|
||||
args: {
|
||||
name: string;
|
||||
key: string;
|
||||
shard?: number;
|
||||
now?: number;
|
||||
ttlMs?: number;
|
||||
},
|
||||
) {
|
||||
const now = Number.isFinite(args.now) ? (args.now ?? Date.now()) : Date.now();
|
||||
const ttlMs =
|
||||
Number.isFinite(args.ttlMs) && (args.ttlMs ?? 0) > 0
|
||||
? Math.floor(args.ttlMs ?? DEFAULT_HTTP_RATE_LIMIT_KEY_TTL_MS)
|
||||
: DEFAULT_HTTP_RATE_LIMIT_KEY_TTL_MS;
|
||||
const expiresAt = now + ttlMs;
|
||||
const shard = normalizeHttpRateLimitKeyShard(args.shard);
|
||||
const matches = await ctx.db
|
||||
.query("httpRateLimitKeys")
|
||||
.withIndex("by_name_and_key_and_shard", (q) =>
|
||||
q.eq("name", args.name).eq("key", args.key).eq("shard", shard),
|
||||
)
|
||||
.take(10);
|
||||
const [existing, ...duplicates] = matches;
|
||||
|
||||
if (existing) {
|
||||
await ctx.db.patch(existing._id, { lastTouchedAt: now, expiresAt });
|
||||
for (const duplicate of duplicates) {
|
||||
await ctx.db.delete(duplicate._id);
|
||||
}
|
||||
return { action: "updated" as const, expiresAt, shard };
|
||||
}
|
||||
|
||||
await ctx.db.insert("httpRateLimitKeys", {
|
||||
name: args.name,
|
||||
key: args.key,
|
||||
shard,
|
||||
lastTouchedAt: now,
|
||||
expiresAt,
|
||||
});
|
||||
return { action: "inserted" as const, expiresAt, shard };
|
||||
}
|
||||
|
||||
export const consumeHttpRateLimitKeyInternal = internalMutation({
|
||||
args: {
|
||||
name: v.string(),
|
||||
key: v.string(),
|
||||
limit: v.number(),
|
||||
windowMs: v.number(),
|
||||
config: fixedWindowRateLimitConfigValidator,
|
||||
now: v.optional(v.number()),
|
||||
ttlMs: v.optional(v.number()),
|
||||
shard: v.optional(v.number()),
|
||||
},
|
||||
returns: rateLimitStatusValidator,
|
||||
handler: async (ctx, args) => {
|
||||
const now = Date.now();
|
||||
const windowStart = Math.floor(now / args.windowMs) * args.windowMs;
|
||||
const resetAt = windowStart + args.windowMs;
|
||||
const requestedShard = Number.isFinite(args.shard) ? Math.floor(args.shard ?? 0) : 0;
|
||||
const shard = Math.max(0, Math.min(RATE_LIMIT_COUNTER_SHARDS - 1, requestedShard));
|
||||
|
||||
const existing = await ctx.db
|
||||
.query("rateLimitCounters")
|
||||
.withIndex("by_key_window_shard", (q) =>
|
||||
q.eq("key", args.key).eq("windowStart", windowStart).eq("shard", shard),
|
||||
)
|
||||
.first();
|
||||
|
||||
if (!existing) {
|
||||
await ctx.db.insert("rateLimitCounters", {
|
||||
key: args.key,
|
||||
windowStart,
|
||||
shard,
|
||||
count: 1,
|
||||
limit: args.limit,
|
||||
updatedAt: now,
|
||||
expiresAt: resetAt + RATE_LIMIT_COUNTER_RETENTION_BUFFER_MS,
|
||||
});
|
||||
return { allowed: true, remaining: Math.max(0, args.limit - 1) };
|
||||
}
|
||||
|
||||
await ctx.db.patch(existing._id, {
|
||||
count: existing.count + 1,
|
||||
limit: args.limit,
|
||||
updatedAt: now,
|
||||
expiresAt: resetAt + RATE_LIMIT_COUNTER_RETENTION_BUFFER_MS,
|
||||
const status = await ctx.runMutation(components.rateLimiter.lib.rateLimit, {
|
||||
name: args.name,
|
||||
key: args.key,
|
||||
config: args.config,
|
||||
});
|
||||
return {
|
||||
allowed: true,
|
||||
remaining: Math.max(0, args.limit - 1),
|
||||
};
|
||||
await touchHttpRateLimitKey(ctx, args);
|
||||
return status;
|
||||
},
|
||||
});
|
||||
|
||||
export const pruneRateLimitCountersInternal = internalMutation({
|
||||
export const touchHttpRateLimitKeyInternal = internalMutation({
|
||||
args: {
|
||||
name: v.string(),
|
||||
key: v.string(),
|
||||
shard: v.optional(v.number()),
|
||||
now: v.optional(v.number()),
|
||||
ttlMs: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
return await touchHttpRateLimitKey(ctx, args);
|
||||
},
|
||||
});
|
||||
|
||||
async function hasActiveHttpRateLimitKeyMetadata(
|
||||
ctx: HttpRateLimitKeyWriteCtx,
|
||||
row: { name: string; key: string },
|
||||
now: number,
|
||||
) {
|
||||
const activeRows = await ctx.db
|
||||
.query("httpRateLimitKeys")
|
||||
.withIndex("by_name_and_key_and_expires_at", (q) =>
|
||||
q.eq("name", row.name).eq("key", row.key).gte("expiresAt", now),
|
||||
)
|
||||
.take(1);
|
||||
return activeRows.length > 0;
|
||||
}
|
||||
|
||||
async function deleteExpiredHttpRateLimitKeyMetadata(
|
||||
ctx: HttpRateLimitKeyWriteCtx,
|
||||
row: { name: string; key: string },
|
||||
now: number,
|
||||
deletedRowIds: Set<string>,
|
||||
) {
|
||||
const expiredRows = await ctx.db
|
||||
.query("httpRateLimitKeys")
|
||||
.withIndex("by_name_and_key_and_expires_at", (q) =>
|
||||
q.eq("name", row.name).eq("key", row.key).lt("expiresAt", now),
|
||||
)
|
||||
.take(MAX_EXPIRED_HTTP_RATE_LIMIT_KEY_ROWS_PER_KEY);
|
||||
|
||||
let deleted = 0;
|
||||
for (const expiredRow of expiredRows) {
|
||||
if (deletedRowIds.has(expiredRow._id)) continue;
|
||||
await ctx.db.delete(expiredRow._id);
|
||||
deletedRowIds.add(expiredRow._id);
|
||||
deleted += 1;
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
|
||||
export const pruneHttpRateLimitKeysInternal = internalMutation({
|
||||
args: {
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const requestedBatchSize = Number.isFinite(args.batchSize)
|
||||
? Math.floor(args.batchSize ?? DEFAULT_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE)
|
||||
: DEFAULT_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE;
|
||||
const batchSize = Math.max(
|
||||
1,
|
||||
Math.min(requestedBatchSize, MAX_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE),
|
||||
const now = Date.now();
|
||||
const batchSize = clampBatchSize(
|
||||
args.batchSize,
|
||||
DEFAULT_PRUNE_HTTP_RATE_LIMIT_KEYS_BATCH_SIZE,
|
||||
MAX_PRUNE_HTTP_RATE_LIMIT_KEYS_BATCH_SIZE,
|
||||
);
|
||||
const stale = await ctx.db
|
||||
.query("rateLimitCounters")
|
||||
.withIndex("by_expires_at", (q) => q.lt("expiresAt", Date.now()))
|
||||
.query("httpRateLimitKeys")
|
||||
.withIndex("by_expires_at", (q) => q.lt("expiresAt", now))
|
||||
.take(batchSize);
|
||||
|
||||
const resetKeys = new Set<string>();
|
||||
const deletedRowIds = new Set<string>();
|
||||
let deleted = 0;
|
||||
|
||||
for (const row of stale) {
|
||||
await ctx.db.delete(row._id);
|
||||
if (deletedRowIds.has(row._id)) continue;
|
||||
|
||||
const key = `${row.name}\0${row.key}`;
|
||||
if (await hasActiveHttpRateLimitKeyMetadata(ctx, row, now)) {
|
||||
await ctx.db.delete(row._id);
|
||||
deletedRowIds.add(row._id);
|
||||
deleted += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!resetKeys.has(key)) {
|
||||
await ctx.runMutation(components.rateLimiter.lib.resetRateLimit, {
|
||||
name: row.name,
|
||||
key: row.key,
|
||||
});
|
||||
resetKeys.add(key);
|
||||
}
|
||||
deleted += await deleteExpiredHttpRateLimitKeyMetadata(ctx, row, now, deletedRowIds);
|
||||
}
|
||||
|
||||
const hasMore = stale.length === batchSize;
|
||||
if (hasMore) {
|
||||
await ctx.scheduler.runAfter(0, internal.rateLimits.pruneRateLimitCountersInternal, {
|
||||
await ctx.scheduler.runAfter(0, internal.rateLimits.pruneHttpRateLimitKeysInternal, {
|
||||
batchSize,
|
||||
});
|
||||
}
|
||||
|
||||
return { deleted: stale.length, hasMore };
|
||||
return { deleted, hasMore };
|
||||
},
|
||||
});
|
||||
|
||||
+7
-9
@@ -2727,17 +2727,15 @@ const cliDeviceCodes = defineTable({
|
||||
.index("by_user_code_hash", ["userCodeHash"])
|
||||
.index("by_status_expires", ["status", "expiresAt"]);
|
||||
|
||||
const rateLimitCounters = defineTable({
|
||||
const httpRateLimitKeys = defineTable({
|
||||
name: v.string(),
|
||||
key: v.string(),
|
||||
windowStart: v.number(),
|
||||
shard: v.number(),
|
||||
count: v.number(),
|
||||
limit: v.number(),
|
||||
updatedAt: v.number(),
|
||||
shard: v.optional(v.number()),
|
||||
lastTouchedAt: v.number(),
|
||||
expiresAt: v.number(),
|
||||
})
|
||||
.index("by_key_window", ["key", "windowStart"])
|
||||
.index("by_key_window_shard", ["key", "windowStart", "shard"])
|
||||
.index("by_name_and_key_and_shard", ["name", "key", "shard"])
|
||||
.index("by_name_and_key_and_expires_at", ["name", "key", "expiresAt"])
|
||||
.index("by_expires_at", ["expiresAt"]);
|
||||
|
||||
const downloadMetricTargetKind = v.union(v.literal("skill"), v.literal("package"));
|
||||
@@ -2955,7 +2953,7 @@ export default defineSchema({
|
||||
vtScanLogs,
|
||||
apiTokens,
|
||||
cliDeviceCodes,
|
||||
rateLimitCounters,
|
||||
httpRateLimitKeys,
|
||||
downloadMetricDedupes,
|
||||
packageInstallMetricDedupes,
|
||||
installTelemetryDedupes,
|
||||
|
||||
+5
-1
@@ -41,12 +41,16 @@ Auth-aware enforcement:
|
||||
- Write: 300/min per IP, 3000/min per key
|
||||
- Download: 1200/min per IP, 6000/min per key
|
||||
|
||||
Headers: `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset`, `RateLimit-Limit`, `RateLimit-Remaining`, `RateLimit-Reset`, `Retry-After` (on 429).
|
||||
Headers: `X-RateLimit-Limit`, `X-RateLimit-Reset`, `RateLimit-Limit`, `RateLimit-Reset`;
|
||||
`X-RateLimit-Remaining`, `RateLimit-Remaining`, and `Retry-After` are included on `429`.
|
||||
|
||||
Semantics:
|
||||
|
||||
- `X-RateLimit-Reset`: Unix epoch seconds (absolute reset time)
|
||||
- `RateLimit-Reset`: delay seconds until reset
|
||||
- `X-RateLimit-Remaining` / `RateLimit-Remaining`: exact remaining budget when
|
||||
present; sharded successful requests omit it rather than returning an approximate
|
||||
global value
|
||||
- `Retry-After`: delay seconds to wait on `429`
|
||||
|
||||
Example `429`:
|
||||
|
||||
+11
-4
@@ -39,14 +39,17 @@ Enforcement model:
|
||||
|
||||
Headers:
|
||||
|
||||
- Legacy compatibility: `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset`
|
||||
- Standardized: `RateLimit-Limit`, `RateLimit-Remaining`, `RateLimit-Reset`
|
||||
- Legacy compatibility: `X-RateLimit-Limit`, `X-RateLimit-Reset`
|
||||
- Standardized: `RateLimit-Limit`, `RateLimit-Reset`
|
||||
- On `429`: `X-RateLimit-Remaining: 0` and `RateLimit-Remaining: 0`
|
||||
- On `429`: `Retry-After`
|
||||
|
||||
Header semantics:
|
||||
|
||||
- `X-RateLimit-Reset`: absolute Unix epoch seconds
|
||||
- `RateLimit-Reset`: seconds until reset (delay)
|
||||
- `X-RateLimit-Remaining` / `RateLimit-Remaining`: exact remaining budget when present.
|
||||
Sharded successful requests omit this header instead of returning an approximate global value.
|
||||
- `Retry-After`: seconds to wait before retry (delay) on `429`
|
||||
|
||||
Example `429` response:
|
||||
@@ -73,9 +76,13 @@ Client guidance:
|
||||
|
||||
IP source:
|
||||
|
||||
- Uses `cf-connecting-ip` (Cloudflare) for client IP by default.
|
||||
- Uses trusted client IP headers, including `cf-connecting-ip`, only when the
|
||||
deployment explicitly enables trusted forwarded headers.
|
||||
- ClawHub uses trusted forwarding headers to identify client IPs at the edge.
|
||||
- If no trusted client IP is available, anonymous download requests use an endpoint-scoped fallback bucket instead of one global `ip:unknown` bucket. Anonymous read/write requests still use the shared unknown bucket so missing-IP routing remains visible and conservative.
|
||||
- If no trusted client IP is available, anonymous requests use fallback buckets
|
||||
scoped only by rate-limit kind. These fallback buckets do not include
|
||||
caller-supplied paths, slugs, package names, versions, query strings, or other
|
||||
artifact parameters.
|
||||
|
||||
## Error responses
|
||||
|
||||
|
||||
@@ -31,7 +31,8 @@ clawhub login --token clh_...
|
||||
Read the retry information in the response:
|
||||
|
||||
- `Retry-After`: seconds to wait before retrying.
|
||||
- `RateLimit-Remaining` and `RateLimit-Limit`: your current budget.
|
||||
- `RateLimit-Limit`: the limit applied to this request.
|
||||
- `RateLimit-Remaining`: your exact remaining budget when the header is present. On `429`, it is `0`.
|
||||
- `RateLimit-Reset` or `X-RateLimit-Reset`: reset timing.
|
||||
|
||||
If many users share one egress IP, anonymous IP limits can be hit even when each
|
||||
|
||||
@@ -74,6 +74,7 @@
|
||||
"@auth/core": "0.41.2",
|
||||
"@convex-dev/auth": "0.0.94",
|
||||
"@convex-dev/migrations": "0.3.5",
|
||||
"@convex-dev/rate-limiter": "0.3.2",
|
||||
"@fontsource/bricolage-grotesque": "5.2.10",
|
||||
"@fontsource/ibm-plex-mono": "5.2.7",
|
||||
"@fontsource/manrope": "5.2.8",
|
||||
|
||||
@@ -131,6 +131,11 @@ Ensure Convex env is set (auth + embeddings):
|
||||
- Optional fallback: `GITHUB_TOKEN` (used when GitHub App auth is unavailable,
|
||||
and for arbitrary public repository lookups such as trusted-publisher setup)
|
||||
|
||||
Do not set `TRUST_FORWARDED_IPS=true` while the Convex `*.convex.site` HTTP
|
||||
origin remains publicly reachable. That flag makes rate limits and download
|
||||
metrics trust forwarded client IP headers, so it is only safe behind a
|
||||
header-sanitizing edge that prevents direct origin requests.
|
||||
|
||||
## 2) Deploy web app (Vercel)
|
||||
|
||||
Set env vars:
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# Rate Limiting
|
||||
|
||||
## Security Intent
|
||||
|
||||
All public HTTP routes, including legacy `/api/...` and `/api/cli/...` routes
|
||||
kept for compatibility, must pass through `applyRateLimit` before doing
|
||||
expensive work, auth parsing, publish mutations, upload ticket creation,
|
||||
telemetry writes, delete/undelete mutations, or search queries.
|
||||
|
||||
Client IP headers are not trustworthy on direct Convex HTTP endpoints. Treat
|
||||
`cf-connecting-ip`, `x-forwarded-for`, `x-real-ip`, and `fly-client-ip` as
|
||||
trusted only when the deployment explicitly enables trusted forwarded headers.
|
||||
Do not enable that opt-in while the Convex `*.convex.site` HTTP origin remains
|
||||
publicly reachable. Without the opt-in, anonymous traffic must use conservative
|
||||
missing-IP fallback buckets scoped only by rate-limit kind. Missing-IP buckets
|
||||
must not include user-controlled paths, dynamic path segments, query parameters,
|
||||
package names, skill slugs, or artifact versions.
|
||||
Artifact-specific download scoping is only safe after the caller has an
|
||||
authenticated identity or a trusted client IP.
|
||||
|
||||
HTTP rate-limit counters are owned by the `@convex-dev/rate-limiter` component.
|
||||
The app defines named fixed-window buckets for each public HTTP policy
|
||||
(`read`, `write`, `trustedPublish`, `download`, and `export`) and for each
|
||||
subject class (`ip`, authenticated API token user, and admin API token user).
|
||||
Anonymous requests consume the `Ip` bucket for the route policy. Authenticated
|
||||
requests consume only the `Key` or `AdminKey` bucket for their user, so shared
|
||||
egress IPs do not drain user quota.
|
||||
Component rows are operational data, not product history. Do not use the
|
||||
component `clearAll(before)` helper as a TTL job for active HTTP buckets:
|
||||
`clearAll` deletes by row creation time while normal rate-limit use patches the
|
||||
row's current timestamp and value in place. Clearing old creation times can
|
||||
reset still-active buckets and weaken the limit.
|
||||
The app-owned `httpRateLimitKeys` table exists only to make component cleanup
|
||||
safe: limiter checks refresh sharded metadata for the observed `(name, key)`
|
||||
with an `expiresAt`, and retention resets the exact component key only after no
|
||||
metadata shard for that `(name, key)` remains active. Active allowed and denied
|
||||
traffic must both refresh this metadata so a client that is still hitting a
|
||||
limit is not reset by cleanup. Metadata sharding is required because shared
|
||||
buckets such as missing-IP anonymous reads can receive many concurrent requests;
|
||||
one metadata row per key would become a separate Convex write-contention point.
|
||||
|
||||
Component sharding is an implementation detail, not a quota multiplier. Keep
|
||||
the public quota in `RATE_LIMITS`; the component's `shards` setting exists to
|
||||
spread writes across counter rows while preserving the configured limit. Low
|
||||
rate buckets should stay unsharded or keep enough per-shard capacity that
|
||||
normal clients are not randomly denied well before the public limit.
|
||||
Because the component does not expose an exact total remaining count for
|
||||
sharded buckets, successful HTTP responses must not synthesize a global
|
||||
`RateLimit-Remaining` value. Denied responses can still return an exact
|
||||
remaining value of `0`.
|
||||
Reference in New Issue
Block a user