fix: harden API rate limits (#2783)

This commit is contained in:
Jesse Merhi
2026-06-25 16:22:00 +10:00
committed by GitHub
parent aebbce2710
commit 088339b5d2
32 changed files with 1949 additions and 578 deletions
+3
View File
@@ -8,6 +8,7 @@
"@auth/core": "0.41.2",
"@convex-dev/auth": "0.0.94",
"@convex-dev/migrations": "0.3.5",
"@convex-dev/rate-limiter": "0.3.2",
"@fontsource/bricolage-grotesque": "5.2.10",
"@fontsource/ibm-plex-mono": "5.2.7",
"@fontsource/manrope": "5.2.8",
@@ -226,6 +227,8 @@
"@convex-dev/migrations": ["@convex-dev/migrations@0.3.5", "", { "peerDependencies": { "convex": "^1.35.0", "convex-helpers": "^0.1.115" } }, "sha512-vj5qjY5XB8laX9WvvxeIFZe8sNW8DGvJ5cPo1zvpjJzYnPLqs5C+VG9HP6sSkEW2QGhsfYOZv56FUNkHiLe1wA=="],
"@convex-dev/rate-limiter": ["@convex-dev/rate-limiter@0.3.2", "", { "peerDependencies": { "convex": "^1.24.8", "react": "^18.2.0 || ^19.0.0" }, "optionalPeers": ["react"] }, "sha512-+oBPsBfFbzdxiF/9XaaTQmVnvDlvEfg/c69/v8LxTbw4VLuiflIKlfnPQL8OS0azXQQ11hcPWHmU8ytFmHKDXA=="],
"@csstools/color-helpers": ["@csstools/color-helpers@6.0.2", "", {}, "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q=="],
"@csstools/css-calc": ["@csstools/css-calc@3.2.1", "", { "peerDependencies": { "@csstools/css-parser-algorithms": "^4.0.0", "@csstools/css-tokenizer": "^4.0.0" } }, "sha512-DtdHlgXh5ZkA43cwBcAm+huzgJiwx3ZTWVjBs94kwz2xKqSimDA3lBgCjphYgwgVUMWatSM0pDd8TILB1yrVVg=="],
+5 -2
View File
@@ -73,7 +73,9 @@ import type * as lib_githubSkillScans from "../lib/githubSkillScans.js";
import type * as lib_githubSkillSync from "../lib/githubSkillSync.js";
import type * as lib_globalStats from "../lib/globalStats.js";
import type * as lib_httpHeaders from "../lib/httpHeaders.js";
import type * as lib_httpPathSegments from "../lib/httpPathSegments.js";
import type * as lib_httpRateLimit from "../lib/httpRateLimit.js";
import type * as lib_httpRouteRateLimit from "../lib/httpRouteRateLimit.js";
import type * as lib_httpUtils from "../lib/httpUtils.js";
import type * as lib_installResolver from "../lib/installResolver.js";
import type * as lib_leaderboards from "../lib/leaderboards.js";
@@ -95,7 +97,6 @@ import type * as lib_publisherAbuseScoring from "../lib/publisherAbuseScoring.js
import type * as lib_publisherCatalogDisplay from "../lib/publisherCatalogDisplay.js";
import type * as lib_publisherStats from "../lib/publisherStats.js";
import type * as lib_publishers from "../lib/publishers.js";
import type * as lib_rateLimitConfig from "../lib/rateLimitConfig.js";
import type * as lib_recommendationScore from "../lib/recommendationScore.js";
import type * as lib_reporting from "../lib/reporting.js";
import type * as lib_reservedHandles from "../lib/reservedHandles.js";
@@ -226,7 +227,9 @@ declare const fullApi: ApiFromModules<{
"lib/githubSkillSync": typeof lib_githubSkillSync;
"lib/globalStats": typeof lib_globalStats;
"lib/httpHeaders": typeof lib_httpHeaders;
"lib/httpPathSegments": typeof lib_httpPathSegments;
"lib/httpRateLimit": typeof lib_httpRateLimit;
"lib/httpRouteRateLimit": typeof lib_httpRouteRateLimit;
"lib/httpUtils": typeof lib_httpUtils;
"lib/installResolver": typeof lib_installResolver;
"lib/leaderboards": typeof lib_leaderboards;
@@ -248,7 +251,6 @@ declare const fullApi: ApiFromModules<{
"lib/publisherCatalogDisplay": typeof lib_publisherCatalogDisplay;
"lib/publisherStats": typeof lib_publisherStats;
"lib/publishers": typeof lib_publishers;
"lib/rateLimitConfig": typeof lib_rateLimitConfig;
"lib/recommendationScore": typeof lib_recommendationScore;
"lib/reporting": typeof lib_reporting;
"lib/reservedHandles": typeof lib_reservedHandles;
@@ -336,4 +338,5 @@ export declare const internal: FilterApi<
export declare const components: {
migrations: import("@convex-dev/migrations/_generated/component.js").ComponentApi<"migrations">;
rateLimiter: import("@convex-dev/rate-limiter/_generated/component.js").ComponentApi<"rateLimiter">;
};
+2
View File
@@ -1,7 +1,9 @@
import migrations from "@convex-dev/migrations/convex.config.js";
import rateLimiter from "@convex-dev/rate-limiter/convex.config.js";
import { defineApp } from "convex/server";
const app = defineApp();
app.use(migrations);
app.use(rateLimiter);
export default app;
+7 -7
View File
@@ -5,7 +5,7 @@ const mocks = vi.hoisted(() => {
const interval = vi.fn();
const githubSkillSyncRef = Symbol("github-skill-source-sync");
const installTelemetryDedupePruneRef = Symbol("install-telemetry-dedupe-prune");
const rateLimitCountersPruneRef = Symbol("rate-limit-counters-prune");
const httpRateLimitKeysPruneRef = Symbol("http-rate-limit-keys-prune");
const skillStatEventPruneRef = Symbol("skill-stat-event-prune");
const packageStatEventPruneRef = Symbol("package-stat-event-prune");
const authSessionsPruneRef = Symbol("auth-sessions-prune");
@@ -14,7 +14,7 @@ const mocks = vi.hoisted(() => {
interval,
githubSkillSyncRef,
installTelemetryDedupePruneRef,
rateLimitCountersPruneRef,
httpRateLimitKeysPruneRef,
skillStatEventPruneRef,
packageStatEventPruneRef,
authSessionsPruneRef,
@@ -63,7 +63,7 @@ vi.mock("./_generated/api", () => ({
pruneInstallTelemetryDedupesInternal: mocks.installTelemetryDedupePruneRef,
},
rateLimits: {
pruneRateLimitCountersInternal: mocks.rateLimitCountersPruneRef,
pruneHttpRateLimitKeysInternal: mocks.httpRateLimitKeysPruneRef,
},
retention: {
pruneExpiredAuthSessionsInternal: mocks.authSessionsPruneRef,
@@ -124,13 +124,13 @@ describe("crons", () => {
);
});
it("prunes expired rate limit counters frequently", async () => {
it("prunes stale component HTTP rate limit keys hourly", async () => {
await import("./crons");
expect(mocks.interval).toHaveBeenCalledWith(
"rate-limit-counters-prune",
{ minutes: 15 },
mocks.rateLimitCountersPruneRef,
"http-rate-limit-keys-prune",
{ hours: 1 },
mocks.httpRateLimitKeysPruneRef,
{ batchSize: 500 },
);
});
+3 -3
View File
@@ -155,9 +155,9 @@ if (process.env.CLAWHUB_DISABLE_CRONS !== "1") {
);
crons.interval(
"rate-limit-counters-prune",
{ minutes: 15 },
internal.rateLimits.pruneRateLimitCountersInternal,
"http-rate-limit-keys-prune",
{ hours: 1 },
internal.rateLimits.pruneHttpRateLimitKeysInternal,
{ batchSize: RETENTION_STANDARD_BATCH_SIZE },
);
}
+2
View File
@@ -89,6 +89,7 @@ function makeDb(
describe("download metric helpers", () => {
afterEach(() => {
vi.useRealTimers();
vi.unstubAllEnvs();
});
it("uses a day bucket for download dedupe", () => {
@@ -97,6 +98,7 @@ describe("download metric helpers", () => {
});
it("prefers user identity and falls back to IP identity", () => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
const request = new Request("https://example.com", {
headers: { "cf-connecting-ip": "203.0.113.10" },
});
+16 -18
View File
@@ -1,24 +1,25 @@
import type { RateLimitArgs, RateLimitReturns } from "@convex-dev/rate-limiter";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { ActionCtx } from "./_generated/server";
import { __test, downloadZipHandler } from "./downloads";
type RateLimitArgs = { key: string; limit: number; windowMs: number };
function isRateLimitArgs(args: unknown): args is RateLimitArgs {
if (!args || typeof args !== "object") return false;
const value = args as Record<string, unknown>;
const config = value.config as Record<string, unknown> | undefined;
return (
typeof value.key === "string" &&
typeof value.limit === "number" &&
typeof value.windowMs === "number"
typeof value.name === "string" &&
(!("key" in value) || typeof value.key === "string") &&
!!config &&
typeof config === "object" &&
(config.kind === "fixed window" || config.kind === "token bucket") &&
typeof config.rate === "number" &&
typeof config.period === "number"
);
}
const okRate = () => ({
allowed: true,
remaining: 10,
limit: 100,
resetAt: Date.now() + 60_000,
const okRate = (): RateLimitReturns => ({
ok: true,
});
function stubZipResponse() {
@@ -55,7 +56,8 @@ describe("downloads helpers", () => {
expect(__test.getDownloadIdentityValue(request, "users_123")).toBe("user:users_123");
});
it("uses cf-connecting-ip for anonymous identity", () => {
it("uses cf-connecting-ip for anonymous identity when trusted headers are enabled", () => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
const request = new Request("https://example.com", {
headers: { "cf-connecting-ip": "1.2.3.4" },
});
@@ -76,10 +78,10 @@ describe("downloads helpers", () => {
});
it("schedules zip download stats outside the response path", async () => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
stubZipResponse();
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
@@ -152,7 +154,6 @@ describe("downloads helpers", () => {
it("threads owner handle through the skill lookup", async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
@@ -202,7 +203,6 @@ describe("downloads helpers", () => {
it("does not serve a tag that points at another skill's version", async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
@@ -262,7 +262,6 @@ describe("downloads helpers", () => {
it("returns ownerHandle guidance when a slug-only download is ambiguous", async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) return { skill: null, ambiguous: true };
return null;
});
@@ -290,7 +289,6 @@ describe("downloads helpers", () => {
it("blocks the exact requested skill version when its ClawScan verdict is malicious", async () => {
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
@@ -369,7 +367,6 @@ describe("downloads helpers", () => {
stubZipResponse();
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("tokenHash" in args) {
return { _id: "apiTokens:1", revokedAt: undefined };
}
@@ -435,10 +432,10 @@ describe("downloads helpers", () => {
});
it("returns zip downloads when download metering is scheduled", async () => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
stubZipResponse();
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
if ("slug" in args) {
return {
skill: {
@@ -497,6 +494,7 @@ describe("downloads helpers", () => {
it.each(["clean", "suspicious"] as const)(
"returns a metered public GitHub handoff descriptor for %s scan without scan metadata",
async (scanStatus) => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
const commit = "1".repeat(40);
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return okRate();
+226
View File
@@ -0,0 +1,226 @@
import { ApiRoutes, LegacyApiRoutes } from "clawhub-schema";
/* @vitest-environment node */
import { describe, expect, it } from "vitest";
import { vi } from "vitest";
import type { ActionCtx } from "./_generated/server";
import http from "./http";
import { RATE_LIMITS } from "./lib/httpRateLimit";
type WrappedHttpAction = {
_handler: (ctx: ActionCtx, request: Request) => Promise<Response>;
};
type RateLimitBucketName = `${"read" | "write" | "trustedPublish" | "download" | "export"}Ip`;
function makeDeniedRateLimitCtx() {
const runQuery = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`);
});
const runMutation = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
if ("name" in args && "config" in args) {
const config = args.config as { period: number };
return { ok: false, retryAfter: config.period };
}
throw new Error(`Unexpected runMutation args: ${JSON.stringify(args)}`);
});
return {
ctx: {
runQuery,
runMutation,
} as unknown as ActionCtx,
runQuery,
runMutation,
};
}
function makeAllowedRateLimitCtx() {
const runQuery = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`);
});
const runMutation = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
if ("name" in args && "config" in args) {
return { ok: true };
}
throw new Error(`Unexpected runMutation args: ${JSON.stringify(args)}`);
});
return {
ctx: {
runQuery,
runMutation,
runAction: vi.fn(async () => []),
} as unknown as ActionCtx,
runMutation,
};
}
async function expectRouteUsesIpBucket({
path,
method,
requestUrl = `https://example.com${path}`,
bucket,
rate,
}: {
path: string;
method: "GET" | "POST";
requestUrl?: string;
bucket: RateLimitBucketName;
rate: number;
}) {
const route = http.lookup(path, method);
if (!route) throw new Error(`Expected route for ${method} ${path}`);
const [action] = route;
const { ctx, runMutation } = makeDeniedRateLimitCtx();
const response = await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request(requestUrl, { method }),
);
expect(response.status).toBe(429);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
name: bucket,
config: expect.objectContaining({ rate }),
}),
);
}
describe("HTTP route rate limit defaults", () => {
it("registers package version downloads behind the download limit", async () => {
await expectRouteUsesIpBucket({
path: "/api/v1/packages/demo/versions/1.0.0/download",
method: "GET",
bucket: "downloadIp",
rate: RATE_LIMITS.download.ip,
});
});
it.each([
["legacy download", LegacyApiRoutes.download, "downloadIp", RATE_LIMITS.download.ip],
["plugins export", ApiRoutes.pluginsExport, "exportIp", RATE_LIMITS.export.ip],
[
"package inspector artifact",
"/api/v1/package-inspector/artifact",
"downloadIp",
RATE_LIMITS.download.ip,
],
[
"package artifact download",
"/api/v1/packages/demo/versions/1.0.0/artifact/download",
"downloadIp",
RATE_LIMITS.download.ip,
],
] as const)(
"registers %s behind the central special-case bucket",
async (_name, path, bucket, rate) => {
await expectRouteUsesIpBucket({
path,
method: "GET",
bucket,
rate,
});
},
);
it("registers security verdict submission behind the read limit", async () => {
await expectRouteUsesIpBucket({
path: `${ApiRoutes.skills}/-/security-verdicts`,
method: "POST",
bucket: "readIp",
rate: RATE_LIMITS.read.ip,
});
});
it("registers auth sign-in routes behind the router-level default limit", async () => {
const route = http.lookup("/api/auth/signin/github", "GET");
if (!route) throw new Error("Expected auth sign-in route");
const [action] = route;
const { ctx, runMutation } = makeDeniedRateLimitCtx();
const response = await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request("https://example.com/api/auth/signin/github"),
);
expect(response.status).toBe(429);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
name: "readIp",
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
}),
);
});
it.each([
["search", LegacyApiRoutes.search, "GET"],
["skill detail", LegacyApiRoutes.skill, "GET"],
["skill resolve", LegacyApiRoutes.skillResolve, "GET"],
["whoami", LegacyApiRoutes.cliWhoami, "GET"],
["upload URL", LegacyApiRoutes.cliUploadUrl, "POST"],
["publish", LegacyApiRoutes.cliPublish, "POST"],
["install telemetry", LegacyApiRoutes.cliTelemetryInstall, "POST"],
["skill delete", LegacyApiRoutes.cliSkillDelete, "POST"],
["skill undelete", LegacyApiRoutes.cliSkillUndelete, "POST"],
])("registers legacy %s behind the router-level default limit", async (_name, path, method) => {
const route = http.lookup(path, method as "GET" | "POST");
if (!route) throw new Error(`Expected legacy route for ${path}`);
const [action] = route;
const { ctx, runMutation } = makeDeniedRateLimitCtx();
const response = await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request(`https://example.com${path}`, { method }),
);
expect(response.status).toBe(429);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
name: method === "GET" ? "readIp" : "writeIp",
config: expect.objectContaining({
rate: method === "GET" ? RATE_LIMITS.read.ip : RATE_LIMITS.write.ip,
}),
}),
);
});
it("does not double-consume routes that already rate limit inside their handler", async () => {
const route = http.lookup("/api/v1/search", "GET");
if (!route) throw new Error("Expected v1 search route");
const [action] = route;
const { ctx, runMutation } = makeAllowedRateLimitCtx();
const response = await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request("https://example.com/api/v1/search?q="),
);
expect(response.status).toBe(200);
expect(runMutation).toHaveBeenCalledTimes(1);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
name: "readIp",
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
}),
);
});
it("leaves API preflights unmetered", async () => {
const route = http.lookup("/api/v1/skills", "OPTIONS");
if (!route) throw new Error("Expected API preflight route");
const [action] = route;
const { ctx, runQuery, runMutation } = makeDeniedRateLimitCtx();
const response = await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request("https://example.com/api/v1/skills", { method: "OPTIONS" }),
);
expect(response.status).toBe(204);
expect(runQuery).not.toHaveBeenCalled();
expect(runMutation).not.toHaveBeenCalled();
});
});
+2 -1
View File
@@ -56,13 +56,14 @@ import {
contentRightsV1Http,
} from "./httpApiV1";
import { preflightHandler } from "./httpPreflight";
import { installRateLimitedRoutes } from "./lib/httpRouteRateLimit";
import {
packageInspectorArtifactHttp,
packageInspectorClaimHttp,
packageInspectorResultsHttp,
} from "./packageInspectorHttp";
const http = httpRouter();
const http = installRateLimitedRoutes(httpRouter());
auth.addHttpRoutes(http);
+6 -24
View File
@@ -400,16 +400,8 @@ describe("httpApi handlers", () => {
expires_in: 900,
interval: 5,
};
const runQuery = vi.fn().mockResolvedValue({
allowed: true,
remaining: 300,
limit: 300,
resetAt: Date.now() + 60_000,
});
const runMutation = vi
.fn()
.mockResolvedValueOnce({ allowed: true, remaining: 299 })
.mockResolvedValueOnce(result);
const runQuery = vi.fn();
const runMutation = vi.fn().mockResolvedValueOnce({ ok: true }).mockResolvedValueOnce(result);
const response = await __handlers.cliDeviceCodeHandler(
makeCtx({ runQuery, runMutation }),
@@ -436,13 +428,8 @@ describe("httpApi handlers", () => {
it("cliDeviceTokenHttp requires the device grant type", async () => {
vi.mocked(getOptionalApiTokenUser).mockResolvedValueOnce(null);
const runQuery = vi.fn().mockResolvedValue({
allowed: true,
remaining: 300,
limit: 300,
resetAt: Date.now() + 60_000,
});
const runMutation = vi.fn().mockResolvedValueOnce({ allowed: true, remaining: 299 });
const runQuery = vi.fn();
const runMutation = vi.fn().mockResolvedValueOnce({ ok: true });
const response = await __handlers.cliDeviceTokenHandler(
makeCtx({ runQuery, runMutation }),
@@ -460,15 +447,10 @@ describe("httpApi handlers", () => {
it("cliDeviceTokenHttp returns pending as retryable", async () => {
vi.mocked(getOptionalApiTokenUser).mockResolvedValueOnce(null);
const runQuery = vi.fn().mockResolvedValue({
allowed: true,
remaining: 300,
limit: 300,
resetAt: Date.now() + 60_000,
});
const runQuery = vi.fn();
const runMutation = vi
.fn()
.mockResolvedValueOnce({ allowed: true, remaining: 299 })
.mockResolvedValueOnce({ ok: true })
.mockResolvedValueOnce({ error: "authorization_pending" });
const response = await __handlers.cliDeviceTokenHandler(
+1 -17
View File
@@ -27,30 +27,14 @@ const { __handlers } = await import("./httpApiV1");
type ActionCtx = import("./_generated/server").ActionCtx;
type RateLimitArgs = { key: string; limit: number; windowMs: number };
function isRateLimitArgs(args: unknown): args is RateLimitArgs {
if (!args || typeof args !== "object") return false;
const value = args as Record<string, unknown>;
return (
typeof value.key === "string" &&
typeof value.limit === "number" &&
typeof value.windowMs === "number"
);
}
const okRate = () => ({
allowed: true,
remaining: 10,
limit: 100,
resetAt: Date.now() + 60_000,
ok: true,
});
function makeCtx(partial: {
runQuery?: (query: unknown, args: Record<string, unknown>) => unknown;
}) {
const runQuery = vi.fn(async (query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return { ...okRate(), limit: args.limit };
return partial.runQuery ? await partial.runQuery(query, args) : null;
});
return { runQuery, runMutation: vi.fn().mockResolvedValue(okRate()) } as unknown as ActionCtx;
+39 -31
View File
@@ -1,4 +1,5 @@
/* @vitest-environment node */
import type { RateLimitArgs, RateLimitReturns } from "@convex-dev/rate-limiter";
import { gzipSync, strFromU8, unzipSync } from "fflate";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { api, internal } from "./_generated/api";
@@ -41,15 +42,18 @@ const { __handlers } = await import("./httpApiV1");
type ActionCtx = import("./_generated/server").ActionCtx;
type RateLimitArgs = { key: string; limit: number; windowMs: number };
function isRateLimitArgs(args: unknown): args is RateLimitArgs {
if (!args || typeof args !== "object") return false;
const value = args as Record<string, unknown>;
const config = value.config as Record<string, unknown> | undefined;
return (
typeof value.key === "string" &&
typeof value.limit === "number" &&
typeof value.windowMs === "number"
typeof value.name === "string" &&
(!("key" in value) || typeof value.key === "string") &&
!!config &&
typeof config === "object" &&
(config.kind === "fixed window" || config.kind === "token bucket") &&
typeof config.rate === "number" &&
typeof config.period === "number"
);
}
@@ -229,31 +233,26 @@ function makeCtx(partial: Record<string, unknown>) {
? (partial.runQuery as (query: unknown, args: Record<string, unknown>) => unknown)
: null;
const runQuery = vi.fn(async (query: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) {
return rateLimitStatus?.(args) ?? { ...okRate(), limit: args.limit };
}
return partialRunQuery ? await partialRunQuery(query, args) : null;
});
const runMutation =
typeof partial.runMutation === "function"
? partial.runMutation
: vi.fn().mockResolvedValue(okRate());
: vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return rateLimitStatus?.(args) ?? okRate();
return okRate();
});
return { ...partial, runQuery, runMutation } as unknown as ActionCtx;
}
const okRate = () => ({
allowed: true,
remaining: 10,
limit: 100,
resetAt: Date.now() + 60_000,
const okRate = (): RateLimitReturns => ({
ok: true,
});
const blockedRate = () => ({
allowed: false,
remaining: 0,
limit: 100,
resetAt: Date.now() + 60_000,
const blockedRate = (): RateLimitReturns => ({
ok: false,
retryAfter: 60_000,
});
beforeEach(() => {
@@ -378,7 +377,7 @@ describe("httpApiV1 handlers", () => {
} as never);
const runMutation = vi.fn(async (_mutation: unknown, args: Record<string, unknown>) => {
if (isRateLimitArgs(args)) return { ...okRate(), limit: args.limit };
if (isRateLimitArgs(args)) return okRate();
return { ok: true };
});
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
@@ -399,7 +398,8 @@ describe("httpApiV1 handlers", () => {
expect.anything(),
expect.objectContaining({
key: "user:users:actor:export",
limit: RATE_LIMITS.export.key,
name: "exportKey",
config: expect.objectContaining({ rate: RATE_LIMITS.export.key }),
}),
);
});
@@ -1453,7 +1453,9 @@ describe("httpApiV1 handlers", () => {
if (response.status !== 200) throw new Error(await response.text());
const slugCalls = runMutation.mock.calls.filter(([, args]) => hasSlugArgs(args));
const packageCalls = runMutation.mock.calls.filter(([, args]) => hasPackageNameArgs(args));
const packageCalls = runMutation.mock.calls.filter(
([, args]) => hasPackageNameArgs(args) && !isRateLimitArgs(args),
);
expect(slugCalls).toHaveLength(1);
expect(slugCalls[0]?.[1]).toMatchObject({
actorUserId: "users:admin",
@@ -2005,18 +2007,16 @@ describe("httpApiV1 handlers", () => {
});
it("search rate limits", async () => {
const runMutation = vi.fn().mockResolvedValue(blockedRate());
const response = await __handlers.searchSkillsV1Handler(
makeCtx({ runAction: vi.fn(), runMutation }),
makeCtx({ runAction: vi.fn(), rateLimitStatus: () => blockedRate() }),
new Request("https://example.com/api/v1/search?q=test"),
);
expect(response.status).toBe(429);
});
it("429 Retry-After is a relative delay, not an absolute epoch", async () => {
const runMutation = vi.fn().mockResolvedValue(blockedRate());
const response = await __handlers.searchSkillsV1Handler(
makeCtx({ runAction: vi.fn(), runMutation }),
makeCtx({ runAction: vi.fn(), rateLimitStatus: () => blockedRate() }),
new Request("https://example.com/api/v1/search?q=test"),
);
expect(response.status).toBe(429);
@@ -8548,7 +8548,8 @@ describe("httpApiV1 handlers", () => {
);
expect(findRateLimitCallArgs(runMutation)).toMatchObject({
key: expect.stringMatching(/^ip:/),
limit: RATE_LIMITS.read.ip,
name: "readIp",
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
});
expect(response.headers.get("RateLimit-Limit")).toBeTruthy();
});
@@ -12631,6 +12632,7 @@ describe("httpApiV1 handlers", () => {
});
it("npm mirror tarball downloads record package installs and download metrics", async () => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ("name" in args && !("paginationOpts" in args)) {
return {
@@ -12927,7 +12929,8 @@ describe("httpApiV1 handlers", () => {
expect(response.headers.get("RateLimit-Limit")).toBeTruthy();
expect(findRateLimitCallArgs(runMutation)).toMatchObject({
key: expect.stringMatching(/^ip:/),
limit: RATE_LIMITS.download.ip,
name: "downloadIp",
config: expect.objectContaining({ rate: RATE_LIMITS.download.ip }),
});
});
@@ -12987,7 +12990,8 @@ describe("httpApiV1 handlers", () => {
expect(response.headers.get("RateLimit-Limit")).toBeTruthy();
expect(findRateLimitCallArgs(runMutation)).toMatchObject({
key: expect.stringMatching(/^ip:/),
limit: RATE_LIMITS.read.ip,
name: "readIp",
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
});
});
@@ -13048,6 +13052,7 @@ describe("httpApiV1 handlers", () => {
});
it("package download uses a package/ root without registry metadata", async () => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
const runMutation = vi.fn().mockResolvedValue(okRate());
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ("name" in args) {
@@ -13202,6 +13207,7 @@ describe("httpApiV1 handlers", () => {
});
it("package downloads succeed and record download metrics", async () => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
const runMutation = vi.fn().mockResolvedValue(okRate());
const runQuery = vi.fn(async (_query: unknown, args: Record<string, unknown>) => {
if ("name" in args) {
@@ -13590,7 +13596,8 @@ describe("httpApiV1 handlers", () => {
expect(response.headers.get("RateLimit-Limit")).toBeTruthy();
expect(findRateLimitCallArgs(runMutation)).toMatchObject({
key: "user:users:1:write",
limit: RATE_LIMITS.write.key,
name: "writeKey",
config: expect.objectContaining({ rate: RATE_LIMITS.write.key }),
});
expect(runAction).toHaveBeenCalledWith(
expect.anything(),
@@ -14259,7 +14266,8 @@ describe("httpApiV1 handlers", () => {
expect.anything(),
expect.objectContaining({
key: "ip:unknown:trustedPublish",
limit: RATE_LIMITS.trustedPublish.ip,
name: "trustedPublishIp",
config: expect.objectContaining({ rate: RATE_LIMITS.trustedPublish.ip }),
}),
);
expect(runMutation).toHaveBeenCalledWith(
+1 -38
View File
@@ -76,6 +76,7 @@ import {
MAX_RAW_FILE_BYTES,
getPathSegments,
json,
parsePackagePathSegments,
publicApiOrigin,
resolveTagsBatch,
requireApiTokenUserOrResponse,
@@ -4058,44 +4059,6 @@ function parseNpmMirrorPath(request: Request) {
return parsePackagePathSegments(segments);
}
function decodePackagePathSegment(segment: string) {
let decoded = segment;
for (let i = 0; i < 2 && decoded.includes("%"); i += 1) {
try {
const next = decodeURIComponent(decoded);
if (next === decoded) break;
decoded = next;
} catch {
break;
}
}
return decoded;
}
function parsePackagePathSegments(segments: string[]) {
if (segments.length === 0) return null;
const firstSegment = decodePackagePathSegment(segments[0]!);
if (firstSegment.startsWith("@")) {
if (firstSegment.includes("/")) {
const [scope, name, ...encodedRest] = firstSegment.split("/");
if (!scope || !name) return null;
return {
packageName: `${scope}/${name}`,
rest: [...encodedRest, ...segments.slice(1)],
};
}
if (segments.length < 2) return null;
return {
packageName: `${firstSegment}/${decodePackagePathSegment(segments[1]!)}`,
rest: segments.slice(2),
};
}
return {
packageName: firstSegment,
rest: segments.slice(1),
};
}
type NpmPackReleasePage = {
page: ReleaseLike[];
isDone: boolean;
+1 -11
View File
@@ -7,6 +7,7 @@ import { requireApiTokenUser, requirePackagePublishAuth } from "../lib/apiTokenA
import { corsHeaders, mergeHeaders } from "../lib/httpHeaders";
import { getPublishFileSizeError, MAX_PUBLISH_FILE_BYTES } from "../lib/publishLimits";
import { isMacJunkPath } from "../lib/skills";
export { getPathSegments, parsePackagePathSegments } from "../lib/httpPathSegments";
export const MAX_RAW_FILE_BYTES = 200 * 1024;
const DEFAULT_PUBLIC_SITE_URL = "https://clawhub.ai";
@@ -220,17 +221,6 @@ export function requireAdminOrResponse(user: Doc<"users">, headers: HeadersInit)
}
}
export function getPathSegments(request: Request, prefix: string) {
const pathname = new URL(request.url).pathname;
if (!pathname.startsWith(prefix)) return [];
const rest = pathname.slice(prefix.length);
return rest
.split("/")
.map((segment) => segment.trim())
.filter(Boolean)
.map((segment) => decodeURIComponent(segment));
}
export function toOptionalNumber(value: string | null) {
if (!value) return undefined;
const parsed = Number.parseInt(value, 10);
+6 -10
View File
@@ -1,16 +1,12 @@
function toHeaderRecord(init?: HeadersInit): Record<string, string> {
if (!init) return {};
if (init instanceof Headers) return Object.fromEntries(init.entries());
if (Array.isArray(init)) return Object.fromEntries(init);
return { ...(init as Record<string, string>) };
}
export function mergeHeaders(...inits: Array<HeadersInit | undefined>): Record<string, string> {
const out: Record<string, string> = {};
const out = new Headers();
for (const init of inits) {
Object.assign(out, toHeaderRecord(init));
if (!init) continue;
for (const [key, value] of new Headers(init)) {
out.set(key, value);
}
}
return out;
return Object.fromEntries(out.entries());
}
export function corsHeaders(origin: string = "*"): Record<string, string> {
+53
View File
@@ -0,0 +1,53 @@
export type PackagePathRoute = {
packageName: string;
rest: string[];
};
export function getPathSegments(request: Request, prefix: string) {
const pathname = new URL(request.url).pathname;
if (!pathname.startsWith(prefix)) return [];
const rest = pathname.slice(prefix.length);
return rest
.split("/")
.map((segment) => segment.trim())
.filter(Boolean)
.map((segment) => decodeURIComponent(segment));
}
export function parsePackagePathSegments(segments: string[]): PackagePathRoute | null {
if (segments.length === 0) return null;
const firstSegment = decodePackagePathSegment(segments[0]!);
if (firstSegment.startsWith("@")) {
if (firstSegment.includes("/")) {
const [scope, name, ...encodedRest] = firstSegment.split("/");
if (!scope || !name) return null;
return {
packageName: `${scope}/${name}`,
rest: [...encodedRest, ...segments.slice(1)],
};
}
if (segments.length < 2) return null;
return {
packageName: `${firstSegment}/${decodePackagePathSegment(segments[1]!)}`,
rest: segments.slice(2),
};
}
return {
packageName: firstSegment,
rest: segments.slice(1),
};
}
function decodePackagePathSegment(segment: string) {
let decoded = segment;
for (let i = 0; i < 2 && decoded.includes("%"); i += 1) {
try {
const next = decodeURIComponent(decoded);
if (next === decoded) break;
decoded = next;
} catch {
break;
}
}
return decoded;
}
+222 -76
View File
@@ -32,19 +32,24 @@ function makeRateLimitCtx(plan: MockRateLimitPlan) {
role: plan.userRole ?? "user",
};
}
if ("key" in args && "limit" in args && "windowMs" in args) {
const key = String(args.key);
if (key.startsWith("ip:")) return plan.ip;
if (key.startsWith("user:")) return plan.user;
}
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`);
});
const runMutation = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
if ("name" in args && "key" in args && !("config" in args)) {
return { action: "updated", expiresAt: Date.now() + 86_400_000 };
}
if (!("name" in args && "config" in args)) {
throw new Error(`Unexpected runMutation args: ${JSON.stringify(args)}`);
}
const key = String(args.key);
const source = key.startsWith("user:") ? plan.user : plan.ip;
if (!source) throw new Error(`Missing rate limit source for ${key}`);
return { allowed: source.allowed, remaining: source.remaining };
if (source.allowed) {
source.remaining = Math.max(0, source.remaining - 1);
return { ok: true };
}
return { ok: false, retryAfter: Math.max(1, source.resetAt - Date.now()) };
});
return {
@@ -53,6 +58,12 @@ function makeRateLimitCtx(plan: MockRateLimitPlan) {
} as unknown as Parameters<typeof applyRateLimit>[0];
}
function componentRateLimitCalls(runMutation: ReturnType<typeof vi.fn>) {
return runMutation.mock.calls.filter(([, args]) => {
return Boolean(args && typeof args === "object" && "config" in args);
}) as [unknown, Record<string, unknown>][];
}
describe("getClientIp", () => {
let prev: string | undefined;
beforeEach(() => {
@@ -86,12 +97,23 @@ describe("getClientIp", () => {
expect(getClientIp(request)).toBeNull();
});
it("returns first ip from cf-connecting-ip", () => {
it("ignores cf-connecting-ip unless client ip headers are explicitly trusted", () => {
const request = new Request("https://example.com", {
headers: {
"cf-connecting-ip": "203.0.113.1, 198.51.100.2",
},
});
delete process.env.TRUST_FORWARDED_IPS;
expect(getClientIp(request)).toBeNull();
});
it("returns first ip from cf-connecting-ip when trusted mode is enabled", () => {
const request = new Request("https://example.com", {
headers: {
"cf-connecting-ip": "203.0.113.1, 198.51.100.2",
},
});
process.env.TRUST_FORWARDED_IPS = "true";
expect(getClientIp(request)).toBe("203.0.113.1");
});
@@ -142,20 +164,20 @@ describe("RATE_LIMITS", () => {
describe("applyRateLimit headers", () => {
afterEach(() => {
vi.restoreAllMocks();
vi.useRealTimers();
vi.unstubAllEnvs();
});
it("returns delay-seconds Retry-After on 429 (not epoch)", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
const runMutation = vi.fn();
const ctx = {
runQuery: vi.fn().mockResolvedValue({
const ctx = makeRateLimitCtx({
ip: {
allowed: false,
remaining: 0,
limit: 20,
limit: RATE_LIMITS.download.ip,
resetAt: 1_030_500,
}),
runMutation,
} as unknown as Parameters<typeof applyRateLimit>[0];
},
});
const request = new Request("https://example.com", {
headers: { "cf-connecting-ip": "203.0.113.1" },
});
@@ -167,23 +189,18 @@ describe("applyRateLimit headers", () => {
expect(result.response.headers.get("Retry-After")).toBe("31");
expect(result.response.headers.get("X-RateLimit-Reset")).toBe("1031");
expect(result.response.headers.get("RateLimit-Reset")).toBe("31");
expect(runMutation).not.toHaveBeenCalled();
});
it("includes rate-limit headers without Retry-After when allowed", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_000_000);
const ctx = {
runQuery: vi.fn().mockResolvedValue({
const ctx = makeRateLimitCtx({
ip: {
allowed: true,
remaining: 19,
limit: 20,
limit: RATE_LIMITS.download.ip,
resetAt: 2_015_000,
}),
runMutation: vi.fn().mockResolvedValue({
allowed: true,
remaining: 18,
}),
} as unknown as Parameters<typeof applyRateLimit>[0];
},
});
const request = new Request("https://example.com", {
headers: { "cf-connecting-ip": "203.0.113.1" },
});
@@ -192,30 +209,22 @@ describe("applyRateLimit headers", () => {
expect(result.ok).toBe(true);
if (!result.ok) return;
const headers = new Headers(result.headers);
expect(headers.get("X-RateLimit-Limit")).toBe("20");
expect(headers.get("X-RateLimit-Remaining")).toBe("18");
expect(headers.get("X-RateLimit-Reset")).toBe("2015");
expect(headers.get("RateLimit-Limit")).toBe("20");
expect(headers.get("RateLimit-Remaining")).toBe("18");
expect(headers.get("RateLimit-Reset")).toBe("15");
expect(headers.get("X-RateLimit-Limit")).toBe(String(RATE_LIMITS.download.ip));
expect(headers.get("X-RateLimit-Remaining")).toBeNull();
expect(headers.get("X-RateLimit-Reset")).toBe("2040");
expect(headers.get("RateLimit-Limit")).toBe(String(RATE_LIMITS.download.ip));
expect(headers.get("RateLimit-Remaining")).toBeNull();
expect(headers.get("RateLimit-Reset")).toBe("40");
expect(headers.get("Retry-After")).toBeNull();
});
it("converts shard write conflicts into a rate-limit response", async () => {
it("returns retryable unavailable response when component counter writes conflict", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_500_000);
const ctx = {
runQuery: vi.fn().mockResolvedValue({
allowed: true,
remaining: 19,
limit: 20,
resetAt: 2_530_000,
}),
runMutation: vi
.fn()
.mockRejectedValue(
new Error(
'Document in table "rateLimitCounters" changed while this mutation was being run',
),
new Error('Document in table "rateLimits" changed while this mutation was being run'),
),
} as unknown as Parameters<typeof applyRateLimit>[0];
const request = new Request("https://example.com", {
@@ -226,13 +235,38 @@ describe("applyRateLimit headers", () => {
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.response.status).toBe(429);
expect(result.response.headers.get("Retry-After")).toBe("30");
expect(result.response.status).toBe(503);
expect(result.response.headers.get("Retry-After")).toBe("1");
await expect(result.response.text()).resolves.toBe("Rate limit temporarily unavailable");
});
it("selects one of 16 active counter shards", async () => {
it("returns retryable unavailable response when metadata key writes conflict", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_550_000);
const ctx = {
runMutation: vi
.fn()
.mockRejectedValue(
new Error(
'Document in table "httpRateLimitKeys" changed while this mutation was being run',
),
),
} as unknown as Parameters<typeof applyRateLimit>[0];
const result = await applyRateLimit(
ctx,
new Request("https://example.com/api/v1/packages/demo"),
"read",
);
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.response.status).toBe(503);
expect(result.response.headers.get("Retry-After")).toBe("1");
await expect(result.response.text()).resolves.toBe("Rate limit temporarily unavailable");
});
it("configures component-backed HTTP limits with 16 shards", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_600_000);
vi.spyOn(Math, "random").mockReturnValue(0.999);
const ctx = makeRateLimitCtx({
ip: {
allowed: true,
@@ -249,8 +283,111 @@ describe("applyRateLimit headers", () => {
expect(result.ok).toBe(true);
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
const [, args] = runMutation.mock.calls[0] as [unknown, Record<string, unknown>];
expect(args.shard).toBe(15);
const [, args] = componentRateLimitCalls(runMutation)[0];
expect(args).toMatchObject({
name: "downloadIp",
key: "ip:unknown:download",
config: expect.objectContaining({
kind: "fixed window",
rate: RATE_LIMITS.download.ip,
period: 60_000,
start: 0,
shards: 16,
}),
});
});
it("passes component key metadata after an allowed limiter check", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_650_000);
const ctx = makeRateLimitCtx({
ip: {
allowed: true,
remaining: 19,
limit: RATE_LIMITS.download.ip,
resetAt: 2_700_000,
},
});
const result = await applyRateLimit(
ctx,
new Request("https://example.com/api/v1/download?slug=demo"),
"download",
);
expect(result.ok).toBe(true);
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
expect(componentRateLimitCalls(runMutation).map(([, args]) => args)).toContainEqual(
expect.objectContaining({
config: expect.any(Object),
name: "downloadIp",
key: "ip:unknown:download",
now: 2_650_000,
ttlMs: 86_400_000,
}),
);
});
it("passes component key metadata after a denied limiter check", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_660_000);
const ctx = makeRateLimitCtx({
ip: {
allowed: false,
remaining: 0,
limit: RATE_LIMITS.download.ip,
resetAt: 2_690_000,
},
});
const result = await applyRateLimit(
ctx,
new Request("https://example.com/api/v1/download?slug=demo"),
"download",
);
expect(result.ok).toBe(false);
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
expect(componentRateLimitCalls(runMutation).map(([, args]) => args)).toContainEqual(
expect.objectContaining({
config: expect.any(Object),
name: "downloadIp",
key: "ip:unknown:download",
now: 2_660_000,
ttlMs: 86_400_000,
}),
);
});
it("does not shard low-rate export ip buckets", async () => {
vi.spyOn(Date, "now").mockReturnValue(2_700_000);
const ctx = makeRateLimitCtx({
ip: {
allowed: true,
remaining: 9,
limit: RATE_LIMITS.export.ip,
resetAt: 2_740_000,
},
});
const result = await applyRateLimit(
ctx,
new Request("https://example.com/api/v1/skills/export"),
"export",
);
expect(result.ok).toBe(true);
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
const [, args] = componentRateLimitCalls(runMutation)[0];
expect(args).toMatchObject({
name: "exportIp",
key: "ip:unknown:export",
config: expect.objectContaining({
kind: "fixed window",
rate: RATE_LIMITS.export.ip,
period: 60_000,
start: 0,
shards: 1,
}),
});
});
it("allows authenticated users when user bucket is healthy and shared ip bucket is exhausted", async () => {
@@ -280,8 +417,8 @@ describe("applyRateLimit headers", () => {
expect(result.ok).toBe(true);
if (!result.ok) return;
const headers = new Headers(result.headers);
expect(headers.get("X-RateLimit-Limit")).toBe("120");
expect(headers.get("X-RateLimit-Remaining")).toBe("41");
expect(headers.get("X-RateLimit-Limit")).toBe(String(RATE_LIMITS.download.key));
expect(headers.get("X-RateLimit-Remaining")).toBeNull();
expect(headers.get("Retry-After")).toBeNull();
});
@@ -311,7 +448,7 @@ describe("applyRateLimit headers", () => {
const result = await applyRateLimit(ctx, request, "download");
expect(result.ok).toBe(true);
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
const consumedKeys = runMutation.mock.calls.map(([, args]) => String(args.key));
const consumedKeys = componentRateLimitCalls(runMutation).map(([, args]) => String(args.key));
expect(consumedKeys.some((key) => key.startsWith("user:"))).toBe(true);
expect(consumedKeys.some((key) => key.startsWith("ip:"))).toBe(false);
});
@@ -343,14 +480,12 @@ describe("applyRateLimit headers", () => {
const result = await applyRateLimit(ctx, request, "write");
expect(result.ok).toBe(true);
const runQuery = (ctx as unknown as { runQuery: ReturnType<typeof vi.fn> }).runQuery;
const rateLimitStatusCalls = runQuery.mock.calls
.map(([, args]) => args as Record<string, unknown>)
.filter((args) => "key" in args && "limit" in args);
expect(rateLimitStatusCalls).toContainEqual(
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
expect(componentRateLimitCalls(runMutation).map(([, args]) => args)).toContainEqual(
expect.objectContaining({
name: "writeAdminKey",
key: "user:users_123:write",
limit: RATE_LIMITS.write.adminKey,
config: expect.objectContaining({ rate: RATE_LIMITS.write.adminKey }),
}),
);
if (!result.ok) return;
@@ -386,12 +521,12 @@ describe("applyRateLimit headers", () => {
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.response.status).toBe(429);
expect(result.response.headers.get("X-RateLimit-Limit")).toBe("120");
expect(result.response.headers.get("X-RateLimit-Limit")).toBe(String(RATE_LIMITS.download.key));
expect(result.response.headers.get("X-RateLimit-Remaining")).toBe("0");
expect(result.response.headers.get("Retry-After")).toBe("30");
});
it("scopes anonymous download fallback buckets when client ip is missing", async () => {
it("uses one anonymous download fallback bucket when client ip is missing", async () => {
vi.spyOn(Date, "now").mockReturnValue(4_500_000);
const ctx = makeRateLimitCtx({
ip: {
@@ -401,21 +536,27 @@ describe("applyRateLimit headers", () => {
resetAt: 4_530_000,
},
});
const request = new Request(
"https://example.com/api/v1/packages/tickflow-assist/download?version=0.2.10",
await applyRateLimit(
ctx,
new Request("https://example.com/api/v1/download?slug=first&version=1.0.0"),
"download",
);
await applyRateLimit(
ctx,
new Request("https://example.com/api/v1/packages/second-plugin/download?version=0.2.0"),
"download",
);
const result = await applyRateLimit(ctx, request, "download");
expect(result.ok).toBe(true);
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
const consumedKeys = runMutation.mock.calls.map(([, args]) => String(args.key));
expect(consumedKeys).toContain(
"ip:unknown:download:/api/v1/packages/tickflow-assist/download?version=0.2.10",
);
expect(componentRateLimitCalls(runMutation).map(([, args]) => String(args.key))).toEqual([
"ip:unknown:download",
"ip:unknown:download",
]);
});
it("scopes known-ip anonymous buckets by rate limit kind", async () => {
vi.stubEnv("TRUST_FORWARDED_IPS", "true");
vi.spyOn(Date, "now").mockReturnValue(4_550_000);
const readCtx = makeRateLimitCtx({
ip: {
@@ -444,10 +585,10 @@ describe("applyRateLimit headers", () => {
.runMutation;
const downloadMutation = (downloadCtx as unknown as { runMutation: ReturnType<typeof vi.fn> })
.runMutation;
expect(readMutation.mock.calls.map(([, args]) => String(args.key))).toContain(
expect(componentRateLimitCalls(readMutation).map(([, args]) => String(args.key))).toContain(
"ip:203.0.113.1:read",
);
expect(downloadMutation.mock.calls.map(([, args]) => String(args.key))).toContain(
expect(componentRateLimitCalls(downloadMutation).map(([, args]) => String(args.key))).toContain(
"ip:203.0.113.1:download",
);
});
@@ -479,12 +620,12 @@ describe("applyRateLimit headers", () => {
expect(result.ok).toBe(true);
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
expect(runMutation.mock.calls.map(([, args]) => String(args.key))).toContain(
expect(componentRateLimitCalls(runMutation).map(([, args]) => String(args.key))).toContain(
"user:users_123:download",
);
});
it("scopes non-download missing-ip anonymous requests by rate limit kind", async () => {
it("uses one anonymous read fallback bucket when client ip is missing", async () => {
vi.spyOn(Date, "now").mockReturnValue(4_600_000);
const ctx = makeRateLimitCtx({
ip: {
@@ -494,14 +635,19 @@ describe("applyRateLimit headers", () => {
resetAt: 4_630_000,
},
});
const request = new Request("https://example.com/api/v1/search?q=demo");
const result = await applyRateLimit(ctx, request, "read");
await applyRateLimit(ctx, new Request("https://example.com/api/v1/search?q=demo"), "read");
await applyRateLimit(
ctx,
new Request("https://example.com/api/v1/packages/second-plugin"),
"read",
);
expect(result.ok).toBe(true);
const runMutation = (ctx as unknown as { runMutation: ReturnType<typeof vi.fn> }).runMutation;
const consumedKeys = runMutation.mock.calls.map(([, args]) => String(args.key));
expect(consumedKeys).toContain("ip:unknown:read");
expect(componentRateLimitCalls(runMutation).map(([, args]) => String(args.key))).toEqual([
"ip:unknown:read",
"ip:unknown:read",
]);
});
it("falls back to ip enforcement when bearer token is invalid", async () => {
@@ -526,7 +672,7 @@ describe("applyRateLimit headers", () => {
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.response.status).toBe(429);
expect(result.response.headers.get("X-RateLimit-Limit")).toBe("20");
expect(result.response.headers.get("X-RateLimit-Limit")).toBe(String(RATE_LIMITS.download.ip));
expect(result.response.headers.get("Retry-After")).toBe("30");
});
});
+146 -77
View File
@@ -1,9 +1,9 @@
import { MINUTE, type RateLimitConfig } from "@convex-dev/rate-limiter";
import { internal } from "../_generated/api";
import type { Doc } from "../_generated/dataModel";
import type { ActionCtx } from "../_generated/server";
import { getOptionalApiTokenUser } from "./apiTokenAuth";
import { corsHeaders, mergeHeaders } from "./httpHeaders";
import { RATE_LIMIT_COUNTER_SHARDS, RATE_LIMIT_WINDOW_MS } from "./rateLimitConfig";
export const RATE_LIMITS = {
read: { ip: 3000, key: 12000, adminKey: 120000 },
@@ -13,18 +13,76 @@ export const RATE_LIMITS = {
export: { ip: 10, key: 60, adminKey: 60 },
} as const;
const RATE_LIMIT_WINDOW_MS = 60_000;
const HTTP_RATE_LIMIT_SHARDS = 16;
const HTTP_RATE_LIMIT_MIN_SHARD_CAPACITY = 10;
const HTTP_RATE_LIMIT_KEY_TTL_MS = 24 * 60 * 60 * 1000;
type RateLimitResult = {
allowed: boolean;
remaining: number;
// The component does not expose an exact global remaining count for sharded
// buckets, so successful responses omit this instead of guessing.
remaining?: number;
limit: number;
resetAt: number;
unavailable?: boolean;
};
export type ApplyRateLimitResult =
| { ok: true; headers: HeadersInit }
| { ok: false; response: Response };
type RateLimitKind = keyof typeof RATE_LIMITS;
type RateLimitSubject = "ip" | "key" | "adminKey";
type HttpRateLimitName = `${RateLimitKind}${Capitalize<RateLimitSubject>}`;
type FixedWindowRateLimitConfig = Extract<RateLimitConfig, { kind: "fixed window" }>;
const preappliedRateLimitHeaders = new WeakMap<Request, HeadersInit>();
function fixedWindowRateLimit(rate: number): FixedWindowRateLimitConfig {
const shards = Math.max(
1,
Math.min(HTTP_RATE_LIMIT_SHARDS, Math.floor(rate / HTTP_RATE_LIMIT_MIN_SHARD_CAPACITY)),
);
return {
kind: "fixed window",
rate,
period: MINUTE,
start: 0,
shards,
};
}
const HTTP_RATE_LIMIT_CONFIGS = {
readIp: fixedWindowRateLimit(RATE_LIMITS.read.ip),
readKey: fixedWindowRateLimit(RATE_LIMITS.read.key),
readAdminKey: fixedWindowRateLimit(RATE_LIMITS.read.adminKey),
writeIp: fixedWindowRateLimit(RATE_LIMITS.write.ip),
writeKey: fixedWindowRateLimit(RATE_LIMITS.write.key),
writeAdminKey: fixedWindowRateLimit(RATE_LIMITS.write.adminKey),
trustedPublishIp: fixedWindowRateLimit(RATE_LIMITS.trustedPublish.ip),
trustedPublishKey: fixedWindowRateLimit(RATE_LIMITS.trustedPublish.key),
trustedPublishAdminKey: fixedWindowRateLimit(RATE_LIMITS.trustedPublish.adminKey),
downloadIp: fixedWindowRateLimit(RATE_LIMITS.download.ip),
downloadKey: fixedWindowRateLimit(RATE_LIMITS.download.key),
downloadAdminKey: fixedWindowRateLimit(RATE_LIMITS.download.adminKey),
exportIp: fixedWindowRateLimit(RATE_LIMITS.export.ip),
exportKey: fixedWindowRateLimit(RATE_LIMITS.export.key),
exportAdminKey: fixedWindowRateLimit(RATE_LIMITS.export.adminKey),
} as const satisfies Record<HttpRateLimitName, RateLimitConfig>;
export function markRateLimitApplied(request: Request, headers: HeadersInit): void {
preappliedRateLimitHeaders.set(request, headers);
}
export async function applyRateLimit(
ctx: ActionCtx,
request: Request,
kind: keyof typeof RATE_LIMITS,
): Promise<{ ok: true; headers: HeadersInit } | { ok: false; response: Response }> {
kind: RateLimitKind,
): Promise<ApplyRateLimitResult> {
const preappliedHeaders = preappliedRateLimitHeaders.get(request);
if (preappliedHeaders) return { ok: true, headers: preappliedHeaders };
const auth = await getOptionalApiTokenUser(ctx, request);
const ip = getClientIp(request) ?? "unknown";
const ipSource = getClientIpSource(request);
@@ -37,9 +95,11 @@ export async function applyRateLimit(
const userResult = await checkRateLimit(
ctx,
getAuthenticatedRateLimitKey(auth.userId, kind),
userLimit,
userLimit.name,
userLimit.limit,
);
const headers = rateHeaders(userResult);
if (userResult.unavailable) return rateLimitUnavailable(headers);
if (!userResult.allowed) {
console.info("rate_limit_denied", {
kind,
@@ -71,10 +131,12 @@ export async function applyRateLimit(
// Anonymous requests remain IP-enforced.
const ipResult = await checkRateLimit(
ctx,
getAnonymousRateLimitKey(request, kind, ip),
getAnonymousRateLimitKey(kind, ip),
getHttpRateLimitName(kind, "ip"),
RATE_LIMITS[kind].ip,
);
const headers = rateHeaders(ipResult);
if (ipResult.unavailable) return rateLimitUnavailable(headers);
if (!ipResult.allowed) {
console.info("rate_limit_denied", {
@@ -104,29 +166,29 @@ export async function applyRateLimit(
return { ok: true, headers };
}
function getAnonymousRateLimitKey(request: Request, kind: keyof typeof RATE_LIMITS, ip: string) {
function getAnonymousRateLimitKey(kind: RateLimitKind, ip: string) {
if (ip !== "unknown") return `ip:${ip}:${kind}`;
if (kind !== "download") return `ip:unknown:${kind}`;
return `ip:unknown:download:${getDownloadRateLimitScope(request)}`;
return `ip:unknown:${kind}`;
}
function getAuthenticatedRateLimitKey(userId: string, kind: keyof typeof RATE_LIMITS) {
function getAuthenticatedRateLimitKey(userId: string, kind: RateLimitKind) {
return `user:${userId}:${kind}`;
}
function getAuthenticatedRateLimit(
kind: keyof typeof RATE_LIMITS,
user: Pick<Doc<"users">, "role">,
) {
return user.role === "admin" ? RATE_LIMITS[kind].adminKey : RATE_LIMITS[kind].key;
function getAuthenticatedRateLimit(kind: RateLimitKind, user: Pick<Doc<"users">, "role">) {
const subject = user.role === "admin" ? "adminKey" : "key";
return {
name: getHttpRateLimitName(kind, subject),
limit: RATE_LIMITS[kind][subject],
};
}
export function getClientIp(request: Request) {
export function getClientIp(request: Request): string | null {
if (!shouldTrustClientIpHeaders()) return null;
const cfHeader = request.headers.get("cf-connecting-ip");
if (cfHeader) return splitFirstIp(cfHeader);
if (!shouldTrustForwardedIps()) return null;
const forwarded =
request.headers.get("x-forwarded-for") ??
request.headers.get("x-real-ip") ??
@@ -136,8 +198,8 @@ export function getClientIp(request: Request) {
}
function getClientIpSource(request: Request) {
if (!shouldTrustClientIpHeaders()) return "none";
if (request.headers.get("cf-connecting-ip")) return "cf-connecting-ip";
if (!shouldTrustForwardedIps()) return "none";
if (request.headers.get("x-forwarded-for")) return "x-forwarded-for";
if (request.headers.get("x-real-ip")) return "x-real-ip";
if (request.headers.get("fly-client-ip")) return "fly-client-ip";
@@ -147,45 +209,62 @@ function getClientIpSource(request: Request) {
async function checkRateLimit(
ctx: ActionCtx,
key: string,
name: HttpRateLimitName,
limit: number,
): Promise<RateLimitResult> {
// Step 1: Read-only check to avoid write conflicts on denied requests.
const status = (await ctx.runQuery(internal.rateLimits.getRateLimitStatusInternal, {
key,
limit,
windowMs: RATE_LIMIT_WINDOW_MS,
})) as RateLimitResult;
if (!status.allowed) {
return status;
}
// Step 2: Consume with a mutation only when still allowed.
let result: { allowed: boolean; remaining: number };
const now = Date.now();
try {
result = (await ctx.runMutation(internal.rateLimits.consumeRateLimitInternal, {
const status = await ctx.runMutation(internal.rateLimits.consumeHttpRateLimitKeyInternal, {
name,
key,
limit,
windowMs: RATE_LIMIT_WINDOW_MS,
shard: Math.floor(Math.random() * RATE_LIMIT_COUNTER_SHARDS),
})) as { allowed: boolean; remaining: number };
} catch (error) {
if (isRateLimitWriteConflict(error)) {
config: HTTP_RATE_LIMIT_CONFIGS[name],
now,
ttlMs: HTTP_RATE_LIMIT_KEY_TTL_MS,
});
if (!status.ok) {
return {
allowed: false,
remaining: 0,
limit: status.limit,
resetAt: status.resetAt,
limit,
resetAt: now + status.retryAfter,
};
}
throw error;
}
return {
allowed: true,
limit,
resetAt: getCurrentWindowResetAt(now),
};
} catch (error) {
if (!isRateLimitWriteConflict(error)) throw error;
return {
allowed: false,
remaining: 0,
limit,
resetAt: now + 1000,
unavailable: true,
};
}
}
function rateLimitUnavailable(headers: HeadersInit): Extract<ApplyRateLimitResult, { ok: false }> {
console.warn("rate_limit_unavailable", {
reason: "counter_write_contention",
});
return {
allowed: result.allowed,
remaining: Math.max(0, status.remaining - 1),
limit: status.limit,
resetAt: status.resetAt,
ok: false,
response: new Response("Rate limit temporarily unavailable", {
status: 503,
headers: mergeHeaders(
{
"Content-Type": "text/plain; charset=utf-8",
"Cache-Control": "no-store",
"Retry-After": "1",
},
headers,
corsHeaders(),
),
}),
};
}
@@ -193,15 +272,27 @@ function rateHeaders(result: RateLimitResult): HeadersInit {
const nowMs = Date.now();
const resetSeconds = Math.ceil(result.resetAt / 1000);
const resetDelaySeconds = Math.max(1, Math.ceil((result.resetAt - nowMs) / 1000));
return {
const headers: Record<string, string> = {
"X-RateLimit-Limit": String(result.limit),
"X-RateLimit-Remaining": String(result.remaining),
"X-RateLimit-Reset": String(resetSeconds),
"RateLimit-Limit": String(result.limit),
"RateLimit-Remaining": String(result.remaining),
"RateLimit-Reset": String(resetDelaySeconds),
...(result.allowed ? {} : { "Retry-After": String(resetDelaySeconds) }),
};
if (result.remaining !== undefined) {
headers["X-RateLimit-Remaining"] = String(result.remaining);
headers["RateLimit-Remaining"] = String(result.remaining);
}
if (!result.allowed) headers["Retry-After"] = String(resetDelaySeconds);
return headers;
}
function getCurrentWindowResetAt(now: number) {
return Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS + RATE_LIMIT_WINDOW_MS;
}
function getHttpRateLimitName(kind: RateLimitKind, subject: RateLimitSubject): HttpRateLimitName {
const suffix = subject === "ip" ? "Ip" : subject === "key" ? "Key" : "AdminKey";
return `${kind}${suffix}` as HttpRateLimitName;
}
export function parseBearerToken(request: Request) {
@@ -220,32 +311,10 @@ function splitFirstIp(header: string | null) {
return trimmed || null;
}
function getDownloadRateLimitScope(request: Request) {
try {
const url = new URL(request.url);
const path = normalizeRateLimitKeyPart(url.pathname.replace(/\/{2,}/g, "/") || "/");
const params = new URLSearchParams();
for (const name of ["slug", "version", "tag"] as const) {
const value = url.searchParams.get(name)?.trim();
if (value) params.set(name, normalizeRateLimitKeyPart(value));
}
const query = params.toString();
return query ? `${path}?${query}` : path;
} catch {
return "unknown";
}
}
function normalizeRateLimitKeyPart(value: string) {
return value.slice(0, 500);
}
function shouldTrustForwardedIps() {
function shouldTrustClientIpHeaders() {
const value = (process.env.TRUST_FORWARDED_IPS ?? "").trim().toLowerCase();
// Hardening default: CF-only. Forwarded headers are trivial to spoof unless you
// control the trusted proxy layer.
// Direct Convex HTTP endpoints can be reached without ClawHub's edge. Trust
// client IP headers only when the deployment is explicitly behind that edge.
if (!value) return false;
if (value === "1" || value === "true" || value === "yes") return true;
return false;
@@ -254,7 +323,7 @@ function shouldTrustForwardedIps() {
function isRateLimitWriteConflict(error: unknown) {
if (!(error instanceof Error)) return false;
return (
(error.message.includes("rateLimitCounters") || error.message.includes("rateLimits")) &&
(error.message.includes("rateLimits") || error.message.includes("httpRateLimitKeys")) &&
error.message.includes("changed while this mutation was being run")
);
}
+366
View File
@@ -0,0 +1,366 @@
/* @vitest-environment node */
import { httpRouter } from "convex/server";
import { describe, expect, it, vi } from "vitest";
import type { ActionCtx } from "../_generated/server";
import { httpAction } from "../functions";
import { applyRateLimit, RATE_LIMITS } from "./httpRateLimit";
import { installRateLimitedRoutes, rateLimitedHttpAction } from "./httpRouteRateLimit";
type WrappedHttpAction = {
_handler: (ctx: ActionCtx, request: Request) => Promise<Response>;
};
function makeCtx({
allowed = true,
}: {
allowed?: boolean;
} = {}) {
const runQuery = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
throw new Error(`Unexpected runQuery args: ${JSON.stringify(args)}`);
});
const runMutation = vi.fn(async (_fn: unknown, args: Record<string, unknown>) => {
if ("name" in args && "config" in args) {
const config = args.config as { period: number };
return allowed ? { ok: true } : { ok: false, retryAfter: config.period };
}
throw new Error(`Unexpected runMutation args: ${JSON.stringify(args)}`);
});
return { ctx: { runQuery, runMutation } as unknown as ActionCtx, runQuery, runMutation };
}
describe("rateLimitedHttpAction", () => {
it("blocks before the wrapped HTTP handler runs", async () => {
const handler = vi.fn(async () => new Response("ok"));
const action = httpAction(handler) as unknown as Parameters<typeof rateLimitedHttpAction>[0];
const wrapped = rateLimitedHttpAction(action, {
resolveRateLimit: () => ({ kind: "read" }),
}) as unknown as WrappedHttpAction;
const { ctx, runMutation } = makeCtx({ allowed: false });
const response = await wrapped._handler(ctx, new Request("https://example.com/api/v1/skills"));
expect(response.status).toBe(429);
expect(await response.text()).toBe("Rate limit exceeded");
expect(handler).not.toHaveBeenCalled();
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
name: "readIp",
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
}),
);
});
it("merges route rate-limit headers into successful responses", async () => {
const action = httpAction(
async () => new Response("ok", { headers: { "X-App": "1" } }),
) as unknown as Parameters<typeof rateLimitedHttpAction>[0];
const wrapped = rateLimitedHttpAction(action, {
resolveRateLimit: () => ({ kind: "read" }),
}) as unknown as WrappedHttpAction;
const { ctx } = makeCtx();
const response = await wrapped._handler(ctx, new Request("https://example.com/api/v1/skills"));
expect(response.status).toBe(200);
expect(response.headers.get("X-App")).toBe("1");
expect(response.headers.get("RateLimit-Limit")).toBe(String(RATE_LIMITS.read.ip));
});
it("preserves multiple Set-Cookie headers on successful responses", async () => {
const action = httpAction(async () => {
const headers = new Headers({ "X-App": "1" });
headers.append("Set-Cookie", "oauth_state=abc; HttpOnly; Path=/");
headers.append("Set-Cookie", "redirect_to=%2Fdashboard; HttpOnly; Path=/");
return new Response("ok", { headers });
}) as unknown as Parameters<typeof rateLimitedHttpAction>[0];
const wrapped = rateLimitedHttpAction(action, {
resolveRateLimit: () => ({ kind: "read" }),
}) as unknown as WrappedHttpAction;
const { ctx } = makeCtx();
const response = await wrapped._handler(
ctx,
new Request("https://example.com/api/auth/signin"),
);
expect(response.status).toBe(200);
expect(response.headers.get("X-App")).toBe("1");
expect(response.headers.get("RateLimit-Limit")).toBe(String(RATE_LIMITS.read.ip));
expect([...response.headers.entries()].filter(([key]) => key === "set-cookie")).toEqual([
["set-cookie", "oauth_state=abc; HttpOnly; Path=/"],
["set-cookie", "redirect_to=%2Fdashboard; HttpOnly; Path=/"],
]);
});
it("supports dynamic route-specific rate limit overrides", async () => {
const action = httpAction(async () => new Response("ok")) as unknown as Parameters<
typeof rateLimitedHttpAction
>[0];
const wrapped = rateLimitedHttpAction(action, {
resolveRateLimit: (request) =>
new URL(request.url).pathname.endsWith("/download")
? { kind: "download" }
: { kind: "read" },
}) as unknown as WrappedHttpAction;
const { ctx, runMutation } = makeCtx();
await wrapped._handler(ctx, new Request("https://example.com/api/v1/packages/demo/download"));
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
name: "downloadIp",
config: expect.objectContaining({ rate: RATE_LIMITS.download.ip }),
}),
);
});
it("makes wrapper-applied limits authoritative for nested handler checks", async () => {
const handler = vi.fn(async (ctx: ActionCtx, request: Request) => {
const nestedRate = await applyRateLimit(ctx, request, "write");
if (!nestedRate.ok) return nestedRate.response;
return new Response("ok", { headers: nestedRate.headers });
});
const action = httpAction(handler) as unknown as Parameters<typeof rateLimitedHttpAction>[0];
const wrapped = rateLimitedHttpAction(action, {
resolveRateLimit: () => ({ kind: "read" }),
}) as unknown as WrappedHttpAction;
const { ctx, runMutation } = makeCtx();
const response = await wrapped._handler(ctx, new Request("https://example.com/api/v1/search"));
expect(response.status).toBe(200);
expect(runMutation).toHaveBeenCalledTimes(1);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
name: "readIp",
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
}),
);
expect(response.headers.get("RateLimit-Limit")).toBe(String(RATE_LIMITS.read.ip));
expect(response.headers.get("RateLimit-Limit")).not.toContain(",");
});
it("allows explicit route opt-outs", async () => {
const action = httpAction(async () => new Response("ok")) as unknown as Parameters<
typeof rateLimitedHttpAction
>[0];
const wrapped = rateLimitedHttpAction(action, {
resolveRateLimit: () => ({ kind: "none" }),
}) as unknown as WrappedHttpAction;
const { ctx, runQuery, runMutation } = makeCtx({ allowed: false });
const response = await wrapped._handler(ctx, new Request("https://example.com/api/v1/health"));
expect(response.status).toBe(200);
expect(runQuery).not.toHaveBeenCalled();
expect(runMutation).not.toHaveBeenCalled();
});
});
describe("installRateLimitedRoutes", () => {
it.each([
[
"read default",
"/api/v1/example",
"GET",
"https://example.com/api/v1/example",
RATE_LIMITS.read.ip,
],
[
"write default",
"/api/v1/example",
"POST",
"https://example.com/api/v1/example",
RATE_LIMITS.write.ip,
],
[
"download exact",
"/api/v1/download",
"GET",
"https://example.com/api/v1/download?slug=demo",
RATE_LIMITS.download.ip,
],
[
"export exact",
"/api/v1/skills/export",
"GET",
"https://example.com/api/v1/skills/export",
RATE_LIMITS.export.ip,
],
[
"trusted publish",
"/api/v1/publish/token/mint",
"POST",
"https://example.com/api/v1/publish/token/mint",
RATE_LIMITS.trustedPublish.ip,
],
])("applies the central %s bucket", async (_name, path, method, requestUrl, expectedLimit) => {
const router = installRateLimitedRoutes(httpRouter());
router.route({
path,
method: method as "GET" | "POST",
handler: httpAction(async () => new Response("ok")),
});
const route = router.lookup(path, method as "GET" | "POST");
if (!route) throw new Error(`Expected route for ${path}`);
const [action] = route;
const { ctx, runMutation } = makeCtx();
await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request(requestUrl, { method }),
);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
config: expect.objectContaining({ rate: expectedLimit }),
}),
);
});
it.each([
["package detail", "/api/v1/packages/", "/api/v1/packages/demo", RATE_LIMITS.read.ip],
[
"package version download",
"/api/v1/packages/",
"/api/v1/packages/demo/versions/1.0.0/download",
RATE_LIMITS.download.ip,
],
[
"package artifact",
"/api/v1/packages/",
"/api/v1/packages/demo/versions/1.0.0/artifact",
RATE_LIMITS.download.ip,
],
[
"scoped package named download",
"/api/v1/packages/",
"/api/v1/packages/@scope/download",
RATE_LIMITS.read.ip,
],
[
"scoped package named artifact",
"/api/v1/packages/",
"/api/v1/packages/@scope/artifact",
RATE_LIMITS.read.ip,
],
["npm metadata", "/api/npm/", "/api/npm/demo", RATE_LIMITS.read.ip],
["npm tarball", "/api/npm/", "/api/npm/demo/-/demo-1.0.0.tgz", RATE_LIMITS.download.ip],
[
"npm tarball with encoded separator",
"/api/npm/",
"/api/npm/demo/%2D/demo-1.0.0.tgz",
RATE_LIMITS.download.ip,
],
])(
"classifies central prefix policy for %s",
async (_name, pathPrefix, requestPath, expectedLimit) => {
const router = installRateLimitedRoutes(httpRouter());
router.route({
pathPrefix,
method: "GET",
handler: httpAction(async () => new Response("ok")),
});
const route = router.lookup(requestPath, "GET");
if (!route) throw new Error(`Expected route for ${requestPath}`);
const [action] = route;
const { ctx, runMutation } = makeCtx();
await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request(`https://example.com${requestPath}`),
);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
config: expect.objectContaining({ rate: expectedLimit }),
}),
);
},
);
it("wraps ordinary route registrations with the default method limit", async () => {
const router = installRateLimitedRoutes(httpRouter());
const handler = vi.fn(async () => new Response("ok"));
router.route({
path: "/api/v1/example",
method: "GET",
handler: httpAction(handler),
});
const route = router.lookup("/api/v1/example", "GET");
if (!route) throw new Error("Expected wrapped route");
const [action] = route;
const { ctx, runMutation } = makeCtx({ allowed: false });
const response = await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request("https://example.com/api/v1/example"),
);
expect(response.status).toBe(429);
expect(handler).not.toHaveBeenCalled();
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
name: "readIp",
config: expect.objectContaining({ rate: RATE_LIMITS.read.ip }),
}),
);
});
it("leaves auth metadata route registrations explicitly unmetered", async () => {
const router = installRateLimitedRoutes(httpRouter());
router.route({
path: "/.well-known/jwks.json",
method: "GET",
handler: httpAction(async () => new Response("jwks")),
});
const route = router.lookup("/.well-known/jwks.json", "GET");
if (!route) throw new Error("Expected auth metadata route");
const [action] = route;
const { ctx, runQuery, runMutation } = makeCtx({ allowed: false });
const response = await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request("https://example.com/.well-known/jwks.json"),
);
expect(response.status).toBe(200);
expect(await response.text()).toBe("jwks");
expect(runQuery).not.toHaveBeenCalled();
expect(runMutation).not.toHaveBeenCalled();
});
it("supports central route policy overrides", async () => {
const router = installRateLimitedRoutes(httpRouter(), {
resolveRateLimit: () => ({ kind: "export" }),
});
router.route({
path: "/custom-export",
method: "GET",
handler: httpAction(async () => new Response("ok")),
});
const route = router.lookup("/custom-export", "GET");
if (!route) throw new Error("Expected custom export route");
const [action] = route;
const { ctx, runMutation } = makeCtx();
await (action as unknown as WrappedHttpAction)._handler(
ctx,
new Request("https://example.com/custom-export"),
);
expect(runMutation).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
name: "exportIp",
config: expect.objectContaining({ rate: RATE_LIMITS.export.ip }),
}),
);
});
});
+162
View File
@@ -0,0 +1,162 @@
import { ApiRoutes, LegacyApiRoutes } from "clawhub-schema";
import type { HttpRouter, PublicHttpAction, RouteSpec } from "convex/server";
import { httpAction } from "../functions";
import { getPathSegments, parsePackagePathSegments } from "./httpPathSegments";
import { applyRateLimit, markRateLimitApplied, RATE_LIMITS } from "./httpRateLimit";
type HttpHandler = Parameters<typeof httpAction>[0];
type ConvexHttpActionWithHandler = PublicHttpAction & {
readonly _handler: HttpHandler;
};
type RouteRateLimitKind = keyof typeof RATE_LIMITS;
type RouteRateLimitDecision = { kind: "none" } | { kind: RouteRateLimitKind };
type RateLimitedHttpActionOptions = {
resolveRateLimit: (request: Request) => RouteRateLimitDecision;
};
type RouteRateLimitResolver = (spec: RouteSpec, request: Request) => RouteRateLimitDecision;
type InstallRateLimitedRoutesOptions = {
resolveRateLimit?: RouteRateLimitResolver;
};
const authMetadataPaths = new Set(["/.well-known/openid-configuration", "/.well-known/jwks.json"]);
export function installRateLimitedRoutes(
http: HttpRouter,
options: InstallRateLimitedRoutesOptions = {},
): HttpRouter {
const route = http.route.bind(http);
// Convex has no HTTP middleware hook, so install one wrapper at registration time.
http.route = ((spec: RouteSpec) => {
route({
...spec,
handler: rateLimitedHttpAction(spec.handler, {
resolveRateLimit: (request) =>
options.resolveRateLimit?.(spec, request) ?? resolveDefaultRouteRateLimit(spec, request),
}),
});
}) as typeof http.route;
return http;
}
function resolveDefaultRouteRateLimit(spec: RouteSpec, request: Request): RouteRateLimitDecision {
if (spec.method === "OPTIONS") return { kind: "none" };
const routedPath = getRoutedPath(spec);
if (authMetadataPaths.has(routedPath)) return { kind: "none" };
if (routedPath === ApiRoutes.publishTokenMint) return { kind: "trustedPublish" };
if (routedPath === ApiRoutes.skillsExport || routedPath === ApiRoutes.pluginsExport) {
return { kind: "export" };
}
if (spec.method === "GET") {
if (routedPath === ApiRoutes.download || routedPath === LegacyApiRoutes.download) {
return { kind: "download" };
}
if (routedPath === "/api/v1/package-inspector/artifact") return { kind: "download" };
if ("pathPrefix" in spec && spec.pathPrefix === `${ApiRoutes.packages}/`) {
return packageReadRouteRateLimitKind(request);
}
if ("pathPrefix" in spec && spec.pathPrefix === "/api/npm/") {
return npmMirrorRouteRateLimitKind(request);
}
}
if (spec.method === "POST" && routedPath === `${ApiRoutes.skills}/-/security-verdicts`) {
return { kind: "read" };
}
return defaultRouteRateLimitKind(spec.method);
}
function defaultRouteRateLimitKind(method: RouteSpec["method"]): RouteRateLimitDecision {
if (method === "GET") return { kind: "read" };
if (method === "OPTIONS") return { kind: "none" };
return { kind: "write" };
}
export function rateLimitedHttpAction(
action: PublicHttpAction,
options: RateLimitedHttpActionOptions,
): PublicHttpAction {
const handler = getRegisteredHttpHandler(action);
return httpAction(async (ctx, request) => {
const decision = options.resolveRateLimit(request);
if (decision.kind === "none") {
return await handler(ctx, request);
}
const rate = await applyRateLimit(ctx, request, decision.kind);
if (!rate.ok) return rate.response;
markRateLimitApplied(request, rate.headers);
const response = await handler(ctx, request);
return addRateLimitHeaders(response, rate.headers);
});
}
function addRateLimitHeaders(response: Response, headers: HeadersInit): Response {
const rateHeaders = new Headers(headers);
try {
for (const [key, value] of rateHeaders) {
response.headers.set(key, value);
}
return response;
} catch {
const mergedHeaders = new Headers(response.headers);
for (const [key, value] of rateHeaders) {
mergedHeaders.set(key, value);
}
return new Response(response.body, {
status: response.status,
statusText: response.statusText,
headers: mergedHeaders,
});
}
}
function getRegisteredHttpHandler(action: PublicHttpAction): HttpHandler {
if (!hasRegisteredHttpHandler(action)) {
throw new Error("HTTP action is missing its registered handler");
}
return action._handler;
}
function hasRegisteredHttpHandler(action: PublicHttpAction): action is ConvexHttpActionWithHandler {
return typeof (action as { _handler?: unknown })._handler === "function";
}
function getRoutedPath(spec: RouteSpec): string {
return "path" in spec ? spec.path : spec.pathPrefix;
}
function packageReadRouteRateLimitKind(request: Request): RouteRateLimitDecision {
const packageRoute = parsePackagePathSegments(getPathSegments(request, `${ApiRoutes.packages}/`));
const packageSegments = packageRoute?.rest ?? [];
if (
packageSegments[0] === "download" ||
(packageSegments[0] === "versions" &&
packageSegments[1] &&
(packageSegments[2] === "download" ||
packageSegments[2] === "artifact" ||
packageSegments[3] === "download"))
) {
return { kind: "download" };
}
return { kind: "read" };
}
function npmMirrorRouteRateLimitKind(request: Request): RouteRateLimitDecision {
const packageRoute = parsePackagePathSegments(getPathSegments(request, "/api/npm/"));
return packageRoute?.rest[0] === "-" && packageRoute.rest[1]
? { kind: "download" }
: { kind: "read" };
}
-3
View File
@@ -1,3 +0,0 @@
export const RATE_LIMIT_WINDOW_MS = 60_000;
// Keep enough shards to spread bursty writes while limiting per-window row growth.
export const RATE_LIMIT_COUNTER_SHARDS = 16;
+2 -1
View File
@@ -33,10 +33,11 @@ describe("retention policies", () => {
});
it("documents active expiring operational tables", () => {
expect(getRetentionPolicy("rateLimitCounters")).toMatchObject({
expect(getRetentionPolicy("httpRateLimitKeys")).toMatchObject({
classification: "ephemeral",
expirationField: "expiresAt",
expirationIndex: "by_expires_at",
prune: "rateLimits.pruneHttpRateLimitKeysInternal",
});
});
+9 -6
View File
@@ -200,12 +200,15 @@ export const RETENTION_POLICIES = {
prune: "usage-time expiry plus pending retention cleanup",
retention: "Device code TTL.",
}),
rateLimitCounters: ephemeral("Active rate-limit counters expire after their rate-limit window.", {
expirationField: "expiresAt",
expirationIndex: "by_expires_at",
prune: "rateLimits.pruneRateLimitCountersInternal",
retention: "Rate-limit window plus buffer.",
}),
httpRateLimitKeys: ephemeral(
"Component-backed HTTP rate-limit key metadata is operational cleanup state.",
{
expirationField: "expiresAt",
expirationIndex: "by_expires_at",
prune: "rateLimits.pruneHttpRateLimitKeysInternal",
retention: "Idle component key window plus buffer before component reset.",
},
),
downloadMetricDedupes: ephemeral(
"Download dedupe rows are only needed for recent metric windows.",
{
+402 -144
View File
@@ -2,191 +2,449 @@
import { describe, expect, it, vi } from "vitest";
import {
consumeRateLimitInternal,
getRateLimitStatusInternal,
pruneRateLimitCountersInternal,
consumeHttpRateLimitKeyInternal,
pruneHttpRateLimitKeysInternal,
touchHttpRateLimitKeyInternal,
} from "./rateLimits";
type WrappedHandler<TArgs, TResult> = {
_handler: (ctx: unknown, args: TArgs) => Promise<TResult>;
};
const getStatusHandler = (
getRateLimitStatusInternal as unknown as WrappedHandler<
{ key: string; limit: number; windowMs: number },
{ allowed: boolean; remaining: number; limit: number; resetAt: number }
const touchHttpKeyHandler = (
touchHttpRateLimitKeyInternal as unknown as WrappedHandler<
{ name: string; key: string; shard?: number; now?: number; ttlMs?: number },
{ action: "inserted" | "updated"; expiresAt: number; shard: number }
>
)._handler;
const consumeHandler = (
consumeRateLimitInternal as unknown as WrappedHandler<
{ key: string; limit: number; windowMs: number; shard?: number },
{ allowed: boolean; remaining: number }
const consumeHttpKeyHandler = (
consumeHttpRateLimitKeyInternal as unknown as WrappedHandler<
{
name: string;
key: string;
config: {
kind: "fixed window";
rate: number;
period: number;
start?: number;
shards?: number;
};
now?: number;
ttlMs?: number;
shard?: number;
},
{ ok: true; retryAfter?: number } | { ok: false; retryAfter: number }
>
)._handler;
const pruneHandler = (
pruneRateLimitCountersInternal as unknown as WrappedHandler<
const pruneHttpKeyHandler = (
pruneHttpRateLimitKeysInternal as unknown as WrappedHandler<
{ batchSize?: number },
{ deleted: number; hasMore: boolean }
>
)._handler;
describe("rate limit sharding", () => {
it("sums active counter rows without reading legacy rate limit tables", async () => {
const ctx = {
db: {
query: vi.fn(() => ({
withIndex: vi.fn(() => ({
collect: vi.fn(async () => [{ count: 4 }, { count: 5 }]),
})),
})),
},
};
function makeDb(overrides: { query: ReturnType<typeof vi.fn>; delete: ReturnType<typeof vi.fn> }) {
return {
get: vi.fn(),
insert: vi.fn(),
patch: vi.fn(),
replace: vi.fn(),
delete: overrides.delete,
query: overrides.query,
normalizeId: vi.fn(() => null),
system: {
get: vi.fn(),
query: vi.fn(),
},
};
}
const result = await getStatusHandler(ctx, {
key: "ip:test",
limit: 20,
windowMs: 60_000,
});
expect(result.allowed).toBe(true);
expect(result.remaining).toBe(11);
expect(ctx.db.query).toHaveBeenCalledTimes(1);
expect(ctx.db.query).toHaveBeenCalledWith("rateLimitCounters");
expect(ctx.db.query).not.toHaveBeenCalledWith("rateLimits");
});
it("writes only the selected active counter shard when consuming", async () => {
describe("component HTTP rate limit key metadata", () => {
it("consumes a component bucket and refreshes sharded metadata in one mutation", async () => {
const take = vi.fn(async () => []);
const insert = vi.fn();
const runMutation = vi.fn(async () => ({ ok: true }));
const eqShard = vi.fn();
const eqKey = vi.fn(() => ({ eq: eqShard }));
const eqName = vi.fn(() => ({ eq: eqKey }));
const withIndex = vi.fn((_index, builder) => {
builder({
eq: vi.fn(() => ({
eq: vi.fn(() => ({
eq: vi.fn(),
})),
})),
});
return { first: vi.fn(async () => null) };
builder({ eq: eqName });
return { take };
});
const ctx = {
db: {
query: vi.fn(() => ({ withIndex })),
get: vi.fn(),
normalizeId: vi.fn(),
insert,
patch: vi.fn(),
replace: vi.fn(),
runMutation,
db: makeDb({
query: vi.fn(() => ({
withIndex,
})),
delete: vi.fn(),
system: {
get: vi.fn(),
query: vi.fn(),
},
}),
scheduler: {
runAfter: vi.fn(),
},
};
ctx.db.insert = insert;
await consumeHandler(ctx, {
key: "ip:test",
limit: 20,
windowMs: 60_000,
const result = await consumeHttpKeyHandler(ctx, {
name: "downloadIp",
key: "ip:203.0.113.1:download",
config: { kind: "fixed window", rate: 1200, period: 60_000, start: 0, shards: 16 },
now: 10_000,
ttlMs: 60_000,
shard: 7,
});
expect(withIndex).toHaveBeenCalledWith("by_key_window_shard", expect.any(Function));
expect(insert).toHaveBeenCalledWith(
"rateLimitCounters",
expect.objectContaining({
key: "ip:test",
shard: 7,
count: 1,
expiresAt: expect.any(Number),
}),
);
expect(result).toEqual({ ok: true });
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
name: "downloadIp",
key: "ip:203.0.113.1:download",
config: { kind: "fixed window", rate: 1200, period: 60_000, start: 0, shards: 16 },
});
expect(withIndex).toHaveBeenCalledWith("by_name_and_key_and_shard", expect.any(Function));
expect(eqName).toHaveBeenCalledWith("name", "downloadIp");
expect(eqKey).toHaveBeenCalledWith("key", "ip:203.0.113.1:download");
expect(eqShard).toHaveBeenCalledWith("shard", 7);
expect(insert).toHaveBeenCalledWith("httpRateLimitKeys", {
name: "downloadIp",
key: "ip:203.0.113.1:download",
shard: 7,
lastTouchedAt: 10_000,
expiresAt: 70_000,
});
});
it("clamps out-of-range shard inputs to the active counter shard range", async () => {
const insert = vi.fn();
it("inserts sharded metadata for a newly observed component key", async () => {
const take = vi.fn(async () => []);
const eqShard = vi.fn();
const eqKey = vi.fn(() => ({ eq: eqShard }));
const eqName = vi.fn(() => ({ eq: eqKey }));
const withIndex = vi.fn((_index, builder) => {
builder({
eq: vi.fn(() => ({
eq: vi.fn(() => ({
eq: vi.fn(),
})),
})),
});
return { first: vi.fn(async () => null) };
});
const ctx = {
db: {
query: vi.fn(() => ({ withIndex })),
get: vi.fn(),
normalizeId: vi.fn(),
insert,
patch: vi.fn(),
replace: vi.fn(),
delete: vi.fn(),
system: {
get: vi.fn(),
query: vi.fn(),
},
},
};
await consumeHandler(ctx, {
key: "ip:test",
limit: 20,
windowMs: 60_000,
shard: 999,
});
expect(insert).toHaveBeenCalledWith(
"rateLimitCounters",
expect.objectContaining({
shard: 15,
}),
);
});
it("prunes only expired active counter rows in bounded batches", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
const stale = [
{ _id: "rateLimitCounters:a", expiresAt: 930_000 },
{ _id: "rateLimitCounters:b", expiresAt: 940_000 },
];
const take = vi.fn(async () => stale);
const withIndex = vi.fn((_index, builder) => {
builder({ lt: vi.fn() });
builder({ eq: eqName });
return { take };
});
const deleteRow = vi.fn();
const insert = vi.fn();
const ctx = {
db: {
db: makeDb({
query: vi.fn(() => ({ withIndex })),
get: vi.fn(),
normalizeId: vi.fn(),
insert: vi.fn(),
patch: vi.fn(),
replace: vi.fn(),
delete: deleteRow,
system: {
get: vi.fn(),
query: vi.fn(),
},
delete: vi.fn(),
}),
scheduler: {
runAfter: vi.fn(),
},
};
ctx.db.insert = insert;
const result = await touchHttpKeyHandler(ctx, {
name: "downloadIp",
key: "ip:203.0.113.1:download",
shard: 3,
now: 10_000,
ttlMs: 60_000,
});
expect(result).toEqual({ action: "inserted", expiresAt: 70_000, shard: 3 });
expect(ctx.db.query).toHaveBeenCalledWith("httpRateLimitKeys");
expect(withIndex).toHaveBeenCalledWith("by_name_and_key_and_shard", expect.any(Function));
expect(eqName).toHaveBeenCalledWith("name", "downloadIp");
expect(eqKey).toHaveBeenCalledWith("key", "ip:203.0.113.1:download");
expect(eqShard).toHaveBeenCalledWith("shard", 3);
expect(insert).toHaveBeenCalledWith("httpRateLimitKeys", {
name: "downloadIp",
key: "ip:203.0.113.1:download",
shard: 3,
lastTouchedAt: 10_000,
expiresAt: 70_000,
});
});
it("refreshes metadata for an existing component key shard", async () => {
const existing = {
_id: "httpRateLimitKeys:1",
name: "readKey",
key: "user:users_123:read",
shard: 11,
lastTouchedAt: 1_000,
expiresAt: 61_000,
};
const ctx = {
db: makeDb({
query: vi.fn(() => ({
withIndex: vi.fn(() => ({ take: vi.fn(async () => [existing]) })),
})),
delete: vi.fn(),
}),
scheduler: {
runAfter: vi.fn(),
},
};
const result = await pruneHandler(ctx, { batchSize: 10 });
const result = await touchHttpKeyHandler(ctx, {
name: "readKey",
key: "user:users_123:read",
shard: 11,
now: 20_000,
ttlMs: 60_000,
});
expect(result).toEqual({ action: "updated", expiresAt: 80_000, shard: 11 });
expect(ctx.db.patch).toHaveBeenCalledWith("httpRateLimitKeys:1", {
lastTouchedAt: 20_000,
expiresAt: 80_000,
});
expect(ctx.db.insert).not.toHaveBeenCalled();
});
it("repairs duplicate metadata rows while refreshing a component key shard", async () => {
const existing = {
_id: "httpRateLimitKeys:1",
name: "readKey",
key: "user:users_123:read",
shard: 11,
lastTouchedAt: 1_000,
expiresAt: 61_000,
};
const duplicate = {
...existing,
_id: "httpRateLimitKeys:2",
};
const ctx = {
db: makeDb({
query: vi.fn(() => ({
withIndex: vi.fn(() => ({ take: vi.fn(async () => [existing, duplicate]) })),
})),
delete: vi.fn(),
}),
scheduler: {
runAfter: vi.fn(),
},
};
const result = await touchHttpKeyHandler(ctx, {
name: "readKey",
key: "user:users_123:read",
shard: 11,
now: 20_000,
ttlMs: 60_000,
});
expect(result).toEqual({ action: "updated", expiresAt: 80_000, shard: 11 });
expect(ctx.db.patch).toHaveBeenCalledWith("httpRateLimitKeys:1", {
lastTouchedAt: 20_000,
expiresAt: 80_000,
});
expect(ctx.db.delete).toHaveBeenCalledWith("httpRateLimitKeys:2");
});
it("keeps component buckets when another shard is still active", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
const stale = {
_id: "httpRateLimitKeys:a",
name: "downloadIp",
key: "ip:203.0.113.1:download",
shard: 2,
expiresAt: 900_000,
};
const active = {
_id: "httpRateLimitKeys:b",
name: "downloadIp",
key: "ip:203.0.113.1:download",
shard: 9,
expiresAt: 1_050_000,
};
const staleTake = vi.fn(async () => [stale]);
const activeTake = vi.fn(async () => [active]);
const withIndex = vi
.fn()
.mockImplementationOnce((_index, builder) => {
builder({ lt: vi.fn() });
return { take: staleTake };
})
.mockImplementationOnce((_index, builder) => {
builder({ eq: vi.fn(() => ({ eq: vi.fn(() => ({ gte: vi.fn() })) })) });
return { take: activeTake };
});
const runMutation = vi.fn();
const deleteRow = vi.fn();
const ctx = {
runMutation,
db: makeDb({
query: vi.fn(() => ({ withIndex })),
delete: deleteRow,
}),
scheduler: {
runAfter: vi.fn(),
},
};
const result = await pruneHttpKeyHandler(ctx, { batchSize: 10 });
expect(result).toEqual({ deleted: 1, hasMore: false });
expect(withIndex).toHaveBeenNthCalledWith(1, "by_expires_at", expect.any(Function));
expect(withIndex).toHaveBeenNthCalledWith(
2,
"by_name_and_key_and_expires_at",
expect.any(Function),
);
expect(activeTake).toHaveBeenCalledWith(1);
expect(runMutation).not.toHaveBeenCalled();
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:a");
});
it("resets component buckets once before deleting fully expired key metadata", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
const stale = [
{
_id: "httpRateLimitKeys:a",
name: "downloadIp",
key: "ip:203.0.113.1:download",
shard: 2,
expiresAt: 900_000,
},
{
_id: "httpRateLimitKeys:b",
name: "downloadIp",
key: "ip:203.0.113.1:download",
shard: 9,
expiresAt: 910_000,
},
];
const staleTake = vi.fn(async () => stale);
const activeTake = vi.fn(async () => []);
const expiredTake = vi.fn(async () => stale);
const withIndex = vi
.fn()
.mockImplementationOnce((_index, builder) => {
builder({ lt: vi.fn() });
return { take: staleTake };
})
.mockImplementationOnce((_index, builder) => {
builder({ eq: vi.fn(() => ({ eq: vi.fn(() => ({ gte: vi.fn() })) })) });
return { take: activeTake };
})
.mockImplementationOnce((_index, builder) => {
builder({ eq: vi.fn(() => ({ eq: vi.fn(() => ({ lt: vi.fn() })) })) });
return { take: expiredTake };
});
const runMutation = vi.fn();
const deleteRow = vi.fn();
const ctx = {
runMutation,
db: makeDb({
query: vi.fn(() => ({ withIndex })),
delete: deleteRow,
}),
scheduler: {
runAfter: vi.fn(),
},
};
const result = await pruneHttpKeyHandler(ctx, { batchSize: 10 });
expect(ctx.db.query).toHaveBeenCalledWith("rateLimitCounters");
expect(withIndex).toHaveBeenCalledWith("by_expires_at", expect.any(Function));
expect(take).toHaveBeenCalledWith(10);
expect(deleteRow).toHaveBeenCalledTimes(2);
expect(deleteRow).toHaveBeenCalledWith("rateLimitCounters:a");
expect(deleteRow).toHaveBeenCalledWith("rateLimitCounters:b");
expect(ctx.scheduler.runAfter).not.toHaveBeenCalled();
expect(result).toEqual({ deleted: 2, hasMore: false });
expect(staleTake).toHaveBeenCalledWith(10);
expect(activeTake).toHaveBeenCalledWith(1);
expect(expiredTake).toHaveBeenCalledWith(128);
expect(runMutation).toHaveBeenCalledTimes(1);
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
name: "downloadIp",
key: "ip:203.0.113.1:download",
});
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:a");
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:b");
expect(ctx.scheduler.runAfter).not.toHaveBeenCalled();
});
it("resets independent expired component keys before deleting metadata rows", async () => {
vi.spyOn(Date, "now").mockReturnValue(1_000_000);
const first = {
_id: "httpRateLimitKeys:a",
name: "downloadIp",
key: "ip:203.0.113.1:download",
shard: 2,
expiresAt: 900_000,
};
const second = {
_id: "httpRateLimitKeys:b",
name: "writeKey",
key: "user:users_123:write",
shard: 4,
expiresAt: 910_000,
};
const staleTake = vi.fn(async () => [first, second]);
const withIndex = vi.fn((_index, builder) => {
builder({
eq: vi.fn(() => ({ eq: vi.fn(() => ({ gte: vi.fn(), lt: vi.fn() })) })),
lt: vi.fn(),
});
return { take };
});
const take = vi
.fn()
.mockResolvedValueOnce([first, second])
.mockResolvedValueOnce([])
.mockResolvedValueOnce([first])
.mockResolvedValueOnce([])
.mockResolvedValueOnce([second]);
const runMutation = vi.fn();
const deleteRow = vi.fn();
const ctx = {
runMutation,
db: makeDb({
query: vi.fn(() => ({ withIndex })),
delete: deleteRow,
}),
scheduler: {
runAfter: vi.fn(),
},
};
const result = await pruneHttpKeyHandler(ctx, { batchSize: 10 });
expect(result).toEqual({ deleted: 2, hasMore: false });
expect(ctx.db.query).toHaveBeenCalledWith("httpRateLimitKeys");
expect(withIndex).toHaveBeenCalledWith("by_expires_at", expect.any(Function));
expect(staleTake).not.toHaveBeenCalled();
expect(take).toHaveBeenCalledWith(10);
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
name: "downloadIp",
key: "ip:203.0.113.1:download",
});
expect(runMutation).toHaveBeenCalledWith(expect.anything(), {
name: "writeKey",
key: "user:users_123:write",
});
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:a");
expect(deleteRow).toHaveBeenCalledWith("httpRateLimitKeys:b");
expect(ctx.scheduler.runAfter).not.toHaveBeenCalled();
});
it("continues pruning expired component keys when a full bounded batch is deleted", async () => {
const stale = Array.from({ length: 3 }, (_, index) => ({
_id: `httpRateLimitKeys:${index}`,
name: "downloadIp",
key: `ip:203.0.113.${index}:download`,
expiresAt: 900_000 + index,
}));
const ctx = {
runMutation: vi.fn(),
db: makeDb({
query: vi.fn(() => ({
withIndex: vi.fn(() => ({ take: vi.fn(async () => stale) })),
})),
delete: vi.fn(),
}),
scheduler: {
runAfter: vi.fn(),
},
};
const result = await pruneHttpKeyHandler(ctx, { batchSize: 3 });
expect(result).toEqual({ deleted: 3, hasMore: true });
expect(ctx.scheduler.runAfter).toHaveBeenCalledWith(
0,
expect.anything(),
expect.objectContaining({ batchSize: 3 }),
);
});
});
+186 -94
View File
@@ -1,127 +1,219 @@
import { v } from "convex/values";
import { internal } from "./_generated/api";
import { internalMutation, internalQuery } from "./functions";
import { RATE_LIMIT_COUNTER_SHARDS } from "./lib/rateLimitConfig";
import { components, internal } from "./_generated/api";
import type { MutationCtx } from "./_generated/server";
import { internalMutation } from "./functions";
import { RETENTION_STANDARD_BATCH_SIZE } from "./lib/retentionPolicy";
const RATE_LIMIT_COUNTER_RETENTION_BUFFER_MS = 5 * 60_000;
const DEFAULT_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE = RETENTION_STANDARD_BATCH_SIZE;
const MAX_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE = 1_000;
/**
* Read-only rate limit check. Returns current status without writing anything.
* This eliminates write conflicts for denied requests entirely.
*/
export const getRateLimitStatusInternal = internalQuery({
args: {
key: v.string(),
limit: v.number(),
windowMs: v.number(),
},
handler: async (ctx, args) => {
const now = Date.now();
const windowStart = Math.floor(now / args.windowMs) * args.windowMs;
const resetAt = windowStart + args.windowMs;
if (args.limit <= 0) {
return { allowed: false, remaining: 0, limit: args.limit, resetAt };
}
const shardRows = await ctx.db
.query("rateLimitCounters")
.withIndex("by_key_window", (q) => q.eq("key", args.key).eq("windowStart", windowStart))
.collect();
const count = shardRows.reduce((sum, row) => sum + row.count, 0);
const allowed = count < args.limit;
return {
allowed,
remaining: Math.max(0, args.limit - count),
limit: args.limit,
resetAt,
};
},
const DEFAULT_HTTP_RATE_LIMIT_KEY_TTL_MS = 24 * 60 * 60 * 1000;
const DEFAULT_PRUNE_HTTP_RATE_LIMIT_KEYS_BATCH_SIZE = RETENTION_STANDARD_BATCH_SIZE;
const MAX_PRUNE_HTTP_RATE_LIMIT_KEYS_BATCH_SIZE = 1_000;
const HTTP_RATE_LIMIT_KEY_METADATA_SHARDS = 64;
const MAX_EXPIRED_HTTP_RATE_LIMIT_KEY_ROWS_PER_KEY = HTTP_RATE_LIMIT_KEY_METADATA_SHARDS * 2;
const fixedWindowRateLimitConfigValidator = v.object({
kind: v.literal("fixed window"),
rate: v.number(),
period: v.number(),
capacity: v.optional(v.number()),
maxReserved: v.optional(v.number()),
shards: v.optional(v.number()),
start: v.optional(v.number()),
});
const rateLimitStatusValidator = v.union(
v.object({
ok: v.literal(true),
retryAfter: v.optional(v.number()),
}),
v.object({
ok: v.literal(false),
retryAfter: v.number(),
}),
);
/**
* Consume one rate limit token. Only call this after getRateLimitStatusInternal
* returns allowed=true. Includes a double-check to handle races between the
* query and this mutation.
*/
export const consumeRateLimitInternal = internalMutation({
type HttpRateLimitKeyWriteCtx = Pick<MutationCtx, "db">;
function clampBatchSize(
requested: number | undefined,
defaultBatchSize: number,
maxBatchSize: number,
) {
const requestedBatchSize = Number.isFinite(requested)
? Math.floor(requested ?? defaultBatchSize)
: defaultBatchSize;
return Math.max(1, Math.min(requestedBatchSize, maxBatchSize));
}
function normalizeHttpRateLimitKeyShard(shard: number | undefined) {
const candidate = Number.isFinite(shard)
? Math.floor(shard ?? 0)
: Math.floor(Math.random() * HTTP_RATE_LIMIT_KEY_METADATA_SHARDS);
return (
((candidate % HTTP_RATE_LIMIT_KEY_METADATA_SHARDS) + HTTP_RATE_LIMIT_KEY_METADATA_SHARDS) %
HTTP_RATE_LIMIT_KEY_METADATA_SHARDS
);
}
async function touchHttpRateLimitKey(
ctx: HttpRateLimitKeyWriteCtx,
args: {
name: string;
key: string;
shard?: number;
now?: number;
ttlMs?: number;
},
) {
const now = Number.isFinite(args.now) ? (args.now ?? Date.now()) : Date.now();
const ttlMs =
Number.isFinite(args.ttlMs) && (args.ttlMs ?? 0) > 0
? Math.floor(args.ttlMs ?? DEFAULT_HTTP_RATE_LIMIT_KEY_TTL_MS)
: DEFAULT_HTTP_RATE_LIMIT_KEY_TTL_MS;
const expiresAt = now + ttlMs;
const shard = normalizeHttpRateLimitKeyShard(args.shard);
const matches = await ctx.db
.query("httpRateLimitKeys")
.withIndex("by_name_and_key_and_shard", (q) =>
q.eq("name", args.name).eq("key", args.key).eq("shard", shard),
)
.take(10);
const [existing, ...duplicates] = matches;
if (existing) {
await ctx.db.patch(existing._id, { lastTouchedAt: now, expiresAt });
for (const duplicate of duplicates) {
await ctx.db.delete(duplicate._id);
}
return { action: "updated" as const, expiresAt, shard };
}
await ctx.db.insert("httpRateLimitKeys", {
name: args.name,
key: args.key,
shard,
lastTouchedAt: now,
expiresAt,
});
return { action: "inserted" as const, expiresAt, shard };
}
export const consumeHttpRateLimitKeyInternal = internalMutation({
args: {
name: v.string(),
key: v.string(),
limit: v.number(),
windowMs: v.number(),
config: fixedWindowRateLimitConfigValidator,
now: v.optional(v.number()),
ttlMs: v.optional(v.number()),
shard: v.optional(v.number()),
},
returns: rateLimitStatusValidator,
handler: async (ctx, args) => {
const now = Date.now();
const windowStart = Math.floor(now / args.windowMs) * args.windowMs;
const resetAt = windowStart + args.windowMs;
const requestedShard = Number.isFinite(args.shard) ? Math.floor(args.shard ?? 0) : 0;
const shard = Math.max(0, Math.min(RATE_LIMIT_COUNTER_SHARDS - 1, requestedShard));
const existing = await ctx.db
.query("rateLimitCounters")
.withIndex("by_key_window_shard", (q) =>
q.eq("key", args.key).eq("windowStart", windowStart).eq("shard", shard),
)
.first();
if (!existing) {
await ctx.db.insert("rateLimitCounters", {
key: args.key,
windowStart,
shard,
count: 1,
limit: args.limit,
updatedAt: now,
expiresAt: resetAt + RATE_LIMIT_COUNTER_RETENTION_BUFFER_MS,
});
return { allowed: true, remaining: Math.max(0, args.limit - 1) };
}
await ctx.db.patch(existing._id, {
count: existing.count + 1,
limit: args.limit,
updatedAt: now,
expiresAt: resetAt + RATE_LIMIT_COUNTER_RETENTION_BUFFER_MS,
const status = await ctx.runMutation(components.rateLimiter.lib.rateLimit, {
name: args.name,
key: args.key,
config: args.config,
});
return {
allowed: true,
remaining: Math.max(0, args.limit - 1),
};
await touchHttpRateLimitKey(ctx, args);
return status;
},
});
export const pruneRateLimitCountersInternal = internalMutation({
export const touchHttpRateLimitKeyInternal = internalMutation({
args: {
name: v.string(),
key: v.string(),
shard: v.optional(v.number()),
now: v.optional(v.number()),
ttlMs: v.optional(v.number()),
},
handler: async (ctx, args) => {
return await touchHttpRateLimitKey(ctx, args);
},
});
async function hasActiveHttpRateLimitKeyMetadata(
ctx: HttpRateLimitKeyWriteCtx,
row: { name: string; key: string },
now: number,
) {
const activeRows = await ctx.db
.query("httpRateLimitKeys")
.withIndex("by_name_and_key_and_expires_at", (q) =>
q.eq("name", row.name).eq("key", row.key).gte("expiresAt", now),
)
.take(1);
return activeRows.length > 0;
}
async function deleteExpiredHttpRateLimitKeyMetadata(
ctx: HttpRateLimitKeyWriteCtx,
row: { name: string; key: string },
now: number,
deletedRowIds: Set<string>,
) {
const expiredRows = await ctx.db
.query("httpRateLimitKeys")
.withIndex("by_name_and_key_and_expires_at", (q) =>
q.eq("name", row.name).eq("key", row.key).lt("expiresAt", now),
)
.take(MAX_EXPIRED_HTTP_RATE_LIMIT_KEY_ROWS_PER_KEY);
let deleted = 0;
for (const expiredRow of expiredRows) {
if (deletedRowIds.has(expiredRow._id)) continue;
await ctx.db.delete(expiredRow._id);
deletedRowIds.add(expiredRow._id);
deleted += 1;
}
return deleted;
}
export const pruneHttpRateLimitKeysInternal = internalMutation({
args: {
batchSize: v.optional(v.number()),
},
handler: async (ctx, args) => {
const requestedBatchSize = Number.isFinite(args.batchSize)
? Math.floor(args.batchSize ?? DEFAULT_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE)
: DEFAULT_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE;
const batchSize = Math.max(
1,
Math.min(requestedBatchSize, MAX_PRUNE_RATE_LIMIT_COUNTERS_BATCH_SIZE),
const now = Date.now();
const batchSize = clampBatchSize(
args.batchSize,
DEFAULT_PRUNE_HTTP_RATE_LIMIT_KEYS_BATCH_SIZE,
MAX_PRUNE_HTTP_RATE_LIMIT_KEYS_BATCH_SIZE,
);
const stale = await ctx.db
.query("rateLimitCounters")
.withIndex("by_expires_at", (q) => q.lt("expiresAt", Date.now()))
.query("httpRateLimitKeys")
.withIndex("by_expires_at", (q) => q.lt("expiresAt", now))
.take(batchSize);
const resetKeys = new Set<string>();
const deletedRowIds = new Set<string>();
let deleted = 0;
for (const row of stale) {
await ctx.db.delete(row._id);
if (deletedRowIds.has(row._id)) continue;
const key = `${row.name}\0${row.key}`;
if (await hasActiveHttpRateLimitKeyMetadata(ctx, row, now)) {
await ctx.db.delete(row._id);
deletedRowIds.add(row._id);
deleted += 1;
continue;
}
if (!resetKeys.has(key)) {
await ctx.runMutation(components.rateLimiter.lib.resetRateLimit, {
name: row.name,
key: row.key,
});
resetKeys.add(key);
}
deleted += await deleteExpiredHttpRateLimitKeyMetadata(ctx, row, now, deletedRowIds);
}
const hasMore = stale.length === batchSize;
if (hasMore) {
await ctx.scheduler.runAfter(0, internal.rateLimits.pruneRateLimitCountersInternal, {
await ctx.scheduler.runAfter(0, internal.rateLimits.pruneHttpRateLimitKeysInternal, {
batchSize,
});
}
return { deleted: stale.length, hasMore };
return { deleted, hasMore };
},
});
+7 -9
View File
@@ -2727,17 +2727,15 @@ const cliDeviceCodes = defineTable({
.index("by_user_code_hash", ["userCodeHash"])
.index("by_status_expires", ["status", "expiresAt"]);
const rateLimitCounters = defineTable({
const httpRateLimitKeys = defineTable({
name: v.string(),
key: v.string(),
windowStart: v.number(),
shard: v.number(),
count: v.number(),
limit: v.number(),
updatedAt: v.number(),
shard: v.optional(v.number()),
lastTouchedAt: v.number(),
expiresAt: v.number(),
})
.index("by_key_window", ["key", "windowStart"])
.index("by_key_window_shard", ["key", "windowStart", "shard"])
.index("by_name_and_key_and_shard", ["name", "key", "shard"])
.index("by_name_and_key_and_expires_at", ["name", "key", "expiresAt"])
.index("by_expires_at", ["expiresAt"]);
const downloadMetricTargetKind = v.union(v.literal("skill"), v.literal("package"));
@@ -2955,7 +2953,7 @@ export default defineSchema({
vtScanLogs,
apiTokens,
cliDeviceCodes,
rateLimitCounters,
httpRateLimitKeys,
downloadMetricDedupes,
packageInstallMetricDedupes,
installTelemetryDedupes,
+5 -1
View File
@@ -41,12 +41,16 @@ Auth-aware enforcement:
- Write: 300/min per IP, 3000/min per key
- Download: 1200/min per IP, 6000/min per key
Headers: `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset`, `RateLimit-Limit`, `RateLimit-Remaining`, `RateLimit-Reset`, `Retry-After` (on 429).
Headers: `X-RateLimit-Limit`, `X-RateLimit-Reset`, `RateLimit-Limit`, `RateLimit-Reset`;
`X-RateLimit-Remaining`, `RateLimit-Remaining`, and `Retry-After` are included on `429`.
Semantics:
- `X-RateLimit-Reset`: Unix epoch seconds (absolute reset time)
- `RateLimit-Reset`: delay seconds until reset
- `X-RateLimit-Remaining` / `RateLimit-Remaining`: exact remaining budget when
present; sharded successful requests omit it rather than returning an approximate
global value
- `Retry-After`: delay seconds to wait on `429`
Example `429`:
+11 -4
View File
@@ -39,14 +39,17 @@ Enforcement model:
Headers:
- Legacy compatibility: `X-RateLimit-Limit`, `X-RateLimit-Remaining`, `X-RateLimit-Reset`
- Standardized: `RateLimit-Limit`, `RateLimit-Remaining`, `RateLimit-Reset`
- Legacy compatibility: `X-RateLimit-Limit`, `X-RateLimit-Reset`
- Standardized: `RateLimit-Limit`, `RateLimit-Reset`
- On `429`: `X-RateLimit-Remaining: 0` and `RateLimit-Remaining: 0`
- On `429`: `Retry-After`
Header semantics:
- `X-RateLimit-Reset`: absolute Unix epoch seconds
- `RateLimit-Reset`: seconds until reset (delay)
- `X-RateLimit-Remaining` / `RateLimit-Remaining`: exact remaining budget when present.
Sharded successful requests omit this header instead of returning an approximate global value.
- `Retry-After`: seconds to wait before retry (delay) on `429`
Example `429` response:
@@ -73,9 +76,13 @@ Client guidance:
IP source:
- Uses `cf-connecting-ip` (Cloudflare) for client IP by default.
- Uses trusted client IP headers, including `cf-connecting-ip`, only when the
deployment explicitly enables trusted forwarded headers.
- ClawHub uses trusted forwarding headers to identify client IPs at the edge.
- If no trusted client IP is available, anonymous download requests use an endpoint-scoped fallback bucket instead of one global `ip:unknown` bucket. Anonymous read/write requests still use the shared unknown bucket so missing-IP routing remains visible and conservative.
- If no trusted client IP is available, anonymous requests use fallback buckets
scoped only by rate-limit kind. These fallback buckets do not include
caller-supplied paths, slugs, package names, versions, query strings, or other
artifact parameters.
## Error responses
+2 -1
View File
@@ -31,7 +31,8 @@ clawhub login --token clh_...
Read the retry information in the response:
- `Retry-After`: seconds to wait before retrying.
- `RateLimit-Remaining` and `RateLimit-Limit`: your current budget.
- `RateLimit-Limit`: the limit applied to this request.
- `RateLimit-Remaining`: your exact remaining budget when the header is present. On `429`, it is `0`.
- `RateLimit-Reset` or `X-RateLimit-Reset`: reset timing.
If many users share one egress IP, anonymous IP limits can be hit even when each
+1
View File
@@ -74,6 +74,7 @@
"@auth/core": "0.41.2",
"@convex-dev/auth": "0.0.94",
"@convex-dev/migrations": "0.3.5",
"@convex-dev/rate-limiter": "0.3.2",
"@fontsource/bricolage-grotesque": "5.2.10",
"@fontsource/ibm-plex-mono": "5.2.7",
"@fontsource/manrope": "5.2.8",
+5
View File
@@ -131,6 +131,11 @@ Ensure Convex env is set (auth + embeddings):
- Optional fallback: `GITHUB_TOKEN` (used when GitHub App auth is unavailable,
and for arbitrary public repository lookups such as trusted-publisher setup)
Do not set `TRUST_FORWARDED_IPS=true` while the Convex `*.convex.site` HTTP
origin remains publicly reachable. That flag makes rate limits and download
metrics trust forwarded client IP headers, so it is only safe behind a
header-sanitizing edge that prevents direct origin requests.
## 2) Deploy web app (Vercel)
Set env vars:
+50
View File
@@ -0,0 +1,50 @@
# Rate Limiting
## Security Intent
All public HTTP routes, including legacy `/api/...` and `/api/cli/...` routes
kept for compatibility, must pass through `applyRateLimit` before doing
expensive work, auth parsing, publish mutations, upload ticket creation,
telemetry writes, delete/undelete mutations, or search queries.
Client IP headers are not trustworthy on direct Convex HTTP endpoints. Treat
`cf-connecting-ip`, `x-forwarded-for`, `x-real-ip`, and `fly-client-ip` as
trusted only when the deployment explicitly enables trusted forwarded headers.
Do not enable that opt-in while the Convex `*.convex.site` HTTP origin remains
publicly reachable. Without the opt-in, anonymous traffic must use conservative
missing-IP fallback buckets scoped only by rate-limit kind. Missing-IP buckets
must not include user-controlled paths, dynamic path segments, query parameters,
package names, skill slugs, or artifact versions.
Artifact-specific download scoping is only safe after the caller has an
authenticated identity or a trusted client IP.
HTTP rate-limit counters are owned by the `@convex-dev/rate-limiter` component.
The app defines named fixed-window buckets for each public HTTP policy
(`read`, `write`, `trustedPublish`, `download`, and `export`) and for each
subject class (`ip`, authenticated API token user, and admin API token user).
Anonymous requests consume the `Ip` bucket for the route policy. Authenticated
requests consume only the `Key` or `AdminKey` bucket for their user, so shared
egress IPs do not drain user quota.
Component rows are operational data, not product history. Do not use the
component `clearAll(before)` helper as a TTL job for active HTTP buckets:
`clearAll` deletes by row creation time while normal rate-limit use patches the
row's current timestamp and value in place. Clearing old creation times can
reset still-active buckets and weaken the limit.
The app-owned `httpRateLimitKeys` table exists only to make component cleanup
safe: limiter checks refresh sharded metadata for the observed `(name, key)`
with an `expiresAt`, and retention resets the exact component key only after no
metadata shard for that `(name, key)` remains active. Active allowed and denied
traffic must both refresh this metadata so a client that is still hitting a
limit is not reset by cleanup. Metadata sharding is required because shared
buckets such as missing-IP anonymous reads can receive many concurrent requests;
one metadata row per key would become a separate Convex write-contention point.
Component sharding is an implementation detail, not a quota multiplier. Keep
the public quota in `RATE_LIMITS`; the component's `shards` setting exists to
spread writes across counter rows while preserving the configured limit. Low
rate buckets should stay unsharded or keep enough per-shard capacity that
normal clients are not randomly denied well before the public limit.
Because the component does not expose an exact total remaining count for
sharded buckets, successful HTTP responses must not synthesize a global
`RateLimit-Remaining` value. Denied responses can still return an exact
remaining value of `0`.