fix: harden GitHub account age lookup

This commit is contained in:
Peter Steinberger
2026-06-02 11:32:16 +01:00
parent dcbc38999f
commit 0abdbf4a50
6 changed files with 393 additions and 21 deletions
+1
View File
@@ -4,6 +4,7 @@
### Fixes
- Auth/Ops: keep GitHub account-age lookups on immutable numeric IDs, retry without auth when a configured GitHub token is rejected, and add an operator backfill for missing cached account ages.
- API/CLI: report Skill Card verification with flattened skill/version metadata, ClawScan verdict fields at `security.*`, and supporting scanner evidence under `security.signals`.
## 0.18.0 - 2026-05-25
+4
View File
@@ -19,6 +19,7 @@ import type * as devSeed from "../devSeed.js";
import type * as devSeedExtra from "../devSeedExtra.js";
import type * as downloads from "../downloads.js";
import type * as functions from "../functions.js";
import type * as githubAccountAgeBackfill from "../githubAccountAgeBackfill.js";
import type * as githubBackups from "../githubBackups.js";
import type * as githubBackupsNode from "../githubBackupsNode.js";
import type * as githubIdentity from "../githubIdentity.js";
@@ -93,6 +94,7 @@ import type * as lib_securityPrompt from "../lib/securityPrompt.js";
import type * as lib_skillBackfill from "../lib/skillBackfill.js";
import type * as lib_skillCapabilityTags from "../lib/skillCapabilityTags.js";
import type * as lib_skillCards from "../lib/skillCards.js";
import type * as lib_skillFileAccess from "../lib/skillFileAccess.js";
import type * as lib_skillIcon from "../lib/skillIcon.js";
import type * as lib_skillPublish from "../lib/skillPublish.js";
import type * as lib_skillQuality from "../lib/skillQuality.js";
@@ -158,6 +160,7 @@ declare const fullApi: ApiFromModules<{
devSeedExtra: typeof devSeedExtra;
downloads: typeof downloads;
functions: typeof functions;
githubAccountAgeBackfill: typeof githubAccountAgeBackfill;
githubBackups: typeof githubBackups;
githubBackupsNode: typeof githubBackupsNode;
githubIdentity: typeof githubIdentity;
@@ -232,6 +235,7 @@ declare const fullApi: ApiFromModules<{
"lib/skillBackfill": typeof lib_skillBackfill;
"lib/skillCapabilityTags": typeof lib_skillCapabilityTags;
"lib/skillCards": typeof lib_skillCards;
"lib/skillFileAccess": typeof lib_skillFileAccess;
"lib/skillIcon": typeof lib_skillIcon;
"lib/skillPublish": typeof lib_skillPublish;
"lib/skillQuality": typeof lib_skillQuality;
+240
View File
@@ -0,0 +1,240 @@
import { ConvexError, v } from "convex/values";
import { internal } from "./_generated/api";
import type { Id } from "./_generated/dataModel";
import type { ActionCtx } from "./_generated/server";
import { internalAction, internalMutation, internalQuery } from "./functions";
import { fetchGitHubCreatedAtByProviderAccountId } from "./lib/githubAccount";
import { getGitHubProviderAccountId } from "./lib/githubIdentity";
import { getUserByHandleOrPersonalPublisher } from "./lib/publishers";
const DEFAULT_BATCH_SIZE = 25;
const MAX_BATCH_SIZE = 50;
const DEFAULT_MAX_PAGES = 1;
const MAX_MAX_PAGES = 20;
type BackfillCandidate = {
userId: Id<"users">;
providerAccountId: string;
handle: string | null;
};
type BackfillStats = {
scanned: number;
candidates: number;
fetched: number;
patched: number;
failed: number;
missingHandles: string[];
errors: Array<{ userId: string; handle: string | null; message: string }>;
};
type BackfillPageResult = {
candidates: BackfillCandidate[];
scanned: number;
cursor: string | null;
isDone: boolean;
};
type BackfillHandlesResult = {
candidates: BackfillCandidate[];
missingHandles: string[];
};
type BackfillResult =
| { ok: true; stats: BackfillStats; cursor: string | null; isDone: boolean }
| { ok: false; rateLimited: true; stats: BackfillStats; cursor: string | null; isDone: false };
function clampPositiveInteger(value: number | undefined, fallback: number, max: number) {
if (!value || !Number.isFinite(value)) return fallback;
return Math.max(1, Math.min(max, Math.floor(value)));
}
async function candidateForUser(
ctx: Parameters<typeof getGitHubProviderAccountId>[0],
userId: Id<"users">,
): Promise<BackfillCandidate | null> {
const user = await ctx.db.get(userId);
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) return null;
const providerAccountId = await getGitHubProviderAccountId(ctx, userId);
if (!providerAccountId || !/^\d+$/.test(providerAccountId)) return null;
return { userId, providerAccountId, handle: user.handle ?? null };
}
export const listGitHubCreatedAtBackfillPageInternal = internalQuery({
args: {
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
},
handler: async (ctx, args) => {
const batchSize = clampPositiveInteger(args.batchSize, DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE);
const page = await ctx.db
.query("authAccounts")
.withIndex("providerAndAccountId", (q) => q.eq("provider", "github"))
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
const candidates: BackfillCandidate[] = [];
for (const account of page.page) {
if (!/^\d+$/.test(account.providerAccountId)) continue;
const user = await ctx.db.get(account.userId);
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) continue;
candidates.push({
userId: account.userId,
providerAccountId: account.providerAccountId,
handle: user.handle ?? null,
});
}
return {
candidates,
scanned: page.page.length,
cursor: page.continueCursor,
isDone: page.isDone,
};
},
});
export const listGitHubCreatedAtBackfillHandlesInternal = internalQuery({
args: { handles: v.array(v.string()) },
handler: async (ctx, args) => {
const seen = new Set<string>();
const candidates: BackfillCandidate[] = [];
const missingHandles: string[] = [];
for (const handle of args.handles) {
const user = await getUserByHandleOrPersonalPublisher(ctx, handle);
if (!user) {
missingHandles.push(handle);
continue;
}
if (seen.has(user._id)) continue;
seen.add(user._id);
const candidate = await candidateForUser(ctx, user._id);
if (candidate) candidates.push(candidate);
}
return { candidates, missingHandles };
},
});
export const applyGitHubCreatedAtBackfillInternal = internalMutation({
args: {
userId: v.id("users"),
githubCreatedAt: v.number(),
fetchedAt: v.number(),
dryRun: v.optional(v.boolean()),
},
handler: async (ctx, args) => {
const user = await ctx.db.get(args.userId);
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) {
return { patched: false };
}
if (args.dryRun) return { patched: false };
await ctx.db.patch(args.userId, {
githubCreatedAt: args.githubCreatedAt,
githubFetchedAt: args.fetchedAt,
updatedAt: Date.now(),
});
return { patched: true };
},
});
export const backfillGitHubCreatedAtInternal = internalAction({
args: {
cursor: v.optional(v.string()),
batchSize: v.optional(v.number()),
maxPages: v.optional(v.number()),
dryRun: v.optional(v.boolean()),
handles: v.optional(v.array(v.string())),
},
handler: async (ctx: ActionCtx, args): Promise<BackfillResult> => {
const batchSize = clampPositiveInteger(args.batchSize, DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE);
const maxPages = clampPositiveInteger(args.maxPages, DEFAULT_MAX_PAGES, MAX_MAX_PAGES);
const dryRun = args.dryRun ?? false;
const fetchedAt = Date.now();
const stats = {
scanned: 0,
candidates: 0,
fetched: 0,
patched: 0,
failed: 0,
missingHandles: [] as string[],
errors: [] as Array<{ userId: string; handle: string | null; message: string }>,
};
let cursor = args.cursor ?? null;
let isDone = true;
let pages = 0;
while (pages < maxPages) {
pages += 1;
const page: BackfillPageResult | BackfillHandlesResult = args.handles
? ((await ctx.runQuery(
internal.githubAccountAgeBackfill.listGitHubCreatedAtBackfillHandlesInternal,
{
handles: args.handles,
},
)) as BackfillHandlesResult)
: ((await ctx.runQuery(
internal.githubAccountAgeBackfill.listGitHubCreatedAtBackfillPageInternal,
{
cursor: cursor ?? undefined,
batchSize,
},
)) as BackfillPageResult);
const candidates = page.candidates;
stats.scanned += "scanned" in page ? page.scanned : (args.handles?.length ?? 0);
if ("missingHandles" in page) stats.missingHandles.push(...page.missingHandles);
stats.candidates += candidates.length;
for (const candidate of candidates) {
try {
const githubCreatedAt = await fetchGitHubCreatedAtByProviderAccountId(
candidate.providerAccountId,
);
stats.fetched += 1;
const result: { patched: boolean } = await ctx.runMutation(
internal.githubAccountAgeBackfill.applyGitHubCreatedAtBackfillInternal,
{
userId: candidate.userId,
githubCreatedAt,
fetchedAt,
dryRun,
},
);
if (result.patched) stats.patched += 1;
} catch (error) {
stats.failed += 1;
const message = error instanceof ConvexError ? String(error.data) : String(error);
if (stats.errors.length < 10) {
stats.errors.push({
userId: candidate.userId,
handle: candidate.handle,
message,
});
}
if (/rate limit/i.test(message)) {
return { ok: false as const, rateLimited: true as const, stats, cursor, isDone: false };
}
}
}
if (args.handles) return { ok: true as const, stats, cursor: null, isDone: true };
cursor = "cursor" in page ? page.cursor : null;
isDone = "isDone" in page ? page.isDone : true;
if (isDone) break;
}
if (!dryRun && !isDone && cursor) {
await ctx.scheduler.runAfter(
0,
internal.githubAccountAgeBackfill.backfillGitHubCreatedAtInternal,
{
cursor,
batchSize,
maxPages,
},
);
}
return { ok: true as const, stats, cursor, isDone };
},
});
+113
View File
@@ -289,6 +289,119 @@ describe("requireGitHubAccountAge", () => {
}),
);
});
it("omits Authorization header when GITHUB_TOKEN is blank", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
vi.stubEnv("GITHUB_TOKEN", " ");
const runQuery = vi
.fn()
.mockResolvedValueOnce({
_id: "users:1",
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
created_at: "2020-01-01T00:00:00Z",
}),
});
vi.stubGlobal("fetch", fetchMock);
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
expect(fetchMock).toHaveBeenCalledWith(
"https://api.github.com/user/12345",
expect.objectContaining({
headers: { "User-Agent": "clawhub" },
}),
);
});
it("retries without Authorization when GITHUB_TOKEN is rejected", async () => {
vi.useFakeTimers();
const now = new Date("2026-02-02T12:00:00Z");
vi.setSystemTime(now);
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
vi.stubEnv("GITHUB_TOKEN", "ghp_expired");
const runQuery = vi
.fn()
.mockResolvedValueOnce({
_id: "users:1",
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
const fetchMock = vi
.fn()
.mockResolvedValueOnce({ ok: false, status: 401 })
.mockResolvedValueOnce({
ok: true,
json: async () => ({
created_at: "2020-01-01T00:00:00Z",
}),
});
vi.stubGlobal("fetch", fetchMock);
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
expect(fetchMock).toHaveBeenNthCalledWith(
1,
"https://api.github.com/user/12345",
expect.objectContaining({
headers: {
"User-Agent": "clawhub",
Authorization: "Bearer ghp_expired",
},
}),
);
expect(fetchMock).toHaveBeenNthCalledWith(
2,
"https://api.github.com/user/12345",
expect.objectContaining({
headers: { "User-Agent": "clawhub" },
}),
);
expect(runMutation).toHaveBeenCalledWith(internal.users.setGitHubCreatedAtInternal, {
userId: "users:1",
githubCreatedAt: Date.parse("2020-01-01T00:00:00Z"),
});
expect(warnSpy).toHaveBeenCalledWith(
"[githubAccount] GITHUB_TOKEN was rejected; retrying lookup without auth",
);
});
it("does not retry unauthenticated 401 responses", async () => {
const runQuery = vi
.fn()
.mockResolvedValueOnce({
_id: "users:1",
githubCreatedAt: undefined,
})
.mockResolvedValueOnce("12345");
const runMutation = vi.fn();
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 401 });
vi.stubGlobal("fetch", fetchMock);
await expect(
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
).rejects.toThrow(/GitHub account lookup failed/i);
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith(
"https://api.github.com/user/12345",
expect.objectContaining({
headers: { "User-Agent": "clawhub" },
}),
);
});
});
describe("syncGitHubProfile", () => {
+32 -21
View File
@@ -24,13 +24,42 @@ function assertGitHubNumericId(providerAccountId: string) {
function buildGitHubHeaders() {
const headers: Record<string, string> = { "User-Agent": "clawhub" };
const token = process.env.GITHUB_TOKEN;
const token = process.env.GITHUB_TOKEN?.trim();
if (token) {
headers.Authorization = `Bearer ${token}`;
}
return headers;
}
async function fetchGitHubUserByNumericId(providerAccountId: string) {
assertGitHubNumericId(providerAccountId);
const url = `${GITHUB_API}/user/${providerAccountId}`;
const response = await fetch(url, {
headers: buildGitHubHeaders(),
});
if (response.status !== 401 || !process.env.GITHUB_TOKEN?.trim()) return response;
console.warn("[githubAccount] GITHUB_TOKEN was rejected; retrying lookup without auth");
return await fetch(url, {
headers: { "User-Agent": "clawhub" },
});
}
export async function fetchGitHubCreatedAtByProviderAccountId(providerAccountId: string) {
const response = await fetchGitHubUserByNumericId(providerAccountId);
if (!response.ok) {
if (response.status === 403 || response.status === 429) {
throw new ConvexError("GitHub API rate limit exceeded — please try again in a few minutes");
}
throw new ConvexError("GitHub account lookup failed");
}
const payload = (await response.json()) as GitHubUser;
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN;
if (!Number.isFinite(parsed)) throw new ConvexError("GitHub account lookup failed");
return parsed;
}
export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId: Id<"users">) {
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId });
if (!user || user.deletedAt || user.deactivatedAt) throw new ConvexError("User not found");
@@ -48,24 +77,8 @@ export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId:
// Invariant: GitHub is our only auth provider, so this should never happen.
throw new ConvexError("GitHub account required");
}
assertGitHubNumericId(providerAccountId);
// Fetch by immutable GitHub numeric ID to avoid username swap attacks entirely.
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
headers: buildGitHubHeaders(),
});
if (!response.ok) {
if (response.status === 403 || response.status === 429) {
throw new ConvexError("GitHub API rate limit exceeded — please try again in a few minutes");
}
throw new ConvexError("GitHub account lookup failed");
}
const payload = (await response.json()) as GitHubUser;
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN;
if (!Number.isFinite(parsed)) throw new ConvexError("GitHub account lookup failed");
createdAt = parsed;
createdAt = await fetchGitHubCreatedAtByProviderAccountId(providerAccountId);
await ctx.runMutation(internal.users.setGitHubCreatedAtInternal, {
userId,
githubCreatedAt: createdAt,
@@ -107,9 +120,7 @@ export async function syncGitHubProfile(ctx: ActionCtx, userId: Id<"users">) {
assertGitHubNumericId(providerAccountId);
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
headers: buildGitHubHeaders(),
});
const response = await fetchGitHubUserByNumericId(providerAccountId);
if (!response.ok) {
// Silently fail - this is a best-effort sync, not critical path
console.warn(`[syncGitHubProfile] GitHub API error for user ${userId}: ${response.status}`);
+3
View File
@@ -299,6 +299,9 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
- To reduce rate-limit failures, set `GITHUB_TOKEN` in Convex env for authenticated
GitHub API requests. The same token is used for trusted-publisher repository
identity lookups.
- If a configured `GITHUB_TOKEN` is rejected with `401`, retry the account-age
lookup without auth before failing. Never fall back to mutable GitHub usernames
for this gate; use the operator backfill to cache missing ages for existing users.
## Empty-skill cleanup (backfill)