mirror of
https://github.com/openclaw/clawhub.git
synced 2026-08-14 00:47:57 +00:00
fix: harden GitHub account age lookup
This commit is contained in:
@@ -4,6 +4,7 @@
|
||||
|
||||
### Fixes
|
||||
|
||||
- Auth/Ops: keep GitHub account-age lookups on immutable numeric IDs, retry without auth when a configured GitHub token is rejected, and add an operator backfill for missing cached account ages.
|
||||
- API/CLI: report Skill Card verification with flattened skill/version metadata, ClawScan verdict fields at `security.*`, and supporting scanner evidence under `security.signals`.
|
||||
|
||||
## 0.18.0 - 2026-05-25
|
||||
|
||||
Vendored
+4
@@ -19,6 +19,7 @@ import type * as devSeed from "../devSeed.js";
|
||||
import type * as devSeedExtra from "../devSeedExtra.js";
|
||||
import type * as downloads from "../downloads.js";
|
||||
import type * as functions from "../functions.js";
|
||||
import type * as githubAccountAgeBackfill from "../githubAccountAgeBackfill.js";
|
||||
import type * as githubBackups from "../githubBackups.js";
|
||||
import type * as githubBackupsNode from "../githubBackupsNode.js";
|
||||
import type * as githubIdentity from "../githubIdentity.js";
|
||||
@@ -93,6 +94,7 @@ import type * as lib_securityPrompt from "../lib/securityPrompt.js";
|
||||
import type * as lib_skillBackfill from "../lib/skillBackfill.js";
|
||||
import type * as lib_skillCapabilityTags from "../lib/skillCapabilityTags.js";
|
||||
import type * as lib_skillCards from "../lib/skillCards.js";
|
||||
import type * as lib_skillFileAccess from "../lib/skillFileAccess.js";
|
||||
import type * as lib_skillIcon from "../lib/skillIcon.js";
|
||||
import type * as lib_skillPublish from "../lib/skillPublish.js";
|
||||
import type * as lib_skillQuality from "../lib/skillQuality.js";
|
||||
@@ -158,6 +160,7 @@ declare const fullApi: ApiFromModules<{
|
||||
devSeedExtra: typeof devSeedExtra;
|
||||
downloads: typeof downloads;
|
||||
functions: typeof functions;
|
||||
githubAccountAgeBackfill: typeof githubAccountAgeBackfill;
|
||||
githubBackups: typeof githubBackups;
|
||||
githubBackupsNode: typeof githubBackupsNode;
|
||||
githubIdentity: typeof githubIdentity;
|
||||
@@ -232,6 +235,7 @@ declare const fullApi: ApiFromModules<{
|
||||
"lib/skillBackfill": typeof lib_skillBackfill;
|
||||
"lib/skillCapabilityTags": typeof lib_skillCapabilityTags;
|
||||
"lib/skillCards": typeof lib_skillCards;
|
||||
"lib/skillFileAccess": typeof lib_skillFileAccess;
|
||||
"lib/skillIcon": typeof lib_skillIcon;
|
||||
"lib/skillPublish": typeof lib_skillPublish;
|
||||
"lib/skillQuality": typeof lib_skillQuality;
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
import { ConvexError, v } from "convex/values";
|
||||
import { internal } from "./_generated/api";
|
||||
import type { Id } from "./_generated/dataModel";
|
||||
import type { ActionCtx } from "./_generated/server";
|
||||
import { internalAction, internalMutation, internalQuery } from "./functions";
|
||||
import { fetchGitHubCreatedAtByProviderAccountId } from "./lib/githubAccount";
|
||||
import { getGitHubProviderAccountId } from "./lib/githubIdentity";
|
||||
import { getUserByHandleOrPersonalPublisher } from "./lib/publishers";
|
||||
|
||||
const DEFAULT_BATCH_SIZE = 25;
|
||||
const MAX_BATCH_SIZE = 50;
|
||||
const DEFAULT_MAX_PAGES = 1;
|
||||
const MAX_MAX_PAGES = 20;
|
||||
|
||||
type BackfillCandidate = {
|
||||
userId: Id<"users">;
|
||||
providerAccountId: string;
|
||||
handle: string | null;
|
||||
};
|
||||
|
||||
type BackfillStats = {
|
||||
scanned: number;
|
||||
candidates: number;
|
||||
fetched: number;
|
||||
patched: number;
|
||||
failed: number;
|
||||
missingHandles: string[];
|
||||
errors: Array<{ userId: string; handle: string | null; message: string }>;
|
||||
};
|
||||
|
||||
type BackfillPageResult = {
|
||||
candidates: BackfillCandidate[];
|
||||
scanned: number;
|
||||
cursor: string | null;
|
||||
isDone: boolean;
|
||||
};
|
||||
|
||||
type BackfillHandlesResult = {
|
||||
candidates: BackfillCandidate[];
|
||||
missingHandles: string[];
|
||||
};
|
||||
|
||||
type BackfillResult =
|
||||
| { ok: true; stats: BackfillStats; cursor: string | null; isDone: boolean }
|
||||
| { ok: false; rateLimited: true; stats: BackfillStats; cursor: string | null; isDone: false };
|
||||
|
||||
function clampPositiveInteger(value: number | undefined, fallback: number, max: number) {
|
||||
if (!value || !Number.isFinite(value)) return fallback;
|
||||
return Math.max(1, Math.min(max, Math.floor(value)));
|
||||
}
|
||||
|
||||
async function candidateForUser(
|
||||
ctx: Parameters<typeof getGitHubProviderAccountId>[0],
|
||||
userId: Id<"users">,
|
||||
): Promise<BackfillCandidate | null> {
|
||||
const user = await ctx.db.get(userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) return null;
|
||||
const providerAccountId = await getGitHubProviderAccountId(ctx, userId);
|
||||
if (!providerAccountId || !/^\d+$/.test(providerAccountId)) return null;
|
||||
return { userId, providerAccountId, handle: user.handle ?? null };
|
||||
}
|
||||
|
||||
export const listGitHubCreatedAtBackfillPageInternal = internalQuery({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const batchSize = clampPositiveInteger(args.batchSize, DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE);
|
||||
const page = await ctx.db
|
||||
.query("authAccounts")
|
||||
.withIndex("providerAndAccountId", (q) => q.eq("provider", "github"))
|
||||
.paginate({ cursor: args.cursor ?? null, numItems: batchSize });
|
||||
|
||||
const candidates: BackfillCandidate[] = [];
|
||||
for (const account of page.page) {
|
||||
if (!/^\d+$/.test(account.providerAccountId)) continue;
|
||||
const user = await ctx.db.get(account.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) continue;
|
||||
candidates.push({
|
||||
userId: account.userId,
|
||||
providerAccountId: account.providerAccountId,
|
||||
handle: user.handle ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
candidates,
|
||||
scanned: page.page.length,
|
||||
cursor: page.continueCursor,
|
||||
isDone: page.isDone,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const listGitHubCreatedAtBackfillHandlesInternal = internalQuery({
|
||||
args: { handles: v.array(v.string()) },
|
||||
handler: async (ctx, args) => {
|
||||
const seen = new Set<string>();
|
||||
const candidates: BackfillCandidate[] = [];
|
||||
const missingHandles: string[] = [];
|
||||
for (const handle of args.handles) {
|
||||
const user = await getUserByHandleOrPersonalPublisher(ctx, handle);
|
||||
if (!user) {
|
||||
missingHandles.push(handle);
|
||||
continue;
|
||||
}
|
||||
if (seen.has(user._id)) continue;
|
||||
seen.add(user._id);
|
||||
const candidate = await candidateForUser(ctx, user._id);
|
||||
if (candidate) candidates.push(candidate);
|
||||
}
|
||||
return { candidates, missingHandles };
|
||||
},
|
||||
});
|
||||
|
||||
export const applyGitHubCreatedAtBackfillInternal = internalMutation({
|
||||
args: {
|
||||
userId: v.id("users"),
|
||||
githubCreatedAt: v.number(),
|
||||
fetchedAt: v.number(),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
},
|
||||
handler: async (ctx, args) => {
|
||||
const user = await ctx.db.get(args.userId);
|
||||
if (!user || user.deletedAt || user.deactivatedAt || user.githubCreatedAt) {
|
||||
return { patched: false };
|
||||
}
|
||||
if (args.dryRun) return { patched: false };
|
||||
await ctx.db.patch(args.userId, {
|
||||
githubCreatedAt: args.githubCreatedAt,
|
||||
githubFetchedAt: args.fetchedAt,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
return { patched: true };
|
||||
},
|
||||
});
|
||||
|
||||
export const backfillGitHubCreatedAtInternal = internalAction({
|
||||
args: {
|
||||
cursor: v.optional(v.string()),
|
||||
batchSize: v.optional(v.number()),
|
||||
maxPages: v.optional(v.number()),
|
||||
dryRun: v.optional(v.boolean()),
|
||||
handles: v.optional(v.array(v.string())),
|
||||
},
|
||||
handler: async (ctx: ActionCtx, args): Promise<BackfillResult> => {
|
||||
const batchSize = clampPositiveInteger(args.batchSize, DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE);
|
||||
const maxPages = clampPositiveInteger(args.maxPages, DEFAULT_MAX_PAGES, MAX_MAX_PAGES);
|
||||
const dryRun = args.dryRun ?? false;
|
||||
const fetchedAt = Date.now();
|
||||
const stats = {
|
||||
scanned: 0,
|
||||
candidates: 0,
|
||||
fetched: 0,
|
||||
patched: 0,
|
||||
failed: 0,
|
||||
missingHandles: [] as string[],
|
||||
errors: [] as Array<{ userId: string; handle: string | null; message: string }>,
|
||||
};
|
||||
|
||||
let cursor = args.cursor ?? null;
|
||||
let isDone = true;
|
||||
let pages = 0;
|
||||
|
||||
while (pages < maxPages) {
|
||||
pages += 1;
|
||||
const page: BackfillPageResult | BackfillHandlesResult = args.handles
|
||||
? ((await ctx.runQuery(
|
||||
internal.githubAccountAgeBackfill.listGitHubCreatedAtBackfillHandlesInternal,
|
||||
{
|
||||
handles: args.handles,
|
||||
},
|
||||
)) as BackfillHandlesResult)
|
||||
: ((await ctx.runQuery(
|
||||
internal.githubAccountAgeBackfill.listGitHubCreatedAtBackfillPageInternal,
|
||||
{
|
||||
cursor: cursor ?? undefined,
|
||||
batchSize,
|
||||
},
|
||||
)) as BackfillPageResult);
|
||||
|
||||
const candidates = page.candidates;
|
||||
stats.scanned += "scanned" in page ? page.scanned : (args.handles?.length ?? 0);
|
||||
if ("missingHandles" in page) stats.missingHandles.push(...page.missingHandles);
|
||||
stats.candidates += candidates.length;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const githubCreatedAt = await fetchGitHubCreatedAtByProviderAccountId(
|
||||
candidate.providerAccountId,
|
||||
);
|
||||
stats.fetched += 1;
|
||||
const result: { patched: boolean } = await ctx.runMutation(
|
||||
internal.githubAccountAgeBackfill.applyGitHubCreatedAtBackfillInternal,
|
||||
{
|
||||
userId: candidate.userId,
|
||||
githubCreatedAt,
|
||||
fetchedAt,
|
||||
dryRun,
|
||||
},
|
||||
);
|
||||
if (result.patched) stats.patched += 1;
|
||||
} catch (error) {
|
||||
stats.failed += 1;
|
||||
const message = error instanceof ConvexError ? String(error.data) : String(error);
|
||||
if (stats.errors.length < 10) {
|
||||
stats.errors.push({
|
||||
userId: candidate.userId,
|
||||
handle: candidate.handle,
|
||||
message,
|
||||
});
|
||||
}
|
||||
if (/rate limit/i.test(message)) {
|
||||
return { ok: false as const, rateLimited: true as const, stats, cursor, isDone: false };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (args.handles) return { ok: true as const, stats, cursor: null, isDone: true };
|
||||
cursor = "cursor" in page ? page.cursor : null;
|
||||
isDone = "isDone" in page ? page.isDone : true;
|
||||
if (isDone) break;
|
||||
}
|
||||
|
||||
if (!dryRun && !isDone && cursor) {
|
||||
await ctx.scheduler.runAfter(
|
||||
0,
|
||||
internal.githubAccountAgeBackfill.backfillGitHubCreatedAtInternal,
|
||||
{
|
||||
cursor,
|
||||
batchSize,
|
||||
maxPages,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return { ok: true as const, stats, cursor, isDone };
|
||||
},
|
||||
});
|
||||
@@ -289,6 +289,119 @@ describe("requireGitHubAccountAge", () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("omits Authorization header when GITHUB_TOKEN is blank", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = new Date("2026-02-02T12:00:00Z");
|
||||
vi.setSystemTime(now);
|
||||
|
||||
vi.stubEnv("GITHUB_TOKEN", " ");
|
||||
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
_id: "users:1",
|
||||
githubCreatedAt: undefined,
|
||||
})
|
||||
.mockResolvedValueOnce("12345");
|
||||
const runMutation = vi.fn();
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
created_at: "2020-01-01T00:00:00Z",
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: { "User-Agent": "clawhub" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("retries without Authorization when GITHUB_TOKEN is rejected", async () => {
|
||||
vi.useFakeTimers();
|
||||
const now = new Date("2026-02-02T12:00:00Z");
|
||||
vi.setSystemTime(now);
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
|
||||
vi.stubEnv("GITHUB_TOKEN", "ghp_expired");
|
||||
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
_id: "users:1",
|
||||
githubCreatedAt: undefined,
|
||||
})
|
||||
.mockResolvedValueOnce("12345");
|
||||
const runMutation = vi.fn();
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ ok: false, status: 401 })
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
created_at: "2020-01-01T00:00:00Z",
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never);
|
||||
|
||||
expect(fetchMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: {
|
||||
"User-Agent": "clawhub",
|
||||
Authorization: "Bearer ghp_expired",
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(fetchMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: { "User-Agent": "clawhub" },
|
||||
}),
|
||||
);
|
||||
expect(runMutation).toHaveBeenCalledWith(internal.users.setGitHubCreatedAtInternal, {
|
||||
userId: "users:1",
|
||||
githubCreatedAt: Date.parse("2020-01-01T00:00:00Z"),
|
||||
});
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
"[githubAccount] GITHUB_TOKEN was rejected; retrying lookup without auth",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not retry unauthenticated 401 responses", async () => {
|
||||
const runQuery = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
_id: "users:1",
|
||||
githubCreatedAt: undefined,
|
||||
})
|
||||
.mockResolvedValueOnce("12345");
|
||||
const runMutation = vi.fn();
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: false, status: 401 });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
await expect(
|
||||
requireGitHubAccountAge({ runQuery, runMutation } as never, "users:1" as never),
|
||||
).rejects.toThrow(/GitHub account lookup failed/i);
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://api.github.com/user/12345",
|
||||
expect.objectContaining({
|
||||
headers: { "User-Agent": "clawhub" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("syncGitHubProfile", () => {
|
||||
|
||||
+32
-21
@@ -24,13 +24,42 @@ function assertGitHubNumericId(providerAccountId: string) {
|
||||
|
||||
function buildGitHubHeaders() {
|
||||
const headers: Record<string, string> = { "User-Agent": "clawhub" };
|
||||
const token = process.env.GITHUB_TOKEN;
|
||||
const token = process.env.GITHUB_TOKEN?.trim();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
async function fetchGitHubUserByNumericId(providerAccountId: string) {
|
||||
assertGitHubNumericId(providerAccountId);
|
||||
const url = `${GITHUB_API}/user/${providerAccountId}`;
|
||||
const response = await fetch(url, {
|
||||
headers: buildGitHubHeaders(),
|
||||
});
|
||||
if (response.status !== 401 || !process.env.GITHUB_TOKEN?.trim()) return response;
|
||||
|
||||
console.warn("[githubAccount] GITHUB_TOKEN was rejected; retrying lookup without auth");
|
||||
return await fetch(url, {
|
||||
headers: { "User-Agent": "clawhub" },
|
||||
});
|
||||
}
|
||||
|
||||
export async function fetchGitHubCreatedAtByProviderAccountId(providerAccountId: string) {
|
||||
const response = await fetchGitHubUserByNumericId(providerAccountId);
|
||||
if (!response.ok) {
|
||||
if (response.status === 403 || response.status === 429) {
|
||||
throw new ConvexError("GitHub API rate limit exceeded — please try again in a few minutes");
|
||||
}
|
||||
throw new ConvexError("GitHub account lookup failed");
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as GitHubUser;
|
||||
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN;
|
||||
if (!Number.isFinite(parsed)) throw new ConvexError("GitHub account lookup failed");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId: Id<"users">) {
|
||||
const user = await ctx.runQuery(internal.users.getByIdInternal, { userId });
|
||||
if (!user || user.deletedAt || user.deactivatedAt) throw new ConvexError("User not found");
|
||||
@@ -48,24 +77,8 @@ export async function requireGitHubAccountAge(ctx: GitHubAccountGateCtx, userId:
|
||||
// Invariant: GitHub is our only auth provider, so this should never happen.
|
||||
throw new ConvexError("GitHub account required");
|
||||
}
|
||||
assertGitHubNumericId(providerAccountId);
|
||||
|
||||
// Fetch by immutable GitHub numeric ID to avoid username swap attacks entirely.
|
||||
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
|
||||
headers: buildGitHubHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
if (response.status === 403 || response.status === 429) {
|
||||
throw new ConvexError("GitHub API rate limit exceeded — please try again in a few minutes");
|
||||
}
|
||||
throw new ConvexError("GitHub account lookup failed");
|
||||
}
|
||||
|
||||
const payload = (await response.json()) as GitHubUser;
|
||||
const parsed = payload.created_at ? Date.parse(payload.created_at) : Number.NaN;
|
||||
if (!Number.isFinite(parsed)) throw new ConvexError("GitHub account lookup failed");
|
||||
|
||||
createdAt = parsed;
|
||||
createdAt = await fetchGitHubCreatedAtByProviderAccountId(providerAccountId);
|
||||
await ctx.runMutation(internal.users.setGitHubCreatedAtInternal, {
|
||||
userId,
|
||||
githubCreatedAt: createdAt,
|
||||
@@ -107,9 +120,7 @@ export async function syncGitHubProfile(ctx: ActionCtx, userId: Id<"users">) {
|
||||
|
||||
assertGitHubNumericId(providerAccountId);
|
||||
|
||||
const response = await fetch(`${GITHUB_API}/user/${providerAccountId}`, {
|
||||
headers: buildGitHubHeaders(),
|
||||
});
|
||||
const response = await fetchGitHubUserByNumericId(providerAccountId);
|
||||
if (!response.ok) {
|
||||
// Silently fail - this is a best-effort sync, not critical path
|
||||
console.warn(`[syncGitHubProfile] GitHub API error for user ${userId}: ${response.status}`);
|
||||
|
||||
@@ -299,6 +299,9 @@ See also: [acceptable-usage.md](./acceptable-usage.md) for the marketplace polic
|
||||
- To reduce rate-limit failures, set `GITHUB_TOKEN` in Convex env for authenticated
|
||||
GitHub API requests. The same token is used for trusted-publisher repository
|
||||
identity lookups.
|
||||
- If a configured `GITHUB_TOKEN` is rejected with `401`, retry the account-age
|
||||
lookup without auth before failing. Never fall back to mutable GitHub usernames
|
||||
for this gate; use the operator backfill to cache missing ages for existing users.
|
||||
|
||||
## Empty-skill cleanup (backfill)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user