build: generate llms.txt from canonical metadata (#2483)

* docs: add llms.txt

* docs: avoid dead llms skill examples

* refactor: centralize public registry routes

* refactor: share canonical ClawHub links

* feat: generate llms.txt from canonical metadata

* build: generate llms.txt for dev and production

* test: follow canonical docs navigation link
This commit is contained in:
Vyctor H. Brzezowski
2026-06-22 13:04:07 -07:00
committed by GitHub
parent 519f56301d
commit 6dfe780a29
7 changed files with 430 additions and 16 deletions
+1
View File
@@ -32,6 +32,7 @@ eval/results/
playwright-report
test-results
.playwright
/public/llms.txt
convex/_generated/*
!convex/_generated/ai/
convex/_generated/ai/*
+5 -3
View File
@@ -9,7 +9,7 @@
"type": "module",
"scripts": {
"admin": "bun packages/clawhub-admin/src/cli.ts",
"build": "vite build && bun scripts/copy-og-assets.ts",
"build": "bun run llms:generate && vite build && bun scripts/copy-og-assets.ts",
"check": "bun run lint",
"check:peers": "bun scripts/check-peer-deps.ts",
"check:secrets": "bun scripts/check-staged-secrets.mjs",
@@ -18,7 +18,7 @@
"ci:playwright": "VITE_CONVEX_URL=https://wry-manatee-359.convex.cloud VITE_CONVEX_SITE_URL=https://wry-manatee-359.convex.site bun run build && VITE_CONVEX_URL=https://wry-manatee-359.convex.cloud VITE_CONVEX_SITE_URL=https://wry-manatee-359.convex.site bun run test:pw",
"ci:playwright-smoke": "VITE_CONVEX_URL=https://wry-manatee-359.convex.cloud VITE_CONVEX_SITE_URL=https://wry-manatee-359.convex.site bun run build && VITE_CONVEX_URL=https://wry-manatee-359.convex.cloud VITE_CONVEX_SITE_URL=https://wry-manatee-359.convex.site bun run test:pw -- --project=chromium e2e/ci-smoke.pw.test.ts e2e/public-routes-smoke.pw.test.ts",
"ci:pr": "bun run ci:static && bun run ci:unit && bun run ci:packages && bun run ci:types-build && bun run ci:e2e-http",
"ci:static": "bun run check:peers && bun audit --ignore GHSA-rmmr-r34h-pfm5 --ignore GHSA-gv7w-rqvm-qjhr --ignore GHSA-g7r4-m6w7-qqqr --ignore GHSA-x4vx-rjvf-j5p4 --ignore GHSA-76mc-f452-cxcm --ignore GHSA-hpcv-96wg-7vj8 --ignore GHSA-r47g-fvhr-h676 --ignore GHSA-vxr8-fq34-vvx9 --ignore GHSA-gvmj-g25r-r7wr --ignore GHSA-rp9w-3fw7-7cwq --ignore GHSA-cmwh-pvxp-8882 --ignore GHSA-vmh5-mc38-953g --ignore GHSA-pr7r-676h-xcf6 && bun run format:check && bun run lint && bun run deadcode:ci",
"ci:static": "bun run check:peers && bun audit --ignore GHSA-rmmr-r34h-pfm5 --ignore GHSA-gv7w-rqvm-qjhr --ignore GHSA-g7r4-m6w7-qqqr --ignore GHSA-x4vx-rjvf-j5p4 --ignore GHSA-76mc-f452-cxcm --ignore GHSA-hpcv-96wg-7vj8 --ignore GHSA-r47g-fvhr-h676 --ignore GHSA-vxr8-fq34-vvx9 --ignore GHSA-gvmj-g25r-r7wr --ignore GHSA-rp9w-3fw7-7cwq --ignore GHSA-cmwh-pvxp-8882 --ignore GHSA-vmh5-mc38-953g --ignore GHSA-pr7r-676h-xcf6 && bun run llms:check && bun run format:check && bun run lint && bun run deadcode:ci",
"ci:types-build": "bunx tsc --noEmit && bunx tsc -p packages/schema/tsconfig.json --noEmit && bunx tsc -p packages/clawhub/tsconfig.json --noEmit && bun run --cwd packages/clawhub-admin typecheck && VITE_CONVEX_URL=https://example.invalid bun run build",
"ci:unit": "VITE_CONVEX_URL=https://example.invalid bun run coverage",
"clawscan:local": "bun scripts/local-clawscan-dry-run.ts",
@@ -35,7 +35,7 @@
"deadcode:exports": "KNIP_INCLUDE_TESTS=1 bunx knip@6.8.0 --config knip.config.ts --no-progress --reporter compact --exports --no-config-hints",
"deadcode:files": "bunx knip@6.8.0 --config knip.config.ts --production --no-progress --reporter compact --files --no-config-hints",
"deadcode:knip": "bun run deadcode:files && bun run deadcode:dependencies && bun run deadcode:exports",
"dev": "bun --bun vite dev --port 3000",
"dev": "bun run llms:generate && bun --bun vite dev --port 3000",
"dev:workers": "bun scripts/dev-workers.ts",
"dev:worktree": "wt --yes dev",
"docs:list": "bun scripts/docs-list.ts",
@@ -47,6 +47,8 @@
"lint": "bun run lint:oxlint",
"lint:fix": "oxlint --type-aware --tsconfig ./tsconfig.oxlint.json ./src ./convex ./packages/clawhub/src ./packages/clawhub-admin/src ./packages/schema/src --fix && bun run format",
"lint:oxlint": "oxlint --type-aware --tsconfig ./tsconfig.oxlint.json ./src ./convex ./packages/clawhub/src ./packages/clawhub-admin/src ./packages/schema/src",
"llms:check": "bun scripts/generate-llms-txt.ts --check",
"llms:generate": "bun scripts/generate-llms-txt.ts",
"preinstall": "bunx only-allow bun",
"preview": "bun --bun vite preview",
"proof:publish": "node scripts/ui-proof-publish.mjs",
+100
View File
@@ -0,0 +1,100 @@
/* @vitest-environment node */
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { readLlmsDocs, renderLlmsTxt, writeLlmsTxt } from "./generate-llms-txt";
const temporaryDirectories: string[] = [];
async function temporaryDocs() {
const root = await mkdtemp(join(tmpdir(), "clawhub-llms-"));
temporaryDirectories.push(root);
const docsDir = join(root, "docs");
await mkdir(docsDir);
return docsDir;
}
async function writeDoc(docsDir: string, filename: string, content: string) {
await writeFile(join(docsDir, filename), content, "utf8");
}
afterEach(async () => {
await Promise.all(
temporaryDirectories
.splice(0)
.map((directory) => rm(directory, { recursive: true, force: true })),
);
});
describe("llms.txt generation", () => {
it("discovers publishable docs from frontmatter without maintaining a second index", async () => {
const docsDir = await temporaryDocs();
await writeDoc(docsDir, "zebra.md", `---\nsummary: "Zebra reference."\n---\n\n# Zebra\n`);
await writeDoc(
docsDir,
"alpha.md",
`---\nsummary: "Alpha reference."\ntitle: "Alpha Guide"\n---\n\n# Ignored fallback\n`,
);
await writeDoc(
docsDir,
"hidden.md",
`---\nsummary: "Internal note."\nllms: false\n---\n\n# Hidden\n`,
);
await writeDoc(docsDir, "README.md", `---\nsummary: "Directory index."\n---\n\n# Docs\n`);
await expect(readLlmsDocs(docsDir)).resolves.toEqual([
{
slug: "alpha",
summary: "Alpha reference.",
title: "Alpha Guide",
},
{
slug: "zebra",
summary: "Zebra reference.",
title: "Zebra",
},
]);
});
it("rejects a public doc without the metadata required by the machine index", async () => {
const docsDir = await temporaryDocs();
await writeDoc(docsDir, "broken.md", `---\nread_when:\n - Testing\n---\n\n# Broken\n`);
await expect(readLlmsDocs(docsDir)).rejects.toThrow(
"docs/broken.md must define a non-empty summary",
);
});
it("renders the current registry map from canonical routes and doc metadata", async () => {
const output = await renderLlmsTxt();
expect(output).toContain(
"- [Skills](https://clawhub.ai/skills): Browse and search skill bundles centered on SKILL.md.",
);
expect(output).toContain(
"- [Plugins and packages](https://clawhub.ai/plugins): Browse and search OpenClaw plugin package records.",
);
expect(output).toContain(
"- [CLI](https://docs.openclaw.ai/clawhub/cli): CLI reference: commands, flags, config, and lockfile behavior.",
);
expect(output).toContain(
"- [OpenAPI v1](https://clawhub.ai/api/v1/openapi.json): Machine-readable schema for the current ClawHub HTTP API.",
);
expect(output).not.toMatch(
/clawhub sync|https:\/\/clawhub\.ai\/packages(?:\/|\b)|souls?|soul-format/i,
);
expect(output.split("").every((character) => character.charCodeAt(0) <= 127)).toBe(true);
expect(output.endsWith("\n")).toBe(true);
});
it("writes the static asset once and leaves an identical file untouched", async () => {
const root = await mkdtemp(join(tmpdir(), "clawhub-llms-output-"));
temporaryDirectories.push(root);
const outputFile = join(root, "llms.txt");
await expect(writeLlmsTxt(outputFile)).resolves.toBe(true);
await expect(readFile(outputFile, "utf8")).resolves.toBe(await renderLlmsTxt());
await expect(writeLlmsTxt(outputFile)).resolves.toBe(false);
});
});
+237
View File
@@ -0,0 +1,237 @@
import { readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
import { dirname, extname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parse as parseYaml } from "yaml";
import {
CLAWHUB_DOCS_URL,
CLAWHUB_OPENAPI_URL,
CLAWHUB_REPOSITORY_URL,
CLAWHUB_SITE_URL,
OPENCLAW_DOCS_LLMS_URL,
PUBLIC_REGISTRY_SURFACES,
clawhubDocsUrl,
publicRegistryUrl,
} from "../src/lib/publicRegistry";
const REPOSITORY_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const DEFAULT_DOCS_DIRECTORY = join(REPOSITORY_ROOT, "docs");
const DEFAULT_OUTPUT_FILE = join(REPOSITORY_ROOT, "public", "llms.txt");
interface LlmsDoc {
slug: string;
summary: string;
title: string;
}
interface Frontmatter {
llms?: unknown;
summary?: unknown;
title?: unknown;
}
function normalizeInlineText(value: string) {
return value.trim().replace(/\s+/g, " ");
}
function isAscii(value: string) {
for (const character of value) {
if ((character.codePointAt(0) ?? 128) > 127) return false;
}
return true;
}
function parseFrontmatter(source: string, filename: string) {
const match = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(source);
if (!match) throw new Error(`docs/${filename} must start with YAML frontmatter`);
const parsed: unknown = parseYaml(match[1]);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new Error(`docs/${filename} frontmatter must be a YAML object`);
}
return parsed as Frontmatter;
}
function findHeading(source: string, filename: string) {
const heading = /^#\s+(.+)$/m.exec(source)?.[1];
if (!heading) throw new Error(`docs/${filename} must define a level-one heading or title`);
return normalizeInlineText(heading);
}
export async function readLlmsDocs(docsDirectory = DEFAULT_DOCS_DIRECTORY): Promise<LlmsDoc[]> {
const entries = await readdir(docsDirectory, { withFileTypes: true });
const docs = await Promise.all(
entries
.filter(
(entry) => entry.isFile() && extname(entry.name) === ".md" && entry.name !== "README.md",
)
.map(async (entry) => {
const source = await readFile(join(docsDirectory, entry.name), "utf8");
const frontmatter = parseFrontmatter(source, entry.name);
if (frontmatter.llms === false) return undefined;
if (typeof frontmatter.summary !== "string" || !frontmatter.summary.trim()) {
throw new Error(`docs/${entry.name} must define a non-empty summary`);
}
const title =
typeof frontmatter.title === "string" && frontmatter.title.trim()
? normalizeInlineText(frontmatter.title)
: findHeading(source, entry.name);
return {
slug: entry.name.slice(0, -extname(entry.name).length),
summary: normalizeInlineText(frontmatter.summary),
title,
} satisfies LlmsDoc;
}),
);
return docs
.filter((doc): doc is LlmsDoc => doc !== undefined)
.sort((left, right) => {
if (left.slug === "clawhub") return -1;
if (right.slug === "clawhub") return 1;
if (left.title === right.title) return left.slug < right.slug ? -1 : 1;
return left.title < right.title ? -1 : 1;
});
}
function markdownLink(label: string, url: string, summary: string) {
return `- [${label}](${url}): ${summary}`;
}
export async function renderLlmsTxt(options: { docsDirectory?: string } = {}) {
const docs = await readLlmsDocs(options.docsDirectory);
const lines = [
"# ClawHub",
"",
"> ClawHub is the public registry for OpenClaw skills and plugins.",
"",
"Use OpenClaw to search, install, and update registry content. Use the ClawHub CLI for registry authentication, publishing, and listing management. Follow the canonical documentation below for current commands and workflows.",
"",
"This generated file is a machine-readable map, not an access-control policy or a substitute for reviewing an artifact before installation.",
"",
"## Primary",
"",
markdownLink(
"ClawHub",
CLAWHUB_SITE_URL,
"Browse the public registry for skills, plugins, and publishers.",
),
markdownLink(
"ClawHub documentation",
CLAWHUB_DOCS_URL,
"Canonical user, publisher, API, trust, and operations documentation.",
),
markdownLink(
"OpenClaw docs llms.txt",
OPENCLAW_DOCS_LLMS_URL,
"Machine-readable map for the full OpenClaw documentation set.",
),
markdownLink(
"OpenAPI v1",
CLAWHUB_OPENAPI_URL,
"Machine-readable schema for the current ClawHub HTTP API.",
),
"",
"## Public Registry Surfaces",
"",
...PUBLIC_REGISTRY_SURFACES.map((surface) =>
markdownLink(surface.label, publicRegistryUrl(surface.path), surface.summary),
),
"",
"## Documentation",
"",
...docs.map((doc) =>
markdownLink(
doc.title,
clawhubDocsUrl(doc.slug === "clawhub" ? undefined : doc.slug),
doc.summary,
),
),
"",
"## Source",
"",
markdownLink(
"Source repository",
CLAWHUB_REPOSITORY_URL,
"TanStack Start app, Convex backend, CLI packages, public docs, and specifications.",
),
markdownLink(
"Public docs source",
`${CLAWHUB_REPOSITORY_URL}/tree/main/docs`,
"Source Markdown mirrored into the OpenClaw documentation site.",
),
markdownLink(
"Schema package",
`${CLAWHUB_REPOSITORY_URL}/tree/main/packages/schema`,
"Shared API routes, package schemas, and compatibility contracts.",
),
markdownLink(
"ClawHub CLI",
`${CLAWHUB_REPOSITORY_URL}/tree/main/packages/clawhub`,
"CLI implementation for authentication, discovery, publishing, and listing management.",
),
markdownLink(
"Convex backend",
`${CLAWHUB_REPOSITORY_URL}/tree/main/convex`,
"Registry data, HTTP handlers, publishing, scans, moderation, and search.",
),
"",
"## Optional",
"",
markdownLink(
"robots.txt",
publicRegistryUrl("/robots.txt"),
"Crawler policy for the ClawHub site.",
),
markdownLink(
"Web app manifest",
publicRegistryUrl("/manifest.json"),
"Installable web app metadata.",
),
"",
];
const output = lines.join("\n");
if (!isAscii(output)) {
throw new Error("Generated public/llms.txt must contain ASCII text only");
}
return output;
}
export async function writeLlmsTxt(outputFile = DEFAULT_OUTPUT_FILE) {
const output = await renderLlmsTxt();
const current = await readFile(outputFile, "utf8").catch(() => undefined);
if (current === output) return false;
const temporaryFile = `${outputFile}.${process.pid}.tmp`;
await writeFile(temporaryFile, output, "utf8");
try {
await rename(temporaryFile, outputFile);
} catch (error) {
await unlink(temporaryFile).catch(() => undefined);
throw error;
}
return true;
}
async function main() {
const args = new Set(process.argv.slice(2));
const unknown = [...args].filter((arg) => arg !== "--check" && arg !== "--stdout");
if (unknown.length > 0) throw new Error(`Unknown argument: ${unknown.join(", ")}`);
if (args.has("--stdout")) {
process.stdout.write(await renderLlmsTxt());
return;
}
if (args.has("--check")) {
await renderLlmsTxt();
return;
}
await writeLlmsTxt();
}
if (import.meta.main) {
await main();
}
+6 -1
View File
@@ -65,6 +65,7 @@ describe("restored UI design contract", () => {
const footer = () => read("src/components/Footer.tsx");
const home = () => read("src/routes/index.tsx");
const navItems = () => read("src/lib/nav-items.ts");
const publicRegistry = () => read("src/lib/publicRegistry.ts");
const settings = () => read("src/routes/settings.tsx");
const styles = () => read("src/styles.css");
const theme = () => read("src/lib/theme.ts");
@@ -83,6 +84,7 @@ describe("restored UI design contract", () => {
it("requires the responsive header rail, search overlay, and theme controls", () => {
const headerSource = header();
const navSource = navItems();
const publicRegistrySource = publicRegistry();
const css = styles();
expect(headerSource).toContain('className="navbar-top"');
@@ -112,7 +114,10 @@ describe("restored UI design contract", () => {
expect(navSource).toContain("export const SECONDARY_NAV_ITEMS");
expect(navSource).toContain('label: "Publishers"');
expect(navSource).toContain('label: "Docs"');
expect(navSource).toContain('href: "https://docs.openclaw.ai/clawhub/"');
expect(navSource).toContain("href: CLAWHUB_DOCS_URL");
expect(publicRegistrySource).toContain(
'export const CLAWHUB_DOCS_URL = "https://docs.openclaw.ai/clawhub/"',
);
expect(navSource).not.toContain('icon: "wrench"');
expect(navSource).not.toContain('icon: "plug"');
expect(navSource).not.toContain('label: "About"');
+19 -12
View File
@@ -1,3 +1,5 @@
import { CLAWHUB_DOCS_URL, CLAWHUB_REPOSITORY_URL, PublicRegistryPaths } from "./publicRegistry";
/**
* Shared navigation configuration used by Header and Footer to eliminate
* triple duplication of nav link definitions.
@@ -48,18 +50,18 @@ const SKILLS_SEARCH = {
export const PRIMARY_NAV_ITEMS: NavItem[] = [
{
label: "Skills",
to: "/skills",
to: PublicRegistryPaths.skills,
search: SKILLS_SEARCH,
activePathPrefixes: ["/skill/"],
},
{
label: "Plugins",
to: "/plugins",
to: PublicRegistryPaths.plugins,
activePathPrefixes: ["/plugin/"],
},
{
label: "Publishers",
to: "/publishers",
to: PublicRegistryPaths.publishers,
activePathPrefixes: ["/user/"],
},
];
@@ -71,7 +73,7 @@ export const PRIMARY_NAV_ITEMS: NavItem[] = [
export const SECONDARY_NAV_ITEMS: NavItem[] = [
{
label: "Docs",
href: "https://docs.openclaw.ai/clawhub/",
href: CLAWHUB_DOCS_URL,
activePathPrefixes: ["/docs"],
},
];
@@ -83,7 +85,7 @@ export const SECONDARY_NAV_ITEMS: NavItem[] = [
export const OPENCLAW_SITE_URL = "https://openclaw.ai";
export const OPENCLAW_ECOSYSTEM_URL = `${OPENCLAW_SITE_URL}/ecosystem`;
const OPENCLAW_BLOG_CLAWHUB_URL = `${OPENCLAW_SITE_URL}/blog#clawhub`;
export const OPENCLAW_CLAWHUB_DOCS_URL = "https://docs.openclaw.ai/clawhub/";
export const OPENCLAW_CLAWHUB_DOCS_URL = CLAWHUB_DOCS_URL;
/** Compact mark for stack avatars (not the full wordmark). */
export const OPENCLAW_LOGO_URL = `${OPENCLAW_SITE_URL}/favicon.svg`;
@@ -113,10 +115,15 @@ export const FOOTER_NAV_SECTIONS: FooterNavSection[] = [
{
title: "Browse",
items: [
{ kind: "link", label: "Skills", to: "/skills", search: SKILLS_SEARCH },
{ kind: "link", label: "Plugins", to: "/plugins" },
{ kind: "link", label: "Publishers", to: "/publishers" },
{ kind: "link", label: "Audits", to: "/audits", search: { type: undefined } },
{ kind: "link", label: "Skills", to: PublicRegistryPaths.skills, search: SKILLS_SEARCH },
{ kind: "link", label: "Plugins", to: PublicRegistryPaths.plugins },
{ kind: "link", label: "Publishers", to: PublicRegistryPaths.publishers },
{
kind: "link",
label: "Audits",
to: PublicRegistryPaths.audits,
search: { type: undefined },
},
],
},
{
@@ -125,13 +132,13 @@ export const FOOTER_NAV_SECTIONS: FooterNavSection[] = [
{
kind: "link",
label: "Publish Skill",
to: "/skills/publish",
to: PublicRegistryPaths.publishSkill,
search: { updateSlug: undefined },
},
{
kind: "link",
label: "Publish Plugin",
to: "/plugins/publish",
to: PublicRegistryPaths.publishPlugin,
search: {
ownerHandle: undefined,
name: undefined,
@@ -164,7 +171,7 @@ export const FOOTER_NAV_SECTIONS: FooterNavSection[] = [
{
kind: "external",
label: "GitHub",
href: "https://github.com/openclaw/clawhub",
href: CLAWHUB_REPOSITORY_URL,
icon: "github",
},
{
+62
View File
@@ -0,0 +1,62 @@
export const CLAWHUB_SITE_URL = "https://clawhub.ai";
export const CLAWHUB_REPOSITORY_URL = "https://github.com/openclaw/clawhub";
export const CLAWHUB_DOCS_URL = "https://docs.openclaw.ai/clawhub/";
export const OPENCLAW_DOCS_LLMS_URL = "https://docs.openclaw.ai/llms.txt";
export const CLAWHUB_OPENAPI_URL = `${CLAWHUB_SITE_URL}/api/v1/openapi.json`;
export const PublicRegistryPaths = {
home: "/",
skills: "/skills",
plugins: "/plugins",
publishers: "/publishers",
search: "/search",
audits: "/audits",
publishSkill: "/skills/publish",
publishPlugin: "/plugins/publish",
} as const;
export const PUBLIC_REGISTRY_SURFACES = [
{
label: "Skills",
path: PublicRegistryPaths.skills,
summary: "Browse and search skill bundles centered on SKILL.md.",
},
{
label: "Plugins and packages",
path: PublicRegistryPaths.plugins,
summary: "Browse and search OpenClaw plugin package records.",
},
{
label: "Publishers",
path: PublicRegistryPaths.publishers,
summary: "Browse public user and organization publishers.",
},
{
label: "Search",
path: PublicRegistryPaths.search,
summary: "Search the public ClawHub catalog.",
},
{
label: "Security audits",
path: PublicRegistryPaths.audits,
summary: "Browse public artifact security audit results.",
},
{
label: "Publish a skill",
path: PublicRegistryPaths.publishSkill,
summary: "Publish a new skill or update an existing skill.",
},
{
label: "Publish a plugin",
path: PublicRegistryPaths.publishPlugin,
summary: "Publish an OpenClaw plugin package.",
},
] as const;
export function publicRegistryUrl(path: string) {
return new URL(path, CLAWHUB_SITE_URL).href;
}
export function clawhubDocsUrl(slug?: string) {
return new URL(slug ?? "", CLAWHUB_DOCS_URL).href;
}