diff --git a/.gitignore b/.gitignore index 3c145df0e25..d7a2c67c1fe 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,12 @@ mini-swe-agent/ .nix-stamps/ result website/static/api/skills-index.json +# skills.json + skills-meta.json are build artifacts emitted by +# website/scripts/extract-skills.py during prebuild — keep them out of +# git for the same reason as skills-index.json (large, generated, change +# every build). +website/static/api/skills.json +website/static/api/skills-meta.json models-dev-upstream/ hermes_cli/tui_dist/* hermes_cli/scripts/ diff --git a/website/scripts/extract-skills.py b/website/scripts/extract-skills.py index dd648589db8..f72598b05af 100644 --- a/website/scripts/extract-skills.py +++ b/website/scripts/extract-skills.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Extract skill metadata into website/src/data/skills.json for the Skills Hub page. +"""Extract skill metadata into website/static/api/skills.json for the Skills Hub page. Two data sources: @@ -32,8 +32,12 @@ LOCAL_SKILL_DIRS = [ ] UNIFIED_INDEX_PATH = os.path.join(REPO_ROOT, "website", "static", "api", "skills-index.json") LEGACY_INDEX_CACHE_DIR = os.path.join(REPO_ROOT, "skills", "index-cache") -OUTPUT = os.path.join(REPO_ROOT, "website", "src", "data", "skills.json") -META_OUTPUT = os.path.join(REPO_ROOT, "website", "src", "data", "skills-meta.json") +# Output to static/api/ so the file is CDN-served at /api/skills.json +# rather than bundled into the page's JS chunk. At 50k+ skills the +# bundled payload was ~26 MB; lazy-fetch keeps the initial page load +# fast and shrinks the JS chunk back to a few hundred KB. +OUTPUT = os.path.join(REPO_ROOT, "website", "static", "api", "skills.json") +META_OUTPUT = os.path.join(REPO_ROOT, "website", "static", "api", "skills-meta.json") CATEGORY_LABELS = { "apple": "Apple", @@ -531,7 +535,9 @@ def main(): os.makedirs(os.path.dirname(OUTPUT), exist_ok=True) with open(OUTPUT, "w", encoding="utf-8") as f: - json.dump(all_skills, f, indent=2) + # Minified — file is served over the wire, not read by humans. + # At 50k+ skills the indented version was ~30% larger. + json.dump(all_skills, f, separators=(",", ":"), ensure_ascii=False) # Sidecar meta file so the page can render a "Last refreshed" badge # without changing the shape of skills.json. @@ -547,7 +553,7 @@ def main(): if index_meta: meta.update(index_meta) with open(META_OUTPUT, "w", encoding="utf-8") as f: - json.dump(meta, f, indent=2) + json.dump(meta, f, separators=(",", ":"), ensure_ascii=False) print(f"Extracted {len(all_skills)} skills to {OUTPUT}") print(f" {len(local)} local ({sum(1 for s in local if s['source'] == 'built-in')} built-in, " diff --git a/website/scripts/prebuild.mjs b/website/scripts/prebuild.mjs index 32e050bd933..11f5e07521e 100644 --- a/website/scripts/prebuild.mjs +++ b/website/scripts/prebuild.mjs @@ -1,7 +1,8 @@ #!/usr/bin/env node // Runs website/scripts/extract-skills.py and generate-llms-txt.py before // docusaurus build/start so that: -// - website/src/data/skills.json (imported by src/pages/skills/index.tsx) +// - website/static/api/skills.json (lazy-fetched by src/pages/skills/index.tsx) +// - website/static/api/skills-meta.json (sidecar metadata for the Skills Hub) // - website/static/llms.txt (agent-friendly short docs index) // - website/static/llms-full.txt (full docs concat for LLM context) // all exist without contributors remembering to run Python scripts manually. @@ -30,7 +31,7 @@ const scriptDir = dirname(fileURLToPath(import.meta.url)); const websiteDir = resolve(scriptDir, ".."); const extractScript = join(scriptDir, "extract-skills.py"); const llmsScript = join(scriptDir, "generate-llms-txt.py"); -const outputFile = join(websiteDir, "src", "data", "skills.json"); +const outputFile = join(websiteDir, "static", "api", "skills.json"); const unifiedIndexFile = join(websiteDir, "static", "api", "skills-index.json"); const UNIFIED_INDEX_URL = "https://hermes-agent.nousresearch.com/docs/api/skills-index.json"; diff --git a/website/src/pages/skills/index.tsx b/website/src/pages/skills/index.tsx index 0ef6f64abc2..a86a0205edf 100644 --- a/website/src/pages/skills/index.tsx +++ b/website/src/pages/skills/index.tsx @@ -1,7 +1,5 @@ import React, { useState, useMemo, useCallback, useRef, useEffect } from "react"; import Layout from "@theme/Layout"; -import skills from "../../data/skills.json"; -import meta from "../../data/skills-meta.json"; import styles from "./styles.module.css"; interface Skill { @@ -21,9 +19,14 @@ interface Skill { docsPath?: string; identifier?: string; installCmd?: string; + /** Lowercase pre-joined haystack used by the search filter. + * Built once at load time so per-keystroke filtering is a single + * `.includes()` per skill instead of array-join + toLowerCase on + * every render. Skipped on the wire — added in the loader. */ + _search?: string; } -const allSkills: Skill[] = skills as Skill[]; +const allSkills: Skill[] = []; interface IndexMeta { extractedAt?: string; @@ -32,7 +35,7 @@ interface IndexMeta { externalSource?: string; bySource?: Record; } -const indexMeta: IndexMeta = meta as IndexMeta; +const indexMeta: IndexMeta = {}; function formatRelativeTime(iso?: string): string | null { if (!iso) return null; @@ -398,8 +401,43 @@ function StatCard({ value, label, color }: { value: number; label: string; color const PAGE_SIZE = 60; +// Routes Docusaurus serves the static API JSON from. `baseUrl` is `/docs/`, +// `static/api/` ends up at `/docs/api/`. Hardcoding here is fine because the +// same `baseUrl` is enforced repo-wide; if it ever changes, this is the only +// place that needs to follow. +const SKILLS_URL = "/docs/api/skills.json"; +const META_URL = "/docs/api/skills-meta.json"; + +function buildSearchHaystack(s: Skill): string { + // Pre-compute the lowercase blob the search filter scans. Done once at + // load time instead of per-keystroke per-skill. With 50k+ skills the + // per-keystroke variant was unusably slow. + return [ + s.name, + s.description, + s.overview, + s.categoryLabel, + s.author, + ...(s.tags || []), + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); +} + export default function SkillsDashboard() { + // Lazy-loaded data. Was bundled into the JS chunk (~22 MB at 50k skills, + // which made the initial page load unusable on mobile). Now fetched on + // mount from the same CDN that serves the docs. + const [data, setData] = useState<{ skills: Skill[]; meta: IndexMeta } | null>(null); + const [loadError, setLoadError] = useState(null); + const [search, setSearch] = useState(""); + // Debounced copy of `search` — used by the filter. Without the debounce, + // typing into the search box ran .filter() over the whole catalog on + // every keystroke, which on a 50k-item list felt like the page had + // hung. 150ms gives a snappy feel without lagging behind the user. + const [debouncedSearch, setDebouncedSearch] = useState(""); const [sourceFilter, setSourceFilter] = useState("all"); const [categoryFilter, setCategoryFilter] = useState("all"); const [expandedCard, setExpandedCard] = useState(null); @@ -408,6 +446,42 @@ export default function SkillsDashboard() { const searchRef = useRef(null); const gridRef = useRef(null); + useEffect(() => { + let cancelled = false; + (async () => { + try { + const [sk, mt] = await Promise.all([ + fetch(SKILLS_URL).then((r) => { + if (!r.ok) throw new Error(`skills.json HTTP ${r.status}`); + return r.json(); + }), + fetch(META_URL).then((r) => (r.ok ? r.json() : {})).catch(() => ({})), + ]); + if (cancelled) return; + const skillsArr = Array.isArray(sk) ? (sk as Skill[]) : []; + // Stamp the precomputed search haystack onto each row. + for (const s of skillsArr) s._search = buildSearchHaystack(s); + setData({ skills: skillsArr, meta: mt || {} }); + } catch (err) { + if (cancelled) return; + setLoadError(err instanceof Error ? err.message : String(err)); + } + })(); + return () => { + cancelled = true; + }; + }, []); + + // Debounce the search input — 150ms feels instant while preventing the + // filter from running on every individual keystroke. + useEffect(() => { + const t = setTimeout(() => setDebouncedSearch(search), 150); + return () => clearTimeout(t); + }, [search]); + + const allSkillsLocal: Skill[] = data?.skills ?? []; + const indexMetaLocal: IndexMeta = data?.meta ?? indexMeta; + useEffect(() => { const handler = (e: KeyboardEvent) => { if (e.key === "/" && document.activeElement?.tagName !== "INPUT") { @@ -424,15 +498,15 @@ export default function SkillsDashboard() { }, []); const sources = useMemo(() => { - const set = new Set(allSkills.map((s) => s.source)); + const set = new Set(allSkillsLocal.map((s) => s.source)); return SOURCE_ORDER.filter((s) => s === "all" || set.has(s)); }, []); const categoryEntries = useMemo(() => { const pool = sourceFilter === "all" - ? allSkills - : allSkills.filter((s) => s.source === sourceFilter); + ? allSkillsLocal + : allSkillsLocal.filter((s) => s.source === sourceFilter); const map = new Map(); for (const s of pool) { const key = s.category || "uncategorized"; @@ -452,24 +526,22 @@ export default function SkillsDashboard() { }, [sourceFilter]); const filtered = useMemo(() => { - const q = search.toLowerCase().trim(); - return allSkills.filter((s) => { + const q = debouncedSearch.toLowerCase().trim(); + return allSkillsLocal.filter((s) => { if (sourceFilter !== "all" && s.source !== sourceFilter) return false; if (categoryFilter !== "all" && s.category !== categoryFilter) return false; if (q) { - const haystack = [s.name, s.description, s.overview, s.categoryLabel, s.author, ...(s.tags || [])] - .join(" ") - .toLowerCase(); - return haystack.includes(q); + // _search is pre-built in the load effect — single .includes() per row. + return (s._search || "").includes(q); } return true; }); - }, [search, sourceFilter, categoryFilter]); + }, [debouncedSearch, sourceFilter, categoryFilter, allSkillsLocal]); useEffect(() => { setVisibleCount(PAGE_SIZE); setExpandedCard(null); - }, [search, sourceFilter, categoryFilter]); + }, [debouncedSearch, sourceFilter, categoryFilter]); const visible = filtered.slice(0, visibleCount); const hasMore = visibleCount < filtered.length; @@ -512,15 +584,22 @@ export default function SkillsDashboard() {

Skills Hub

Discover, search, and install from{" "} - {allSkills.length} skills - across {sources.length - 1} registries + + {data ? allSkillsLocal.length.toLocaleString() : "…"} + {" "} + skills across {sources.length - 1} registries + {loadError && ( + + · failed to load catalog ({loadError}) + + )}

- {(indexMeta?.indexGeneratedAt || indexMeta?.extractedAt) && ( + {(indexMetaLocal?.indexGeneratedAt || indexMetaLocal?.extractedAt) && (

Catalog refreshed{" "} - + {formatRelativeTime( - indexMeta.indexGeneratedAt || indexMeta.extractedAt, + indexMetaLocal.indexGeneratedAt || indexMetaLocal.extractedAt, ) || "recently"} {" "}· auto-rebuilt twice daily @@ -529,18 +608,18 @@ export default function SkillsDashboard() {

s.source === "built-in").length} + value={allSkillsLocal.filter((s) => s.source === "built-in").length} label="Built-in" color="#4ade80" /> s.source === "optional").length} + value={allSkillsLocal.filter((s) => s.source === "optional").length} label="Optional" color="#fbbf24" /> s.source !== "built-in" && s.source !== "optional" ).length } @@ -548,7 +627,7 @@ export default function SkillsDashboard() { color="#60a5fa" /> s.category)).size} + value={new Set(allSkillsLocal.map((s) => s.category)).size} label="Categories" color="#a78bfa" /> @@ -592,8 +671,8 @@ export default function SkillsDashboard() { const conf = SOURCE_CONFIG[src]; const count = src === "all" - ? allSkills.length - : allSkills.filter((s) => s.source === src).length; + ? allSkillsLocal.length + : allSkillsLocal.filter((s) => s.source === src).length; return (