#!/usr/bin/env python3 """Extract skill metadata into website/static/api/skills.json for the Skills Hub page. Two data sources: 1. Local SKILL.md files under ``skills/`` (built-in) and ``optional-skills/`` (official optional). These give us full metadata — overview prose, version, license, env vars, commands — that the unified index doesn't carry. 2. The unified Hermes Skills Index at ``website/static/api/skills-index.json``, built twice daily by ``scripts/build_skills_index.py`` (workflow ``.github/workflows/skills-index.yml``). Covers skills.sh, ClawHub, browse.sh, LobeHub, Claude Marketplace, well-known endpoints, and the GitHub taps (openai/skills, anthropics/skills, huggingface/skills, VoltAgent, etc.). Legacy fallback: if the unified index is missing AND ``skills/index-cache/`` contains pre-baked JSON dumps, we read those (preserves behaviour from before the unified index existed). """ import json import os from collections import Counter from datetime import datetime, timezone import yaml REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) LOCAL_SKILL_DIRS = [ ("skills", "built-in"), ("optional-skills", "optional"), ] UNIFIED_INDEX_PATH = os.path.join(REPO_ROOT, "website", "static", "api", "skills-index.json") LEGACY_INDEX_CACHE_DIR = os.path.join(REPO_ROOT, "skills", "index-cache") # Output to static/api/ so the file is CDN-served at /api/skills.json # rather than bundled into the page's JS chunk. At 50k+ skills the # bundled payload was ~26 MB; lazy-fetch keeps the initial page load # fast and shrinks the JS chunk back to a few hundred KB. OUTPUT = os.path.join(REPO_ROOT, "website", "static", "api", "skills.json") META_OUTPUT = os.path.join(REPO_ROOT, "website", "static", "api", "skills-meta.json") CATEGORY_LABELS = { "apple": "Apple", "autonomous-ai-agents": "AI Agents", "blockchain": "Blockchain", "communication": "Communication", "creative": "Creative", "data-science": "Data Science", "devops": "DevOps", "dogfood": "Dogfood", "domain": "Business & Finance", "email": "Email", "gaming": "Gaming", "gifs": "GIFs", "github": "GitHub", "health": "Health", "inference-sh": "Inference", "leisure": "Leisure", "mcp": "MCP", "media": "Media", "migration": "Migration", "mlops": "MLOps", "note-taking": "Note-Taking", "productivity": "Productivity", "red-teaming": "Red Teaming", "research": "Research", "security": "Security", "smart-home": "Smart Home", "social-media": "Social Media", "software-development": "Software Dev", "translation": "Translation", "other": "Other", } # Map the source ids the unified index emits to the friendly labels the # Skills Hub UI uses. Keep these in sync with the SOURCE_CONFIG dict in # website/src/pages/skills/index.tsx. UNIFIED_SOURCE_LABELS = { "official": "official", # treated as our "optional" tier in the UI "skills.sh": "skills.sh", "skills-sh": "skills.sh", "clawhub": "ClawHub", "browse-sh": "browse.sh", "lobehub": "LobeHub", "claude-marketplace": "Claude Marketplace", "well-known": "Well-Known", "github": "GitHub", # default for non-named GitHub taps } # Repo-specific labels for the unified index's "github" source. Lets us # call out the well-known taps with their vendor name instead of a generic # "GitHub" pill. Match is checked against the leading "owner/repo/" prefix # of the identifier. GITHUB_TAP_LABELS = { "openai/skills": "OpenAI", "anthropics/skills": "Anthropic", "huggingface/skills": "HuggingFace", "NVIDIA/skills": "NVIDIA", "VoltAgent/awesome-agent-skills": "VoltAgent", "garrytan/gstack": "gstack", "MiniMax-AI/cli": "MiniMax", } # Legacy filename -> label mapping for the deprecated skills/index-cache/ # fallback. Used only when website/static/api/skills-index.json is absent. LEGACY_SOURCE_LABELS = { "anthropics_skills": "Anthropic", "openai_skills": "OpenAI", "claude_marketplace": "Claude Marketplace", "lobehub": "LobeHub", } def _extract_overview(body: str) -> str: """Pull the first non-heading paragraph from a SKILL.md body.""" if not body: return "" paragraphs = [p.strip() for p in body.split("\n\n") if p.strip()] for p in paragraphs[:6]: if p.startswith("#"): lines = [ln for ln in p.split("\n") if ln.strip() and not ln.lstrip().startswith("#")] if lines: p = "\n".join(lines).strip() else: continue if p.startswith(":::"): continue if p.startswith("```") or p.startswith("~~~"): continue if len(p) > 500: cut = p[:500] last_period = cut.rfind(". ") if last_period > 200: p = cut[: last_period + 1] else: p = cut.rstrip() + "…" return p return "" def _docs_page_path(rel_dir: str, source_label: str) -> str: """Compute the per-skill docs-site URL slug for a given SKILL.md location. Mirrors the slug logic in website/scripts/generate-skill-docs.py: bundled + skills///SKILL.md -> bundled//- bundled + skills//_{//SKILL.md -> bundled//-_{-
optional + optional-skills///SKILL.md -> optional//-
"""
parts = [p for p in rel_dir.split(os.sep) if p]
if not parts:
return ""
source_dir = "bundled" if source_label == "built-in" else "optional"
if len(parts) == 1:
category, slug = parts[0], parts[0]
return f"{source_dir}/{category}/{category}-{slug}"
if len(parts) == 2:
category, slug = parts
return f"{source_dir}/{category}/{category}-{slug}"
if len(parts) == 3:
category, sub, slug = parts
return f"{source_dir}/{category}/{category}-{sub}-{slug}"
return ""

def _install_command(source: str, identifier: str, name: str) -> str:
"""Build the ``hermes skills install …`` command for a unified-index entry.

These show up in the SkillCard panel so users can copy-paste them. We try
to use the most idiomatic identifier per source.
"""
if not identifier:
return f"hermes skills install {name}"
src = source.lower()
if src in {"official", "built-in", "optional"}:
# OptionalSkillSource emits identifiers like "official/security/1password"
return f"hermes skills install {identifier}"
if src in {"skills.sh", "skills-sh"}:
# Already wrapped as "skills-sh/owner/repo/skill" by the source
return f"hermes skills install {identifier}"
if src == "clawhub":
return f"hermes skills install clawhub/{identifier}"
if src == "browse-sh":
# Identifier already includes the "browse-sh/" prefix from BrowseShSource
return f"hermes skills install {identifier}"
if src == "lobehub":
return f"hermes skills install {identifier}"
if src == "claude-marketplace":
return f"hermes skills install {identifier}"
if src == "github":
return f"hermes skills install {identifier}"
if src == "well-known":
return f"hermes skills install {identifier}"
return f"hermes skills install {identifier}"

def _source_url(source: str, identifier: str, extra: dict) -> str:
"""Best-effort clickable URL to the skill's origin (repo / detail page).

Community skills have no generated docs page, so without this the
expanded card on the Skills Hub gives users nowhere to go to read the
actual SKILL.md before installing. We prefer an explicit URL the source
adapter already collected (``extra.detail_url`` / ``extra.repo_url``),
then fall back to synthesizing one from the identifier shape.
"""
extra = extra or {}
for key in ("detail_url", "source_url", "repo_url", "url", "index_url"):
val = extra.get(key)
if isinstance(val, str) and val.startswith("http"):
return val

if not identifier:
return ""
src = (source or "").lower()

# GitHub-backed taps (openai/anthropic/nvidia/hf/gstack/VoltAgent/...):
# identifier is "owner/repo/" — link to the directory on GitHub.
if src in {"github", "openai", "anthropic", "huggingface", "nvidia",
"gstack", "voltagent", "minimax", "claude marketplace",
"claude-marketplace"}:
parts = [p for p in identifier.split("/") if p]
if len(parts) >= 2:
owner, repo = parts[0], parts[1]
sub = "/".join(parts[2:])
base = f"https://github.com/{owner}/{repo}"
return f"{base}/tree/main/{sub}" if sub else base
return ""

if src == "clawhub":
# identifier is a bare slug (the "clawhub/" prefix is added at install time)
slug = identifier[len("clawhub/"):] if identifier.startswith("clawhub/") else identifier
return f"https://clawhub.ai/skills/{slug}"

if src in {"skills.sh", "skills-sh"}:
# "skills-sh/owner/repo/skill" -> the skills.sh detail page
rest = identifier[len("skills-sh/"):] if identifier.startswith("skills-sh/") else identifier
return f"https://skills.sh/skills/{rest}"

if src == "lobehub":
slug = identifier[len("lobehub/"):] if identifier.startswith("lobehub/") else identifier
return f"https://lobehub.com/agent/{slug}"

if src in {"browse.sh", "browse-sh"}:
# "browse-sh//" -> browse.sh task page
rest = identifier[len("browse-sh/"):] if identifier.startswith("browse-sh/") else identifier
return f"https://browse.sh/skills/{rest}"

return ""

def extract_local_skills():
skills = []

for base_dir, source_label in LOCAL_SKILL_DIRS:
base_path = os.path.join(REPO_ROOT, base_dir)
if not os.path.isdir(base_path):
continue

for root, _dirs, files in os.walk(base_path):
if "SKILL.md" not in files:
continue

skill_path = os.path.join(root, "SKILL.md")
with open(skill_path, encoding="utf-8") as f:
content = f.read()

if not content.startswith("---"):
continue

parts = content.split("---", 2)
if len(parts) < 3:
continue

try:
fm = yaml.safe_load(parts[1])
except yaml.YAMLError:
continue

if not fm or not isinstance(fm, dict):
continue

body = parts[2].strip()
overview = _extract_overview(body)

rel = os.path.relpath(root, base_path)
category = rel.split(os.sep)[0]

tags = []
metadata = fm.get("metadata")
if isinstance(metadata, dict):
hermes_meta = metadata.get("hermes", {})
if isinstance(hermes_meta, dict):
tags = hermes_meta.get("tags", [])
if not tags:
tags = fm.get("tags", [])
if isinstance(tags, str):
tags = [tags]

prereq = fm.get("prerequisites") or {}
env_vars = []
commands = []
if isinstance(prereq, dict):
ev = prereq.get("env_vars")
if isinstance(ev, list):
env_vars = [str(x) for x in ev if x]
elif isinstance(ev, str) and ev.strip():
env_vars = [ev.strip()]
cmds = prereq.get("commands")
if isinstance(cmds, list):
commands = [str(x) for x in cmds if x]
elif isinstance(cmds, str) and cmds.strip():
commands = [cmds.strip()]

skills.append({
"name": fm.get("name", os.path.basename(root)),
"description": fm.get("description", ""),
"overview": overview,
"category": category,
"categoryLabel": CATEGORY_LABELS.get(category, category.replace("-", " ").title()),
"source": source_label,
"tags": tags or [],
"platforms": fm.get("platforms", []),
"author": fm.get("author", ""),
"version": fm.get("version", ""),
"license": fm.get("license", ""),
"envVars": env_vars,
"commands": commands,
"docsPath": _docs_page_path(rel, source_label),
})

return skills

def _label_for_github_identifier(identifier: str) -> str:
"""Return a friendly source label for a unified-index 'github' entry."""
if not identifier:
return "GitHub"
for prefix, label in GITHUB_TAP_LABELS.items():
if identifier.startswith(prefix + "/") or identifier == prefix:
return label
return "GitHub"

def extract_unified_index_skills():
"""Read website/static/api/skills-index.json — the canonical multi-source index.

Returns ``(skills, meta)`` where ``meta`` carries the index's
``generated_at`` timestamp and total count so the Skills Hub page can
show a "Last refreshed …" badge. Returns ``(None, None)`` when the
index file is absent or malformed (caller falls back to the legacy
cache).
"""
if not os.path.isfile(UNIFIED_INDEX_PATH):
return None, None

try:
with open(UNIFIED_INDEX_PATH, encoding="utf-8") as f:
data = json.load(f)
except (json.JSONDecodeError, OSError) as e:
print(f"[extract-skills] Failed to read unified index: {e}")
return None, None

if not isinstance(data, dict) or "skills" not in data:
return None, None

meta = {
"indexGeneratedAt": data.get("generated_at", ""),
"indexSkillCount": data.get("skill_count", 0),
"indexVersion": data.get("version", 0),
}

out = []
for entry in data.get("skills", []):
if not isinstance(entry, dict):
continue
source_id = (entry.get("source") or "").lower()
identifier = entry.get("identifier", "") or ""
name = entry.get("name") or identifier.split("/")[-1] or "unknown"
description = (entry.get("description") or "").split("\n")[0]
if len(description) > 280:
description = description[:277] + "…"
tags = entry.get("tags", []) or []
if not isinstance(tags, list):
tags = []

# Skip official entries here — extract_local_skills() already covered
# those from optional-skills/ with full metadata (overview, version, etc.).
if source_id == "official":
continue

# Map source id -> display label
if source_id == "github":
source_label = _label_for_github_identifier(identifier)
else:
source_label = UNIFIED_SOURCE_LABELS.get(source_id, source_id or "community")

# Guess a category from tags so the UI's category filter has a chance.
category = _guess_category(tags)
extra = entry.get("extra", {}) or {}

# A skills.sh.json grouping sidecar (if the tap ships one) gives us a
# real, human-readable category — prefer it over the tag heuristic.
# extra["category"] holds the grouping title, e.g. "Inference AI".
sidecar_category = extra.get("category") if isinstance(extra, dict) else None
category_label_override = ""
if isinstance(sidecar_category, str) and sidecar_category.strip():
category_label_override = sidecar_category.strip()
category = category_label_override.lower().replace(" ", "-")

# Author hint from extras when available (skills.sh has installs;
# clawhub doesn't expose author).
author = ""
if source_id in {"skills.sh", "skills-sh"}:
repo = entry.get("repo", "")
if repo:
author = repo.split("/")[0]

install_cmd = _install_command(source_id, identifier, name)
source_url = _source_url(source_id, identifier, extra)

out.append({
"name": name,
"description": description,
"overview": "",
"category": category,
"categoryLabel": category_label_override, # set from sidecar, else filled in _consolidate_small_categories
"fixedCategory": bool(category_label_override), # sidecar categories are exempt from small-cat collapse
"source": source_label,
"tags": tags,
"platforms": [],
"author": author,
"version": "",
"license": "",
"envVars": [],
"commands": [],
"docsPath": "",
"identifier": identifier,
"installCmd": install_cmd,
"sourceUrl": source_url,
})

return out, meta

def extract_legacy_cache_skills():
"""Read the deprecated skills/index-cache/ snapshots — fallback only."""
skills = []

if not os.path.isdir(LEGACY_INDEX_CACHE_DIR):
return skills

for filename in os.listdir(LEGACY_INDEX_CACHE_DIR):
if not filename.endswith(".json"):
continue

filepath = os.path.join(LEGACY_INDEX_CACHE_DIR, filename)
try:
with open(filepath, encoding="utf-8") as f:
data = json.load(f)
except (json.JSONDecodeError, OSError):
continue

stem = filename.replace(".json", "")
source_label = "community"
for key, label in LEGACY_SOURCE_LABELS.items():
if key in stem:
source_label = label
break

if isinstance(data, dict) and "agents" in data:
for agent in data["agents"]:
if not isinstance(agent, dict):
continue
skills.append({
"name": agent.get("identifier", agent.get("meta", {}).get("title", "unknown")),
"description": (agent.get("meta", {}).get("description", "") or "").split("\n")[0][:200],
"category": _guess_category(agent.get("meta", {}).get("tags", [])),
"categoryLabel": "",
"source": source_label,
"tags": agent.get("meta", {}).get("tags", []),
"platforms": [],
"author": agent.get("author", ""),
"version": "",
})
continue

if isinstance(data, list):
for entry in data:
if not isinstance(entry, dict) or not entry.get("name"):
continue
if "skills" in entry and isinstance(entry["skills"], list):
continue
skills.append({
"name": entry.get("name", ""),
"description": entry.get("description", ""),
"category": "uncategorized",
"categoryLabel": "",
"source": source_label,
"tags": entry.get("tags", []),
"platforms": [],
"author": "",
"version": "",
})

for s in skills:
if not s["categoryLabel"]:
s["categoryLabel"] = CATEGORY_LABELS.get(
s["category"],
s["category"].replace("-", " ").title() if s["category"] else "Uncategorized",
)

return skills

TAG_TO_CATEGORY = {}
for _cat, _tags in {
"software-development": [
"programming", "code", "coding", "software-development",
"frontend-development", "backend-development", "web-development",
"react", "python", "typescript", "java", "rust", "cli",
"developer-tools", "development", "api", "database", "debugging",
"documentation", "testing", "test", "architecture",
],
"autonomous-ai-agents": [
"ai", "agent", "agents", "ai-agent", "ai-agents", "agentic",
"agentic-ai", "ai-assistant", "assistant", "multi-agent",
"autonomous", "llm", "rag", "prompt", "prompts", "a2a", "acp",
],
"creative": [
"writing", "design", "creative", "art", "image-generation",
"image", "content", "video-editing", "content-creation",
],
"research": ["education", "academic", "academic-writing", "research", "knowledge"],
"social-media": ["marketing", "seo", "social-media", "advertising", "creator"],
"productivity": [
"productivity", "business", "automation", "calendar", "email",
"document", "documents", "office", "notes", "note-taking",
"collaboration", "workflow", "crm",
],
"data-science": ["data", "data-science", "analytics", "analysis", "visualization"],
"mlops": ["machine-learning", "deep-learning", "mlops", "training", "fine-tuning"],
"devops": ["devops", "docker", "kubernetes", "infrastructure", "deployment", "monitoring", "ci-cd"],
"gaming": ["gaming", "game", "game-development"],
"media": ["music", "media", "video", "audio", "podcast", "youtube"],
"health": ["health", "fitness", "medical", "wellness"],
"translation": ["translation", "language-learning", "i18n", "localization"],
"security": ["security", "cybersecurity", "auth", "compliance", "audit", "privacy"],
"blockchain": [
"blockchain", "crypto", "cryptocurrency", "defi", "web3",
"bitcoin", "ethereum", "nft", "trading", "arbitrage",
],
"communication": ["communication", "chat", "messaging", "slack", "discord"],
"domain": [
"finance", "accounting", "banking", "ecommerce", "e-commerce",
"shopping", "travel", "booking", "real-estate", "legal",
"government", "b2b", "b2b-sales", "entrepreneur", "budget",
],
}.items():
for _t in _tags:
TAG_TO_CATEGORY[_t] = _cat

def _guess_category(tags: list) -> str:
"""Map a skill's tags to a curated category, or 'uncategorized'.

Previously this fell back to ``tags[0]`` verbatim, which produced
hundreds of junk one-off "categories" in the sidebar (e.g.
"Doramagic Crystal", "0.10.7 Dev", "Ap2") — version strings, brand
names, and tag noise. We now ONLY accept categories that map to a
known curated bucket; everything else becomes "uncategorized", which
_consolidate_small_categories folds into "Other". Sidecar-declared
categories (skills.sh groupings) bypass this entirely via fixedCategory.
"""
if not tags:
return "uncategorized"
for tag in tags:
if not isinstance(tag, str):
continue
cat = TAG_TO_CATEGORY.get(tag.lower())
if cat:
return cat
# Also accept a tag that's already a known curated category key
# (e.g. a skill tagged literally "security" or "devops").
normalized = tag.lower().replace(" ", "-")
if normalized in CATEGORY_LABELS and normalized != "other":
return normalized
return "uncategorized"

MIN_CATEGORY_SIZE = 4

def _consolidate_small_categories(skills: list) -> list:
for s in skills:
if s["category"] in {"uncategorized", ""}:
s["category"] = "other"
s["categoryLabel"] = "Other"

# Skills with a sidecar-declared category (skills.sh.json grouping) keep
# their category even if it's the only skill in it — the tap explicitly
# chose that label, so it's not a heuristic guess to collapse away.
counts = Counter(
s["category"] for s in skills if not s.get("fixedCategory")
)
small_cats = {cat for cat, n in counts.items() if n < MIN_CATEGORY_SIZE}

for s in skills:
if s.get("fixedCategory"):
continue
if s["category"] in small_cats:
s["category"] = "other"
s["categoryLabel"] = "Other"
elif not s["categoryLabel"]:
s["categoryLabel"] = CATEGORY_LABELS.get(
s["category"],
s["category"].replace("-", " ").title() if s["category"] else "Uncategorized",
)

return skills

def main():
local = extract_local_skills()

unified, index_meta = extract_unified_index_skills()
if unified is not None:
external = unified
external_source = "unified index"
else:
external = extract_legacy_cache_skills()
external_source = "legacy index-cache"
index_meta = None
print(
f"[extract-skills] WARNING: unified index not found at "
f"{UNIFIED_INDEX_PATH}; falling back to {external_source}. "
f"Run `python3 scripts/build_skills_index.py` to refresh."
)

all_skills = _consolidate_small_categories(local + external)

source_order = {"built-in": 0, "optional": 1}
all_skills.sort(key=lambda s: (
source_order.get(s["source"], 2),
1 if s["category"] == "other" else 0,
s["category"],
s["name"],
))

os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
with open(OUTPUT, "w", encoding="utf-8") as f:
# Minified — file is served over the wire, not read by humans.
# At 50k+ skills the indented version was ~30% larger.
json.dump(all_skills, f, separators=(",", ":"), ensure_ascii=False)

# Sidecar meta file so the page can render a "Last refreshed" badge
# without changing the shape of skills.json.
by_source = Counter(s["source"] for s in all_skills)
meta = {
"extractedAt": datetime.now(timezone.utc).isoformat(),
"totalSkills": len(all_skills),
"localSkills": len(local),
"externalSkills": len(external),
"externalSource": external_source,
"bySource": dict(by_source.most_common()),
}
if index_meta:
meta.update(index_meta)
with open(META_OUTPUT, "w", encoding="utf-8") as f:
json.dump(meta, f, separators=(",", ":"), ensure_ascii=False)

print(f"Extracted {len(all_skills)} skills to {OUTPUT}")
print(f" {len(local)} local ({sum(1 for s in local if s['source'] == 'built-in')} built-in, "
f"{sum(1 for s in local if s['source'] == 'optional')} optional)")
print(f" {len(external)} from {external_source}")

print("By source:")
for src, count in by_source.most_common():
print(f" {src}: {count}")
if index_meta and index_meta.get("indexGeneratedAt"):
print(f"Unified index built at: {index_meta['indexGeneratedAt']}")

if __name__ == "__main__":
main()}}