mirror of
https://github.com/nicepkg/ai-workflow.git
synced 2026-11-18 18:04:23 +00:00
313 lines
8.4 KiB
Python
313 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Claude Code 文档爬虫脚本
|
|
将 https://code.claude.com/docs 的文档爬取并转换为 Markdown 文件
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
import requests
|
|
from urllib.parse import urljoin, urlparse
|
|
from bs4 import BeautifulSoup
|
|
from markdownify import markdownify as md
|
|
|
|
# 配置
|
|
BASE_URL = "https://code.claude.com/docs"
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
OUTPUT_DIR = os.path.join(SCRIPT_DIR, "docs")
|
|
LANGUAGE = "en" # 爬取英文文档
|
|
|
|
# 文档页面列表
|
|
DOC_PAGES = [
|
|
# Getting started
|
|
"overview",
|
|
"quickstart",
|
|
"common-workflows",
|
|
"changelog",
|
|
# Outside of the terminal
|
|
"claude-code-on-the-web",
|
|
"desktop",
|
|
"chrome",
|
|
"vs-code",
|
|
"jetbrains",
|
|
"github-actions",
|
|
"gitlab-ci-cd",
|
|
"slack",
|
|
# Build with Claude Code
|
|
"sub-agents",
|
|
"plugins",
|
|
"discover-plugins",
|
|
"skills",
|
|
"output-styles",
|
|
"hooks-guide",
|
|
"headless",
|
|
"mcp",
|
|
"troubleshooting",
|
|
# Deployment
|
|
"third-party-integrations",
|
|
"amazon-bedrock",
|
|
"google-vertex-ai",
|
|
"microsoft-foundry",
|
|
"network-config",
|
|
"llm-gateway",
|
|
"devcontainer",
|
|
"sandboxing",
|
|
# Administration
|
|
"setup",
|
|
"iam",
|
|
"security",
|
|
"data-usage",
|
|
"monitoring-usage",
|
|
"costs",
|
|
"analytics",
|
|
"plugin-marketplaces",
|
|
# Configuration
|
|
"settings",
|
|
"terminal-config",
|
|
"model-config",
|
|
"memory",
|
|
"statusline",
|
|
# Reference
|
|
"cli-reference",
|
|
"interactive-mode",
|
|
"slash-commands",
|
|
"checkpointing",
|
|
"hooks",
|
|
"plugins-reference",
|
|
# Resources
|
|
"legal-and-compliance",
|
|
]
|
|
|
|
# HTTP 请求头
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
}
|
|
|
|
|
|
def fetch_page(url: str) -> str | None:
|
|
"""获取页面 HTML 内容"""
|
|
try:
|
|
response = requests.get(url, headers=HEADERS, timeout=30)
|
|
response.raise_for_status()
|
|
return response.text
|
|
except requests.RequestException as e:
|
|
print(f" ❌ 获取失败: {e}")
|
|
return None
|
|
|
|
|
|
def extract_main_content(html: str) -> tuple[str, str]:
|
|
"""从 HTML 中提取主要内容区域,返回 (title, content_html)"""
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
# 提取标题
|
|
title = ""
|
|
title_tag = soup.find("h1")
|
|
if title_tag:
|
|
title = title_tag.get_text(strip=True)
|
|
|
|
# 尝试找到主内容区域 - 通常在 article 或 main 标签中
|
|
main_content = None
|
|
|
|
# 尝试不同的选择器
|
|
selectors = [
|
|
"article",
|
|
"main",
|
|
"[role='main']",
|
|
".docs-content",
|
|
".content",
|
|
"#content",
|
|
".markdown-body",
|
|
]
|
|
|
|
for selector in selectors:
|
|
main_content = soup.select_one(selector)
|
|
if main_content:
|
|
break
|
|
|
|
# 如果没找到,使用 body
|
|
if not main_content:
|
|
main_content = soup.find("body")
|
|
|
|
if main_content:
|
|
# 移除不需要的元素
|
|
for element in main_content.select("nav, header, footer, script, style, .sidebar, .toc, .navigation"):
|
|
element.decompose()
|
|
|
|
return title, str(main_content)
|
|
|
|
return title, ""
|
|
|
|
|
|
def html_to_markdown(html: str, base_url: str) -> str:
|
|
"""将 HTML 转换为 Markdown"""
|
|
# 先用 BeautifulSoup 清理 HTML
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
# 移除按钮和交互元素
|
|
for btn in soup.select("button, .copy-button, [data-copy], .ask-ai"):
|
|
btn.decompose()
|
|
|
|
# 移除 "Copy" 和 "Ask AI" 文本节点
|
|
for text_node in soup.find_all(string=re.compile(r"^(Copy|Ask AI)$")):
|
|
if text_node.parent and text_node.parent.name not in ["code", "pre"]:
|
|
text_node.extract()
|
|
|
|
# 修复代码块语言
|
|
for code in soup.select("code"):
|
|
classes = code.get("class", [])
|
|
for cls in classes:
|
|
if "language-shiki" in cls or cls == "shiki":
|
|
# 尝试从父元素或 data 属性获取真实语言
|
|
code["class"] = [c.replace("shiki", "bash") for c in classes]
|
|
|
|
html = str(soup)
|
|
|
|
# 使用 markdownify 转换
|
|
markdown = md(
|
|
html,
|
|
heading_style="ATX",
|
|
bullets="-",
|
|
code_language_callback=lambda el: el.get("class", [""])[0].replace("language-", "") if el.get("class") else "",
|
|
)
|
|
|
|
# 清理 shiki 语言标记
|
|
markdown = re.sub(r"```shiki\b", "```bash", markdown)
|
|
markdown = re.sub(r"```language-shiki\b", "```bash", markdown)
|
|
|
|
# 移除残留的 "Copy" 和 "Ask AI"
|
|
markdown = re.sub(r"\nCopy\n", "\n", markdown)
|
|
markdown = re.sub(r"\nAsk AI\n", "\n", markdown)
|
|
markdown = re.sub(r"^Copy$", "", markdown, flags=re.MULTILINE)
|
|
markdown = re.sub(r"^Ask AI$", "", markdown, flags=re.MULTILINE)
|
|
|
|
# 清理多余的空行
|
|
markdown = re.sub(r"\n{3,}", "\n\n", markdown)
|
|
|
|
# 清理行首行尾空白
|
|
lines = [line.rstrip() for line in markdown.split("\n")]
|
|
markdown = "\n".join(lines)
|
|
|
|
return markdown.strip()
|
|
|
|
|
|
def save_markdown(content: str, filename: str):
|
|
"""保存 Markdown 文件"""
|
|
filepath = os.path.join(OUTPUT_DIR, filename)
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
f.write(content)
|
|
print(f" ✅ 已保存: {filename}")
|
|
|
|
|
|
def crawl_page(page_slug: str) -> bool:
|
|
"""爬取单个页面"""
|
|
url = f"{BASE_URL}/{LANGUAGE}/{page_slug}"
|
|
print(f"\n📄 正在爬取: {page_slug}")
|
|
print(f" URL: {url}")
|
|
|
|
html = fetch_page(url)
|
|
if not html:
|
|
return False
|
|
|
|
title, content_html = extract_main_content(html)
|
|
if not content_html:
|
|
print(" ⚠️ 未找到内容")
|
|
return False
|
|
|
|
markdown = html_to_markdown(content_html, url)
|
|
|
|
# 添加文档头信息
|
|
header = f"""---
|
|
source: {url}
|
|
title: {title}
|
|
---
|
|
|
|
"""
|
|
|
|
full_content = header + markdown
|
|
|
|
# 保存文件
|
|
filename = f"{page_slug}.md"
|
|
save_markdown(full_content, filename)
|
|
|
|
return True
|
|
|
|
|
|
def create_index(successful_pages: list[str]):
|
|
"""创建索引文件"""
|
|
index_content = """---
|
|
title: Claude Code Documentation Index
|
|
---
|
|
|
|
# Claude Code 文档索引
|
|
|
|
此文件夹包含从 https://code.claude.com/docs 爬取的文档。
|
|
|
|
## 文档列表
|
|
|
|
"""
|
|
|
|
categories = {
|
|
"Getting Started": ["overview", "quickstart", "common-workflows", "changelog"],
|
|
"Outside of the Terminal": ["claude-code-on-the-web", "desktop", "chrome", "vs-code", "jetbrains", "github-actions", "gitlab-ci-cd", "slack"],
|
|
"Build with Claude Code": ["sub-agents", "plugins", "discover-plugins", "skills", "output-styles", "hooks-guide", "headless", "mcp", "troubleshooting"],
|
|
"Deployment": ["third-party-integrations", "amazon-bedrock", "google-vertex-ai", "microsoft-foundry", "network-config", "llm-gateway", "devcontainer", "sandboxing"],
|
|
"Administration": ["setup", "iam", "security", "data-usage", "monitoring-usage", "costs", "analytics", "plugin-marketplaces"],
|
|
"Configuration": ["settings", "terminal-config", "model-config", "memory", "statusline"],
|
|
"Reference": ["cli-reference", "interactive-mode", "slash-commands", "checkpointing", "hooks", "plugins-reference"],
|
|
"Resources": ["legal-and-compliance"],
|
|
}
|
|
|
|
for category, pages in categories.items():
|
|
index_content += f"### {category}\n\n"
|
|
for page in pages:
|
|
if page in successful_pages:
|
|
index_content += f"- [{page}](./{page}.md)\n"
|
|
index_content += "\n"
|
|
|
|
save_markdown(index_content, "INDEX.md")
|
|
|
|
|
|
def main():
|
|
"""主函数"""
|
|
# 确保输出目录存在
|
|
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
|
|
|
print("=" * 60)
|
|
print("Claude Code 文档爬虫")
|
|
print("=" * 60)
|
|
print(f"输出目录: {OUTPUT_DIR}")
|
|
print(f"文档数量: {len(DOC_PAGES)}")
|
|
|
|
successful = []
|
|
failed = []
|
|
|
|
for page_slug in DOC_PAGES:
|
|
if crawl_page(page_slug):
|
|
successful.append(page_slug)
|
|
else:
|
|
failed.append(page_slug)
|
|
|
|
# 礼貌性延迟,避免请求过快
|
|
time.sleep(0.5)
|
|
|
|
# 创建索引
|
|
print("\n📑 创建索引文件...")
|
|
create_index(successful)
|
|
|
|
# 打印统计
|
|
print("\n" + "=" * 60)
|
|
print("爬取完成!")
|
|
print(f"成功: {len(successful)}/{len(DOC_PAGES)}")
|
|
|
|
if failed:
|
|
print(f"失败: {failed}")
|
|
|
|
print("=" * 60)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|