# robots.txt for index.zshipu.com

User-agent: *
Allow: /

# 禁止索引的目录
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /scripts/

# 搜索结果页：21 个垂类各有一个 /search/，本身是导航型页面、没有发布商内容。
# Google 明确建议不要让站内搜索结果页进入索引，放开抓取只会产出一批
# 「没有内容的屏幕」，与 AdSense 的广告资源价值条款直接冲突。
Disallow: /*/search/



# 允许CSS和JS（利于渲染）
Allow: /css/
Allow: /js/
Allow: /images/

# Sitemap位置
Sitemap: https://index.zshipu.com/sitemap.xml

# 爬取延迟（可选）
Crawl-delay: 1

# webagent:ai-crawlers
# 由 webagent 维护：AI 抓取白名单与各垂类 sitemap。
# robots.txt 只在域名根生效，子路径副本仅供各垂类自查。

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: CCBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Bytespider
Allow: /

# 各垂类 sitemap
Sitemap: https://index.zshipu.com/ai/sitemap.xml
Sitemap: https://index.zshipu.com/ai001/sitemap.xml
Sitemap: https://index.zshipu.com/ai002/sitemap.xml
Sitemap: https://index.zshipu.com/dchat/sitemap.xml
Sitemap: https://index.zshipu.com/ecg/sitemap.xml
Sitemap: https://index.zshipu.com/ecg001/sitemap.xml
Sitemap: https://index.zshipu.com/edudaily/sitemap.xml
Sitemap: https://index.zshipu.com/geek/sitemap.xml
Sitemap: https://index.zshipu.com/geek001/sitemap.xml
Sitemap: https://index.zshipu.com/geek002/sitemap.xml
Sitemap: https://index.zshipu.com/go/sitemap.xml
Sitemap: https://index.zshipu.com/gpt/sitemap.xml
Sitemap: https://index.zshipu.com/idea-of-the-day/sitemap.xml
Sitemap: https://index.zshipu.com/prompt-gallery/sitemap.xml
Sitemap: https://index.zshipu.com/stock/sitemap.xml
Sitemap: https://index.zshipu.com/stock001/sitemap.xml
Sitemap: https://index.zshipu.com/stock002/sitemap.xml
Sitemap: https://index.zshipu.com/stock003/sitemap.xml
Sitemap: https://index.zshipu.com/stocktactics/sitemap.xml
Sitemap: https://index.zshipu.com/weekly/sitemap.xml