# robots.txt for 西祠胡同广告营销平台
# https://www.xici.net

# 允许所有搜索引擎爬取营销官网
User-agent: *
Allow: /marketing/
Allow: /marketing/products/
Allow: /marketing/solutions/
Allow: /marketing/cases
Allow: /marketing/help
Allow: /marketing/about

# 禁止爬取用户私密信息和后台管理页面
Disallow: /profile/
Disallow: /admin/
Disallow: /api/
Disallow: /user/
Disallow: /dating/chat
Disallow: /complaints/my
Disallow: /complaints/notifications

# 禁止爬取动态生成的搜索结果页
Disallow: /search?
Disallow: /*?*

# 禁止爬取登录和注册页面
Disallow: /login
Disallow: /register

# 禁止爬取购物车和订单页面
Disallow: /cart
Disallow: /checkout
Disallow: /orders

# 允许爬取静态资源
Allow: /images/
Allow: /css/
Allow: /js/

# 网站地图位置
Sitemap: https://www.xici.net/sitemap.xml

# 爬取延迟（秒）- 避免对服务器造成过大压力
Crawl-delay: 1

# 针对特定搜索引擎的规则

# 百度爬虫
User-agent: Baiduspider
Allow: /marketing/
Crawl-delay: 1

# 谷歌爬虫
User-agent: Googlebot
Allow: /marketing/
Crawl-delay: 1

# 搜狗爬虫
User-agent: Sogou web spider
Allow: /marketing/
Crawl-delay: 1

# 360搜索爬虫
User-agent: 360Spider
Allow: /marketing/
Crawl-delay: 1

# 必应爬虫
User-agent: Bingbot
Allow: /marketing/
Crawl-delay: 1

# 禁止恶意爬虫
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /