# ==================================================
# SERVICEAGENT ROBOTS POLICY
# Public marketing + product pages are crawlable.
# Private/internal/system surfaces are blocked.
# /directory and other public marketing routes are crawlable (not disallowed).
# ==================================================

# ==================================================
# AI SEARCH & ASSISTANT ANSWERING (ALLOWED)
# ==================================================
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: YouBot
User-agent: FirecrawlAgent
Allow: /
Allow: /directory
Allow: /directory/
Disallow: /api/
Disallow: /internal/
Disallow: /admin/

# ==================================================
# AI TRAINING / DATA COLLECTION (ALLOWED)
# ==================================================
User-agent: GPTBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: cohere-ai
User-agent: CohereBot
User-agent: MistralAI
User-agent: xAI-Bot
User-agent: HuggingFace
User-agent: HuggingFaceBot
User-agent: OpenRouter
User-agent: TogetherAI
User-agent: StabilityAI
Allow: /
Allow: /directory
Allow: /directory/
Disallow: /api/
Disallow: /internal/
Disallow: /admin/

# ==================================================
# TRADITIONAL SEARCH ENGINES
# ==================================================
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: PetalBot
Allow: /
Allow: /directory
Allow: /directory/
Disallow: /api/
Disallow: /internal/
Disallow: /admin/

# ==================================================
# BLOCK AGGRESSIVE SCRAPERS
# ==================================================
User-agent: Scrapy
Disallow: /

# ==================================================
# GLOBAL RULES (ALL OTHER BOTS)
# ==================================================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /internal/
Disallow: /admin/
Disallow: /_next/
Disallow: /tmp/
Disallow: /test/

# Query parameter hygiene
Disallow: /*?session=
Disallow: /*?sid=
Disallow: /*?token=
Disallow: /*?utm_*
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?ref=

# Explicit allow (directory is indexed; no Disallow above would block it)
Allow: /directory
Allow: /directory/

# ==================================================
# SITEMAPS
# ==================================================
Sitemap: https://serviceagent.ai/sitemap.xml
Sitemap: https://serviceagent.ai/blogs/sitemap.xml
Sitemap: https://serviceagent.ai/directory/sitemap.xml
Agentmap: https://serviceagent.ai/.well-known/ard.json