# AI training bots: allow marketing/SEO pages, block aggregated job data
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Bytespider
User-agent: anthropic-ai
User-agent: Google-Extended
Allow: /$
Allow: /about
Allow: /blog/
Allow: /privacy
Allow: /terms
Disallow: /assignments/
Disallow: /g/
Disallow: /api/
Disallow: /apply/
Disallow: /i/
Disallow: /search
Disallow: /preview

# AI answer/search bots: explicit read access to assignment pages
# (policy decision 2026-06-10). These bots fetch pages to answer user
# queries and drive our top referrer channel; they are deliberately NOT in
# the training-bot block above. Abuse protection is handled outside
# robots.txt (WAF verified-bot exemption plus search API rate limits), so
# this file only states read intent.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
Allow: /assignments/
Disallow: /g/
Disallow: /api/
Disallow: /apply/
Disallow: /i/
Disallow: /saved
Disallow: /applied
Disallow: /recent
Disallow: /preview

# Regular search engine crawlers. /g/, /apply/ and /i/ are redirect links
# (short links, source postings, invites), not pages.
User-agent: *
Allow: /
Disallow: /g/
Disallow: /api/
Disallow: /apply/
Disallow: /i/
Disallow: /saved
Disallow: /applied
Disallow: /recent
Disallow: /preview

Sitemap: https://consultant.dev/sitemap.xml