# robots.txt for iamarunraj.com
# Last updated: April 2026

# ============================
# General search engine crawlers
# ============================
User-agent: *
Allow: /
Disallow: /private/

# ============================
# Sitemap location
# ============================
Sitemap: https://iamarunraj.com/sitemap.xml

# ============================
# Major search engines (allowed)
# ============================
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# ============================
# AI training crawlers
# Default position: NOT permitted without explicit consent.
# See /content-usage.html and /llms.txt for details.
# ============================

# OpenAI / ChatGPT training crawler
User-agent: GPTBot
Disallow: /

# OpenAI search crawler (allowed for citation/discovery)
User-agent: OAI-SearchBot
Allow: /

# Anthropic / Claude training crawler
User-agent: ClaudeBot
Disallow: /

# Anthropic / Claude user-triggered fetch (allowed)
User-agent: Claude-User
Allow: /

# Google AI training crawler
User-agent: Google-Extended
Disallow: /

# Common Crawl (used by many AI training datasets)
User-agent: CCBot
Disallow: /

# Perplexity bot (allowed for citation)
User-agent: PerplexityBot
Allow: /

# Apple AI crawler
User-agent: Applebot-Extended
Disallow: /

# Meta AI crawler
User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# Bytedance / TikTok crawler
User-agent: Bytespider
Disallow: /

# Amazonbot
User-agent: Amazonbot
Disallow: /

# ============================
# For permission to use this content for AI training,
# email: hello@iamarunraj.com
# ============================