# robots.txt for StartupStage (startupstage.com)
# SEO + crawler directives | Last updated: April 8, 2026

########################################
# DEFAULT CRAWLER RULES
########################################
User-agent: *
Allow: /

# Admin & system paths
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /checkout/
Disallow: /cart/
Disallow: /internal/
Disallow: /401
Disallow: /404
Disallow: /cet4t2ha17bwx7bav4dqjfzhj2g5mft7

# Block error/toast state URLs (these create junk crawl-budget waste)
Disallow: /*?toast=
Disallow: /*&toast=

# Block stacked SearchAction template parameter URLs
# These were generated by a SearchAction JSON-LD schema and are not real pages
Disallow: /*?q%3D%7B
Disallow: /*&q%3D%7B

# Block duplicate comment parameter pages (canonical URL is without this param)
Disallow: /*?comments=
Disallow: /*&comments=

# Block pagination parameters that create thin/duplicate content
Disallow: /*?0cb279a2_page=
Disallow: /*&0cb279a2_page=
Disallow: /*?page=
Disallow: /*&page=

Crawl-delay: 1

########################################
# AI / LLM / AGENT CRAWLERS (FULL ACCESS)
########################################

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

# Apple Intelligence & Siri Knowledge
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Meta / Facebook AI
User-agent: FacebookBot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# TikTok / ByteDance AI
User-agent: Bytespider
Allow: /

########################################
# SITEMAPS
########################################
Sitemap: https://startupstage.com/sitemap.xml

Sitemap: https://startupstage.com/sitemap.xml