# robots.txt — auto-generated from CMS settings

User-agent: *
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# AI Crawlers

# Common Crawl (feeds many LLMs)
User-agent: CCBot
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# OpenAI / ChatGPT training
User-agent: GPTBot
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# Cohere LLM
User-agent: cohere-ai
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# ByteDance / TikTok AI
User-agent: Bytespider
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# Meta AI
User-agent: FacebookBot
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# ChatGPT browsing mode
User-agent: ChatGPT-User
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# Google Gemini AI training
User-agent: Google-Extended
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# Apple Intelligence / Siri
User-agent: Applebot-Extended
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# Search Engines

# Bing Search
User-agent: Bingbot
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# Google Search
User-agent: Googlebot
Allow: /
Disallow: /noindex/
Disallow: /api/
Disallow: /thank-you
Disallow: /seo-settings-general.json
Disallow: /seo-settings-schema.json
Disallow: /seo-extraction.json

# Sitemap
Sitemap: https://www.newcastlenumiscan.co.uk/sitemap.xml

# LLM Information Files
# Summary: https://www.newcastlenumiscan.co.uk/llms.txt
# Detailed: https://www.newcastlenumiscan.co.uk/llms-full.txt