# Searlixa — robots.txt
# AI crawler policy is a deliberate decision, not a default. See /company/methodology/

# --- Search + AI answer discovery: allowed ---
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

# --- Model training: separate choice from search discovery ---
# GPTBot is a training crawler, not the ChatGPT Search crawler.
# Decision required from the project owner before launch. Default below is permissive
# so that discovery is never blocked by accident; flip to Disallow if training is opted out.
User-agent: GPTBot
Allow: /

# Applebot serves Siri and Spotlight. Applebot-Extended controls Apple Intelligence
# training separately, the same split as OAI-SearchBot and GPTBot.
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

# --- Bulk corpus crawlers: feed many downstream training sets ---
# CCBot is Common Crawl. Its archive is used as source data by a large number of
# model builders, so allowing it has wider reach than allowing any single vendor.
# Same reasoning as GPTBot: permissive by default, flip to Disallow to opt out.
User-agent: CCBot
Allow: /

User-agent: *
Allow: /

# Upload helper files — should be deleted from the server after setup
Disallow: /form-handler.php
Disallow: /company/contact/thank-you/
Disallow: /UPLOAD-INSTRUCTIONS.txt
Disallow: /htaccess-RENAME-TO-DOT-htaccess.txt

Sitemap: https://searlixa.com/sitemap.xml