# =====================================================
# All East Bay Properties robots.txt (2025)
# AI + Search Engine Crawler Whitelist
# Blocks sensitive paths, allows everything else
# =====================================================

# --- BLOCK SENSITIVE PATHS FOR *ALL* BOTS --------
# Must come FIRST – robots.txt is parsed top-down
User-agent: *
Allow: /
Disallow: /stage
Disallow: /test
Disallow: /xmlrpc/
Disallow: /wp-admin

# Ensure AI / LLM metadata files are always accessible
Allow: /ai.txt
Allow: /llms.txt
Allow: /llms-listings.txt
Allow: /llms-listings.json
Allow: /llms-personas.json
Allow: /llms-questions.json
Allow: /.well-known/llms-compressed.txt
Allow: /.well-known/ai-plugin.json
Allow: /wp-json/
Allow: /llms-*.json

# --- ALLOW ALL KNOWN AI BOTS (FULL ACCESS EXCEPT ABOVE) ---
User-agent: Grok
User-agent: xAI
User-agent: Grok-Spider
User-agent: GrokBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OpenAI
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Anthropic
User-agent: Google-Extended
User-agent: GeminiBot
User-agent: GoogleOther
User-agent: PerplexityBot
User-agent: Perplexity
User-agent: Cohere
User-agent: CohereBot
User-agent: MistralBot
User-agent: LeChat
User-agent: Meta-ExternalAgent
User-agent: LlamaBot
User-agent: Firecrawl
User-agent: llms-txt-crawler
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: YouBot
Allow: /
Disallow: /stage
Disallow: /test
Disallow: /xmlrpc/
Disallow: /wp-admin

# --- ALLOW MAJOR SEARCH ENGINE CRAWLERS -------------
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Bingbot
Crawl-delay: 10
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: Sogou Spider
Allow: /
Disallow: /stage
Disallow: /test
Disallow: /xmlrpc/
Disallow: /wp-admin

# --- ALLOW OTHER NOTABLE CRAWLERS (OPTIONAL) --------
User-agent: Exabot
User-agent: SeznamBot
User-agent: FacebookExternalHit
User-agent: Facebot
User-agent: Twitterbot
User-agent: LinkedInBot
Allow: /
Disallow: /stage
Disallow: /test
Disallow: /xmlrpc/
Disallow: /wp-admin

# --- ENSURE SITEMAPS ARE DISCOVERABLE ----
Sitemap: https://alleastbayproperties.com/sitemap_index.xml
Sitemap: https://alleastbayproperties.com/schema-sitemap.xml

schemamap: https://alleastbayproperties.com/schema-sitemap.xml


# ================================
# AI / LLM RESOURCE DISCOVERY
# These lines are for AI crawlers. Safe for search engines to ignore.
# ================================

# Make schema aggregated ld+json available to AI
SCHEMA-FEED: https://alleastbayproperties.com/wp-json/aebp/v1/schema-map?page=1&per_page=50

# General AI policy file
AI: https://alleastbayproperties.com/ai.txt

# AI catalog / index of site resources
LLMS: https://alleastbayproperties.com/llms.txt
LLMS-COMPRESSED: https://alleastbayproperties.com/.well-known/llms-compressed.txt
LLMS-LIST: https://alleastbayproperties.com/llms-listings.txt
LLMS-SITEMAP: https://alleastbayproperties.com/schema-sitemap.xml
LLMS-PERSONAS: https://alleastbayproperties.com/llms-personas.json
LLMS-QUESTIONS: https://alleastbayproperties.com/llms-questions.json

# OpenAI / Claude / general LLM plugin file
AI-PLUGIN: https://alleastbayproperties.com/.well-known/ai-plugin.json

# PRIMARY LISTINGS SOURCES (for rental search AIs)
AI-Index: https://alleastbayproperties.com/llms-listings.txt
AI-JSON: https://alleastbayproperties.com/llms-listings.json