# Default rules
User-agent: *
Crawl-delay: 1
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
# Global AI discovery hints
Allow: /llms.txt
Allow: /ai.txt
Allow: /ai.json
Allow: /identity.json

Sitemap: https://www.berger.team/sitemap_index.xml
Sitemap: https://www.berger.team/it/sitemap_index.xml
Sitemap: https://www.berger.team/en/sitemap_index.xml

# Block aggressive scrapers/crawlers
User-agent: AhrefsBot
User-agent: Barkrowler
User-agent: SemrushBot
User-agent: MJ12bot
Disallow: /

# BEGIN WP Agent Gateway

# =========================================
# AI Agent Discovery (WP Agent Gateway)
# =========================================

User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Anthropic
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Gemini
User-agent: Bingbot
User-agent: BingPreview
User-agent: Copilot
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: PerplexityBot
User-agent: cohere-ai
User-agent: MistralBot
User-agent: DeepSeekBot
User-agent: Applebot
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: BraveBot
User-agent: PhindBot
User-agent: CCBot
User-agent: AI2Bot
User-agent: AI-Agent
User-agent: MCP-Client
User-agent: AIWebIndex
User-agent: amazon-kendra
User-agent: Anomura
User-agent: atlassian-bot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Cloudflare-AutoRAG
User-agent: Devin
User-agent: Diffbot-User
User-agent: Google-Agent
User-agent: Kimi-SearchBot
User-agent: KlaviyoAIBot
User-agent: MistralAI-Index
User-agent: PetalBot
User-agent: SemrushBot-SWA
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /llms.html
Allow: /llm.txt
Allow: /ai.txt
Allow: /ai.json
Allow: /brand.txt
Allow: /faq-ai.txt
Allow: /developer-ai.txt
Allow: /robots-ai.txt
Allow: /identity.json
Allow: /.well-known/security.txt
Allow: /wp-json/wp-agent-gateway/

# AI Discovery Resources
# llms.txt:           https://www.berger.team/llms.txt
# llms-full.txt:      https://www.berger.team/llms-full.txt
# Tools API:          https://www.berger.team/wp-json/wp-agent-gateway/v1/tools
# OpenAPI Spec:       https://www.berger.team/wp-json/wp-agent-gateway/v1/openapi.json
# Content Search:     https://www.berger.team/wp-json/wp-agent-gateway/v1/search?q={query}
# Single Content:     https://www.berger.team/wp-json/wp-agent-gateway/v1/content/{id}
# FAQ Database:       https://www.berger.team/wp-json/wp-agent-gateway/v1/faq-database
# Contact:            https://www.berger.team/wp-json/wp-agent-gateway/v1/contact
# Brand Guidelines:   https://www.berger.team/brand.txt
# Identity (JSON):    https://www.berger.team/identity.json
# LLMs HTML:          https://www.berger.team/llms.html
# AI Policy (JSON):   https://www.berger.team/ai.json
# FAQ AI:             https://www.berger.team/faq-ai.txt
# Developer AI:       https://www.berger.team/developer-ai.txt
# Robots AI:          https://www.berger.team/robots-ai.txt

# END WP Agent Gateway