# Bots blokkeren (overlast/spam)
User-agent: AhrefsBot
User-agent: AlphaSeoBot
User-agent: Amazonbot
User-agent: Baiduspider
User-agent: BLEXBot
User-agent: Bingbot
User-agent: BingPreview
User-agent: CazoodleBot
User-agent: coccocbot
User-agent: coccocbot-web
User-agent: coccocbot-image
User-agent: DataForSeoBot
User-agent: DoCoMo
User-agent: DotBot
User-agent: Exabot
User-agent: Gigabot
User-agent: infoobot
User-agent: Mail.RU_Bot
User-agent: MJ12bot
User-agent: MojeekBot
User-agent: MSLBot
User-agent: Nimbostratus-Bot
User-agent: Orbbot
User-agent: PetalBot
User-agent: pricebot
User-agent: Purebot
User-agent: SBIder
User-agent: SBIder/Nutch-1
User-agent: Scrapy
User-agent: SeekportBot
User-agent: Seekport Crawler
User-agent: SEBot-WA
User-agent: SemrushBot
User-agent: SEOkicks
User-agent: SEOkicks-Robot
User-agent: serpstatbot
User-agent: SeznamBot
User-agent: SiteExplorer
User-agent: Slurp
User-agent: Slurp Bot
User-agent: SMTBot
User-agent: Sosospider
User-agent: Sosospider+
User-agent: Streamline3Bot
User-agent: TurnitinBot
User-agent: TwengaBot
User-agent: Vagabondo
User-agent: VelenPublicWebCrawler
User-agent: Yahoo
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexImages
User-agent: ZoomInfo
User-agent: ZoominfoBot
Disallow: /

# Toegestane belangrijke zoekmachines
User-agent: Googlebot
User-agent: Googlebot-image
User-agent: GoogleBot-News
User-agent: GoogleBot-News-Video
User-agent: GoogleBot-Video
User-agent: AdsBot-Google
User-agent: GoogleOther
User-agent: Applebot
User-agent: DuckDuckBot
Disallow:

#Social media
User-agent: facebot
User-agent: facebookexternalhit
User-agent: FacebookBot
User-agent: InstagramBot
User-agent: Pinterestbot
User-agent: LinkedInBot
User-agent: TikTokbot
User-agent: Twitterbot
Disallow:
Crawl-delay: 20

# AI-crawlers toestaan met delay
User-agent: Anthropic
User-agent: Anthropic-API
User-agent: anthropic-ai
User-agent: Bard
User-agent: ChatGPT
User-agent: ChatGPTBot
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: CharacterAI
User-agent: CharacterAI-Bot
User-agent: Claude
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Cohere
User-agent: Google-Extended
User-agent: GPTBot
User-agent: HuggingFace
User-agent: Jasper
User-agent: JasperBot
User-agent: LLaMA
User-agent: Mistral
User-agent: MosaicML
User-agent: Neeva
User-agent: NeevaBot
User-agent: OAI-SearchBot
User-agent: OpenAI
User-agent: OpenAI-API
User-agent: PerplexityBot
User-agent: Poe
User-agent: PoeBot
User-agent: YouChat
User-agent: YouChatBot
User-agent: Grok
Disallow: 
Crawl-delay: 20

# Algemene blokkades (CS-Cart / duplicate content / beveiliging)
User-agent: *
Disallow: /app/
Disallow: /store_closed.html


# Sitemap URL toevoegen (voor zoekmachines)
Sitemap: https://www.savannahbay.nl/sitemap.xml
Sitemap: https://www.savannahbay.nl/shop/sitemap.xml