# Blocks crawlers that are kind enough to obey robots
User-agent: AhrefsBot
User-agent: BUbiNG
User-agent: Bloglines/3.1
User-agent: BrandVerity
User-agent: Bytespider
User-agent: CrazyWebCrawler-Spider
User-agent: DeepSeekBot
User-agent: DOC
User-agent: Domain Re-Animator Bot
User-agent: Download Ninja
User-agent: Exabot
User-agent: Fetch
User-agent: HTTrack
User-agent: Jyxobot/1
User-agent: Linguee
User-agent: MSIECrawler
User-agent: MauiBot
User-agent: Microsoft.URL.Control
User-agent: NPBot
User-agent: Offline Explorer
User-agent: PetalBot
User-agent: SeekportBot
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SiteSnagger
User-agent: Speedy
User-agent: Teleport
User-agent: TeleportPro
User-agent: UbiCrawler
User-agent: Vegi
User-agent: WebCopier
User-agent: WebReaper
User-agent: WebStripper
User-agent: WebZIP
User-agent: Xenu
User-agent: Zao
User-agent: Zealbot
User-agent: ZyBORG
User-agent: cityreview
User-agent: dotbot
User-agent: grub-client
User-agent: k2spider
User-agent: larbin
User-agent: libwww
User-agent: linko
User-agent: psbot
User-agent: rogerbot
User-agent: sitecheck.internetseer.com
User-agent: wget
Disallow: /

User-agent: *
Allow: /*?vsn=d$
Allow: /sitemap/*?page=
Allow: /directory/*?page=
Allow: /blog?page=
Disallow: /*?*
Disallow: /cdn-cgi/
Disallow: /x/*
Disallow: /sem/