User-agent: *   # directed to all robots
Disallow: /App_Code
Disallow: /App_Data
Disallow: /App_WebReferences
Disallow: /aspnet_client
Disallow: /Bin
Disallow: /BlogAdmin
Disallow: /CacheManager
Disallow: /images
Disallow: /AdmapEdition
Disallow: /MarketLeaderEdition
Disallow: /Offers
Disallow: /Player
Disallow: /Services
Disallow: /Test
Disallow: /Security
Disallow: /FullText
Disallow: /Search?q=
SITEMAP: https://www.warc.com/sitemap/sitemap.xml

# LLMs
User-agent: AhrefsBot
Disallow: /
User-Agent: AI-Ingest
Disallow: /
User-agent: Anthropic-ai
Disallow: /
User-agent: AwarioRssBot
Disallow: /
User-agent: AwarioSmartBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: Cohere-ai
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: FriendlyCrawler
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: img2dataset
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: Magpie-crawler
Disallow: /
User-agent: Omgili
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: Peer39_crawler
Disallow: /
User-agent: Peer39_crawler/1.0
Disallow: /