# AI Crawlers
 
User-agent: GPTBot
Disallow: /
 
User-agent: ChatGPT-User
Disallow: /
 
User-agent: ClaudeBot
Disallow: /
 
User-agent: Claude-Web
Disallow: /
 
User-agent: CCBot
Disallow: /
 
User-agent: Bytespider
Disallow: /
 
User-agent: Amazonbot
Disallow: /
 
User-agent: PerplexityBot
Disallow: /
 
User-agent: Meta-ExternalAgent
Disallow: /
 
User-agent: Meta-ExternalFetcher
Disallow: /
 
# General rules
 
User-agent: *
 
# Magento system directories
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /pub/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
 
# Customer areas
Disallow: /customer/
Disallow: /checkout/
Disallow: /wishlist/
Disallow: /review/
Disallow: /sales/
Disallow: /catalog/product_compare/
 
# Search
Disallow: /catalogsearch/
Disallow: /search/
 
# APIs
Disallow: /graphql
Disallow: /rest/
Disallow: /media/customer/
 
# Block all URLs with query strings
Disallow: /*?*
 
# Allow canonical product/category pages
Allow: /*.html$

Sitemap: https://insightcommodity.com/sitemap.xml