# Optimized Robots.txt for petplanet.co.il
# Sitemap
Sitemap: https://petplanet.co.il/sitemap.xml

# -------------------------
# Googlebot (normal crawl)
User-agent: Googlebot
Allow: /
Crawl-delay: 5

# -------------------------
# Aggressive bots throttling
User-agent: AdsBot-Google
Crawl-delay: 30

User-agent: SemrushBot
Crawl-delay: 20

User-agent: Bingbot
Crawl-delay: 10

# -------------------------
# Block heavy WooCommerce pages for all bots
User-agent: *
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# General crawl delay for other bots
User-agent: *
Crawl-delay: 10