# MC Cars - Robots Configuration
# For demo.lago.dev domain

User-agent: *
Allow: /
Allow: /index.html
Allow: /agb.html
Allow: /datenschutz.html
Allow: /impressum.html
Allow: /mietbedingungen.html
Allow: /styles.css
Allow: /images/

# Disallow admin panel from search engines
Disallow: /admin.html
Disallow: /admin/
Disallow: /config.js

# Prevent crawling of query strings (pagination, filters, etc.)
Disallow: /*?*

# Sitemap location
Sitemap: https://demo.lago.dev/sitemap.xml

# Crawl delay for responsible crawling (optional)
Crawl-delay: 1

# Request rate: 1 request per second
Request-rate: 1/1s

# Specific rules for Google
User-agent: Googlebot
Allow: /
Disallow: /admin.html
Crawl-delay: 0

# Specific rules for Bing
User-agent: Bingbot
Allow: /
Disallow: /admin.html
Crawl-delay: 1

# Block bad bots and scrapers
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: AiHitBot
Disallow: /