fix: simplify robots.txt to valid standard directives

frontend/robots.txt

@@ -1,49 +1,6 @@
-# MC Cars - Robots Configuration
-# For demo.lago.dev domain
-
 User-agent: *
-Allow: /
-Allow: /index.html
-Allow: /agb.html
-Allow: /datenschutz.html
-Allow: /impressum.html
-Allow: /mietbedingungen.html
-Allow: /styles.css
-Allow: /images/
-
-# Disallow admin panel from search engines
 Disallow: /admin.html
 Disallow: /admin/
 Disallow: /config.js
 
-# Prevent crawling of query strings (pagination, filters, etc.)
+Sitemap: https://www.mc-cars.at/sitemap.xml
-Disallow: /*?*
-
-# Sitemap location
-Sitemap: https://demo.lago.dev/sitemap.xml
-
-# Crawl delay for responsible crawling (optional)
-Crawl-delay: 1
-
-# Request rate: 1 request per second
-Request-rate: 1/1s
-
-# Specific rules for Google
-User-agent: Googlebot
-Allow: /
-Disallow: /admin.html
-Crawl-delay: 0
-
-# Specific rules for Bing
-User-agent: Bingbot
-Allow: /
-Disallow: /admin.html
-Crawl-delay: 1
-
-# Block bad bots and scrapers
-User-agent: AhrefsBot
-User-agent: SemrushBot
-User-agent: DotBot
-User-agent: MJ12bot
-User-agent: AiHitBot
-Disallow: /