Lago
3fb0369367
- Added meta tags for SEO optimization across multiple pages including descriptions, robots directives, and Open Graph tags. - Implemented JSON-LD structured data for better search visibility and rich snippets. - Configured nginx for improved performance with gzip compression, caching strategies, and security headers. - Created robots.txt and sitemap.xml for better search engine indexing and crawl management. - Developed comprehensive SEO documentation including a guide and quick reference for ongoing optimization efforts.
50 lines
941 B
Plaintext
50 lines
941 B
Plaintext
# MC Cars - Robots Configuration
|
|
# For demo.lago.dev domain
|
|
|
|
User-agent: *
|
|
Allow: /
|
|
Allow: /index.html
|
|
Allow: /agb.html
|
|
Allow: /datenschutz.html
|
|
Allow: /impressum.html
|
|
Allow: /mietbedingungen.html
|
|
Allow: /styles.css
|
|
Allow: /images/
|
|
|
|
# Disallow admin panel from search engines
|
|
Disallow: /admin.html
|
|
Disallow: /admin/
|
|
Disallow: /config.js
|
|
|
|
# Prevent crawling of query strings (pagination, filters, etc.)
|
|
Disallow: /*?*
|
|
|
|
# Sitemap location
|
|
Sitemap: https://demo.lago.dev/sitemap.xml
|
|
|
|
# Crawl delay for responsible crawling (optional)
|
|
Crawl-delay: 1
|
|
|
|
# Request rate: 1 request per second
|
|
Request-rate: 1/1s
|
|
|
|
# Specific rules for Google
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
Disallow: /admin.html
|
|
Crawl-delay: 0
|
|
|
|
# Specific rules for Bing
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
Disallow: /admin.html
|
|
Crawl-delay: 1
|
|
|
|
# Block bad bots and scrapers
|
|
User-agent: AhrefsBot
|
|
User-agent: SemrushBot
|
|
User-agent: DotBot
|
|
User-agent: MJ12bot
|
|
User-agent: AiHitBot
|
|
Disallow: /
|