# LKGetaways Robots.txt
# Last Updated: 2026-06-07

# Default: allow all crawlers, block admin and system paths
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /tmp/
Disallow: /cgi-bin/
Disallow: /instance/
Disallow: /migrations/
Disallow: /*.php$
Disallow: /*/wp-admin/
Disallow: /*/wp-includes/
Disallow: /*.log$
Disallow: /*.py$
Disallow: /*.db$
Disallow: /*.fcgi$
Disallow: /requirements.txt
Disallow: /thank-you
Disallow: /thank-you-bus

# Rate limiting for aggressive SEO crawlers
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

# Explicitly allow AI search crawlers (override any future restrictive rules)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Googlebot-Extended
Allow: /

# XML Sitemap
Sitemap: https://lkgetaways.com/sitemap.xml