# Austin Locksmiths - Robots.txt
# Last updated: 2026-02-08

User-agent: *
Allow: /

# Block non-indexable pages
Disallow: /thank-you
Disallow: /thank-you/

# Block old WordPress structures (now redirected)
Disallow: /admin/
Disallow: /private/
Disallow: /author/
Disallow: /feed/
Disallow: /*/feed/
Disallow: /blog/page/

# Block spam URLs (legacy hacked content)
Disallow: /*casino*
Disallow: /casino-
Disallow: /-casino

# ALLOW tracking parameters (critical for Google Ads attribution)
Allow: /*?utm_*
Allow: /*?gclid=*
Allow: /*?fbclid=*

# Block dynamic parameters that create duplicate content
Disallow: /*?sid=*
Disallow: /*?session*
Disallow: /*?sort=*

# AI Search Bots - ALLOW for local business visibility
# When users ask AI "find a locksmith near me" we need to be cited
User-agent: GPTBot
Allow: /
Disallow: /private/

User-agent: Google-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Throttle aggressive SEO tool crawlers
User-agent: SemrushBot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

# Sitemaps
Sitemap: https://austinlocksmiths.com/sitemap-index.xml

# SEO by Optymizer | https://optymizer.com