# Robots.txt for Professional Barber Shop
# Generated: 2025-09-21T00:00:00.000Z
# Contact: (613) 235-0322

# Default access for all crawlers
User-agent: *
Allow: /
Crawl-delay: 1

# Disallow access to admin areas and development files
Disallow: /admin/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /.next/
Disallow: /scripts/
Disallow: /lib/private/
Disallow: /api/private/

# Search Engine Crawlers - Full access
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: YandexBot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# AI Answer Engines - Allowed for business discovery
User-agent: Claude-Web
Allow: /
Crawl-delay: 2

User-agent: ClaudeBot
Allow: /
Crawl-delay: 2

User-agent: PerplexityBot
Allow: /
Crawl-delay: 2

User-agent: YouBot
Allow: /
Crawl-delay: 2

User-agent: SearchGPT
Allow: /
Crawl-delay: 2

User-agent: OpenAI-SearchBot
Allow: /
Crawl-delay: 2

# AI Training Crawlers - Selective blocking
# Remove these blocks if you want to allow AI training on your content
User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: Anthropic-AI
Allow: /
Crawl-delay: 5

# Social Media Crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# SEO and Analytics Tools
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: MJ12bot
Allow: /
Crawl-delay: 10

# Aggressive or unwanted crawlers
User-agent: ia_archiver
Disallow: /

User-agent: ScrapyBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemaps
Sitemap: https://ottawabarber.ca/sitemap.xml 
# Additional information
# Business: Professional Barber Shop Ottawa
# Website: https://ottawabarber.ca
# Contact: (613) 235-0322