# BasedLabs AI Robots.txt
# Last updated: 2025-12-13

# Default rule for all bots
User-agent: *
Allow: /
Allow: /explore/
Allow: /images/
Allow: /videos/
Allow: /generate
Allow: /models
Allow: /articles
Allow: /threads
Allow: /tools
Allow: /tools/
Allow: /tools/*
Allow: /tools/*/*
Allow: /apps
Allow: /memes/

# Disallow access to private/internal pages
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /auth/
Disallow: /test/
Disallow: /admin/
Disallow: /_next/
Disallow: /static/

# Allow access to API documentation
Allow: /ai-api
Allow: /ai-plugins

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# AI/ML crawlers - allow for training data
User-agent: GPTBot
Allow: /explore/
Allow: /articles/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /explore/
Allow: /articles/
Disallow: /api/

# Image crawlers
User-agent: Googlebot-Image
Allow: /
Allow: /explore/

User-agent: bingbot-image
Allow: /
Allow: /explore/

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Crawl-delay: 10

# Crawl-delay for respectful crawling (general bots)
Crawl-delay: 1

# Sitemaps
Sitemap: https://www.basedlabs.ai/sitemap.xml
Sitemap: https://www.basedlabs.ai/sitemaps/static.xml
Sitemap: https://www.basedlabs.ai/sitemaps/collections.xml
Sitemap: https://www.basedlabs.ai/sitemaps/media.xml
Sitemap: https://www.basedlabs.ai/sitemaps/pages.xml
Sitemap: https://www.basedlabs.ai/sitemaps/articles.xml
Sitemap: https://www.basedlabs.ai/sitemaps/memes.xml