# robots.txt for https://algoleap.com/
# Last reviewed: 2026-09-13
#
# Section 1 of our Terms and Conditions reserves all rights in respect of
# text and data mining. This file is the machine-readable expression of that
# reservation for the purposes of Article 4(3) of Directive (EU) 2019/790.
# https://algoleap.com/termsandconditions.html

# ---------------------------------------------------------------------------
# 1. General crawling - search engines are welcome
# ---------------------------------------------------------------------------

User-agent: *
Allow: /

# ---------------------------------------------------------------------------
# 2. Text and data mining reservation (EU DSM Directive, Art. 4(3))
#    Reproduction or extraction of any content on this site for the purposes
#    of text and data mining, or for training, fine-tuning or evaluating any
#    machine learning or generative AI system, is expressly reserved and is
#    not permitted without prior written licence from Algoleap.
# ---------------------------------------------------------------------------

# --- AI training crawlers (model training corpora) ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Scrapy
Disallow: /

# --- Commercial SEO / content scrapers ---
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# ---------------------------------------------------------------------------
# 3. AI assistants that fetch a page live to answer a user's question.
#    These drive referral traffic and do NOT add pages to a training corpus.
#    We allow them. If you would rather not appear in AI answers at all,
#    change each Allow to Disallow.
# ---------------------------------------------------------------------------

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Googlebot
Allow: /

# ---------------------------------------------------------------------------
# 4. Sitemap
# ---------------------------------------------------------------------------

Sitemap: https://algoleap.com/sitemap.xml