# Cyprus Business Review
# Search engines are welcome, and so are the assistants that answer a question
# by citing a source and linking to it. Bulk collection for training models is
# not, which is the same position the copyright page takes in prose:
# https://cyprusbusinessreview.com/copyright
#
# The distinction matters because "AI crawler" covers two different jobs. One
# reads a page in order to quote it and send the reader here. The other copies
# it into a training set and the reader never arrives. The first is how this
# publication gets found; the second is how it gets absorbed.
#
# /subscribed is deliberately NOT disallowed here. It carries a noindex tag,
# and a crawler has to be allowed to fetch a page in order to read that tag.

User-agent: *
Allow: /

Sitemap: https://cyprusbusinessreview.com/sitemap.xml

# Retrieval and citation. These fetch a page to answer a live question and
# name the source in the answer, which is the same bargain as a search engine.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Bulk text collection for training generative models. Honouring robots.txt is
# voluntary, so this is a stated position rather than an enforcement mechanism.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /