# robots.txt for flownest.xyz
#
# Two different things are controlled here, and they are easy to confuse:
#
#   TRAINING crawlers  collect pages that may be used to train future models.
#                      Allowing them builds long-term brand knowledge, so when
#                      someone asks an assistant "who does AI automation in
#                      Leeds", FlowNest is part of what the model knows.
#
#   SEARCH / ANSWER    fetch pages live to answer a question right now, and
#   crawlers           cite the source. These produce referral traffic.
#                      Blocking one makes citation impossible on that
#                      assistant, however good the content is.
#
# Both are allowed below. For a services business, being findable is worth
# more than withholding marketing copy already public on the site.
#
# Last reviewed: July 2026. Re-check quarterly, as vendors keep splitting
# and renaming these agents.

# --- Traditional search engines ---
User-agent: *
Allow: /
Disallow: /assets/cube/

# =============================================================
# OpenAI / ChatGPT
# =============================================================

# Training crawler.
User-agent: GPTBot
Allow: /

# Builds the ChatGPT search index. Blocking this removes FlowNest from
# ChatGPT search answers entirely, the costliest mistake in this file.
User-agent: OAI-SearchBot
Allow: /

# Live fetch when a ChatGPT user follows a citation to the site.
User-agent: ChatGPT-User
Allow: /

# =============================================================
# Anthropic / Claude
# =============================================================

# Training crawler.
User-agent: ClaudeBot
Allow: /

# Crawls for the search feature inside Claude.
User-agent: Claude-SearchBot
Allow: /

# Live fetch when a Claude user asks it to visit a page.
User-agent: Claude-User
Allow: /

# =============================================================
# Google / Gemini and AI Overviews
# =============================================================
#
# AI Overviews are served from the normal Google Search index via
# Googlebot, already allowed by the wildcard rule above. Google-Extended
# is a separate control token governing whether that content may also be
# used for Gemini and Vertex AI. It does not affect Search rankings.

User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

# =============================================================
# Perplexity
# =============================================================

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# =============================================================
# Other assistants
# =============================================================

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: MistralAI-User
Allow: /

# --- Bulk scrapers with no citation or referral benefit ---
# Common Crawl is a bulk archive redistributed to third parties. Unlike
# the crawlers above it sends no traffic back and offers no attribution.

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

Sitemap: https://flownest.xyz/sitemap.xml