# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
# Served for every host, including organizer custom domains. Custom-domain event
# pages are not blocked here — they carry rel="canonical" back to the primary
# host instead, so their signals consolidate onto movemint.cc rather than being
# dropped.
#
# IMPORTANT: a crawler that matches a named User-agent group ignores the `*`
# group completely — robots.txt groups do not merge. The Disallow list below is
# therefore repeated verbatim in the AI crawler group. Change one, change both.
# Deliberately NOT disallowed, though it is tempting to: registration, donation
# and store paths, and /athletes/sign_in. Two reasons. They earn real traffic
# (/athletes/sign_in alone: 725 clicks at average position 4.9; /register pages:
# 226 clicks across 340 URLs). And more fundamentally, a page blocked here can
# never be crawled, so Google never reads its rel="canonical" — blocking and
# canonical consolidation are mutually exclusive. Thin transactional pages get
# pointed at their event page with a canonical, not hidden behind a Disallow.
User-agent: *
Disallow: /event_confirmation
Disallow: /api/
Disallow: /oauth/
Disallow: /participants/*/waivers
Disallow: /volunteers/*/waivers
Disallow: /admin/
Disallow: /*/*/admin/
# AI crawlers are welcome, on the same terms as everyone else. Two families,
# and they behave differently:
#
# Training crawlers — GPTBot, ClaudeBot, Google-Extended, Applebot-Extended.
# Feed model pre-training. Allowing them is what makes
# Movemint part of what a model knows unprompted.
# Retrieval crawlers — OAI-SearchBot, Claude-SearchBot, Claude-User,
# ChatGPT-User, PerplexityBot, Perplexity-User. Fetch
# live pages to answer a question right now. These are
# the ones that produce a citation with a link.
#
# Google-Extended and Applebot-Extended are robots.txt opt-out tokens only —
# they are never sent as a User-Agent, so they will not appear in request logs.
# Every other token here is a real user agent, tracked by CrawlerRequestLogger.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
Disallow: /event_confirmation
Disallow: /api/
Disallow: /oauth/
Disallow: /participants/*/waivers
Disallow: /volunteers/*/waivers
Disallow: /admin/
Disallow: /*/*/admin/
Sitemap: https://www.movemint.cc/sitemap.xml