Robots.txt

Robots.txt Rules

Crawler Rules
All
  • Allow: /
  • Allow: /__origami/
  • Allow: /__assets/
  • Disallow: /__
  • Disallow: /_fs-ch
  • Disallow: /article/e63d8640-eeb3-4473-b0ad-aa6aec16d9fe
  • Disallow: /buy/
  • Disallow: /cancel/
  • Disallow: /join/
  • Disallow: /login
  • Disallow: /myft
  • Disallow: /offline/
  • Disallow: /page-resources
  • Disallow: /preferences/manage-cookies
  • Disallow: /register/
  • Disallow: /search
  • Disallow: /signup
ccbot
  • Allow: /
  • Disallow: /
google-extended
  • Allow: /
  • Allow: /*?syn-25a6b1a6=1
  • Disallow: /
googleother
  • Allow: /
  • Disallow: /
anthropic-ai
  • Allow: /
  • Disallow: /
omgili
  • Allow: /
  • Disallow: /
omgilibot
  • Allow: /
  • Disallow: /
bytespider
  • Allow: /
  • Disallow: /
cohere-ai
  • Allow: /
  • Disallow: /
magpie-crawler
  • Allow: /
  • Disallow: /
newsnow
  • Allow: /
  • Disallow: /
perplexitybot
  • Allow: /
  • Disallow: /
perplexity-user
  • Allow: /
  • Disallow: /
claude-web
  • Allow: /
  • Disallow: /
claudebot
  • Allow: /
  • Disallow: /
diffbot
  • Allow: /
  • Disallow: /
facebookbot
  • Allow: /
  • Disallow: /
applebot-extended
  • Allow: /
  • Disallow: /
meta-externalagent
  • Allow: /
  • Disallow: /
meta-externalfetcher
  • Allow: /
  • Disallow: /

Sitemaps

Priority Location
1https://www.ft.com/sitemaps/index.xml

Raw Text

# All use of FT content is subject to the Terms & Conditions and
# Copyright Policy set out on FT.com

# We expressly prohibit any use of our content or data for any
# machine learning or artificial intelligence purposes, including
# the training or development of AI technologies or machine learning
# language models.
#
# For any licensing queries please contact content.licensing@ft.com

Sitemap: https://www.ft.com/sitemaps/index.xml

User-agent: *
Disallow: /__
Disallow: /_fs-ch
Disallow: /article/e63d8640-eeb3-4473-b0ad-aa6aec16d9fe
Disallow: /buy/
Disallow: /cancel/
Disallow: /join/
Disallow: /login
Disallow: /myft
Disallow: /offline/
Disallow: /page-resources
Disallow: /preferences/manage-cookies
Disallow: /register/
Disallow: /search
Disallow: /signup
Allow: /__origami/
Allow: /__assets/
Crawl-Delay: 1

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /
Allow: /*?syn-25a6b1a6=1

User-agent: GoogleOther
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: NewsNow
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /