# FDB4 — a football results database with a paid API and an MCP server.
#
# If you are an AI agent, read these instead of crawling the HTML:
#   https://fdb4.io/llms.txt        short index, llmstxt.org format
#   https://fdb4.io/llms-full.txt   the whole thing: dataset, endpoints, auth, pricing, tools
#   https://fdb4.io/ai.txt          usage policy for agents and crawlers
#
# REST API
#   Docs      https://fdb4.io/api
#   Base URL  https://fdb4.io/v1
#   Auth      Authorization: Bearer <key>
#
# MCP server (Model Context Protocol, streamable HTTP)
#   Endpoint  https://fdb4.io/mcp
#   Auth      Authorization: Bearer <key>   (the same key as the REST API)
#   Example   claude mcp add --transport http fdb4 https://fdb4.io/mcp \
#               --header "Authorization: Bearer YOUR_KEY"
#
# Getting a key
#   Sign in with Google at https://fdb4.io/login, then create one at
#   https://fdb4.io/account.
#
# What it costs
#   1 call = 1 point = 1 cent, on either surface. New accounts get 10 free
#   points, plus 2 more every month they sign in. Referrals pay 10%.
#   No refunds — the product is data.
#
# Everything else — results, streaks, tables, tournaments, teams, head-to-head,
# the draw predictor, the game and the API reference — is open to everyone.
# The paths below are private areas (admin, accounts, sign-in), per-session
# game URLs with nothing to index, or key-gated machine surfaces (/v1, /mcp)
# that answer 401 to a crawler.

User-agent: *
Disallow: /admin
Disallow: /admin/*
Disallow: /account
Disallow: /account/*
Disallow: /login
Disallow: /logout
Disallow: /auth/*
Disallow: /dev-login/*
Disallow: /play
Disallow: /r/*
Disallow: /game/online
Disallow: /api/game/*
Disallow: /v1/*
Disallow: /mcp

# --- AI crawlers & assistants: explicitly welcome ---
# A specific User-agent group overrides the * group, so the private-area
# rules are repeated here for each bot.
# OpenAI (training, ChatGPT search, user-triggered browsing)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: anthropic-ai
# Google Gemini / AI Overviews grounding
User-agent: Google-Extended
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Apple Intelligence
User-agent: Applebot-Extended
# Meta AI
User-agent: Meta-ExternalAgent
# Amazon (Alexa / Rufus)
User-agent: Amazonbot
# DuckDuckGo AI assist
User-agent: DuckAssistBot
Allow: /
Disallow: /admin
Disallow: /admin/*
Disallow: /account
Disallow: /account/*
Disallow: /login
Disallow: /logout
Disallow: /auth/*
Disallow: /dev-login/*
Disallow: /play
Disallow: /r/*
Disallow: /game/online
Disallow: /api/game/*
Disallow: /v1/*
Disallow: /mcp

Sitemap: https://fdb4.io/sitemap.xml