AI Agents Are Scraping Your Site.
Block Them at Every Layer.

# ============================================================
# AI Crawler Blocking — robots.txt
# Comprehensive block of known AI training, RAG, and
# content-synthesis crawlers. Update monthly from the
# AI Tools Blocklist crawler registry.
# ============================================================

# --- OpenAI crawlers ---
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

# --- Anthropic crawlers ---
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# --- Google AI crawlers (separate from Googlebot search) ---
User-agent: Google-Extended
Disallow: /

# --- Common Crawl (used by many AI training pipelines) ---
User-agent: CCBot
Disallow: /

# --- ByteDance / TikTok ---
User-agent: Bytespider
Disallow: /

# --- Meta / Facebook AI ---
User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# --- Apple AI training ---
User-agent: Applebot-Extended
Disallow: /

# --- AI search / answer engines ---
User-agent: PerplexityBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Cohere-ai
Disallow: /

# --- Other known AI crawlers ---
User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: img2telegrambot
Disallow: /

User-agent: Scrapy
Disallow: /

# --- Preserve search engine access ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: *
Allow: /

// ============================================================
// Cloudflare Worker — AI Agent Detection & Blocking
// Deploy as a route-level Worker on your zone.
// Combines user-agent, TLS fingerprint, and behavioral signals.
// ============================================================

const AI_BOT_PATTERNS = [
  /GPTBot/i, /ChatGPT-User/i, /OAI-SearchBot/i,
  /ClaudeBot/i, /anthropic-ai/i,
  /Google-Extended/i, /CCBot/i,
  /Bytespider/i, /FacebookBot/i,
  /Meta-ExternalAgent/i, /Applebot-Extended/i,
  /PerplexityBot/i, /YouBot/i, /Cohere-ai/i,
  /Diffbot/i, /Omgilibot/i, /Timpibot/i,
  /PetalBot/i, /Scrapy/i, /img2dataset/i,
];

// Known AI crawler TLS fingerprints (JA3 hashes)
const AI_JA3_HASHES = new Set([
  "b32309a26951912be7dba376398abc3b",  // Python requests
  "3b5074b1b5d032e5620f69f9f700ff0e",  // Go net/http
  "e7d705a3286e19ea42f587b344ee6865",  // Node.js fetch
  "cd08e31494f9531f0ab8fd18ff2af55c",  // aiohttp
]);

async function handleRequest(request) {
  const ua = request.headers.get("user-agent") || "";
  const cf = request.cf || {};
  let score = 0;
  let signals = [];

  // Signal 1: User-agent match (highest confidence)
  if (AI_BOT_PATTERNS.some(p => p.test(ua))) {
    score += 100;
    signals.push("ua_match");
  }

  // Signal 2: Empty or missing user-agent
  if (!ua || ua.length < 10) {
    score += 40;
    signals.push("empty_ua");
  }

  // Signal 3: TLS fingerprint match
  if (cf.tlsClientAuth &&
      AI_JA3_HASHES.has(cf.ja3Hash)) {
    score += 60;
    signals.push("ja3_match");
  }

  // Signal 4: Missing typical browser headers
  if (!request.headers.get("accept-language") &&
      !request.headers.get("sec-fetch-mode")) {
    score += 30;
    signals.push("missing_browser_headers");
  }

  // Signal 5: Datacenter ASN (non-residential IP)
  if (cf.asOrganization &&
      /(amazon|google|microsoft|digital.?ocean|hetzner|ovh)/i
        .test(cf.asOrganization)) {
    score += 20;
    signals.push("datacenter_ip");
  }

  // Block if score exceeds threshold
  if (score >= 70) {
    return new Response(
      "Access denied: automated AI agent traffic is not permitted.",
      {
        status: 403,
        headers: {
          "X-Block-Reason": signals.join(","),
          "X-Bot-Score": String(score),
          "Cache-Control": "no-store",
        },
      }
    );
  }

  return fetch(request);
}

addEventListener("fetch", (event) => {
  event.respondWith(handleRequest(event.request));
});