From 06748ee0a313b8b64fcc2c95146b095e990fd6be Mon Sep 17 00:00:00 2001
From: Anthony Ettinger <anthony@profullstack.com>
Date: Sat, 5 Sep 2026 21:31:32 +0000
Subject: [PATCH] Charge AI training crawlers for access
 (@profullstack/x402-gateway)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2
---
 bun.lock      |  3 +++
 package.json  |  1 +
 src/server.ts | 24 ++++++++++++++++++++++++
 3 files changed, 28 insertions(+)

diff --git a/bun.lock b/bun.lock
index 415b27b..23206f9 100644
--- a/bun.lock
+++ b/bun.lock
@@ -6,6 +6,7 @@
       "name": "transcript-search",
       "dependencies": {
         "@libsql/client": "^0.15.0",
+        "@profullstack/x402-gateway": "^0.1.0",
         "commander": "^12.1.0",
         "smol-toml": "^1.3.1",
         "zod": "^3.23.8",
@@ -69,6 +70,8 @@
 
     "@neon-rs/load": ["@neon-rs/load@0.0.4", "", {}, "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw=="],
 
+    "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.1.0", "", {}, "sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ=="],
+
     "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="],
 
     "@types/jsdom": ["@types/jsdom@28.0.3", "", { "dependencies": { "@types/node": "*", "@types/tough-cookie": "*", "parse5": "^8.0.0", "undici-types": "^7.21.0" } }, "sha512-/HQ2uFoetFTXuye8vzIcHw2z6Fwi7Hi/qcgC+RoS9NCyewiqxhVGqlG+ViGB6lkax481R6dmhf1I7lIGlzJStQ=="],
diff --git a/package.json b/package.json
index f5ef46b..a5525e7 100644
--- a/package.json
+++ b/package.json
@@ -20,6 +20,7 @@
   },
   "dependencies": {
     "@libsql/client": "^0.15.0",
+    "@profullstack/x402-gateway": "^0.1.0",
     "commander": "^12.1.0",
     "smol-toml": "^1.3.1",
     "zod": "^3.23.8"
diff --git a/src/server.ts b/src/server.ts
index 6648b0b..5d07b8f 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -19,6 +19,7 @@
  *   GET /*                               -> static assets / SPA shell
  */
 import { join, normalize } from "node:path";
+import { createGateway } from "@profullstack/x402-gateway";
 import { loadConfig } from "./config.ts";
 import { getDb, migrate } from "./db/index.ts";
 import { buildRegistry, getAiProvider } from "./registry.ts";
@@ -407,6 +408,19 @@ async function candidateTickers(topic: string | null, limit: number): Promise<st
   }
 }
 
+/**
+ * Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider,
+ * Applebot-Extended) pay by the day over x402 (@profullstack/x402-gateway).
+ * People, search engines and retrieval crawlers pass through untouched.
+ */
+const crawlGateway = createGateway({
+  siteUrl: process.env.SITE_URL ?? "https://advis0r.com",
+  siteName: "advis0r",
+  coinpay: { apiKey: process.env.COINPAY_X402_KEY },
+  payTo: process.env.CRAWL_PAY_TO,
+  contact: "mailto:anthony@profullstack.com",
+});
+
 const server = Bun.serve({
   port,
   idleTimeout: 60,
@@ -414,6 +428,16 @@ const server = Bun.serve({
     const url = new URL(req.url);
     const p = url.pathname;
 
+    // Crawl gateway first: a refused training crawler gets 402 (or the sales
+    // page at /crawl) before anything below runs.
+    const refused = await crawlGateway.handle(req);
+    if (refused) return refused;
+    if (p === "/robots.txt") {
+      return new Response(crawlGateway.robotsTxt({ disallow: ["/api/"] }), {
+        headers: { "content-type": "text/plain; charset=utf-8" },
+      });
+    }
+
     // Canonicalize: strip a leading "www." from the host on every request.
     const host = req.headers.get("host") ?? "";
     if (host.toLowerCase().startsWith("www.")) {