From c51a1c59015512b475cb15c72f015cc98e3679de Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:29:55 +0000 Subject: [PATCH] Charge AI training crawlers for access (@profullstack/x402-gateway) Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, Applebot-Extended) get 402 Payment Required with an x402 offer, or the sales page at /crawl, and a paid pass opens the site for a day. People, search engines and retrieval crawlers pass through untouched. robots.txt is now generated from the same lists. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- apps/web/package.json | 1 + apps/web/src/app/robots.ts | 34 ---------------------------- apps/web/src/app/robots.txt/route.ts | 9 ++++++++ apps/web/src/lib/crawl-gateway.ts | 27 ++++++++++++++++++++++ apps/web/src/proxy.ts | 7 ++++++ pnpm-lock.yaml | 19 +++++++++------- 6 files changed, 55 insertions(+), 42 deletions(-) delete mode 100644 apps/web/src/app/robots.ts create mode 100644 apps/web/src/app/robots.txt/route.ts create mode 100644 apps/web/src/lib/crawl-gateway.ts diff --git a/apps/web/package.json b/apps/web/package.json index 7539e88..53d64c9 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -22,6 +22,7 @@ "@profullstack/coinpay": "^0.6.11", "@profullstack/pluginstore": "^0.1.1", "@profullstack/stack": "^0.1.3", + "@profullstack/x402-gateway": "^0.1.0", "@supabase/ssr": "latest", "@supabase/supabase-js": "latest", "lightweight-charts": "^5.2.0", diff --git a/apps/web/src/app/robots.ts b/apps/web/src/app/robots.ts deleted file mode 100644 index 0efbd4c..0000000 --- a/apps/web/src/app/robots.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { MetadataRoute } from 'next'; - -const BASE_URL = 'https://b1dz.com'; - -// Explicit allow rules for the major AI / answer-engine crawlers so they -// don't fall back to defaults, plus a sitemap pointer for discovery. -const AI_BOTS = [ - 'GPTBot', - 'OAI-SearchBot', - 'ChatGPT-User', - 'ClaudeBot', - 'Claude-Web', - 'PerplexityBot', - 'Google-Extended', - 'Applebot-Extended', - 'CCBot', - 'Bytespider', -]; - -export default function robots(): MetadataRoute.Robots { - return { - rules: [ - { - userAgent: '*', - allow: '/', - // Authenticated app surfaces — keep out of the index. - disallow: ['/api/', '/dashboard', '/console', '/settings'], - }, - ...AI_BOTS.map((userAgent) => ({ userAgent, allow: '/' })), - ], - sitemap: `${BASE_URL}/sitemap.xml`, - host: BASE_URL, - }; -} diff --git a/apps/web/src/app/robots.txt/route.ts b/apps/web/src/app/robots.txt/route.ts new file mode 100644 index 0000000..7941acc --- /dev/null +++ b/apps/web/src/app/robots.txt/route.ts @@ -0,0 +1,9 @@ +import { robotsRoute } from "@profullstack/x402-gateway/next"; +import { gateway } from "@/lib/crawl-gateway"; + +// Generated from the same crawler lists the gateway enforces: training +// crawlers are refused everywhere but /crawl (where they can buy a pass), +// retrieval crawlers are named as welcome, everyone else gets the rules below. +export const GET = robotsRoute(gateway, { + disallow: ["/api/", "/console", "/dashboard", "/settings"], +}); diff --git a/apps/web/src/lib/crawl-gateway.ts b/apps/web/src/lib/crawl-gateway.ts new file mode 100644 index 0000000..717cd79 --- /dev/null +++ b/apps/web/src/lib/crawl-gateway.ts @@ -0,0 +1,27 @@ +import { createGateway } from "@profullstack/x402-gateway"; +import { x402Proxy } from "@profullstack/x402-gateway/next"; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, + * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over + * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval + * crawlers behind AI search pass through untouched. + * + * Runs inside the middleware, so nothing here may import Node-only modules. + * The env is read through a non-literal key on purpose: Next inlines + * `process.env.NAME` at build time, and these are runtime secrets. Without + * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training + * crawlers with 402, just with an empty offer. + */ +const env = (name: string) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://b1dz.com", + siteName: "b1dz", + coinpay: { apiKey: env("COINPAY_X402_KEY") }, + payTo: env("CRAWL_PAY_TO"), + contact: "mailto:support@b1dz.com", +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/apps/web/src/proxy.ts b/apps/web/src/proxy.ts index f90e668..358fbc9 100644 --- a/apps/web/src/proxy.ts +++ b/apps/web/src/proxy.ts @@ -1,3 +1,4 @@ +import { gate } from "@/lib/crawl-gateway"; /** * Auth proxy — redirects www → apex, refreshes the Supabase session on every * request, and gates non-public routes. Lifted from the official @supabase/ssr docs. @@ -18,6 +19,12 @@ const PUBLIC_EXACT = new Set(['/']); let loggedVersion = false; export async function proxy(request: NextRequest) { + // Crawl gateway first: AI training crawlers get 402 Payment Required (or the + // sales page at /crawl) unless they present a paid pass. People, Googlebot + // and retrieval crawlers fall through to everything below. + const answer = await gate(request); + if (answer) return answer; + const host = request.headers.get('host') ?? ''; // Redirect www → non-www before touching auth. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 55c80fe..7c04dcb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -253,6 +253,9 @@ importers: '@profullstack/stack': specifier: ^0.1.3 version: 0.1.3(@supabase/ssr@0.12.6(@supabase/supabase-js@2.115.0))(next@16.3.4(@types/node@26.4.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8) + '@profullstack/x402-gateway': + specifier: ^0.1.0 + version: 0.1.0 '@supabase/ssr': specifier: latest version: 0.12.6(@supabase/supabase-js@2.115.0) @@ -1432,6 +1435,10 @@ packages: engines: {node: '>=20.19'} hasBin: true + '@profullstack/x402-gateway@0.1.0': + resolution: {integrity: sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ==} + engines: {node: '>=20.11'} + '@puppeteer/browsers@3.2.2': resolution: {integrity: sha512-q2BU4YfO9h/Wt7IcWPcggpOOqLk2Tbs1hDwolvKZrweRjy751OJBKMN9zO5bfD0pzU7X/tvKw/exQds4pM/LOg==} engines: {node: '>=22.12.0'} @@ -2827,10 +2834,6 @@ packages: picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - picomatch@4.0.5: - resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} - engines: {node: '>=12'} - picomatch@4.0.7: resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} engines: {node: '>=12'} @@ -3767,6 +3770,8 @@ snapshots: '@noble/curves': 2.4.0 '@noble/hashes': 2.4.0 + '@profullstack/x402-gateway@0.1.0': {} + '@puppeteer/browsers@3.2.2': dependencies: modern-tar: 0.8.4 @@ -5003,8 +5008,6 @@ snapshots: picocolors@1.1.1: {} - picomatch@4.0.5: {} - picomatch@4.0.7: {} picture-tuber@1.0.2: @@ -5441,8 +5444,8 @@ snapshots: vite@8.0.7(@types/node@26.4.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.23.13): dependencies: lightningcss: 1.32.0 - picomatch: 4.0.5 - postcss: 8.5.19 + picomatch: 4.0.7 + postcss: 8.5.23 rolldown: 1.0.0-rc.13 tinyglobby: 0.2.17 optionalDependencies: