From b0f0ab1b7ec5ddf96b68356ba3f3cec0d3623c47 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 9 Sep 2026 16:40:32 +0000 Subject: [PATCH 1/2] Meter every route, and sell a pass to whoever goes over The limiter here watched three API prefixes and two page paths. That is the shape every site in the fleet had, and it is the shape that failed on coinpayportal on 2026-09-08: a headless browser found /explorer fifteen hours after it shipped and walked 19,000 of its URLs a day for two days. It declared nothing, so no crawler list matched it. It never touched a listed path, so no limiter saw it. The expensive routes were never the ones at risk -- the unlisted ones were. So the tuned numbers stay (30/min on /api/search/, /api/dht/ and /api/torrent-search, 60/min on /search and /dht) and everything NOT in that list is now metered too, at 100/min per caller, via @profullstack/throttle. Going over is answered 402 with the gate's own offer rather than 429, so a scraper that declares nothing is sold the same pass GPTBot buys. The good-bot and bad-bot tiers are untouched and still run first; this is the floor under them, not a replacement. A signed-in reader and an API bearer get the 600/min credentialed budget keyed on the credential itself -- the gate exempts both, because it is deciding whether to charge a crawler and a session is good evidence of a person, but this is deciding whether anyone at all reads 19,000 pages an hour. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YDGCxTmEPs3ecwjjLJDQXh --- package.json | 3 ++- pnpm-lock.yaml | 21 +++++++++++---- src/lib/throttle.ts | 66 +++++++++++++++++++++++++++++++++++++++++++++ src/proxy.test.ts | 47 +++++++++++++++++++++++++++++++- src/proxy.ts | 41 +++++++++------------------- 5 files changed, 142 insertions(+), 36 deletions(-) create mode 100644 src/lib/throttle.ts diff --git a/package.json b/package.json index b53f273..3b45bb4 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "@profullstack/player": "0.3.1", "@profullstack/referrals": "^0.1.0", "@profullstack/stack": "^0.1.3", - "@profullstack/x402-gateway": "0.3.0", + "@profullstack/throttle": "^0.2.1", + "@profullstack/x402-gateway": "^0.5.0", "@radix-ui/react-dialog": "^1.1.23", "@radix-ui/react-dropdown-menu": "^2.1.24", "@radix-ui/react-icons": "^1.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 64cd537..314e81a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,9 +37,12 @@ importers: '@profullstack/stack': specifier: ^0.1.3 version: 0.1.3(next@16.3.3(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8) + '@profullstack/throttle': + specifier: ^0.2.1 + version: 0.2.1 '@profullstack/x402-gateway': - specifier: 0.3.0 - version: 0.3.0 + specifier: ^0.5.0 + version: 0.5.0 '@radix-ui/react-dialog': specifier: ^1.1.23 version: 1.1.23(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -1317,8 +1320,12 @@ packages: react: optional: true - '@profullstack/x402-gateway@0.3.0': - resolution: {integrity: sha512-aaSMdtVInaAD6te/RnNnnqZW2+oo/dUsOTFTJCl0hOn58s85Yw3vZGk3KCtZmjrwV2O2rdTjKlZjbKp3CGgntw==} + '@profullstack/throttle@0.2.1': + resolution: {integrity: sha512-QnrQ2y2pjKch8s3aLswn6MCoW+XjjdpwiG9PDRx/KGVjBcfstlisg8dI6TRv1RooaMou7tp+kTEW999kbkmiow==} + engines: {node: '>=20.11'} + + '@profullstack/x402-gateway@0.5.0': + resolution: {integrity: sha512-QnSDRKXbVbswB6lQ9eSdDUUt64A90Tmd1U2tuDFgqAtgJtTr/CoXDHHzXcJWJxv5f3f8FafdoxwO+/bgl0MhtQ==} engines: {node: '>=20.11'} '@puppeteer/browsers@3.2.1': @@ -6211,7 +6218,11 @@ snapshots: next: 16.3.3(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) react: 19.2.8 - '@profullstack/x402-gateway@0.3.0': {} + '@profullstack/throttle@0.2.1': + dependencies: + '@profullstack/x402-gateway': 0.5.0 + + '@profullstack/x402-gateway@0.5.0': {} '@puppeteer/browsers@3.2.1(yauzl@2.10.0)': dependencies: diff --git a/src/lib/throttle.ts b/src/lib/throttle.ts new file mode 100644 index 0000000..123eed7 --- /dev/null +++ b/src/lib/throttle.ts @@ -0,0 +1,66 @@ +/** + * The site-wide allowance: a hundred requests a minute, per caller, on every + * route. Going over is answered 402 with the crawl gateway's offer, not 429. + * + * WHY EVERY ROUTE. The limiter this replaces watched three API prefixes and + * two page paths. That is the shape every site in the fleet had, and it is the + * shape that failed on coinpayportal: a headless browser found a route nobody + * had listed and walked 19,000 of its URLs a day for two days, declaring + * nothing, tripping no list, rendering every page server-side against a + * metered upstream. The expensive routes were never the ones at risk. The + * unlisted ones were. + * + * The tuned numbers below are kept. What changes is that everything NOT in + * this list is now metered too, at the house default. + * + * Imports nothing Node-only: the proxy may run at the edge. + */ + +import { createThrottle } from '@profullstack/throttle'; +import { gateway, hasApiBearer } from '@/lib/crawl-gateway'; + +const SESSION_COOKIE_NAME = 'sb-auth-token'; + +/** The session's own access token, as a bucket key -- not a boolean. */ +function sessionKey(request: Request): string | null { + const raw = /(?:^|;\s*)sb-auth-token=([^;]+)/.exec(request.headers.get('cookie') ?? '')?.[1]; + if (!raw) return null; + try { + const session = JSON.parse(decodeURIComponent(raw)) as { access_token?: unknown }; + return typeof session.access_token === 'string' ? session.access_token : null; + } catch { + return null; + } +} + +export const throttle = createThrottle({ + gateway, + /* + * A signed-in reader and an API integration each get the larger budget, + * keyed on the credential itself so two of them never share one. The gate + * exempts both outright -- it is deciding whether to charge a crawler, and a + * session is good evidence of a person. Here they are still metered, because + * this is deciding whether anyone at all is reading 19,000 pages an hour. + */ + credentialFrom: (request) => + sessionKey(request) ?? + (hasApiBearer(request) + ? (/^Bearer\s+(\S+)/i.exec(request.headers.get('authorization') ?? '')?.[1] ?? null) + : null), + credential: { limit: 600, ceiling: 1200 }, + rules: [ + /* The expensive ones, at the numbers they were already tuned to. */ + { path: '/api/search/', limit: 30 }, + { path: '/api/dht/', limit: 30 }, + { path: '/api/torrent-search', limit: 30 }, + { path: '/search', limit: 60 }, + { path: '/dht', limit: 60 }, + /* Sign-in stays address-bucketed, or a guess buys the session budget. */ + { path: '/api/auth/', limit: 10, credential: false }, + ], +}); + +/** Resolves to a Response for a caller over the allowance, or undefined. */ +export const meter = (request: Request) => throttle.handle(request); + +export { SESSION_COOKIE_NAME }; diff --git a/src/proxy.test.ts b/src/proxy.test.ts index 9017c7f..3b00934 100644 --- a/src/proxy.test.ts +++ b/src/proxy.test.ts @@ -7,7 +7,9 @@ * - Good bots (Googlebot, Bingbot, Applebot): rate-limited (10/min), NOT blocked * - Bad bots on expensive routes (/api/search/*, /api/dht/*): blocked (403) * - Bad bots on other API routes: rate-limited (5/min), allowed through - * - Normal browsers: rate-limited on expensive routes (30/min) + * - Normal browsers: rate-limited on expensive routes (30/min) and on every + * other route at the house default (100/min), which is the point of the + * throttle: the routes nobody listed are the ones that get walked * - Supabase session: refreshed (cookie rewritten) when the access token expires within 60s * - ?ref=CODE: stored in the referral_code cookie when valid */ @@ -455,3 +457,46 @@ describe('Edge controls: hosting ranges and spoofed browsers', () => { }); }); }); + +const BROWSER_UA = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'; + +describe('The site-wide allowance', () => { + /** Every case needs its own address: the counter is per caller and module-level. */ + function call(pathname: string, ip: string) { + return middleware( + new NextRequest(new URL(`http://localhost${pathname}`), { + headers: { + 'user-agent': BROWSER_UA, + 'sec-fetch-mode': 'navigate', + 'x-real-ip': ip, + }, + }) + ); + } + + async function countUntilLimited(pathname: string, ip: string, attempts: number) { + let allowed = 0; + for (let i = 0; i < attempts; i++) { + const res = await call(pathname, ip); + if (res && res.status !== 200) break; + allowed++; + } + return allowed; + } + + // The gap this closes. A route nobody thought to list used to be unmetered + // however hard it was hit; on coinpayportal that was 19,000 URLs a day. + it('meters a route nobody listed', async () => { + expect(await countUntilLimited('/torrent/abc123', '10.9.0.1', 140)).toBe(100); + }); + + it('keeps the expensive routes at the number they were tuned to', async () => { + expect(await countUntilLimited('/api/torrent-search', '10.9.0.2', 60)).toBe(30); + }); + + it('gives each caller its own allowance', async () => { + expect(await countUntilLimited('/torrent/abc123', '10.9.0.3', 5)).toBe(5); + expect(await countUntilLimited('/torrent/abc123', '10.9.0.4', 5)).toBe(5); + }); +}); diff --git a/src/proxy.ts b/src/proxy.ts index 4eb5984..8ade483 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -4,7 +4,7 @@ * The one live proxy file. In order: * - Charges AI training crawlers for access (402 + x402 offer) via the crawl gateway * - Refreshes the Supabase session cookie when the access token is about to expire - * - Rate limits expensive API routes (sliding window, per-IP) + * - Meters every route at 100 req/min per caller, selling a pass to whoever goes over * - Blocks known bots/crawlers from hitting API routes (with exceptions for good bots) * - Enforces profile selection for authenticated users * - Stores a valid ?ref= referral code in a cookie @@ -17,6 +17,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { trackReferralCode } from '@profullstack/stack/referrals'; import { gateway } from '@/lib/crawl-gateway'; +import { meter } from '@/lib/throttle'; // ============================================================================= // Rate Limiting (in-memory sliding window) @@ -36,8 +37,6 @@ const WINDOW_MS = 60_000; // 1 minute sliding window /** Rate limit tiers (requests per minute) */ const RATE_LIMITS = { - api: 30, // /api/search/*, /api/dht/*, /api/torrent-search - page: 60, // /search, /dht page routes goodBot: 10, // Googlebot, Bingbot, Applebot badBot: 5, // All other bots } as const; @@ -180,20 +179,10 @@ const EXPENSIVE_API_PATHS = [ '/api/torrent-search', ]; -/** Page paths that should be rate limited (more generous) */ -const RATE_LIMITED_PAGE_PATHS = [ - '/search', - '/dht', -]; - function isExpensiveApiRoute(pathname: string): boolean { return EXPENSIVE_API_PATHS.some(p => pathname.startsWith(p)); } -function isRateLimitedPageRoute(pathname: string): boolean { - return RATE_LIMITED_PAGE_PATHS.some(p => pathname === p || pathname.startsWith(p + '/')); -} - // ============================================================================= // Supabase session refresh // ============================================================================= @@ -495,22 +484,16 @@ export async function proxy(request: NextRequest): Promise { } } - // --- Rate limiting for expensive API routes (non-bot requests) --- - if (!isBotRequest && isExpensiveApiRoute(pathname)) { - const result = checkRateLimit(`api:${clientIp}`, RATE_LIMITS.api); - if (!result.allowed) { - console.log(`[rate-limit] API rate limited: IP=${clientIp} path=${pathname}`); - return withSession(make429Response(result.retryAfterSec ?? 60, true), session); - } - } - - // --- Rate limiting for page routes --- - if (!isBotRequest && !isApiRoute && isRateLimitedPageRoute(pathname)) { - const result = checkRateLimit(`page:${clientIp}`, RATE_LIMITS.page); - if (!result.allowed) { - console.log(`[rate-limit] Page rate limited: IP=${clientIp} path=${pathname}`); - return withSession(make429Response(result.retryAfterSec ?? 60, false), session); - } + // --- The site-wide allowance --- + // 100 requests a minute per caller on EVERY route, with the expensive ones + // kept at the numbers they were tuned to (see lib/throttle.ts). Going over + // is answered 402 with the gate's own offer rather than 429, so a scraper + // that declares nothing is sold the same pass GPTBot buys. The bot tiers + // above still run first and stay tighter; this is the floor under them. + const overLimit = await meter(request); + if (overLimit) { + console.log(`[throttle] ${overLimit.status}: IP=${clientIp} path=${pathname} UA=${userAgent?.slice(0, 80)}`); + return withSession(overLimit as NextResponse, session); } // --- Profile enforcement: authenticated users must select a profile --- From d42f88571fcd880085d4c15b00e8ad25366e83eb Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 9 Sep 2026 17:11:47 +0000 Subject: [PATCH 2/2] Take throttle 0.2.2 and gateway 0.6.0 Keeps this on the same two versions as the rest of the rollout, and drops the second nested copy of the gateway: a caret range on a 0.x version only matches patches, so the throttle's old ^0.5.0 floor could not resolve 0.6.0. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YDGCxTmEPs3ecwjjLJDQXh --- package.json | 4 ++-- pnpm-lock.yaml | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 3b45bb4..fc6efa0 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,8 @@ "@profullstack/player": "0.3.1", "@profullstack/referrals": "^0.1.0", "@profullstack/stack": "^0.1.3", - "@profullstack/throttle": "^0.2.1", - "@profullstack/x402-gateway": "^0.5.0", + "@profullstack/throttle": "^0.2.2", + "@profullstack/x402-gateway": "^0.6.0", "@radix-ui/react-dialog": "^1.1.23", "@radix-ui/react-dropdown-menu": "^2.1.24", "@radix-ui/react-icons": "^1.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 314e81a..b22af72 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,11 +38,11 @@ importers: specifier: ^0.1.3 version: 0.1.3(next@16.3.3(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8) '@profullstack/throttle': - specifier: ^0.2.1 - version: 0.2.1 + specifier: ^0.2.2 + version: 0.2.2 '@profullstack/x402-gateway': - specifier: ^0.5.0 - version: 0.5.0 + specifier: ^0.6.0 + version: 0.6.0 '@radix-ui/react-dialog': specifier: ^1.1.23 version: 1.1.23(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -1320,12 +1320,12 @@ packages: react: optional: true - '@profullstack/throttle@0.2.1': - resolution: {integrity: sha512-QnrQ2y2pjKch8s3aLswn6MCoW+XjjdpwiG9PDRx/KGVjBcfstlisg8dI6TRv1RooaMou7tp+kTEW999kbkmiow==} + '@profullstack/throttle@0.2.2': + resolution: {integrity: sha512-qpb9yKE7frwTP29YILx+QePqyEPWGVZxPYve7ITqSfARzOl0ZKhfjeizxPKZUgYhFQfNk6kWCa6/53RVoj304g==} engines: {node: '>=20.11'} - '@profullstack/x402-gateway@0.5.0': - resolution: {integrity: sha512-QnSDRKXbVbswB6lQ9eSdDUUt64A90Tmd1U2tuDFgqAtgJtTr/CoXDHHzXcJWJxv5f3f8FafdoxwO+/bgl0MhtQ==} + '@profullstack/x402-gateway@0.6.0': + resolution: {integrity: sha512-n9/7NixIM9wVOMGWDc+3RtFsGABG+HAYfkZ11JxoEIlF/FDRGEXbvHFlNL/M97RNy01964FhAbQp34OMJoxqZg==} engines: {node: '>=20.11'} '@puppeteer/browsers@3.2.1': @@ -6218,11 +6218,11 @@ snapshots: next: 16.3.3(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) react: 19.2.8 - '@profullstack/throttle@0.2.1': + '@profullstack/throttle@0.2.2': dependencies: - '@profullstack/x402-gateway': 0.5.0 + '@profullstack/x402-gateway': 0.6.0 - '@profullstack/x402-gateway@0.5.0': {} + '@profullstack/x402-gateway@0.6.0': {} '@puppeteer/browsers@3.2.1(yauzl@2.10.0)': dependencies: