diff --git a/package.json b/package.json index 440ec59..80c7cc7 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "@profullstack/player": "0.6.0", "@profullstack/referrals": "^0.1.0", "@profullstack/stack": "^0.1.3", - "@profullstack/x402-gateway": "0.3.0", + "@profullstack/throttle": "^0.2.2", + "@profullstack/x402-gateway": "^0.6.0", "@radix-ui/react-dialog": "^1.1.23", "@radix-ui/react-dropdown-menu": "^2.1.24", "@radix-ui/react-icons": "^1.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 02653ab..371e3a8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,9 +37,12 @@ importers: '@profullstack/stack': specifier: ^0.1.3 version: 0.1.3(next@16.3.4(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8))(react@19.2.8) + '@profullstack/throttle': + specifier: ^0.2.2 + version: 0.2.2 '@profullstack/x402-gateway': - specifier: 0.3.0 - version: 0.3.0 + specifier: ^0.6.0 + version: 0.6.0 '@radix-ui/react-dialog': specifier: ^1.1.23 version: 1.1.23(@types/react-dom@19.2.7(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -1322,8 +1325,12 @@ packages: react: optional: true - '@profullstack/x402-gateway@0.3.0': - resolution: {integrity: sha512-aaSMdtVInaAD6te/RnNnnqZW2+oo/dUsOTFTJCl0hOn58s85Yw3vZGk3KCtZmjrwV2O2rdTjKlZjbKp3CGgntw==} + '@profullstack/throttle@0.2.2': + resolution: {integrity: sha512-qpb9yKE7frwTP29YILx+QePqyEPWGVZxPYve7ITqSfARzOl0ZKhfjeizxPKZUgYhFQfNk6kWCa6/53RVoj304g==} + engines: {node: '>=20.11'} + + '@profullstack/x402-gateway@0.6.0': + resolution: {integrity: sha512-n9/7NixIM9wVOMGWDc+3RtFsGABG+HAYfkZ11JxoEIlF/FDRGEXbvHFlNL/M97RNy01964FhAbQp34OMJoxqZg==} engines: {node: '>=20.11'} '@puppeteer/browsers@3.2.2': @@ -6185,7 +6192,11 @@ snapshots: next: 16.3.4(@babel/core@7.29.7)(@playwright/test@1.62.1)(@types/node@26.4.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) react: 19.2.8 - '@profullstack/x402-gateway@0.3.0': {} + '@profullstack/throttle@0.2.2': + dependencies: + '@profullstack/x402-gateway': 0.6.0 + + '@profullstack/x402-gateway@0.6.0': {} '@puppeteer/browsers@3.2.2(yauzl@2.10.0)': dependencies: diff --git a/src/lib/throttle.ts b/src/lib/throttle.ts new file mode 100644 index 0000000..123eed7 --- /dev/null +++ b/src/lib/throttle.ts @@ -0,0 +1,66 @@ +/** + * The site-wide allowance: a hundred requests a minute, per caller, on every + * route. Going over is answered 402 with the crawl gateway's offer, not 429. + * + * WHY EVERY ROUTE. The limiter this replaces watched three API prefixes and + * two page paths. That is the shape every site in the fleet had, and it is the + * shape that failed on coinpayportal: a headless browser found a route nobody + * had listed and walked 19,000 of its URLs a day for two days, declaring + * nothing, tripping no list, rendering every page server-side against a + * metered upstream. The expensive routes were never the ones at risk. The + * unlisted ones were. + * + * The tuned numbers below are kept. What changes is that everything NOT in + * this list is now metered too, at the house default. + * + * Imports nothing Node-only: the proxy may run at the edge. + */ + +import { createThrottle } from '@profullstack/throttle'; +import { gateway, hasApiBearer } from '@/lib/crawl-gateway'; + +const SESSION_COOKIE_NAME = 'sb-auth-token'; + +/** The session's own access token, as a bucket key -- not a boolean. */ +function sessionKey(request: Request): string | null { + const raw = /(?:^|;\s*)sb-auth-token=([^;]+)/.exec(request.headers.get('cookie') ?? '')?.[1]; + if (!raw) return null; + try { + const session = JSON.parse(decodeURIComponent(raw)) as { access_token?: unknown }; + return typeof session.access_token === 'string' ? session.access_token : null; + } catch { + return null; + } +} + +export const throttle = createThrottle({ + gateway, + /* + * A signed-in reader and an API integration each get the larger budget, + * keyed on the credential itself so two of them never share one. The gate + * exempts both outright -- it is deciding whether to charge a crawler, and a + * session is good evidence of a person. Here they are still metered, because + * this is deciding whether anyone at all is reading 19,000 pages an hour. + */ + credentialFrom: (request) => + sessionKey(request) ?? + (hasApiBearer(request) + ? (/^Bearer\s+(\S+)/i.exec(request.headers.get('authorization') ?? '')?.[1] ?? null) + : null), + credential: { limit: 600, ceiling: 1200 }, + rules: [ + /* The expensive ones, at the numbers they were already tuned to. */ + { path: '/api/search/', limit: 30 }, + { path: '/api/dht/', limit: 30 }, + { path: '/api/torrent-search', limit: 30 }, + { path: '/search', limit: 60 }, + { path: '/dht', limit: 60 }, + /* Sign-in stays address-bucketed, or a guess buys the session budget. */ + { path: '/api/auth/', limit: 10, credential: false }, + ], +}); + +/** Resolves to a Response for a caller over the allowance, or undefined. */ +export const meter = (request: Request) => throttle.handle(request); + +export { SESSION_COOKIE_NAME }; diff --git a/src/proxy.test.ts b/src/proxy.test.ts index 9017c7f..3b00934 100644 --- a/src/proxy.test.ts +++ b/src/proxy.test.ts @@ -7,7 +7,9 @@ * - Good bots (Googlebot, Bingbot, Applebot): rate-limited (10/min), NOT blocked * - Bad bots on expensive routes (/api/search/*, /api/dht/*): blocked (403) * - Bad bots on other API routes: rate-limited (5/min), allowed through - * - Normal browsers: rate-limited on expensive routes (30/min) + * - Normal browsers: rate-limited on expensive routes (30/min) and on every + * other route at the house default (100/min), which is the point of the + * throttle: the routes nobody listed are the ones that get walked * - Supabase session: refreshed (cookie rewritten) when the access token expires within 60s * - ?ref=CODE: stored in the referral_code cookie when valid */ @@ -455,3 +457,46 @@ describe('Edge controls: hosting ranges and spoofed browsers', () => { }); }); }); + +const BROWSER_UA = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36'; + +describe('The site-wide allowance', () => { + /** Every case needs its own address: the counter is per caller and module-level. */ + function call(pathname: string, ip: string) { + return middleware( + new NextRequest(new URL(`http://localhost${pathname}`), { + headers: { + 'user-agent': BROWSER_UA, + 'sec-fetch-mode': 'navigate', + 'x-real-ip': ip, + }, + }) + ); + } + + async function countUntilLimited(pathname: string, ip: string, attempts: number) { + let allowed = 0; + for (let i = 0; i < attempts; i++) { + const res = await call(pathname, ip); + if (res && res.status !== 200) break; + allowed++; + } + return allowed; + } + + // The gap this closes. A route nobody thought to list used to be unmetered + // however hard it was hit; on coinpayportal that was 19,000 URLs a day. + it('meters a route nobody listed', async () => { + expect(await countUntilLimited('/torrent/abc123', '10.9.0.1', 140)).toBe(100); + }); + + it('keeps the expensive routes at the number they were tuned to', async () => { + expect(await countUntilLimited('/api/torrent-search', '10.9.0.2', 60)).toBe(30); + }); + + it('gives each caller its own allowance', async () => { + expect(await countUntilLimited('/torrent/abc123', '10.9.0.3', 5)).toBe(5); + expect(await countUntilLimited('/torrent/abc123', '10.9.0.4', 5)).toBe(5); + }); +}); diff --git a/src/proxy.ts b/src/proxy.ts index 4eb5984..8ade483 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -4,7 +4,7 @@ * The one live proxy file. In order: * - Charges AI training crawlers for access (402 + x402 offer) via the crawl gateway * - Refreshes the Supabase session cookie when the access token is about to expire - * - Rate limits expensive API routes (sliding window, per-IP) + * - Meters every route at 100 req/min per caller, selling a pass to whoever goes over * - Blocks known bots/crawlers from hitting API routes (with exceptions for good bots) * - Enforces profile selection for authenticated users * - Stores a valid ?ref= referral code in a cookie @@ -17,6 +17,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { trackReferralCode } from '@profullstack/stack/referrals'; import { gateway } from '@/lib/crawl-gateway'; +import { meter } from '@/lib/throttle'; // ============================================================================= // Rate Limiting (in-memory sliding window) @@ -36,8 +37,6 @@ const WINDOW_MS = 60_000; // 1 minute sliding window /** Rate limit tiers (requests per minute) */ const RATE_LIMITS = { - api: 30, // /api/search/*, /api/dht/*, /api/torrent-search - page: 60, // /search, /dht page routes goodBot: 10, // Googlebot, Bingbot, Applebot badBot: 5, // All other bots } as const; @@ -180,20 +179,10 @@ const EXPENSIVE_API_PATHS = [ '/api/torrent-search', ]; -/** Page paths that should be rate limited (more generous) */ -const RATE_LIMITED_PAGE_PATHS = [ - '/search', - '/dht', -]; - function isExpensiveApiRoute(pathname: string): boolean { return EXPENSIVE_API_PATHS.some(p => pathname.startsWith(p)); } -function isRateLimitedPageRoute(pathname: string): boolean { - return RATE_LIMITED_PAGE_PATHS.some(p => pathname === p || pathname.startsWith(p + '/')); -} - // ============================================================================= // Supabase session refresh // ============================================================================= @@ -495,22 +484,16 @@ export async function proxy(request: NextRequest): Promise { } } - // --- Rate limiting for expensive API routes (non-bot requests) --- - if (!isBotRequest && isExpensiveApiRoute(pathname)) { - const result = checkRateLimit(`api:${clientIp}`, RATE_LIMITS.api); - if (!result.allowed) { - console.log(`[rate-limit] API rate limited: IP=${clientIp} path=${pathname}`); - return withSession(make429Response(result.retryAfterSec ?? 60, true), session); - } - } - - // --- Rate limiting for page routes --- - if (!isBotRequest && !isApiRoute && isRateLimitedPageRoute(pathname)) { - const result = checkRateLimit(`page:${clientIp}`, RATE_LIMITS.page); - if (!result.allowed) { - console.log(`[rate-limit] Page rate limited: IP=${clientIp} path=${pathname}`); - return withSession(make429Response(result.retryAfterSec ?? 60, false), session); - } + // --- The site-wide allowance --- + // 100 requests a minute per caller on EVERY route, with the expensive ones + // kept at the numbers they were tuned to (see lib/throttle.ts). Going over + // is answered 402 with the gate's own offer rather than 429, so a scraper + // that declares nothing is sold the same pass GPTBot buys. The bot tiers + // above still run first and stay tighter; this is the floor under them. + const overLimit = await meter(request); + if (overLimit) { + console.log(`[throttle] ${overLimit.status}: IP=${clientIp} path=${pathname} UA=${userAgent?.slice(0, 80)}`); + return withSession(overLimit as NextResponse, session); } // --- Profile enforcement: authenticated users must select a profile ---