Meter every route, and sell a pass to whoever goes over

package.json

@@ -40,7 +40,8 @@
     "@profullstack/player": "0.6.0",
     "@profullstack/referrals": "^0.1.0",
     "@profullstack/stack": "^0.1.3",
-    "@profullstack/x402-gateway": "0.3.0",
+    "@profullstack/throttle": "^0.2.2",
+    "@profullstack/x402-gateway": "^0.6.0",
     "@radix-ui/react-dialog": "^1.1.23",
     "@radix-ui/react-dropdown-menu": "^2.1.24",
     "@radix-ui/react-icons": "^1.3.2",

src/lib/throttle.ts

@@ -0,0 +1,66 @@
+/**
+ * The site-wide allowance: a hundred requests a minute, per caller, on every
+ * route. Going over is answered 402 with the crawl gateway's offer, not 429.
+ *
+ * WHY EVERY ROUTE. The limiter this replaces watched three API prefixes and
+ * two page paths. That is the shape every site in the fleet had, and it is the
+ * shape that failed on coinpayportal: a headless browser found a route nobody
+ * had listed and walked 19,000 of its URLs a day for two days, declaring
+ * nothing, tripping no list, rendering every page server-side against a
+ * metered upstream. The expensive routes were never the ones at risk. The
+ * unlisted ones were.
+ *
+ * The tuned numbers below are kept. What changes is that everything NOT in
+ * this list is now metered too, at the house default.
+ *
+ * Imports nothing Node-only: the proxy may run at the edge.
+ */
+
+import { createThrottle } from '@profullstack/throttle';
+import { gateway, hasApiBearer } from '@/lib/crawl-gateway';
+
+const SESSION_COOKIE_NAME = 'sb-auth-token';
+
+/** The session's own access token, as a bucket key -- not a boolean. */
+function sessionKey(request: Request): string | null {
+  const raw = /(?:^|;\s*)sb-auth-token=([^;]+)/.exec(request.headers.get('cookie') ?? '')?.[1];
+  if (!raw) return null;
+  try {
+    const session = JSON.parse(decodeURIComponent(raw)) as { access_token?: unknown };
+    return typeof session.access_token === 'string' ? session.access_token : null;
+  } catch {
+    return null;
+  }
+}
+
+export const throttle = createThrottle({
+  gateway,
+  /*
+   * A signed-in reader and an API integration each get the larger budget,
+   * keyed on the credential itself so two of them never share one. The gate
+   * exempts both outright -- it is deciding whether to charge a crawler, and a
+   * session is good evidence of a person. Here they are still metered, because
+   * this is deciding whether anyone at all is reading 19,000 pages an hour.
+   */
+  credentialFrom: (request) =>
+    sessionKey(request) ??
+    (hasApiBearer(request)
+      ? (/^Bearer\s+(\S+)/i.exec(request.headers.get('authorization') ?? '')?.[1] ?? null)
+      : null),
+  credential: { limit: 600, ceiling: 1200 },
+  rules: [
+    /* The expensive ones, at the numbers they were already tuned to. */
+    { path: '/api/search/', limit: 30 },
+    { path: '/api/dht/', limit: 30 },
+    { path: '/api/torrent-search', limit: 30 },
+    { path: '/search', limit: 60 },
+    { path: '/dht', limit: 60 },
+    /* Sign-in stays address-bucketed, or a guess buys the session budget. */
+    { path: '/api/auth/', limit: 10, credential: false },
+  ],
+});
+
+/** Resolves to a Response for a caller over the allowance, or undefined. */
+export const meter = (request: Request) => throttle.handle(request);
+
+export { SESSION_COOKIE_NAME };

src/proxy.test.ts

@@ -7,7 +7,9 @@
  * - Good bots (Googlebot, Bingbot, Applebot): rate-limited (10/min), NOT blocked
  * - Bad bots on expensive routes (/api/search/*, /api/dht/*): blocked (403)
  * - Bad bots on other API routes: rate-limited (5/min), allowed through
- * - Normal browsers: rate-limited on expensive routes (30/min)
+ * - Normal browsers: rate-limited on expensive routes (30/min) and on every
+ *   other route at the house default (100/min), which is the point of the
+ *   throttle: the routes nobody listed are the ones that get walked
  * - Supabase session: refreshed (cookie rewritten) when the access token expires within 60s
  * - ?ref=CODE: stored in the referral_code cookie when valid
  */
@@ -455,3 +457,46 @@ describe('Edge controls: hosting ranges and spoofed browsers', () => {
     });
   });
 });
+
+const BROWSER_UA =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36';
+
+describe('The site-wide allowance', () => {
+  /** Every case needs its own address: the counter is per caller and module-level. */
+  function call(pathname: string, ip: string) {
+    return middleware(
+      new NextRequest(new URL(`http://localhost${pathname}`), {
+        headers: {
+          'user-agent': BROWSER_UA,
+          'sec-fetch-mode': 'navigate',
+          'x-real-ip': ip,
+        },
+      })
+    );
+  }
+
+  async function countUntilLimited(pathname: string, ip: string, attempts: number) {
+    let allowed = 0;
+    for (let i = 0; i < attempts; i++) {
+      const res = await call(pathname, ip);
+      if (res && res.status !== 200) break;
+      allowed++;
+    }
+    return allowed;
+  }
+
+  // The gap this closes. A route nobody thought to list used to be unmetered
+  // however hard it was hit; on coinpayportal that was 19,000 URLs a day.
+  it('meters a route nobody listed', async () => {
+    expect(await countUntilLimited('/torrent/abc123', '10.9.0.1', 140)).toBe(100);
+  });
+
+  it('keeps the expensive routes at the number they were tuned to', async () => {
+    expect(await countUntilLimited('/api/torrent-search', '10.9.0.2', 60)).toBe(30);
+  });
+
+  it('gives each caller its own allowance', async () => {
+    expect(await countUntilLimited('/torrent/abc123', '10.9.0.3', 5)).toBe(5);
+    expect(await countUntilLimited('/torrent/abc123', '10.9.0.4', 5)).toBe(5);
+  });
+});

src/proxy.ts

@@ -4,7 +4,7 @@
  * The one live proxy file. In order:
  * - Charges AI training crawlers for access (402 + x402 offer) via the crawl gateway
  * - Refreshes the Supabase session cookie when the access token is about to expire
- * - Rate limits expensive API routes (sliding window, per-IP)
+ * - Meters every route at 100 req/min per caller, selling a pass to whoever goes over
  * - Blocks known bots/crawlers from hitting API routes (with exceptions for good bots)
  * - Enforces profile selection for authenticated users
  * - Stores a valid ?ref= referral code in a cookie
@@ -17,6 +17,7 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { trackReferralCode } from '@profullstack/stack/referrals';
 import { gateway } from '@/lib/crawl-gateway';
+import { meter } from '@/lib/throttle';
 
 // =============================================================================
 // Rate Limiting (in-memory sliding window)
@@ -36,8 +37,6 @@ const WINDOW_MS = 60_000; // 1 minute sliding window
 
 /** Rate limit tiers (requests per minute) */
 const RATE_LIMITS = {
-  api: 30,        // /api/search/*, /api/dht/*, /api/torrent-search
-  page: 60,       // /search, /dht page routes
   goodBot: 10,    // Googlebot, Bingbot, Applebot
   badBot: 5,      // All other bots
 } as const;
@@ -180,20 +179,10 @@ const EXPENSIVE_API_PATHS = [
   '/api/torrent-search',
 ];
 
-/** Page paths that should be rate limited (more generous) */
-const RATE_LIMITED_PAGE_PATHS = [
-  '/search',
-  '/dht',
-];
-
 function isExpensiveApiRoute(pathname: string): boolean {
   return EXPENSIVE_API_PATHS.some(p => pathname.startsWith(p));
 }
 
-function isRateLimitedPageRoute(pathname: string): boolean {
-  return RATE_LIMITED_PAGE_PATHS.some(p => pathname === p || pathname.startsWith(p + '/'));
-}
-
 // =============================================================================
 // Supabase session refresh
 // =============================================================================
@@ -495,22 +484,16 @@ export async function proxy(request: NextRequest): Promise<Response> {
     }
   }
 
-  // --- Rate limiting for expensive API routes (non-bot requests) ---
-  if (!isBotRequest && isExpensiveApiRoute(pathname)) {
-    const result = checkRateLimit(`api:${clientIp}`, RATE_LIMITS.api);
-    if (!result.allowed) {
-      console.log(`[rate-limit] API rate limited: IP=${clientIp} path=${pathname}`);
-      return withSession(make429Response(result.retryAfterSec ?? 60, true), session);
-    }
-  }
-
-  // --- Rate limiting for page routes ---
-  if (!isBotRequest && !isApiRoute && isRateLimitedPageRoute(pathname)) {
-    const result = checkRateLimit(`page:${clientIp}`, RATE_LIMITS.page);
-    if (!result.allowed) {
-      console.log(`[rate-limit] Page rate limited: IP=${clientIp} path=${pathname}`);
-      return withSession(make429Response(result.retryAfterSec ?? 60, false), session);
-    }
+  // --- The site-wide allowance ---
+  // 100 requests a minute per caller on EVERY route, with the expensive ones
+  // kept at the numbers they were tuned to (see lib/throttle.ts). Going over
+  // is answered 402 with the gate's own offer rather than 429, so a scraper
+  // that declares nothing is sold the same pass GPTBot buys. The bot tiers
+  // above still run first and stay tighter; this is the floor under them.
+  const overLimit = await meter(request);
+  if (overLimit) {
+    console.log(`[throttle] ${overLimit.status}: IP=${clientIp} path=${pathname} UA=${userAgent?.slice(0, 80)}`);
+    return withSession(overLimit as NextResponse, session);
   }
 
   // --- Profile enforcement: authenticated users must select a profile ---