From 97f046cfe22c3893eccb81cfb20264bbde9b9870 Mon Sep 17 00:00:00 2001 From: barckcode Date: Tue, 15 Sep 2026 21:55:17 +0100 Subject: [PATCH] feat(docs): publish per-model concurrency with tier numbers and per-key ceiling --- i18n/en.json | 2 +- i18n/es.json | 2 +- src/components/docs/RateLimits.astro | 36 +++- src/lib/__fixtures__/ratelimits.expected.md | 16 +- src/lib/mdxToText.test.ts | 49 ++++- src/lib/mdxToText.ts | 26 ++- src/lib/openapiSpec.test.ts | 48 ++++- src/lib/rateLimits.test.ts | 77 +++++++- src/lib/rateLimits.ts | 192 +++++++++++++++++++- src/lib/rateLimitsI18n.test.ts | 26 ++- src/tests/landing/pricing.test.ts | 5 +- src/tests/lib/rateLimits.test.ts | 38 +++- 12 files changed, 476 insertions(+), 41 deletions(-) diff --git a/i18n/en.json b/i18n/en.json index 858f4d6..cfc99b4 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -395,7 +395,7 @@ "GLM 5.3, frontier open model with reasoning", "3,000M token allowance per billing period", "400M tokens per rolling 4h window: the limit a coding agent reaches first", - "1M context · 5 concurrent requests", + "1M context · 10 concurrent requests", "Everything in nan_member", "Access switches on a few minutes after payment" ], diff --git a/i18n/es.json b/i18n/es.json index 6ca1e15..905f59c 100644 --- a/i18n/es.json +++ b/i18n/es.json @@ -395,7 +395,7 @@ "GLM 5.3, modelo abierto frontier con razonamiento", "Asignación de 3.000M de tokens por periodo de facturación", "400M de tokens por ventana deslizante de 4h: el límite que un agente de código alcanza primero", - "Contexto de 1M · 5 peticiones en paralelo", + "Contexto de 1M · 10 peticiones en paralelo", "Todo lo incluido en nan_member", "El acceso se activa pocos minutos después del pago" ], diff --git a/src/components/docs/RateLimits.astro b/src/components/docs/RateLimits.astro index b0e9fa2..b2f88dd 100644 --- a/src/components/docs/RateLimits.astro +++ b/src/components/docs/RateLimits.astro @@ -1,15 +1,19 @@ --- import { env } from 'cloudflare:workers'; import { + concurrencyValue, formatTokens, getRateLimitsConfig, + perKeyOuterCapValue, + premiumConcurrency, windowedModelBody, windowedModelHeadline, rateLimitsLabels, } from '../../lib/rateLimits'; -const { perKey, tokensPerMinuteByModel, requestsPerMinuteByModel, windowedModels } = - getRateLimitsConfig(env); +const config = getRateLimitsConfig(env); +const { perKey, tokensPerMinuteByModel, requestsPerMinuteByModel, windowedModels, concurrencyByModel } = + config; // This card is embedded from both the English and the Spanish guides, and MDX // content cannot pass props down from the layout, so the locale is read off the @@ -29,10 +33,32 @@ const T = rateLimitsLabels(lang);
{perKey.requestsPerMinute} rpm
-
{T.maxParallel}
-
{perKey.maxParallel} {T.concurrent}
+
{T.perKeyConcurrency}
+
{T.concurrencyPointer}
+
+
{T.perKeyAcrossModels}
+
{perKeyOuterCapValue(perKey, lang)}
+
+ + + +
+

+ {T.concurrencyPerModel} +

+

{T.concurrencyNote}

+
+ { + concurrencyByModel.map((c) => ( +
+
{c.model}
+
{concurrencyValue(c, lang)}
+
+ )) + }
+

{T.concurrencyExempt}

{ @@ -62,7 +88,7 @@ const T = rateLimitsLabels(lang);
{T.concurrentRequests}
-
{m.maxParallel}
+
{premiumConcurrency(config, m.model, m.maxParallel)}

diff --git a/src/lib/__fixtures__/ratelimits.expected.md b/src/lib/__fixtures__/ratelimits.expected.md index 02dbb8a..59263f4 100644 --- a/src/lib/__fixtures__/ratelimits.expected.md +++ b/src/lib/__fixtures__/ratelimits.expected.md @@ -1,14 +1,26 @@ **rate limits per API key** - Requests / min: 60 rpm -- Max parallel: 5 concurrent +- Concurrent requests: per model — see the per-model limits below + +**concurrent requests per model** + +- glm5.3: 7 (base plan) · 10 (premium plan) +- glm5.3-flash: 7 (base plan) · 10 (premium plan) +- deepseek-v4-flash: 7 (base plan) · 10 (premium plan) +- qwen3.8-flash: 7 (base plan) · 10 (premium plan) +- mimo-v2.5: 5 +- qwen3.6: 5 +- gemma4: 5 + +Audio, embedding and rerank endpoints have no concurrency limit. **glm5.3 · premium tier limits** - Rolling 4h window: 400M tokens - Allowance / billing period: 3,000M tokens - Context window: 1M tokens -- Concurrent requests: 5 +- Concurrent requests: 10 400M tokens per rolling 4 hours is the limit a heavy coding-agent run reaches first, well before the allowance. Once you hit it, glm5.3 requests are rejected until the window slides forward: it is a rolling window, not a daily reset. The allowance counter goes back to zero when your billing period starts, and if you upgrade part-way into a period that first allowance is prorated to the share of the period you paid for. diff --git a/src/lib/mdxToText.test.ts b/src/lib/mdxToText.test.ts index c963639..a5c6ca0 100644 --- a/src/lib/mdxToText.test.ts +++ b/src/lib/mdxToText.test.ts @@ -89,9 +89,13 @@ describe('mdxToText rate limits', () => { it('serves the values from the injected config, not hardcoded ones', async () => { const out = await mdxToText(input, { - perKey: { requestsPerMinute: 120, maxParallel: 8 }, + perKey: { requestsPerMinute: 120, maxParallel: 8, tierMaxParallel: { inference: 7, premium: 10 } }, tokensPerMinuteByModel: [{ model: 'foo', label: '2M tpm' }], requestsPerMinuteByModel: [{ model: 'bar', label: '500 rpm' }], + concurrencyByModel: [ + { model: 'foo', maxParallel: 5, tierMaxParallel: { inference: 7, premium: 10 } }, + { model: 'plain', maxParallel: 5 }, + ], windowedModels: [ { model: 'baz', @@ -104,35 +108,70 @@ describe('mdxToText rate limits', () => { ], }); expect(out).toContain('- Requests / min: 120 rpm'); - expect(out).toContain('- Max parallel: 8 concurrent'); + // maxParallel is the legacy outer cap: no surface renders it, the + // per-key row points at the per-model table instead. + expect(out).not.toContain('- Max parallel'); + expect(out).toContain('- Concurrent requests: per model — see the per-model limits below'); expect(out).toContain('- foo: 2M tpm'); expect(out).toContain('- bar: 500 rpm'); + expect(out).toContain('**concurrent requests per model**'); + expect(out).toContain('- foo: 7 (base plan) · 10 (premium plan)'); + expect(out).toContain('- plain: 5'); + // The card names the exempt endpoints under the per-model table; the + // plain-text renderer says the same thing. + expect(out).toContain('Audio, embedding and rerank endpoints have no concurrency limit.'); expect(out).toContain('**baz · premium tier limits**'); expect(out).toContain('- Rolling 6h window: 7M tokens'); expect(out).toContain('- Allowance / billing period: 9M tokens'); expect(out).toContain('- Context window: 128K tokens'); + // baz has no row in concurrencyByModel, so the card falls back to the + // windowed entry's own flat default. expect(out).toContain('- Concurrent requests: 2'); }); it('defaults to the same numbers renders', async () => { const out = await mdxToText(input); expect(out).toContain('- Requests / min: 60 rpm'); - expect(out).toContain('- Max parallel: 5 concurrent'); + // The per-key block no longer publishes a flat concurrency number (it + // said 5, which the per-model tiers made false): it points at the + // per-model table below instead. + expect(out).not.toContain('- Max parallel'); + expect(out).not.toContain('5 concurrent'); + expect(out).toContain('- Concurrent requests: per model — see the per-model limits below'); + // The per-model concurrency card, with the tier numbers of the four + // frontier models and the flat 5 of the rest. + expect(out).toContain('**concurrent requests per model**'); + expect(out).toContain('- glm5.3: 7 (base plan) · 10 (premium plan)'); + expect(out).toContain('- glm5.3-flash: 7 (base plan) · 10 (premium plan)'); + expect(out).toContain('- deepseek-v4-flash: 7 (base plan) · 10 (premium plan)'); + expect(out).toContain('- qwen3.8-flash: 7 (base plan) · 10 (premium plan)'); + expect(out).toContain('- mimo-v2.5: 5'); + expect(out).toContain('- qwen3.6: 5'); + expect(out).toContain('- gemma4: 5'); // glm5.3 is gated by the window, not by a per-minute rate, and the docs - // had no row for it at all while the model was already being served. + // had no row for it at all while the model was already being served. Its + // concurrency row resolves the premium tier's number. expect(out).toContain('- Rolling 4h window: 400M tokens'); expect(out).toContain('- Allowance / billing period: 3,000M tokens'); + expect(out).toContain('- Concurrent requests: 10'); + // The exempt-endpoints note travels with the per-model table on every + // surface, so a consumer never reads the list as exhaustive. + expect(out).toContain('Audio, embedding and rerank endpoints have no concurrency limit.'); }); it('omits the per-model blocks when they are empty', async () => { const out = await mdxToText(input, { - perKey: { requestsPerMinute: 60, maxParallel: 5 }, + perKey: { requestsPerMinute: 60, maxParallel: 5, tierMaxParallel: { inference: 7, premium: 10 } }, tokensPerMinuteByModel: [], requestsPerMinuteByModel: [], + concurrencyByModel: [], windowedModels: [], }); expect(out).not.toContain('tokens / min per model'); expect(out).not.toContain('requests / min per model'); + expect(out).not.toContain('concurrent requests per model'); + // The note is tied to the table's presence, not a standalone block. + expect(out).not.toContain('Audio, embedding and rerank endpoints have no concurrency limit.'); expect(out).not.toContain('premium tier limits'); }); }); diff --git a/src/lib/mdxToText.ts b/src/lib/mdxToText.ts index 433661c..ecddac5 100644 --- a/src/lib/mdxToText.ts +++ b/src/lib/mdxToText.ts @@ -4,8 +4,11 @@ import remarkGfm from 'remark-gfm'; import remarkMdx from 'remark-mdx'; import remarkStringify from 'remark-stringify'; import { + concurrencyValue, DEFAULT_RATE_LIMITS, formatTokens, + premiumConcurrency, + rateLimitsLabels, windowedModelNote, type RateLimitsConfig, } from './rateLimits'; @@ -416,13 +419,26 @@ function brandIntroToMd(node: MdxNode): string { } function rateLimitsToMd(config: RateLimitsConfig): string { + // The canonical text is English-only, so the labels resolve to the English + // table: the per-key block and the card must not be able to drift apart. + const L = rateLimitsLabels('en'); const lines = [ - '**rate limits per API key**', + `**${L.perKey}**`, '', - `- Requests / min: ${config.perKey.requestsPerMinute} rpm`, - `- Max parallel: ${config.perKey.maxParallel} concurrent`, + `- ${L.requestsPerMin}: ${config.perKey.requestsPerMinute} rpm`, + // Concurrency is enforced per model, so the per-key block points at the + // per-model table below instead of a flat number the tiers made false. + `- ${L.perKeyConcurrency}: ${L.concurrencyPointer}`, '', ]; + if (config.concurrencyByModel.length) { + lines.push('**concurrent requests per model**', ''); + for (const c of config.concurrencyByModel) lines.push(`- ${c.model}: ${concurrencyValue(c)}`); + // The card names the endpoints the list above does not cover; the bot + // consumer gets the same sentence, or the list reads as exhaustive. The + // blank line first keeps the note out of the last list item. + lines.push('', L.concurrencyExempt, ''); + } for (const m of config.windowedModels) { lines.push( `**${m.model} · premium tier limits**`, @@ -430,7 +446,9 @@ function rateLimitsToMd(config: RateLimitsConfig): string { `- Rolling ${m.windowHours}h window: ${formatTokens(m.windowTokens)} tokens`, `- Allowance / billing period: ${formatTokens(m.periodCapTokens)} tokens`, `- Context window: ${formatTokens(m.contextTokens)} tokens`, - `- Concurrent requests: ${m.maxParallel}`, + // The premium card addresses premium members, so it resolves the + // premium tier's number rather than the flat default. + `- Concurrent requests: ${premiumConcurrency(config, m.model, m.maxParallel)}`, '', windowedModelNote(m), '', diff --git a/src/lib/openapiSpec.test.ts b/src/lib/openapiSpec.test.ts index a418454..229bb02 100644 --- a/src/lib/openapiSpec.test.ts +++ b/src/lib/openapiSpec.test.ts @@ -197,20 +197,23 @@ describe('openapi.json: rate limits come from the single source of truth', () => expect(description).toContain( `| Requests per minute | ${DEFAULT_RATE_LIMITS.perKey.requestsPerMinute} |`, ); - expect(description).toContain( - `| Concurrent requests | ${DEFAULT_RATE_LIMITS.perKey.maxParallel} |`, - ); + // Concurrency is enforced per model, so the row points at the per-model + // table below instead of a flat per-key number. + expect(description).toContain('| Concurrent requests | per model — see the per-model limits below |'); }); /** An env override has to reach /docs/api, not only /docs/models. */ - it('follows an env override of the per-key limits', () => { + it('follows an env override of the per-key rate, and leaks no legacy parallel cap', () => { const overridden = getRateLimitsConfig({ RATE_LIMIT_RPM: '250', RATE_LIMIT_PARALLEL: '9' }); const description = resolveSpec(overridden).info.description; expect(description).toContain('| Requests per minute | 250 |'); - expect(description).toContain('| Concurrent requests | 9 |'); expect(description).not.toContain( `| Requests per minute | ${DEFAULT_RATE_LIMITS.perKey.requestsPerMinute} |`, ); + // RATE_LIMIT_PARALLEL is the legacy outer cap: it still parses (env + // overrides must not crash) but no surface publishes it any more. + expect(description).not.toContain('| Concurrent requests | 9 |'); + expect(description).toContain('| Concurrent requests | per model — see the per-model limits below |'); }); it('publishes every model that carries a per-minute limit', () => { @@ -231,6 +234,41 @@ describe('openapi.json: rate limits come from the single source of truth', () => expect(description).toContain(`${formatTokens(m.contextTokens)} tokens`); } }); + + it('publishes the per-model concurrency with the tier numbers', () => { + const description = resolveSpec(DEFAULT_RATE_LIMITS).info.description; + expect(description).toContain('Concurrency is enforced per model'); + // Grouped the way the per-minute rows are, so the Rate limits section + // grows no `| `glm5.3` |` row of its own that could shadow the Model + // catalog's row for the same model. + expect(description).toContain( + '| `glm5.3`, `glm5.3-flash`, `deepseek-v4-flash`, `qwen3.8-flash` | 7 (base plan) · 10 (premium plan) |', + ); + expect(description).toContain('| `mimo-v2.5`, `qwen3.6`, `gemma4` | 5 |'); + }); + + it('names the endpoints the per-model concurrency table does not cover', () => { + const description = resolveSpec(DEFAULT_RATE_LIMITS).info.description; + expect(description).toContain('Audio, embedding and rerank endpoints have no concurrency limit.'); + }); + + it('gives a model whose numbers differ its own row, not its neighbour\'s', () => { + const config = { + ...DEFAULT_RATE_LIMITS, + concurrencyByModel: [ + ...DEFAULT_RATE_LIMITS.concurrencyByModel, + { model: 'frontier-next', maxParallel: 5, tierMaxParallel: { inference: 8, premium: 12 } }, + ], + }; + const description = resolveSpec(config).info.description; + expect(description).toContain('| `frontier-next` | 8 (base plan) · 12 (premium plan) |'); + }); + + it('resolves the premium concurrency in the windowed note', () => { + const description = resolveSpec(DEFAULT_RATE_LIMITS).info.description; + // glm5.3 is premium-only, so the note states the premium tier's number. + expect(description).toMatch(/Context window: 1M tokens, 10 concurrent requests\./); + }); }); /** diff --git a/src/lib/rateLimits.test.ts b/src/lib/rateLimits.test.ts index de6190a..689f3fc 100644 --- a/src/lib/rateLimits.test.ts +++ b/src/lib/rateLimits.test.ts @@ -1,5 +1,10 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { DEFAULT_RATE_LIMITS, getRateLimitsConfig } from './rateLimits'; +import { + DEFAULT_RATE_LIMITS, + getRateLimitsConfig, + perKeyOuterCapValue, + premiumConcurrency, +} from './rateLimits'; afterEach(() => { vi.restoreAllMocks(); @@ -15,13 +20,20 @@ describe('getRateLimitsConfig', () => { RATE_LIMIT_RPM: '120', RATE_LIMIT_PARALLEL: '8', }); - expect(config.perKey).toEqual({ requestsPerMinute: 120, maxParallel: 8 }); + // The env moves the two env-driven fields; the outer ceiling per tier is a + // code constant (LiteLLM enforces it on the key) and stays put. + expect(config.perKey).toEqual({ + ...DEFAULT_RATE_LIMITS.perKey, + requestsPerMinute: 120, + maxParallel: 8, + }); }); it('leaves the per-model tables untouched', () => { const config = getRateLimitsConfig({ RATE_LIMIT_RPM: '120' }); expect(config.tokensPerMinuteByModel).toEqual(DEFAULT_RATE_LIMITS.tokensPerMinuteByModel); expect(config.requestsPerMinuteByModel).toEqual(DEFAULT_RATE_LIMITS.requestsPerMinuteByModel); + expect(config.concurrencyByModel).toEqual(DEFAULT_RATE_LIMITS.concurrencyByModel); }); it.each(['0', '-1', 'abc', '1.5', '60rpm'])( @@ -41,7 +53,64 @@ describe('getRateLimitsConfig', () => { expect(warn).not.toHaveBeenCalled(); }); - it('defaults to the values main settled on: 60 rpm, 5 concurrentes', () => { - expect(DEFAULT_RATE_LIMITS.perKey).toEqual({ requestsPerMinute: 60, maxParallel: 5 }); + it('defaults to the values main settled on: 60 rpm, legacy flat cap 5, outer ceiling 7/10', () => { + // maxParallel stays in the data model for RATE_LIMIT_PARALLEL env-override + // compatibility, but no published surface renders it any more. The real + // per-key ceiling is tierMaxParallel: LiteLLM caps the key at 7 (base) / + // 10 (premium) across ALL models combined, and the per-key card publishes it. + expect(DEFAULT_RATE_LIMITS.perKey).toEqual({ + requestsPerMinute: 60, + maxParallel: 5, + tierMaxParallel: { inference: 7, premium: 10 }, + }); + }); +}); + +describe('the per-key outer ceiling', () => { + it('publishes the key-level cap in the member vocabulary, both locales', () => { + // The exact value the per-key card renders next to the "Across all + // models" / "En todos los modelos" label: LiteLLM caps the KEY at + // max_parallel_requests across all models combined, so a base member who + // sums the per-model allowances (7+7+5+5) plans past it and hits an + // unexplained 429 unless this is published. + expect(perKeyOuterCapValue(DEFAULT_RATE_LIMITS.perKey, 'en')).toBe( + '7 (base plan) · 10 (premium plan) simultaneous requests per key', + ); + expect(perKeyOuterCapValue(DEFAULT_RATE_LIMITS.perKey, 'es')).toBe( + '7 (plan base) · 10 (plan premium) peticiones simultáneas por key', + ); + }); +}); + +describe('per-model concurrency', () => { + const TIERED = ['glm5.3', 'glm5.3-flash', 'deepseek-v4-flash', 'qwen3.8-flash']; + + it('keeps the flat default at 5 for every model, the tiered ones included', () => { + for (const c of DEFAULT_RATE_LIMITS.concurrencyByModel) { + expect(c.maxParallel, c.model).toBe(5); + } + }); + + it('raises the four frontier models to 7 (inference) / 10 (premium)', () => { + for (const model of TIERED) { + const entry = DEFAULT_RATE_LIMITS.concurrencyByModel.find((c) => c.model === model); + expect(entry, model).toBeDefined(); + expect(entry!.tierMaxParallel).toEqual({ inference: 7, premium: 10 }); + } + }); + + it('leaves every other model at the flat default, with no tier variant', () => { + const rest = DEFAULT_RATE_LIMITS.concurrencyByModel.filter((c) => !TIERED.includes(c.model)); + expect(rest.length).toBeGreaterThan(0); + for (const c of rest) { + expect(c.maxParallel, c.model).toBe(5); + expect(c.tierMaxParallel, c.model).toBeUndefined(); + } + }); + + it('resolves the premium number the premium card publishes', () => { + expect(premiumConcurrency(DEFAULT_RATE_LIMITS, 'glm5.3', 5)).toBe(10); + // A model without a tier variant falls back to the flat default. + expect(premiumConcurrency(DEFAULT_RATE_LIMITS, 'mimo-v2.5', 5)).toBe(5); }); }); diff --git a/src/lib/rateLimits.ts b/src/lib/rateLimits.ts index 5add960..0d3df26 100644 --- a/src/lib/rateLimits.ts +++ b/src/lib/rateLimits.ts @@ -12,7 +12,26 @@ export interface PerKeyRateLimits { requestsPerMinute: number; + /** + * Legacy flat outer cap default. Kept only so the RATE_LIMIT_PARALLEL env + * override keeps parsing (an override must not crash the config), but no + * published surface renders it any more: concurrency is enforced per model + * (see concurrencyByModel), and the real per-key ceiling is tiered (see + * tierMaxParallel below). It used to render as "Max parallel: 5 concurrent", + * which the per-model tiers made false. + */ maxParallel: number; + /** + * The per-key outer ceiling, per member tier: LiteLLM caps the KEY at + * max_parallel_requests across ALL models combined — 7 for the base + * (inference) plan, 10 for premium. It is enforced on the key itself, so a + * base member who sums the per-model allowances (7+7+5+5) and plans more + * than 7 in flight hits this cap first, on models whose own limit was never + * reached. Enforced by LiteLLM on the key, not by this site's env, so it is + * a code constant like the per-model tables, and the per-key card publishes + * it (perKeyOuterCapValue). + */ + tierMaxParallel: { inference: number; premium: number }; } export interface ModelRate { @@ -20,6 +39,27 @@ export interface ModelRate { label: string; } +/** + * Concurrency a model allows, per member tier. + * + * Concurrency is enforced per model, not per key: every model allows the flat + * default below, and the frontier models raise it for the member's tier, 7 + * concurrent for inference-tier members and 10 for premium (glm_access) + * members. glm5.2 is premium too but intentionally stays at the flat 5, and + * it is hidden by owner decision; served via the glm5.3 group alias, so it + * has no row here. + */ +export interface ModelConcurrency { + model: string; + /** The flat default every tier falls back to; mirrors the backend default. */ + maxParallel: number; + /** Per-tier overrides; when present they supersede maxParallel for that tier. + * `inference` is the backend's name for the base plan; the published labels + * render it as "base plan" — the vocabulary a member knows — not as + * "inference tier". */ + tierMaxParallel?: { inference: number; premium: number }; +} + /** * Limits for a model that is not gated by a per-minute rate but by a sliding * window plus an allowance per billing period. Those are the two numbers a @@ -39,6 +79,7 @@ export interface RateLimitsConfig { perKey: PerKeyRateLimits; tokensPerMinuteByModel: ModelRate[]; requestsPerMinuteByModel: ModelRate[]; + concurrencyByModel: ModelConcurrency[]; windowedModels: WindowedModelLimits[]; } @@ -52,7 +93,12 @@ export interface RateLimitsEnv { * model is added or removed, which is a code change anyway. */ export const DEFAULT_RATE_LIMITS: RateLimitsConfig = { - perKey: { requestsPerMinute: 60, maxParallel: 5 }, + // maxParallel is the legacy outer cap default (see PerKeyRateLimits): no + // surface renders it; it exists for RATE_LIMIT_PARALLEL env-override compat. + // tierMaxParallel is the real per-key ceiling: LiteLLM caps the key at + // max_parallel_requests across all models combined, and the per-key card + // publishes it. + perKey: { requestsPerMinute: 60, maxParallel: 5, tierMaxParallel: { inference: 7, premium: 10 } }, tokensPerMinuteByModel: [ { model: 'deepseek-v4-flash', label: '1.5M tpm' }, { model: 'mimo-v2.5', label: '1.5M tpm' }, @@ -60,6 +106,24 @@ export const DEFAULT_RATE_LIMITS: RateLimitsConfig = { { model: 'gemma4', label: '1.5M tpm' }, ], requestsPerMinuteByModel: [{ model: 'rerank', label: '1000 rpm' }], + // Per-model concurrency. The flat default (5) is what every model allows; + // the four frontier models raise it per tier. The chat models are + // enumerated because they are the ones a member runs agents against; the + // utility endpoints have no rows because the hook does not govern them: + // audio, embeddings and rerank are exempt from the concurrency limit + // (FALLBACK_EXEMPT_MODELS), and images are not hook-governed at all. + // Env overrides do not reach this table: RATE_LIMIT_PARALLEL moves + // only the legacy per-key outer cap (PerKeyRateLimits.maxParallel), which + // no surface renders any more. + concurrencyByModel: [ + { model: 'glm5.3', maxParallel: 5, tierMaxParallel: { inference: 7, premium: 10 } }, + { model: 'glm5.3-flash', maxParallel: 5, tierMaxParallel: { inference: 7, premium: 10 } }, + { model: 'deepseek-v4-flash', maxParallel: 5, tierMaxParallel: { inference: 7, premium: 10 } }, + { model: 'qwen3.8-flash', maxParallel: 5, tierMaxParallel: { inference: 7, premium: 10 } }, + { model: 'mimo-v2.5', maxParallel: 5 }, + { model: 'qwen3.6', maxParallel: 5 }, + { model: 'gemma4', maxParallel: 5 }, + ], // glm5.3 (premium tier) is absent from the per-minute tables on purpose: its // gate is the 4h sliding window plus the allowance per billing period. These // mirror the backend policy (cloud-api modelRateLimits + the token cap for @@ -166,6 +230,50 @@ export function windowedModelNote(m: WindowedModelLimits, lang: DocsLocale = 'en return `${windowedModelHeadline(m, lang)} ${windowedModelBody(m, lang)}`; } +/** + * The concurrency a model allows a premium member: the number the premium + * card ("glm5.3 · premium tier limits") and the spec's windowed note publish. + * Those surfaces are addressed to premium members, so they resolve the + * premium variant; a model without a tier variant falls back to the flat + * default the caller hands in. + */ +export function premiumConcurrency( + config: RateLimitsConfig, + model: string, + fallback: number, +): number { + const entry = config.concurrencyByModel.find((c) => c.model === model); + return entry?.tierMaxParallel?.premium ?? fallback; +} + +/** + * The value of a per-model concurrency row: the flat number, or the pair of + * tier numbers when the model carries them. Shared by , + * rateLimitsToMd() and rateLimitsToSpecMarkdown() so the three surfaces + * cannot disagree about the wording either. + */ +export function concurrencyValue(c: ModelConcurrency, lang: DocsLocale = 'en'): string { + if (!c.tierMaxParallel) return `${c.maxParallel}`; + const L = rateLimitsLabels(lang); + return `${c.tierMaxParallel.inference} (${L.tierBase}) · ${c.tierMaxParallel.premium} (${L.tierPremium})`; +} + +/** + * The per-key card's outer-ceiling value: the number a member plans against + * when they run requests across models, because the key caps the total before + * any per-model limit is reached. Composed from the label table (the way + * concurrencyValue composes the per-model values) so the two locales cannot + * drift, and so the numbers come from the config instead of being retyped on + * the page. + */ +export function perKeyOuterCapValue(perKey: PerKeyRateLimits, lang: DocsLocale = 'en'): string { + const L = rateLimitsLabels(lang); + return ( + `${perKey.tierMaxParallel.inference} (${L.tierBase}) · ` + + `${perKey.tierMaxParallel.premium} (${L.tierPremium}) ${L.simultaneousPerKey}` + ); +} + function parsePositiveInt(raw: string | undefined, fallback: number, varName: string): number { if (raw === undefined || raw.trim() === '') return fallback; const n = Number(raw); @@ -191,6 +299,9 @@ export function getRateLimitsConfig(env: RateLimitsEnv = {}): RateLimitsConfig { DEFAULT_RATE_LIMITS.perKey.maxParallel, 'RATE_LIMIT_PARALLEL', ), + // The outer ceiling per tier is a code constant like the per-model + // table: LiteLLM enforces it on the key, so no env var reaches it. + tierMaxParallel: DEFAULT_RATE_LIMITS.perKey.tierMaxParallel, }, }; } @@ -212,11 +323,16 @@ export function getRateLimitsConfig(env: RateLimitsEnv = {}): RateLimitsConfig { */ export function rateLimitsToSpecMarkdown(config: RateLimitsConfig): string { const tpm = config.tokensPerMinuteByModel; + // The spec is English-only, so the labels resolve to the English table. + const L = rateLimitsLabels('en'); const rows = [ '| Limit | Value |', '| --- | --- |', `| Requests per minute | ${config.perKey.requestsPerMinute} |`, - `| Concurrent requests | ${config.perKey.maxParallel} |`, + // Concurrency is enforced per model, so instead of a flat per-key number + // (which the per-model tiers made false) the row points at the per-model + // table below. + `| ${L.perKeyConcurrency} | ${L.concurrencyPointer} |`, ]; if (tpm.length) { // The label already carries its unit ("1.5M tpm"), so the value column @@ -229,11 +345,48 @@ export function rateLimitsToSpecMarkdown(config: RateLimitsConfig): string { } const out = [ - 'Limits apply per API key (RPM and concurrency), not on total token volume:', + 'Limits apply per API key (requests per minute) and per model (concurrent requests), not on total token volume:', '', ...rows, ]; + if (config.concurrencyByModel.length) { + // Rows are grouped by value, the way the per-minute rows above group + // models comma-joined in the limit column. One row per model would also + // put `| `glm5.3` |` rows in the Rate limits section, ahead of the Model + // catalog's row for the same model, and any consumer reading "the first + // catalog row for X" would pick the wrong one. Grouping by VALUE rather + // than by shape keeps a model whose numbers differ from its neighbours on + // a row of its own instead of publishing their numbers for it. + out.push( + '', + 'Concurrency is enforced per model, not per key. The frontier models raise the default for the member\'s tier:', + '', + '| Model | Concurrent requests |', + '| --- | --- |', + ); + let group: string[] = []; + let groupValue = ''; + const flush = () => { + if (group.length) { + out.push(`| ${group.map((m) => `\`${m}\``).join(', ')} | ${groupValue} |`); + } + }; + for (const c of config.concurrencyByModel) { + const value = concurrencyValue(c); + if (value !== groupValue) { + flush(); + group = []; + groupValue = value; + } + group.push(c.model); + } + flush(); + // The card names the endpoints the table does not cover; the spec says + // the same, or a client author reads the rows above as exhaustive. + out.push('', L.concurrencyExempt); + } + for (const m of config.windowedModels) { out.push( '', @@ -242,7 +395,7 @@ export function rateLimitsToSpecMarkdown(config: RateLimitsConfig): string { `${m.windowHours} hours and a ${formatTokens(m.periodCapTokens)}-token allowance that ` + `returns to zero when your billing period starts. The window is rolling, not a daily ` + `reset. Context window: ${formatTokens(m.contextTokens)} tokens, ` + - `${m.maxParallel} concurrent requests.`, + `${premiumConcurrency(config, m.model, m.maxParallel)} concurrent requests.`, ); } @@ -259,8 +412,16 @@ export function rateLimitsLabels(lang: DocsLocale) { en: { perKey: 'rate limits per API key', requestsPerMin: 'Requests / min', - maxParallel: 'Max parallel', - concurrent: 'concurrent', + // The per-key concurrency row: concurrency is enforced per model, so the + // value points at the per-model card below instead of a flat number. + perKeyConcurrency: 'Concurrent requests', + concurrencyPointer: 'per model — see the per-model limits below', + // The per-key outer ceiling row: the key caps the total across all + // models, so the card states the ceiling next to the pointer — or a + // member summing the per-model numbers plans past it and eats an + // unexplained 429. + perKeyAcrossModels: 'Across all models', + simultaneousPerKey: 'simultaneous requests per key', premium: 'premium tier limits', window: (h: number) => `Rolling ${h}h window`, allowance: 'Allowance / billing period', @@ -268,12 +429,22 @@ export function rateLimitsLabels(lang: DocsLocale) { concurrentRequests: 'Concurrent requests', tokensPerModel: 'tokens / min per model', requestsPerModel: 'requests / min per model', + concurrencyPerModel: 'concurrent requests per model', + concurrencyNote: 'Concurrency is enforced per model, not per API key.', + // The utility endpoints have no per-model concurrency row: name them, so + // a member scripting against them does not read the list above as + // applying to every endpoint. + concurrencyExempt: 'Audio, embedding and rerank endpoints have no concurrency limit.', + tierBase: 'base plan', + tierPremium: 'premium plan', }, es: { perKey: 'límites por API key', requestsPerMin: 'Peticiones / min', - maxParallel: 'Máximo en paralelo', - concurrent: 'concurrentes', + perKeyConcurrency: 'Peticiones en paralelo', + concurrencyPointer: 'por modelo — ver los límites por modelo abajo', + perKeyAcrossModels: 'En todos los modelos', + simultaneousPerKey: 'peticiones simultáneas por key', premium: 'límites del tier premium', window: (h: number) => `Ventana móvil de ${h}h`, allowance: 'Cuota / periodo de facturación', @@ -281,6 +452,11 @@ export function rateLimitsLabels(lang: DocsLocale) { concurrentRequests: 'Peticiones concurrentes', tokensPerModel: 'tokens / min por modelo', requestsPerModel: 'peticiones / min por modelo', + concurrencyPerModel: 'peticiones concurrentes por modelo', + concurrencyNote: 'La concurrencia se aplica por modelo, no por API key.', + concurrencyExempt: 'Los endpoints de audio, embeddings y rerank no tienen límite de concurrencia.', + tierBase: 'plan base', + tierPremium: 'plan premium', }, }[lang]; } diff --git a/src/lib/rateLimitsI18n.test.ts b/src/lib/rateLimitsI18n.test.ts index 8a3b797..fb0827c 100644 --- a/src/lib/rateLimitsI18n.test.ts +++ b/src/lib/rateLimitsI18n.test.ts @@ -5,6 +5,7 @@ import { dirname, resolve } from 'node:path'; import { DEFAULT_RATE_LIMITS, formatTokens, + perKeyOuterCapValue, rateLimitsLabels, windowedModelBody, windowedModelHeadline, @@ -48,8 +49,11 @@ function visible(lang: 'en' | 'es'): string { return [ t.perKey, t.requestsPerMin, - t.maxParallel, - t.concurrent, + t.perKeyConcurrency, + t.concurrencyPointer, + t.perKeyAcrossModels, + t.simultaneousPerKey, + perKeyOuterCapValue(DEFAULT_RATE_LIMITS.perKey, lang), t.premium, t.window(model.windowHours), t.allowance, @@ -57,6 +61,11 @@ function visible(lang: 'en' | 'es'): string { t.concurrentRequests, t.tokensPerModel, t.requestsPerModel, + t.concurrencyPerModel, + t.concurrencyNote, + t.concurrencyExempt, + t.tierBase, + t.tierPremium, windowedModelHeadline(model, lang), windowedModelBody(model, lang), ] @@ -113,6 +122,19 @@ describe('the card renders no hardcoded copy', () => { ].map((m) => m[1].trim()); expect(literals, `inlined: ${literals.join(' | ')}`).toEqual([]); }); + + it('publishes the per-key outer ceiling through the shared composition', () => { + // The ceiling is the number a member hits first when they spread requests + // across models; if the card drops the row again this fails, instead of a + // member eating an unexplained 429. The pointer row stays: the ceiling + // does not replace the per-model table. + const here = dirname(fileURLToPath(import.meta.url)); + const source = readFileSync(resolve(here, '../components/docs/RateLimits.astro'), 'utf-8'); + const markup = source.slice(source.lastIndexOf('---') + 3); + expect(markup).toContain('{perKeyOuterCapValue(perKey, lang)}'); + expect(markup).toContain('{T.perKeyAcrossModels}'); + expect(markup).toContain('{T.concurrencyPointer}'); + }); }); /** diff --git a/src/tests/landing/pricing.test.ts b/src/tests/landing/pricing.test.ts index 7d8f2f1..941528d 100644 --- a/src/tests/landing/pricing.test.ts +++ b/src/tests/landing/pricing.test.ts @@ -167,7 +167,10 @@ describe.each(locales)('Pricing copy — %s', (locale) => { test('publishes context and concurrency', () => { const copy = premium().join(' '); expect(copy).toMatch(/1M context|Contexto de 1M/); - expect(copy).toMatch(/5 (concurrent requests|peticiones en paralelo)/); + // The bullet is the glm5.3 family's concurrency for a PREMIUM member: + // 10 under the per-model tier model (inference members get 7). + expect(copy).toMatch(/10 (concurrent requests|peticiones en paralelo)/); + expect(copy).not.toMatch(/5 (concurrent requests|peticiones en paralelo)/); }); test('refers to the member tier by the name the section renders', () => { diff --git a/src/tests/lib/rateLimits.test.ts b/src/tests/lib/rateLimits.test.ts index a29a300..fa49e12 100644 --- a/src/tests/lib/rateLimits.test.ts +++ b/src/tests/lib/rateLimits.test.ts @@ -3,10 +3,10 @@ import { DEFAULT_RATE_LIMITS, formatTokens, getRateLimitsConfig, + premiumConcurrency, windowedModelBody, windowedModelHeadline, windowedModelNote, - formatTokens, } from '../../lib/rateLimits'; /** @@ -18,7 +18,9 @@ import { * to what the platform actually enforces: * * context 1,048,576 cloud-api usage_quota.go modelRateLimits - * concurrency 5 idem, and the ratelimit hook + * concurrency 5 flat per-model default (idem, and the ratelimit hook); + * the frontier models raise it per tier, see + * concurrencyByModel: 7 inference / 10 premium * 400M per rolling 4h ratelimit hook ROLLING_WINDOW_S / rolling budget * 3,000M per period cloud-api usage_quota.go monthlyTokenCaps * @@ -78,8 +80,38 @@ describe('rateLimits — glm5.3 windowed limits', () => { test('getRateLimitsConfig keeps the windowed models when env overrides the per-key values', () => { const config = getRateLimitsConfig({ RATE_LIMIT_RPM: '120', RATE_LIMIT_PARALLEL: '9' }); - expect(config.perKey).toEqual({ requestsPerMinute: 120, maxParallel: 9 }); + // The env moves the two env-driven fields; the outer ceiling per tier is a + // code constant (LiteLLM enforces it on the key) and stays put. + expect(config.perKey).toEqual({ + ...DEFAULT_RATE_LIMITS.perKey, + requestsPerMinute: 120, + maxParallel: 9, + }); expect(config.windowedModels).toEqual(DEFAULT_RATE_LIMITS.windowedModels); + expect(config.concurrencyByModel).toEqual(DEFAULT_RATE_LIMITS.concurrencyByModel); + }); + + test('the premium card resolves the tier raise: 10 concurrent for premium members', () => { + // The windowed card is titled "glm5.3 · premium tier limits", so its + // concurrency row is the premium tier's number, not the flat default. + expect(premiumConcurrency(DEFAULT_RATE_LIMITS, 'glm5.3', glm!.maxParallel)).toBe(10); + }); + + test('the frontier models carry the tier variants, the rest stay flat', () => { + const tiered = DEFAULT_RATE_LIMITS.concurrencyByModel.filter((c) => c.tierMaxParallel); + expect(tiered.map((c) => c.model)).toEqual([ + 'glm5.3', + 'glm5.3-flash', + 'deepseek-v4-flash', + 'qwen3.8-flash', + ]); + for (const c of tiered) expect(c.tierMaxParallel).toEqual({ inference: 7, premium: 10 }); + // glm5.2 is premium too but intentionally stays at the flat 5, and it is + // hidden by owner decision; served via the glm5.3 group alias, so it has + // no row here at all. + expect(DEFAULT_RATE_LIMITS.concurrencyByModel.map((c) => c.model)).not.toContain('glm5.2'); + const flat = DEFAULT_RATE_LIMITS.concurrencyByModel.filter((c) => !c.tierMaxParallel); + for (const c of flat) expect(c.maxParallel, c.model).toBe(5); }); });