From 0d78c5b8d7dcec0391de76dc05369928ef3565c1 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 11 Sep 2026 06:42:59 +0000 Subject: [PATCH] Pair discovery and equity backtests can read nichedb Every five minutes the arb daemon asked CoinGecko and four exchanges which pairs exist, how much they traded and what they are worth; a backtest asked Yahoo for a year of bars. nichedb.dev keeps both now. With NICHEDB_CRYPTO=1 discovery reads the dollar-quoted pairs and the assets by market cap from nichedb, about ten requests, and applies the same rules: two venues or more, fifty thousand dollars a day on each, ten million of market cap, a null volume passing the way the Gemini sentinel did. With NICHEDB_MARKETS=1 a backtest reads the symbol's history item and falls back to Yahoo when the window is longer than the item, the item is stale, or the symbol is too new. Either switch off means no request; any failure means the old path. The client is its own package, since the web route and the CLI need it and must not pull the exchange feeds in to read a bar. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014X1XCwt5iD3nqZfbddZqEP --- apps/cli/package.json | 1 + apps/cli/src/strategy-backtest.test.ts | 80 +++++- apps/cli/src/strategy-backtest.ts | 38 ++- apps/web/next.config.ts | 1 + apps/web/package.json | 1 + .../app/api/strategies/backtest/route.test.ts | 97 ++++++- .../src/app/api/strategies/backtest/route.ts | 34 ++- packages/source-crypto-arb/package.json | 1 + .../src/pair-discovery.test.ts | 171 ++++++++++++ .../source-crypto-arb/src/pair-discovery.ts | 81 ++++++ packages/source-nichedb/package.json | 22 ++ packages/source-nichedb/src/client.test.ts | 131 +++++++++ packages/source-nichedb/src/client.ts | 174 ++++++++++++ packages/source-nichedb/src/crypto.test.ts | 254 ++++++++++++++++++ packages/source-nichedb/src/crypto.ts | 233 ++++++++++++++++ packages/source-nichedb/src/env.ts | 28 ++ packages/source-nichedb/src/index.ts | 54 ++++ packages/source-nichedb/src/markets.test.ts | 139 ++++++++++ packages/source-nichedb/src/markets.ts | 148 ++++++++++ packages/source-nichedb/tsconfig.build.json | 9 + packages/source-nichedb/tsconfig.json | 4 + pnpm-lock.yaml | 21 ++ 22 files changed, 1711 insertions(+), 11 deletions(-) create mode 100644 packages/source-crypto-arb/src/pair-discovery.test.ts create mode 100644 packages/source-nichedb/package.json create mode 100644 packages/source-nichedb/src/client.test.ts create mode 100644 packages/source-nichedb/src/client.ts create mode 100644 packages/source-nichedb/src/crypto.test.ts create mode 100644 packages/source-nichedb/src/crypto.ts create mode 100644 packages/source-nichedb/src/env.ts create mode 100644 packages/source-nichedb/src/index.ts create mode 100644 packages/source-nichedb/src/markets.test.ts create mode 100644 packages/source-nichedb/src/markets.ts create mode 100644 packages/source-nichedb/tsconfig.build.json create mode 100644 packages/source-nichedb/tsconfig.json diff --git a/apps/cli/package.json b/apps/cli/package.json index eac9c58..73bd029 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -19,6 +19,7 @@ "@b1dz/storage-supabase": "workspace:*", "@b1dz/source-crypto-arb": "workspace:*", "@b1dz/source-crypto-trade": "workspace:*", + "@b1dz/source-nichedb": "workspace:*", "@b1dz/source-strategies": "workspace:*", "@b1dz/storage-b1dz-api": "workspace:*", "@b1dz/sdk": "workspace:*", diff --git a/apps/cli/src/strategy-backtest.test.ts b/apps/cli/src/strategy-backtest.test.ts index a2cf368..11b9854 100644 --- a/apps/cli/src/strategy-backtest.test.ts +++ b/apps/cli/src/strategy-backtest.test.ts @@ -1,5 +1,81 @@ -import { describe, it, expect } from 'vitest'; -import { parseArgs } from './strategy-backtest.js'; +import { afterEach, describe, it, expect, vi } from 'vitest'; +import { fetchDailySnapshots, parseArgs } from './strategy-backtest.js'; + +describe('strategy-backtest daily bars via nichedb (NICHEDB_MARKETS)', () => { + const DAY = 24 * 60 * 60 * 1000; + const today = Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate()); + + function historyItem(symbol: string, count = 400) { + const bars: (string | number | null)[][] = []; + let ms = today - DAY; + while (bars.length < count) { + const dow = new Date(ms).getUTCDay(); + if (dow !== 0 && dow !== 6) bars.unshift([new Date(ms).toISOString().slice(0, 10), 10, 11, 9, 10 + bars.length, 1000, null]); + ms -= DAY; + } + return { + id: 7, collection: 'markets', kind: 'history', external_id: `history:${symbol}`, title: `${symbol} daily bars`, + published_at: null, updated_at: new Date().toISOString(), tags: ['history', `symbol:${symbol.toLowerCase()}`, 'feed:iex'], + data: { symbol, timeframe: '1Day', feed: 'iex', adjustment: 'split', bars, first: bars[0]![0], last: bars[bars.length - 1]![0], count: bars.length }, + }; + } + + function installFetch(nichedbItems: unknown[]) { + const urls: string[] = []; + vi.stubGlobal('fetch', vi.fn(async (input: string | URL) => { + const url = typeof input === 'string' ? input : input.href; + urls.push(url); + const host = new URL(url).host; + if (host === 'nichedb.dev') return Response.json({ count: nichedbItems.length, items: nichedbItems }); + if (host === 'query1.finance.yahoo.com') { + return Response.json({ chart: { result: [{ timestamp: [1_700_000_000, 1_700_086_400], indicators: { quote: [{ close: [1, 2] }] } }] } }); + } + return new Response('nope', { status: 404 }); + })); + return urls; + } + + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + }); + + it('uses Yahoo only when the switch is off', async () => { + vi.stubEnv('NICHEDB_MARKETS', ''); + const urls = installFetch([historyItem('SPY')]); + const snaps = await fetchDailySnapshots('SPY', today - 365 * DAY, today); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false); + expect(snaps.map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']); + }); + + it('maps nichedb bars to snapshots for an equity and never asks nichedb for crypto', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const urls = installFetch([historyItem('SPY')]); + const snaps = await fetchDailySnapshots('SPY', today - 365 * DAY, today); + expect(urls).toHaveLength(1); + expect(new URL(urls[0]!).searchParams.get('tags')).toBe('symbol:spy'); + expect(snaps.length).toBeGreaterThan(240); + expect(snaps[0]).toMatchObject({ exchange: 'nichedb', pair: 'SPY', assetClass: 'equity', bidSize: 1, askSize: 1 }); + expect(snaps[0]!.bid).toBe(snaps[0]!.ask); + expect(snaps.every((s, i) => i === 0 || s.ts > snaps[i - 1]!.ts)).toBe(true); + urls.length = 0; + const crypto = await fetchDailySnapshots('BTC-USD', today - 365 * DAY, today); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false); + expect(crypto.map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']); + }); + + it('falls back to Yahoo when nichedb is missing the symbol or the window is too long', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const urls = installFetch([]); + expect((await fetchDailySnapshots('ZZZZ', today - 90 * DAY, today)).map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']); + expect(urls.filter((u) => u.includes('nichedb.dev'))).toHaveLength(1); + urls.length = 0; + expect((await fetchDailySnapshots('SPY', today - 5 * 365 * DAY, today)).map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']); + expect(urls.filter((u) => u.includes('nichedb.dev'))).toHaveLength(0); + stderr.mockRestore(); + }); +}); describe('strategy-backtest parseArgs', () => { it('defaults to backtesting both asset classes', () => { diff --git a/apps/cli/src/strategy-backtest.ts b/apps/cli/src/strategy-backtest.ts index aa64d75..e19b45d 100644 --- a/apps/cli/src/strategy-backtest.ts +++ b/apps/cli/src/strategy-backtest.ts @@ -4,7 +4,8 @@ * Unlike `b1dz backtest ` (the server-side multi-pair crypto candle sim), * this runs locally against the deterministic StrategyPlugin engine in * @b1dz/source-strategies: it replays the strategy's own buy/sell signals - * long-only over Yahoo daily bars. + * long-only over Yahoo daily bars (equities come from nichedb.dev's `markets` + * history first when NICHEDB_MARKETS=1, Yahoo when nichedb cannot serve them). * * Crucially it scores crypto and equities SEPARATELY so you can see which asset * class a strategy suits. Run both (default, with a head-to-head verdict), @@ -19,6 +20,7 @@ import { readFileSync } from 'node:fs'; import chalk from 'chalk'; import Table from 'cli-table3'; import { PLUGIN_CATALOG, type MarketSnapshot, type StrategyPlugin } from '@b1dz/core'; +import { createNichedbClient, fetchDailyBars, nichedbEnabled } from '@b1dz/source-nichedb'; import { STRATEGY_PLUGINS, replayStrategy, @@ -105,7 +107,37 @@ function subtract(end: Date, h: (typeof HORIZONS)[number]): Date { return d; } -async function fetchDailySnapshots(symbol: string, startMs: number, endMs: number): Promise { +/** + * Equity bars from nichedb's `markets` history (NICHEDB_MARKETS=1). Returns + * null when nichedb cannot serve the window: missing symbol, last bar older + * than 5 days, or more than its 400-bar window asked for. Crypto symbols are + * not in that collection, so they never ask. + */ +async function fetchNichedbSnapshots(symbol: string, startMs: number, endMs: number): Promise { + if (symbol.includes('-USD')) return null; + const result = await fetchDailyBars(createNichedbClient(), symbol, startMs, endMs); + if (!result.ok) { + process.stderr.write(chalk.dim(` (nichedb cannot serve ${symbol}: ${result.reason}; using Yahoo)\n`)); + return null; + } + return result.bars.map((b) => ({ + exchange: 'nichedb', pair: symbol, bid: b.close, ask: b.close, bidSize: 1, askSize: 1, ts: b.ts, assetClass: 'equity' as const, + })); +} + +export async function fetchDailySnapshots(symbol: string, startMs: number, endMs: number): Promise { + if (nichedbEnabled('NICHEDB_MARKETS')) { + try { + const fromNichedb = await fetchNichedbSnapshots(symbol, startMs, endMs); + if (fromNichedb && fromNichedb.length > 0) return fromNichedb; + } catch (e) { + process.stderr.write(chalk.dim(` (nichedb error for ${symbol}: ${(e as Error).message}; using Yahoo)\n`)); + } + } + return fetchYahooSnapshots(symbol, startMs, endMs); +} + +async function fetchYahooSnapshots(symbol: string, startMs: number, endMs: number): Promise { const period1 = Math.floor((startMs - 7 * DAY_MS) / 1000); const period2 = Math.floor((endMs + 7 * DAY_MS) / 1000); const url = @@ -244,7 +276,7 @@ export async function runStrategyBacktestCli(argv: string[]): Promise { console.log( chalk.dim( - `Long-only signal replay · $${args.amount}/entry · Yahoo daily · classes: ${args.classes.join(' + ')} · ignores fees/slippage`, + `Long-only signal replay · $${args.amount}/entry · ${nichedbEnabled('NICHEDB_MARKETS') ? 'nichedb + Yahoo' : 'Yahoo'} daily · classes: ${args.classes.join(' + ')} · ignores fees/slippage`, ), ); diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts index 3f17e82..8a9f9c0 100644 --- a/apps/web/next.config.ts +++ b/apps/web/next.config.ts @@ -67,6 +67,7 @@ const config: NextConfig = { transpilePackages: [ '@b1dz/core', '@b1dz/sdk', + '@b1dz/source-nichedb', '@b1dz/source-strategies', '@b1dz/storage-json', '@b1dz/storage-supabase', diff --git a/apps/web/package.json b/apps/web/package.json index 53d64c9..2d4319f 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -16,6 +16,7 @@ "@b1dz/projection-engine": "workspace:*", "@b1dz/source-crypto-arb": "workspace:*", "@b1dz/source-crypto-trade": "workspace:*", + "@b1dz/source-nichedb": "workspace:*", "@b1dz/source-strategies": "workspace:*", "@b1dz/storage-json": "workspace:*", "@b1dz/storage-supabase": "workspace:*", diff --git a/apps/web/src/app/api/strategies/backtest/route.test.ts b/apps/web/src/app/api/strategies/backtest/route.test.ts index 1273c85..9edaef7 100644 --- a/apps/web/src/app/api/strategies/backtest/route.test.ts +++ b/apps/web/src/app/api/strategies/backtest/route.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const authenticateMock = vi.fn(); const unauthorizedMock = vi.fn(() => new Response(JSON.stringify({ error: 'unauthorized' }), { status: 401 })); @@ -108,3 +108,98 @@ describe('POST /api/strategies/backtest', () => { expect(runBacktestMock).not.toHaveBeenCalled(); }); }); + +describe('equity closes via nichedb (NICHEDB_MARKETS)', () => { + const DAY = 24 * 60 * 60 * 1000; + const today = Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate()); + + /** A fresh 400-bar history item ending yesterday, weekdays only. */ + function historyItem(symbol: string) { + const bars: (string | number | null)[][] = []; + let ms = today - DAY; + while (bars.length < 400) { + const dow = new Date(ms).getUTCDay(); + if (dow !== 0 && dow !== 6) bars.unshift([new Date(ms).toISOString().slice(0, 10), 10, 11, 9, 10 + bars.length, 1000, null]); + ms -= DAY; + } + return { + id: 7, collection: 'markets', kind: 'history', external_id: `history:${symbol}`, title: `${symbol} daily bars`, + published_at: null, updated_at: new Date().toISOString(), tags: ['history', `symbol:${symbol.toLowerCase()}`, 'feed:iex'], + data: { symbol, timeframe: '1Day', feed: 'iex', adjustment: 'split', bars, first: bars[0]![0], last: bars[399]![0], count: 400 }, + }; + } + + function installFetch(nichedbItems: unknown[]) { + const urls: string[] = []; + vi.stubGlobal('fetch', vi.fn(async (input: string | URL) => { + const url = typeof input === 'string' ? input : input.href; + urls.push(url); + const host = new URL(url).host; + if (host === 'nichedb.dev') return Response.json({ count: nichedbItems.length, items: nichedbItems }); + if (host === 'query1.finance.yahoo.com') { + return Response.json({ chart: { result: [{ timestamp: [1_700_000_000, 1_700_086_400], indicators: { quote: [{ close: [1, 2] }] } }] } }); + } + return new Response('nope', { status: 404 }); + })); + return urls; + } + + async function equityCloses(symbol: string, days: number) { + const { POST } = await importRoute(); + await POST(makeReq({ definition: validDoc, classes: ['equity'] }) as never); + const [, opts] = runBacktestMock.mock.calls[0]!; + return opts.fetchCloses(symbol, today - days * DAY, today) as Promise<{ ts: number; close: number }[]>; + } + + beforeEach(() => { + vi.stubEnv('ALPACA_API_KEY_ID', ''); + vi.stubEnv('ALPACA_API_SECRET_KEY', ''); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it('makes no nichedb request when the switch is off', async () => { + vi.stubEnv('NICHEDB_MARKETS', ''); + const urls = installFetch([historyItem('AAPL')]); + const rows = await equityCloses('AAPL', 365); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false); + expect(urls.some((u) => u.includes('finance.yahoo.com'))).toBe(true); + expect(rows.map((r) => r.close)).toEqual([1, 2]); + }); + + it('serves a one-year window from nichedb bars without touching Yahoo', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const urls = installFetch([historyItem('AAPL')]); + const rows = await equityCloses('AAPL', 365); + expect(urls).toHaveLength(1); + const q = new URL(urls[0]!).searchParams; + expect(q.get('collection')).toBe('markets'); + expect(q.get('kind')).toBe('history'); + expect(q.get('tags')).toBe('symbol:aapl'); + expect(q.get('limit')).toBe('1'); + expect(rows.length).toBeGreaterThan(240); + expect(rows[rows.length - 1]!.ts).toBe(rows[rows.length - 1]!.ts - (rows[rows.length - 1]!.ts % DAY)); // midnight UTC + expect(rows.every((r) => Number.isFinite(r.close))).toBe(true); + }); + + it('falls back to Yahoo when nichedb has no history for the symbol', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const urls = installFetch([]); + const rows = await equityCloses('ZZZZ', 90); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(true); + expect(urls.some((u) => u.includes('finance.yahoo.com'))).toBe(true); + expect(rows.map((r) => r.close)).toEqual([1, 2]); + }); + + it('skips nichedb for a window longer than its 400 bars and uses Yahoo', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const urls = installFetch([historyItem('AAPL')]); + const rows = await equityCloses('AAPL', 5 * 365); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false); + expect(rows.map((r) => r.close)).toEqual([1, 2]); + }); +}); diff --git a/apps/web/src/app/api/strategies/backtest/route.ts b/apps/web/src/app/api/strategies/backtest/route.ts index 0c43010..1093dd2 100644 --- a/apps/web/src/app/api/strategies/backtest/route.ts +++ b/apps/web/src/app/api/strategies/backtest/route.ts @@ -14,13 +14,16 @@ * - Crypto → Kraken daily OHLC via @b1dz/source-crypto-trade's * fetchHistoricalCandles (keyless; the same path the daemon + /api/backtest * use, proven to work from Railway). - * - Equities → Alpaca daily bars (when ALPACA_API_KEY_ID/SECRET are set), + * - Equities → nichedb.dev's `markets` history (when NICHEDB_MARKETS=1 and + * it holds a fresh window covering the request; keyless, one request per + * symbol), then Alpaca daily bars (when ALPACA_API_KEY_ID/SECRET are set), * falling back to Yahoo best-effort. Yahoo is frequently blocked from - * datacenter IPs, so without Alpaca keys equities may return no data; - * crypto is unaffected. + * datacenter IPs, so without nichedb or Alpaca keys equities may return + * no data; crypto is unaffected. */ import type { NextRequest } from 'next/server'; import { fetchHistoricalCandles } from '@b1dz/source-crypto-trade'; +import { createNichedbClient, fetchDailyBars, nichedbEnabled } from '@b1dz/source-nichedb'; import { tsp } from '@b1dz/source-strategies'; import { authenticate, unauthorized } from '@/lib/api-auth'; import { @@ -101,10 +104,24 @@ async function fetchAlpacaCloses(symbol: string, startMs: number, endMs: number) return rows; } +/** + * Equity daily closes from nichedb's `markets` history item (400 daily bars, + * split adjusted). Throws when nichedb has no fresh window covering the + * request (missing symbol, last bar older than 5 days, more than 400 days + * asked for) so the caller moves on to Alpaca and Yahoo. + */ +async function fetchNichedbCloses(symbol: string, startMs: number, endMs: number): Promise { + const result = await fetchDailyBars(createNichedbClient(), symbol, startMs, endMs); + if (!result.ok) throw new Error(`nichedb ${result.reason}`); + const rows = result.bars.map((b) => ({ ts: b.ts, close: b.close })); + if (rows.length === 0) throw new Error('nichedb returned no rows'); + return rows; +} + /** * Route closes by asset class: * - crypto → Kraken (reliable from Railway, keyless). - * - equities → Alpaca (if keys set) → Yahoo fallback. + * - equities → nichedb (if NICHEDB_MARKETS=1) → Alpaca (if keys set) → Yahoo fallback. * Returns [] only if every source for that symbol fails. */ const fetchCloses: FetchCloses = async (symbol, startMs, endMs) => { @@ -116,7 +133,14 @@ const fetchCloses: FetchCloses = async (symbol, startMs, endMs) => { return []; } } - // Equity: Alpaca first, then Yahoo. + // Equity: nichedb when switched on, then Alpaca, then Yahoo. + if (nichedbEnabled('NICHEDB_MARKETS')) { + try { + return await fetchNichedbCloses(symbol, startMs, endMs); + } catch (err) { + console.warn(`[backtest] nichedb cannot serve ${symbol} (${(err as Error).message}); falling back`); + } + } try { return await fetchAlpacaCloses(symbol, startMs, endMs); } catch (alpacaErr) { diff --git a/packages/source-crypto-arb/package.json b/packages/source-crypto-arb/package.json index 3f0823c..f056395 100644 --- a/packages/source-crypto-arb/package.json +++ b/packages/source-crypto-arb/package.json @@ -17,6 +17,7 @@ "dependencies": { "@b1dz/adapters-evm": "workspace:*", "@b1dz/core": "workspace:*", + "@b1dz/source-nichedb": "workspace:*", "undici": "^8.0.2", "ws": "^8.20.0" }, diff --git a/packages/source-crypto-arb/src/pair-discovery.test.ts b/packages/source-crypto-arb/src/pair-discovery.test.ts new file mode 100644 index 0000000..a15b78d --- /dev/null +++ b/packages/source-crypto-arb/src/pair-discovery.test.ts @@ -0,0 +1,171 @@ +/** + * Pair discovery behind NICHEDB_CRYPTO: nichedb rows in, the same `BTC-USD` + * list out; live CoinGecko + venue tickers when the switch is off or nichedb + * fails. Every request goes through a fake global fetch; nothing is live. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { CryptoAssetItem, CryptoPairItem } from '@b1dz/source-nichedb'; + +const RUN = '2026-09-11T06:00:00.000Z'; +let nextId = 1; + +function pairRow(venue: string, base: string, volume: number | null, quote = 'USD'): CryptoPairItem { + const sym = venue === 'coinbase' ? `${base}-${quote}` : venue === 'gemini' ? `${base}${quote}`.toLowerCase() : `${base}${quote}`; + return { + id: nextId++, collection: 'crypto', kind: 'pair', external_id: `pair:${venue}:${base}-${quote}`, title: `${base}/${quote}`, + published_at: RUN, updated_at: RUN, tags: ['pair', `venue:${venue}`, `base:${base.toLowerCase()}`, `quote:${quote.toLowerCase()}`, 'stable-quote'], + data: { + venue, base, quote, venueSymbol: sym, status: 'online', price: 100, bid: null, ask: null, high24h: null, low24h: null, change24hPct: 1, + volume24hBase: volume == null ? null : volume / 100, volume24hQuote: volume, vwap24h: null, priceUsd: 100, updatedAt: RUN, + }, + }; +} + +function assetRow(symbol: string, marketCapUsd: number, rank: number): CryptoAssetItem { + return { + id: nextId++, collection: 'crypto', kind: 'asset', external_id: `coingecko:${symbol.toLowerCase()}`, title: symbol, + published_at: RUN, updated_at: RUN, tags: ['asset', `symbol:${symbol.toLowerCase()}`], + data: { + id: symbol.toLowerCase(), symbol: symbol.toLowerCase(), name: symbol, rank, priceUsd: 100, marketCapUsd, fullyDilutedUsd: null, volume24hUsd: null, + change24hPct: null, high24h: null, low24h: null, supply: { circulating: null, total: null, max: null }, ath: null, athDate: null, atl: null, atlDate: null, updatedAt: RUN, + }, + }; +} + +const PAIRS: CryptoPairItem[] = [ + pairRow('kraken', 'BTC', 900e6), pairRow('coinbase', 'BTC', 1200e6), pairRow('binance-us', 'BTC', 80e6), pairRow('gemini', 'BTC', null), + pairRow('kraken', 'ETH', 300e6), pairRow('coinbase', 'ETH', 500e6), + pairRow('kraken', 'ONLYK', 5e6), // one venue + pairRow('kraken', 'THIN', 20_000), pairRow('coinbase', 'THIN', 30_000), // under $50k everywhere + pairRow('kraken', 'GEMNULL', 75_000), pairRow('gemini', 'GEMNULL', null), // Gemini null counts + pairRow('kraken', 'SMALL', 100_000), pairRow('coinbase', 'SMALL', 100_000), // $4M cap + pairRow('kraken', 'BTC', 50e6, 'USDT'), // not a USD book +]; +const ASSETS: CryptoAssetItem[] = [assetRow('BTC', 1.2e12, 1), assetRow('ETH', 3.6e11, 2), assetRow('SMALL', 4e6, 480), assetRow('GEMNULL', 5e7, 200)]; + +type Mode = 'ok' | 'fail' | 'empty'; + +/** Fake fetch for nichedb plus the live fallback hosts. Records every URL. */ +function installFetch(mode: Mode) { + const urls: string[] = []; + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + urls.push(url); + const u = new URL(url); + const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); + if (u.host === 'nichedb.dev') { + if (mode === 'fail') return json({ error: 'boom' }, 503); + if (mode === 'empty') return json({ count: 0, items: [] }); + const kind = u.searchParams.get('kind'); + const after = Number(u.searchParams.get('after') ?? 0); + const rows = (kind === 'pair' ? PAIRS : ASSETS).filter((r) => r.id > after); + return json({ count: rows.length, items: rows }); + } + // The live path (ETH and SOL on Kraken + Binance.US; Coinbase needs keys and is skipped). + // Kraken's XXBTZUSD is left out on purpose: normalizeKrakenBase strips "XX" to "BT", a + // pre-existing quirk of the live path that this file does not test. + if (u.host === 'api.kraken.com') { + return json({ error: [], result: { XETHZUSD: { v: ['0', '100000'], c: ['3000', '0'] }, SOLUSD: { v: ['0', '100000'], c: ['150', '0'] } } }); + } + if (u.host === 'api.binance.us') { + return json([ + { symbol: 'ETHUSD', quoteVolume: '30000000', priceChangePercent: '1' }, + { symbol: 'SOLUSD', quoteVolume: '3000000', priceChangePercent: '1' }, + ]); + } + if (u.host === 'api.coingecko.com') { + return json(u.searchParams.get('page') === '1' ? [{ symbol: 'eth', market_cap: 3.6e11 }, { symbol: 'sol', market_cap: 8e10 }] : []); + } + return json({ error: 'unexpected host' }, 404); + }); + vi.stubGlobal('fetch', fetchMock); + return { urls, nichedb: () => urls.filter((x) => x.includes('nichedb.dev')), live: () => urls.filter((x) => !x.includes('nichedb.dev')) }; +} + +async function load() { + vi.resetModules(); + return import('./pair-discovery.js'); +} + +describe('pair discovery via nichedb', () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-09-11T06:05:00Z')); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.stubEnv('COINBASE_API_KEY_NAME', ''); + vi.stubEnv('MIN_VOLUME_USD', ''); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('reads the universe from nichedb only, with the same filters, when NICHEDB_CRYPTO=1', async () => { + vi.stubEnv('NICHEDB_CRYPTO', '1'); + const f = installFetch('ok'); + const { getActivePairs, discoverPairsFromNichedb } = await load(); + expect(await getActivePairs()).toEqual(['BTC-USD', 'ETH-USD', 'GEMNULL-USD']); + expect(f.live()).toEqual([]); + expect(f.nichedb()).toHaveLength(2); // one page of pairs, one page of assets + const pairUrl = new URL(f.nichedb().find((u) => u.includes('kind=pair'))!); + expect(pairUrl.searchParams.get('collection')).toBe('crypto'); + expect(pairUrl.searchParams.get('tags')).toBe('stable-quote'); + expect(pairUrl.searchParams.get('limit')).toBe('200'); + expect(f.nichedb().some((u) => u.includes('kind=asset'))).toBe(true); + // venue symbols travel with each pair for anyone who needs the venue's own name + const btc = (await discoverPairsFromNichedb()).find((p) => p.pair === 'BTC-USD')!; + expect(btc.venues['kraken']!.venueSymbol).toBe('BTCUSD'); + expect(btc.venues['gemini']!.volume24hQuote).toBeNull(); + expect(console.error).not.toHaveBeenCalled(); + }); + + it('caches the answer for five minutes', async () => { + vi.stubEnv('NICHEDB_CRYPTO', '1'); + const f = installFetch('ok'); + const { getActivePairs } = await load(); + await getActivePairs(); + await getActivePairs(); + vi.advanceTimersByTime(4 * 60_000); + await getActivePairs(); + expect(f.nichedb()).toHaveLength(2); + vi.advanceTimersByTime(61_000); + await getActivePairs(); + expect(f.nichedb()).toHaveLength(4); + }); + + it('falls back to CoinGecko + venue tickers on a nichedb failure and logs once', async () => { + vi.stubEnv('NICHEDB_CRYPTO', '1'); + const f = installFetch('fail'); + const { getActivePairs } = await load(); + expect(await getActivePairs()).toEqual(['ETH-USD', 'SOL-USD']); + expect(f.nichedb().length).toBeGreaterThan(0); + expect(f.live().some((u) => u.includes('api.kraken.com'))).toBe(true); + expect(f.live().some((u) => u.includes('api.coingecko.com'))).toBe(true); + expect(console.error).toHaveBeenCalledTimes(1); + expect((console.error as unknown as { mock: { calls: string[][] } }).mock.calls[0]![0]).toMatch(/nichedb unavailable .* falling back/); + vi.advanceTimersByTime(6 * 60_000); + await getActivePairs(); + expect(console.error).toHaveBeenCalledTimes(1); + }); + + it('falls back on an empty nichedb answer', async () => { + vi.stubEnv('NICHEDB_CRYPTO', '1'); + const f = installFetch('empty'); + const { getActivePairs } = await load(); + expect(await getActivePairs()).toEqual(['ETH-USD', 'SOL-USD']); + expect(f.live().some((u) => u.includes('api.kraken.com'))).toBe(true); + expect(console.error).toHaveBeenCalledTimes(1); + }); + + it('never asks nichedb when the switch is off', async () => { + vi.stubEnv('NICHEDB_CRYPTO', ''); + const f = installFetch('ok'); + const { getActivePairs } = await load(); + expect(await getActivePairs()).toEqual(['ETH-USD', 'SOL-USD']); + expect(f.nichedb()).toEqual([]); + expect(f.live().some((u) => u.includes('api.kraken.com'))).toBe(true); + }); +}); diff --git a/packages/source-crypto-arb/src/pair-discovery.ts b/packages/source-crypto-arb/src/pair-discovery.ts index 444b142..c9c8158 100644 --- a/packages/source-crypto-arb/src/pair-discovery.ts +++ b/packages/source-crypto-arb/src/pair-discovery.ts @@ -7,9 +7,24 @@ * 4. Return every pair that clears the liquidity + market-cap filters * * Refreshes every 5 minutes. + * + * With NICHEDB_CRYPTO=1 the universe (per-venue tickers and CoinGecko market + * caps) is read from nichedb.dev in ~10 requests instead of CoinGecko plus the + * four venues' ticker lists; the filters are the same. Any nichedb failure or + * an empty answer falls back to the live path below, logged once per outage. */ import { createSign, randomBytes } from 'node:crypto'; +import { + createNichedbClient, + fetchCryptoAssets, + fetchCryptoPairs, + nichedbEnabled, + selectCryptoPairs, + type CryptoAssetItem, + type DiscoveredPair, + type FetchLike, +} from '@b1dz/source-nichedb'; import { getCoinbasePem } from './feeds/coinbase-pem.js'; import { fetchJson } from './feeds/http.js'; @@ -135,9 +150,75 @@ async function getBinanceVolumes(): Promise= MIN_VOLUME_USD` per venue + * (a null volume, which is every Gemini book, passes, exactly as the old + * Gemini sentinel did), on at least two venues, market cap >= $10M when known. + * + * Throws on a nichedb failure or an empty pair set so the caller can fall + * back; a failed asset read only drops the market-cap filter, as a CoinGecko + * failure did. + */ +export async function discoverPairsFromNichedb(fetchImpl?: FetchLike): Promise { + const client = createNichedbClient({ fetch: fetchImpl }); + const [pairsResult, assetsResult] = await Promise.allSettled([fetchCryptoPairs(client), fetchCryptoAssets(client)]); + // No pairs means no universe: throw so the caller falls back (an asset error alongside is the same outage). + if (pairsResult.status === 'rejected') throw pairsResult.reason; + const pairRows = pairsResult.value; + if (pairRows.length === 0) throw new Error('nichedb returned no pairs'); + let assetRows: CryptoAssetItem[] = []; + if (assetsResult.status === 'fulfilled') assetRows = assetsResult.value; + else console.error(`[discovery] nichedb assets error (skipping mcap filter): ${(assetsResult.reason as Error).message}`); + + const sel = selectCryptoPairs(pairRows, assetRows, { + minVolumeUsd: minVolumeUsd(), + minMarketCapUsd: MIN_MARKET_CAP_USD, + minVenues: MIN_EXCHANGES, + excludedBases: EXCLUDED, + quote: 'USD', + }); + if (sel.pairs.length === 0) throw new Error(`nichedb: no pair cleared the filters (${pairRows.length} rows)`); + + console.log( + `[discovery] nichedb: ${sel.pairs.length} pairs from ${pairRows.length} pair rows + ${assetRows.length} assets in ${client.requestCount} requests ` + + `(${sel.filteredByVenues} filtered by <${MIN_EXCHANGES} exchanges, ${sel.filteredByMarketCap} filtered by <$${MIN_MARKET_CAP_USD / 1e6}M mcap, min vol $${(minVolumeUsd() / 1e6).toFixed(2)}M)`, + ); + for (const p of sel.pairs.slice(0, 12)) { + const chg = p.change24hPct >= 0 ? `+${p.change24hPct.toFixed(1)}%` : `${p.change24hPct.toFixed(1)}%`; + const mcapStr = p.marketCapUsd > 0 ? `mcap=$${(p.marketCapUsd / 1e9).toFixed(1)}B` : 'mcap=?'; + console.log(` ${p.pair.padEnd(12)} vol=$${(p.totalVolumeUsd / 1e6).toFixed(1)}M 24h=${chg} ${mcapStr} venues=${Object.keys(p.venues).join(',')}`); + } + if (sel.pairs.length > 12) console.log(` ... +${sel.pairs.length - 12} more`); + return sel.pairs; +} + // ─── Discovery ──────────────────────────────────────────────── async function discoverPairs(): Promise { + if (nichedbEnabled('NICHEDB_CRYPTO')) { + try { + const pairs = await discoverPairsFromNichedb(); + nichedbFallbackWarned = false; + return pairs.map((p) => p.pair); + } catch (e) { + if (!nichedbFallbackWarned) { + nichedbFallbackWarned = true; + console.error(`[discovery] nichedb unavailable (${(e as Error).message}); falling back to CoinGecko + venue tickers`); + } + } + } + return discoverPairsFromVenues(); +} + +/** The live path: CoinGecko (2 pages x 250) plus Kraken, Coinbase and Binance.US tickers. */ +async function discoverPairsFromVenues(): Promise { const [krakenVols, coinbaseData, binanceData] = await Promise.all([ getKrakenVolumes(), getCoinbaseVolumes(), diff --git a/packages/source-nichedb/package.json b/packages/source-nichedb/package.json new file mode 100644 index 0000000..6562c8a --- /dev/null +++ b/packages/source-nichedb/package.json @@ -0,0 +1,22 @@ +{ + "name": "@b1dz/source-nichedb", + "version": "0.3.10", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "lint": "eslint src", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "latest", + "typescript": "latest", + "vitest": "latest" + } +} diff --git a/packages/source-nichedb/src/client.test.ts b/packages/source-nichedb/src/client.test.ts new file mode 100644 index 0000000..fa679c7 --- /dev/null +++ b/packages/source-nichedb/src/client.test.ts @@ -0,0 +1,131 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createNichedbClient, NichedbError, type FetchLike } from './client.js'; +import { nichedbBaseUrl, nichedbEnabled } from './env.js'; + +type Call = { url: string; init?: Parameters[1] }; + +function fakeFetch(handler: (url: URL, call: Call) => { status?: number; body: unknown } | Promise<{ status?: number; body: unknown }>) { + const calls: Call[] = []; + const fetch: FetchLike = async (url, init) => { + const call = { url, init }; + calls.push(call); + const { status = 200, body } = await handler(new URL(url), call); + return { ok: status >= 200 && status < 300, status, statusText: status === 200 ? 'OK' : 'ERR', json: async () => body }; + }; + return { fetch, calls }; +} + +const row = (id: number, extra: Record = {}) => ({ + id, collection: 'crypto', kind: 'pair', external_id: `x:${id}`, title: `#${id}`, published_at: null, updated_at: '2026-09-11T00:00:00Z', tags: [], data: {}, ...extra, +}); + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('createNichedbClient', () => { + it('builds the items URL from the query with a comma tag list and a capped limit', () => { + const client = createNichedbClient({ baseUrl: 'https://example.test/', fetch: fakeFetch(() => ({ body: { items: [] } })).fetch }); + const url = new URL(client.itemsUrl({ collection: 'crypto', kind: 'pair', tags: ['stable-quote', 'venue:kraken'], limit: 2500, since: '2026-09-11T00:00:00Z', after: 12 })); + expect(url.origin + url.pathname).toBe('https://example.test/api/v1/items'); + expect(url.searchParams.get('collection')).toBe('crypto'); + expect(url.searchParams.get('kind')).toBe('pair'); + expect(url.searchParams.get('tags')).toBe('stable-quote,venue:kraken'); + expect(url.searchParams.get('limit')).toBe('200'); + expect(url.searchParams.get('since')).toBe('2026-09-11T00:00:00Z'); + expect(url.searchParams.get('after')).toBe('12'); + }); + + it('defaults the base to NICHEDB_URL, else nichedb.dev', () => { + expect(nichedbBaseUrl({})).toBe('https://nichedb.dev'); + expect(nichedbBaseUrl({ NICHEDB_URL: 'http://localhost:3000/' })).toBe('http://localhost:3000'); + vi.stubEnv('NICHEDB_URL', 'https://mirror.test'); + expect(createNichedbClient({ fetch: fakeFetch(() => ({ body: { items: [] } })).fetch }).baseUrl).toBe('https://mirror.test'); + }); + + it('walks after= keyset pages of 200 in id order and stops on a short page', async () => { + const total = 450; + const all = Array.from({ length: total }, (_, i) => row(i + 1)); + const { fetch, calls } = fakeFetch((url) => { + const after = Number(url.searchParams.get('after') ?? 0); + const limit = Number(url.searchParams.get('limit')); + expect(url.searchParams.get('sort')).toBe('id'); + expect(url.searchParams.get('order')).toBe('asc'); + const page = all.filter((r) => r.id > after).slice(0, limit); + return { body: { count: page.length, items: page } }; + }); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + const items = await client.walk({ collection: 'crypto', kind: 'pair', tags: ['stable-quote'] }); + expect(items).toHaveLength(total); + expect(items.map((i) => i.id)).toEqual(all.map((r) => r.id)); + expect(calls).toHaveLength(3); // 200 + 200 + 50 + expect(new URL(calls[1]!.url).searchParams.get('after')).toBe('200'); + expect(new URL(calls[2]!.url).searchParams.get('after')).toBe('400'); + expect(client.requestCount).toBe(3); + }); + + it('makes one extra request when the last page is exactly full', async () => { + const all = Array.from({ length: 400 }, (_, i) => row(i + 1)); + const { fetch, calls } = fakeFetch((url) => { + const after = Number(url.searchParams.get('after') ?? 0); + return { body: { items: all.filter((r) => r.id > after).slice(0, 200) } }; + }); + const items = await createNichedbClient({ baseUrl: 'https://n.test', fetch }).walk({ collection: 'crypto' }); + expect(items).toHaveLength(400); + expect(calls).toHaveLength(3); + }); + + it('honours maxItems and maxPages', async () => { + const all = Array.from({ length: 1000 }, (_, i) => row(i + 1)); + const { fetch, calls } = fakeFetch((url) => { + const after = Number(url.searchParams.get('after') ?? 0); + return { body: { items: all.filter((r) => r.id > after).slice(0, 200) } }; + }); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + expect(await client.walk({ collection: 'crypto', kind: 'asset' }, { maxItems: 500 })).toHaveLength(500); + expect(calls).toHaveLength(3); + calls.length = 0; + expect(await client.walk({ collection: 'crypto', kind: 'asset' }, { maxPages: 2 })).toHaveLength(400); + expect(calls).toHaveLength(2); + }); + + it('passes since= through on every page', async () => { + const { fetch, calls } = fakeFetch(() => ({ body: { items: [] } })); + await createNichedbClient({ baseUrl: 'https://n.test', fetch }).walk({ collection: 'crypto', since: new Date('2026-09-11T05:00:00Z') }); + expect(new URL(calls[0]!.url).searchParams.get('since')).toBe('2026-09-11T05:00:00.000Z'); + }); + + it('throws a NichedbError on a non-2xx answer, an error body, or a malformed body', async () => { + const c1 = createNichedbClient({ baseUrl: 'https://n.test', fetch: fakeFetch(() => ({ status: 503, body: {} })).fetch }); + await expect(c1.items({ collection: 'crypto' })).rejects.toBeInstanceOf(NichedbError); + const c2 = createNichedbClient({ baseUrl: 'https://n.test', fetch: fakeFetch(() => ({ body: { error: 'No collection named crypto' } })).fetch }); + await expect(c2.items({ collection: 'crypto' })).rejects.toThrow(/No collection named crypto/); + const c3 = createNichedbClient({ baseUrl: 'https://n.test', fetch: fakeFetch(() => ({ body: { nope: true } })).fetch }); + await expect(c3.items({ collection: 'crypto' })).rejects.toThrow(/malformed/); + }); + + it('aborts a request that exceeds the timeout', async () => { + vi.useFakeTimers(); + try { + const fetch: FetchLike = (_url, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(Object.assign(new Error('aborted'), { name: 'AbortError' }))); + }); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch, timeoutMs: 50 }); + const p = client.items({ collection: 'crypto' }); + const assertion = expect(p).rejects.toThrow(/timeout after 50ms/); + await vi.advanceTimersByTimeAsync(60); + await assertion; + } finally { + vi.useRealTimers(); + } + }); +}); + +describe('nichedbEnabled', () => { + it('is on for 1/true/yes/on and off otherwise', () => { + for (const v of ['1', 'true', 'TRUE', 'yes', ' on ']) expect(nichedbEnabled('NICHEDB_CRYPTO', { NICHEDB_CRYPTO: v })).toBe(true); + for (const v of ['0', 'false', '', 'no']) expect(nichedbEnabled('NICHEDB_CRYPTO', { NICHEDB_CRYPTO: v })).toBe(false); + expect(nichedbEnabled('NICHEDB_MARKETS', {})).toBe(false); + }); +}); diff --git a/packages/source-nichedb/src/client.ts b/packages/source-nichedb/src/client.ts new file mode 100644 index 0000000..d9f7106 --- /dev/null +++ b/packages/source-nichedb/src/client.ts @@ -0,0 +1,174 @@ +/** + * Tiny client for nichedb.dev's public, keyless items API. + * + * GET /api/v1/items?collection=&kind=&tags=a,b&since=&sort=&order=&limit=&after= + * + * The API caps `limit` at 200 and answers `{ count, items }` with no cursor, + * so a full read walks keyset pages: `sort=id&order=asc&after=` until + * a page comes back short. About 600 requests an hour per IP are tolerated; + * callers are expected to cache. + */ + +import { nichedbBaseUrl } from './env.js'; + +/** One row of `/api/v1/items`; `data` is the adapter's full record. */ +export interface NichedbItem { + id: number; + collection: string; + source?: string; + adapter?: string; + kind: string; + external_id: string; + title: string; + summary?: string | null; + url?: string | null; + image_url?: string | null; + published_at: string | null; + updated_at: string; + tags: string[]; + data: T; +} + +export interface ItemsQuery { + collection: string; + kind?: string; + /** All named tags must be on the item (comma list on the wire). */ + tags?: string[]; + /** Only rows whose `updated_at` is at or after this stamp. */ + since?: string | Date; + sort?: 'id' | 'published' | 'updated'; + order?: 'asc' | 'desc'; + /** Page size, capped by the API at 200. */ + limit?: number; + /** Keyset cursor: rows with `id` greater than this. */ + after?: number; +} + +/** The slice of fetch the client needs, so tests and proxies can hand in their own. */ +export type FetchLike = ( + input: string, + init?: { signal?: AbortSignal; headers?: Record }, +) => Promise<{ ok: boolean; status: number; statusText: string; json(): Promise }>; + +export interface NichedbClientOptions { + /** Base URL; defaults to NICHEDB_URL or https://nichedb.dev. */ + baseUrl?: string; + /** Injectable fetch (tests, proxies). Defaults to globalThis.fetch. */ + fetch?: FetchLike; + /** Per-request timeout. */ + timeoutMs?: number; + /** Sent as User-Agent so nichedb can see who is reading. */ + userAgent?: string; +} + +export interface WalkOptions { + /** Hard stop on the number of pages, so a bug upstream cannot loop forever. */ + maxPages?: number; + /** Stop once this many rows have been collected. */ + maxItems?: number; +} + +export const NICHEDB_PAGE_LIMIT = 200; +const DEFAULT_TIMEOUT_MS = 10_000; +const DEFAULT_MAX_PAGES = 25; + +export class NichedbError extends Error { + constructor( + message: string, + readonly status?: number, + readonly url?: string, + ) { + super(message); + this.name = 'NichedbError'; + } +} + +export interface NichedbClient { + readonly baseUrl: string; + /** Build the items URL for a query (exposed for logging and tests). */ + itemsUrl(query: ItemsQuery): string; + /** One page of items. */ + items(query: ItemsQuery): Promise[]>; + /** Every item matching the query, walking `after=` keyset pages in id order. */ + walk(query: ItemsQuery, opts?: WalkOptions): Promise[]>; + /** Requests made so far by this client (for budget checks and tests). */ + readonly requestCount: number; +} + +export function createNichedbClient(opts: NichedbClientOptions = {}): NichedbClient { + const baseUrl = (opts.baseUrl ?? nichedbBaseUrl()).replace(/\/+$/, ''); + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const userAgent = opts.userAgent ?? 'b1dz (+https://b1dz.com)'; + let requestCount = 0; + + const resolveFetch = (): FetchLike => { + const f = opts.fetch ?? (globalThis.fetch as unknown as FetchLike | undefined); + if (!f) throw new NichedbError('no fetch available'); + return f; + }; + + function itemsUrl(query: ItemsQuery): string { + const params = new URLSearchParams(); + params.set('collection', query.collection); + if (query.kind) params.set('kind', query.kind); + if (query.tags && query.tags.length > 0) params.set('tags', query.tags.join(',')); + if (query.since != null) { + params.set('since', query.since instanceof Date ? query.since.toISOString() : query.since); + } + if (query.sort) params.set('sort', query.sort); + if (query.order) params.set('order', query.order); + if (query.limit != null) params.set('limit', String(Math.min(Math.max(1, query.limit), NICHEDB_PAGE_LIMIT))); + if (query.after != null) params.set('after', String(query.after)); + return `${baseUrl}/api/v1/items?${params.toString()}`; + } + + async function items(query: ItemsQuery): Promise[]> { + const url = itemsUrl(query); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + requestCount++; + try { + const res = await resolveFetch()(url, { + signal: controller.signal, + headers: { Accept: 'application/json', 'User-Agent': userAgent }, + }); + if (!res.ok) throw new NichedbError(`nichedb ${res.status} ${res.statusText}`, res.status, url); + const body = (await res.json()) as { count?: number; items?: unknown; error?: string } | null; + if (body && typeof body.error === 'string') throw new NichedbError(`nichedb: ${body.error}`, res.status, url); + if (!body || !Array.isArray(body.items)) { + throw new NichedbError('nichedb: malformed answer (no items array)', res.status, url); + } + return body.items as NichedbItem[]; + } catch (e) { + if ((e as Error).name === 'AbortError') throw new NichedbError(`nichedb: timeout after ${timeoutMs}ms`, undefined, url); + throw e; + } finally { + clearTimeout(timer); + } + } + + async function walk(query: ItemsQuery, walkOpts: WalkOptions = {}): Promise[]> { + const maxPages = walkOpts.maxPages ?? DEFAULT_MAX_PAGES; + const maxItems = walkOpts.maxItems ?? Number.POSITIVE_INFINITY; + const pageSize = Math.min(query.limit ?? NICHEDB_PAGE_LIMIT, NICHEDB_PAGE_LIMIT); + const out: NichedbItem[] = []; + let after = query.after; + for (let page = 0; page < maxPages; page++) { + const rows = await items({ ...query, sort: 'id', order: 'asc', limit: pageSize, after }); + out.push(...rows); + if (rows.length < pageSize || out.length >= maxItems) break; + after = rows[rows.length - 1]!.id; + } + return out.length > maxItems ? out.slice(0, maxItems) : out; + } + + return { + baseUrl, + itemsUrl, + items, + walk, + get requestCount() { + return requestCount; + }, + }; +} diff --git a/packages/source-nichedb/src/crypto.test.ts b/packages/source-nichedb/src/crypto.test.ts new file mode 100644 index 0000000..66c28fc --- /dev/null +++ b/packages/source-nichedb/src/crypto.test.ts @@ -0,0 +1,254 @@ +/** + * Fixture built from niche-db docs/crypto.md: Kraken, Coinbase, Binance.US and + * Gemini rows for a handful of bases, then the grouping and the filters b1dz + * applied to CoinGecko plus the venue ticker lists. + */ +import { describe, expect, it } from 'vitest'; +import { fetchCryptoAssets, fetchCryptoPairs, marketCapsBySymbol, selectCryptoPairs, type CryptoAssetItem, type CryptoPairItem } from './crypto.js'; +import { createNichedbClient, type FetchLike } from './client.js'; + +let nextId = 1; +const RUN = '2026-09-11T06:00:00.000Z'; + +const VENUE_NAME: Record = { kraken: 'Kraken', coinbase: 'Coinbase', 'binance-us': 'Binance.US', gemini: 'Gemini' }; + +function venueSymbol(venue: string, base: string, quote: string): string { + switch (venue) { + case 'kraken': return base === 'BTC' ? `XXBTZ${quote}` : `${base}${quote}`; + case 'coinbase': return `${base}-${quote}`; + case 'binance-us': return `${base}${quote}`; + default: return `${base}${quote}`.toLowerCase(); + } +} + +export function pairRow( + venue: string, + base: string, + opts: { quote?: string; price?: number; volume?: number | null; status?: string; change?: number | null } = {}, +): CryptoPairItem { + const quote = opts.quote ?? 'USD'; + const price = opts.price ?? 100; + const volume = opts.volume === undefined ? (venue === 'gemini' ? null : 1_000_000) : opts.volume; + const stable = ['USD', 'USDT', 'USDC'].includes(quote); + const isGemini = venue === 'gemini'; + const isCoinbase = venue === 'coinbase'; + return { + id: nextId++, + collection: 'crypto', + source: 'crypto-pairs', + kind: 'pair', + external_id: `pair:${venue}:${base}-${quote}`, + title: `${base}/${quote} on ${VENUE_NAME[venue]}`, + published_at: RUN, + updated_at: RUN, + tags: ['pair', `venue:${venue}`, `base:${base.toLowerCase()}`, `quote:${quote.toLowerCase()}`, ...(stable ? ['stable-quote'] : [])], + data: { + venue, + venueName: VENUE_NAME[venue], + base, + quote, + venueSymbol: venueSymbol(venue, base, quote), + status: opts.status ?? 'online', + price, + bid: isGemini || isCoinbase ? null : price * 0.999, + ask: isGemini || isCoinbase ? null : price * 1.001, + high24h: isGemini ? null : price * 1.05, + low24h: isGemini ? null : price * 0.95, + open24h: isGemini ? null : price, + change24hPct: opts.change === undefined ? 1.2 : opts.change, + volume24hBase: volume == null ? null : volume / price, + volume24hQuote: volume, + vwap24h: isGemini || isCoinbase ? null : price, + priceUsd: stable ? price : null, + updatedAt: RUN, + }, + }; +} + +export function assetRow(symbol: string, marketCapUsd: number | null, rank: number | null = 1): CryptoAssetItem { + return { + id: nextId++, + collection: 'crypto', + source: 'coingecko-assets', + kind: 'asset', + external_id: `coingecko:${symbol.toLowerCase()}`, + title: `${symbol} (${symbol})`, + published_at: RUN, + updated_at: RUN, + tags: ['asset', `symbol:${symbol.toLowerCase()}`, ...(rank != null && rank <= 100 ? ['rank:top100'] : [])], + data: { + id: symbol.toLowerCase(), symbol: symbol.toLowerCase(), name: symbol, rank, priceUsd: 100, marketCapUsd, fullyDilutedUsd: marketCapUsd, + volume24hUsd: 1e9, change24hPct: 1, high24h: 105, low24h: 95, supply: { circulating: 1, total: 1, max: null }, + ath: 1, athDate: null, atl: 1, atlDate: null, updatedAt: RUN, + }, + }; +} + +const RULES = { minVolumeUsd: 50_000, minMarketCapUsd: 10_000_000, minVenues: 2, excludedBases: ['USDT', 'USDC', 'DAI', 'EUR'] }; + +/** BTC and ETH everywhere; ONLYK on Kraken alone; THIN under the volume floor on + * every venue but one; GEMNULL on Kraken (known) + Gemini (null volume); SMALL + * on two venues with a tiny market cap; USDT quoted books to be ignored. */ +function fixture() { + const pairs: CryptoPairItem[] = [ + pairRow('kraken', 'BTC', { price: 60_000, volume: 900_000_000, change: 0.9 }), + pairRow('coinbase', 'BTC', { price: 60_000, volume: 1_200_000_000, change: -1.3 }), + pairRow('binance-us', 'BTC', { price: 60_000, volume: 80_000_000, change: -1.4 }), + pairRow('gemini', 'BTC', { price: 60_000, change: -1.2 }), + pairRow('kraken', 'BTC', { quote: 'USDT', price: 60_000, volume: 50_000_000 }), + pairRow('kraken', 'ETH', { price: 3_000, volume: 300_000_000 }), + pairRow('coinbase', 'ETH', { price: 3_000, volume: 500_000_000 }), + pairRow('binance-us', 'ETH', { price: 3_000, volume: 30_000_000 }), + pairRow('gemini', 'ETH', { price: 3_000 }), + pairRow('kraken', 'ONLYK', { volume: 5_000_000 }), + pairRow('kraken', 'THIN', { volume: 20_000 }), + pairRow('coinbase', 'THIN', { volume: 49_999 }), + pairRow('binance-us', 'THIN', { volume: 400_000 }), + pairRow('kraken', 'GEMNULL', { volume: 75_000 }), + pairRow('gemini', 'GEMNULL'), + pairRow('kraken', 'SMALL', { volume: 100_000 }), + pairRow('coinbase', 'SMALL', { volume: 100_000 }), + pairRow('kraken', 'NOCAP', { volume: 100_000 }), + pairRow('coinbase', 'NOCAP', { volume: 100_000 }), + pairRow('kraken', 'USDT', { quote: 'USD', volume: 900_000_000 }), + pairRow('coinbase', 'USDT', { quote: 'USD', volume: 900_000_000 }), + pairRow('kraken', 'OFFLINE', { volume: 1_000_000, status: 'cancel_only' }), + pairRow('coinbase', 'OFFLINE', { volume: 1_000_000 }), + pairRow('kraken', 'BTC', { quote: 'EUR', price: 55_000, volume: 100_000_000 }), + ]; + const assets: CryptoAssetItem[] = [ + assetRow('BTC', 1.2e12, 1), + assetRow('ETH', 3.6e11, 2), + assetRow('USDT', 1.2e11, 3), + assetRow('SMALL', 4_000_000, 480), + assetRow('GEMNULL', 50_000_000, 200), + assetRow('THIN', 90_000_000, 150), + assetRow('ONLYK', 200_000_000, 90), + assetRow('OFFLINE', 200_000_000, 91), + // a second coin sharing ETH's ticker, ranked far lower: must not clobber ETH's cap + assetRow('eth', 1_000_000, 499), + ]; + return { pairs, assets }; +} + +describe('selectCryptoPairs', () => { + it('groups by base across venues and keeps the venue symbols', () => { + const { pairs, assets } = fixture(); + const { pairs: out } = selectCryptoPairs(pairs, assets, RULES); + const btc = out.find((p) => p.pair === 'BTC-USD')!; + expect(btc).toBeDefined(); + expect(Object.keys(btc.venues).sort()).toEqual(['binance-us', 'coinbase', 'gemini', 'kraken']); + expect(btc.venues['kraken']!.venueSymbol).toBe('XXBTZUSD'); + expect(btc.venues['coinbase']!.venueSymbol).toBe('BTC-USD'); + expect(btc.venues['binance-us']!.venueSymbol).toBe('BTCUSD'); + expect(btc.venues['gemini']!.venueSymbol).toBe('btcusd'); + expect(btc.marketCapUsd).toBe(1.2e12); + // Gemini's null volume adds nothing to the total; the USDT and EUR books are not counted. + expect(btc.totalVolumeUsd).toBe(900_000_000 + 1_200_000_000 + 80_000_000); + // Coinbase's change wins for the log line, as it did before. + expect(btc.change24hPct).toBe(-1.3); + }); + + it('applies the two-venue, $50k per venue and $10M market-cap rules', () => { + const { pairs, assets } = fixture(); + const sel = selectCryptoPairs(pairs, assets, RULES); + const names = sel.pairs.map((p) => p.pair); + expect(names).toContain('BTC-USD'); + expect(names).toContain('ETH-USD'); + expect(names).not.toContain('ONLYK-USD'); // one venue + expect(names).not.toContain('THIN-USD'); // clears $50k on one venue only + expect(names).not.toContain('SMALL-USD'); // $4M cap + expect(names).not.toContain('USDT-USD'); // excluded base + expect(names).not.toContain('OFFLINE-USD'); // cancel_only on Kraken leaves one venue + expect(names).toContain('NOCAP-USD'); // unknown cap passes, as before + expect(sel.filteredByVenues).toBe(3); // ONLYK, THIN, OFFLINE + expect(sel.filteredByMarketCap).toBe(1); // SMALL + expect(sel.marketCapCount).toBe(8); + }); + + it('treats a null (Gemini) volume as passing, as the old sentinel did', () => { + const { pairs, assets } = fixture(); + const { pairs: out } = selectCryptoPairs(pairs, assets, RULES); + const g = out.find((p) => p.pair === 'GEMNULL-USD')!; + expect(g).toBeDefined(); + expect(Object.keys(g.venues).sort()).toEqual(['gemini', 'kraken']); + expect(g.venues['gemini']!.volume24hQuote).toBeNull(); + expect(g.totalVolumeUsd).toBe(75_000); + // Two Gemini-null venues alone would also pass: nothing known says otherwise. + const twoNull = selectCryptoPairs([pairRow('gemini', 'X'), pairRow('gemini', 'X', { quote: 'USD' })], [], RULES); + expect(twoNull.pairs).toHaveLength(0); // same venue twice is still one venue + }); + + it('sorts by total known volume descending', () => { + const { pairs, assets } = fixture(); + const names = selectCryptoPairs(pairs, assets, RULES).pairs.map((p) => p.pair); + expect(names.slice(0, 2)).toEqual(['BTC-USD', 'ETH-USD']); + }); + + it('honours the MIN_VOLUME_USD style floor and the venue list', () => { + const { pairs, assets } = fixture(); + const strict = selectCryptoPairs(pairs, assets, { ...RULES, minVolumeUsd: 100_000_000 }); + expect(strict.pairs.map((p) => p.pair)).toEqual(['BTC-USD', 'ETH-USD']); + // Without Gemini, GEMNULL is a one-venue coin. + const three = selectCryptoPairs(pairs, assets, { ...RULES, venues: ['kraken', 'coinbase', 'binance-us'] }); + expect(three.pairs.map((p) => p.pair)).not.toContain('GEMNULL-USD'); + }); + + it('passes every base through the cap filter when there are no assets at all', () => { + const { pairs } = fixture(); + const sel = selectCryptoPairs(pairs, [], RULES); + expect(sel.pairs.map((p) => p.pair)).toContain('SMALL-USD'); + expect(sel.marketCapCount).toBe(0); + }); + + it('keys market caps by upper-cased symbol and lets the better-ranked duplicate win', () => { + const caps = marketCapsBySymbol([assetRow('eth', 1_000_000, 499), assetRow('ETH', 3.6e11, 2)]); + expect(caps.get('ETH')).toBe(3.6e11); + }); + + it('ignores rows with a missing or malformed data record', () => { + const bad = { ...pairRow('kraken', 'BTC'), data: null } as unknown as CryptoPairItem; + const sel = selectCryptoPairs([bad, pairRow('coinbase', 'BTC')], [], RULES); + expect(sel.pairs).toHaveLength(0); + }); +}); + +describe('fetchCryptoPairs / fetchCryptoAssets', () => { + function serve(rows: { id: number; kind: string; tags: string[] }[]) { + const urls: URL[] = []; + const fetch: FetchLike = async (url) => { + const u = new URL(url); + urls.push(u); + const kind = u.searchParams.get('kind'); + const tags = (u.searchParams.get('tags') ?? '').split(',').filter(Boolean); + const after = Number(u.searchParams.get('after') ?? 0); + const limit = Number(u.searchParams.get('limit')); + const page = rows.filter((r) => r.kind === kind && tags.every((t) => r.tags.includes(t)) && r.id > after).slice(0, limit); + return { ok: true, status: 200, statusText: 'OK', json: async () => ({ count: page.length, items: page }) }; + }; + return { fetch, urls }; + } + + it('reads kind=pair&tags=stable-quote in 200-row pages', async () => { + const rows = Array.from({ length: 1_400 }, (_, i) => pairRow(['kraken', 'coinbase', 'binance-us', 'gemini'][i % 4]!, `C${i}`)); + const { fetch, urls } = serve(rows); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + const out = await fetchCryptoPairs(client); + expect(out).toHaveLength(1_400); + expect(urls).toHaveLength(8); // 7 full pages + the short (empty) one + expect(urls[0]!.searchParams.get('collection')).toBe('crypto'); + expect(urls[0]!.searchParams.get('kind')).toBe('pair'); + expect(urls[0]!.searchParams.get('tags')).toBe('stable-quote'); + expect(urls[0]!.searchParams.get('limit')).toBe('200'); + }); + + it('reads kind=asset up to 500 rows (three pages)', async () => { + const rows = Array.from({ length: 900 }, (_, i) => assetRow(`A${i}`, 1e9, i + 1)); + const { fetch, urls } = serve(rows); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + const out = await fetchCryptoAssets(client); + expect(out).toHaveLength(500); + expect(urls).toHaveLength(3); + expect(urls[0]!.searchParams.get('kind')).toBe('asset'); + }); +}); diff --git a/packages/source-nichedb/src/crypto.ts b/packages/source-nichedb/src/crypto.ts new file mode 100644 index 0000000..1908854 --- /dev/null +++ b/packages/source-nichedb/src/crypto.ts @@ -0,0 +1,233 @@ +/** + * The `crypto` collection: CoinGecko assets and per-venue spot pairs, per + * niche-db docs/crypto.md. This module reads the rows and turns them into the + * per-base venue grouping b1dz's pair discovery filters on. + */ + +import type { NichedbClient, NichedbItem } from './client.js'; + +export type CryptoVenue = 'kraken' | 'coinbase' | 'binance-us' | 'gemini'; + +/** `data` of a kind=asset row (source coingecko-assets). */ +export interface CryptoAssetData { + id: string; + symbol: string; + name: string; + rank: number | null; + priceUsd: number | null; + marketCapUsd: number | null; + fullyDilutedUsd: number | null; + volume24hUsd: number | null; + change24hPct: number | null; + high24h: number | null; + low24h: number | null; + supply: { circulating: number | null; total: number | null; max: number | null }; + ath: number | null; + athDate: string | null; + atl: number | null; + atlDate: string | null; + updatedAt: string; +} + +/** `data` of a kind=pair row (source crypto-pairs). */ +export interface CryptoPairData { + venue: CryptoVenue | string; + venueName?: string; + base: string; + quote: string; + venueSymbol: string; + status: string; + price: number | null; + bid: number | null; + ask: number | null; + high24h: number | null; + low24h: number | null; + open24h?: number | null; + change24hPct: number | null; + volume24hBase: number | null; + /** Dollars when the quote is a dollar stable. Null on Gemini (unknown, not zero). */ + volume24hQuote: number | null; + vwap24h: number | null; + priceUsd: number | null; + updatedAt: string; +} + +export type CryptoAssetItem = NichedbItem; +export type CryptoPairItem = NichedbItem; + +export const CRYPTO_COLLECTION = 'crypto'; + +/** Every stable-quoted spot pair across the four venues (about 1,400 rows, 7-8 pages). */ +export function fetchCryptoPairs(client: NichedbClient, opts: { since?: string | Date } = {}): Promise { + return client.walk({ + collection: CRYPTO_COLLECTION, + kind: 'pair', + tags: ['stable-quote'], + since: opts.since, + limit: 200, + }); +} + +/** The top assets by market cap (CoinGecko's two pages of 250; 3 pages of 200 here). */ +export function fetchCryptoAssets(client: NichedbClient, opts: { max?: number; since?: string | Date } = {}): Promise { + const max = opts.max ?? 500; + return client.walk( + { collection: CRYPTO_COLLECTION, kind: 'asset', since: opts.since, limit: 200 }, + { maxItems: max, maxPages: Math.ceil(max / 200) + 1 }, + ); +} + +// ─── Grouping and filters ──────────────────────────────────── + +export interface VenueListing { + venue: string; + venueSymbol: string; + /** Quote-currency 24h volume; null when the venue does not report it (Gemini). */ + volume24hQuote: number | null; + priceUsd: number | null; + change24hPct: number | null; +} + +/** One tradeable base grouped across venues, in b1dz's canonical `BASE-USD` naming. */ +export interface DiscoveredPair { + /** Canonical b1dz pair, e.g. `BTC-USD`. */ + pair: string; + base: string; + quote: string; + /** Venue slug → that venue's own symbol and ticker. */ + venues: Record; + /** Sum of the known per-venue volumes (a null volume adds nothing). */ + totalVolumeUsd: number; + /** 24h change from the first venue that reports one, for logging only. */ + change24hPct: number; + /** CoinGecko market cap, 0 when the asset is not in the top list. */ + marketCapUsd: number; +} + +export interface CryptoSelectionRules { + /** Per-venue 24h quote-volume floor in dollars. */ + minVolumeUsd: number; + /** Market-cap floor; a base with an unknown cap passes, as the CoinGecko path did. */ + minMarketCapUsd: number; + /** A base must clear the volume floor on at least this many venues. */ + minVenues: number; + /** Bases never traded (stables and fiat). */ + excludedBases?: Iterable; + /** Quote to keep; b1dz's feeds only know `BASE-USD` books. Default `USD`. */ + quote?: string; + /** Venues to consider; default the four nichedb carries. */ + venues?: Iterable; +} + +export const DEFAULT_CRYPTO_VENUES: readonly CryptoVenue[] = ['kraken', 'coinbase', 'binance-us', 'gemini']; + +export interface CryptoSelection { + pairs: DiscoveredPair[]; + /** Bases dropped for being on fewer than `minVenues` venues (after the volume floor). */ + filteredByVenues: number; + /** Bases dropped by the market-cap floor. */ + filteredByMarketCap: number; + /** Assets found for the market-cap lookup. */ + marketCapCount: number; +} + +/** + * Market caps keyed by upper-cased symbol. Two CoinGecko coins can share a + * ticker; the better-ranked one wins, as it did when CoinGecko's ranked list + * was walked in order. + */ +export function marketCapsBySymbol(assets: ReadonlyArray): Map { + const best = new Map(); + for (const a of assets) { + const d = a.data; + if (!d || typeof d.symbol !== 'string') continue; + const symbol = d.symbol.toUpperCase(); + const rank = typeof d.rank === 'number' && Number.isFinite(d.rank) ? d.rank : Number.POSITIVE_INFINITY; + const cap = typeof d.marketCapUsd === 'number' && Number.isFinite(d.marketCapUsd) ? d.marketCapUsd : 0; + const prev = best.get(symbol); + if (!prev || rank < prev.rank || (rank === prev.rank && cap > prev.cap)) best.set(symbol, { rank, cap }); + } + return new Map([...best].map(([s, v]) => [s, v.cap])); +} + +/** + * Group pair rows by base across venues and apply b1dz's discovery rules: + * + * 1. only online books quoted in `rules.quote` (USD) on a known venue; + * 2. a venue counts when its `volume24hQuote` is at or above the floor, or + * is null: an unknown volume passes, exactly as the old Gemini sentinel + * (`minVolumeUsd * 2`) made every Gemini book pass; + * 3. a base needs `minVenues` counting venues; + * 4. a base with a known market cap under the floor is dropped; an unknown + * cap (not in the top list, or no assets at all) passes. + * + * Sorted by total known volume, descending. + */ +export function selectCryptoPairs( + pairRows: ReadonlyArray, + assetRows: ReadonlyArray, + rules: CryptoSelectionRules, +): CryptoSelection { + const quote = (rules.quote ?? 'USD').toUpperCase(); + const excluded = new Set([...(rules.excludedBases ?? [])].map((b) => b.toUpperCase())); + const venues = new Set([...(rules.venues ?? DEFAULT_CRYPTO_VENUES)]); + const marketCaps = marketCapsBySymbol(assetRows); + + const byBase = new Map(); + for (const row of pairRows) { + const d = row.data; + if (!d || typeof d.base !== 'string' || typeof d.venue !== 'string') continue; + if (!venues.has(d.venue)) continue; + if ((d.quote ?? '').toUpperCase() !== quote) continue; + if (d.status && d.status !== 'online') continue; + const base = d.base.toUpperCase(); + if (base.length === 0 || excluded.has(base)) continue; + + const vol = typeof d.volume24hQuote === 'number' && Number.isFinite(d.volume24hQuote) ? d.volume24hQuote : null; + if (vol !== null && vol < rules.minVolumeUsd) continue; + + const listing: VenueListing = { + venue: d.venue, + venueSymbol: d.venueSymbol, + volume24hQuote: vol, + priceUsd: typeof d.priceUsd === 'number' ? d.priceUsd : null, + change24hPct: typeof d.change24hPct === 'number' && Number.isFinite(d.change24hPct) ? d.change24hPct : null, + }; + + let entry = byBase.get(base); + if (!entry) { + entry = { pair: `${base}-${quote}`, base, quote, venues: {}, totalVolumeUsd: 0, change24hPct: 0, marketCapUsd: 0 }; + byBase.set(base, entry); + } + const prev = entry.venues[d.venue]; + // The same book twice (a stale row plus a fresh one): keep the larger volume. + if (prev && (prev.volume24hQuote ?? -1) >= (vol ?? -1)) continue; + if (prev?.volume24hQuote != null) entry.totalVolumeUsd -= prev.volume24hQuote; + entry.venues[d.venue] = listing; + if (vol !== null) entry.totalVolumeUsd += vol; + } + + const pairs: DiscoveredPair[] = []; + let filteredByVenues = 0; + let filteredByMarketCap = 0; + for (const entry of byBase.values()) { + const listings = Object.values(entry.venues); + if (listings.length < rules.minVenues) { + filteredByVenues++; + continue; + } + const cap = marketCaps.get(entry.base) ?? 0; + entry.marketCapUsd = cap; + if (marketCaps.size > 0 && cap > 0 && cap < rules.minMarketCapUsd) { + filteredByMarketCap++; + continue; + } + // Coinbase's reported change first (the old path let Coinbase overwrite), then any venue. + const change = entry.venues['coinbase']?.change24hPct ?? listings.find((l) => l.change24hPct != null)?.change24hPct ?? 0; + entry.change24hPct = change; + pairs.push(entry); + } + pairs.sort((a, b) => b.totalVolumeUsd - a.totalVolumeUsd || a.pair.localeCompare(b.pair)); + + return { pairs, filteredByVenues, filteredByMarketCap, marketCapCount: marketCaps.size }; +} diff --git a/packages/source-nichedb/src/env.ts b/packages/source-nichedb/src/env.ts new file mode 100644 index 0000000..fb2a5fc --- /dev/null +++ b/packages/source-nichedb/src/env.ts @@ -0,0 +1,28 @@ +/** + * Feature switches for reading from nichedb.dev. + * + * NICHEDB_CRYPTO=1 pair discovery reads the crypto universe from nichedb + * NICHEDB_MARKETS=1 strategy backtests read daily equity bars from nichedb + * NICHEDB_URL base URL (default https://nichedb.dev) + * + * Read through `process.env[name]` on purpose: Next.js inlines + * `process.env.NAME` member expressions at build time, and these are runtime + * switches that must be flippable on the deployed daemon and web app. + */ + +export type NichedbSwitch = 'NICHEDB_CRYPTO' | 'NICHEDB_MARKETS'; + +const ON = new Set(['1', 'true', 'yes', 'on']); + +export function nichedbEnabled(name: NichedbSwitch, env: NodeJS.ProcessEnv = process.env): boolean { + const raw = env[name]; + if (raw == null) return false; + return ON.has(raw.trim().toLowerCase()); +} + +export const DEFAULT_NICHEDB_URL = 'https://nichedb.dev'; + +export function nichedbBaseUrl(env: NodeJS.ProcessEnv = process.env): string { + const raw = env['NICHEDB_URL']?.trim(); + return (raw && raw.length > 0 ? raw : DEFAULT_NICHEDB_URL).replace(/\/+$/, ''); +} diff --git a/packages/source-nichedb/src/index.ts b/packages/source-nichedb/src/index.ts new file mode 100644 index 0000000..b60cdfe --- /dev/null +++ b/packages/source-nichedb/src/index.ts @@ -0,0 +1,54 @@ +/** + * @b1dz/source-nichedb — read-only client for nichedb.dev's public items API + * plus the two readers b1dz uses: the crypto universe (pair discovery) and + * daily equity bars (strategy backtests). Everything real-time stays on the + * venues' own feeds; this package only replaces slow-moving catalogue reads. + */ + +export { nichedbEnabled, nichedbBaseUrl, DEFAULT_NICHEDB_URL, type NichedbSwitch } from './env.js'; +export { + createNichedbClient, + NichedbError, + NICHEDB_PAGE_LIMIT, + type NichedbClient, + type NichedbClientOptions, + type NichedbItem, + type ItemsQuery, + type WalkOptions, + type FetchLike, +} from './client.js'; +export { + CRYPTO_COLLECTION, + DEFAULT_CRYPTO_VENUES, + fetchCryptoPairs, + fetchCryptoAssets, + marketCapsBySymbol, + selectCryptoPairs, + type CryptoVenue, + type CryptoAssetData, + type CryptoPairData, + type CryptoAssetItem, + type CryptoPairItem, + type CryptoSelectionRules, + type CryptoSelection, + type DiscoveredPair, + type VenueListing, +} from './crypto.js'; +export { + MARKETS_COLLECTION, + HISTORY_MAX_BARS, + HISTORY_MAX_AGE_DAYS, + historySymbolTag, + dayToUtcMs, + barsFromHistory, + historyWindow, + fetchHistoryItem, + fetchDailyBars, + type HistoryBarTuple, + type HistoryData, + type HistoryItem, + type HistoryMiss, + type HistoryResult, + type HistoryOptions, + type DailyBar, +} from './markets.js'; diff --git a/packages/source-nichedb/src/markets.test.ts b/packages/source-nichedb/src/markets.test.ts new file mode 100644 index 0000000..3422006 --- /dev/null +++ b/packages/source-nichedb/src/markets.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest'; +import { createNichedbClient, type FetchLike } from './client.js'; +import { barsFromHistory, dayToUtcMs, fetchDailyBars, historySymbolTag, historyWindow, DAY_MS, type HistoryItem } from './markets.js'; + +const NOW = Date.UTC(2026, 8, 16, 14, 0, 0); // 2026-09-16 14:00Z, a Wednesday + +function ymd(ms: number): string { + return new Date(ms).toISOString().slice(0, 10); +} + +/** `count` weekday bars ending on `lastDay`, oldest first, close = 100 + i. */ +export function historyItem(symbol: string, lastMs: number, count: number): HistoryItem { + const bars: HistoryItem['data']['bars'] = []; + let ms = lastMs; + while (bars.length < count) { + const dow = new Date(ms).getUTCDay(); + if (dow !== 0 && dow !== 6) bars.unshift([ymd(ms), 100, 101, 99, 100 + bars.length, 1_000, 100.5]); + ms -= DAY_MS; + } + // close values were assigned newest-first above; make them ascend oldest-first for readable tests + bars.forEach((b, i) => { b[4] = 100 + i; }); + return { + id: 1, + collection: 'markets', + source: 'equity-history', + kind: 'history', + external_id: `history:${symbol}`, + title: `${symbol} daily bars`, + published_at: `${bars[bars.length - 1]![0]}T00:00:00.000Z`, + updated_at: '2026-09-11T00:30:00.000Z', + tags: ['history', `symbol:${symbol.toLowerCase()}`, 'feed:iex'], + data: { symbol, timeframe: '1Day', feed: 'iex', adjustment: 'split', bars, first: bars[0]![0], last: bars[bars.length - 1]![0], count: bars.length }, + }; +} + +describe('history mapping', () => { + it('tags symbols lower-case in Alpaca spelling', () => { + expect(historySymbolTag('AAPL')).toBe('symbol:aapl'); + expect(historySymbolTag('BRK-B')).toBe('symbol:brk.b'); + }); + + it('maps YYYY-MM-DD to midnight UTC', () => { + expect(dayToUtcMs('2026-09-10')).toBe(Date.UTC(2026, 8, 10)); + expect(dayToUtcMs('nope')).toBeNull(); + }); + + it('maps bar tuples to OHLCV rows, oldest first, dropping malformed ones', () => { + const item = historyItem('AAPL', NOW - DAY_MS, 3); + item.data.bars.push(['bad', 1, 1, 1, 1, 1, null], ['2026-09-12', Number.NaN, 1, 1, 1, 1, null]); + const bars = barsFromHistory(item.data); + expect(bars).toHaveLength(3); + expect(bars[0]!.ts).toBeLessThan(bars[2]!.ts); + expect(bars[0]).toMatchObject({ open: 100, high: 101, low: 99, close: 100, volume: 1_000, vwap: 100.5 }); + expect(bars[0]!.ts).toBe(dayToUtcMs(bars[0]!.day)); + }); +}); + +describe('historyWindow', () => { + const start = NOW - 365 * DAY_MS; + + it('serves a one-year request from a fresh 400-bar window, padded a week either side', () => { + const item = historyItem('AAPL', NOW - DAY_MS, 400); + const r = historyWindow(item, start, NOW, { now: NOW }); + expect(r.ok).toBe(true); + if (!r.ok) return; + expect(r.bars[0]!.ts).toBeGreaterThanOrEqual(start - 7 * DAY_MS); + expect(r.bars[0]!.ts).toBeLessThanOrEqual(start); + expect(r.bars[r.bars.length - 1]!.day).toBe(ymd(NOW - DAY_MS)); + expect(r.bars.length).toBeGreaterThan(250); + }); + + it('falls back when the item is missing', () => { + expect(historyWindow(null, start, NOW, { now: NOW })).toEqual({ ok: false, reason: 'missing' }); + }); + + it('falls back when the last bar is older than 5 days', () => { + // Wednesday now: a bar from last Friday (5 days) still serves; last Thursday (6 days) does not. + const fresh = historyItem('AAPL', NOW - 5 * DAY_MS, 400); + expect(fresh.data.last).toBe('2026-09-11'); + expect(historyWindow(fresh, start, NOW, { now: NOW }).ok).toBe(true); + const stale = historyItem('AAPL', NOW - 6 * DAY_MS, 400); + expect(stale.data.last).toBe('2026-09-10'); + expect(historyWindow(stale, start, NOW, { now: NOW })).toMatchObject({ ok: false, reason: 'stale' }); + }); + + it('falls back when the request spans more than 400 days', () => { + const item = historyItem('AAPL', NOW - DAY_MS, 400); + expect(historyWindow(item, NOW - 401 * DAY_MS, NOW, { now: NOW })).toEqual({ ok: false, reason: 'window-too-long' }); + expect(historyWindow(item, NOW - 400 * DAY_MS, NOW, { now: NOW }).ok).toBe(true); + }); + + it('falls back when the window does not reach the requested start', () => { + const young = historyItem('NEWCO', NOW - DAY_MS, 40); + expect(historyWindow(young, start, NOW, { now: NOW })).toMatchObject({ ok: false, reason: 'short' }); + expect(historyWindow(young, NOW - 30 * DAY_MS, NOW, { now: NOW }).ok).toBe(true); + }); + + it('falls back on an item with no usable bars', () => { + const item = historyItem('AAPL', NOW - DAY_MS, 1); + item.data.bars = []; + expect(historyWindow(item, start, NOW, { now: NOW })).toMatchObject({ ok: false, reason: 'empty' }); + }); +}); + +describe('fetchDailyBars', () => { + function serve(items: HistoryItem[]) { + const urls: URL[] = []; + const fetch: FetchLike = async (url) => { + const u = new URL(url); + urls.push(u); + const tags = (u.searchParams.get('tags') ?? '').split(','); + const hit = items.filter((i) => tags.every((t) => i.tags.includes(t))).slice(0, Number(u.searchParams.get('limit'))); + return { ok: true, status: 200, statusText: 'OK', json: async () => ({ count: hit.length, items: hit }) }; + }; + return { fetch, urls }; + } + + it('asks for collection=markets&kind=history&tags=symbol:&limit=1', async () => { + const { fetch, urls } = serve([historyItem('AAPL', NOW - DAY_MS, 400)]); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + const r = await fetchDailyBars(client, 'AAPL', NOW - 365 * DAY_MS, NOW, { now: NOW }); + expect(r.ok).toBe(true); + expect(urls).toHaveLength(1); + const q = urls[0]!.searchParams; + expect(q.get('collection')).toBe('markets'); + expect(q.get('kind')).toBe('history'); + expect(q.get('tags')).toBe('symbol:aapl'); + expect(q.get('limit')).toBe('1'); + }); + + it('reports missing for an unknown symbol and makes no request for an over-long window', async () => { + const { fetch, urls } = serve([]); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + expect(await fetchDailyBars(client, 'ZZZZ', NOW - 30 * DAY_MS, NOW, { now: NOW })).toEqual({ ok: false, reason: 'missing' }); + expect(urls).toHaveLength(1); + expect(await fetchDailyBars(client, 'AAPL', NOW - 5 * 365 * DAY_MS, NOW, { now: NOW })).toEqual({ ok: false, reason: 'window-too-long' }); + expect(urls).toHaveLength(1); + }); +}); diff --git a/packages/source-nichedb/src/markets.ts b/packages/source-nichedb/src/markets.ts new file mode 100644 index 0000000..7f4c6c3 --- /dev/null +++ b/packages/source-nichedb/src/markets.ts @@ -0,0 +1,148 @@ +/** + * The `markets` collection's `history` kind: one item per US equity symbol + * carrying its last 400 daily bars, oldest first, per niche-db docs/markets.md. + * + * data.bars = [[ 'YYYY-MM-DD', open, high, low, close, volume, vwap ], ...] + */ + +import type { NichedbClient, NichedbItem } from './client.js'; + +export const MARKETS_COLLECTION = 'markets'; + +/** One bar as nichedb stores it: day, open, high, low, close, volume, vwap (null when the feed sent none). */ +export type HistoryBarTuple = [string, number, number, number, number, number, (number | null)?]; + +export interface HistoryData { + symbol: string; + timeframe: '1Day' | string; + feed: 'iex' | 'sip' | string; + adjustment: string; + bars: HistoryBarTuple[]; + first: string; + last: string; + count: number; +} + +export type HistoryItem = NichedbItem; + +/** A daily bar in the shape b1dz's backtesters consume. `ts` is midnight UTC of the bar's day. */ +export interface DailyBar { + ts: number; + day: string; + open: number; + high: number; + low: number; + close: number; + volume: number; + vwap: number | null; +} + +export const DAY_MS = 24 * 60 * 60 * 1000; +/** nichedb keeps at most this many bars per symbol; a longer request cannot be served from it. */ +export const HISTORY_MAX_BARS = 400; +/** A window whose last bar is older than this is stale (long weekend plus a holiday still fits). */ +export const HISTORY_MAX_AGE_DAYS = 5; +/** The Yahoo path pads each end of the window by a week; keep the same slack. */ +const WINDOW_PAD_MS = 7 * DAY_MS; + +/** nichedb tags symbols lower-case in Alpaca's spelling (`brk.b`); b1dz and Yahoo write `BRK-B`. */ +export function historySymbolTag(symbol: string): string { + return `symbol:${symbol.trim().toLowerCase().replace(/-/g, '.')}`; +} + +/** Midnight UTC of a `YYYY-MM-DD` day, or null when the string is not one. */ +export function dayToUtcMs(day: string): number | null { + const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(day); + if (!m) return null; + const ms = Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3])); + return Number.isFinite(ms) ? ms : null; +} + +/** Turn a history item's tuples into bars, dropping malformed rows, oldest first. */ +export function barsFromHistory(data: HistoryData | null | undefined): DailyBar[] { + const out: DailyBar[] = []; + for (const tuple of data?.bars ?? []) { + if (!Array.isArray(tuple) || tuple.length < 6) continue; + const [day, open, high, low, close, volume, vwap] = tuple; + const ts = typeof day === 'string' ? dayToUtcMs(day) : null; + if (ts === null) continue; + if (![open, high, low, close].every((n) => typeof n === 'number' && Number.isFinite(n))) continue; + out.push({ + ts, + day, + open: open as number, + high: high as number, + low: low as number, + close: close as number, + volume: typeof volume === 'number' && Number.isFinite(volume) ? volume : 0, + vwap: typeof vwap === 'number' && Number.isFinite(vwap) ? vwap : null, + }); + } + out.sort((a, b) => a.ts - b.ts); + return out; +} + +export type HistoryMiss = + | 'window-too-long' // more than 400 days asked for; nichedb holds at most 400 bars + | 'missing' // no history item for the symbol + | 'empty' // an item with no usable bars + | 'stale' // last bar older than HISTORY_MAX_AGE_DAYS + | 'short'; // the window does not reach back to the requested start + +export type HistoryResult = { ok: true; bars: DailyBar[]; item: HistoryItem } | { ok: false; reason: HistoryMiss; item?: HistoryItem }; + +export interface HistoryOptions { + /** "Now" for the staleness check; defaults to Date.now(). */ + now?: number; + maxAgeDays?: number; +} + +/** + * Decide whether a history item can serve a `[startMs, endMs]` request and, + * if so, return its bars for that window (padded a week either side, as the + * Yahoo path pads). Pure: the caller fetches the item. + */ +export function historyWindow(item: HistoryItem | null | undefined, startMs: number, endMs: number, opts: HistoryOptions = {}): HistoryResult { + const now = opts.now ?? Date.now(); + const maxAgeMs = (opts.maxAgeDays ?? HISTORY_MAX_AGE_DAYS) * DAY_MS; + if (endMs - startMs > HISTORY_MAX_BARS * DAY_MS) return { ok: false, reason: 'window-too-long' }; + if (!item) return { ok: false, reason: 'missing' }; + const all = barsFromHistory(item.data); + if (all.length === 0) return { ok: false, reason: 'empty', item }; + const last = all[all.length - 1]!; + // `last.ts` is midnight of the bar's day: the bar is stale once its day is more than maxAgeDays before now's day. + if (now - last.ts >= maxAgeMs + DAY_MS) return { ok: false, reason: 'stale', item }; + const first = all[0]!; + // A symbol first seen recently has a window that starts after the request; do not truncate silently. + if (first.ts > startMs + WINDOW_PAD_MS) return { ok: false, reason: 'short', item }; + const lo = startMs - WINDOW_PAD_MS; + const hi = endMs + WINDOW_PAD_MS; + return { ok: true, bars: all.filter((b) => b.ts >= lo && b.ts <= hi), item }; +} + +/** One request: the history item for a symbol, or null when nichedb has none. */ +export async function fetchHistoryItem(client: NichedbClient, symbol: string): Promise { + const rows = await client.items({ + collection: MARKETS_COLLECTION, + kind: 'history', + tags: [historySymbolTag(symbol)], + limit: 1, + }); + return rows[0] ?? null; +} + +/** + * Daily bars for a symbol from nichedb, or a reason it cannot serve them. + * Makes no request when the window is longer than nichedb can hold. + */ +export async function fetchDailyBars( + client: NichedbClient, + symbol: string, + startMs: number, + endMs: number, + opts: HistoryOptions = {}, +): Promise { + if (endMs - startMs > HISTORY_MAX_BARS * DAY_MS) return { ok: false, reason: 'window-too-long' }; + const item = await fetchHistoryItem(client, symbol); + return historyWindow(item, startMs, endMs, opts); +} diff --git a/packages/source-nichedb/tsconfig.build.json b/packages/source-nichedb/tsconfig.build.json new file mode 100644 index 0000000..2e725b7 --- /dev/null +++ b/packages/source-nichedb/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "noEmit": false + }, + "exclude": ["**/*.test.ts"] +} diff --git a/packages/source-nichedb/tsconfig.json b/packages/source-nichedb/tsconfig.json new file mode 100644 index 0000000..bf5a36d --- /dev/null +++ b/packages/source-nichedb/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7c04dcb..b634b6a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,6 +68,9 @@ importers: '@b1dz/source-crypto-trade': specifier: workspace:* version: link:../../packages/source-crypto-trade + '@b1dz/source-nichedb': + specifier: workspace:* + version: link:../../packages/source-nichedb '@b1dz/source-strategies': specifier: workspace:* version: link:../../packages/source-strategies @@ -235,6 +238,9 @@ importers: '@b1dz/source-crypto-trade': specifier: workspace:* version: link:../../packages/source-crypto-trade + '@b1dz/source-nichedb': + specifier: workspace:* + version: link:../../packages/source-nichedb '@b1dz/source-strategies': specifier: workspace:* version: link:../../packages/source-strategies @@ -554,6 +560,9 @@ importers: '@b1dz/core': specifier: workspace:* version: link:../core + '@b1dz/source-nichedb': + specifier: workspace:* + version: link:../source-nichedb undici: specifier: ^8.0.2 version: 8.0.2 @@ -640,6 +649,18 @@ importers: specifier: latest version: 5.0.0(@types/node@26.4.1)(vite@8.0.7(@types/node@26.4.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.23.13)) + packages/source-nichedb: + devDependencies: + '@types/node': + specifier: latest + version: 26.4.1 + typescript: + specifier: latest + version: 7.0.2 + vitest: + specifier: latest + version: 5.0.0(@types/node@26.4.1)(vite@8.0.7(@types/node@26.4.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.23.13)) + packages/source-schwab: dependencies: '@b1dz/core':