diff --git a/apps/cli/package.json b/apps/cli/package.json index eac9c58..73bd029 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -19,6 +19,7 @@ "@b1dz/storage-supabase": "workspace:*", "@b1dz/source-crypto-arb": "workspace:*", "@b1dz/source-crypto-trade": "workspace:*", + "@b1dz/source-nichedb": "workspace:*", "@b1dz/source-strategies": "workspace:*", "@b1dz/storage-b1dz-api": "workspace:*", "@b1dz/sdk": "workspace:*", diff --git a/apps/cli/src/strategy-backtest.test.ts b/apps/cli/src/strategy-backtest.test.ts index a2cf368..11b9854 100644 --- a/apps/cli/src/strategy-backtest.test.ts +++ b/apps/cli/src/strategy-backtest.test.ts @@ -1,5 +1,81 @@ -import { describe, it, expect } from 'vitest'; -import { parseArgs } from './strategy-backtest.js'; +import { afterEach, describe, it, expect, vi } from 'vitest'; +import { fetchDailySnapshots, parseArgs } from './strategy-backtest.js'; + +describe('strategy-backtest daily bars via nichedb (NICHEDB_MARKETS)', () => { + const DAY = 24 * 60 * 60 * 1000; + const today = Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate()); + + function historyItem(symbol: string, count = 400) { + const bars: (string | number | null)[][] = []; + let ms = today - DAY; + while (bars.length < count) { + const dow = new Date(ms).getUTCDay(); + if (dow !== 0 && dow !== 6) bars.unshift([new Date(ms).toISOString().slice(0, 10), 10, 11, 9, 10 + bars.length, 1000, null]); + ms -= DAY; + } + return { + id: 7, collection: 'markets', kind: 'history', external_id: `history:${symbol}`, title: `${symbol} daily bars`, + published_at: null, updated_at: new Date().toISOString(), tags: ['history', `symbol:${symbol.toLowerCase()}`, 'feed:iex'], + data: { symbol, timeframe: '1Day', feed: 'iex', adjustment: 'split', bars, first: bars[0]![0], last: bars[bars.length - 1]![0], count: bars.length }, + }; + } + + function installFetch(nichedbItems: unknown[]) { + const urls: string[] = []; + vi.stubGlobal('fetch', vi.fn(async (input: string | URL) => { + const url = typeof input === 'string' ? input : input.href; + urls.push(url); + const host = new URL(url).host; + if (host === 'nichedb.dev') return Response.json({ count: nichedbItems.length, items: nichedbItems }); + if (host === 'query1.finance.yahoo.com') { + return Response.json({ chart: { result: [{ timestamp: [1_700_000_000, 1_700_086_400], indicators: { quote: [{ close: [1, 2] }] } }] } }); + } + return new Response('nope', { status: 404 }); + })); + return urls; + } + + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + }); + + it('uses Yahoo only when the switch is off', async () => { + vi.stubEnv('NICHEDB_MARKETS', ''); + const urls = installFetch([historyItem('SPY')]); + const snaps = await fetchDailySnapshots('SPY', today - 365 * DAY, today); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false); + expect(snaps.map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']); + }); + + it('maps nichedb bars to snapshots for an equity and never asks nichedb for crypto', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const urls = installFetch([historyItem('SPY')]); + const snaps = await fetchDailySnapshots('SPY', today - 365 * DAY, today); + expect(urls).toHaveLength(1); + expect(new URL(urls[0]!).searchParams.get('tags')).toBe('symbol:spy'); + expect(snaps.length).toBeGreaterThan(240); + expect(snaps[0]).toMatchObject({ exchange: 'nichedb', pair: 'SPY', assetClass: 'equity', bidSize: 1, askSize: 1 }); + expect(snaps[0]!.bid).toBe(snaps[0]!.ask); + expect(snaps.every((s, i) => i === 0 || s.ts > snaps[i - 1]!.ts)).toBe(true); + urls.length = 0; + const crypto = await fetchDailySnapshots('BTC-USD', today - 365 * DAY, today); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false); + expect(crypto.map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']); + }); + + it('falls back to Yahoo when nichedb is missing the symbol or the window is too long', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const urls = installFetch([]); + expect((await fetchDailySnapshots('ZZZZ', today - 90 * DAY, today)).map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']); + expect(urls.filter((u) => u.includes('nichedb.dev'))).toHaveLength(1); + urls.length = 0; + expect((await fetchDailySnapshots('SPY', today - 5 * 365 * DAY, today)).map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']); + expect(urls.filter((u) => u.includes('nichedb.dev'))).toHaveLength(0); + stderr.mockRestore(); + }); +}); describe('strategy-backtest parseArgs', () => { it('defaults to backtesting both asset classes', () => { diff --git a/apps/cli/src/strategy-backtest.ts b/apps/cli/src/strategy-backtest.ts index aa64d75..e19b45d 100644 --- a/apps/cli/src/strategy-backtest.ts +++ b/apps/cli/src/strategy-backtest.ts @@ -4,7 +4,8 @@ * Unlike `b1dz backtest ` (the server-side multi-pair crypto candle sim), * this runs locally against the deterministic StrategyPlugin engine in * @b1dz/source-strategies: it replays the strategy's own buy/sell signals - * long-only over Yahoo daily bars. + * long-only over Yahoo daily bars (equities come from nichedb.dev's `markets` + * history first when NICHEDB_MARKETS=1, Yahoo when nichedb cannot serve them). * * Crucially it scores crypto and equities SEPARATELY so you can see which asset * class a strategy suits. Run both (default, with a head-to-head verdict), @@ -19,6 +20,7 @@ import { readFileSync } from 'node:fs'; import chalk from 'chalk'; import Table from 'cli-table3'; import { PLUGIN_CATALOG, type MarketSnapshot, type StrategyPlugin } from '@b1dz/core'; +import { createNichedbClient, fetchDailyBars, nichedbEnabled } from '@b1dz/source-nichedb'; import { STRATEGY_PLUGINS, replayStrategy, @@ -105,7 +107,37 @@ function subtract(end: Date, h: (typeof HORIZONS)[number]): Date { return d; } -async function fetchDailySnapshots(symbol: string, startMs: number, endMs: number): Promise { +/** + * Equity bars from nichedb's `markets` history (NICHEDB_MARKETS=1). Returns + * null when nichedb cannot serve the window: missing symbol, last bar older + * than 5 days, or more than its 400-bar window asked for. Crypto symbols are + * not in that collection, so they never ask. + */ +async function fetchNichedbSnapshots(symbol: string, startMs: number, endMs: number): Promise { + if (symbol.includes('-USD')) return null; + const result = await fetchDailyBars(createNichedbClient(), symbol, startMs, endMs); + if (!result.ok) { + process.stderr.write(chalk.dim(` (nichedb cannot serve ${symbol}: ${result.reason}; using Yahoo)\n`)); + return null; + } + return result.bars.map((b) => ({ + exchange: 'nichedb', pair: symbol, bid: b.close, ask: b.close, bidSize: 1, askSize: 1, ts: b.ts, assetClass: 'equity' as const, + })); +} + +export async function fetchDailySnapshots(symbol: string, startMs: number, endMs: number): Promise { + if (nichedbEnabled('NICHEDB_MARKETS')) { + try { + const fromNichedb = await fetchNichedbSnapshots(symbol, startMs, endMs); + if (fromNichedb && fromNichedb.length > 0) return fromNichedb; + } catch (e) { + process.stderr.write(chalk.dim(` (nichedb error for ${symbol}: ${(e as Error).message}; using Yahoo)\n`)); + } + } + return fetchYahooSnapshots(symbol, startMs, endMs); +} + +async function fetchYahooSnapshots(symbol: string, startMs: number, endMs: number): Promise { const period1 = Math.floor((startMs - 7 * DAY_MS) / 1000); const period2 = Math.floor((endMs + 7 * DAY_MS) / 1000); const url = @@ -244,7 +276,7 @@ export async function runStrategyBacktestCli(argv: string[]): Promise { console.log( chalk.dim( - `Long-only signal replay · $${args.amount}/entry · Yahoo daily · classes: ${args.classes.join(' + ')} · ignores fees/slippage`, + `Long-only signal replay · $${args.amount}/entry · ${nichedbEnabled('NICHEDB_MARKETS') ? 'nichedb + Yahoo' : 'Yahoo'} daily · classes: ${args.classes.join(' + ')} · ignores fees/slippage`, ), ); diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts index 3f17e82..8a9f9c0 100644 --- a/apps/web/next.config.ts +++ b/apps/web/next.config.ts @@ -67,6 +67,7 @@ const config: NextConfig = { transpilePackages: [ '@b1dz/core', '@b1dz/sdk', + '@b1dz/source-nichedb', '@b1dz/source-strategies', '@b1dz/storage-json', '@b1dz/storage-supabase', diff --git a/apps/web/package.json b/apps/web/package.json index 53d64c9..2d4319f 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -16,6 +16,7 @@ "@b1dz/projection-engine": "workspace:*", "@b1dz/source-crypto-arb": "workspace:*", "@b1dz/source-crypto-trade": "workspace:*", + "@b1dz/source-nichedb": "workspace:*", "@b1dz/source-strategies": "workspace:*", "@b1dz/storage-json": "workspace:*", "@b1dz/storage-supabase": "workspace:*", diff --git a/apps/web/src/app/api/strategies/backtest/route.test.ts b/apps/web/src/app/api/strategies/backtest/route.test.ts index 1273c85..9edaef7 100644 --- a/apps/web/src/app/api/strategies/backtest/route.test.ts +++ b/apps/web/src/app/api/strategies/backtest/route.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const authenticateMock = vi.fn(); const unauthorizedMock = vi.fn(() => new Response(JSON.stringify({ error: 'unauthorized' }), { status: 401 })); @@ -108,3 +108,98 @@ describe('POST /api/strategies/backtest', () => { expect(runBacktestMock).not.toHaveBeenCalled(); }); }); + +describe('equity closes via nichedb (NICHEDB_MARKETS)', () => { + const DAY = 24 * 60 * 60 * 1000; + const today = Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate()); + + /** A fresh 400-bar history item ending yesterday, weekdays only. */ + function historyItem(symbol: string) { + const bars: (string | number | null)[][] = []; + let ms = today - DAY; + while (bars.length < 400) { + const dow = new Date(ms).getUTCDay(); + if (dow !== 0 && dow !== 6) bars.unshift([new Date(ms).toISOString().slice(0, 10), 10, 11, 9, 10 + bars.length, 1000, null]); + ms -= DAY; + } + return { + id: 7, collection: 'markets', kind: 'history', external_id: `history:${symbol}`, title: `${symbol} daily bars`, + published_at: null, updated_at: new Date().toISOString(), tags: ['history', `symbol:${symbol.toLowerCase()}`, 'feed:iex'], + data: { symbol, timeframe: '1Day', feed: 'iex', adjustment: 'split', bars, first: bars[0]![0], last: bars[399]![0], count: 400 }, + }; + } + + function installFetch(nichedbItems: unknown[]) { + const urls: string[] = []; + vi.stubGlobal('fetch', vi.fn(async (input: string | URL) => { + const url = typeof input === 'string' ? input : input.href; + urls.push(url); + const host = new URL(url).host; + if (host === 'nichedb.dev') return Response.json({ count: nichedbItems.length, items: nichedbItems }); + if (host === 'query1.finance.yahoo.com') { + return Response.json({ chart: { result: [{ timestamp: [1_700_000_000, 1_700_086_400], indicators: { quote: [{ close: [1, 2] }] } }] } }); + } + return new Response('nope', { status: 404 }); + })); + return urls; + } + + async function equityCloses(symbol: string, days: number) { + const { POST } = await importRoute(); + await POST(makeReq({ definition: validDoc, classes: ['equity'] }) as never); + const [, opts] = runBacktestMock.mock.calls[0]!; + return opts.fetchCloses(symbol, today - days * DAY, today) as Promise<{ ts: number; close: number }[]>; + } + + beforeEach(() => { + vi.stubEnv('ALPACA_API_KEY_ID', ''); + vi.stubEnv('ALPACA_API_SECRET_KEY', ''); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it('makes no nichedb request when the switch is off', async () => { + vi.stubEnv('NICHEDB_MARKETS', ''); + const urls = installFetch([historyItem('AAPL')]); + const rows = await equityCloses('AAPL', 365); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false); + expect(urls.some((u) => u.includes('finance.yahoo.com'))).toBe(true); + expect(rows.map((r) => r.close)).toEqual([1, 2]); + }); + + it('serves a one-year window from nichedb bars without touching Yahoo', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const urls = installFetch([historyItem('AAPL')]); + const rows = await equityCloses('AAPL', 365); + expect(urls).toHaveLength(1); + const q = new URL(urls[0]!).searchParams; + expect(q.get('collection')).toBe('markets'); + expect(q.get('kind')).toBe('history'); + expect(q.get('tags')).toBe('symbol:aapl'); + expect(q.get('limit')).toBe('1'); + expect(rows.length).toBeGreaterThan(240); + expect(rows[rows.length - 1]!.ts).toBe(rows[rows.length - 1]!.ts - (rows[rows.length - 1]!.ts % DAY)); // midnight UTC + expect(rows.every((r) => Number.isFinite(r.close))).toBe(true); + }); + + it('falls back to Yahoo when nichedb has no history for the symbol', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const urls = installFetch([]); + const rows = await equityCloses('ZZZZ', 90); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(true); + expect(urls.some((u) => u.includes('finance.yahoo.com'))).toBe(true); + expect(rows.map((r) => r.close)).toEqual([1, 2]); + }); + + it('skips nichedb for a window longer than its 400 bars and uses Yahoo', async () => { + vi.stubEnv('NICHEDB_MARKETS', '1'); + const urls = installFetch([historyItem('AAPL')]); + const rows = await equityCloses('AAPL', 5 * 365); + expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false); + expect(rows.map((r) => r.close)).toEqual([1, 2]); + }); +}); diff --git a/apps/web/src/app/api/strategies/backtest/route.ts b/apps/web/src/app/api/strategies/backtest/route.ts index 0c43010..1093dd2 100644 --- a/apps/web/src/app/api/strategies/backtest/route.ts +++ b/apps/web/src/app/api/strategies/backtest/route.ts @@ -14,13 +14,16 @@ * - Crypto → Kraken daily OHLC via @b1dz/source-crypto-trade's * fetchHistoricalCandles (keyless; the same path the daemon + /api/backtest * use, proven to work from Railway). - * - Equities → Alpaca daily bars (when ALPACA_API_KEY_ID/SECRET are set), + * - Equities → nichedb.dev's `markets` history (when NICHEDB_MARKETS=1 and + * it holds a fresh window covering the request; keyless, one request per + * symbol), then Alpaca daily bars (when ALPACA_API_KEY_ID/SECRET are set), * falling back to Yahoo best-effort. Yahoo is frequently blocked from - * datacenter IPs, so without Alpaca keys equities may return no data; - * crypto is unaffected. + * datacenter IPs, so without nichedb or Alpaca keys equities may return + * no data; crypto is unaffected. */ import type { NextRequest } from 'next/server'; import { fetchHistoricalCandles } from '@b1dz/source-crypto-trade'; +import { createNichedbClient, fetchDailyBars, nichedbEnabled } from '@b1dz/source-nichedb'; import { tsp } from '@b1dz/source-strategies'; import { authenticate, unauthorized } from '@/lib/api-auth'; import { @@ -101,10 +104,24 @@ async function fetchAlpacaCloses(symbol: string, startMs: number, endMs: number) return rows; } +/** + * Equity daily closes from nichedb's `markets` history item (400 daily bars, + * split adjusted). Throws when nichedb has no fresh window covering the + * request (missing symbol, last bar older than 5 days, more than 400 days + * asked for) so the caller moves on to Alpaca and Yahoo. + */ +async function fetchNichedbCloses(symbol: string, startMs: number, endMs: number): Promise { + const result = await fetchDailyBars(createNichedbClient(), symbol, startMs, endMs); + if (!result.ok) throw new Error(`nichedb ${result.reason}`); + const rows = result.bars.map((b) => ({ ts: b.ts, close: b.close })); + if (rows.length === 0) throw new Error('nichedb returned no rows'); + return rows; +} + /** * Route closes by asset class: * - crypto → Kraken (reliable from Railway, keyless). - * - equities → Alpaca (if keys set) → Yahoo fallback. + * - equities → nichedb (if NICHEDB_MARKETS=1) → Alpaca (if keys set) → Yahoo fallback. * Returns [] only if every source for that symbol fails. */ const fetchCloses: FetchCloses = async (symbol, startMs, endMs) => { @@ -116,7 +133,14 @@ const fetchCloses: FetchCloses = async (symbol, startMs, endMs) => { return []; } } - // Equity: Alpaca first, then Yahoo. + // Equity: nichedb when switched on, then Alpaca, then Yahoo. + if (nichedbEnabled('NICHEDB_MARKETS')) { + try { + return await fetchNichedbCloses(symbol, startMs, endMs); + } catch (err) { + console.warn(`[backtest] nichedb cannot serve ${symbol} (${(err as Error).message}); falling back`); + } + } try { return await fetchAlpacaCloses(symbol, startMs, endMs); } catch (alpacaErr) { diff --git a/packages/source-crypto-arb/package.json b/packages/source-crypto-arb/package.json index 3f0823c..f056395 100644 --- a/packages/source-crypto-arb/package.json +++ b/packages/source-crypto-arb/package.json @@ -17,6 +17,7 @@ "dependencies": { "@b1dz/adapters-evm": "workspace:*", "@b1dz/core": "workspace:*", + "@b1dz/source-nichedb": "workspace:*", "undici": "^8.0.2", "ws": "^8.20.0" }, diff --git a/packages/source-crypto-arb/src/pair-discovery.test.ts b/packages/source-crypto-arb/src/pair-discovery.test.ts new file mode 100644 index 0000000..a15b78d --- /dev/null +++ b/packages/source-crypto-arb/src/pair-discovery.test.ts @@ -0,0 +1,171 @@ +/** + * Pair discovery behind NICHEDB_CRYPTO: nichedb rows in, the same `BTC-USD` + * list out; live CoinGecko + venue tickers when the switch is off or nichedb + * fails. Every request goes through a fake global fetch; nothing is live. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { CryptoAssetItem, CryptoPairItem } from '@b1dz/source-nichedb'; + +const RUN = '2026-09-11T06:00:00.000Z'; +let nextId = 1; + +function pairRow(venue: string, base: string, volume: number | null, quote = 'USD'): CryptoPairItem { + const sym = venue === 'coinbase' ? `${base}-${quote}` : venue === 'gemini' ? `${base}${quote}`.toLowerCase() : `${base}${quote}`; + return { + id: nextId++, collection: 'crypto', kind: 'pair', external_id: `pair:${venue}:${base}-${quote}`, title: `${base}/${quote}`, + published_at: RUN, updated_at: RUN, tags: ['pair', `venue:${venue}`, `base:${base.toLowerCase()}`, `quote:${quote.toLowerCase()}`, 'stable-quote'], + data: { + venue, base, quote, venueSymbol: sym, status: 'online', price: 100, bid: null, ask: null, high24h: null, low24h: null, change24hPct: 1, + volume24hBase: volume == null ? null : volume / 100, volume24hQuote: volume, vwap24h: null, priceUsd: 100, updatedAt: RUN, + }, + }; +} + +function assetRow(symbol: string, marketCapUsd: number, rank: number): CryptoAssetItem { + return { + id: nextId++, collection: 'crypto', kind: 'asset', external_id: `coingecko:${symbol.toLowerCase()}`, title: symbol, + published_at: RUN, updated_at: RUN, tags: ['asset', `symbol:${symbol.toLowerCase()}`], + data: { + id: symbol.toLowerCase(), symbol: symbol.toLowerCase(), name: symbol, rank, priceUsd: 100, marketCapUsd, fullyDilutedUsd: null, volume24hUsd: null, + change24hPct: null, high24h: null, low24h: null, supply: { circulating: null, total: null, max: null }, ath: null, athDate: null, atl: null, atlDate: null, updatedAt: RUN, + }, + }; +} + +const PAIRS: CryptoPairItem[] = [ + pairRow('kraken', 'BTC', 900e6), pairRow('coinbase', 'BTC', 1200e6), pairRow('binance-us', 'BTC', 80e6), pairRow('gemini', 'BTC', null), + pairRow('kraken', 'ETH', 300e6), pairRow('coinbase', 'ETH', 500e6), + pairRow('kraken', 'ONLYK', 5e6), // one venue + pairRow('kraken', 'THIN', 20_000), pairRow('coinbase', 'THIN', 30_000), // under $50k everywhere + pairRow('kraken', 'GEMNULL', 75_000), pairRow('gemini', 'GEMNULL', null), // Gemini null counts + pairRow('kraken', 'SMALL', 100_000), pairRow('coinbase', 'SMALL', 100_000), // $4M cap + pairRow('kraken', 'BTC', 50e6, 'USDT'), // not a USD book +]; +const ASSETS: CryptoAssetItem[] = [assetRow('BTC', 1.2e12, 1), assetRow('ETH', 3.6e11, 2), assetRow('SMALL', 4e6, 480), assetRow('GEMNULL', 5e7, 200)]; + +type Mode = 'ok' | 'fail' | 'empty'; + +/** Fake fetch for nichedb plus the live fallback hosts. Records every URL. */ +function installFetch(mode: Mode) { + const urls: string[] = []; + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + urls.push(url); + const u = new URL(url); + const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); + if (u.host === 'nichedb.dev') { + if (mode === 'fail') return json({ error: 'boom' }, 503); + if (mode === 'empty') return json({ count: 0, items: [] }); + const kind = u.searchParams.get('kind'); + const after = Number(u.searchParams.get('after') ?? 0); + const rows = (kind === 'pair' ? PAIRS : ASSETS).filter((r) => r.id > after); + return json({ count: rows.length, items: rows }); + } + // The live path (ETH and SOL on Kraken + Binance.US; Coinbase needs keys and is skipped). + // Kraken's XXBTZUSD is left out on purpose: normalizeKrakenBase strips "XX" to "BT", a + // pre-existing quirk of the live path that this file does not test. + if (u.host === 'api.kraken.com') { + return json({ error: [], result: { XETHZUSD: { v: ['0', '100000'], c: ['3000', '0'] }, SOLUSD: { v: ['0', '100000'], c: ['150', '0'] } } }); + } + if (u.host === 'api.binance.us') { + return json([ + { symbol: 'ETHUSD', quoteVolume: '30000000', priceChangePercent: '1' }, + { symbol: 'SOLUSD', quoteVolume: '3000000', priceChangePercent: '1' }, + ]); + } + if (u.host === 'api.coingecko.com') { + return json(u.searchParams.get('page') === '1' ? [{ symbol: 'eth', market_cap: 3.6e11 }, { symbol: 'sol', market_cap: 8e10 }] : []); + } + return json({ error: 'unexpected host' }, 404); + }); + vi.stubGlobal('fetch', fetchMock); + return { urls, nichedb: () => urls.filter((x) => x.includes('nichedb.dev')), live: () => urls.filter((x) => !x.includes('nichedb.dev')) }; +} + +async function load() { + vi.resetModules(); + return import('./pair-discovery.js'); +} + +describe('pair discovery via nichedb', () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-09-11T06:05:00Z')); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.stubEnv('COINBASE_API_KEY_NAME', ''); + vi.stubEnv('MIN_VOLUME_USD', ''); + }); + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('reads the universe from nichedb only, with the same filters, when NICHEDB_CRYPTO=1', async () => { + vi.stubEnv('NICHEDB_CRYPTO', '1'); + const f = installFetch('ok'); + const { getActivePairs, discoverPairsFromNichedb } = await load(); + expect(await getActivePairs()).toEqual(['BTC-USD', 'ETH-USD', 'GEMNULL-USD']); + expect(f.live()).toEqual([]); + expect(f.nichedb()).toHaveLength(2); // one page of pairs, one page of assets + const pairUrl = new URL(f.nichedb().find((u) => u.includes('kind=pair'))!); + expect(pairUrl.searchParams.get('collection')).toBe('crypto'); + expect(pairUrl.searchParams.get('tags')).toBe('stable-quote'); + expect(pairUrl.searchParams.get('limit')).toBe('200'); + expect(f.nichedb().some((u) => u.includes('kind=asset'))).toBe(true); + // venue symbols travel with each pair for anyone who needs the venue's own name + const btc = (await discoverPairsFromNichedb()).find((p) => p.pair === 'BTC-USD')!; + expect(btc.venues['kraken']!.venueSymbol).toBe('BTCUSD'); + expect(btc.venues['gemini']!.volume24hQuote).toBeNull(); + expect(console.error).not.toHaveBeenCalled(); + }); + + it('caches the answer for five minutes', async () => { + vi.stubEnv('NICHEDB_CRYPTO', '1'); + const f = installFetch('ok'); + const { getActivePairs } = await load(); + await getActivePairs(); + await getActivePairs(); + vi.advanceTimersByTime(4 * 60_000); + await getActivePairs(); + expect(f.nichedb()).toHaveLength(2); + vi.advanceTimersByTime(61_000); + await getActivePairs(); + expect(f.nichedb()).toHaveLength(4); + }); + + it('falls back to CoinGecko + venue tickers on a nichedb failure and logs once', async () => { + vi.stubEnv('NICHEDB_CRYPTO', '1'); + const f = installFetch('fail'); + const { getActivePairs } = await load(); + expect(await getActivePairs()).toEqual(['ETH-USD', 'SOL-USD']); + expect(f.nichedb().length).toBeGreaterThan(0); + expect(f.live().some((u) => u.includes('api.kraken.com'))).toBe(true); + expect(f.live().some((u) => u.includes('api.coingecko.com'))).toBe(true); + expect(console.error).toHaveBeenCalledTimes(1); + expect((console.error as unknown as { mock: { calls: string[][] } }).mock.calls[0]![0]).toMatch(/nichedb unavailable .* falling back/); + vi.advanceTimersByTime(6 * 60_000); + await getActivePairs(); + expect(console.error).toHaveBeenCalledTimes(1); + }); + + it('falls back on an empty nichedb answer', async () => { + vi.stubEnv('NICHEDB_CRYPTO', '1'); + const f = installFetch('empty'); + const { getActivePairs } = await load(); + expect(await getActivePairs()).toEqual(['ETH-USD', 'SOL-USD']); + expect(f.live().some((u) => u.includes('api.kraken.com'))).toBe(true); + expect(console.error).toHaveBeenCalledTimes(1); + }); + + it('never asks nichedb when the switch is off', async () => { + vi.stubEnv('NICHEDB_CRYPTO', ''); + const f = installFetch('ok'); + const { getActivePairs } = await load(); + expect(await getActivePairs()).toEqual(['ETH-USD', 'SOL-USD']); + expect(f.nichedb()).toEqual([]); + expect(f.live().some((u) => u.includes('api.kraken.com'))).toBe(true); + }); +}); diff --git a/packages/source-crypto-arb/src/pair-discovery.ts b/packages/source-crypto-arb/src/pair-discovery.ts index 444b142..c9c8158 100644 --- a/packages/source-crypto-arb/src/pair-discovery.ts +++ b/packages/source-crypto-arb/src/pair-discovery.ts @@ -7,9 +7,24 @@ * 4. Return every pair that clears the liquidity + market-cap filters * * Refreshes every 5 minutes. + * + * With NICHEDB_CRYPTO=1 the universe (per-venue tickers and CoinGecko market + * caps) is read from nichedb.dev in ~10 requests instead of CoinGecko plus the + * four venues' ticker lists; the filters are the same. Any nichedb failure or + * an empty answer falls back to the live path below, logged once per outage. */ import { createSign, randomBytes } from 'node:crypto'; +import { + createNichedbClient, + fetchCryptoAssets, + fetchCryptoPairs, + nichedbEnabled, + selectCryptoPairs, + type CryptoAssetItem, + type DiscoveredPair, + type FetchLike, +} from '@b1dz/source-nichedb'; import { getCoinbasePem } from './feeds/coinbase-pem.js'; import { fetchJson } from './feeds/http.js'; @@ -135,9 +150,75 @@ async function getBinanceVolumes(): Promise= MIN_VOLUME_USD` per venue + * (a null volume, which is every Gemini book, passes, exactly as the old + * Gemini sentinel did), on at least two venues, market cap >= $10M when known. + * + * Throws on a nichedb failure or an empty pair set so the caller can fall + * back; a failed asset read only drops the market-cap filter, as a CoinGecko + * failure did. + */ +export async function discoverPairsFromNichedb(fetchImpl?: FetchLike): Promise { + const client = createNichedbClient({ fetch: fetchImpl }); + const [pairsResult, assetsResult] = await Promise.allSettled([fetchCryptoPairs(client), fetchCryptoAssets(client)]); + // No pairs means no universe: throw so the caller falls back (an asset error alongside is the same outage). + if (pairsResult.status === 'rejected') throw pairsResult.reason; + const pairRows = pairsResult.value; + if (pairRows.length === 0) throw new Error('nichedb returned no pairs'); + let assetRows: CryptoAssetItem[] = []; + if (assetsResult.status === 'fulfilled') assetRows = assetsResult.value; + else console.error(`[discovery] nichedb assets error (skipping mcap filter): ${(assetsResult.reason as Error).message}`); + + const sel = selectCryptoPairs(pairRows, assetRows, { + minVolumeUsd: minVolumeUsd(), + minMarketCapUsd: MIN_MARKET_CAP_USD, + minVenues: MIN_EXCHANGES, + excludedBases: EXCLUDED, + quote: 'USD', + }); + if (sel.pairs.length === 0) throw new Error(`nichedb: no pair cleared the filters (${pairRows.length} rows)`); + + console.log( + `[discovery] nichedb: ${sel.pairs.length} pairs from ${pairRows.length} pair rows + ${assetRows.length} assets in ${client.requestCount} requests ` + + `(${sel.filteredByVenues} filtered by <${MIN_EXCHANGES} exchanges, ${sel.filteredByMarketCap} filtered by <$${MIN_MARKET_CAP_USD / 1e6}M mcap, min vol $${(minVolumeUsd() / 1e6).toFixed(2)}M)`, + ); + for (const p of sel.pairs.slice(0, 12)) { + const chg = p.change24hPct >= 0 ? `+${p.change24hPct.toFixed(1)}%` : `${p.change24hPct.toFixed(1)}%`; + const mcapStr = p.marketCapUsd > 0 ? `mcap=$${(p.marketCapUsd / 1e9).toFixed(1)}B` : 'mcap=?'; + console.log(` ${p.pair.padEnd(12)} vol=$${(p.totalVolumeUsd / 1e6).toFixed(1)}M 24h=${chg} ${mcapStr} venues=${Object.keys(p.venues).join(',')}`); + } + if (sel.pairs.length > 12) console.log(` ... +${sel.pairs.length - 12} more`); + return sel.pairs; +} + // ─── Discovery ──────────────────────────────────────────────── async function discoverPairs(): Promise { + if (nichedbEnabled('NICHEDB_CRYPTO')) { + try { + const pairs = await discoverPairsFromNichedb(); + nichedbFallbackWarned = false; + return pairs.map((p) => p.pair); + } catch (e) { + if (!nichedbFallbackWarned) { + nichedbFallbackWarned = true; + console.error(`[discovery] nichedb unavailable (${(e as Error).message}); falling back to CoinGecko + venue tickers`); + } + } + } + return discoverPairsFromVenues(); +} + +/** The live path: CoinGecko (2 pages x 250) plus Kraken, Coinbase and Binance.US tickers. */ +async function discoverPairsFromVenues(): Promise { const [krakenVols, coinbaseData, binanceData] = await Promise.all([ getKrakenVolumes(), getCoinbaseVolumes(), diff --git a/packages/source-nichedb/package.json b/packages/source-nichedb/package.json new file mode 100644 index 0000000..6562c8a --- /dev/null +++ b/packages/source-nichedb/package.json @@ -0,0 +1,22 @@ +{ + "name": "@b1dz/source-nichedb", + "version": "0.3.10", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "lint": "eslint src", + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "latest", + "typescript": "latest", + "vitest": "latest" + } +} diff --git a/packages/source-nichedb/src/client.test.ts b/packages/source-nichedb/src/client.test.ts new file mode 100644 index 0000000..fa679c7 --- /dev/null +++ b/packages/source-nichedb/src/client.test.ts @@ -0,0 +1,131 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createNichedbClient, NichedbError, type FetchLike } from './client.js'; +import { nichedbBaseUrl, nichedbEnabled } from './env.js'; + +type Call = { url: string; init?: Parameters[1] }; + +function fakeFetch(handler: (url: URL, call: Call) => { status?: number; body: unknown } | Promise<{ status?: number; body: unknown }>) { + const calls: Call[] = []; + const fetch: FetchLike = async (url, init) => { + const call = { url, init }; + calls.push(call); + const { status = 200, body } = await handler(new URL(url), call); + return { ok: status >= 200 && status < 300, status, statusText: status === 200 ? 'OK' : 'ERR', json: async () => body }; + }; + return { fetch, calls }; +} + +const row = (id: number, extra: Record = {}) => ({ + id, collection: 'crypto', kind: 'pair', external_id: `x:${id}`, title: `#${id}`, published_at: null, updated_at: '2026-09-11T00:00:00Z', tags: [], data: {}, ...extra, +}); + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('createNichedbClient', () => { + it('builds the items URL from the query with a comma tag list and a capped limit', () => { + const client = createNichedbClient({ baseUrl: 'https://example.test/', fetch: fakeFetch(() => ({ body: { items: [] } })).fetch }); + const url = new URL(client.itemsUrl({ collection: 'crypto', kind: 'pair', tags: ['stable-quote', 'venue:kraken'], limit: 2500, since: '2026-09-11T00:00:00Z', after: 12 })); + expect(url.origin + url.pathname).toBe('https://example.test/api/v1/items'); + expect(url.searchParams.get('collection')).toBe('crypto'); + expect(url.searchParams.get('kind')).toBe('pair'); + expect(url.searchParams.get('tags')).toBe('stable-quote,venue:kraken'); + expect(url.searchParams.get('limit')).toBe('200'); + expect(url.searchParams.get('since')).toBe('2026-09-11T00:00:00Z'); + expect(url.searchParams.get('after')).toBe('12'); + }); + + it('defaults the base to NICHEDB_URL, else nichedb.dev', () => { + expect(nichedbBaseUrl({})).toBe('https://nichedb.dev'); + expect(nichedbBaseUrl({ NICHEDB_URL: 'http://localhost:3000/' })).toBe('http://localhost:3000'); + vi.stubEnv('NICHEDB_URL', 'https://mirror.test'); + expect(createNichedbClient({ fetch: fakeFetch(() => ({ body: { items: [] } })).fetch }).baseUrl).toBe('https://mirror.test'); + }); + + it('walks after= keyset pages of 200 in id order and stops on a short page', async () => { + const total = 450; + const all = Array.from({ length: total }, (_, i) => row(i + 1)); + const { fetch, calls } = fakeFetch((url) => { + const after = Number(url.searchParams.get('after') ?? 0); + const limit = Number(url.searchParams.get('limit')); + expect(url.searchParams.get('sort')).toBe('id'); + expect(url.searchParams.get('order')).toBe('asc'); + const page = all.filter((r) => r.id > after).slice(0, limit); + return { body: { count: page.length, items: page } }; + }); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + const items = await client.walk({ collection: 'crypto', kind: 'pair', tags: ['stable-quote'] }); + expect(items).toHaveLength(total); + expect(items.map((i) => i.id)).toEqual(all.map((r) => r.id)); + expect(calls).toHaveLength(3); // 200 + 200 + 50 + expect(new URL(calls[1]!.url).searchParams.get('after')).toBe('200'); + expect(new URL(calls[2]!.url).searchParams.get('after')).toBe('400'); + expect(client.requestCount).toBe(3); + }); + + it('makes one extra request when the last page is exactly full', async () => { + const all = Array.from({ length: 400 }, (_, i) => row(i + 1)); + const { fetch, calls } = fakeFetch((url) => { + const after = Number(url.searchParams.get('after') ?? 0); + return { body: { items: all.filter((r) => r.id > after).slice(0, 200) } }; + }); + const items = await createNichedbClient({ baseUrl: 'https://n.test', fetch }).walk({ collection: 'crypto' }); + expect(items).toHaveLength(400); + expect(calls).toHaveLength(3); + }); + + it('honours maxItems and maxPages', async () => { + const all = Array.from({ length: 1000 }, (_, i) => row(i + 1)); + const { fetch, calls } = fakeFetch((url) => { + const after = Number(url.searchParams.get('after') ?? 0); + return { body: { items: all.filter((r) => r.id > after).slice(0, 200) } }; + }); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + expect(await client.walk({ collection: 'crypto', kind: 'asset' }, { maxItems: 500 })).toHaveLength(500); + expect(calls).toHaveLength(3); + calls.length = 0; + expect(await client.walk({ collection: 'crypto', kind: 'asset' }, { maxPages: 2 })).toHaveLength(400); + expect(calls).toHaveLength(2); + }); + + it('passes since= through on every page', async () => { + const { fetch, calls } = fakeFetch(() => ({ body: { items: [] } })); + await createNichedbClient({ baseUrl: 'https://n.test', fetch }).walk({ collection: 'crypto', since: new Date('2026-09-11T05:00:00Z') }); + expect(new URL(calls[0]!.url).searchParams.get('since')).toBe('2026-09-11T05:00:00.000Z'); + }); + + it('throws a NichedbError on a non-2xx answer, an error body, or a malformed body', async () => { + const c1 = createNichedbClient({ baseUrl: 'https://n.test', fetch: fakeFetch(() => ({ status: 503, body: {} })).fetch }); + await expect(c1.items({ collection: 'crypto' })).rejects.toBeInstanceOf(NichedbError); + const c2 = createNichedbClient({ baseUrl: 'https://n.test', fetch: fakeFetch(() => ({ body: { error: 'No collection named crypto' } })).fetch }); + await expect(c2.items({ collection: 'crypto' })).rejects.toThrow(/No collection named crypto/); + const c3 = createNichedbClient({ baseUrl: 'https://n.test', fetch: fakeFetch(() => ({ body: { nope: true } })).fetch }); + await expect(c3.items({ collection: 'crypto' })).rejects.toThrow(/malformed/); + }); + + it('aborts a request that exceeds the timeout', async () => { + vi.useFakeTimers(); + try { + const fetch: FetchLike = (_url, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(Object.assign(new Error('aborted'), { name: 'AbortError' }))); + }); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch, timeoutMs: 50 }); + const p = client.items({ collection: 'crypto' }); + const assertion = expect(p).rejects.toThrow(/timeout after 50ms/); + await vi.advanceTimersByTimeAsync(60); + await assertion; + } finally { + vi.useRealTimers(); + } + }); +}); + +describe('nichedbEnabled', () => { + it('is on for 1/true/yes/on and off otherwise', () => { + for (const v of ['1', 'true', 'TRUE', 'yes', ' on ']) expect(nichedbEnabled('NICHEDB_CRYPTO', { NICHEDB_CRYPTO: v })).toBe(true); + for (const v of ['0', 'false', '', 'no']) expect(nichedbEnabled('NICHEDB_CRYPTO', { NICHEDB_CRYPTO: v })).toBe(false); + expect(nichedbEnabled('NICHEDB_MARKETS', {})).toBe(false); + }); +}); diff --git a/packages/source-nichedb/src/client.ts b/packages/source-nichedb/src/client.ts new file mode 100644 index 0000000..d9f7106 --- /dev/null +++ b/packages/source-nichedb/src/client.ts @@ -0,0 +1,174 @@ +/** + * Tiny client for nichedb.dev's public, keyless items API. + * + * GET /api/v1/items?collection=&kind=&tags=a,b&since=&sort=&order=&limit=&after= + * + * The API caps `limit` at 200 and answers `{ count, items }` with no cursor, + * so a full read walks keyset pages: `sort=id&order=asc&after=` until + * a page comes back short. About 600 requests an hour per IP are tolerated; + * callers are expected to cache. + */ + +import { nichedbBaseUrl } from './env.js'; + +/** One row of `/api/v1/items`; `data` is the adapter's full record. */ +export interface NichedbItem { + id: number; + collection: string; + source?: string; + adapter?: string; + kind: string; + external_id: string; + title: string; + summary?: string | null; + url?: string | null; + image_url?: string | null; + published_at: string | null; + updated_at: string; + tags: string[]; + data: T; +} + +export interface ItemsQuery { + collection: string; + kind?: string; + /** All named tags must be on the item (comma list on the wire). */ + tags?: string[]; + /** Only rows whose `updated_at` is at or after this stamp. */ + since?: string | Date; + sort?: 'id' | 'published' | 'updated'; + order?: 'asc' | 'desc'; + /** Page size, capped by the API at 200. */ + limit?: number; + /** Keyset cursor: rows with `id` greater than this. */ + after?: number; +} + +/** The slice of fetch the client needs, so tests and proxies can hand in their own. */ +export type FetchLike = ( + input: string, + init?: { signal?: AbortSignal; headers?: Record }, +) => Promise<{ ok: boolean; status: number; statusText: string; json(): Promise }>; + +export interface NichedbClientOptions { + /** Base URL; defaults to NICHEDB_URL or https://nichedb.dev. */ + baseUrl?: string; + /** Injectable fetch (tests, proxies). Defaults to globalThis.fetch. */ + fetch?: FetchLike; + /** Per-request timeout. */ + timeoutMs?: number; + /** Sent as User-Agent so nichedb can see who is reading. */ + userAgent?: string; +} + +export interface WalkOptions { + /** Hard stop on the number of pages, so a bug upstream cannot loop forever. */ + maxPages?: number; + /** Stop once this many rows have been collected. */ + maxItems?: number; +} + +export const NICHEDB_PAGE_LIMIT = 200; +const DEFAULT_TIMEOUT_MS = 10_000; +const DEFAULT_MAX_PAGES = 25; + +export class NichedbError extends Error { + constructor( + message: string, + readonly status?: number, + readonly url?: string, + ) { + super(message); + this.name = 'NichedbError'; + } +} + +export interface NichedbClient { + readonly baseUrl: string; + /** Build the items URL for a query (exposed for logging and tests). */ + itemsUrl(query: ItemsQuery): string; + /** One page of items. */ + items(query: ItemsQuery): Promise[]>; + /** Every item matching the query, walking `after=` keyset pages in id order. */ + walk(query: ItemsQuery, opts?: WalkOptions): Promise[]>; + /** Requests made so far by this client (for budget checks and tests). */ + readonly requestCount: number; +} + +export function createNichedbClient(opts: NichedbClientOptions = {}): NichedbClient { + const baseUrl = (opts.baseUrl ?? nichedbBaseUrl()).replace(/\/+$/, ''); + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const userAgent = opts.userAgent ?? 'b1dz (+https://b1dz.com)'; + let requestCount = 0; + + const resolveFetch = (): FetchLike => { + const f = opts.fetch ?? (globalThis.fetch as unknown as FetchLike | undefined); + if (!f) throw new NichedbError('no fetch available'); + return f; + }; + + function itemsUrl(query: ItemsQuery): string { + const params = new URLSearchParams(); + params.set('collection', query.collection); + if (query.kind) params.set('kind', query.kind); + if (query.tags && query.tags.length > 0) params.set('tags', query.tags.join(',')); + if (query.since != null) { + params.set('since', query.since instanceof Date ? query.since.toISOString() : query.since); + } + if (query.sort) params.set('sort', query.sort); + if (query.order) params.set('order', query.order); + if (query.limit != null) params.set('limit', String(Math.min(Math.max(1, query.limit), NICHEDB_PAGE_LIMIT))); + if (query.after != null) params.set('after', String(query.after)); + return `${baseUrl}/api/v1/items?${params.toString()}`; + } + + async function items(query: ItemsQuery): Promise[]> { + const url = itemsUrl(query); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + requestCount++; + try { + const res = await resolveFetch()(url, { + signal: controller.signal, + headers: { Accept: 'application/json', 'User-Agent': userAgent }, + }); + if (!res.ok) throw new NichedbError(`nichedb ${res.status} ${res.statusText}`, res.status, url); + const body = (await res.json()) as { count?: number; items?: unknown; error?: string } | null; + if (body && typeof body.error === 'string') throw new NichedbError(`nichedb: ${body.error}`, res.status, url); + if (!body || !Array.isArray(body.items)) { + throw new NichedbError('nichedb: malformed answer (no items array)', res.status, url); + } + return body.items as NichedbItem[]; + } catch (e) { + if ((e as Error).name === 'AbortError') throw new NichedbError(`nichedb: timeout after ${timeoutMs}ms`, undefined, url); + throw e; + } finally { + clearTimeout(timer); + } + } + + async function walk(query: ItemsQuery, walkOpts: WalkOptions = {}): Promise[]> { + const maxPages = walkOpts.maxPages ?? DEFAULT_MAX_PAGES; + const maxItems = walkOpts.maxItems ?? Number.POSITIVE_INFINITY; + const pageSize = Math.min(query.limit ?? NICHEDB_PAGE_LIMIT, NICHEDB_PAGE_LIMIT); + const out: NichedbItem[] = []; + let after = query.after; + for (let page = 0; page < maxPages; page++) { + const rows = await items({ ...query, sort: 'id', order: 'asc', limit: pageSize, after }); + out.push(...rows); + if (rows.length < pageSize || out.length >= maxItems) break; + after = rows[rows.length - 1]!.id; + } + return out.length > maxItems ? out.slice(0, maxItems) : out; + } + + return { + baseUrl, + itemsUrl, + items, + walk, + get requestCount() { + return requestCount; + }, + }; +} diff --git a/packages/source-nichedb/src/crypto.test.ts b/packages/source-nichedb/src/crypto.test.ts new file mode 100644 index 0000000..66c28fc --- /dev/null +++ b/packages/source-nichedb/src/crypto.test.ts @@ -0,0 +1,254 @@ +/** + * Fixture built from niche-db docs/crypto.md: Kraken, Coinbase, Binance.US and + * Gemini rows for a handful of bases, then the grouping and the filters b1dz + * applied to CoinGecko plus the venue ticker lists. + */ +import { describe, expect, it } from 'vitest'; +import { fetchCryptoAssets, fetchCryptoPairs, marketCapsBySymbol, selectCryptoPairs, type CryptoAssetItem, type CryptoPairItem } from './crypto.js'; +import { createNichedbClient, type FetchLike } from './client.js'; + +let nextId = 1; +const RUN = '2026-09-11T06:00:00.000Z'; + +const VENUE_NAME: Record = { kraken: 'Kraken', coinbase: 'Coinbase', 'binance-us': 'Binance.US', gemini: 'Gemini' }; + +function venueSymbol(venue: string, base: string, quote: string): string { + switch (venue) { + case 'kraken': return base === 'BTC' ? `XXBTZ${quote}` : `${base}${quote}`; + case 'coinbase': return `${base}-${quote}`; + case 'binance-us': return `${base}${quote}`; + default: return `${base}${quote}`.toLowerCase(); + } +} + +export function pairRow( + venue: string, + base: string, + opts: { quote?: string; price?: number; volume?: number | null; status?: string; change?: number | null } = {}, +): CryptoPairItem { + const quote = opts.quote ?? 'USD'; + const price = opts.price ?? 100; + const volume = opts.volume === undefined ? (venue === 'gemini' ? null : 1_000_000) : opts.volume; + const stable = ['USD', 'USDT', 'USDC'].includes(quote); + const isGemini = venue === 'gemini'; + const isCoinbase = venue === 'coinbase'; + return { + id: nextId++, + collection: 'crypto', + source: 'crypto-pairs', + kind: 'pair', + external_id: `pair:${venue}:${base}-${quote}`, + title: `${base}/${quote} on ${VENUE_NAME[venue]}`, + published_at: RUN, + updated_at: RUN, + tags: ['pair', `venue:${venue}`, `base:${base.toLowerCase()}`, `quote:${quote.toLowerCase()}`, ...(stable ? ['stable-quote'] : [])], + data: { + venue, + venueName: VENUE_NAME[venue], + base, + quote, + venueSymbol: venueSymbol(venue, base, quote), + status: opts.status ?? 'online', + price, + bid: isGemini || isCoinbase ? null : price * 0.999, + ask: isGemini || isCoinbase ? null : price * 1.001, + high24h: isGemini ? null : price * 1.05, + low24h: isGemini ? null : price * 0.95, + open24h: isGemini ? null : price, + change24hPct: opts.change === undefined ? 1.2 : opts.change, + volume24hBase: volume == null ? null : volume / price, + volume24hQuote: volume, + vwap24h: isGemini || isCoinbase ? null : price, + priceUsd: stable ? price : null, + updatedAt: RUN, + }, + }; +} + +export function assetRow(symbol: string, marketCapUsd: number | null, rank: number | null = 1): CryptoAssetItem { + return { + id: nextId++, + collection: 'crypto', + source: 'coingecko-assets', + kind: 'asset', + external_id: `coingecko:${symbol.toLowerCase()}`, + title: `${symbol} (${symbol})`, + published_at: RUN, + updated_at: RUN, + tags: ['asset', `symbol:${symbol.toLowerCase()}`, ...(rank != null && rank <= 100 ? ['rank:top100'] : [])], + data: { + id: symbol.toLowerCase(), symbol: symbol.toLowerCase(), name: symbol, rank, priceUsd: 100, marketCapUsd, fullyDilutedUsd: marketCapUsd, + volume24hUsd: 1e9, change24hPct: 1, high24h: 105, low24h: 95, supply: { circulating: 1, total: 1, max: null }, + ath: 1, athDate: null, atl: 1, atlDate: null, updatedAt: RUN, + }, + }; +} + +const RULES = { minVolumeUsd: 50_000, minMarketCapUsd: 10_000_000, minVenues: 2, excludedBases: ['USDT', 'USDC', 'DAI', 'EUR'] }; + +/** BTC and ETH everywhere; ONLYK on Kraken alone; THIN under the volume floor on + * every venue but one; GEMNULL on Kraken (known) + Gemini (null volume); SMALL + * on two venues with a tiny market cap; USDT quoted books to be ignored. */ +function fixture() { + const pairs: CryptoPairItem[] = [ + pairRow('kraken', 'BTC', { price: 60_000, volume: 900_000_000, change: 0.9 }), + pairRow('coinbase', 'BTC', { price: 60_000, volume: 1_200_000_000, change: -1.3 }), + pairRow('binance-us', 'BTC', { price: 60_000, volume: 80_000_000, change: -1.4 }), + pairRow('gemini', 'BTC', { price: 60_000, change: -1.2 }), + pairRow('kraken', 'BTC', { quote: 'USDT', price: 60_000, volume: 50_000_000 }), + pairRow('kraken', 'ETH', { price: 3_000, volume: 300_000_000 }), + pairRow('coinbase', 'ETH', { price: 3_000, volume: 500_000_000 }), + pairRow('binance-us', 'ETH', { price: 3_000, volume: 30_000_000 }), + pairRow('gemini', 'ETH', { price: 3_000 }), + pairRow('kraken', 'ONLYK', { volume: 5_000_000 }), + pairRow('kraken', 'THIN', { volume: 20_000 }), + pairRow('coinbase', 'THIN', { volume: 49_999 }), + pairRow('binance-us', 'THIN', { volume: 400_000 }), + pairRow('kraken', 'GEMNULL', { volume: 75_000 }), + pairRow('gemini', 'GEMNULL'), + pairRow('kraken', 'SMALL', { volume: 100_000 }), + pairRow('coinbase', 'SMALL', { volume: 100_000 }), + pairRow('kraken', 'NOCAP', { volume: 100_000 }), + pairRow('coinbase', 'NOCAP', { volume: 100_000 }), + pairRow('kraken', 'USDT', { quote: 'USD', volume: 900_000_000 }), + pairRow('coinbase', 'USDT', { quote: 'USD', volume: 900_000_000 }), + pairRow('kraken', 'OFFLINE', { volume: 1_000_000, status: 'cancel_only' }), + pairRow('coinbase', 'OFFLINE', { volume: 1_000_000 }), + pairRow('kraken', 'BTC', { quote: 'EUR', price: 55_000, volume: 100_000_000 }), + ]; + const assets: CryptoAssetItem[] = [ + assetRow('BTC', 1.2e12, 1), + assetRow('ETH', 3.6e11, 2), + assetRow('USDT', 1.2e11, 3), + assetRow('SMALL', 4_000_000, 480), + assetRow('GEMNULL', 50_000_000, 200), + assetRow('THIN', 90_000_000, 150), + assetRow('ONLYK', 200_000_000, 90), + assetRow('OFFLINE', 200_000_000, 91), + // a second coin sharing ETH's ticker, ranked far lower: must not clobber ETH's cap + assetRow('eth', 1_000_000, 499), + ]; + return { pairs, assets }; +} + +describe('selectCryptoPairs', () => { + it('groups by base across venues and keeps the venue symbols', () => { + const { pairs, assets } = fixture(); + const { pairs: out } = selectCryptoPairs(pairs, assets, RULES); + const btc = out.find((p) => p.pair === 'BTC-USD')!; + expect(btc).toBeDefined(); + expect(Object.keys(btc.venues).sort()).toEqual(['binance-us', 'coinbase', 'gemini', 'kraken']); + expect(btc.venues['kraken']!.venueSymbol).toBe('XXBTZUSD'); + expect(btc.venues['coinbase']!.venueSymbol).toBe('BTC-USD'); + expect(btc.venues['binance-us']!.venueSymbol).toBe('BTCUSD'); + expect(btc.venues['gemini']!.venueSymbol).toBe('btcusd'); + expect(btc.marketCapUsd).toBe(1.2e12); + // Gemini's null volume adds nothing to the total; the USDT and EUR books are not counted. + expect(btc.totalVolumeUsd).toBe(900_000_000 + 1_200_000_000 + 80_000_000); + // Coinbase's change wins for the log line, as it did before. + expect(btc.change24hPct).toBe(-1.3); + }); + + it('applies the two-venue, $50k per venue and $10M market-cap rules', () => { + const { pairs, assets } = fixture(); + const sel = selectCryptoPairs(pairs, assets, RULES); + const names = sel.pairs.map((p) => p.pair); + expect(names).toContain('BTC-USD'); + expect(names).toContain('ETH-USD'); + expect(names).not.toContain('ONLYK-USD'); // one venue + expect(names).not.toContain('THIN-USD'); // clears $50k on one venue only + expect(names).not.toContain('SMALL-USD'); // $4M cap + expect(names).not.toContain('USDT-USD'); // excluded base + expect(names).not.toContain('OFFLINE-USD'); // cancel_only on Kraken leaves one venue + expect(names).toContain('NOCAP-USD'); // unknown cap passes, as before + expect(sel.filteredByVenues).toBe(3); // ONLYK, THIN, OFFLINE + expect(sel.filteredByMarketCap).toBe(1); // SMALL + expect(sel.marketCapCount).toBe(8); + }); + + it('treats a null (Gemini) volume as passing, as the old sentinel did', () => { + const { pairs, assets } = fixture(); + const { pairs: out } = selectCryptoPairs(pairs, assets, RULES); + const g = out.find((p) => p.pair === 'GEMNULL-USD')!; + expect(g).toBeDefined(); + expect(Object.keys(g.venues).sort()).toEqual(['gemini', 'kraken']); + expect(g.venues['gemini']!.volume24hQuote).toBeNull(); + expect(g.totalVolumeUsd).toBe(75_000); + // Two Gemini-null venues alone would also pass: nothing known says otherwise. + const twoNull = selectCryptoPairs([pairRow('gemini', 'X'), pairRow('gemini', 'X', { quote: 'USD' })], [], RULES); + expect(twoNull.pairs).toHaveLength(0); // same venue twice is still one venue + }); + + it('sorts by total known volume descending', () => { + const { pairs, assets } = fixture(); + const names = selectCryptoPairs(pairs, assets, RULES).pairs.map((p) => p.pair); + expect(names.slice(0, 2)).toEqual(['BTC-USD', 'ETH-USD']); + }); + + it('honours the MIN_VOLUME_USD style floor and the venue list', () => { + const { pairs, assets } = fixture(); + const strict = selectCryptoPairs(pairs, assets, { ...RULES, minVolumeUsd: 100_000_000 }); + expect(strict.pairs.map((p) => p.pair)).toEqual(['BTC-USD', 'ETH-USD']); + // Without Gemini, GEMNULL is a one-venue coin. + const three = selectCryptoPairs(pairs, assets, { ...RULES, venues: ['kraken', 'coinbase', 'binance-us'] }); + expect(three.pairs.map((p) => p.pair)).not.toContain('GEMNULL-USD'); + }); + + it('passes every base through the cap filter when there are no assets at all', () => { + const { pairs } = fixture(); + const sel = selectCryptoPairs(pairs, [], RULES); + expect(sel.pairs.map((p) => p.pair)).toContain('SMALL-USD'); + expect(sel.marketCapCount).toBe(0); + }); + + it('keys market caps by upper-cased symbol and lets the better-ranked duplicate win', () => { + const caps = marketCapsBySymbol([assetRow('eth', 1_000_000, 499), assetRow('ETH', 3.6e11, 2)]); + expect(caps.get('ETH')).toBe(3.6e11); + }); + + it('ignores rows with a missing or malformed data record', () => { + const bad = { ...pairRow('kraken', 'BTC'), data: null } as unknown as CryptoPairItem; + const sel = selectCryptoPairs([bad, pairRow('coinbase', 'BTC')], [], RULES); + expect(sel.pairs).toHaveLength(0); + }); +}); + +describe('fetchCryptoPairs / fetchCryptoAssets', () => { + function serve(rows: { id: number; kind: string; tags: string[] }[]) { + const urls: URL[] = []; + const fetch: FetchLike = async (url) => { + const u = new URL(url); + urls.push(u); + const kind = u.searchParams.get('kind'); + const tags = (u.searchParams.get('tags') ?? '').split(',').filter(Boolean); + const after = Number(u.searchParams.get('after') ?? 0); + const limit = Number(u.searchParams.get('limit')); + const page = rows.filter((r) => r.kind === kind && tags.every((t) => r.tags.includes(t)) && r.id > after).slice(0, limit); + return { ok: true, status: 200, statusText: 'OK', json: async () => ({ count: page.length, items: page }) }; + }; + return { fetch, urls }; + } + + it('reads kind=pair&tags=stable-quote in 200-row pages', async () => { + const rows = Array.from({ length: 1_400 }, (_, i) => pairRow(['kraken', 'coinbase', 'binance-us', 'gemini'][i % 4]!, `C${i}`)); + const { fetch, urls } = serve(rows); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + const out = await fetchCryptoPairs(client); + expect(out).toHaveLength(1_400); + expect(urls).toHaveLength(8); // 7 full pages + the short (empty) one + expect(urls[0]!.searchParams.get('collection')).toBe('crypto'); + expect(urls[0]!.searchParams.get('kind')).toBe('pair'); + expect(urls[0]!.searchParams.get('tags')).toBe('stable-quote'); + expect(urls[0]!.searchParams.get('limit')).toBe('200'); + }); + + it('reads kind=asset up to 500 rows (three pages)', async () => { + const rows = Array.from({ length: 900 }, (_, i) => assetRow(`A${i}`, 1e9, i + 1)); + const { fetch, urls } = serve(rows); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + const out = await fetchCryptoAssets(client); + expect(out).toHaveLength(500); + expect(urls).toHaveLength(3); + expect(urls[0]!.searchParams.get('kind')).toBe('asset'); + }); +}); diff --git a/packages/source-nichedb/src/crypto.ts b/packages/source-nichedb/src/crypto.ts new file mode 100644 index 0000000..1908854 --- /dev/null +++ b/packages/source-nichedb/src/crypto.ts @@ -0,0 +1,233 @@ +/** + * The `crypto` collection: CoinGecko assets and per-venue spot pairs, per + * niche-db docs/crypto.md. This module reads the rows and turns them into the + * per-base venue grouping b1dz's pair discovery filters on. + */ + +import type { NichedbClient, NichedbItem } from './client.js'; + +export type CryptoVenue = 'kraken' | 'coinbase' | 'binance-us' | 'gemini'; + +/** `data` of a kind=asset row (source coingecko-assets). */ +export interface CryptoAssetData { + id: string; + symbol: string; + name: string; + rank: number | null; + priceUsd: number | null; + marketCapUsd: number | null; + fullyDilutedUsd: number | null; + volume24hUsd: number | null; + change24hPct: number | null; + high24h: number | null; + low24h: number | null; + supply: { circulating: number | null; total: number | null; max: number | null }; + ath: number | null; + athDate: string | null; + atl: number | null; + atlDate: string | null; + updatedAt: string; +} + +/** `data` of a kind=pair row (source crypto-pairs). */ +export interface CryptoPairData { + venue: CryptoVenue | string; + venueName?: string; + base: string; + quote: string; + venueSymbol: string; + status: string; + price: number | null; + bid: number | null; + ask: number | null; + high24h: number | null; + low24h: number | null; + open24h?: number | null; + change24hPct: number | null; + volume24hBase: number | null; + /** Dollars when the quote is a dollar stable. Null on Gemini (unknown, not zero). */ + volume24hQuote: number | null; + vwap24h: number | null; + priceUsd: number | null; + updatedAt: string; +} + +export type CryptoAssetItem = NichedbItem; +export type CryptoPairItem = NichedbItem; + +export const CRYPTO_COLLECTION = 'crypto'; + +/** Every stable-quoted spot pair across the four venues (about 1,400 rows, 7-8 pages). */ +export function fetchCryptoPairs(client: NichedbClient, opts: { since?: string | Date } = {}): Promise { + return client.walk({ + collection: CRYPTO_COLLECTION, + kind: 'pair', + tags: ['stable-quote'], + since: opts.since, + limit: 200, + }); +} + +/** The top assets by market cap (CoinGecko's two pages of 250; 3 pages of 200 here). */ +export function fetchCryptoAssets(client: NichedbClient, opts: { max?: number; since?: string | Date } = {}): Promise { + const max = opts.max ?? 500; + return client.walk( + { collection: CRYPTO_COLLECTION, kind: 'asset', since: opts.since, limit: 200 }, + { maxItems: max, maxPages: Math.ceil(max / 200) + 1 }, + ); +} + +// ─── Grouping and filters ──────────────────────────────────── + +export interface VenueListing { + venue: string; + venueSymbol: string; + /** Quote-currency 24h volume; null when the venue does not report it (Gemini). */ + volume24hQuote: number | null; + priceUsd: number | null; + change24hPct: number | null; +} + +/** One tradeable base grouped across venues, in b1dz's canonical `BASE-USD` naming. */ +export interface DiscoveredPair { + /** Canonical b1dz pair, e.g. `BTC-USD`. */ + pair: string; + base: string; + quote: string; + /** Venue slug → that venue's own symbol and ticker. */ + venues: Record; + /** Sum of the known per-venue volumes (a null volume adds nothing). */ + totalVolumeUsd: number; + /** 24h change from the first venue that reports one, for logging only. */ + change24hPct: number; + /** CoinGecko market cap, 0 when the asset is not in the top list. */ + marketCapUsd: number; +} + +export interface CryptoSelectionRules { + /** Per-venue 24h quote-volume floor in dollars. */ + minVolumeUsd: number; + /** Market-cap floor; a base with an unknown cap passes, as the CoinGecko path did. */ + minMarketCapUsd: number; + /** A base must clear the volume floor on at least this many venues. */ + minVenues: number; + /** Bases never traded (stables and fiat). */ + excludedBases?: Iterable; + /** Quote to keep; b1dz's feeds only know `BASE-USD` books. Default `USD`. */ + quote?: string; + /** Venues to consider; default the four nichedb carries. */ + venues?: Iterable; +} + +export const DEFAULT_CRYPTO_VENUES: readonly CryptoVenue[] = ['kraken', 'coinbase', 'binance-us', 'gemini']; + +export interface CryptoSelection { + pairs: DiscoveredPair[]; + /** Bases dropped for being on fewer than `minVenues` venues (after the volume floor). */ + filteredByVenues: number; + /** Bases dropped by the market-cap floor. */ + filteredByMarketCap: number; + /** Assets found for the market-cap lookup. */ + marketCapCount: number; +} + +/** + * Market caps keyed by upper-cased symbol. Two CoinGecko coins can share a + * ticker; the better-ranked one wins, as it did when CoinGecko's ranked list + * was walked in order. + */ +export function marketCapsBySymbol(assets: ReadonlyArray): Map { + const best = new Map(); + for (const a of assets) { + const d = a.data; + if (!d || typeof d.symbol !== 'string') continue; + const symbol = d.symbol.toUpperCase(); + const rank = typeof d.rank === 'number' && Number.isFinite(d.rank) ? d.rank : Number.POSITIVE_INFINITY; + const cap = typeof d.marketCapUsd === 'number' && Number.isFinite(d.marketCapUsd) ? d.marketCapUsd : 0; + const prev = best.get(symbol); + if (!prev || rank < prev.rank || (rank === prev.rank && cap > prev.cap)) best.set(symbol, { rank, cap }); + } + return new Map([...best].map(([s, v]) => [s, v.cap])); +} + +/** + * Group pair rows by base across venues and apply b1dz's discovery rules: + * + * 1. only online books quoted in `rules.quote` (USD) on a known venue; + * 2. a venue counts when its `volume24hQuote` is at or above the floor, or + * is null: an unknown volume passes, exactly as the old Gemini sentinel + * (`minVolumeUsd * 2`) made every Gemini book pass; + * 3. a base needs `minVenues` counting venues; + * 4. a base with a known market cap under the floor is dropped; an unknown + * cap (not in the top list, or no assets at all) passes. + * + * Sorted by total known volume, descending. + */ +export function selectCryptoPairs( + pairRows: ReadonlyArray, + assetRows: ReadonlyArray, + rules: CryptoSelectionRules, +): CryptoSelection { + const quote = (rules.quote ?? 'USD').toUpperCase(); + const excluded = new Set([...(rules.excludedBases ?? [])].map((b) => b.toUpperCase())); + const venues = new Set([...(rules.venues ?? DEFAULT_CRYPTO_VENUES)]); + const marketCaps = marketCapsBySymbol(assetRows); + + const byBase = new Map(); + for (const row of pairRows) { + const d = row.data; + if (!d || typeof d.base !== 'string' || typeof d.venue !== 'string') continue; + if (!venues.has(d.venue)) continue; + if ((d.quote ?? '').toUpperCase() !== quote) continue; + if (d.status && d.status !== 'online') continue; + const base = d.base.toUpperCase(); + if (base.length === 0 || excluded.has(base)) continue; + + const vol = typeof d.volume24hQuote === 'number' && Number.isFinite(d.volume24hQuote) ? d.volume24hQuote : null; + if (vol !== null && vol < rules.minVolumeUsd) continue; + + const listing: VenueListing = { + venue: d.venue, + venueSymbol: d.venueSymbol, + volume24hQuote: vol, + priceUsd: typeof d.priceUsd === 'number' ? d.priceUsd : null, + change24hPct: typeof d.change24hPct === 'number' && Number.isFinite(d.change24hPct) ? d.change24hPct : null, + }; + + let entry = byBase.get(base); + if (!entry) { + entry = { pair: `${base}-${quote}`, base, quote, venues: {}, totalVolumeUsd: 0, change24hPct: 0, marketCapUsd: 0 }; + byBase.set(base, entry); + } + const prev = entry.venues[d.venue]; + // The same book twice (a stale row plus a fresh one): keep the larger volume. + if (prev && (prev.volume24hQuote ?? -1) >= (vol ?? -1)) continue; + if (prev?.volume24hQuote != null) entry.totalVolumeUsd -= prev.volume24hQuote; + entry.venues[d.venue] = listing; + if (vol !== null) entry.totalVolumeUsd += vol; + } + + const pairs: DiscoveredPair[] = []; + let filteredByVenues = 0; + let filteredByMarketCap = 0; + for (const entry of byBase.values()) { + const listings = Object.values(entry.venues); + if (listings.length < rules.minVenues) { + filteredByVenues++; + continue; + } + const cap = marketCaps.get(entry.base) ?? 0; + entry.marketCapUsd = cap; + if (marketCaps.size > 0 && cap > 0 && cap < rules.minMarketCapUsd) { + filteredByMarketCap++; + continue; + } + // Coinbase's reported change first (the old path let Coinbase overwrite), then any venue. + const change = entry.venues['coinbase']?.change24hPct ?? listings.find((l) => l.change24hPct != null)?.change24hPct ?? 0; + entry.change24hPct = change; + pairs.push(entry); + } + pairs.sort((a, b) => b.totalVolumeUsd - a.totalVolumeUsd || a.pair.localeCompare(b.pair)); + + return { pairs, filteredByVenues, filteredByMarketCap, marketCapCount: marketCaps.size }; +} diff --git a/packages/source-nichedb/src/env.ts b/packages/source-nichedb/src/env.ts new file mode 100644 index 0000000..fb2a5fc --- /dev/null +++ b/packages/source-nichedb/src/env.ts @@ -0,0 +1,28 @@ +/** + * Feature switches for reading from nichedb.dev. + * + * NICHEDB_CRYPTO=1 pair discovery reads the crypto universe from nichedb + * NICHEDB_MARKETS=1 strategy backtests read daily equity bars from nichedb + * NICHEDB_URL base URL (default https://nichedb.dev) + * + * Read through `process.env[name]` on purpose: Next.js inlines + * `process.env.NAME` member expressions at build time, and these are runtime + * switches that must be flippable on the deployed daemon and web app. + */ + +export type NichedbSwitch = 'NICHEDB_CRYPTO' | 'NICHEDB_MARKETS'; + +const ON = new Set(['1', 'true', 'yes', 'on']); + +export function nichedbEnabled(name: NichedbSwitch, env: NodeJS.ProcessEnv = process.env): boolean { + const raw = env[name]; + if (raw == null) return false; + return ON.has(raw.trim().toLowerCase()); +} + +export const DEFAULT_NICHEDB_URL = 'https://nichedb.dev'; + +export function nichedbBaseUrl(env: NodeJS.ProcessEnv = process.env): string { + const raw = env['NICHEDB_URL']?.trim(); + return (raw && raw.length > 0 ? raw : DEFAULT_NICHEDB_URL).replace(/\/+$/, ''); +} diff --git a/packages/source-nichedb/src/index.ts b/packages/source-nichedb/src/index.ts new file mode 100644 index 0000000..b60cdfe --- /dev/null +++ b/packages/source-nichedb/src/index.ts @@ -0,0 +1,54 @@ +/** + * @b1dz/source-nichedb — read-only client for nichedb.dev's public items API + * plus the two readers b1dz uses: the crypto universe (pair discovery) and + * daily equity bars (strategy backtests). Everything real-time stays on the + * venues' own feeds; this package only replaces slow-moving catalogue reads. + */ + +export { nichedbEnabled, nichedbBaseUrl, DEFAULT_NICHEDB_URL, type NichedbSwitch } from './env.js'; +export { + createNichedbClient, + NichedbError, + NICHEDB_PAGE_LIMIT, + type NichedbClient, + type NichedbClientOptions, + type NichedbItem, + type ItemsQuery, + type WalkOptions, + type FetchLike, +} from './client.js'; +export { + CRYPTO_COLLECTION, + DEFAULT_CRYPTO_VENUES, + fetchCryptoPairs, + fetchCryptoAssets, + marketCapsBySymbol, + selectCryptoPairs, + type CryptoVenue, + type CryptoAssetData, + type CryptoPairData, + type CryptoAssetItem, + type CryptoPairItem, + type CryptoSelectionRules, + type CryptoSelection, + type DiscoveredPair, + type VenueListing, +} from './crypto.js'; +export { + MARKETS_COLLECTION, + HISTORY_MAX_BARS, + HISTORY_MAX_AGE_DAYS, + historySymbolTag, + dayToUtcMs, + barsFromHistory, + historyWindow, + fetchHistoryItem, + fetchDailyBars, + type HistoryBarTuple, + type HistoryData, + type HistoryItem, + type HistoryMiss, + type HistoryResult, + type HistoryOptions, + type DailyBar, +} from './markets.js'; diff --git a/packages/source-nichedb/src/markets.test.ts b/packages/source-nichedb/src/markets.test.ts new file mode 100644 index 0000000..3422006 --- /dev/null +++ b/packages/source-nichedb/src/markets.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest'; +import { createNichedbClient, type FetchLike } from './client.js'; +import { barsFromHistory, dayToUtcMs, fetchDailyBars, historySymbolTag, historyWindow, DAY_MS, type HistoryItem } from './markets.js'; + +const NOW = Date.UTC(2026, 8, 16, 14, 0, 0); // 2026-09-16 14:00Z, a Wednesday + +function ymd(ms: number): string { + return new Date(ms).toISOString().slice(0, 10); +} + +/** `count` weekday bars ending on `lastDay`, oldest first, close = 100 + i. */ +export function historyItem(symbol: string, lastMs: number, count: number): HistoryItem { + const bars: HistoryItem['data']['bars'] = []; + let ms = lastMs; + while (bars.length < count) { + const dow = new Date(ms).getUTCDay(); + if (dow !== 0 && dow !== 6) bars.unshift([ymd(ms), 100, 101, 99, 100 + bars.length, 1_000, 100.5]); + ms -= DAY_MS; + } + // close values were assigned newest-first above; make them ascend oldest-first for readable tests + bars.forEach((b, i) => { b[4] = 100 + i; }); + return { + id: 1, + collection: 'markets', + source: 'equity-history', + kind: 'history', + external_id: `history:${symbol}`, + title: `${symbol} daily bars`, + published_at: `${bars[bars.length - 1]![0]}T00:00:00.000Z`, + updated_at: '2026-09-11T00:30:00.000Z', + tags: ['history', `symbol:${symbol.toLowerCase()}`, 'feed:iex'], + data: { symbol, timeframe: '1Day', feed: 'iex', adjustment: 'split', bars, first: bars[0]![0], last: bars[bars.length - 1]![0], count: bars.length }, + }; +} + +describe('history mapping', () => { + it('tags symbols lower-case in Alpaca spelling', () => { + expect(historySymbolTag('AAPL')).toBe('symbol:aapl'); + expect(historySymbolTag('BRK-B')).toBe('symbol:brk.b'); + }); + + it('maps YYYY-MM-DD to midnight UTC', () => { + expect(dayToUtcMs('2026-09-10')).toBe(Date.UTC(2026, 8, 10)); + expect(dayToUtcMs('nope')).toBeNull(); + }); + + it('maps bar tuples to OHLCV rows, oldest first, dropping malformed ones', () => { + const item = historyItem('AAPL', NOW - DAY_MS, 3); + item.data.bars.push(['bad', 1, 1, 1, 1, 1, null], ['2026-09-12', Number.NaN, 1, 1, 1, 1, null]); + const bars = barsFromHistory(item.data); + expect(bars).toHaveLength(3); + expect(bars[0]!.ts).toBeLessThan(bars[2]!.ts); + expect(bars[0]).toMatchObject({ open: 100, high: 101, low: 99, close: 100, volume: 1_000, vwap: 100.5 }); + expect(bars[0]!.ts).toBe(dayToUtcMs(bars[0]!.day)); + }); +}); + +describe('historyWindow', () => { + const start = NOW - 365 * DAY_MS; + + it('serves a one-year request from a fresh 400-bar window, padded a week either side', () => { + const item = historyItem('AAPL', NOW - DAY_MS, 400); + const r = historyWindow(item, start, NOW, { now: NOW }); + expect(r.ok).toBe(true); + if (!r.ok) return; + expect(r.bars[0]!.ts).toBeGreaterThanOrEqual(start - 7 * DAY_MS); + expect(r.bars[0]!.ts).toBeLessThanOrEqual(start); + expect(r.bars[r.bars.length - 1]!.day).toBe(ymd(NOW - DAY_MS)); + expect(r.bars.length).toBeGreaterThan(250); + }); + + it('falls back when the item is missing', () => { + expect(historyWindow(null, start, NOW, { now: NOW })).toEqual({ ok: false, reason: 'missing' }); + }); + + it('falls back when the last bar is older than 5 days', () => { + // Wednesday now: a bar from last Friday (5 days) still serves; last Thursday (6 days) does not. + const fresh = historyItem('AAPL', NOW - 5 * DAY_MS, 400); + expect(fresh.data.last).toBe('2026-09-11'); + expect(historyWindow(fresh, start, NOW, { now: NOW }).ok).toBe(true); + const stale = historyItem('AAPL', NOW - 6 * DAY_MS, 400); + expect(stale.data.last).toBe('2026-09-10'); + expect(historyWindow(stale, start, NOW, { now: NOW })).toMatchObject({ ok: false, reason: 'stale' }); + }); + + it('falls back when the request spans more than 400 days', () => { + const item = historyItem('AAPL', NOW - DAY_MS, 400); + expect(historyWindow(item, NOW - 401 * DAY_MS, NOW, { now: NOW })).toEqual({ ok: false, reason: 'window-too-long' }); + expect(historyWindow(item, NOW - 400 * DAY_MS, NOW, { now: NOW }).ok).toBe(true); + }); + + it('falls back when the window does not reach the requested start', () => { + const young = historyItem('NEWCO', NOW - DAY_MS, 40); + expect(historyWindow(young, start, NOW, { now: NOW })).toMatchObject({ ok: false, reason: 'short' }); + expect(historyWindow(young, NOW - 30 * DAY_MS, NOW, { now: NOW }).ok).toBe(true); + }); + + it('falls back on an item with no usable bars', () => { + const item = historyItem('AAPL', NOW - DAY_MS, 1); + item.data.bars = []; + expect(historyWindow(item, start, NOW, { now: NOW })).toMatchObject({ ok: false, reason: 'empty' }); + }); +}); + +describe('fetchDailyBars', () => { + function serve(items: HistoryItem[]) { + const urls: URL[] = []; + const fetch: FetchLike = async (url) => { + const u = new URL(url); + urls.push(u); + const tags = (u.searchParams.get('tags') ?? '').split(','); + const hit = items.filter((i) => tags.every((t) => i.tags.includes(t))).slice(0, Number(u.searchParams.get('limit'))); + return { ok: true, status: 200, statusText: 'OK', json: async () => ({ count: hit.length, items: hit }) }; + }; + return { fetch, urls }; + } + + it('asks for collection=markets&kind=history&tags=symbol:&limit=1', async () => { + const { fetch, urls } = serve([historyItem('AAPL', NOW - DAY_MS, 400)]); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + const r = await fetchDailyBars(client, 'AAPL', NOW - 365 * DAY_MS, NOW, { now: NOW }); + expect(r.ok).toBe(true); + expect(urls).toHaveLength(1); + const q = urls[0]!.searchParams; + expect(q.get('collection')).toBe('markets'); + expect(q.get('kind')).toBe('history'); + expect(q.get('tags')).toBe('symbol:aapl'); + expect(q.get('limit')).toBe('1'); + }); + + it('reports missing for an unknown symbol and makes no request for an over-long window', async () => { + const { fetch, urls } = serve([]); + const client = createNichedbClient({ baseUrl: 'https://n.test', fetch }); + expect(await fetchDailyBars(client, 'ZZZZ', NOW - 30 * DAY_MS, NOW, { now: NOW })).toEqual({ ok: false, reason: 'missing' }); + expect(urls).toHaveLength(1); + expect(await fetchDailyBars(client, 'AAPL', NOW - 5 * 365 * DAY_MS, NOW, { now: NOW })).toEqual({ ok: false, reason: 'window-too-long' }); + expect(urls).toHaveLength(1); + }); +}); diff --git a/packages/source-nichedb/src/markets.ts b/packages/source-nichedb/src/markets.ts new file mode 100644 index 0000000..7f4c6c3 --- /dev/null +++ b/packages/source-nichedb/src/markets.ts @@ -0,0 +1,148 @@ +/** + * The `markets` collection's `history` kind: one item per US equity symbol + * carrying its last 400 daily bars, oldest first, per niche-db docs/markets.md. + * + * data.bars = [[ 'YYYY-MM-DD', open, high, low, close, volume, vwap ], ...] + */ + +import type { NichedbClient, NichedbItem } from './client.js'; + +export const MARKETS_COLLECTION = 'markets'; + +/** One bar as nichedb stores it: day, open, high, low, close, volume, vwap (null when the feed sent none). */ +export type HistoryBarTuple = [string, number, number, number, number, number, (number | null)?]; + +export interface HistoryData { + symbol: string; + timeframe: '1Day' | string; + feed: 'iex' | 'sip' | string; + adjustment: string; + bars: HistoryBarTuple[]; + first: string; + last: string; + count: number; +} + +export type HistoryItem = NichedbItem; + +/** A daily bar in the shape b1dz's backtesters consume. `ts` is midnight UTC of the bar's day. */ +export interface DailyBar { + ts: number; + day: string; + open: number; + high: number; + low: number; + close: number; + volume: number; + vwap: number | null; +} + +export const DAY_MS = 24 * 60 * 60 * 1000; +/** nichedb keeps at most this many bars per symbol; a longer request cannot be served from it. */ +export const HISTORY_MAX_BARS = 400; +/** A window whose last bar is older than this is stale (long weekend plus a holiday still fits). */ +export const HISTORY_MAX_AGE_DAYS = 5; +/** The Yahoo path pads each end of the window by a week; keep the same slack. */ +const WINDOW_PAD_MS = 7 * DAY_MS; + +/** nichedb tags symbols lower-case in Alpaca's spelling (`brk.b`); b1dz and Yahoo write `BRK-B`. */ +export function historySymbolTag(symbol: string): string { + return `symbol:${symbol.trim().toLowerCase().replace(/-/g, '.')}`; +} + +/** Midnight UTC of a `YYYY-MM-DD` day, or null when the string is not one. */ +export function dayToUtcMs(day: string): number | null { + const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(day); + if (!m) return null; + const ms = Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3])); + return Number.isFinite(ms) ? ms : null; +} + +/** Turn a history item's tuples into bars, dropping malformed rows, oldest first. */ +export function barsFromHistory(data: HistoryData | null | undefined): DailyBar[] { + const out: DailyBar[] = []; + for (const tuple of data?.bars ?? []) { + if (!Array.isArray(tuple) || tuple.length < 6) continue; + const [day, open, high, low, close, volume, vwap] = tuple; + const ts = typeof day === 'string' ? dayToUtcMs(day) : null; + if (ts === null) continue; + if (![open, high, low, close].every((n) => typeof n === 'number' && Number.isFinite(n))) continue; + out.push({ + ts, + day, + open: open as number, + high: high as number, + low: low as number, + close: close as number, + volume: typeof volume === 'number' && Number.isFinite(volume) ? volume : 0, + vwap: typeof vwap === 'number' && Number.isFinite(vwap) ? vwap : null, + }); + } + out.sort((a, b) => a.ts - b.ts); + return out; +} + +export type HistoryMiss = + | 'window-too-long' // more than 400 days asked for; nichedb holds at most 400 bars + | 'missing' // no history item for the symbol + | 'empty' // an item with no usable bars + | 'stale' // last bar older than HISTORY_MAX_AGE_DAYS + | 'short'; // the window does not reach back to the requested start + +export type HistoryResult = { ok: true; bars: DailyBar[]; item: HistoryItem } | { ok: false; reason: HistoryMiss; item?: HistoryItem }; + +export interface HistoryOptions { + /** "Now" for the staleness check; defaults to Date.now(). */ + now?: number; + maxAgeDays?: number; +} + +/** + * Decide whether a history item can serve a `[startMs, endMs]` request and, + * if so, return its bars for that window (padded a week either side, as the + * Yahoo path pads). Pure: the caller fetches the item. + */ +export function historyWindow(item: HistoryItem | null | undefined, startMs: number, endMs: number, opts: HistoryOptions = {}): HistoryResult { + const now = opts.now ?? Date.now(); + const maxAgeMs = (opts.maxAgeDays ?? HISTORY_MAX_AGE_DAYS) * DAY_MS; + if (endMs - startMs > HISTORY_MAX_BARS * DAY_MS) return { ok: false, reason: 'window-too-long' }; + if (!item) return { ok: false, reason: 'missing' }; + const all = barsFromHistory(item.data); + if (all.length === 0) return { ok: false, reason: 'empty', item }; + const last = all[all.length - 1]!; + // `last.ts` is midnight of the bar's day: the bar is stale once its day is more than maxAgeDays before now's day. + if (now - last.ts >= maxAgeMs + DAY_MS) return { ok: false, reason: 'stale', item }; + const first = all[0]!; + // A symbol first seen recently has a window that starts after the request; do not truncate silently. + if (first.ts > startMs + WINDOW_PAD_MS) return { ok: false, reason: 'short', item }; + const lo = startMs - WINDOW_PAD_MS; + const hi = endMs + WINDOW_PAD_MS; + return { ok: true, bars: all.filter((b) => b.ts >= lo && b.ts <= hi), item }; +} + +/** One request: the history item for a symbol, or null when nichedb has none. */ +export async function fetchHistoryItem(client: NichedbClient, symbol: string): Promise { + const rows = await client.items({ + collection: MARKETS_COLLECTION, + kind: 'history', + tags: [historySymbolTag(symbol)], + limit: 1, + }); + return rows[0] ?? null; +} + +/** + * Daily bars for a symbol from nichedb, or a reason it cannot serve them. + * Makes no request when the window is longer than nichedb can hold. + */ +export async function fetchDailyBars( + client: NichedbClient, + symbol: string, + startMs: number, + endMs: number, + opts: HistoryOptions = {}, +): Promise { + if (endMs - startMs > HISTORY_MAX_BARS * DAY_MS) return { ok: false, reason: 'window-too-long' }; + const item = await fetchHistoryItem(client, symbol); + return historyWindow(item, startMs, endMs, opts); +} diff --git a/packages/source-nichedb/tsconfig.build.json b/packages/source-nichedb/tsconfig.build.json new file mode 100644 index 0000000..2e725b7 --- /dev/null +++ b/packages/source-nichedb/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "noEmit": false + }, + "exclude": ["**/*.test.ts"] +} diff --git a/packages/source-nichedb/tsconfig.json b/packages/source-nichedb/tsconfig.json new file mode 100644 index 0000000..bf5a36d --- /dev/null +++ b/packages/source-nichedb/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7c04dcb..b634b6a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,6 +68,9 @@ importers: '@b1dz/source-crypto-trade': specifier: workspace:* version: link:../../packages/source-crypto-trade + '@b1dz/source-nichedb': + specifier: workspace:* + version: link:../../packages/source-nichedb '@b1dz/source-strategies': specifier: workspace:* version: link:../../packages/source-strategies @@ -235,6 +238,9 @@ importers: '@b1dz/source-crypto-trade': specifier: workspace:* version: link:../../packages/source-crypto-trade + '@b1dz/source-nichedb': + specifier: workspace:* + version: link:../../packages/source-nichedb '@b1dz/source-strategies': specifier: workspace:* version: link:../../packages/source-strategies @@ -554,6 +560,9 @@ importers: '@b1dz/core': specifier: workspace:* version: link:../core + '@b1dz/source-nichedb': + specifier: workspace:* + version: link:../source-nichedb undici: specifier: ^8.0.2 version: 8.0.2 @@ -640,6 +649,18 @@ importers: specifier: latest version: 5.0.0(@types/node@26.4.1)(vite@8.0.7(@types/node@26.4.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.23.13)) + packages/source-nichedb: + devDependencies: + '@types/node': + specifier: latest + version: 26.4.1 + typescript: + specifier: latest + version: 7.0.2 + vitest: + specifier: latest + version: 5.0.0(@types/node@26.4.1)(vite@8.0.7(@types/node@26.4.1)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.23.13)) + packages/source-schwab: dependencies: '@b1dz/core':