Pair discovery and equity backtests can read nichedb

apps/cli/package.json

@@ -19,6 +19,7 @@
     "@b1dz/storage-supabase": "workspace:*",
     "@b1dz/source-crypto-arb": "workspace:*",
     "@b1dz/source-crypto-trade": "workspace:*",
+    "@b1dz/source-nichedb": "workspace:*",
     "@b1dz/source-strategies": "workspace:*",
     "@b1dz/storage-b1dz-api": "workspace:*",
     "@b1dz/sdk": "workspace:*",

apps/cli/src/strategy-backtest.test.ts

@@ -1,5 +1,81 @@
-import { describe, it, expect } from 'vitest';
-import { parseArgs } from './strategy-backtest.js';
+import { afterEach, describe, it, expect, vi } from 'vitest';
+import { fetchDailySnapshots, parseArgs } from './strategy-backtest.js';
+
+describe('strategy-backtest daily bars via nichedb (NICHEDB_MARKETS)', () => {
+  const DAY = 24 * 60 * 60 * 1000;
+  const today = Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate());
+
+  function historyItem(symbol: string, count = 400) {
+    const bars: (string | number | null)[][] = [];
+    let ms = today - DAY;
+    while (bars.length < count) {
+      const dow = new Date(ms).getUTCDay();
+      if (dow !== 0 && dow !== 6) bars.unshift([new Date(ms).toISOString().slice(0, 10), 10, 11, 9, 10 + bars.length, 1000, null]);
+      ms -= DAY;
+    }
+    return {
+      id: 7, collection: 'markets', kind: 'history', external_id: `history:${symbol}`, title: `${symbol} daily bars`,
+      published_at: null, updated_at: new Date().toISOString(), tags: ['history', `symbol:${symbol.toLowerCase()}`, 'feed:iex'],
+      data: { symbol, timeframe: '1Day', feed: 'iex', adjustment: 'split', bars, first: bars[0]![0], last: bars[bars.length - 1]![0], count: bars.length },
+    };
+  }
+
+  function installFetch(nichedbItems: unknown[]) {
+    const urls: string[] = [];
+    vi.stubGlobal('fetch', vi.fn(async (input: string | URL) => {
+      const url = typeof input === 'string' ? input : input.href;
+      urls.push(url);
+      const host = new URL(url).host;
+      if (host === 'nichedb.dev') return Response.json({ count: nichedbItems.length, items: nichedbItems });
+      if (host === 'query1.finance.yahoo.com') {
+        return Response.json({ chart: { result: [{ timestamp: [1_700_000_000, 1_700_086_400], indicators: { quote: [{ close: [1, 2] }] } }] } });
+      }
+      return new Response('nope', { status: 404 });
+    }));
+    return urls;
+  }
+
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    vi.unstubAllGlobals();
+  });
+
+  it('uses Yahoo only when the switch is off', async () => {
+    vi.stubEnv('NICHEDB_MARKETS', '');
+    const urls = installFetch([historyItem('SPY')]);
+    const snaps = await fetchDailySnapshots('SPY', today - 365 * DAY, today);
+    expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false);
+    expect(snaps.map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']);
+  });
+
+  it('maps nichedb bars to snapshots for an equity and never asks nichedb for crypto', async () => {
+    vi.stubEnv('NICHEDB_MARKETS', '1');
+    const urls = installFetch([historyItem('SPY')]);
+    const snaps = await fetchDailySnapshots('SPY', today - 365 * DAY, today);
+    expect(urls).toHaveLength(1);
+    expect(new URL(urls[0]!).searchParams.get('tags')).toBe('symbol:spy');
+    expect(snaps.length).toBeGreaterThan(240);
+    expect(snaps[0]).toMatchObject({ exchange: 'nichedb', pair: 'SPY', assetClass: 'equity', bidSize: 1, askSize: 1 });
+    expect(snaps[0]!.bid).toBe(snaps[0]!.ask);
+    expect(snaps.every((s, i) => i === 0 || s.ts > snaps[i - 1]!.ts)).toBe(true);
+    urls.length = 0;
+    const crypto = await fetchDailySnapshots('BTC-USD', today - 365 * DAY, today);
+    expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false);
+    expect(crypto.map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']);
+  });
+
+  it('falls back to Yahoo when nichedb is missing the symbol or the window is too long', async () => {
+    vi.stubEnv('NICHEDB_MARKETS', '1');
+    const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
+    const urls = installFetch([]);
+    expect((await fetchDailySnapshots('ZZZZ', today - 90 * DAY, today)).map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']);
+    expect(urls.filter((u) => u.includes('nichedb.dev'))).toHaveLength(1);
+    urls.length = 0;
+    expect((await fetchDailySnapshots('SPY', today - 5 * 365 * DAY, today)).map((s) => s.exchange)).toEqual(['yahoo', 'yahoo']);
+    expect(urls.filter((u) => u.includes('nichedb.dev'))).toHaveLength(0);
+    stderr.mockRestore();
+  });
+});
 
 describe('strategy-backtest parseArgs', () => {
   it('defaults to backtesting both asset classes', () => {

apps/cli/src/strategy-backtest.ts

@@ -4,7 +4,8 @@
  * Unlike `b1dz backtest <tf>` (the server-side multi-pair crypto candle sim),
  * this runs locally against the deterministic StrategyPlugin engine in
  * @b1dz/source-strategies: it replays the strategy's own buy/sell signals
- * long-only over Yahoo daily bars.
+ * long-only over Yahoo daily bars (equities come from nichedb.dev's `markets`
+ * history first when NICHEDB_MARKETS=1, Yahoo when nichedb cannot serve them).
  *
  * Crucially it scores crypto and equities SEPARATELY so you can see which asset
  * class a strategy suits. Run both (default, with a head-to-head verdict),
@@ -19,6 +20,7 @@ import { readFileSync } from 'node:fs';
 import chalk from 'chalk';
 import Table from 'cli-table3';
 import { PLUGIN_CATALOG, type MarketSnapshot, type StrategyPlugin } from '@b1dz/core';
+import { createNichedbClient, fetchDailyBars, nichedbEnabled } from '@b1dz/source-nichedb';
 import {
   STRATEGY_PLUGINS,
   replayStrategy,
@@ -105,7 +107,37 @@ function subtract(end: Date, h: (typeof HORIZONS)[number]): Date {
   return d;
 }
 
-async function fetchDailySnapshots(symbol: string, startMs: number, endMs: number): Promise<MarketSnapshot[]> {
+/**
+ * Equity bars from nichedb's `markets` history (NICHEDB_MARKETS=1). Returns
+ * null when nichedb cannot serve the window: missing symbol, last bar older
+ * than 5 days, or more than its 400-bar window asked for. Crypto symbols are
+ * not in that collection, so they never ask.
+ */
+async function fetchNichedbSnapshots(symbol: string, startMs: number, endMs: number): Promise<MarketSnapshot[] | null> {
+  if (symbol.includes('-USD')) return null;
+  const result = await fetchDailyBars(createNichedbClient(), symbol, startMs, endMs);
+  if (!result.ok) {
+    process.stderr.write(chalk.dim(`  (nichedb cannot serve ${symbol}: ${result.reason}; using Yahoo)\n`));
+    return null;
+  }
+  return result.bars.map((b) => ({
+    exchange: 'nichedb', pair: symbol, bid: b.close, ask: b.close, bidSize: 1, askSize: 1, ts: b.ts, assetClass: 'equity' as const,
+  }));
+}
+
+export async function fetchDailySnapshots(symbol: string, startMs: number, endMs: number): Promise<MarketSnapshot[]> {
+  if (nichedbEnabled('NICHEDB_MARKETS')) {
+    try {
+      const fromNichedb = await fetchNichedbSnapshots(symbol, startMs, endMs);
+      if (fromNichedb && fromNichedb.length > 0) return fromNichedb;
+    } catch (e) {
+      process.stderr.write(chalk.dim(`  (nichedb error for ${symbol}: ${(e as Error).message}; using Yahoo)\n`));
+    }
+  }
+  return fetchYahooSnapshots(symbol, startMs, endMs);
+}
+
+async function fetchYahooSnapshots(symbol: string, startMs: number, endMs: number): Promise<MarketSnapshot[]> {
   const period1 = Math.floor((startMs - 7 * DAY_MS) / 1000);
   const period2 = Math.floor((endMs + 7 * DAY_MS) / 1000);
   const url =
@@ -244,7 +276,7 @@ export async function runStrategyBacktestCli(argv: string[]): Promise<void> {
 
   console.log(
     chalk.dim(
-      `Long-only signal replay · $${args.amount}/entry · Yahoo daily · classes: ${args.classes.join(' + ')} · ignores fees/slippage`,
+      `Long-only signal replay · $${args.amount}/entry · ${nichedbEnabled('NICHEDB_MARKETS') ? 'nichedb + Yahoo' : 'Yahoo'} daily · classes: ${args.classes.join(' + ')} · ignores fees/slippage`,
     ),
   );
 

apps/web/next.config.ts

@@ -67,6 +67,7 @@ const config: NextConfig = {
   transpilePackages: [
     '@b1dz/core',
     '@b1dz/sdk',
+    '@b1dz/source-nichedb',
     '@b1dz/source-strategies',
     '@b1dz/storage-json',
     '@b1dz/storage-supabase',

apps/web/package.json

@@ -16,6 +16,7 @@
     "@b1dz/projection-engine": "workspace:*",
     "@b1dz/source-crypto-arb": "workspace:*",
     "@b1dz/source-crypto-trade": "workspace:*",
+    "@b1dz/source-nichedb": "workspace:*",
     "@b1dz/source-strategies": "workspace:*",
     "@b1dz/storage-json": "workspace:*",
     "@b1dz/storage-supabase": "workspace:*",

apps/web/src/app/api/strategies/backtest/route.test.ts

@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
 const authenticateMock = vi.fn();
 const unauthorizedMock = vi.fn(() => new Response(JSON.stringify({ error: 'unauthorized' }), { status: 401 }));
@@ -108,3 +108,98 @@ describe('POST /api/strategies/backtest', () => {
     expect(runBacktestMock).not.toHaveBeenCalled();
   });
 });
+
+describe('equity closes via nichedb (NICHEDB_MARKETS)', () => {
+  const DAY = 24 * 60 * 60 * 1000;
+  const today = Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate());
+
+  /** A fresh 400-bar history item ending yesterday, weekdays only. */
+  function historyItem(symbol: string) {
+    const bars: (string | number | null)[][] = [];
+    let ms = today - DAY;
+    while (bars.length < 400) {
+      const dow = new Date(ms).getUTCDay();
+      if (dow !== 0 && dow !== 6) bars.unshift([new Date(ms).toISOString().slice(0, 10), 10, 11, 9, 10 + bars.length, 1000, null]);
+      ms -= DAY;
+    }
+    return {
+      id: 7, collection: 'markets', kind: 'history', external_id: `history:${symbol}`, title: `${symbol} daily bars`,
+      published_at: null, updated_at: new Date().toISOString(), tags: ['history', `symbol:${symbol.toLowerCase()}`, 'feed:iex'],
+      data: { symbol, timeframe: '1Day', feed: 'iex', adjustment: 'split', bars, first: bars[0]![0], last: bars[399]![0], count: 400 },
+    };
+  }
+
+  function installFetch(nichedbItems: unknown[]) {
+    const urls: string[] = [];
+    vi.stubGlobal('fetch', vi.fn(async (input: string | URL) => {
+      const url = typeof input === 'string' ? input : input.href;
+      urls.push(url);
+      const host = new URL(url).host;
+      if (host === 'nichedb.dev') return Response.json({ count: nichedbItems.length, items: nichedbItems });
+      if (host === 'query1.finance.yahoo.com') {
+        return Response.json({ chart: { result: [{ timestamp: [1_700_000_000, 1_700_086_400], indicators: { quote: [{ close: [1, 2] }] } }] } });
+      }
+      return new Response('nope', { status: 404 });
+    }));
+    return urls;
+  }
+
+  async function equityCloses(symbol: string, days: number) {
+    const { POST } = await importRoute();
+    await POST(makeReq({ definition: validDoc, classes: ['equity'] }) as never);
+    const [, opts] = runBacktestMock.mock.calls[0]!;
+    return opts.fetchCloses(symbol, today - days * DAY, today) as Promise<{ ts: number; close: number }[]>;
+  }
+
+  beforeEach(() => {
+    vi.stubEnv('ALPACA_API_KEY_ID', '');
+    vi.stubEnv('ALPACA_API_SECRET_KEY', '');
+    vi.spyOn(console, 'warn').mockImplementation(() => {});
+  });
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+  });
+
+  it('makes no nichedb request when the switch is off', async () => {
+    vi.stubEnv('NICHEDB_MARKETS', '');
+    const urls = installFetch([historyItem('AAPL')]);
+    const rows = await equityCloses('AAPL', 365);
+    expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false);
+    expect(urls.some((u) => u.includes('finance.yahoo.com'))).toBe(true);
+    expect(rows.map((r) => r.close)).toEqual([1, 2]);
+  });
+
+  it('serves a one-year window from nichedb bars without touching Yahoo', async () => {
+    vi.stubEnv('NICHEDB_MARKETS', '1');
+    const urls = installFetch([historyItem('AAPL')]);
+    const rows = await equityCloses('AAPL', 365);
+    expect(urls).toHaveLength(1);
+    const q = new URL(urls[0]!).searchParams;
+    expect(q.get('collection')).toBe('markets');
+    expect(q.get('kind')).toBe('history');
+    expect(q.get('tags')).toBe('symbol:aapl');
+    expect(q.get('limit')).toBe('1');
+    expect(rows.length).toBeGreaterThan(240);
+    expect(rows[rows.length - 1]!.ts).toBe(rows[rows.length - 1]!.ts - (rows[rows.length - 1]!.ts % DAY)); // midnight UTC
+    expect(rows.every((r) => Number.isFinite(r.close))).toBe(true);
+  });
+
+  it('falls back to Yahoo when nichedb has no history for the symbol', async () => {
+    vi.stubEnv('NICHEDB_MARKETS', '1');
+    const urls = installFetch([]);
+    const rows = await equityCloses('ZZZZ', 90);
+    expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(true);
+    expect(urls.some((u) => u.includes('finance.yahoo.com'))).toBe(true);
+    expect(rows.map((r) => r.close)).toEqual([1, 2]);
+  });
+
+  it('skips nichedb for a window longer than its 400 bars and uses Yahoo', async () => {
+    vi.stubEnv('NICHEDB_MARKETS', '1');
+    const urls = installFetch([historyItem('AAPL')]);
+    const rows = await equityCloses('AAPL', 5 * 365);
+    expect(urls.some((u) => u.includes('nichedb.dev'))).toBe(false);
+    expect(rows.map((r) => r.close)).toEqual([1, 2]);
+  });
+});

apps/web/src/app/api/strategies/backtest/route.ts

@@ -14,13 +14,16 @@
  *   - Crypto → Kraken daily OHLC via @b1dz/source-crypto-trade's
  *     fetchHistoricalCandles (keyless; the same path the daemon + /api/backtest
  *     use, proven to work from Railway).
- *   - Equities → Alpaca daily bars (when ALPACA_API_KEY_ID/SECRET are set),
+ *   - Equities → nichedb.dev's `markets` history (when NICHEDB_MARKETS=1 and
+ *     it holds a fresh window covering the request; keyless, one request per
+ *     symbol), then Alpaca daily bars (when ALPACA_API_KEY_ID/SECRET are set),
  *     falling back to Yahoo best-effort. Yahoo is frequently blocked from
- *     datacenter IPs, so without Alpaca keys equities may return no data;
- *     crypto is unaffected.
+ *     datacenter IPs, so without nichedb or Alpaca keys equities may return
+ *     no data; crypto is unaffected.
  */
 import type { NextRequest } from 'next/server';
 import { fetchHistoricalCandles } from '@b1dz/source-crypto-trade';
+import { createNichedbClient, fetchDailyBars, nichedbEnabled } from '@b1dz/source-nichedb';
 import { tsp } from '@b1dz/source-strategies';
 import { authenticate, unauthorized } from '@/lib/api-auth';
 import {
@@ -101,10 +104,24 @@ async function fetchAlpacaCloses(symbol: string, startMs: number, endMs: number)
   return rows;
 }
 
+/**
+ * Equity daily closes from nichedb's `markets` history item (400 daily bars,
+ * split adjusted). Throws when nichedb has no fresh window covering the
+ * request (missing symbol, last bar older than 5 days, more than 400 days
+ * asked for) so the caller moves on to Alpaca and Yahoo.
+ */
+async function fetchNichedbCloses(symbol: string, startMs: number, endMs: number): Promise<DailyClose[]> {
+  const result = await fetchDailyBars(createNichedbClient(), symbol, startMs, endMs);
+  if (!result.ok) throw new Error(`nichedb ${result.reason}`);
+  const rows = result.bars.map((b) => ({ ts: b.ts, close: b.close }));
+  if (rows.length === 0) throw new Error('nichedb returned no rows');
+  return rows;
+}
+
 /**
  * Route closes by asset class:
  *   - crypto → Kraken (reliable from Railway, keyless).
- *   - equities → Alpaca (if keys set) → Yahoo fallback.
+ *   - equities → nichedb (if NICHEDB_MARKETS=1) → Alpaca (if keys set) → Yahoo fallback.
  * Returns [] only if every source for that symbol fails.
  */
 const fetchCloses: FetchCloses = async (symbol, startMs, endMs) => {
@@ -116,7 +133,14 @@ const fetchCloses: FetchCloses = async (symbol, startMs, endMs) => {
       return [];
     }
   }
-  // Equity: Alpaca first, then Yahoo.
+  // Equity: nichedb when switched on, then Alpaca, then Yahoo.
+  if (nichedbEnabled('NICHEDB_MARKETS')) {
+    try {
+      return await fetchNichedbCloses(symbol, startMs, endMs);
+    } catch (err) {
+      console.warn(`[backtest] nichedb cannot serve ${symbol} (${(err as Error).message}); falling back`);
+    }
+  }
   try {
     return await fetchAlpacaCloses(symbol, startMs, endMs);
   } catch (alpacaErr) {

packages/source-crypto-arb/package.json

@@ -17,6 +17,7 @@
   "dependencies": {
     "@b1dz/adapters-evm": "workspace:*",
     "@b1dz/core": "workspace:*",
+    "@b1dz/source-nichedb": "workspace:*",
     "undici": "^8.0.2",
     "ws": "^8.20.0"
   },