diff --git a/.env.example b/.env.example index 43134a0..486ccb2 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,17 @@ APCA_API_SECRET_KEY= # Optional: override the data base URL (defaults to https://data.alpaca.markets) APCA_API_DATA_URL= +# --- Shared market data from nichedb.dev --- +# Set to 1 to read daily bars, SEC fundamentals, the symbol directory and the +# market-news wire from nichedb's public `markets` collection (keyless) instead +# of fetching them here. Every read falls back to the live Alpaca/Yahoo/SEC +# path when nichedb has nothing for a ticker or its window is older than five +# days. Snapshots (latest trade/quote) and the SEC filings list stay live. +# Unset or 0: no request to nichedb is ever made. +NICHEDB_MARKETS= +# Optional: a different nichedb deployment (defaults to https://nichedb.dev). +NICHEDB_URL= + # --- Database (libSQL / Turso) --- # For a local embedded database use: file:./data/transcripts.sqlite # For Turso use the libsql:// URL for your database and an auth token. diff --git a/README.md b/README.md index 951f970..efb95a4 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,27 @@ environment variables (see `.env.example`): | `RESEND_API_KEY` / `MAILGUN_API_KEY` | Transactional + digest email transport | | `APP_URL` | Public base URL used for links in emails | | `DIGEST_SCHEDULER` | `0` disables the built-in 04:00 ET digest scheduler | +| `NICHEDB_MARKETS` | `1` reads shared market data from nichedb.dev (below) | +| `NICHEDB_URL` | Another nichedb deployment (default `https://nichedb.dev`) | + +### Shared market data (nichedb.dev) + +With `NICHEDB_MARKETS=1` the site reads what every site needs from +[nichedb.dev](https://nichedb.dev)'s public `markets` collection instead of +fetching it itself: no key, one request per read, and the same shapes the rest +of the app already consumes. + +| Read | nichedb item | Falls back to | +|---|---|---| +| Daily bars for a report build | `kind=history&tags=symbol:` (last 400 bars) | Alpaca → Yahoo, when there is no item or its last bar is older than 5 days | +| Company facts for a report build | `kind=fundamentals&tags=symbol:` | Live SEC companyfacts | +| Symbol directory (`symbols sync`) | `kind=symbol`, paged with a stored `since=` cursor | Alpaca asset list | +| News refresh, before the RSS feeds | `kind=market-news&tags=` (90-day window) | The RSS feeds still run for anything the wire lacks | + +What stays live regardless: snapshots (latest trade and quote — nichedb has no +quotes, so the price on a report is always the provider's), the SEC filings +list, the per-miss Yahoo symbol search, and ValueSERP. Off, no request to +nichedb is ever made. ## Ticker lookup diff --git a/config.example.toml b/config.example.toml index 4ce2f8b..0905e00 100644 --- a/config.example.toml +++ b/config.example.toml @@ -1,5 +1,14 @@ # transcript-search config (PRD §23). Copy to ~/.config/transcripts/config.toml # Secrets (API keys, DB auth token) go in the environment, NOT here. +# +# Shared market data is also an environment switch, not a TOML key: +# NICHEDB_MARKETS=1 read daily bars, SEC fundamentals, the symbol directory +# and the market-news wire from nichedb.dev (keyless), +# falling back to the live Alpaca/Yahoo/SEC path below +# whenever nichedb has nothing fresh for a ticker. +# NICHEDB_URL= another nichedb deployment (default https://nichedb.dev). +# The [alpaca] section still governs snapshots (latest trade/quote), which +# nichedb does not carry, and every fallback read. database = "~/.local/share/transcripts/transcripts.sqlite" downloads = "~/.local/share/transcripts/downloads" diff --git a/src/cli.ts b/src/cli.ts index 2bd998f..7fcc90d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -496,9 +496,29 @@ symbols await withApp(async ({ db, registry }) => { const { fetchAlpacaDirectory } = await import("./symbols/providers.ts"); const { upsertSymbols, directoryAge } = await import("./symbols/directory.ts"); + const { nichedbMarketsFromEnv } = await import("./providers/nichedb-markets.ts"); const before = await directoryAge(db); console.log(`Directory before: ${before.count} symbol(s)`); + // NICHEDB_MARKETS=1: walk nichedb's `kind=symbol` mirror from the stored + // cursor — the whole directory the first time, only what moved after + // that — and only reach for the Alpaca asset list when nichedb fails. + const nichedb = nichedbMarketsFromEnv(); + if (nichedb) { + const { syncSymbolsFromNichedb } = await import("./symbols/nichedb-sync.ts"); + try { + const r = await syncSymbolsFromNichedb(db, nichedb.client, { onProgress: (m) => console.log(` ${m}`) }); + const after = await directoryAge(db); + console.log( + `nichedb: ${r.items} item(s) over ${r.pages} page(s)${r.since ? ` since ${r.since}` : " (full walk)"}, ` + + `wrote ${r.written} symbol(s). Directory now: ${after.count}. Cursor: ${r.cursor ?? "none"}.`, + ); + return; + } catch (err) { + console.error(`nichedb symbol mirror unavailable, falling back to Alpaca: ${String(err).slice(0, 200)}`); + } + } + let rows: Awaited> = []; try { rows = await fetchAlpacaDirectory(registry.alpaca); diff --git a/src/db/schema.sql b/src/db/schema.sql index 4dcf1bf..3912813 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -43,7 +43,7 @@ CREATE TABLE IF NOT EXISTS symbols ( asset_class TEXT, status TEXT, -- active | inactive tradable INTEGER NOT NULL DEFAULT 1, - source TEXT NOT NULL, -- alpaca | yahoo + source TEXT NOT NULL, -- alpaca | yahoo | nichedb updated_at TEXT NOT NULL, -- Name in its match form: lowercased with punctuation collapsed to spaces, so -- "coca cola" finds The Coca-Cola Company. Stored rather than computed per @@ -520,6 +520,16 @@ CREATE TABLE IF NOT EXISTS auth_attempts ( created_at TEXT NOT NULL ); +-- Where a nichedb.dev mirror walk left off (NICHEDB_MARKETS=1). One row per +-- walk, keyed by what is mirrored: `symbols.since` holds the newest +-- `updated_at` seen on the last complete walk of `kind=symbol`, so the next +-- sync asks nichedb only for what moved. +CREATE TABLE IF NOT EXISTS nichedb_cursor ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at TEXT NOT NULL +); + -- Helpful indexes CREATE INDEX IF NOT EXISTS idx_signals_ticker ON signals(ticker); CREATE INDEX IF NOT EXISTS idx_corrob_ticker ON corroborations(ticker, relation); diff --git a/src/pipeline/news-refresh.ts b/src/pipeline/news-refresh.ts index 877fc67..7499097 100644 --- a/src/pipeline/news-refresh.ts +++ b/src/pipeline/news-refresh.ts @@ -21,6 +21,7 @@ import type { Client } from "@libsql/client"; import type { AppConfig } from "../config.ts"; import { NewsProvider } from "../providers/news/index.ts"; +import { nichedbMarketsFromEnv, type NichedbMarkets } from "../providers/nichedb-markets.ts"; import { ingest } from "./ingest.ts"; export interface NewsRefreshOptions { @@ -34,6 +35,12 @@ export interface NewsRefreshOptions { useValueSerp?: boolean; /** Only consider articles published on/after this ISO date. */ from?: string; + /** + * The shared market-news mirror to consult before the RSS feeds. Resolved + * from `NICHEDB_MARKETS` when not given; pass `null` to force the RSS-only + * path regardless of the environment. + */ + nichedb?: NichedbMarkets | null; onProgress?: (message: string) => void; } @@ -103,12 +110,22 @@ export async function refreshTickerNews( return base; } + // Default window: a quarter of coverage is what a 1-2 quarter horizon needs. + const from = opts.from ?? new Date(Date.now() - 90 * 86_400_000).toISOString().slice(0, 10); + + // The shared wire first, when the switch is on: one keyless request for the + // ticker's whole window, dated and attributed, before any RSS feed is read. + // The feeds still run after it for anything the wire did not carry. + const nichedb = opts.nichedb === undefined ? nichedbMarketsFromEnv() : opts.nichedb; + const sinceDays = Math.max(1, Math.ceil((Date.now() - Date.parse(from)) / 86_400_000)); + const provider = new NewsProvider({ downloadsDir: config.downloadsDir, // Deliberately blank unless asked: RSS discovery is free, ValueSERP is not, // and this runs on a user-facing click. valueSerpKey: opts.useValueSerp ? config.secrets.valueSerpApiKey : "", perTicker: opts.perTicker ?? 8, + discover: nichedb ? (t) => nichedb.newsHits(t, { sinceDays }) : undefined, }); const name = await companyNameFor(db, ticker); if (name) provider.setCompanyNames(new Map([[ticker, name]])); @@ -121,10 +138,9 @@ export async function refreshTickerNews( }); provider.setKnownHeadlines(titles.rows.map((r) => String(r.title ?? ""))); - opts.onProgress?.(`Searching news for ${name ? `${name} (${ticker})` : ticker}`); - - // Default window: a quarter of coverage is what a 1-2 quarter horizon needs. - const from = opts.from ?? new Date(Date.now() - 90 * 86_400_000).toISOString().slice(0, 10); + opts.onProgress?.( + `Searching news for ${name ? `${name} (${ticker})` : ticker}${nichedb ? " (nichedb wire first)" : ""}`, + ); const run = ingest( db, diff --git a/src/providers/news/index.ts b/src/providers/news/index.ts index fbbf88b..1d19390 100644 --- a/src/providers/news/index.ts +++ b/src/providers/news/index.ts @@ -42,6 +42,14 @@ export interface NewsProviderOptions { * appears alongside the ticker (see `isAboutSubject`). */ requireSubject?: boolean; + /** + * Hits to consider before the RSS feeds: a shared source that already holds + * dated, publisher-attributed stories for the ticker (nichedb's market-news + * wire when `NICHEDB_MARKETS` is on). They go through the same subject + * check, headline dedupe, tiering and per-ticker cap as every other hit, and + * take the first places in it, so the feeds only add what this did not. + */ + discover?: (ticker: string, from?: string) => Promise; } /** @@ -126,6 +134,16 @@ export class NewsProvider extends BaseTranscriptProvider { const hits: NewsHit[] = []; const name = this.companyNames.get(ticker); + // 0. A shared mirror, when one is wired: already dated and attributed, + // and one request for the ticker's whole window. + if (this.options.discover) { + try { + hits.push(...(await this.options.discover(ticker, query.from))); + } catch { + /* the mirror is an accelerator, not a precondition */ + } + } + // 1. Keyless per-ticker headline feed — cheapest broad coverage. try { const items = await fetchFeed(yahooTickerFeed(ticker)); diff --git a/src/providers/nichedb-markets.ts b/src/providers/nichedb-markets.ts new file mode 100644 index 0000000..c28c5c8 --- /dev/null +++ b/src/providers/nichedb-markets.ts @@ -0,0 +1,341 @@ +/** + * nichedb items → the site's own shapes, behind the `NICHEDB_MARKETS` switch. + * + * Everything downstream — indicators, scoring, the evidence builder, the report + * page — keeps reading `MarketBar[]`, `CompanyFacts`, `SymbolRow[]` and + * `NewsHit[]`. This module is the only place that knows a nichedb item exists, + * and every read here answers `null`/`[]` when nichedb has nothing usable so + * the caller falls through to the live provider it always had. + * + * What stays live on purpose: + * - Snapshots (latest trade/quote): nichedb has no quotes. The report's + * price, timestamp and `delayed` flag still come from Alpaca (or Yahoo). + * - The filings list (SEC submissions): nichedb's `filings` collection is a + * different shape; `getFilings` is untouched. + * - Yahoo per-miss symbol search: a ticker nichedb's directory lacks is + * still found and cached the same way. + */ +import type { CompanyFacts, MarketBar } from "../types.ts"; +import type { SymbolRow } from "../symbols/directory.ts"; +import type { NewsHit } from "./news/valueserp.ts"; +import { normalizeHost, tierFor } from "./news/tiers.ts"; +import { + NichedbClient, + nichedbEnabled, + type NichedbFactPoint, + type NichedbFundamentalsData, + type NichedbHistoryData, + type NichedbItem, + type NichedbNewsData, + type NichedbSymbolData, +} from "./nichedb.ts"; + +/** The source name carried on facts, symbols and bars read from the mirror. */ +export const NICHEDB_SOURCE = "nichedb"; + +/** + * A history window whose last bar is older than this is not used: the mirror + * refreshes after every session, so five days covers a long weekend plus a + * holiday, and anything staler means the mirror stopped and the live path + * should answer. + */ +export const HISTORY_MAX_AGE_DAYS = 5; + +/** + * Daily bars are stamped at 05:00Z: midnight Eastern in winter, 01:00 in + * summer — either way the same Eastern calendar day as the bar, which is what + * `etDate()` in the digest and the `slice(0, 10)` on the report page both read. + * Alpaca stamps its daily bars at midnight ET too, so the two sources agree. + */ +export function barTimestamp(day: string): string { + return `${day}T05:00:00Z`; +} + +export interface BarsWindow { + /** ISO instant or date; bars on/after this day are kept. */ + start?: string; + /** ISO instant or date; bars on/before this day are kept. */ + end?: string; +} + +export interface HistoryBars { + bars: MarketBar[]; + feed: "iex" | "sip"; + /** The IEX free feed is one venue; only SIP is the consolidated tape. */ + delayed: boolean; + /** The last bar's day. */ + last: string; +} + +/** + * A `history` item → `MarketBar[]` in the Alpaca provider's shape, oldest + * first, cut to the window asked for. `null` when the item is missing, not a + * history item, or its last bar is older than `HISTORY_MAX_AGE_DAYS`. + */ +export function historyToBars( + item: NichedbItem | null, + window: BarsWindow = {}, + now: number = Date.now(), +): HistoryBars | null { + if (!item || item.kind !== "history") return null; + const data = item.data as unknown as Partial; + if (!Array.isArray(data.bars) || !data.bars.length) return null; + const symbol = String(data.symbol ?? "").toUpperCase(); + if (!symbol) return null; + + const last = String(data.last ?? data.bars[data.bars.length - 1]![0]); + if (!isFresh(last, now)) return null; + + const feed: "iex" | "sip" = data.feed === "sip" ? "sip" : "iex"; + const from = window.start?.slice(0, 10); + const to = window.end?.slice(0, 10); + const bars: MarketBar[] = []; + for (const tuple of data.bars) { + if (!Array.isArray(tuple) || tuple.length < 6) continue; + const [day, open, high, low, close, volume, vwap] = tuple; + if (typeof day !== "string") continue; + if (from && day < from) continue; + if (to && day > to) continue; + if (![open, high, low, close].every((n) => typeof n === "number" && Number.isFinite(n))) continue; + bars.push({ + symbol, + timestamp: barTimestamp(day), + open: open as number, + high: high as number, + low: low as number, + close: close as number, + volume: typeof volume === "number" ? volume : 0, + vwap: typeof vwap === "number" ? vwap : undefined, + timeframe: "1Day", + // nichedb's window is split-adjusted, whatever the site's Alpaca config + // says; the bar records what it is rather than what was asked for. + adjustment: "split", + }); + } + if (!bars.length) return null; + bars.sort((a, b) => a.timestamp.localeCompare(b.timestamp)); + return { bars, feed, delayed: feed !== "sip", last }; +} + +/** True when `day` (YYYY-MM-DD) is within the staleness limit of `now`. */ +export function isFresh(day: string, now: number, maxAgeDays = HISTORY_MAX_AGE_DAYS): boolean { + const t = Date.parse(day.length === 10 ? `${day}T00:00:00Z` : day); + if (Number.isNaN(t)) return false; + return now - t <= maxAgeDays * 86_400_000; +} + +/** + * A `fundamentals` item → `CompanyFacts`, the shape `SecFundamentalsProvider` + * produces. Derived the same way from the same concepts: revenue through the + * three tags in the same order of preference, debt as current plus + * non-current, shares from the cover page, runway from cash over operating + * burn. `asOf` is honoured point-in-time the way `latestValue` there does it — + * the newest point whose period end is on/before the cutoff — so a report + * built for a past date still reads what was known then. + */ +export function fundamentalsToFacts( + item: NichedbItem | null, + symbol: string, + asOf?: string, +): CompanyFacts | null { + if (!item || item.kind !== "fundamentals") return null; + const data = item.data as unknown as Partial; + const concepts = (data.concepts ?? {}) as Record; + const latest = data.latest ?? {}; + const cutoff = asOf?.slice(0, 10); + const sym = symbol.toUpperCase(); + const nowIso = asOf ?? new Date().toISOString(); + + const pick = (concept: string): number | null => latestValue(concepts[concept], cutoff); + // `latest` is the item's own headline figure. It describes one period + // (`latest.period`), so it is only point-in-time safe when that period ends + // on/before the cutoff — the same `end <= cutoff` rule `latestValue` applies. + const latestUsable = !cutoff || (typeof latest.period === "string" && latest.period <= cutoff); + const orLatest = (v: number | null, key: keyof NichedbFundamentalsData["latest"]): number | null => { + if (v != null) return v; + if (!latestUsable) return null; + const l = latest[key]; + return typeof l === "number" ? l : null; + }; + + const shares = orLatest( + pick("EntityCommonStockSharesOutstanding") ?? pick("CommonStockSharesOutstanding"), + "sharesOutstanding", + ); + const publicFloat = orLatest(pick("EntityPublicFloat"), "publicFloat"); + const revenueConcept = ["RevenueFromContractWithCustomerExcludingAssessedTax", "Revenues", "SalesRevenueNet"].find( + (c) => concepts[c]?.length, + ); + const revenue = orLatest(revenueConcept ? pick(revenueConcept) : null, "revenue"); + const revenueGrowth = revenueConcept ? yoyGrowth(concepts[revenueConcept], cutoff) : undefined; + const cash = orLatest(pick("CashAndCashEquivalentsAtCarryingValue"), "cash"); + const debtParts = [pick("LongTermDebtNoncurrent"), pick("LongTermDebtCurrent")]; + const debt = debtParts.some((d) => d != null) + ? debtParts.reduce((a, b) => a + (b ?? 0), 0) + : orLatest(pick("LongTermDebt"), "longTermDebt"); + const opCashFlow = orLatest(pick("NetCashProvidedByUsedInOperatingActivities"), "operatingCashFlow"); + const runwayMonths = + cash != null && opCashFlow != null && opCashFlow < 0 + ? Math.round((cash / (Math.abs(opCashFlow) / 12)) * 10) / 10 + : undefined; + + const cikRaw = data.cik != null ? String(data.cik).replace(/\D/g, "") : ""; + return { + symbol: sym, + companyName: data.name ? String(data.name) : undefined, + cik: cikRaw ? cikRaw.padStart(10, "0") : undefined, + exchange: data.exchange ? String(data.exchange) : undefined, + sharesOutstanding: shares ?? undefined, + publicFloat: publicFloat ?? undefined, + revenue: revenue ?? undefined, + revenueGrowth, + cashBalance: cash ?? undefined, + totalDebt: debt || undefined, + freeCashFlow: opCashFlow ?? undefined, + runwayMonths, + asOf: nowIso, + source: NICHEDB_SOURCE, + }; +} + +/** Newest point on/before `cutoff` (period end), like sec.ts `latestValue`. */ +function latestValue(points: NichedbFactPoint[] | undefined, cutoff?: string): number | null { + if (!points?.length) return null; + const eligible = points + .filter((p) => p.end && (!cutoff || p.end <= cutoff)) + .sort((a, b) => a.end.localeCompare(b.end)); + const last = eligible.at(-1); + return typeof last?.val === "number" ? last.val : null; +} + +/** Year-over-year growth from the two newest full-year 10-K points, like sec.ts. */ +function yoyGrowth(points: NichedbFactPoint[] | undefined, cutoff?: string): number | undefined { + if (!points?.length) return undefined; + const annual = points + .filter((p) => /^10-K/.test(p.form ?? "") && p.fp === "FY" && p.start && p.end) + .filter((p) => !cutoff || p.end <= cutoff) + .filter((p) => (Date.parse(p.end) - Date.parse(p.start!)) / 86_400_000 >= 300) + .sort((a, b) => a.end.localeCompare(b.end)); + const latest = annual.at(-1); + const prior = annual.at(-2); + if (!latest || !prior || typeof latest.val !== "number" || !prior.val) return undefined; + return Math.round(((latest.val - prior.val) / Math.abs(prior.val)) * 1000) / 10; +} + +/** + * `symbol` items → directory rows. Crypto pairs are skipped: the directory is + * the equity typeahead, and the Alpaca sync it replaces asked for + * `asset_class=us_equity` only. Untradable and OTC names are kept and flagged, + * exactly as the Alpaca rows were. + */ +export function symbolItemsToRows(items: NichedbItem[]): SymbolRow[] { + const rows: SymbolRow[] = []; + for (const item of items) { + if (item.kind !== "symbol") continue; + const d = item.data as unknown as Partial; + if (!d.symbol || !d.name) continue; + if (d.assetClass && d.assetClass !== "us_equity") continue; + rows.push({ + symbol: String(d.symbol).toUpperCase(), + name: String(d.name), + exchange: d.exchange ? String(d.exchange) : undefined, + assetClass: d.assetClass ?? "us_equity", + status: d.status ? String(d.status) : undefined, + tradable: d.tradable !== false && d.status !== "inactive", + source: NICHEDB_SOURCE, + }); + } + return rows; +} + +/** + * Market-news items → the hits the RSS path produces, so they run through the + * same subject check, headline dedupe, tiering and article fetch. The + * publisher is the wire's source name (Benzinga, Reuters...) or the byline; + * the tier comes from the article's own host, exactly as for an RSS hit. + */ +export function newsItemsToHits(items: NichedbItem[]): NewsHit[] { + const hits: NewsHit[] = []; + for (const item of items) { + if (!item.url || !item.title) continue; + const d = (item.data ?? {}) as Partial; + const host = normalizeHost(item.url); + if (!host) continue; + hits.push({ + title: item.title, + url: item.url, + publisher: d.source || d.author || host, + host, + tier: tierFor(item.url), + publishedAt: item.published_at ? item.published_at.slice(0, 10) : undefined, + snippet: item.summary ?? undefined, + }); + } + return hits; +} + +/** + * The reads a report build makes. Each one swallows nichedb errors into a + * `null`/`[]` and reports them through `onMiss`, because the caller has a live + * path to fall back to and a mirror outage must not fail a page that used to + * render without it. + */ +export class NichedbMarkets { + constructor( + readonly client: NichedbClient, + private readonly opts: { now?: () => number; onMiss?: (what: string, why: string) => void } = {}, + ) {} + + private miss(what: string, why: string): void { + this.opts.onMiss?.(what, why); + } + + /** Daily bars for the window, or null when nichedb has none fresh enough. */ + async bars(symbol: string, window: BarsWindow): Promise { + try { + const item = await this.client.historyItem(symbol); + const out = historyToBars(item, window, this.opts.now?.() ?? Date.now()); + if (!out) this.miss("history", item ? "stale or empty window" : "no item"); + return out; + } catch (err) { + this.miss("history", String(err).slice(0, 200)); + return null; + } + } + + /** Company facts, or null when nichedb has no fundamentals for the ticker. */ + async facts(symbol: string, asOf?: string): Promise { + try { + const item = await this.client.fundamentalsItem(symbol); + const out = fundamentalsToFacts(item, symbol, asOf); + if (!out) this.miss("fundamentals", "no item"); + return out; + } catch (err) { + this.miss("fundamentals", String(err).slice(0, 200)); + return null; + } + } + + /** News hits about a ticker from the shared wire, newest first. */ + async newsHits(symbol: string, opts: { sinceDays?: number; limit?: number } = {}): Promise { + try { + return newsItemsToHits(await this.client.newsItems(symbol, opts)); + } catch (err) { + this.miss("news", String(err).slice(0, 200)); + return []; + } + } +} + +/** + * The switch, resolved once at boot. `undefined` when `NICHEDB_MARKETS` is off, + * which is what every call site tests — with it undefined, no code path can + * construct a client, so no request can be made. + */ +export function nichedbMarketsFromEnv( + env: Record = process.env, + opts: { fetch?: typeof fetch; onMiss?: (what: string, why: string) => void } = {}, +): NichedbMarkets | undefined { + if (!nichedbEnabled(env)) return undefined; + return new NichedbMarkets(new NichedbClient({ baseUrl: env.NICHEDB_URL, fetch: opts.fetch }), { onMiss: opts.onMiss }); +} diff --git a/src/providers/nichedb.ts b/src/providers/nichedb.ts new file mode 100644 index 0000000..5f0d6c4 --- /dev/null +++ b/src/providers/nichedb.ts @@ -0,0 +1,317 @@ +/** + * nichedb.dev client — the shared market mirror. + * + * nichedb fetches the US symbol directory (Alpaca assets), daily price history + * (Alpaca bars) and SEC XBRL fundamentals once, for every site, and serves them + * as items in its `markets` collection. This client reads those items; it never + * writes, needs no key, and is rate-limited per IP (~600 requests an hour), so + * every caller here asks for exactly one page and lets the site cache the rest. + * + * The contract (kinds, tags, `data` shapes) is nichedb's docs/markets.md and + * docs/consolidation.md. The shapes are typed here; the mapping into the + * site's own types lives in nichedb-markets.ts so this file stays a wire client. + * + * `fetch` is injectable so the mapping can be tested with fixtures and so the + * switch-off case ("no nichedb request is ever made") is provable. + */ + +export const DEFAULT_NICHEDB_URL = "https://nichedb.dev"; +const USER_AGENT = "advis0r.com/2.0 (research)"; +const TIMEOUT_MS = 15_000; +/** The API caps `limit` at 200 for anonymous reads; a walker asks for the cap. */ +export const PAGE_LIMIT = 200; + +/** One item as `GET /api/v1/items` returns it. */ +export interface NichedbItem { + id: number; + collection: string; + source?: string; + adapter?: string; + kind: string; + external_id: string; + updated_at: string; + title: string | null; + summary: string | null; + url: string | null; + image_url: string | null; + published_at: string | null; + tags: string[]; + data: Record; +} + +/** `data` of a `kind=symbol` item (one per active Alpaca asset). */ +export interface NichedbSymbolData { + assetId: string; + symbol: string; + name: string; + exchange: string; + assetClass: "us_equity" | "crypto"; + status: string; + tradable: boolean; + marginable?: boolean; + shortable?: boolean; + easyToBorrow?: boolean; + fractionable?: boolean; + attributes?: string[]; +} + +/** One bar of a `history` item: day, open, high, low, close, volume, vwap. */ +export type NichedbBarTuple = [string, number, number, number, number, number, number | null]; + +/** `data` of a `kind=history` item (last 400 daily bars, oldest first). */ +export interface NichedbHistoryData { + symbol: string; + timeframe: "1Day"; + feed: "iex" | "sip"; + adjustment: "split"; + bars: NichedbBarTuple[]; + first: string; + last: string; + count: number; +} + +/** One XBRL point in a `fundamentals` item's `concepts`. */ +export interface NichedbFactPoint { + start: string | null; + end: string; + val: number; + fy: number | null; + fp: string | null; + form: string; + filed: string; + unit: string; +} + +/** `data` of a `kind=fundamentals` item (one per CIK). */ +export interface NichedbFundamentalsData { + cik: number | string; + symbol: string; + symbols: string[]; + name: string; + exchange: string | null; + concepts: Record; + latest: { + revenue?: number | null; + grossProfit?: number | null; + operatingIncome?: number | null; + netIncome?: number | null; + epsDiluted?: number | null; + epsBasic?: number | null; + assets?: number | null; + liabilities?: number | null; + equity?: number | null; + cash?: number | null; + operatingCashFlow?: number | null; + sharesOutstanding?: number | null; + publicFloat?: number | null; + longTermDebt?: number | null; + period?: string | null; + fp?: string | null; + fy?: number | null; + form?: string | null; + filed?: string | null; + }; +} + +/** `data` of a market-news item from the `alpaca-news` adapter. */ +export interface NichedbNewsData { + newsId?: number | string; + author: string | null; + source: string | null; + symbols: string[]; + updatedAt?: string | null; +} + +/** + * The kind the alpaca-news adapter emits. The markets doc calls it "news"; the + * adapter (`packages/adapters/src/alpaca.js`, `newsToItem`) writes + * `kind: 'market-news'`, and the kind is also its first tag. + */ +export const NEWS_KIND = "market-news"; + +export interface ItemsQuery { + collection: string; + kind?: string; + /** Every tag named must be on the item. */ + tags?: string[]; + /** `published_at >= from` (ISO). */ + from?: string; + /** `published_at < to` (ISO). */ + to?: string; + /** `updated_at >= since` (ISO) — what a mirror asks. */ + since?: string; + /** Keyset pagination on the item id, pairs with `sort=id&order=asc`. */ + after?: number; + limit?: number; + sort?: "id" | "published" | "updated"; + order?: "asc" | "desc"; +} + +export interface NichedbClientOptions { + /** Base URL; `NICHEDB_URL` in the environment, else https://nichedb.dev. */ + baseUrl?: string; + fetch?: typeof fetch; + timeoutMs?: number; + now?: () => number; +} + +export type MirrorPage = { items: NichedbItem[]; page: number }; + +export class NichedbClient { + readonly baseUrl: string; + private readonly fetchImpl: typeof fetch; + private readonly timeoutMs: number; + private readonly now: () => number; + /** Requests made by this instance, so a caller can budget and a test can count. */ + requests = 0; + + constructor(opts: NichedbClientOptions = {}) { + this.baseUrl = (opts.baseUrl ?? process.env.NICHEDB_URL ?? DEFAULT_NICHEDB_URL).replace(/\/$/, ""); + this.fetchImpl = opts.fetch ?? fetch; + this.timeoutMs = opts.timeoutMs ?? TIMEOUT_MS; + this.now = opts.now ?? Date.now; + } + + /** The `/api/v1/items` URL for a query. Pure, so tests can check exactly what is asked. */ + itemsUrl(q: ItemsQuery): string { + const url = new URL(`${this.baseUrl}/api/v1/items`); + url.searchParams.set("collection", q.collection); + if (q.kind) url.searchParams.set("kind", q.kind); + if (q.tags?.length) url.searchParams.set("tags", q.tags.map(normalizeTag).join(",")); + if (q.from) url.searchParams.set("from", q.from); + if (q.to) url.searchParams.set("to", q.to); + if (q.since) url.searchParams.set("since", q.since); + if (q.after != null) url.searchParams.set("after", String(q.after)); + if (q.sort) url.searchParams.set("sort", q.sort); + if (q.order) url.searchParams.set("order", q.order); + url.searchParams.set("limit", String(Math.min(PAGE_LIMIT, Math.max(1, q.limit ?? 50)))); + return url.toString(); + } + + /** The `/api/v1/match` URL: best items for a name, by trigram similarity on `title`. */ + matchUrl(params: { collection: string; q: string; kind?: string; limit?: number }): string { + const url = new URL(`${this.baseUrl}/api/v1/match`); + url.searchParams.set("collection", params.collection); + url.searchParams.set("q", params.q); + if (params.kind) url.searchParams.set("kind", params.kind); + url.searchParams.set("limit", String(Math.max(1, params.limit ?? 5))); + return url.toString(); + } + + private async getJson(url: string): Promise<{ items?: NichedbItem[] }> { + this.requests += 1; + const res = await this.fetchImpl(url, { + headers: { "User-Agent": USER_AGENT, Accept: "application/json" }, + signal: AbortSignal.timeout(this.timeoutMs), + }); + if (!res.ok) { + throw new Error(`nichedb ${res.status} for ${url.replace(this.baseUrl, "")}`); + } + return (await res.json()) as { items?: NichedbItem[] }; + } + + async items(q: ItemsQuery): Promise { + const body = await this.getJson(this.itemsUrl(q)); + return Array.isArray(body.items) ? body.items : []; + } + + /** + * The `symbol` item for a ticker. + * + * `symbol` items carry no `symbol:` tag (the contract says the directory + * is the lookup), so a single-symbol read goes through `/api/v1/match` on the + * title — `AAPL · Apple Inc. Common Stock` — and is verified against + * `data.symbol`, so a near-miss on a similar ticker is never returned. + */ + async symbolItem(symbol: string): Promise { + const wanted = symbol.toUpperCase(); + const body = await this.getJson(this.matchUrl({ collection: "markets", kind: "symbol", q: wanted, limit: 5 })); + const items = Array.isArray(body.items) ? body.items : []; + return items.find((i) => String((i.data as { symbol?: unknown })?.symbol ?? "").toUpperCase() === wanted) ?? null; + } + + /** The `history` item (last 400 daily bars) for a symbol, or null. */ + async historyItem(symbol: string): Promise { + const items = await this.items({ + collection: "markets", + kind: "history", + tags: [symbolTag(symbol)], + limit: 1, + }); + return items[0] ?? null; + } + + /** + * The `fundamentals` item for a ticker or a CIK. A company that lists several + * tickers is one item tagged with all of them, so either spelling finds it. + */ + async fundamentalsItem(symbolOrCik: string): Promise { + const key = String(symbolOrCik).trim(); + const tag = /^\d+$/.test(key) ? `cik:${Number(key)}` : symbolTag(key); + const items = await this.items({ collection: "markets", kind: "fundamentals", tags: [tag], limit: 1 }); + return items[0] ?? null; + } + + /** + * Market-news items about a symbol, newest first. Stories are tagged with + * their lowercased tickers (`aapl`), and the window is on `published_at`, + * which is what "news from the last N days" means; `updated_at` would let a + * backfill of old stories through. + */ + async newsItems(symbol: string, opts: { sinceDays?: number; limit?: number } = {}): Promise { + const days = opts.sinceDays ?? 90; + const from = new Date(this.now() - days * 86_400_000).toISOString(); + return this.items({ + collection: "markets", + kind: NEWS_KIND, + tags: [symbol.toLowerCase()], + from, + sort: "published", + order: "desc", + limit: opts.limit ?? 50, + }); + } + + /** + * Walk every `symbol` item, a page at a time. + * + * Keyset on the item id (`sort=id&order=asc&after=`), which is stable + * while the walk runs, filtered by `since` on `updated_at` so a daily sync + * after the first only reads what moved. The caller keeps the cursor: the + * newest `updated_at` seen is what to pass as `since` next time. + */ + async *mirrorSymbols(opts: { since?: string; maxPages?: number } = {}): AsyncGenerator { + const maxPages = opts.maxPages ?? 200; + let after: number | undefined; + for (let page = 0; page < maxPages; page++) { + const items = await this.items({ + collection: "markets", + kind: "symbol", + since: opts.since, + after, + sort: "id", + order: "asc", + limit: PAGE_LIMIT, + }); + if (!items.length) return; + yield { items, page }; + if (items.length < PAGE_LIMIT) return; + after = items[items.length - 1]!.id; + } + } +} + +/** `symbol:` tag — lower-case, Alpaca's spelling (`symbol:brk.b`). */ +export function symbolTag(symbol: string): string { + return `symbol:${symbol.trim().toLowerCase()}`; +} + +function normalizeTag(tag: string): string { + return tag.trim().toLowerCase(); +} + +/** The switch. `NICHEDB_MARKETS=1` (or `true`) reads shared market data from nichedb. */ +export function nichedbEnabled(env: Record = process.env): boolean { + const v = (env.NICHEDB_MARKETS ?? "").trim().toLowerCase(); + return v === "1" || v === "true" || v === "on" || v === "yes"; +} diff --git a/src/server.ts b/src/server.ts index 5d07b8f..bf65041 100644 --- a/src/server.ts +++ b/src/server.ts @@ -25,6 +25,7 @@ import { getDb, migrate } from "./db/index.ts"; import { buildRegistry, getAiProvider } from "./registry.ts"; import { analyzeTicker } from "./pipeline/analyze.ts"; import { refreshTickerNews } from "./pipeline/news-refresh.ts"; +import { nichedbMarketsFromEnv } from "./providers/nichedb-markets.ts"; import { calculateIndicators, scoreTechnicalSetup } from "./technical/indicators.ts"; import { buildEvidence } from "./evidence/builder.ts"; import { composeScore, classifyRisk } from "./scoring/score.ts"; @@ -102,6 +103,16 @@ console.log( }`, ); +// Shared market data from nichedb.dev (NICHEDB_MARKETS=1): daily bars and SEC +// fundamentals for a report build, the market-news wire for a news refresh, +// and the symbol directory for `symbols sync`. Every read falls back to the +// live provider it replaced when the mirror has nothing, so a stopped mirror +// costs a log line, not a report. Off, no nichedb request is ever made. +const nichedb = nichedbMarketsFromEnv(process.env, { + onMiss: (what, why) => console.error(`[nichedb] ${what} miss, using live provider: ${why}`), +}); +console.log(`nichedb: ${nichedb ? `on (${nichedb.client.baseUrl})` : "off — set NICHEDB_MARKETS=1 to read shared market data"}`); + const port = Number(process.env.PORT ?? 8080); const PUBLIC_DIR = join(import.meta.dir, "..", "public"); @@ -122,11 +133,22 @@ async function tickerDetail(symbol: string): Promise> { const start = new Date(Date.now() - 400 * 86_400_000).toISOString(); let bars: Awaited> = []; + let barsSource: string | undefined; let snapshot: Awaited>[number] | undefined; let asset: Awaited>[number] | undefined; let marketError: string | undefined; try { - bars = await registry.alpaca.getBars({ symbols: [sym], timeframe: "1Day", start, end: asOf }); + // Bars: the nichedb window first when the switch is on, the live + // Alpaca→Yahoo path when it has no item or one older than five days. + // Snapshots stay live either way — nichedb carries no trades or quotes, so + // the price, its timestamp and the delayed flag are the provider's. + const mirrored = await nichedb?.bars(sym, { start, end: asOf }); + if (mirrored) { + bars = mirrored.bars; + barsSource = `nichedb (${mirrored.feed})`; + } else { + bars = await registry.alpaca.getBars({ symbols: [sym], timeframe: "1Day", start, end: asOf }); + } [snapshot] = await registry.alpaca.getSnapshots([sym]); [asset] = await registry.alpaca.getAssets([sym]); } catch (err) { @@ -140,7 +162,10 @@ async function tickerDetail(symbol: string): Promise> { let facts: Awaited>; let factsError: string | undefined; try { - facts = await registry.fundamentals.getCompanyFacts(sym, asOf); + // The mirrored companyfacts first (same concepts, same derivations, see + // fundamentalsToFacts); live SEC when nichedb has no item for the ticker. + // The filings list is not read here and stays a live SEC call elsewhere. + facts = (await nichedb?.facts(sym, asOf)) ?? (await registry.fundamentals.getCompanyFacts(sym, asOf)); } catch (err) { factsError = String(err).slice(0, 300); // `source` names what produced these facts; "unavailable" is the honest @@ -263,6 +288,8 @@ async function tickerDetail(symbol: string): Promise> { delayed: snapshot?.delayed ?? true, // True per-response provenance: the feed on the snapshot we actually used. marketSource: snapshot?.feed ?? registry.marketSource, + // Where the daily bars came from when it was not the snapshot's provider. + barsSource: barsSource ?? registry.marketSource, marketError, factsError, facts, diff --git a/src/symbols/directory.ts b/src/symbols/directory.ts index ffe3a19..b37dcb5 100644 --- a/src/symbols/directory.ts +++ b/src/symbols/directory.ts @@ -23,7 +23,7 @@ export interface SymbolRow { assetClass?: string; status?: string; tradable?: boolean; - source: "alpaca" | "yahoo"; + source: "alpaca" | "yahoo" | "nichedb"; } export interface SymbolMatch { diff --git a/src/symbols/nichedb-sync.ts b/src/symbols/nichedb-sync.ts new file mode 100644 index 0000000..39df64a --- /dev/null +++ b/src/symbols/nichedb-sync.ts @@ -0,0 +1,82 @@ +/** + * Symbol directory sync from nichedb's `kind=symbol` mirror. + * + * Replaces the Alpaca `/v2/assets` pull when `NICHEDB_MARKETS` is on: the + * first sync walks the whole directory (~14k rows, ~72 pages of 200), every + * later sync passes the stored cursor as `since=` and reads only the rows + * whose `updated_at` moved — between listings that is a page or none. + * + * The cursor is the newest `updated_at` seen on a completed walk, kept in + * `nichedb_cursor` under `symbols.since`. It is only advanced after the walk + * finishes, so a walk cut short by a network error is simply repeated. + * `since` is inclusive, so the boundary row is re-read once; the upsert makes + * that harmless. + */ +import type { Client } from "@libsql/client"; +import type { NichedbClient } from "../providers/nichedb.ts"; +import { symbolItemsToRows } from "../providers/nichedb-markets.ts"; +import { upsertSymbols } from "./directory.ts"; + +export const SYMBOLS_CURSOR_KEY = "symbols.since"; + +export async function readCursor(db: Client, key: string): Promise { + const rs = await db.execute({ sql: "SELECT value FROM nichedb_cursor WHERE key = ?", args: [key] }); + const v = rs.rows[0]?.value; + return v == null ? undefined : String(v); +} + +export async function writeCursor(db: Client, key: string, value: string): Promise { + await db.execute({ + sql: `INSERT INTO nichedb_cursor (key, value, updated_at) VALUES (?, ?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`, + args: [key, value, new Date().toISOString()], + }); +} + +export interface SymbolSyncResult { + /** Rows written to `symbols` (crypto pairs and malformed items excluded). */ + written: number; + /** Items nichedb returned across every page. */ + items: number; + pages: number; + /** The cursor the walk started from, if any. */ + since?: string; + /** The cursor stored for next time. */ + cursor?: string; +} + +/** + * Walk the mirror and upsert each page as it arrives, so a 14k-row first sync + * does not hold every row in memory and a page that lands is kept even if a + * later one fails. Returns what was done; throws only when nichedb itself + * fails, so the caller can fall back to the Alpaca list. + */ +export async function syncSymbolsFromNichedb( + db: Client, + client: NichedbClient, + opts: { onProgress?: (msg: string) => void; upsert?: typeof upsertSymbols } = {}, +): Promise { + const upsert = opts.upsert ?? upsertSymbols; + const since = await readCursor(db, SYMBOLS_CURSOR_KEY); + const result: SymbolSyncResult = { written: 0, items: 0, pages: 0, since }; + let newest = since ?? ""; + + for await (const page of client.mirrorSymbols({ since })) { + result.pages += 1; + result.items += page.items.length; + for (const item of page.items) { + if (item.updated_at && item.updated_at > newest) newest = item.updated_at; + } + const rows = symbolItemsToRows(page.items); + result.written += await upsert(db, rows); + opts.onProgress?.(`page ${page.page + 1}: ${page.items.length} item(s), ${rows.length} row(s)`); + } + + if (newest && newest !== since) { + await writeCursor(db, SYMBOLS_CURSOR_KEY, newest); + result.cursor = newest; + } else { + result.cursor = since; + } + return result; +} diff --git a/test/nichedb.test.ts b/test/nichedb.test.ts new file mode 100644 index 0000000..b91e186 --- /dev/null +++ b/test/nichedb.test.ts @@ -0,0 +1,538 @@ +/** + * nichedb.dev mirror: the client, the mappings into the site's own shapes, the + * symbol walk with its cursor, and the switch. + * + * Everything runs against a fake `fetch` with fixtures in the documented item + * shape (nichedb docs/markets.md), because the `symbol`/`history`/`fundamentals` + * kinds were not deployed when this was written. The test that matters most is + * the last one: with the switch off, no request to nichedb can be made. + */ +import { afterEach, beforeAll, afterAll, describe, expect, test } from "bun:test"; +import { createClient, type Client } from "@libsql/client"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { migrate } from "../src/db/index.ts"; +import { + NEWS_KIND, + NichedbClient, + PAGE_LIMIT, + nichedbEnabled, + symbolTag, + type NichedbItem, +} from "../src/providers/nichedb.ts"; +import { + HISTORY_MAX_AGE_DAYS, + NichedbMarkets, + barTimestamp, + fundamentalsToFacts, + historyToBars, + isFresh, + newsItemsToHits, + nichedbMarketsFromEnv, + symbolItemsToRows, +} from "../src/providers/nichedb-markets.ts"; +import { NewsProvider } from "../src/providers/news/index.ts"; +import { readCursor, syncSymbolsFromNichedb, writeCursor, SYMBOLS_CURSOR_KEY } from "../src/symbols/nichedb-sync.ts"; +import type { SymbolRow } from "../src/symbols/directory.ts"; + +const NOW = Date.parse("2026-09-11T12:00:00Z"); + +/** A fake fetch that answers each URL from `routes` (substring match) and records what was asked. */ +function fakeFetch(routes: Array<[string, unknown]> | ((url: string) => unknown), calls: string[] = []) { + const fetchImpl = (async (input: any, init?: any) => { + const url = String(input); + calls.push(url); + const body = + typeof routes === "function" + ? routes(url) + : (routes.find(([needle]) => url.includes(needle))?.[1] ?? { count: 0, items: [] }); + if (body instanceof Response) return body; + return { + ok: true, + status: 200, + headers: init?.headers, + json: async () => body, + text: async () => JSON.stringify(body), + } as any; + }) as unknown as typeof fetch; + return { fetchImpl, calls }; +} + +const item = (over: Partial): NichedbItem => ({ + id: 1, + collection: "markets", + kind: "history", + external_id: "x", + updated_at: "2026-09-10T00:20:00.000Z", + title: null, + summary: null, + url: null, + image_url: null, + published_at: null, + tags: [], + data: {}, + ...over, +}); + +const HISTORY = item({ + id: 10, + kind: "history", + external_id: "history:AAPL", + published_at: "2026-09-10T00:00:00.000Z", + tags: ["history", "symbol:aapl", "feed:iex"], + data: { + symbol: "AAPL", + timeframe: "1Day", + feed: "iex", + adjustment: "split", + bars: [ + ["2026-09-03", 100, 101, 99, 100.5, 1000, 100.2], + ["2026-09-04", 100.5, 102, 100, 101.5, 1100, null], + ["2026-09-08", 101.5, 103, 101, 102.5, 1200, 102.1], + ["2026-09-09", 102.5, 104, 102, 103.5, 1300, 103.2], + ["2026-09-10", 103.5, 105, 103, 104.5, 1400, 104.1], + ], + first: "2026-09-03", + last: "2026-09-10", + count: 5, + }, +}); + +const FY = (end: string, val: number, start?: string) => ({ + start: start ?? `${Number(end.slice(0, 4)) - 1}-${end.slice(5)}`, + end, val, fy: Number(end.slice(0, 4)), fp: "FY", form: "10-K", filed: `${Number(end.slice(0, 4)) + 1}-02-01`, unit: "USD", +}); +const Q = (end: string, val: number) => ({ + start: null, end, val, fy: Number(end.slice(0, 4)), fp: "Q2", form: "10-Q", filed: end, unit: "USD", +}); + +const FUNDAMENTALS = item({ + id: 20, + kind: "fundamentals", + external_id: "sec:facts:320193", + published_at: "2026-08-01T00:00:00.000Z", + tags: ["fundamentals", "symbol:aapl", "cik:320193", "exchange:nasdaq"], + data: { + cik: 320193, + symbol: "AAPL", + symbols: ["AAPL"], + name: "Apple Inc.", + exchange: "Nasdaq", + concepts: { + RevenueFromContractWithCustomerExcludingAssessedTax: [FY("2024-09-28", 391_000), FY("2025-09-27", 416_000)], + Revenues: [FY("2025-09-27", 1)], + CashAndCashEquivalentsAtCarryingValue: [Q("2025-06-28", 28_000), Q("2026-06-27", 30_000)], + LongTermDebtNoncurrent: [Q("2026-06-27", 80_000)], + LongTermDebtCurrent: [Q("2026-06-27", 10_000)], + LongTermDebt: [Q("2026-06-27", 999_999)], + NetCashProvidedByUsedInOperatingActivities: [Q("2026-06-27", -12_000)], + EntityCommonStockSharesOutstanding: [Q("2026-06-27", 15_000)], + CommonStockSharesOutstanding: [Q("2026-06-27", 14_000)], + EntityPublicFloat: [Q("2025-03-28", 3_000_000)], + }, + latest: { + revenue: 416_000, cash: 30_000, operatingCashFlow: -12_000, sharesOutstanding: 15_000, + publicFloat: 3_000_000, longTermDebt: 90_000, period: "2026-06-27", fp: "Q3", fy: 2026, + form: "10-Q", filed: "2026-08-01", + }, + }, +}); + +const symbolItem = (id: number, symbol: string, over: Record = {}, updated = "2026-09-10T06:00:00.000Z") => + item({ + id, + kind: "symbol", + external_id: `alpaca:asset:${id}`, + updated_at: updated, + title: `${symbol} · ${symbol} Inc.`, + tags: ["symbol", "exchange:nasdaq", "class:us_equity", "tradable"], + data: { + assetId: String(id), symbol, name: `${symbol} Inc.`, exchange: "NASDAQ", assetClass: "us_equity", + status: "active", tradable: true, fractionable: true, attributes: [], ...over, + }, + }); + +const NEWS = [ + item({ + id: 30, kind: NEWS_KIND, external_id: "alpaca-news-1", title: "AAPL beats on iPhone", + summary: "Apple reported...", url: "https://www.benzinga.com/news/1", published_at: "2026-09-09T13:05:00.000Z", + tags: ["market-news", "benzinga", "aapl"], data: { source: "benzinga", author: "A. Writer", symbols: ["AAPL"] }, + }), + item({ + id: 31, kind: NEWS_KIND, external_id: "alpaca-news-2", title: "AAPL supplier note", + summary: null, url: "https://www.reuters.com/x", published_at: "2026-09-08T09:00:00.000Z", + tags: ["market-news", "aapl"], data: { source: null, author: "R. Byline", symbols: ["AAPL"] }, + }), + item({ id: 32, kind: NEWS_KIND, external_id: "alpaca-news-3", title: "no url", url: null, tags: ["aapl"], data: {} }), +]; + +describe("client: URLs and tags", () => { + const client = new NichedbClient({ baseUrl: "https://mirror.example/", fetch: fakeFetch([]).fetchImpl, now: () => NOW }); + + test("history is one item by its symbol tag, lower-cased in Alpaca's spelling", () => { + const u = new URL(client.itemsUrl({ collection: "markets", kind: "history", tags: [symbolTag("BRK.B")], limit: 1 })); + expect(u.origin).toBe("https://mirror.example"); + expect(u.pathname).toBe("/api/v1/items"); + expect(u.searchParams.get("collection")).toBe("markets"); + expect(u.searchParams.get("kind")).toBe("history"); + expect(u.searchParams.get("tags")).toBe("symbol:brk.b"); + expect(u.searchParams.get("limit")).toBe("1"); + }); + + test("fundamentals accepts a ticker or a plain CIK", async () => { + const { fetchImpl, calls } = fakeFetch([]); + const c = new NichedbClient({ baseUrl: "https://mirror.example", fetch: fetchImpl }); + await c.fundamentalsItem("aapl"); + await c.fundamentalsItem("0000320193"); + expect(new URL(calls[0]!).searchParams.get("tags")).toBe("symbol:aapl"); + expect(new URL(calls[1]!).searchParams.get("tags")).toBe("cik:320193"); + expect(new URL(calls[1]!).searchParams.get("kind")).toBe("fundamentals"); + }); + + test("news is the market-news kind, tagged with the bare lower-case symbol, windowed on published_at", async () => { + const { fetchImpl, calls } = fakeFetch([]); + const c = new NichedbClient({ baseUrl: "https://mirror.example", fetch: fetchImpl, now: () => NOW }); + await c.newsItems("AAPL", { sinceDays: 90 }); + const u = new URL(calls[0]!); + expect(u.searchParams.get("kind")).toBe("market-news"); + expect(u.searchParams.get("tags")).toBe("aapl"); + expect(u.searchParams.get("from")).toBe(new Date(NOW - 90 * 86_400_000).toISOString()); + expect(u.searchParams.get("since")).toBeNull(); + expect(u.searchParams.get("sort")).toBe("published"); + expect(u.searchParams.get("order")).toBe("desc"); + }); + + test("limit is capped at the API's 200 and tags are normalized", () => { + const u = new URL(client.itemsUrl({ collection: "markets", tags: [" Symbol:AAPL "], limit: 5000 })); + expect(u.searchParams.get("limit")).toBe(String(PAGE_LIMIT)); + expect(u.searchParams.get("tags")).toBe("symbol:aapl"); + }); + + test("sends the research User-Agent and a timeout signal", async () => { + let seen: any; + const fetchImpl = (async (_u: any, init: any) => { + seen = init; + return { ok: true, status: 200, json: async () => ({ items: [] }) } as any; + }) as unknown as typeof fetch; + await new NichedbClient({ fetch: fetchImpl }).historyItem("AAPL"); + expect(seen.headers["User-Agent"]).toBe("advis0r.com/2.0 (research)"); + expect(seen.signal).toBeInstanceOf(AbortSignal); + }); + + test("base URL comes from NICHEDB_URL when not given, default nichedb.dev", () => { + const prev = process.env.NICHEDB_URL; + delete process.env.NICHEDB_URL; + expect(new NichedbClient().baseUrl).toBe("https://nichedb.dev"); + process.env.NICHEDB_URL = "https://staging.example/"; + expect(new NichedbClient().baseUrl).toBe("https://staging.example"); + if (prev === undefined) delete process.env.NICHEDB_URL; + else process.env.NICHEDB_URL = prev; + }); + + test("a non-2xx answer throws rather than returning an empty page", async () => { + const fetchImpl = (async () => ({ ok: false, status: 503, json: async () => ({}) })) as unknown as typeof fetch; + await expect(new NichedbClient({ fetch: fetchImpl }).historyItem("AAPL")).rejects.toThrow(/503/); + }); + + test("symbolItem goes through /api/v1/match and verifies data.symbol", async () => { + const { fetchImpl, calls } = fakeFetch([ + ["/api/v1/match", { items: [symbolItem(2, "AAPLW"), symbolItem(1, "AAPL")] }], + ]); + const c = new NichedbClient({ baseUrl: "https://mirror.example", fetch: fetchImpl }); + const hit = await c.symbolItem("aapl"); + expect(hit?.id).toBe(1); + const u = new URL(calls[0]!); + expect(u.pathname).toBe("/api/v1/match"); + expect(u.searchParams.get("kind")).toBe("symbol"); + expect(u.searchParams.get("q")).toBe("AAPL"); + expect(await c.symbolItem("ZZZZ")).toBeNull(); + }); +}); + +describe("history item → MarketBar[]", () => { + test("maps the seven-tuple into the Alpaca bar shape, oldest first", () => { + const out = historyToBars(HISTORY, {}, NOW)!; + expect(out.feed).toBe("iex"); + expect(out.delayed).toBe(true); + expect(out.last).toBe("2026-09-10"); + expect(out.bars).toHaveLength(5); + const first = out.bars[0]!; + expect(first).toEqual({ + symbol: "AAPL", timestamp: barTimestamp("2026-09-03"), open: 100, high: 101, low: 99, close: 100.5, + volume: 1000, vwap: 100.2, timeframe: "1Day", adjustment: "split", + }); + // A null vwap is absent, as the Alpaca mapper leaves an unsent `vw`. + expect(out.bars[1]!.vwap).toBeUndefined(); + // Midnight Eastern, so the report page's slice(0, 10) and the digest's + // etDate() both read the bar's own day. + expect(first.timestamp).toBe("2026-09-03T05:00:00Z"); + }); + + test("cuts the window to the days asked for", () => { + const out = historyToBars(HISTORY, { start: "2026-09-04T00:00:00Z", end: "2026-09-09T23:00:00Z" }, NOW)!; + expect(out.bars.map((b) => b.timestamp.slice(0, 10))).toEqual(["2026-09-04", "2026-09-08", "2026-09-09"]); + }); + + test("a SIP feed is not delayed", () => { + const sip = item({ ...HISTORY, data: { ...HISTORY.data, feed: "sip" } }); + expect(historyToBars(sip, {}, NOW)!.delayed).toBe(false); + }); + + test("a window whose last bar is older than five days is refused (live fallback)", () => { + const sixDays = NOW + 6 * 86_400_000; + expect(historyToBars(HISTORY, {}, sixDays)).toBeNull(); + const fiveDays = Date.parse("2026-09-10T00:00:00Z") + HISTORY_MAX_AGE_DAYS * 86_400_000; + expect(historyToBars(HISTORY, {}, fiveDays)).not.toBeNull(); + expect(isFresh("2026-09-10", fiveDays + 1)).toBe(false); + expect(isFresh("not a date", NOW)).toBe(false); + }); + + test("no item, the wrong kind, or an empty window all mean fallback", () => { + expect(historyToBars(null, {}, NOW)).toBeNull(); + expect(historyToBars(item({ kind: "symbol", data: HISTORY.data }), {}, NOW)).toBeNull(); + expect(historyToBars(item({ data: { ...HISTORY.data, bars: [] } }), {}, NOW)).toBeNull(); + // Fresh item but the requested window is entirely before its first bar. + expect(historyToBars(HISTORY, { end: "2026-01-01" }, NOW)).toBeNull(); + }); + + test("NichedbMarkets.bars swallows a mirror error into a fallback", async () => { + const misses: string[] = []; + const fetchImpl = (async () => { throw new Error("offline"); }) as unknown as typeof fetch; + const m = new NichedbMarkets(new NichedbClient({ fetch: fetchImpl }), { onMiss: (w, why) => misses.push(`${w}: ${why}`) }); + expect(await m.bars("AAPL", {})).toBeNull(); + expect(misses[0]).toMatch(/^history: .*offline/); + }); +}); + +describe("fundamentals item → CompanyFacts", () => { + test("derives the same figures the SEC provider does, from the same concepts", () => { + const facts = fundamentalsToFacts(FUNDAMENTALS, "aapl", "2026-09-11T12:00:00Z")!; + expect(facts.symbol).toBe("AAPL"); + expect(facts.source).toBe("nichedb"); + expect(facts.companyName).toBe("Apple Inc."); + expect(facts.cik).toBe("0000320193"); + expect(facts.exchange).toBe("Nasdaq"); + // Revenue: RevenueFromContract... wins over Revenues, like sec.ts. + expect(facts.revenue).toBe(416_000); + expect(facts.revenueGrowth).toBe(6.4); + // Shares from the cover page (dei), not the balance sheet. + expect(facts.sharesOutstanding).toBe(15_000); + expect(facts.publicFloat).toBe(3_000_000); + expect(facts.cashBalance).toBe(30_000); + // Debt is current + non-current; the LongTermDebt total is only a fallback. + expect(facts.totalDebt).toBe(90_000); + expect(facts.freeCashFlow).toBe(-12_000); + // cash / (burn / 12) = 30000 / 1000 = 30 months. + expect(facts.runwayMonths).toBe(30); + expect(facts.asOf).toBe("2026-09-11T12:00:00Z"); + }); + + test("asOf is point-in-time: a past cutoff reads what was known then", () => { + const facts = fundamentalsToFacts(FUNDAMENTALS, "AAPL", "2025-08-01T00:00:00Z")!; + expect(facts.cashBalance).toBe(28_000); + expect(facts.revenue).toBe(391_000); + // Only one full year on/before the cutoff: no growth figure. + expect(facts.revenueGrowth).toBeUndefined(); + // Nothing filed by then: absent, never the item's current headline. + expect(facts.totalDebt).toBeUndefined(); + expect(facts.sharesOutstanding).toBeUndefined(); + }); + + test("falls back to the item's `latest` when concepts are missing", () => { + const thin = item({ kind: "fundamentals", data: { cik: "1", symbol: "X", name: "X Co", latest: { revenue: 5, cash: 7 } } }); + const facts = fundamentalsToFacts(thin, "X")!; + expect(facts.revenue).toBe(5); + expect(facts.cashBalance).toBe(7); + expect(facts.cik).toBe("0000000001"); + }); + + test("no item or the wrong kind means the live SEC path", () => { + expect(fundamentalsToFacts(null, "AAPL")).toBeNull(); + expect(fundamentalsToFacts(HISTORY, "AAPL")).toBeNull(); + }); +}); + +describe("news items → hits for the news pipeline", () => { + test("maps publisher, host, tier, date and snippet like an RSS hit", () => { + const hits = newsItemsToHits(NEWS); + expect(hits).toHaveLength(2); + expect(hits[0]).toEqual({ + title: "AAPL beats on iPhone", + url: "https://www.benzinga.com/news/1", + publisher: "benzinga", + host: "benzinga.com", + tier: 2, + publishedAt: "2026-09-09", + snippet: "Apple reported...", + }); + // No source name: the byline; tier from the article host. + expect(hits[1]!.publisher).toBe("R. Byline"); + expect(hits[1]!.tier).toBe(1); + expect(hits[1]!.snippet).toBeUndefined(); + }); + + test("hits go through NewsProvider first and count against the per-ticker cap", async () => { + const { fetchImpl } = fakeFetch([["/api/v1/items", { count: 2, items: NEWS }]]); + const markets = new NichedbMarkets(new NichedbClient({ fetch: fetchImpl, now: () => NOW })); + const rssCalls: string[] = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (async (input: any) => { + rssCalls.push(String(input)); + return new Response("", { status: 200, headers: { "content-type": "application/rss+xml" } }); + }) as any; + try { + const provider = new NewsProvider({ + downloadsDir: "/tmp", + perTicker: 1, + discover: (t) => markets.newsHits(t, { sinceDays: 90 }), + }); + const docs = await provider.search({ topic: "news", tickers: ["AAPL"], from: "2026-06-13" }); + expect(docs).toHaveLength(1); + expect(docs[0]!.url).toBe("https://www.benzinga.com/news/1"); + expect(docs[0]!.publisher).toBe("benzinga"); + expect(docs[0]!.sourceTier).toBe(2); + expect(docs[0]!.publishedAt).toBe("2026-09-09"); + expect(docs[0]!.tickers).toEqual(["AAPL"]); + // The RSS feeds still ran after the wire. + expect(rssCalls.some((u) => u.includes("feeds.finance.yahoo.com"))).toBe(true); + expect(rssCalls.some((u) => u.includes("nichedb"))).toBe(false); + } finally { + globalThis.fetch = realFetch; + } + }); +}); + +describe("symbol directory walk with a stored cursor", () => { + const dir = mkdtempSync(join(tmpdir(), "advis0r-nichedb-")); + let db: Client; + beforeAll(async () => { + db = createClient({ url: `file:${join(dir, "cursor.sqlite")}` }); + await migrate(db); + }); + afterAll(() => { + db?.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + test("symbol items become directory rows; crypto pairs are left out", () => { + const rows = symbolItemsToRows([ + symbolItem(1, "AAPL"), + symbolItem(2, "BTC/USD", { assetClass: "crypto", exchange: "CRYPTO" }), + symbolItem(3, "DEAD", { tradable: false, status: "inactive", exchange: "OTC" }), + item({ kind: "symbol", data: { symbol: "NONAME" } }), + ]); + expect(rows.map((r) => r.symbol)).toEqual(["AAPL", "DEAD"]); + expect(rows[0]).toEqual({ + symbol: "AAPL", name: "AAPL Inc.", exchange: "NASDAQ", assetClass: "us_equity", status: "active", + tradable: true, source: "nichedb", + }); + expect(rows[1]!.tradable).toBe(false); + }); + + test("a first sync walks every page by id and stores the newest updated_at", async () => { + const page1 = Array.from({ length: PAGE_LIMIT }, (_, i) => symbolItem(1000 + i, `S${i}`, {}, "2026-09-10T06:00:00.000Z")); + const page2 = [symbolItem(5000, "LAST", {}, "2026-09-10T06:30:00.000Z"), symbolItem(5001, "BTC/USD", { assetClass: "crypto" })]; + const { fetchImpl, calls } = fakeFetch((url) => { + const after = new URL(url).searchParams.get("after"); + if (!after) return { count: page1.length, items: page1 }; + if (after === "1199") return { count: page2.length, items: page2 }; + return { count: 0, items: [] }; + }); + const written: SymbolRow[][] = []; + const client = new NichedbClient({ baseUrl: "https://mirror.example", fetch: fetchImpl }); + const r = await syncSymbolsFromNichedb(db, client, { + upsert: async (_db, rows) => { written.push(rows); return rows.length; }, + }); + expect(r.pages).toBe(2); + expect(r.items).toBe(PAGE_LIMIT + 2); + expect(r.written).toBe(PAGE_LIMIT + 1); + expect(r.since).toBeUndefined(); + expect(r.cursor).toBe("2026-09-10T06:30:00.000Z"); + expect(await readCursor(db, SYMBOLS_CURSOR_KEY)).toBe("2026-09-10T06:30:00.000Z"); + // Page one had no cursor and no keyset; page two continued after its last id. + const u1 = new URL(calls[0]!); + expect(u1.searchParams.get("kind")).toBe("symbol"); + expect(u1.searchParams.get("sort")).toBe("id"); + expect(u1.searchParams.get("order")).toBe("asc"); + expect(u1.searchParams.get("limit")).toBe("200"); + expect(u1.searchParams.get("since")).toBeNull(); + expect(u1.searchParams.get("after")).toBeNull(); + expect(new URL(calls[1]!).searchParams.get("after")).toBe("1199"); + // A short page ends the walk without asking for an empty third page. + expect(calls).toHaveLength(2); + }); + + test("the next sync passes the cursor as since= and keeps it when nothing moved", async () => { + const { fetchImpl, calls } = fakeFetch([["since=", { count: 0, items: [] }]]); + const client = new NichedbClient({ baseUrl: "https://mirror.example", fetch: fetchImpl }); + const r = await syncSymbolsFromNichedb(db, client, { upsert: async () => 0 }); + expect(new URL(calls[0]!).searchParams.get("since")).toBe("2026-09-10T06:30:00.000Z"); + expect(r.pages).toBe(0); + expect(r.cursor).toBe("2026-09-10T06:30:00.000Z"); + }); + + test("a delta advances the cursor to the newest row seen", async () => { + const moved = [symbolItem(7, "MOVED", {}, "2026-09-11T06:00:00.000Z")]; + const { fetchImpl } = fakeFetch([["since=", { count: 1, items: moved }]]); + const client = new NichedbClient({ baseUrl: "https://mirror.example", fetch: fetchImpl }); + const r = await syncSymbolsFromNichedb(db, client, { upsert: async (_db, rows) => rows.length }); + expect(r.written).toBe(1); + expect(r.cursor).toBe("2026-09-11T06:00:00.000Z"); + expect(await readCursor(db, SYMBOLS_CURSOR_KEY)).toBe("2026-09-11T06:00:00.000Z"); + }); + + test("a failed walk leaves the cursor where it was, so it is repeated", async () => { + await writeCursor(db, SYMBOLS_CURSOR_KEY, "2026-09-11T06:00:00.000Z"); + const fetchImpl = (async () => ({ ok: false, status: 502, json: async () => ({}) })) as unknown as typeof fetch; + await expect(syncSymbolsFromNichedb(db, new NichedbClient({ fetch: fetchImpl }), { upsert: async () => 0 })).rejects.toThrow(/502/); + expect(await readCursor(db, SYMBOLS_CURSOR_KEY)).toBe("2026-09-11T06:00:00.000Z"); + }); +}); + +describe("the switch", () => { + const realFetch = globalThis.fetch; + afterEach(() => { globalThis.fetch = realFetch; }); + + test("NICHEDB_MARKETS must be set to 1 (or true)", () => { + expect(nichedbEnabled({})).toBe(false); + expect(nichedbEnabled({ NICHEDB_MARKETS: "0" })).toBe(false); + expect(nichedbEnabled({ NICHEDB_MARKETS: "" })).toBe(false); + expect(nichedbEnabled({ NICHEDB_MARKETS: "1" })).toBe(true); + expect(nichedbEnabled({ NICHEDB_MARKETS: "true" })).toBe(true); + expect(nichedbMarketsFromEnv({})).toBeUndefined(); + expect(nichedbMarketsFromEnv({ NICHEDB_MARKETS: "off" })).toBeUndefined(); + expect(nichedbMarketsFromEnv({ NICHEDB_MARKETS: "1", NICHEDB_URL: "https://m.example" })?.client.baseUrl).toBe("https://m.example"); + }); + + test("off, a news search reaches every feed but never nichedb", async () => { + const calls: string[] = []; + globalThis.fetch = (async (input: any) => { + calls.push(String(input)); + return new Response("", { status: 200 }); + }) as any; + const markets = nichedbMarketsFromEnv({ NICHEDB_MARKETS: undefined }); + const provider = new NewsProvider({ + downloadsDir: "/tmp", + perTicker: 8, + discover: markets ? (t) => markets.newsHits(t) : undefined, + }); + await provider.search({ topic: "news", tickers: ["AAPL"] }); + expect(calls.length).toBeGreaterThan(0); + expect(calls.filter((u) => u.includes("nichedb"))).toEqual([]); + }); + + test("on, the reads a report build makes are one request each", async () => { + const { fetchImpl, calls } = fakeFetch([ + ["kind=history", { count: 1, items: [HISTORY] }], + ["kind=fundamentals", { count: 1, items: [FUNDAMENTALS] }], + ]); + const markets = nichedbMarketsFromEnv({ NICHEDB_MARKETS: "1" }, { fetch: fetchImpl }); + const bars = await markets!.bars("AAPL", { start: "2026-09-01", end: "2026-09-11" }); + const facts = await markets!.facts("AAPL", "2026-09-11T00:00:00Z"); + expect(bars?.bars.length).toBe(5); + expect(facts?.revenue).toBe(416_000); + expect(calls).toHaveLength(2); + expect(markets!.client.requests).toBe(2); + }); +});