diff --git a/README.md b/README.md index 070259a..951f970 100644 --- a/README.md +++ b/README.md @@ -364,6 +364,44 @@ above Apple Hospitality REIT, and KO above Coca-Cola Consolidated. | `symbols find ` | Look up a ticker by name or symbol | | `symbols status` | Directory size and freshness | +## Search: transcripts, the web, and a pasted link + +The Search tab takes one box and works out what you meant. + +**Paste a URL** and the page is read back to you: title, publisher and source +tier, author, date, the phrases it keeps using, the tickers it names, and any +feeds it advertises. Retrieval is the same code the news pipeline uses, so the +same rules apply — `robots.txt` is honoured, a publisher that blocks automated +access is reported as blocked, and no body text is ever invented. + +**Type a phrase** and *Auto* searches the indexed transcripts and the open web +together; *News* adds dates and publishers. Alongside the results you get the +things a list of links does not tell you: Google's **related searches**, the +**recurring phrases** shared across the titles that came back, and the +**niches** those results cluster into. Every one of them is clickable, so +following a thread never means retyping it. + +```bash +curl "localhost:8080/api/web?q=ai+infrastructure&kind=web" +curl "localhost:8080/api/web?q=SoundHound&kind=news&time=last_week" +curl "localhost:8080/api/parse?url=https://example.com/story" +``` + +Phrases and niches are computed by **document frequency** — a phrase scores by +how many separate results contain it, so one long article repeating its own +keyword twelve times cannot invent a trend. No LLM is involved; the whole thing +is deterministic and free to run. + +Two costs are worth knowing about. Web and news search is **ValueSERP**, one +credit per page out of a monthly bucket shared with other properties, so +identical queries are cached and each IP is throttled; without +`VALUESERP_API_KEY` the tab still searches transcripts and still parses a URL, +and `/api/web` answers 503 rather than pretending. And `/api/parse` fetches a +URL chosen by an anonymous caller, so the target is checked first: http(s) +only, standard ports only, and the hostname must resolve exclusively to public +addresses — a name pointing at `169.254.169.254` or `10.0.0.5` is refused +before any request is made. + ## Report pages Every ticker that has been looked at has a report at **`/ticker/`** — a diff --git a/public/app.js b/public/app.js index 0b589b4..bb1c2e9 100644 --- a/public/app.js +++ b/public/app.js @@ -7,7 +7,16 @@ const esc = (s) => async function api(path) { const res = await fetch(path); - if (!res.ok) throw new Error(`${res.status}`); + if (!res.ok) { + // The API explains its refusals — out of credits, throttled, not + // configured. Throwing the bare status threw that explanation away and + // every caller rendered "failed (503)". + let detail = ""; + try { detail = (await res.json())?.error || ""; } catch { /* not JSON */ } + const err = new Error(detail || `${res.status}`); + err.status = res.status; + throw err; + } return res.json(); } @@ -378,24 +387,283 @@ $("#wl-topic").addEventListener("keydown", (e) => e.key === "Enter" && runWatchl // Picking a suggestion from the datalist fires an 'input' change → run it. $("#wl-topic").addEventListener("change", () => runWatchlist()); -/* ---- Search ---- */ +/* ---- Search ---- + One box, three jobs, and what you type decides which: + + - a URL -> fetch that page and describe it + - a phrase -> indexed transcripts and the open web, side by side + - either, with a mode picked -> just the one you asked for + + Web and news come from a metered search API. The server caches identical + queries and throttles per IP, so the only thing this side has to get right + is saying plainly when a budget — not a lack of results — is the reason the + page is empty. */ + +/** + * Mirror of the server's `looksLikeUrl`. Auto has to decide which endpoint to + * call before it calls one, so the test lives on both sides; keep them in step + * (src/research/page.ts). + */ +function looksLikeUrl(input) { + const s = String(input || "").trim(); + if (!s || /\s/.test(s)) return false; + if (/^https?:\/\//i.test(s)) return true; + if (/^[a-z][a-z0-9+.-]*:/i.test(s)) return false; + return /^[\w-]+(\.[\w-]+)*\.[a-z]{2,24}(:\d+)?([/?#]|$)/i.test(s); +} + +/** A titled block. Returns "" for empty content so callers can just concatenate. */ +function sect(title, inner, note) { + if (!inner) return ""; + const sub = note ? `${esc(note)}` : ""; + return `

${esc(title)}${sub}

${inner}
`; +} + +/** Clickable phrases. Accepts plain strings or {phrase, count}. */ +function phraseChips(list) { + if (!list || !list.length) return ""; + return `
${list + .map((p) => { + const label = typeof p === "string" ? p : p.phrase; + const count = typeof p === "string" ? 0 : p.count; + const n = count > 1 ? `${count}` : ""; + return ``; + }) + .join("")}
`; +} + +function questionList(questions) { + if (!questions || !questions.length) return ""; + return `
${questions + .slice(0, 8) + .map((q) => ``) + .join("")}
`; +} + +/** A niche is a recurring phrase plus who is writing about it. */ +function nicheList(niches) { + if (!niches || !niches.length) return ""; + return `
${niches + .map((n) => { + const shown = (n.hosts || []).slice(0, 3).join(", "); + const more = (n.hosts || []).length > 3 ? ` +${n.hosts.length - 3}` : ""; + return ``; + }) + .join("")}
`; +} + +function sourceChips(sources) { + if (!sources || !sources.length) return ""; + return `
${sources + .map( + (s) => + `${esc(s.host)}${s.count}`, + ) + .join("")}
`; +} + +function webResultRows(results) { + return results + .map( + (r) => `
+
${esc(r.publisher || r.host)}${r.publishedAt ? `${esc(String(r.publishedAt).slice(0, 10))}` : ""}${esc(r.tierLabel || "")}
+ ${esc(r.title || r.url)} + ${r.snippet ? `
${esc(r.snippet)}
` : ""} +
open ↗
+
`, + ) + .join(""); +} + +function webHtml(data) { + const label = data.kind === "news" ? "News" : "Web"; + const results = data.results || []; + if (!results.length && !(data.trending || []).length) { + return sect(label, `
No ${label.toLowerCase()} results for “${esc(data.query)}”.
`); + } + // Credits are shown because this is the one search on the site that costs + // money; a reader watching it fall knows why it might stop working. + const note = [ + `${results.length} result${results.length === 1 ? "" : "s"}`, + data.cached ? "cached" : null, + Number.isFinite(data.creditsRemaining) ? `${Number(data.creditsRemaining).toLocaleString()} credits left` : null, + ] + .filter(Boolean) + .join(" · "); + + return [ + sect("Trending searches", phraseChips(data.trending || []), "what else is searched around this"), + sect("Recurring phrases", phraseChips(data.phrases || []), "language shared across these titles"), + sect("Niches", nicheList(data.niches || [])), + sect(label, `
${webResultRows(results)}
`, note), + sect("Questions", questionList(data.questions || [])), + sect("Publishers", sourceChips(data.sources || [])), + ].join(""); +} + +function parsedPageHtml(p) { + const m = p.meta || {}; + const line = [ + m.siteName || p.host, + m.author, + m.publishedAt ? String(m.publishedAt).slice(0, 10) : null, + p.wordCount ? `${Number(p.wordCount).toLocaleString()} words` : null, + ].filter(Boolean); + + const tickers = (p.tickers || []).length + ? sect( + "Tickers named", + // Real links, not buttons: a ticker found in an article is worth + // copying, opening in a tab, and sharing. The document-level `.tlink` + // handler still intercepts a plain click. + `
${p.tickers + .map( + (t) => + `${esc(t.symbol)}${t.name ? ` · ${esc(t.name)}` : ""}${t.hasReport ? " ✓" : ""}`, + ) + .join("")}
`, + "✓ has a stored report", + ) + : ""; + + const feeds = (m.feeds || []).length + ? sect( + "Feeds", + `
${m.feeds + .map((f) => `${esc(f)}`) + .join("")}
`, + ) + : ""; + + const blocked = p.blockedReason + ? `
Body not read: ${esc(p.blockedReason)}. Headline and publisher metadata only — nothing here is invented.
` + : ""; + + const card = `
+
${esc(p.host)}${esc(p.tierLabel || "")}${p.cached ? "cached" : ""}
+ ${esc(m.title || p.url)} + ${line.length ? `
${line.map((b) => `${esc(b)}`).join("")}
` : ""} + ${m.description ? `
${esc(m.description)}
` : ""} + ${p.excerpt ? `
${esc(p.excerpt)}
` : ""} + ${blocked} +
open original ↗
+
`; + + return [ + sect("Page", card), + tickers, + sect("Recurring phrases", phraseChips(p.phrases || []), "the page in its own words"), + (m.keywords || []).length ? sect("Publisher keywords", phraseChips(m.keywords), "declared by the site") : "", + (m.headings || []).length + ? sect( + "Outline", + `
${m.headings.slice(0, 12).map((h) => `
${esc(h)}
`).join("")}
`, + ) + : "", + feeds, + ].join(""); +} + +async function transcriptsSection(q, sole) { + try { + const data = await api(`/api/search?q=${encodeURIComponent(q)}&limit=30`); + const rows = data.results || []; + if (!rows.length) { + return sole ? `
No transcript matches for “${esc(q)}”.
` : ""; + } + const inner = rows + .map( + (x) => + `
${esc(x.ticker || "?")}${esc(x.event_date || "")}${esc(x.speaker || "")}
${esc((x.text || "").slice(0, 320))}…
`, + ) + .join(""); + return sect("Transcripts", `
${inner}
`, `${rows.length} segment${rows.length === 1 ? "" : "s"}`); + } catch (e) { + return sect("Transcripts", `
Transcript search failed (${esc(e.message)}).
`); + } +} + +async function webSection(q, kind, time) { + const label = kind === "news" ? "News" : "Web"; + try { + const params = new URLSearchParams({ q, kind }); + if (time) params.set("time", time); + return webHtml(await api(`/api/web?${params}`)); + } catch (e) { + const why = + e.status === 402 + ? "Web search credits are exhausted for this month." + : e.status === 503 + ? "Web search is not configured on this deployment." + : e.status === 429 + ? e.message + : `${label} search failed (${esc(e.message)}).`; + return sect(label, `
${esc(why)}
`); + } +} + +async function renderParse(input) { + const out = $("#search-results"); + out.innerHTML = ``; + try { + out.innerHTML = parsedPageHtml(await api(`/api/parse?url=${encodeURIComponent(input)}`)); + } catch (e) { + out.innerHTML = `
${esc(e.message || "That page could not be read.")}
`; + } +} + async function runSearch() { const q = $("#sq").value.trim(); const out = $("#search-results"); if (!q) return; + const mode = $("#sq-mode") ? $("#sq-mode").value : "auto"; + const time = $("#sq-time") ? $("#sq-time").value : ""; + + if (mode === "auto" && looksLikeUrl(q)) return renderParse(q); + if (mode === "url") return renderParse(q); + out.innerHTML = ``; - try { - const data = await api(`/api/search?q=${encodeURIComponent(q)}&limit=30`); - const r = data.results || []; - out.innerHTML = r.length - ? r.map((x) => `
${esc(x.ticker || "?")}${esc(x.event_date || "")}${esc(x.speaker || "")}
${esc((x.text || "").slice(0, 320))}…
`).join("") - : `
No matches for “${esc(q)}”.
`; - } catch (e) { - out.innerHTML = `
Search failed (${esc(e.message)}).
`; + if (mode === "web" || mode === "news") { + out.innerHTML = await webSection(q, mode, time); + return; + } + if (mode === "transcripts") { + out.innerHTML = await transcriptsSection(q, true); + return; } + + // Auto over a phrase: both sources at once. Neither waits on the other, and + // a web failure (no key, no credits) still leaves the transcripts standing. + const [transcripts, web] = await Promise.all([ + transcriptsSection(q, false), + webSection(q, "web", time), + ]); + out.innerHTML = transcripts + web || `
No matches for “${esc(q)}”.
`; } + $("#sq-run").addEventListener("click", runSearch); $("#sq").addEventListener("keydown", (e) => e.key === "Enter" && runSearch()); +if ($("#sq-mode")) $("#sq-mode").addEventListener("change", () => $("#sq").value.trim() && runSearch()); +if ($("#sq-time")) $("#sq-time").addEventListener("change", () => $("#sq").value.trim() && runSearch()); + +// Every phrase, question and niche in the results is a new search, and every +// result is a page that can be parsed — the point of the tab is following the +// thread without retyping it. `.tlink` is left to the document handler. +$("#search-results").addEventListener("click", (e) => { + const parse = e.target.closest("[data-parse]"); + if (parse) { + e.preventDefault(); + $("#sq").value = parse.dataset.parse; + renderParse(parse.dataset.parse); + return; + } + const again = e.target.closest("[data-q]"); + if (again && again.dataset.q) { + e.preventDefault(); + $("#sq").value = again.dataset.q; + runSearch(); + } +}); /* ---- Signals ---- */ async function runSignals() { diff --git a/public/index.html b/public/index.html index e52eb22..f308faa 100644 --- a/public/index.html +++ b/public/index.html @@ -109,9 +109,24 @@
- + + +
+

Paste a link and it is read back to you: title, publisher, date, the phrases it keeps using, and any tickers it names. Type a phrase and Auto searches the indexed transcripts and the open web together — News adds dates and publishers, with related searches, recurring phrases and the niches the results fall into. Web and news come from a metered search API, so identical queries are cached.

diff --git a/public/styles.css b/public/styles.css index f1a1be1..92db506 100644 --- a/public/styles.css +++ b/public/styles.css @@ -693,3 +693,69 @@ main:has(.view[data-view="watchlist"].active) { max-width: 1340px; } .wl-table th, .wl-table td { padding: 8px 7px; } .wl-toolbar .wl-count { margin-left: 0; } } + +/* ---- Search: web results, parsed pages, phrases and niches ---- + The tab now returns several kinds of thing at once — transcript segments, + web results, a parsed page, and the phrases derived from them — so each gets + a titled block rather than one undifferentiated list. */ +.sect { margin: 0 0 22px; } +.sect h3 { + display: flex; align-items: baseline; gap: 10px; flex-wrap: wrap; + margin: 0 0 10px; font-size: 12px; font-weight: 700; letter-spacing: .07em; + text-transform: uppercase; color: var(--dim); +} +.sect-note { font-size: 11.5px; font-weight: 500; letter-spacing: 0; text-transform: none; color: var(--dim); opacity: .8; } + +/* A chip that re-runs the search, versus one that is only a label. */ +.chip-btn { cursor: pointer; font-family: inherit; text-align: left; } +button.chip-btn:hover, a.chip:hover { border-color: var(--accent-2); color: var(--text); } +.chip-n { + margin-left: 6px; padding: 0 5px; border-radius: 5px; + background: rgba(255,255,255,.06); font-family: var(--mono); font-size: 10.5px; +} + +.res-title { + display: block; margin: 2px 0 6px; color: var(--text); + font-size: 15px; font-weight: 600; line-height: 1.4; text-decoration: none; +} +.res-title:hover { color: var(--accent); text-decoration: underline; } +.res-actions { display: flex; gap: 6px; flex-wrap: wrap; margin-top: 9px; } +.res-actions .chip { text-decoration: none; } + +/* Source reputation, matching the ingest-side tiers: primary, press, opinion, + excluded. Colour repeats the label rather than replacing it. */ +.tier { font-family: var(--mono); font-size: 11px; } +.tier-0 { color: var(--accent); } +.tier-1 { color: var(--accent-2); } +.tier-2 { color: var(--warn); } +.tier-3 { color: var(--neg); } +.chip.tier-0 { border-color: rgba(56,224,176,.3); } +.chip.tier-1 { border-color: rgba(76,141,255,.3); } +.chip.tier-2 { border-color: rgba(255,180,84,.3); } +.chip.tier-3 { border-color: rgba(255,107,107,.35); } + +.niches { display: grid; gap: 8px; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); } +.niche { + display: flex; flex-direction: column; gap: 3px; padding: 10px 12px; + border: 1px solid var(--line); border-radius: 10px; background: var(--panel-2); + color: var(--text); font-family: inherit; text-align: left; cursor: pointer; +} +.niche:hover { border-color: var(--accent-2); } +.niche-label { font-size: 13.5px; font-weight: 600; } +.niche-meta { font-size: 11.5px; color: var(--dim); } + +.qlist { display: grid; gap: 6px; } +.qitem { + padding: 9px 12px; border: 1px solid var(--line); border-radius: 10px; + background: var(--panel-2); color: #c9d4e2; font-family: inherit; + font-size: 13px; text-align: left; line-height: 1.45; +} +button.qitem { cursor: pointer; } +button.qitem:hover { border-color: var(--accent-2); color: var(--text); } + +.page-card .res-title { font-size: 17px; } +/* The excerpt is a quote from someone else's page, so it reads as one. */ +.page-card .excerpt { + margin-top: 8px; padding-left: 11px; border-left: 2px solid var(--line); + color: var(--dim); font-style: italic; +} diff --git a/src/research/page.ts b/src/research/page.ts new file mode 100644 index 0000000..2b19318 --- /dev/null +++ b/src/research/page.ts @@ -0,0 +1,441 @@ +/** + * Pasted-URL parsing: turn a link into the same shape a search result has. + * + * Someone reading about a company does not want to retype what it is called — + * they want to paste the article. This module fetches that page and extracts + * what the Search tab shows: title, publisher, date, the phrases it keeps + * using, and any tickers it names. Body retrieval is `providers/news/article.ts`, + * unchanged, so a pasted URL obeys exactly the same rules as an ingested one: + * robots.txt is honoured, publishers that block us are reported as blocked, + * and no body text is ever invented. + * + * **This endpoint fetches a URL chosen by an anonymous caller**, which makes it + * a server-side request forgery vector unless the target is checked first. + * `assertFetchableUrl` is that check: http(s) only, standard ports only, and + * the hostname must resolve exclusively to public addresses. A DNS name that + * resolves to 169.254.169.254 or 10.0.0.5 is refused before any fetch happens. + */ +import { lookup as dnsLookup } from "node:dns/promises"; +import { fetchArticle } from "../providers/news/article.ts"; +import { normalizeHost, tierFor, tierLabel } from "../providers/news/tiers.ts"; +import type { SourceTier } from "../types.ts"; +import { topPhrases, type Phrase } from "./phrases.ts"; + +/** Refused target: not a URL we will fetch on a stranger's behalf. */ +export class UnfetchableUrlError extends Error { + constructor(message: string) { + super(message); + this.name = "UnfetchableUrlError"; + } +} + +/** + * Does this input look like a link rather than a search phrase? + * + * Deliberately generous — "nvidia.com/newsroom" is a paste, not a query — but + * it must not swallow ordinary searches. A bare word with a dot in it only + * counts when the last label is a plausible TLD, so "rivian earnings" stays a + * query and "3.5% yield" does not become a URL. + */ +export function looksLikeUrl(input: string): boolean { + const s = String(input ?? "").trim(); + if (!s || /\s/.test(s)) return false; + if (/^https?:\/\//i.test(s)) return true; + // Reject other schemes outright rather than letting them reach the parser. + if (/^[a-z][a-z0-9+.-]*:/i.test(s)) return false; + return /^[\w-]+(\.[\w-]+)*\.[a-z]{2,24}(:\d+)?([/?#]|$)/i.test(s); +} + +/** + * Add the scheme a pasted `example.com/x` is missing — and only then. + * + * Prefixing unconditionally turned `ftp://host/x` into `https://ftp://host/x`, + * a URL that parses, has host `ftp`, and so failed later as "does not resolve" + * instead of "we only fetch http and https". An input that already names a + * scheme is left alone so the protocol check is the thing that rejects it. + */ +export function normalizeUrlInput(input: string): string { + const s = String(input ?? "").trim(); + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(s)) return s; // any scheme with an authority + // A bare `word:` scheme (mailto:, javascript:). Dots are excluded from the + // scheme name and digits from what follows, so `example.com:8080` and + // `host:8080` stay hostnames rather than becoming schemes. + if (/^[a-z][a-z0-9+-]*:(?!\d)/i.test(s)) return s; + return `https://${s}`; +} + +/** + * IPv4/IPv6 ranges that are not the public internet: loopback, link-local + * (which is where cloud metadata services live), the RFC1918 blocks, CGNAT, + * benchmarking, multicast and reserved space. + */ +export function isPrivateAddress(address: string, family: number): boolean { + if (family === 4) return isPrivateV4(address); + + const ip = address.toLowerCase().replace(/%.*$/, ""); + // IPv4-mapped (::ffff:10.0.0.1) is an IPv4 address wearing a costume. + const mapped = ip.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/); + if (mapped) return isPrivateV4(mapped[1]!); + if (ip === "::" || ip === "::1") return true; + if (/^f[cd][0-9a-f]{2}:/.test(ip)) return true; // fc00::/7 unique-local + if (/^fe[89ab][0-9a-f]:/.test(ip)) return true; // fe80::/10 link-local + if (/^ff[0-9a-f]{2}:/.test(ip)) return true; // multicast + return false; +} + +function isPrivateV4(address: string): boolean { + const parts = address.split(".").map(Number); + if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) { + return true; // unparseable: refuse rather than guess + } + const [a, b] = parts as [number, number, number, number]; + if (a === 0 || a === 10 || a === 127) return true; + if (a === 169 && b === 254) return true; // link-local + cloud metadata + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 192 && b === 0) return true; // 192.0.0.0/24 IETF protocol assignments + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT + if (a === 198 && (b === 18 || b === 19)) return true; // benchmarking + if (a >= 224) return true; // multicast + reserved + return false; +} + +/** + * Validate a pasted URL and prove its host is on the public internet. + * + * Returns the normalized URL. Throws `UnfetchableUrlError` with a reason the + * UI can show, because "we won't fetch a private address" is a better answer + * than a generic failure. + */ +export async function assertFetchableUrl(input: string): Promise { + let url: URL; + try { + url = new URL(normalizeUrlInput(input)); + } catch { + throw new UnfetchableUrlError("That does not parse as a URL."); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new UnfetchableUrlError("Only http and https URLs can be fetched."); + } + // Non-standard ports are almost never articles and often internal services. + if (url.port && url.port !== "80" && url.port !== "443") { + throw new UnfetchableUrlError("Only the standard web ports (80, 443) are fetched."); + } + const host = url.hostname.replace(/^\[|\]$/g, "").toLowerCase(); + if (!host || host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local") + || host.endsWith(".internal") || host.endsWith(".home.arpa")) { + throw new UnfetchableUrlError("That host is not on the public internet."); + } + + let addresses: Array<{ address: string; family: number }>; + try { + addresses = await dnsLookup(host, { all: true }); + } catch { + throw new UnfetchableUrlError("That host does not resolve."); + } + if (addresses.length === 0) throw new UnfetchableUrlError("That host does not resolve."); + // Every address must be public: one private answer is enough to make the + // fetch a request into someone's network. + for (const { address, family } of addresses) { + if (isPrivateAddress(address, family)) { + throw new UnfetchableUrlError("That host resolves to a private address."); + } + } + return url; +} + +export interface PageMeta { + title?: string; + description?: string; + siteName?: string; + author?: string; + publishedAt?: string; + canonical?: string; + image?: string; + type?: string; + lang?: string; + /** The publisher's own declared keywords, when it declares any. */ + keywords: string[]; + /** Section headings, in document order — the page's own outline. */ + headings: string[]; + /** RSS/Atom feeds the page advertises. */ + feeds: string[]; +} + +/** + * Read a page's declared metadata. + * + * Preference order is publisher-declared before inferred: OpenGraph and + * JSON-LD are what the publisher tells syndicators the page is, while + * `` is decorated for the tab bar ("Story — Section | Publisher"). + */ +export function extractPageMeta(html: string): PageMeta { + const meta: PageMeta = { keywords: [], headings: [], feeds: [] }; + const props = metaMap(html); + const ld = jsonLdNodes(html); + + const ldPick = (key: string): string | undefined => { + for (const node of ld) { + const v = node?.[key]; + if (typeof v === "string" && v.trim()) return v.trim(); + } + return undefined; + }; + + meta.title = clean( + props["og:title"] ?? props["twitter:title"] ?? ldPick("headline") ?? titleTag(html) ?? firstHeading(html), + ); + meta.description = clean( + props["og:description"] ?? props["description"] ?? props["twitter:description"] ?? ldPick("description"), + ); + meta.siteName = clean(props["og:site_name"] ?? props["application-name"]); + meta.author = clean(props["author"] ?? props["article:author"] ?? ldAuthor(ld)); + meta.publishedAt = isoDate( + props["article:published_time"] ?? props["datePublished"] ?? props["date"] + ?? ldPick("datePublished") ?? props["article:modified_time"], + ); + meta.canonical = clean(linkHref(html, "canonical") ?? props["og:url"]); + meta.image = clean(props["og:image"] ?? props["twitter:image"]); + meta.type = clean(props["og:type"] ?? ldType(ld)); + meta.lang = clean(html.match(/<html[^>]*\blang=["']([^"']{2,10})["']/i)?.[1]); + meta.keywords = (props["keywords"] ?? props["news_keywords"] ?? "") + .split(",") + .map((k) => clean(k) ?? "") + .filter((k) => k.length > 1 && k.length < 60) + .slice(0, 20); + meta.headings = headings(html); + meta.feeds = feedLinks(html); + return meta; +} + +/** + * Tickers a page names, read only from the two forms that are unambiguous: + * a cashtag ($NVDA) and an exchange-qualified mention (NASDAQ: NVDA). + * + * Bare capitals are not read. "CEO said the AI and EV markets" would otherwise + * produce three tickers, and a confidently wrong ticker on a research page is + * worse than no ticker at all. + */ +export function extractTickerMentions(text: string): string[] { + const found = new Map<string, number>(); + const add = (raw: string) => { + const sym = raw.toUpperCase(); + if (!/^[A-Z]{1,5}(\.[A-Z]{1,2})?$/.test(sym)) return; + found.set(sym, (found.get(sym) ?? 0) + 1); + }; + const body = String(text ?? ""); + for (const m of body.matchAll(/\$([A-Za-z]{1,5}(?:\.[A-Za-z]{1,2})?)\b/g)) add(m[1]!); + const exchange = + /\b(?:NASDAQ|NYSE(?:\s+(?:American|Arca))?|AMEX|NYSEAMERICAN|OTCQB|OTCQX|OTC(?:\s+Markets)?|CBOE|TSX(?:V)?|LSE)\s*[:：]\s*([A-Za-z]{1,5}(?:\.[A-Za-z]{1,2})?)\b/gi; + for (const m of body.matchAll(exchange)) add(m[1]!); + return [...found.entries()].sort((a, b) => b[1] - a[1]).map(([sym]) => sym); +} + +export interface ParsedPage { + url: string; + host: string; + tier: SourceTier; + tierLabel: string; + ok: boolean; + meta: PageMeta; + /** Recurring phrases in the page's own text — its niche, in its own words. */ + phrases: Phrase[]; + tickers: string[]; + /** Extracted body length in characters; 0 when blocked or unparseable. */ + textLength: number; + wordCount: number; + /** A short attributed quote. Never the whole body — that is never republished. */ + excerpt?: string; + /** Set when the body could not be read: paywall, robots, or a hard block. */ + blockedReason?: string; +} + +/** + * Fetch and describe one page. + * + * The caller is expected to have run `assertFetchableUrl` already (the route + * does), but it is run again here so no future caller can skip it. + */ +export async function parsePage( + input: string, + opts: { timeoutMs?: number } = {}, +): Promise<ParsedPage> { + const url = await assertFetchableUrl(input); + const target = url.toString(); + const host = normalizeHost(target); + const tier = tierFor(target); + + const article = await fetchArticle(target, { timeoutMs: opts.timeoutMs ?? 20_000 }); + const meta = extractPageMeta(article.raw ?? ""); + const text = article.text ?? ""; + + // Phrases come from the page's own sections rather than one blob, so + // document frequency means "recurs through the piece", not "appears". + const documents = [ + meta.title ?? "", + meta.description ?? "", + ...meta.headings, + ...paragraphs(text), + ].filter((d) => d.trim().length > 0); + + const tickerSource = `${meta.title ?? ""} ${meta.description ?? ""} ${text}`; + + return { + url: target, + host, + tier, + tierLabel: tierLabel(tier), + ok: article.ok, + meta, + phrases: topPhrases(documents, { limit: 12, maxWords: 3 }), + tickers: extractTickerMentions(tickerSource).slice(0, 8), + textLength: text.length, + wordCount: text ? text.split(/\s+/).filter(Boolean).length : 0, + excerpt: text ? `${text.slice(0, 320).trim()}${text.length > 320 ? "…" : ""}` : undefined, + blockedReason: article.ok ? undefined : article.reason, + }; +} + +/* ---------------- HTML helpers ---------------- */ + +/** Every `<meta>` on the page, keyed by name/property, lowercased. */ +function metaMap(html: string): Record<string, string> { + const out: Record<string, string> = {}; + for (const tag of html.match(/<meta\b[^>]*>/gi) ?? []) { + const key = ( + attr(tag, "property") ?? attr(tag, "name") ?? attr(tag, "itemprop") ?? "" + ).toLowerCase(); + const value = attr(tag, "content"); + if (key && value && !(key in out)) out[key] = value; + } + return out; +} + +function attr(tag: string, name: string): string | undefined { + const m = + tag.match(new RegExp(`\\b${name}\\s*=\\s*"([^"]*)"`, "i")) ?? + tag.match(new RegExp(`\\b${name}\\s*=\\s*'([^']*)'`, "i")) ?? + tag.match(new RegExp(`\\b${name}\\s*=\\s*([^\\s>]+)`, "i")); + return m?.[1]; +} + +function titleTag(html: string): string | undefined { + return html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]; +} + +function firstHeading(html: string): string | undefined { + return html.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i)?.[1]; +} + +function headings(html: string): string[] { + const out: string[] = []; + for (const m of html.matchAll(/<h([12345])\b[^>]*>([\s\S]*?)<\/h\1>/gi)) { + const text = clean(m[2]); + if (text && text.length > 2 && text.length < 200 && !out.includes(text)) out.push(text); + if (out.length >= 25) break; + } + return out; +} + +function linkHref(html: string, rel: string): string | undefined { + for (const tag of html.match(/<link\b[^>]*>/gi) ?? []) { + if ((attr(tag, "rel") ?? "").toLowerCase().split(/\s+/).includes(rel)) return attr(tag, "href"); + } + return undefined; +} + +function feedLinks(html: string): string[] { + const out: string[] = []; + for (const tag of html.match(/<link\b[^>]*>/gi) ?? []) { + const rel = (attr(tag, "rel") ?? "").toLowerCase(); + const type = (attr(tag, "type") ?? "").toLowerCase(); + const href = attr(tag, "href"); + if (!href || !rel.includes("alternate")) continue; + if (!/rss|atom|feed\+json/.test(type)) continue; + if (!out.includes(href)) out.push(href); + if (out.length >= 5) break; + } + return out; +} + +/* eslint-disable @typescript-eslint/no-explicit-any */ +/** Flattened JSON-LD objects, so `@graph` entries are reachable. */ +function jsonLdNodes(html: string): any[] { + const nodes: any[] = []; + const blocks = + html.match(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi) ?? []; + for (const block of blocks) { + const json = block.replace(/^[\s\S]*?>/, "").replace(/<\/script>$/i, ""); + try { + flatten(JSON.parse(json), nodes); + } catch { + /* one malformed block must not lose the others */ + } + } + return nodes; +} + +function flatten(node: any, out: any[], depth = 0): void { + if (!node || depth > 5) return; + if (Array.isArray(node)) { + for (const item of node) flatten(item, out, depth + 1); + return; + } + if (typeof node !== "object") return; + out.push(node); + if (node["@graph"]) flatten(node["@graph"], out, depth + 1); +} + +function ldAuthor(nodes: any[]): string | undefined { + for (const node of nodes) { + const a = node?.author; + if (typeof a === "string" && a.trim()) return a; + if (Array.isArray(a) && typeof a[0]?.name === "string") return a[0].name; + if (typeof a?.name === "string") return a.name; + } + return undefined; +} + +function ldType(nodes: any[]): string | undefined { + for (const node of nodes) { + const t = node?.["@type"]; + if (typeof t === "string") return t; + if (Array.isArray(t) && typeof t[0] === "string") return t[0]; + } + return undefined; +} + +/** Split extracted body text back into the paragraphs `article.ts` joined. */ +function paragraphs(text: string): string[] { + return text.split(/\n{2,}/).map((p) => p.trim()).filter((p) => p.length > 40).slice(0, 60); +} + +function clean(s: string | undefined): string | undefined { + if (s == null) return undefined; + const out = decodeEntities(String(s).replace(/<[^>]+>/g, " ")) + .replace(/\s+/g, " ") + .trim(); + return out.length > 0 ? out.slice(0, 400) : undefined; +} + +function isoDate(raw: string | undefined): string | undefined { + if (!raw) return undefined; + const t = Date.parse(raw.trim()); + return Number.isNaN(t) ? undefined : new Date(t).toISOString(); +} + +function decodeEntities(s: string): string { + return s + .replace(/ /g, " ") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/�?39;|'|’|’/g, "'") + .replace(/“|”|“|”/g, '"') + .replace(/–|—|–|—/g, "-") + .replace(/&/g, "&") + .replace(/&#(\d+);/g, (_, n) => { + const code = Number(n); + return code > 31 && code < 0x10ffff ? String.fromCodePoint(code) : " "; + }); +} diff --git a/src/research/phrases.ts b/src/research/phrases.ts new file mode 100644 index 0000000..e6cd2af --- /dev/null +++ b/src/research/phrases.ts @@ -0,0 +1,205 @@ +/** + * Phrase and niche extraction from search titles, snippets and page text. + * + * The Search tab answers three questions about a query or a pasted page: what + * is being written about (titles), what language recurs across those writers + * (phrases), and which distinct subjects the results fall into (niches). Only + * the first is given to us; the other two are computed here. + * + * Two decisions shape the output: + * + * - **Document frequency, not term frequency.** A phrase scores by how many + * separate results contain it, so one long article repeating its own + * keyword twelve times cannot invent a trend. A phrase in one document is + * a quirk of that document; a phrase in six is a niche. + * - **Longest wins.** "ai" and "ai infrastructure" occurring in the same six + * documents are one phrase, not two, so the shorter is dropped when it is + * contained in a longer one of equal weight. Without this the chip row is + * the same idea at three lengths. + * + * Pure and deterministic: no network, no LLM, no model of English beyond a + * stopword list. That keeps it free to run on every search and testable. + */ + +/** + * Words that carry no topic. Kept deliberately short — a long list starts + * eating domain vocabulary ("general", "market", "value") that is exactly what + * distinguishes one niche from another. + */ +const STOPWORDS = new Set([ + "a", "about", "after", "again", "against", "all", "also", "am", "an", "and", "any", "are", + "as", "at", "be", "because", "been", "before", "being", "below", "between", "both", "but", + "by", "can", "did", "do", "does", "doing", "down", "during", "each", "few", "for", "from", + "further", "had", "has", "have", "having", "he", "her", "here", "hers", "him", "his", "how", + "i", "if", "in", "into", "is", "it", "its", "just", "me", "might", "more", "most", "must", + "my", "no", "nor", "not", "now", "of", "off", "on", "once", "only", "or", "other", "our", + "out", "over", "own", "per", "said", "same", "says", "she", "should", "so", + "some", "such", "than", "that", "the", "their", "them", "then", "there", "these", + "they", "this", "those", "through", "to", "too", "under", "until", "up", "us", "very", + "was", "we", "were", "what", "when", "where", "which", "while", "who", "whom", "why", + "will", "with", "would", "you", "your", +]); + +/** Editorial furniture that survives tokenisation but names no subject. */ +const NOISE = new Set([ + "read", "reads", "click", "subscribe", "newsletter", "advertisement", "sponsored", + "continue", "reading", "share", "comments", "min", "mins", "ago", "today", "yesterday", + "new", "news", "latest", "update", "updates", "best", "top", "guide", "review", "reviews", + "https", "http", "www", "com", +]); + +export interface Phrase { + phrase: string; + /** How many separate documents contain it. */ + count: number; + /** Words in the phrase, 1-3. */ + words: number; +} + +export interface PhraseOptions { + /** Longest n-gram considered. */ + maxWords?: number; + /** How many phrases to return. */ + limit?: number; + /** + * Documents a phrase must appear in to count. Defaults to 2 once there are + * enough documents to make repetition meaningful, and 1 below that — a + * single pasted article has only one document, and every phrase in it would + * otherwise score 1 and be discarded. + */ + minCount?: number; +} + +/** Split text into lowercase word tokens, keeping intra-word hyphens. */ +export function tokenize(text: string): string[] { + return String(text ?? "") + .toLowerCase() + .replace(/[‘’]/g, "'") + .split(/[^a-z0-9'&-]+/) + .map((w) => w.replace(/^[-']+|[-']+$/g, "")) + .filter((w) => w.length > 1 && w.length < 40 && !/^\d+$/.test(w)); +} + +function isEdgeWord(word: string): boolean { + return !STOPWORDS.has(word) && !NOISE.has(word); +} + +/** + * Rank the phrases that recur across a set of documents. + * + * `documents` are independent texts — one per search result, or one per + * section of a parsed page. Passing a single concatenated string is a mistake + * that turns document frequency back into term frequency. + */ +export function topPhrases(documents: string[], opts: PhraseOptions = {}): Phrase[] { + const maxWords = Math.min(4, Math.max(1, opts.maxWords ?? 3)); + const limit = Math.max(1, opts.limit ?? 12); + const docs = documents.map(tokenize).filter((t) => t.length > 0); + if (docs.length === 0) return []; + const minCount = opts.minCount ?? (docs.length >= 4 ? 2 : 1); + + // Document frequency: a phrase counted once per document however often it + // appears inside that one. + const df = new Map<string, number>(); + for (const tokens of docs) { + const seen = new Set<string>(); + for (let n = 1; n <= maxWords; n++) { + for (let i = 0; i + n <= tokens.length; i++) { + const words = tokens.slice(i, i + n); + if (!isEdgeWord(words[0]!) || !isEdgeWord(words[n - 1]!)) continue; + // A stopword may sit inside a phrase ("state of the art") but a phrase + // made only of them is not a subject. + if (words.every((w) => STOPWORDS.has(w))) continue; + seen.add(words.join(" ")); + } + } + for (const phrase of seen) df.set(phrase, (df.get(phrase) ?? 0) + 1); + } + + const candidates = [...df.entries()] + .map(([phrase, count]) => ({ phrase, count, words: phrase.split(" ").length })) + .filter((p) => p.count >= minCount) + // Longer phrases first at equal weight so containment dedupe keeps the + // specific one ("ai infrastructure") over the generic one ("ai"). + .sort((a, b) => b.count - a.count || b.words - a.words || a.phrase.localeCompare(b.phrase)); + + const chosen: Phrase[] = []; + for (const cand of candidates) { + if (chosen.length >= limit) break; + // Drop a phrase already represented by a kept one of equal or greater + // weight — either direction of containment, since "ai infrastructure" + // makes "ai" redundant and vice versa. + const redundant = chosen.some( + (kept) => + kept.count >= cand.count && + (contains(kept.phrase, cand.phrase) || contains(cand.phrase, kept.phrase)), + ); + if (!redundant) chosen.push(cand); + } + return chosen; +} + +/** Whole-word containment: "ai infrastructure" contains "ai", not "rai". */ +function contains(haystack: string, needle: string): boolean { + if (haystack === needle) return true; + return ` ${haystack} `.includes(` ${needle} `); +} + +export interface NicheItem { + title: string; + snippet?: string; + host?: string; +} + +export interface Niche { + /** The recurring phrase that defines the cluster. */ + label: string; + /** Results containing it. */ + count: number; + /** Distinct publishers writing about it, most frequent first. */ + hosts: string[]; + /** Indices into the input array, so the UI can filter to the cluster. */ + members: number[]; +} + +/** + * Group results into niches: one cluster per recurring phrase, holding the + * results that mention it. + * + * Clusters overlap by design — an article about "ai infrastructure spending" + * belongs to both niches, and forcing a single assignment would misreport the + * size of each. A niche of one is not a niche, so singletons are dropped. + */ +export function deriveNiches(items: NicheItem[], opts: { limit?: number } = {}): Niche[] { + const limit = Math.max(1, opts.limit ?? 6); + const docs = items.map((it) => `${it.title ?? ""} ${it.snippet ?? ""}`); + // Ask for more phrases than niches wanted: many will cluster to one member + // and be discarded below. + const phrases = topPhrases(docs, { limit: limit * 4, maxWords: 3, minCount: 2 }); + + const niches: Niche[] = []; + for (const { phrase } of phrases) { + const members: number[] = []; + const hostCounts = new Map<string, number>(); + docs.forEach((doc, i) => { + if (!containsPhrase(doc, phrase)) return; + members.push(i); + const host = items[i]?.host; + if (host) hostCounts.set(host, (hostCounts.get(host) ?? 0) + 1); + }); + if (members.length < 2) continue; + niches.push({ + label: phrase, + count: members.length, + hosts: [...hostCounts.entries()].sort((a, b) => b[1] - a[1]).map(([h]) => h), + members, + }); + if (niches.length >= limit) break; + } + return niches; +} + +/** Phrase membership test over raw text, matching `tokenize`'s word boundaries. */ +function containsPhrase(text: string, phrase: string): boolean { + return contains(tokenize(text).join(" "), phrase); +} diff --git a/src/research/routes.ts b/src/research/routes.ts new file mode 100644 index 0000000..340da2d --- /dev/null +++ b/src/research/routes.ts @@ -0,0 +1,306 @@ +/** + * Web/news search and URL parsing for the Search tab. + * + * GET /api/web?q=&kind=web|news&pages=&time= -> ranked results + phrases + niches + * GET /api/parse?url= -> one pasted page, described + * + * Public and unauthenticated, like the rest of the research API — but unlike + * the rest of it, `/api/web` spends real money: every page is a ValueSERP + * credit out of a monthly bucket shared with other properties. Two guards keep + * an open endpoint from draining it: + * + * - **Cache first.** Identical queries inside the TTL are answered from + * memory and cost nothing. Search traffic is repetitive (a shared link, a + * reload, a back button), so this absorbs most of it. + * - **Then throttle.** Only a cache miss — an actual credit — counts against + * the per-IP limit. + * + * A cache miss that finds the bucket empty answers 402 with a plain message + * rather than a generic failure, because "out of credits until the bucket + * resets" is something the operator needs to see and a user can understand. + */ +import type { Client } from "@libsql/client"; +import { searchSymbols } from "../symbols/directory.ts"; +import { rateLimit } from "../auth/service.ts"; +import { deriveNiches, topPhrases } from "./phrases.ts"; +import { + SerpClient, + SerpCreditsError, + SerpNotConfiguredError, + type SerpHit, + type SerpKind, +} from "./serp.ts"; +import { UnfetchableUrlError, assertFetchableUrl, looksLikeUrl, parsePage } from "./page.ts"; + +const json = (body: unknown, status = 200, cacheSeconds = 0) => + new Response(JSON.stringify(body, null, 2), { + status, + headers: { + "content-type": "application/json", + "cache-control": cacheSeconds ? `public, max-age=${cacheSeconds}` : "no-store", + }, + }); + +/** + * Throttles. Web search is capped hard because each miss is a credit; parsing + * costs only our own bandwidth, so it is capped loosely and mostly to stop the + * endpoint being used as an open proxy. + */ +export const RESEARCH_LIMITS = { + web: { max: 20, windowMinutes: 60 }, + parse: { max: 60, windowMinutes: 60 }, +} as const; + +/** Pages one request may buy. Three is ~25 results; more is a crawl, not a search. */ +const MAX_PAGES = 3; + +const SERP_TTL_MS = 30 * 60_000; +/** Pages change far more slowly than rankings, and re-fetching one is rude. */ +const PARSE_TTL_MS = 6 * 60 * 60_000; +const CACHE_MAX_ENTRIES = 500; + +interface CacheEntry { + at: number; + value: unknown; +} + +const cache = new Map<string, CacheEntry>(); + +function cacheGet(key: string, ttlMs: number): unknown | undefined { + const hit = cache.get(key); + if (!hit) return undefined; + if (Date.now() - hit.at > ttlMs) { + cache.delete(key); + return undefined; + } + return hit.value; +} + +function cacheSet(key: string, value: unknown): void { + // Insertion-ordered Map: the first key is the oldest write. + if (cache.size >= CACHE_MAX_ENTRIES) { + const oldest = cache.keys().next().value; + if (oldest !== undefined) cache.delete(oldest); + } + cache.set(key, { at: Date.now(), value }); +} + +/** Exposed for tests, and for a future admin endpoint that needs to flush. */ +export function clearResearchCache(): void { + cache.clear(); +} + +function clientIp(req: Request): string { + const fwd = req.headers.get("x-forwarded-for") ?? ""; + return (fwd.split(",")[0] || req.headers.get("x-real-ip") || "unknown").trim(); +} + +export interface ResearchRouteDeps { + db: Client; + serp: SerpClient; +} + +export async function handleResearchRoute( + req: Request, + path: string, + deps: ResearchRouteDeps, +): Promise<Response | null> { + if (path !== "/api/web" && path !== "/api/parse") return null; + if (req.method !== "GET") return json({ error: "method not allowed" }, 405); + + const url = new URL(req.url); + const ip = clientIp(req); + + if (path === "/api/parse") return handleParse(url, ip, deps); + return handleWeb(url, ip, deps); +} + +/* ---------------- /api/web ---------------- */ + +async function handleWeb(url: URL, ip: string, deps: ResearchRouteDeps): Promise<Response> { + const q = (url.searchParams.get("q") ?? "").trim().slice(0, 300); + if (!q) return json({ error: "missing ?q=" }, 400); + + const kind: SerpKind = url.searchParams.get("kind") === "news" ? "news" : "web"; + const pages = Math.min(MAX_PAGES, Math.max(1, Number(url.searchParams.get("pages") ?? 1) || 1)); + const timePeriod = normalizeTimePeriod(url.searchParams.get("time")); + + if (!deps.serp.configured) { + return json( + { + error: "Web search is not configured on this deployment (VALUESERP_API_KEY).", + configured: false, + }, + 503, + ); + } + + const key = `web:${kind}:${pages}:${timePeriod ?? ""}:${q.toLowerCase()}`; + const cached = cacheGet(key, SERP_TTL_MS); + if (cached) return json({ ...(cached as object), cached: true }, 200, 300); + + // Only a real credit is throttled — a cache hit above never reaches here. + const limit = await rateLimit(deps.db, `websearch:${ip}`, RESEARCH_LIMITS.web); + if (!limit.allowed) { + return json( + { error: `Too many web searches. Try again in ${limit.retryAfterMinutes} minutes.` }, + 429, + ); + } + + let search; + try { + search = await deps.serp.search(q, { kind, pages, timePeriod }); + } catch (err) { + if (err instanceof SerpCreditsError) { + return json({ error: "Web search credits are exhausted for this month.", credits: 0 }, 402); + } + if (err instanceof SerpNotConfiguredError) { + return json({ error: err.message, configured: false }, 503); + } + return json({ error: `Web search failed: ${String(err).slice(0, 200)}` }, 502); + } + + const body = summarize(search.hits, { + kind, + query: q, + pages, + trending: search.relatedSearches, + questions: search.questions, + creditsRemaining: search.creditsRemaining, + }); + cacheSet(key, body); + return json({ ...body, cached: false }, 200, 300); +} + +interface SummaryMeta { + kind: SerpKind; + query: string; + pages: number; + trending: string[]; + questions: string[]; + creditsRemaining?: number; +} + +/** + * Turn raw hits into what the tab renders: the results themselves, the + * language recurring across them, the niches they fall into, and who is + * publishing. + */ +function summarize(hits: SerpHit[], meta: SummaryMeta) { + const items = hits.map((h) => ({ title: h.title, snippet: h.snippet, host: h.host })); + const documents = items.map((i) => `${i.title} ${i.snippet ?? ""}`); + + const hostCounts = new Map<string, { count: number; tier: number; tierLabel: string }>(); + for (const h of hits) { + const cur = hostCounts.get(h.host); + if (cur) cur.count++; + else hostCounts.set(h.host, { count: 1, tier: h.tier, tierLabel: h.tierLabel }); + } + + return { + kind: meta.kind, + query: meta.query, + pages: meta.pages, + results: hits, + /** Google's own related searches and questions: what else is being asked. */ + trending: meta.trending, + questions: meta.questions, + /** Phrases recurring across the titles we got back. */ + phrases: topPhrases(documents, { limit: 14, maxWords: 3 }), + niches: deriveNiches(items, { limit: 6 }), + sources: [...hostCounts.entries()] + .map(([host, v]) => ({ host, ...v })) + .sort((a, b) => b.count - a.count || a.host.localeCompare(b.host)), + creditsRemaining: meta.creditsRemaining, + }; +} + +/** Only Google's own vocabulary is passed through; anything else is dropped. */ +function normalizeTimePeriod(raw: string | null): string | undefined { + const allowed = ["last_hour", "last_day", "last_week", "last_month", "last_year"]; + const v = (raw ?? "").trim().toLowerCase(); + return allowed.includes(v) ? v : undefined; +} + +/* ---------------- /api/parse ---------------- */ + +async function handleParse(url: URL, ip: string, deps: ResearchRouteDeps): Promise<Response> { + const raw = (url.searchParams.get("url") ?? "").trim().slice(0, 2000); + if (!raw) return json({ error: "missing ?url=" }, 400); + if (!looksLikeUrl(raw)) return json({ error: "That does not look like a URL." }, 400); + + // Validate before the cache lookup so a refused target is refused + // consistently, and before the throttle so a typo does not cost a slot. + let target: URL; + try { + target = await assertFetchableUrl(raw); + } catch (err) { + if (err instanceof UnfetchableUrlError) return json({ error: err.message }, 400); + return json({ error: "That URL could not be checked." }, 400); + } + + const key = `parse:${target.toString()}`; + const cached = cacheGet(key, PARSE_TTL_MS); + if (cached) return json({ ...(cached as object), cached: true }, 200, 600); + + const limit = await rateLimit(deps.db, `webparse:${ip}`, RESEARCH_LIMITS.parse); + if (!limit.allowed) { + return json({ error: `Too many page fetches. Try again in ${limit.retryAfterMinutes} minutes.` }, 429); + } + + let page; + try { + page = await parsePage(target.toString()); + } catch (err) { + if (err instanceof UnfetchableUrlError) return json({ error: err.message }, 400); + return json({ error: `Could not read that page: ${String(err).slice(0, 200)}` }, 502); + } + + const body = { ...page, tickers: await annotateTickers(deps.db, page.tickers) }; + cacheSet(key, body); + return json({ ...body, cached: false }, 200, 600); +} + +export interface AnnotatedTicker { + symbol: string; + name?: string; + hasReport: boolean; +} + +/** + * Attach the company name and whether a stored report exists, so a ticker + * found in an article is one click from the research already done on it. + * + * Directory misses are kept rather than dropped: a symbol the page states + * explicitly is worth showing even when our directory has not synced it. + */ +async function annotateTickers(db: Client, symbols: string[]): Promise<AnnotatedTicker[]> { + if (symbols.length === 0) return []; + const reports = await reportedTickers(db, symbols); + const out: AnnotatedTicker[] = []; + for (const symbol of symbols) { + let name: string | undefined; + try { + const matches = await searchSymbols(db, symbol, 3); + name = matches.find((m) => m.symbol.toUpperCase() === symbol)?.name; + } catch { + /* the directory is an enrichment, never the answer */ + } + out.push({ symbol, name, hasReport: reports.has(symbol) }); + } + return out; +} + +async function reportedTickers(db: Client, symbols: string[]): Promise<Set<string>> { + try { + const ph = symbols.map(() => "?").join(","); + const rs = await db.execute({ + sql: `SELECT ticker FROM reports WHERE ticker IN (${ph})`, + args: symbols, + }); + return new Set(rs.rows.map((r) => String(r.ticker))); + } catch { + return new Set(); + } +} diff --git a/src/research/serp.ts b/src/research/serp.ts new file mode 100644 index 0000000..9efb859 --- /dev/null +++ b/src/research/serp.ts @@ -0,0 +1,268 @@ +/** + * General web and news search over ValueSERP. + * + * `providers/news/valueserp.ts` is the ingestion-side client: news only, wired + * into the analysis pipeline, spending credits on a schedule. This is the + * interactive one behind the Search tab — web *or* news, one page by default, + * and it returns the things a person searching wants rather than the things + * the ranker wants: titles, publishers, related searches and the questions + * Google thinks the query implies. + * + * Three measured facts about the API shape this client (all verified against + * the live account, see docs/PRD-v3-media-news.md §3.1): + * + * - **`num` is the pagination stride, not the page size.** The offset sent + * upstream is `(page - 1) * num`, and a request returns 8-10 organic + * results whatever `num` says. Asking for `num=100` therefore puts page 2 + * at result 101 — past the end of a truncated result set — so pagination + * dies after one page and the query caps at ~8 results. Always `num=10`, + * always walk `page`. + * - **A page takes 3-34 seconds.** Any timeout under ~45s randomly kills + * slow queries, so the default here is deliberately generous. + * - **Pages must be fetched concurrently.** Sequentially, three pages is a + * minute and a half of a person staring at a spinner. The account allows + * 250 requests/minute, so a handful in parallel is well inside it. + * + * Every request costs a credit from a monthly bucket shared with other + * Profullstack properties, so callers are expected to cache. When the bucket + * empties the API answers HTTP 402, which is raised as `SerpCreditsError` so + * the route can say "out of credits" rather than "search failed". + */ +import { normalizeHost, tierFor, tierLabel } from "../providers/news/tiers.ts"; +import { parseNewsResults, resolveDate } from "../providers/news/valueserp.ts"; +import type { SourceTier } from "../types.ts"; + +const ENDPOINT = "https://api.valueserp.com/search"; + +/** The stride that keeps pagination alive. Not a preference — see above. */ +const PAGE_STRIDE = 10; + +export type SerpKind = "web" | "news"; + +export interface SerpHit { + title: string; + url: string; + host: string; + /** Source reputation, reused from the news tiering so both agree. */ + tier: SourceTier; + tierLabel: string; + publisher?: string; + snippet?: string; + /** ISO date when the result carried a resolvable one. */ + publishedAt?: string; +} + +export interface SerpSearch { + kind: SerpKind; + query: string; + hits: SerpHit[]; + /** Google's "related searches" — what else people search around this. */ + relatedSearches: string[]; + /** People-also-ask questions, the query's implied sub-topics. */ + questions: string[]; + /** Credits left in the monthly bucket, when the response reported it. */ + creditsRemaining?: number; + pagesFetched: number; +} + +/** Raised on HTTP 402: the monthly bucket is empty, the query never ran. */ +export class SerpCreditsError extends Error { + constructor(message = "ValueSERP monthly credits are exhausted") { + super(message); + this.name = "SerpCreditsError"; + } +} + +/** Raised when no API key is configured, so the caller can degrade honestly. */ +export class SerpNotConfiguredError extends Error { + constructor(message = "Web search is not configured (VALUESERP_API_KEY)") { + super(message); + this.name = "SerpNotConfiguredError"; + } +} + +export interface SerpClientOptions { + apiKey: string; + timeoutMs?: number; + /** Hard ceiling on pages per call, so one request cannot spend 20 credits. */ + maxPages?: number; +} + +export interface SerpQueryOptions { + kind?: SerpKind; + /** Pages to walk, 1 credit each. Clamped to `maxPages`. */ + pages?: number; + /** Google `time_period`: last_hour / last_day / last_week / last_month / last_year. */ + timePeriod?: string; + /** Anchors relative dates ("2 days ago") so a replay cannot shift them. */ + asOf?: Date; +} + +export class SerpClient { + constructor(private opts: SerpClientOptions) {} + + get configured(): boolean { + return Boolean(this.opts.apiKey); + } + + async search(query: string, opts: SerpQueryOptions = {}): Promise<SerpSearch> { + if (!this.configured) throw new SerpNotConfiguredError(); + const q = query.trim(); + if (!q) throw new Error("empty query"); + + const kind: SerpKind = opts.kind === "news" ? "news" : "web"; + const asOf = opts.asOf ?? new Date(); + const maxPages = Math.max(1, this.opts.maxPages ?? 3); + const pages = Math.min(maxPages, Math.max(1, opts.pages ?? 1)); + + const bodies = await Promise.all( + Array.from({ length: pages }, (_, i) => this.fetchPage(q, kind, i + 1, opts.timePeriod)), + ); + + const hits: SerpHit[] = []; + const seen = new Set<string>(); + const related = new Set<string>(); + const questions = new Set<string>(); + let creditsRemaining: number | undefined; + + for (const body of bodies) { + for (const hit of kind === "news" ? newsHits(body, asOf) : webHits(body, asOf)) { + // The same URL can surface on two pages when the result set shifts + // between concurrent requests. + if (seen.has(hit.url)) continue; + seen.add(hit.url); + hits.push(hit); + } + for (const s of parseRelatedSearches(body)) related.add(s); + for (const question of parseQuestions(body)) questions.add(question); + const credits = creditsFrom(body); + if (credits !== undefined) { + creditsRemaining = creditsRemaining === undefined ? credits : Math.min(creditsRemaining, credits); + } + } + + return { + kind, + query: q, + hits, + relatedSearches: [...related], + questions: [...questions], + creditsRemaining, + pagesFetched: pages, + }; + } + + /** Remaining monthly credits, for a budget read that costs nothing. */ + async credits(): Promise<{ remaining: number; limit: number } | null> { + if (!this.configured) return null; + try { + const res = await fetch( + `https://api.valueserp.com/account?api_key=${encodeURIComponent(this.opts.apiKey)}`, + { signal: AbortSignal.timeout(15_000) }, + ); + if (!res.ok) return null; + const j = (await res.json()) as any; + return { + remaining: Number(j?.account_info?.monthly_credits_remaining ?? 0), + limit: Number(j?.account_info?.monthly_credits_limit ?? 0), + }; + } catch { + return null; + } + } + + private async fetchPage( + q: string, + kind: SerpKind, + page: number, + timePeriod?: string, + ): Promise<unknown> { + const params = new URLSearchParams({ + api_key: this.opts.apiKey, + q, + num: String(PAGE_STRIDE), + page: String(page), + }); + if (kind === "news") params.set("search_type", "news"); + if (timePeriod) params.set("time_period", timePeriod); + + const res = await fetch(`${ENDPOINT}?${params}`, { + signal: AbortSignal.timeout(this.opts.timeoutMs ?? 60_000), + }); + if (res.status === 402) throw new SerpCreditsError(); + if (!res.ok) { + throw new Error(`ValueSERP ${res.status}: ${(await res.text()).slice(0, 200)}`); + } + return res.json(); + } +} + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +/** Organic web results. */ +export function webHits(body: any, asOf: Date = new Date()): SerpHit[] { + const rows: any[] = body?.organic_results ?? []; + const out: SerpHit[] = []; + for (const r of rows) { + const url = String(r?.link ?? ""); + if (!url) continue; + const host = normalizeHost(url); + const tier = tierFor(url); + out.push({ + title: String(r?.title ?? "").trim(), + url, + host, + tier, + tierLabel: tierLabel(tier), + publisher: r?.domain ? String(r.domain) : host, + snippet: r?.snippet ? String(r.snippet) : undefined, + // `date_utc` is an exact timestamp when the result carries one; `date` + // is the human string ("2 days ago") that has to be resolved against a + // reference point. Prefer the exact one. + publishedAt: resolveDate(r?.date_utc ?? r?.date, asOf), + }); + } + return out; +} + +/** News results, reusing the ingestion parser so both paths tier identically. */ +export function newsHits(body: any, asOf: Date = new Date()): SerpHit[] { + return parseNewsResults(body, asOf).map((n) => ({ + title: n.title, + url: n.url, + host: n.host, + tier: n.tier, + tierLabel: tierLabel(n.tier), + publisher: n.publisher, + snippet: n.snippet, + publishedAt: n.publishedAt, + })); +} + +/** + * Related searches. The field has carried both `{ query }` and `{ q }` across + * result types, so both are read rather than assuming one. + */ +export function parseRelatedSearches(body: any): string[] { + const rows: any[] = body?.related_searches ?? []; + return rows + .map((r) => String(r?.query ?? r?.q ?? r ?? "").trim()) + .filter((s) => s.length > 0 && s.length < 120); +} + +/** People-also-ask questions. */ +export function parseQuestions(body: any): string[] { + const rows: any[] = body?.related_questions ?? body?.people_also_ask ?? []; + return rows + .map((r) => String(r?.question ?? r ?? "").trim()) + .filter((s) => s.length > 0 && s.length < 200); +} + +/** + * Credits left, as reported on the response itself. Free: it saves the extra + * `/account` round trip that would otherwise be needed to show a budget. + */ +export function creditsFrom(body: any): number | undefined { + const n = body?.request_info?.credits_remaining; + return typeof n === "number" && Number.isFinite(n) ? n : undefined; +} diff --git a/src/server.ts b/src/server.ts index ede5ae8..6648b0b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -10,6 +10,8 @@ * GET /health -> liveness probe * GET /api/stats -> index coverage counts * GET /api/search?q=...&limit=.. -> FTS over indexed transcript segments + * GET /api/web?q=..&kind=web|news -> web/news search: titles, phrases, niches + * GET /api/parse?url=.. -> describe one pasted URL * GET /api/signals?ticker=.. -> extracted signals for a ticker * GET /api/tickers -> tickers present in the index * GET /api/discover?topic=..&provider= -> ranked watchlist (offline by default) @@ -37,6 +39,8 @@ import { startDigestScheduler } from "./digest/run.ts"; import { handleReportRoute } from "./reports/routes.ts"; import { loadReport, normalizeSymbol, saveReport } from "./reports/store.ts"; import { handleLookupRoute } from "./symbols/routes.ts"; +import { handleResearchRoute } from "./research/routes.ts"; +import { SerpClient } from "./research/serp.ts"; import { cryptoDeepLinkRedirect, handleCryptoRoute } from "./crypto/routes.ts"; import { cryptoSitemapEntries } from "./crypto/page.ts"; import { SUPPORTED_PAIRS } from "./crypto/pairs.ts"; @@ -60,6 +64,11 @@ const coinpay = new CoinPayClient({ businessId: config.secrets.coinpayBusinessId, webhookSecret: config.secrets.coinpayWebhookSecret, }); +// Interactive web/news search behind the Search tab. Optional: without a key +// the tab still searches transcripts and still parses a pasted URL, and +// /api/web answers 503 rather than pretending. +const serp = new SerpClient({ apiKey: config.secrets.valueSerpApiKey }); + const authDeps = { db, mailer, @@ -435,6 +444,9 @@ const server = Bun.serve({ "POST /api/report/regenerate": "rebuild one snapshot (watchlist members only)", "GET /api/discover?topic=&provider=offline&horizon=2&limit=": "ranked watchlist", "GET /api/lookup?q=&limit=": "find a ticker by company name (e.g. q=rivian -> RIVN)", + "GET /api/web?q=&kind=web|news&pages=&time=": + "web or news search: results, related searches, recurring phrases, niches", + "GET /api/parse?url=": "fetch one URL and describe it: title, publisher, phrases, tickers", "GET /api/watchlist": "your saved tickers (requires sign-in); ?format=csv downloads them", "GET /api/watchlist/overview?range=1M|3M|6M|1Y": "the same tickers priced: per-row changes, sparkline and score, summary statistics, and an equal-weight index against SPY", @@ -839,6 +851,10 @@ const server = Bun.serve({ const lookupResponse = await handleLookupRoute(req, p, { db }); if (lookupResponse) return lookupResponse; + // Web/news search and pasted-URL parsing (/api/web, /api/parse). + const researchResponse = await handleResearchRoute(req, p, { db, serp }); + if (researchResponse) return researchResponse; + const reportResponse = await handleReportRoute(req, p, { db, appUrl: config.appUrl, diff --git a/test/dashboard-search.test.ts b/test/dashboard-search.test.ts new file mode 100644 index 0000000..75b27a3 --- /dev/null +++ b/test/dashboard-search.test.ts @@ -0,0 +1,346 @@ +/** + * Search tab — the real public/index.html and public/app.js, driven in a real DOM. + * + * The tab's whole design is a routing decision made in the browser: one box + * that decides, from what you typed, whether to search transcripts, search the + * web, or fetch a pasted page. That decision is not visible in any API test, + * so it is pinned here. + * + * Two behaviours matter more than the rendering and are asserted directly: + * + * - **A dead web search must not take the transcripts down with it.** Web + * search runs on a metered third-party budget that will empty; when it + * answers 402 the tab still has to show what it got for free. + * - **The server's explanation has to reach the reader.** "Credits are + * exhausted" is actionable; "failed (402)" is not. + * + * Hermetic: every request is answered from the fixtures below, so this never + * needs a server, a key, or a credit. + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { JSDOM, VirtualConsole } from "jsdom"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const PUBLIC_DIR = join(import.meta.dir, "..", "public"); +const read = (f: string) => readFileSync(join(PUBLIC_DIR, f), "utf8"); + +/* ---- Fixtures ---------------------------------------------------------- */ + +const WEB_RESPONSE = { + kind: "web", + query: "ai infrastructure", + pages: 1, + cached: false, + results: [ + { + title: "Data center buildout accelerates", + url: "https://www.reuters.com/tech/buildout", + host: "reuters.com", + tier: 1, + tierLabel: "reputable press", + publisher: "reuters.com", + snippet: "Operators are racing to add capacity.", + publishedAt: "2026-08-27", + }, + { + title: "Inside the data center buildout", + url: "https://www.cnbc.com/2026/08/28/inside", + host: "cnbc.com", + tier: 1, + tierLabel: "reputable press", + publisher: "cnbc.com", + snippet: "A tour of the power problem.", + }, + ], + trending: ["ai infrastructure stocks", "data center reits"], + questions: ["What counts as AI infrastructure?"], + phrases: [{ phrase: "data center buildout", count: 2, words: 3 }], + niches: [ + { label: "data center buildout", count: 2, hosts: ["reuters.com", "cnbc.com"], members: [0, 1] }, + ], + sources: [ + { host: "reuters.com", count: 1, tier: 1, tierLabel: "reputable press" }, + { host: "cnbc.com", count: 1, tier: 1, tierLabel: "reputable press" }, + ], + creditsRemaining: 24113, +}; + +const PARSE_RESPONSE = { + url: "https://www.cnbc.com/2026/08/28/inside", + host: "cnbc.com", + tier: 1, + tierLabel: "reputable press", + ok: true, + cached: false, + meta: { + title: "Inside the data center buildout", + description: "A tour of the power problem.", + siteName: "CNBC", + author: "A Reporter", + publishedAt: "2026-08-28T11:00:00.000Z", + keywords: ["data centers", "power"], + headings: ["The power problem", "What comes next"], + feeds: ["https://www.cnbc.com/id/100003114/device/rss/rss.html"], + }, + phrases: [{ phrase: "data center", count: 4, words: 2 }], + tickers: [ + { symbol: "NVDA", name: "NVIDIA Corporation", hasReport: true }, + { symbol: "SOUN", hasReport: false }, + ], + textLength: 4200, + wordCount: 700, + excerpt: "The buildout is limited by power, not by chips.", +}; + +const TRANSCRIPTS_RESPONSE = { + query: "ai infrastructure", + results: [ + { + ticker: "RIVN", + event_date: "2026-05-01", + speaker: "CEO", + text: "We are expanding our compute footprint this year.", + }, + ], +}; + +/* ---- Harness ------------------------------------------------------------ */ + +let dom: JSDOM; +let win: any; +let pageErrors: string[] = []; +let requests: string[] = []; +/** Per-path status overrides, so error paths (402, 503) can be exercised. */ +let failures: Record<string, { status: number; body: unknown }> = {}; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +const $ = (sel: string) => win.document.querySelector(sel); +const $$ = (sel: string) => [...win.document.querySelectorAll(sel)]; +const text = (sel: string) => $(sel)?.textContent ?? ""; +const click = (el: any) => el?.dispatchEvent(new win.MouseEvent("click", { bubbles: true })); +const sent = (fragment: string) => requests.some((u) => u.includes(fragment)); + +function respond(rawUrl: string): unknown { + const url = new URL(rawUrl, "http://localhost"); + const p = url.pathname; + if (p === "/health") return { ok: true }; + if (p === "/api/stats") return { documents: 1, signals: 2, transcripts: 3, analyses: 4 }; + if (p === "/api/topics") return { topics: ["AI infrastructure"] }; + if (p === "/api/discover") return { candidates: [], disclaimer: "" }; + if (p === "/api/auth/me") return { user: null }; + if (p === "/api/search") return TRANSCRIPTS_RESPONSE; + if (p === "/api/web") return { ...WEB_RESPONSE, kind: url.searchParams.get("kind") ?? "web" }; + if (p === "/api/parse") return { ...PARSE_RESPONSE, url: url.searchParams.get("url") }; + return {}; +} + +/** Type into the search box and run it. */ +async function search(query: string, opts: { mode?: string; time?: string } = {}) { + $("#sq").value = query; + if (opts.mode) $("#sq-mode").value = opts.mode; + if (opts.time) $("#sq-time").value = opts.time; + requests = []; + click($("#sq-run")); + await sleep(60); +} + +async function loadPage(where = "search") { + pageErrors = []; + requests = []; + failures = {}; + const vc = new VirtualConsole(); + vc.on("jsdomError", (e: Error) => pageErrors.push(e.message)); + vc.on("error", (...a: unknown[]) => pageErrors.push(a.join(" "))); + + dom = new JSDOM(read("index.html"), { + url: `http://localhost/${where}`, + runScripts: "outside-only", + pretendToBeVisual: true, + virtualConsole: vc, + }); + win = dom.window as any; + win.LightweightCharts = null; + win.ResizeObserver = class { observe() {} unobserve() {} disconnect() {} }; + win.alert = () => {}; + win.fetch = async (input: any) => { + const u = String(input?.url ?? input); + requests.push(u); + const path = new URL(u, "http://localhost").pathname; + const failure = failures[path]; + if (failure) { + return { + ok: false, + status: failure.status, + json: async () => failure.body, + text: async () => JSON.stringify(failure.body), + }; + } + return { ok: true, status: 200, json: async () => respond(u), text: async () => JSON.stringify(respond(u)) }; + }; + + win.eval([read("app.js"), read("auth.js")].join("\n;\n")); + await sleep(150); +} + +beforeEach(async () => { await loadPage(); }); +afterEach(() => { try { win?.close(); } catch {} }); + +/* ---- Tests -------------------------------------------------------------- */ + +describe("search tab", () => { + test("the page evaluates without errors", () => { + expect(pageErrors).toEqual([]); + }); + + test("/search routes straight to the tab", () => { + expect($('.view[data-view="search"]').classList.contains("active")).toBe(true); + }); + + test("the box offers the modes and time windows", () => { + expect($$("#sq-mode option").map((o: any) => o.value)).toEqual([ + "auto", "transcripts", "web", "news", "url", + ]); + expect($$("#sq-time option").map((o: any) => o.value)).toContain("last_week"); + }); +}); + +describe("auto mode", () => { + test("a phrase searches transcripts and the web together", async () => { + await search("ai infrastructure"); + expect(sent("/api/search?q=ai%20infrastructure")).toBe(true); + expect(sent("/api/web?q=ai+infrastructure&kind=web")).toBe(true); + expect(sent("/api/parse")).toBe(false); + // Both sets of results are on the page at once. + expect(text("#search-results")).toContain("RIVN"); + expect(text("#search-results")).toContain("Data center buildout accelerates"); + }); + + test("a pasted URL is parsed instead of searched", async () => { + await search("https://www.cnbc.com/2026/08/28/inside"); + expect(sent("/api/parse?url=")).toBe(true); + expect(sent("/api/web")).toBe(false); + expect(sent("/api/search?q=")).toBe(false); + }); + + test("a URL without a scheme is still recognised as one", async () => { + await search("cnbc.com/2026/08/28/inside"); + expect(sent("/api/parse?url=")).toBe(true); + expect(sent("/api/web")).toBe(false); + }); +}); + +describe("web results", () => { + beforeEach(async () => { await search("ai infrastructure", { mode: "web" }); }); + + test("each result links out, names its publisher and shows its source tier", () => { + const first = $$("#search-results .res")[0]; + const link = first.querySelector(".res-title"); + expect(link.getAttribute("href")).toBe("https://www.reuters.com/tech/buildout"); + // Outbound links must not leak PageRank or referrer trust. + expect(link.getAttribute("rel")).toContain("noopener"); + expect(link.getAttribute("rel")).toContain("nofollow"); + expect(first.textContent).toContain("reuters.com"); + expect(first.textContent).toContain("reputable press"); + expect(first.textContent).toContain("Operators are racing to add capacity."); + }); + + test("related searches, recurring phrases and niches all render", () => { + const body = text("#search-results"); + expect(body).toContain("ai infrastructure stocks"); // Google's related search + expect(body).toContain("data center buildout"); // computed phrase + niche + expect(body).toContain("What counts as AI infrastructure?"); // people-also-ask + expect(body).toContain("cnbc.com"); // publisher breakdown + }); + + test("the credit balance is shown, because this is the search that costs money", () => { + expect(text("#search-results")).toContain("24,113 credits left"); + }); + + test("clicking a phrase runs it as the next search", async () => { + const chip = $$("#search-results .chip-btn").find((c: any) => c.dataset.q === "data center buildout"); + expect(chip).toBeTruthy(); + requests = []; + click(chip); + await sleep(60); + expect($("#sq").value).toBe("data center buildout"); + expect(sent("q=data+center+buildout")).toBe(true); + }); + + test("a result can be handed straight to the parser", async () => { + const parse = $$("#search-results [data-parse]")[0]; + requests = []; + click(parse); + await sleep(60); + expect(sent("/api/parse?url=https%3A%2F%2Fwww.reuters.com%2Ftech%2Fbuildout")).toBe(true); + }); +}); + +describe("news mode", () => { + test("news is requested as news, with the time window applied", async () => { + await search("soundhound", { mode: "news", time: "last_week" }); + expect(sent("kind=news")).toBe(true); + expect(sent("time=last_week")).toBe(true); + expect(sent("/api/search?q=")).toBe(false); // news mode is news only + }); +}); + +describe("a parsed page", () => { + beforeEach(async () => { await search("https://www.cnbc.com/2026/08/28/inside"); }); + + test("the page is described: title, publisher, byline and length", () => { + const body = text("#search-results"); + expect(body).toContain("Inside the data center buildout"); + expect(body).toContain("CNBC"); + expect(body).toContain("A Reporter"); + expect(body).toContain("700 words"); + expect(body).toContain("The buildout is limited by power, not by chips."); + }); + + test("its outline, declared keywords and feeds come through", () => { + const body = text("#search-results"); + expect(body).toContain("The power problem"); + expect(body).toContain("data centers"); + expect($$("#search-results a").some((a: any) => a.href.includes("rss"))).toBe(true); + }); + + test("a ticker it names links to that stock's report", () => { + const chip = $$("#search-results .tlink").find((c: any) => c.dataset.ticker === "NVDA"); + expect(chip).toBeTruthy(); + expect(chip.textContent).toContain("NVIDIA Corporation"); + // A real link, so it can be copied, middle-clicked and shared; the + // document-level `.tlink` handler intercepts an ordinary click. + expect(chip.tagName).toBe("A"); + expect(chip.getAttribute("href")).toBe("/stocks/NVDA"); + }); +}); + +describe("when the web search cannot run", () => { + test("an exhausted budget says so, and the transcripts still show", async () => { + failures["/api/web"] = { status: 402, body: { error: "Web search credits are exhausted for this month." } }; + await search("ai infrastructure"); + const body = text("#search-results"); + expect(body).toContain("credits are exhausted"); + // The point of the assertion: free results survive a paid failure. + expect(body).toContain("RIVN"); + expect(body).toContain("We are expanding our compute footprint"); + }); + + test("a deployment with no key says that instead of failing vaguely", async () => { + failures["/api/web"] = { status: 503, body: { error: "not configured" } }; + await search("ai infrastructure", { mode: "web" }); + expect(text("#search-results")).toContain("not configured"); + }); + + test("a throttled caller is told when to come back", async () => { + failures["/api/web"] = { status: 429, body: { error: "Too many web searches. Try again in 60 minutes." } }; + await search("ai infrastructure", { mode: "web" }); + expect(text("#search-results")).toContain("Try again in 60 minutes"); + }); + + test("a refused URL shows the reason it was refused", async () => { + failures["/api/parse"] = { status: 400, body: { error: "That host resolves to a private address." } }; + await search("http://10.0.0.5/admin"); + expect(text("#search-results")).toContain("private address"); + }); +}); diff --git a/test/research.test.ts b/test/research.test.ts new file mode 100644 index 0000000..1377d68 --- /dev/null +++ b/test/research.test.ts @@ -0,0 +1,294 @@ +/** + * Search-tab research tests: phrase/niche extraction, SERP parsing, and the + * pasted-URL path. + * + * The three things worth pinning down are the three that would fail quietly: + * a phrase list that reports one loud document as a trend, a SERP parser that + * silently returns nothing when a field is renamed, and a URL guard that lets + * an anonymous caller aim the server at a private address. + */ +import { describe, expect, test } from "bun:test"; +import { deriveNiches, tokenize, topPhrases } from "../src/research/phrases.ts"; +import { + creditsFrom, + newsHits, + parseQuestions, + parseRelatedSearches, + webHits, +} from "../src/research/serp.ts"; +import { + assertFetchableUrl, + extractPageMeta, + extractTickerMentions, + isPrivateAddress, + looksLikeUrl, + normalizeUrlInput, + UnfetchableUrlError, +} from "../src/research/page.ts"; + +describe("phrase extraction", () => { + test("tokenize drops numbers and punctuation but keeps hyphenated words", () => { + expect(tokenize("Q3 2026: AI-infrastructure spend, up 40%!")).toEqual([ + "q3", + "ai-infrastructure", + "spend", + "up", + ]); + }); + + test("scores by document frequency, so one repetitive document is not a trend", () => { + const shouty = "data centers data centers data centers data centers data centers"; + const others = ["quarterly guidance raised", "quarterly guidance raised again"]; + const phrases = topPhrases([shouty, ...others, "margin pressure"]); + const dc = phrases.find((p) => p.phrase === "data centers"); + const guidance = phrases.find((p) => p.phrase === "quarterly guidance raised"); + // Five mentions inside one document still count once. + expect(dc).toBeUndefined(); + expect(guidance?.count).toBe(2); + }); + + test("keeps the specific phrase and drops the generic one it contains", () => { + const docs = [ + "ai infrastructure spending is up", + "ai infrastructure spending accelerates", + "ai infrastructure spending in 2026", + ]; + const phrases = topPhrases(docs, { maxWords: 3 }); + expect(phrases[0]!.phrase).toBe("ai infrastructure spending"); + expect(phrases.map((p) => p.phrase)).not.toContain("ai"); + expect(phrases.map((p) => p.phrase)).not.toContain("infrastructure spending"); + }); + + test("a phrase cannot begin or end on a stopword", () => { + const docs = [ + "the future of solid state batteries", + "the future of solid state batteries explained", + ]; + for (const { phrase } of topPhrases(docs)) { + expect(phrase.startsWith("the ")).toBe(false); + expect(phrase.endsWith(" of")).toBe(false); + } + }); + + test("a single document still yields phrases (minCount falls to 1)", () => { + expect(topPhrases(["rivian delivery guidance for the fourth quarter"]).length).toBeGreaterThan(0); + }); + + test("empty input is empty output, not a crash", () => { + expect(topPhrases([])).toEqual([]); + expect(topPhrases(["", " "])).toEqual([]); + }); +}); + +describe("niches", () => { + const items = [ + { title: "Data center buildout accelerates", host: "reuters.com" }, + { title: "Inside the data center buildout", host: "cnbc.com" }, + { title: "Data center buildout hits power limits", host: "reuters.com" }, + { title: "A lone story about tractors", host: "example.com" }, + ]; + + test("clusters results by the phrase they share and names the publishers", () => { + const niches = deriveNiches(items); + const cluster = niches.find((n) => n.label.includes("data center buildout")); + expect(cluster?.count).toBe(3); + expect(cluster?.hosts[0]).toBe("reuters.com"); // two of the three + expect(cluster?.members).toEqual([0, 1, 2]); + }); + + test("a niche of one is not a niche", () => { + expect(deriveNiches(items).some((n) => n.label.includes("tractors"))).toBe(false); + }); +}); + +describe("SERP parsing", () => { + const body = { + request_info: { success: true, credits_remaining: 24_113 }, + organic_results: [ + { + position: 1, + title: "Nvidia earnings beat", + link: "https://www.reuters.com/tech/nvidia-earnings", + domain: "reuters.com", + snippet: "Revenue rose on data center demand.", + date: "2 days ago", + }, + { position: 2, title: "No link here" }, + ], + related_searches: [{ query: "nvidia stock forecast" }, { q: "nvidia competitors" }], + related_questions: [{ question: "Is Nvidia overvalued?", answer: "…" }], + }; + + test("organic results carry publisher, tier and a resolved date", () => { + const asOf = new Date("2026-08-29T00:00:00Z"); + const hits = webHits(body, asOf); + expect(hits).toHaveLength(1); // the row without a link is dropped + expect(hits[0]!.host).toBe("reuters.com"); + expect(hits[0]!.tier).toBe(1); + expect(hits[0]!.tierLabel).toBe("reputable press"); + expect(hits[0]!.publishedAt).toBe("2026-08-27"); + }); + + test("related searches read both field spellings the API has used", () => { + expect(parseRelatedSearches(body)).toEqual(["nvidia stock forecast", "nvidia competitors"]); + }); + + test("people-also-ask questions are extracted", () => { + expect(parseQuestions(body)).toEqual(["Is Nvidia overvalued?"]); + }); + + test("credits are read off the response, so no extra account call is needed", () => { + expect(creditsFrom(body)).toBe(24_113); + expect(creditsFrom({})).toBeUndefined(); + }); + + test("news results reuse the ingest parser and tier identically", () => { + const hits = newsHits( + { news_results: [{ title: "Promo alert", link: "https://stocktwits.com/x", source: "Stocktwits" }] }, + new Date(), + ); + expect(hits[0]!.publisher).toBe("Stocktwits"); + expect(hits[0]!.tierLabel).toBe("excluded"); + }); + + test("a response with no results is empty, not an exception", () => { + expect(webHits({}, new Date())).toEqual([]); + expect(parseRelatedSearches(null)).toEqual([]); + }); +}); + +describe("URL detection", () => { + test("links are recognised with or without a scheme", () => { + expect(looksLikeUrl("https://reuters.com/tech/story")).toBe(true); + expect(looksLikeUrl("reuters.com/tech/story")).toBe(true); + expect(looksLikeUrl("www.reuters.com")).toBe(true); + }); + + test("ordinary searches are not mistaken for links", () => { + expect(looksLikeUrl("rivian earnings")).toBe(false); + expect(looksLikeUrl("3.5")).toBe(false); + expect(looksLikeUrl("")).toBe(false); + // Other schemes are refused outright rather than parsed. + expect(looksLikeUrl("javascript:alert(1)")).toBe(false); + expect(looksLikeUrl("file:///etc/passwd")).toBe(false); + }); + + test("a missing scheme is filled in as https", () => { + expect(normalizeUrlInput("reuters.com/x")).toBe("https://reuters.com/x"); + expect(normalizeUrlInput("http://reuters.com/x")).toBe("http://reuters.com/x"); + }); +}); + +describe("SSRF guard", () => { + test("private and reserved ranges are recognised", () => { + for (const ip of ["127.0.0.1", "10.1.2.3", "192.168.1.1", "172.16.0.1", "169.254.169.254", + "100.64.0.1", "0.0.0.0", "224.0.0.1"]) { + expect(isPrivateAddress(ip, 4)).toBe(true); + } + for (const ip of ["8.8.8.8", "1.1.1.1", "104.18.32.7", "172.32.0.1"]) { + expect(isPrivateAddress(ip, 4)).toBe(false); + } + }); + + test("IPv6 loopback, unique-local, link-local and v4-mapped are recognised", () => { + expect(isPrivateAddress("::1", 6)).toBe(true); + expect(isPrivateAddress("fd00::1", 6)).toBe(true); + expect(isPrivateAddress("fe80::1", 6)).toBe(true); + expect(isPrivateAddress("::ffff:10.0.0.1", 6)).toBe(true); + expect(isPrivateAddress("2606:4700::1111", 6)).toBe(false); + }); + + test("loopback and internal names are refused before any fetch", async () => { + for (const target of [ + "http://localhost/admin", + "http://foo.local/", + "http://service.internal/", + "http://127.0.0.1/", + "http://169.254.169.254/latest/meta-data/", + "http://[::1]/", + "http://10.0.0.5/", + ]) { + expect(assertFetchableUrl(target)).rejects.toThrow(UnfetchableUrlError); + } + }); + + test("non-web schemes and non-standard ports are refused", async () => { + expect(assertFetchableUrl("ftp://example.com/x")).rejects.toThrow(/http and https/); + expect(assertFetchableUrl("http://example.com:8080/x")).rejects.toThrow(/standard web ports/); + expect(assertFetchableUrl("not a url at all")).rejects.toThrow(UnfetchableUrlError); + }); +}); + +describe("page metadata", () => { + const html = `<!doctype html><html lang="en"><head> + <title>Story — Section | The Publisher + + + + + + + + + +

On-page headline

The buildout

`; + + test("publisher-declared metadata beats the decorated title tag", () => { + const meta = extractPageMeta(html); + expect(meta.title).toBe("Nvidia's data center quarter"); + expect(meta.siteName).toBe("The Publisher"); + expect(meta.description).toBe("What the numbers showed."); + expect(meta.author).toBe("A Reporter"); + expect(meta.publishedAt).toBe("2026-08-20T11:00:00.000Z"); + expect(meta.canonical).toBe("https://example.com/story"); + expect(meta.lang).toBe("en"); + expect(meta.keywords).toEqual(["nvidia", "data centers", "earnings"]); + expect(meta.headings).toEqual(["On-page headline", "The buildout"]); + expect(meta.feeds).toEqual(["https://example.com/feed.xml"]); + }); + + test("the title tag is the fallback when nothing is declared", () => { + const meta = extractPageMeta("Just a title"); + expect(meta.title).toBe("Just a title"); + expect(meta.description).toBeUndefined(); + expect(meta.keywords).toEqual([]); + }); + + test("JSON-LD supplies the author when no meta tag does", () => { + const meta = extractPageMeta( + ``, + ); + expect(meta.author).toBe("LD Reporter"); + expect(meta.title).toBe("Graph headline"); + }); + + test("a malformed JSON-LD block does not lose the page", () => { + const meta = extractPageMeta( + `Fine`, + ); + expect(meta.title).toBe("Fine"); + }); +}); + +describe("ticker mentions", () => { + test("cashtags and exchange-qualified mentions are read", () => { + const tickers = extractTickerMentions( + "Shares of $NVDA rose after (NASDAQ: SOUN) reported. NYSE American: XYZ also moved. $BRK.B held.", + ); + expect(tickers).toContain("NVDA"); + expect(tickers).toContain("SOUN"); + expect(tickers).toContain("XYZ"); + expect(tickers).toContain("BRK.B"); + }); + + test("bare capitals are not tickers", () => { + // The whole point: a confidently wrong ticker is worse than none. + expect(extractTickerMentions("The CEO said AI and EV demand in the US was strong")).toEqual([]); + }); + + test("the most-repeated ticker leads", () => { + expect(extractTickerMentions("$AAPL $MSFT $AAPL")[0]).toBe("AAPL"); + }); +});