From 05e69d28868304af55d8f5d928cf7871d843a0ec Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Thu, 10 Sep 2026 14:44:03 +0000 Subject: [PATCH] The IMDb tables and a torrent's title can come from nichedb instead of the dumps and TMDB bittorrented imported the IMDb dumps nightly and asked TMDB for every torrent page's poster. nichedb.dev keeps both now, so this adds the other path and leaves the old one in place. With NICHEDB_TITLES=1 a torrent page asks nichedb what its name is before TMDB, and takes the answer only when it comes with a poster or a synopsis; a bare IMDb row falls through to TMDB so no page loses its picture. The tmdb_data row written is the same shape either way. With NICHEDB_MIRROR=1 the nightly crontab line runs a walk of the screen collection instead of the dump import, filling imdb_title_basics and imdb_title_ratings from IMDb-sourced rows (TMDB and TVmaze rows only add a title they know the tconst of, and never write a rating, since theirs is a different scale). A cursor in nichedb_mirror_cursor resumes a walk cut short by the request budget; the first run is the backfill, about 1,700 pages seven seconds apart. Crew, principals, episodes, akas and names are not in nichedb, so those tables stop refreshing under the mirror; the director line on a DHT page freezes at its last dump value until nichedb carries crew. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014X1XCwt5iD3nqZfbddZqEP --- .env.example | 8 + package.json | 1 + scripts/mirror-imdb-from-nichedb.sh | 35 +++ scripts/mirror-imdb-from-nichedb.ts | 180 +++++++++++ scripts/setup-server.sh | 41 ++- src/lib/imdb/tmdb.test.ts | 207 +++++++++++++ src/lib/imdb/tmdb.ts | 58 +++- src/lib/nichedb/fixtures.ts | 130 ++++++++ src/lib/nichedb/mirror.test.ts | 245 +++++++++++++++ src/lib/nichedb/mirror.ts | 237 +++++++++++++++ src/lib/nichedb/titles.test.ts | 189 ++++++++++++ src/lib/nichedb/titles.ts | 282 ++++++++++++++++++ .../20260910150000_nichedb_mirror_cursor.sql | 30 ++ 13 files changed, 1638 insertions(+), 5 deletions(-) create mode 100755 scripts/mirror-imdb-from-nichedb.sh create mode 100755 scripts/mirror-imdb-from-nichedb.ts create mode 100644 src/lib/imdb/tmdb.test.ts create mode 100644 src/lib/nichedb/fixtures.ts create mode 100644 src/lib/nichedb/mirror.test.ts create mode 100644 src/lib/nichedb/mirror.ts create mode 100644 src/lib/nichedb/titles.test.ts create mode 100644 src/lib/nichedb/titles.ts create mode 100644 supabase/migrations/20260910150000_nichedb_mirror_cursor.sql diff --git a/.env.example b/.env.example index 34f23c94..dd822bf8 100644 --- a/.env.example +++ b/.env.example @@ -44,6 +44,14 @@ FANART_TV_API_KEY=your-fanart-api-key TMDB_API_KEY=your-tmdb-api-key MUSICBRAINZ_USER_AGENT=BitTorrented/1.0.0 (https://bittorrented.com) +# nichedb.dev `screen` collection as the title source (default off for both) +# NICHEDB_TITLES=1 torrent pages ask nichedb for poster/backdrop/overview before TMDB +# NICHEDB_MIRROR=1 setup-server.sh schedules scripts/mirror-imdb-from-nichedb.sh at 00:00 +# instead of the IMDb dump download (pnpm mirror:imdb) +# NICHEDB_MIRROR_MAX_PAGES=480 NICHEDB_MIRROR_INTERVAL_MS=7000 NICHEDB_MIRROR_TAGS= +#NICHEDB_TITLES=1 +#NICHEDB_MIRROR=1 + # CoinPayPortal (Crypto Payments) # Get these from your CoinPayPortal dashboard: https://coinpayportal.com/dashboard COINPAYPORTAL_API_KEY=cp_live_your_api_key diff --git a/package.json b/package.json index 80c7cc75..f2479e1d 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "enrich-folders:all": "tsx --env-file=.env scripts/enrich-folder-metadata.ts --all-status", "enrich-folders:dry-run": "tsx --env-file=.env scripts/enrich-folder-metadata.ts --all-status --dry-run", "enrich-folders:force": "tsx --env-file=.env scripts/enrich-folder-metadata.ts --all-status --force", + "mirror:imdb": "tsx --env-file=.env scripts/mirror-imdb-from-nichedb.ts", "iptv-worker": "tsx --env-file=.env workers/iptv-cache/index.ts", "iptv-worker:dev": "tsx --watch --env-file=.env workers/iptv-cache/index.ts", "podcast-worker": "tsx --env-file=.env workers/podcast-notifier/index.ts", diff --git a/scripts/mirror-imdb-from-nichedb.sh b/scripts/mirror-imdb-from-nichedb.sh new file mode 100755 index 00000000..7e246e63 --- /dev/null +++ b/scripts/mirror-imdb-from-nichedb.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Nightly imdb_* refresh from nichedb.dev, in place of the IMDb dump download. +# +# Runs scripts/mirror-imdb-from-nichedb.ts (pnpm mirror:imdb) with a lock so two +# cron runs never overlap. One run walks at most NICHEDB_MIRROR_MAX_PAGES pages +# (default 480, ~56 minutes at 7 s a page) and stores its cursor, so the first +# backfill spreads over a few nights unless kicked off by hand with +# pnpm mirror:imdb -- --all +# +# crontab (set up by setup-server.sh when NICHEDB_MIRROR=1): +# 0 0 * * * /home/ubuntu/src/media-streamer/scripts/mirror-imdb-from-nichedb.sh >> /var/log/imdb-update.log 2>&1 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +LOCK_FILE="/tmp/imdb-update.lock" + +if [ -f "$LOCK_FILE" ]; then + pid=$(cat "$LOCK_FILE") + if kill -0 "$pid" 2>/dev/null; then + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Another IMDb update is running (pid $pid), skipping" + exit 0 + fi +fi +echo $$ > "$LOCK_FILE" +trap 'rm -f "$LOCK_FILE"' EXIT + +# cron's PATH has no pnpm; pick up the user's toolchain. +export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$PATH" + +cd "$PROJECT_DIR" +echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Starting nichedb -> imdb_* mirror" +pnpm mirror:imdb "$@" +echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] nichedb mirror finished" diff --git a/scripts/mirror-imdb-from-nichedb.ts b/scripts/mirror-imdb-from-nichedb.ts new file mode 100755 index 00000000..47db0f67 --- /dev/null +++ b/scripts/mirror-imdb-from-nichedb.ts @@ -0,0 +1,180 @@ +#!/usr/bin/env npx tsx +/** + * Mirror IMDb titles from nichedb.dev into imdb_title_basics / imdb_title_ratings. + * + * Replaces the nightly IMDb dump download (scripts/update-imdb-daily.sh) for the + * two tables the site reads on every torrent page. It walks nichedb's `screen` + * collection over the public API, id-ordered, with a cursor kept in + * nichedb_mirror_cursor so a run can stop at its page cap and the next run + * carries on. Once a walk reaches the end, later walks send `since=` and only see + * titles nichedb updated after the previous walk began. + * + * Usage: + * pnpm mirror:imdb one capped run (cron) + * pnpm mirror:imdb -- --all run until the walk completes (first backfill, ~3-4 h) + * pnpm mirror:imdb -- --max-pages=N --interval-ms=MS --dry-run --reset + * + * Options: + * --all No page cap for this run + * --max-pages=N Pages per run (default NICHEDB_MIRROR_MAX_PAGES or 480) + * --interval-ms=MS Pause between pages (default NICHEDB_MIRROR_INTERVAL_MS or 7000). + * nichedb allows 600 requests/hour per IP; 7 s is ~514/h, leaving + * the rest for the site's own match calls from the same box. + * --tags=a,b Only walk items carrying all these tags (default: none, all titles; + * `imdb` walks the IMDb-sourced rows only) + * --reset Forget the cursor and start a full backfill + * --dry-run Fetch and map, write nothing (cursor included) + * + * Required environment variables: + * SUPABASE_URL (or NEXT_PUBLIC_SUPABASE_URL), SUPABASE_SERVICE_ROLE_KEY + * Optional: + * NICHEDB_BASE_URL (default https://nichedb.dev) + */ + +import { config } from 'dotenv'; +import { createClient } from '@supabase/supabase-js'; + +config(); + +import { + MIRROR_CURSOR_NAME, + runMirror, + type MirrorCursor, + type MirrorRows, + type MirrorStore, +} from '../src/lib/nichedb/mirror'; + +interface Options { + maxPages: number; + intervalMs: number; + tags: string[]; + reset: boolean; + dryRun: boolean; +} + +function parseArgs(): Options { + const opts: Options = { + maxPages: Number(process.env.NICHEDB_MIRROR_MAX_PAGES) || 480, + intervalMs: Number(process.env.NICHEDB_MIRROR_INTERVAL_MS) || 7000, + tags: (process.env.NICHEDB_MIRROR_TAGS ?? '').split(',').filter(Boolean), + reset: false, + dryRun: false, + }; + for (const arg of process.argv.slice(2)) { + if (arg === '--all') opts.maxPages = 0; + else if (arg.startsWith('--max-pages=')) opts.maxPages = Number(arg.slice('--max-pages='.length)) || 0; + else if (arg.startsWith('--interval-ms=')) opts.intervalMs = Number(arg.slice('--interval-ms='.length)) || 0; + else if (arg.startsWith('--tags=')) opts.tags = arg.slice('--tags='.length).split(',').filter(Boolean); + else if (arg === '--reset') opts.reset = true; + else if (arg === '--dry-run') opts.dryRun = true; + else if (arg === '--help' || arg === '-h') { + console.log('Usage: pnpm mirror:imdb -- [--all] [--max-pages=N] [--interval-ms=MS] [--tags=a,b] [--reset] [--dry-run]'); + process.exit(0); + } else { + console.error(`Unknown option: ${arg}`); + process.exit(1); + } + } + return opts; +} + +function stamp(): string { + return new Date().toISOString(); +} + +/** --dry-run without Supabase credentials: walk from the start, keep nothing. */ +function memoryStore(): MirrorStore { + let cursor: MirrorCursor | null = null; + return { + async loadCursor() { return cursor; }, + async saveCursor(_name, c) { cursor = { ...c }; }, + async writeRows() {}, + }; +} + +function makeStore(dryRun: boolean): MirrorStore { + const url = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL; + const key = process.env.SUPABASE_SERVICE_ROLE_KEY; + if (!url || !key) { + if (dryRun) { + console.log('no SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY: dry run with an in-memory cursor'); + return memoryStore(); + } + console.error('ERROR: SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY are required'); + process.exit(1); + } + const supabase = createClient(url, key, { auth: { persistSession: false } }); + + return { + async loadCursor(name: string): Promise { + const { data, error } = await supabase + .from('nichedb_mirror_cursor') + .select('after_id, since') + .eq('name', name) + .maybeSingle(); + if (error) throw new Error(`cursor read failed: ${error.message}`); + if (!data) return null; + return { after_id: Number(data.after_id) || 0, since: data.since ?? null }; + }, + async saveCursor(name: string, cursor: MirrorCursor): Promise { + if (dryRun) return; + const { error } = await supabase + .from('nichedb_mirror_cursor') + .upsert( + { name, after_id: cursor.after_id, since: cursor.since, updated_at: new Date().toISOString() }, + { onConflict: 'name' }, + ); + if (error) throw new Error(`cursor write failed: ${error.message}`); + }, + async writeRows(rows: MirrorRows): Promise { + if (dryRun) return; + if (rows.basics.length) { + const { error } = await supabase + .from('imdb_title_basics') + .upsert(rows.basics, { onConflict: 'tconst' }); + if (error) throw new Error(`imdb_title_basics upsert failed: ${error.message}`); + } + if (rows.basicsFill.length) { + // Rows from TMDB/TVmaze that know a tconst: fill gaps, never overwrite IMDb's own row. + const { error } = await supabase + .from('imdb_title_basics') + .upsert(rows.basicsFill, { onConflict: 'tconst', ignoreDuplicates: true }); + if (error) throw new Error(`imdb_title_basics fill failed: ${error.message}`); + } + if (rows.ratings.length) { + const { error } = await supabase + .from('imdb_title_ratings') + .upsert(rows.ratings, { onConflict: 'tconst' }); + if (error) throw new Error(`imdb_title_ratings upsert failed: ${error.message}`); + } + }, + }; +} + +async function main(): Promise { + const opts = parseArgs(); + const store = makeStore(opts.dryRun); + + console.log(`[${stamp()}] nichedb -> imdb_* mirror starting (maxPages=${opts.maxPages || 'unlimited'}, interval=${opts.intervalMs}ms, tags=${opts.tags.join(',') || 'none'}${opts.dryRun ? ', DRY RUN' : ''})`); + + if (opts.reset) { + console.log(`[${stamp()}] resetting cursor ${MIRROR_CURSOR_NAME}`); + await store.saveCursor(MIRROR_CURSOR_NAME, { after_id: 0, since: null }); + } + + const result = await runMirror({ + store, + maxPages: opts.maxPages, + intervalMs: opts.intervalMs, + tags: opts.tags.length ? opts.tags : undefined, + log: (line) => console.log(`[${stamp()}] ${line}`), + }); + + console.log(`[${stamp()}] done: ${result.pages} pages, ${result.items} items, ${result.basics} basics upserted, ${result.basicsFill} basics filled, ${result.ratings} ratings upserted; ${result.stoppedBecause}; cursor after_id=${result.cursor.after_id} since=${result.cursor.since ?? 'none'}`); + if (!result.completed && result.stoppedBecause !== 'page-cap') process.exitCode = 1; +} + +main().catch((err) => { + console.error(`[${stamp()}] FAILED:`, err instanceof Error ? err.message : err); + process.exit(1); +}); diff --git a/scripts/setup-server.sh b/scripts/setup-server.sh index d1c2fd44..6ef6b57f 100644 --- a/scripts/setup-server.sh +++ b/scripts/setup-server.sh @@ -1032,19 +1032,42 @@ sudo systemctl enable ${IPTV_WORKER_SERVICE} 2>/dev/null || true sudo systemctl enable ${PODCAST_WORKER_SERVICE} 2>/dev/null || true # Set up IMDB dataset daily update cron job (runs at midnight UTC) +# +# Two sources for imdb_title_basics / imdb_title_ratings: +# default update-imdb-daily.sh downloads the IMDb dumps and COPYs all seven tables +# NICHEDB_MIRROR=1 mirror-imdb-from-nichedb.sh walks nichedb.dev's `screen` collection +# over its API (basics + ratings only; crew/episode/akas/principals/ +# name.basics are not refreshed by it) +# Switching flips the 00:00 crontab line; the other script stays in the repo. echo "=== Setting up IMDB dataset daily update cron job ===" -IMDB_UPDATE_SCRIPT="${PROJECT_ROOT}/scripts/update-imdb-daily.sh" +if [ "${NICHEDB_MIRROR:-}" = "1" ]; then + IMDB_UPDATE_SCRIPT="${PROJECT_ROOT}/scripts/mirror-imdb-from-nichedb.sh" + IMDB_UPDATE_MARK="mirror-imdb-from-nichedb" + IMDB_UPDATE_OTHER="update-imdb-daily" + IMDB_UPDATE_LABEL="nichedb -> imdb_* mirror" +else + IMDB_UPDATE_SCRIPT="${PROJECT_ROOT}/scripts/update-imdb-daily.sh" + IMDB_UPDATE_MARK="update-imdb-daily" + IMDB_UPDATE_OTHER="mirror-imdb-from-nichedb" + IMDB_UPDATE_LABEL="IMDB dataset update" +fi IMDB_CRON_JOB="0 0 * * * ${IMDB_UPDATE_SCRIPT} >> /var/log/imdb-update.log 2>&1" if [ -f "${IMDB_UPDATE_SCRIPT}" ]; then chmod +x "${IMDB_UPDATE_SCRIPT}" chmod +x "${PROJECT_ROOT}/scripts/import-imdb.sh" 2>/dev/null || true + chmod +x "${PROJECT_ROOT}/scripts/mirror-imdb-from-nichedb.sh" 2>/dev/null || true sudo touch /var/log/imdb-update.log sudo chown ${VPS_USER}:${VPS_USER} /var/log/imdb-update.log - if crontab -l 2>/dev/null | grep -q "update-imdb-daily"; then - echo " IMDB update cron job already exists" + if crontab -l 2>/dev/null | grep -q "${IMDB_UPDATE_OTHER}"; then + # The other source is scheduled; replace it so only one job runs at 00:00. + (crontab -l 2>/dev/null | grep -v "${IMDB_UPDATE_OTHER}" || true) | crontab - + echo " Removed the ${IMDB_UPDATE_OTHER} cron job" + fi + if crontab -l 2>/dev/null | grep -q "${IMDB_UPDATE_MARK}"; then + echo " ${IMDB_UPDATE_LABEL} cron job already exists" else (crontab -l 2>/dev/null || true; echo "${IMDB_CRON_JOB}") | crontab - - echo "✓ Added cron job: IMDB dataset update at midnight daily" + echo "✓ Added cron job: ${IMDB_UPDATE_LABEL} at midnight daily" fi else echo " WARNING: ${IMDB_UPDATE_SCRIPT} not found, skipping IMDB cron setup" @@ -1139,13 +1162,23 @@ echo " Errors: tail -f ${PODCAST_WORKER_ERROR_LOG}" echo "" echo "Scheduled Tasks:" echo " WebTorrent temp cleanup: Daily at midnight" +if [ "${NICHEDB_MIRROR:-}" = "1" ]; then +echo " IMDB titles: Daily at midnight, mirrored from nichedb.dev (basics + ratings)" +else echo " IMDB dataset update: Daily at midnight (incremental)" +fi echo " Directory: ${WEBTORRENT_TMP_DIR}" echo " View cron jobs: crontab -l" echo "" echo "IMDB Datasets:" +if [ "${NICHEDB_MIRROR:-}" = "1" ]; then +echo " First backfill: pnpm mirror:imdb -- --all (~3-4 h at nichedb's 600 req/h)" +echo " Daily update: ./scripts/mirror-imdb-from-nichedb.sh (automatic via cron)" +echo " Note: crew/episode/akas/principals/name tables are not refreshed by the mirror" +else echo " First import: ./scripts/import-imdb.sh ~/tmp/data" echo " Daily update: ./scripts/update-imdb-daily.sh (automatic via cron)" +fi echo " Logs: /var/log/imdb-update.log" # DHT Services output (only if enabled) diff --git a/src/lib/imdb/tmdb.test.ts b/src/lib/imdb/tmdb.test.ts new file mode 100644 index 00000000..f48a902e --- /dev/null +++ b/src/lib/imdb/tmdb.test.ts @@ -0,0 +1,207 @@ +/** + * fetchTmdbData: cache -> nichedb (NICHEDB_TITLES=1) -> TMDB, and what lands in tmdb_data. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { imdbTitle, jsonResponse, tmdbTitle } from '@/lib/nichedb/fixtures'; + +const supabaseMock = vi.hoisted(() => { + const state = { cached: null as Record | null, upserts: [] as Array<{ table: string; row: Record }> }; + const from = vi.fn((table: string) => ({ + select: () => ({ eq: () => ({ single: async () => ({ data: state.cached, error: null }) }) }), + upsert: async (row: Record) => { state.upserts.push({ table, row }); return { error: null }; }, + })); + return { state, from }; +}); + +vi.mock('@supabase/supabase-js', () => ({ + createClient: () => ({ from: supabaseMock.from }), +})); + +import { fetchTmdbData, fetchNichedbData } from './tmdb'; + +function tmdbFind(movie: Record | null) { + return { movie_results: movie ? [movie] : [], tv_results: [] }; +} + +const TMDB_MOVIE = { id: 603, poster_path: '/p.jpg', backdrop_path: '/b.jpg', overview: 'TMDB overview' }; +const TMDB_DETAIL = { + tagline: 'TMDB tagline', + overview: 'TMDB overview', + credits: { cast: [{ name: 'Keanu Reeves' }], crew: [{ department: 'Writing', name: 'Lana Wachowski' }] }, + release_dates: { results: [{ iso_3166_1: 'US', release_dates: [{ certification: 'R' }] }] }, +}; + +/** A fetch that answers nichedb and TMDB by host and records what it was asked. */ +function routedFetch(routes: { nichedb?: (url: URL) => Response; tmdb?: (url: URL) => Response }) { + const calls: URL[] = []; + const fetchMock = vi.fn(async (input: string) => { + const url = new URL(input); + calls.push(url); + if (url.hostname === 'nichedb.dev') return routes.nichedb?.(url) ?? jsonResponse({ count: 0, items: [] }); + if (url.hostname === 'api.themoviedb.org') { + if (routes.tmdb) return routes.tmdb(url); + if (url.pathname.startsWith('/3/find/')) return jsonResponse(tmdbFind(TMDB_MOVIE)); + if (url.pathname.startsWith('/3/movie/')) return jsonResponse(TMDB_DETAIL); + if (url.pathname.startsWith('/3/search/')) return jsonResponse({ results: [] }); + } + return jsonResponse({}, { status: 404 }); + }); + vi.stubGlobal('fetch', fetchMock); + const hosts = () => calls.map((u) => u.hostname + u.pathname); + return { fetchMock, calls, hosts }; +} + +beforeEach(() => { + supabaseMock.state.cached = null; + supabaseMock.state.upserts = []; + vi.stubEnv('SUPABASE_URL', 'https://example.supabase.co'); + vi.stubEnv('SUPABASE_SERVICE_ROLE_KEY', 'service-role'); + vi.stubEnv('TMDB_API_KEY', 'tmdb-key'); + vi.stubEnv('NICHEDB_TITLES', ''); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +describe('fetchTmdbData with NICHEDB_TITLES off (default)', () => { + it('never talks to nichedb and caches the TMDB answer as before', async () => { + const { hosts } = routedFetch({}); + const data = await fetchTmdbData('tt0133093', 'The.Matrix.1999.1080p'); + + expect(hosts().some((h) => h.startsWith('nichedb.dev'))).toBe(false); + expect(hosts()).toEqual(['api.themoviedb.org/3/find/tt0133093', 'api.themoviedb.org/3/movie/603']); + expect(data).toEqual({ + posterUrl: 'https://image.tmdb.org/t/p/w500/p.jpg', + backdropUrl: 'https://image.tmdb.org/t/p/w1280/b.jpg', + overview: 'TMDB overview', + tagline: 'TMDB tagline', + cast: 'Keanu Reeves', + writers: 'Lana Wachowski', + contentRating: 'R', + tmdbId: 603, + }); + expect(supabaseMock.state.upserts).toEqual([{ table: 'tmdb_data', row: expect.objectContaining({ lookup_key: 'tt0133093', tmdb_id: 603 }) }]); + }); + + it('is EMPTY without a TMDB key', async () => { + vi.stubEnv('TMDB_API_KEY', ''); + const { fetchMock } = routedFetch({}); + const data = await fetchTmdbData('tt0133093', 'The Matrix'); + expect(data.posterUrl).toBeNull(); + expect(data.tmdbId).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('fetchTmdbData with NICHEDB_TITLES=1', () => { + beforeEach(() => vi.stubEnv('NICHEDB_TITLES', '1')); + + it('serves the cache first without any network call', async () => { + supabaseMock.state.cached = { poster_url: 'cached.jpg', backdrop_url: null, overview: 'cached', tagline: null, cast_names: null, writers: null, content_rating: null, tmdb_id: 1 }; + const { fetchMock } = routedFetch({}); + const data = await fetchTmdbData('tt0133093', 'The Matrix'); + expect(data.posterUrl).toBe('cached.jpg'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('takes the nichedb TMDB row, skips TMDB, and writes tmdb_data in the same shape', async () => { + const { hosts, calls } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 2, items: [imdbTitle({ score: 1 }), tmdbTitle({ score: 1 })] }), + }); + const data = await fetchTmdbData('tt0133093', 'The.Matrix.1999.1080p.BluRay.x264-GROUP'); + + expect(hosts()).toEqual(['nichedb.dev/api/v1/match']); + expect(calls[0].searchParams.get('q')).toBe('The.Matrix.1999.1080p.BluRay.x264-GROUP'); + expect(data).toEqual({ + posterUrl: 'https://image.tmdb.org/t/p/w342/aOIuZAjPaRIE6CMzbazvcHuHXDc.jpg', + backdropUrl: 'https://image.tmdb.org/t/p/w780/lrtSb1skJayPydZk0OSMAKjBOVe.jpg', + overview: expect.stringMatching(/^Set in the 22nd century/), + tagline: 'Believe the unbelievable.', + cast: expect.stringMatching(/^Keanu Reeves, Laurence Fishburne/), + writers: null, + contentRating: null, + tmdbId: 603, + }); + expect(supabaseMock.state.upserts).toEqual([{ + table: 'tmdb_data', + row: { + lookup_key: 'tt0133093', + tmdb_id: 603, + poster_url: 'https://image.tmdb.org/t/p/w342/aOIuZAjPaRIE6CMzbazvcHuHXDc.jpg', + backdrop_url: 'https://image.tmdb.org/t/p/w780/lrtSb1skJayPydZk0OSMAKjBOVe.jpg', + overview: expect.any(String), + tagline: 'Believe the unbelievable.', + cast_names: expect.any(String), + writers: null, + content_rating: null, + }, + }]); + }); + + it('falls back to TMDB when nichedb only has the bare IMDb row (nothing to show)', async () => { + const { hosts } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 1, items: [imdbTitle({ score: 1 })] }), + }); + const data = await fetchTmdbData('tt0133093', 'The Matrix 1999'); + expect(hosts()).toEqual(['nichedb.dev/api/v1/match', 'api.themoviedb.org/3/find/tt0133093', 'api.themoviedb.org/3/movie/603']); + expect(data.posterUrl).toBe('https://image.tmdb.org/t/p/w500/p.jpg'); + expect(data.writers).toBe('Lana Wachowski'); + }); + + it('falls back to TMDB when nichedb has a different title for that name (tconst mismatch)', async () => { + const { hosts } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 1, items: [tmdbTitle({ score: 1 }, { imdbId: 'tt9999999' })] }), + }); + await fetchTmdbData('tt0133093', 'The Matrix 1999'); + expect(hosts()[0]).toBe('nichedb.dev/api/v1/match'); + expect(hosts()).toContain('api.themoviedb.org/3/find/tt0133093'); + }); + + it('falls back to TMDB when nichedb errors', async () => { + const { hosts } = routedFetch({ nichedb: () => jsonResponse({ error: 'down' }, { status: 503 }) }); + const data = await fetchTmdbData('tt0133093', 'The Matrix 1999'); + expect(hosts()).toContain('api.themoviedb.org/3/find/tt0133093'); + expect(data.tmdbId).toBe(603); + }); + + it('without an IMDb id uses the best match at or above the floor, poster row preferred', async () => { + const { hosts } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 2, items: [imdbTitle({ score: 1 }), tmdbTitle({ score: 1 })] }), + }); + const data = await fetchTmdbData('', 'The.Matrix.1999.1080p'); + expect(hosts()).toEqual(['nichedb.dev/api/v1/match']); + expect(data.tmdbId).toBe(603); + expect(supabaseMock.state.upserts[0].row.lookup_key).toMatch(/^title:[0-9a-f]{32}$/); + }); + + it('without an IMDb id a below-floor match goes on to the TMDB search', async () => { + const { hosts } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 1, items: [tmdbTitle({ score: 0.3 })] }), + tmdb: (url) => url.pathname.startsWith('/3/search/tv') ? jsonResponse({ results: [] }) : url.pathname.startsWith('/3/search/movie') ? jsonResponse({ results: [TMDB_MOVIE] }) : jsonResponse(TMDB_DETAIL), + }); + const data = await fetchTmdbData('', 'The Matrix 1999'); + expect(hosts()[0]).toBe('nichedb.dev/api/v1/match'); + expect(hosts()).toContain('api.themoviedb.org/3/search/movie'); + expect(data.tmdbId).toBe(603); + }); + + it('still asks nichedb without a TMDB key, and is EMPTY (uncached) when nichedb has nothing', async () => { + vi.stubEnv('TMDB_API_KEY', ''); + const { hosts } = routedFetch({}); + const data = await fetchTmdbData('tt0133093', 'The Matrix'); + expect(hosts()).toEqual(['nichedb.dev/api/v1/match']); + expect(data.tmdbId).toBeNull(); + expect(supabaseMock.state.upserts).toEqual([]); + }); +}); + +describe('fetchNichedbData', () => { + it('answers null without a title hint (nothing to ask nichedb for)', async () => { + const { fetchMock } = routedFetch({}); + expect(await fetchNichedbData('tt0133093', '')).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/imdb/tmdb.ts b/src/lib/imdb/tmdb.ts index 26397ede..9bdcb54a 100644 --- a/src/lib/imdb/tmdb.ts +++ b/src/lib/imdb/tmdb.ts @@ -4,10 +4,15 @@ * Falls back to /search if /find returns nothing (common for obscure IMDB entries). * * Results are cached in the tmdb_data table to avoid repeated API calls. + * + * With NICHEDB_TITLES=1 the lookup asks nichedb.dev's `screen` collection first + * (src/lib/nichedb/titles.ts) and only falls through to TMDB when nichedb has + * nothing to show for the title. The cache row is written the same way either way. */ import { createClient } from '@supabase/supabase-js'; import { createHash } from 'crypto'; +import { hasPresentation, matchTitle, titleByImdbId, titleFacts } from '@/lib/nichedb/titles'; export interface TmdbData { posterUrl: string | null; @@ -108,9 +113,49 @@ function cleanTitleForSearch(titleHint: string): string { return cleanTitle; } +/** NICHEDB_TITLES=1 routes lookups through nichedb.dev before TMDB. Default off. */ +export function nichedbTitlesEnabled(): boolean { + return process.env.NICHEDB_TITLES === '1'; +} + +/** + * The nichedb row for this torrent, shaped like a TMDB answer, or null when + * nichedb has nothing to show for it (no poster, no summary, no backdrop), in + * which case the caller goes on to TMDB as it always did. + * + * With an IMDb id the row must carry that tconst; the release name only tells + * nichedb what to search. Without one the best `match` candidate at or above the + * score floor is taken, preferring the row with a poster among exact titles. + */ +export async function fetchNichedbData(imdbId: string, titleHint?: string): Promise { + const name = titleHint?.trim(); + if (!name) return null; + try { + const item = imdbId + ? await titleByImdbId(imdbId, { title: name }) + : await matchTitle(name); + if (!item || !hasPresentation(item)) return null; + const facts = titleFacts(item); + return { + posterUrl: facts.posterUrl, + backdropUrl: facts.backdropUrl, + overview: facts.overview, + tagline: facts.tagline, + cast: facts.cast, + // nichedb carries no writers or certification; the tmdb_data columns stay null. + writers: null, + contentRating: null, + tmdbId: facts.tmdbId, + }; + } catch { + return null; + } +} + export async function fetchTmdbData(imdbId: string, titleHint?: string): Promise { const tmdbKey = process.env.TMDB_API_KEY; - if (!tmdbKey) return EMPTY; + const useNichedb = nichedbTitlesEnabled(); + if (!tmdbKey && !useNichedb) return EMPTY; if (!imdbId && !titleHint) return EMPTY; // Check cache first @@ -118,6 +163,17 @@ export async function fetchTmdbData(imdbId: string, titleHint?: string): Promise const cached = await getCached(cacheKey); if (cached) return cached; + // nichedb first when switched on; the answer lands in tmdb_data exactly as a + // TMDB one would, so every reader of the cache is unchanged. + if (useNichedb) { + const fromNichedb = await fetchNichedbData(imdbId, titleHint); + if (fromNichedb) { + await setCache(cacheKey, fromNichedb); + return fromNichedb; + } + } + if (!tmdbKey) return EMPTY; + try { let tmdbId: number | null = null; let isTV = false; diff --git a/src/lib/nichedb/fixtures.ts b/src/lib/nichedb/fixtures.ts new file mode 100644 index 00000000..5bc9d999 --- /dev/null +++ b/src/lib/nichedb/fixtures.ts @@ -0,0 +1,130 @@ +/** + * nichedb `screen` title items as they come off the wire (0.6.1), for tests. + * Shapes copied from live answers on 2026-09-10. + */ + +import type { NichedbTitleItem } from './titles'; + +export function imdbTitle(over: Partial = {}, data: Record = {}): NichedbTitleItem { + return { + id: 3742539, + collection: 'screen', + kind: 'title', + external_id: 'imdb:title:tt0133093', + updated_at: '2026-09-10T14:05:02.923Z', + title: 'The Matrix', + summary: null, + url: 'https://www.imdb.com/title/tt0133093/', + image_url: null, + published_at: '1999-07-01T12:00:00.000Z', + tags: ['title', 'film', 'imdb', 'genre:action', 'genre:sci-fi'], + ...over, + data: { + form: 'movie', + year: 1999, + watch: [], + genres: ['Action', 'Sci-Fi'], + imdbId: 'tt0133093', + rating: 8.7, + tmdbId: null, + endYear: null, + tagline: null, + category: 'film', + provider: 'imdb', + tvmazeId: null, + anilistId: null, + normTitle: 'the matrix', + titleType: 'movie', + popularity: null, + runtimeMin: 136, + trailerUrl: null, + backdropUrl: null, + ratingCount: 2276418, + originalTitle: 'The Matrix', + ...data, + }, + }; +} + +export function tmdbTitle(over: Partial = {}, data: Record = {}): NichedbTitleItem { + return { + id: 3729335, + collection: 'screen', + kind: 'title', + external_id: 'tmdb:title:603', + updated_at: '2026-09-10T14:03:59.947Z', + title: 'The Matrix', + summary: 'Set in the 22nd century, The Matrix tells the story of a computer hacker who joins a group of underground insurgents fighting the vast and powerful computers who now rule the earth.', + url: 'https://www.themoviedb.org/movie/603', + image_url: 'https://image.tmdb.org/t/p/w342/aOIuZAjPaRIE6CMzbazvcHuHXDc.jpg', + published_at: '1999-03-31T12:00:00.000Z', + tags: ['title', 'film', 'tmdb', 'genre:action', 'genre:science-fiction'], + ...over, + data: { + cast: ['Keanu Reeves', 'Laurence Fishburne', 'Carrie-Anne Moss', 'Hugo Weaving', 'Gloria Foster', 'Joe Pantoliano', 'Marcus Chong', 'Julian Arahanga'], + form: 'movie', + year: 1999, + watch: ['YouTube TV'], + genres: ['Action', 'Science Fiction'], + imdbId: 'tt0133093', + rating: 8.257, + tmdbId: '603', + tagline: 'Believe the unbelievable.', + category: 'film', + provider: 'tmdb', + normTitle: 'the matrix', + popularity: 47.1488, + runtimeMin: 136, + trailerUrl: 'https://www.youtube.com/watch?v=FVI84Dfx2-I', + backdropUrl: 'https://image.tmdb.org/t/p/w780/lrtSb1skJayPydZk0OSMAKjBOVe.jpg', + ratingCount: 28678, + originalTitle: 'The Matrix', + ...data, + }, + }; +} + +export function tvmazeTitle(over: Partial = {}, data: Record = {}): NichedbTitleItem { + return { + id: 3611951, + collection: 'screen', + kind: 'title', + external_id: 'tvmaze:title:93688', + updated_at: '2026-09-10T14:03:28.877Z', + title: 'Perfect Addiction', + summary: 'Akihito is a college student obsessed with good looks.', + url: 'https://www.tvmaze.com/shows/93688/perfect-addiction', + image_url: 'https://static.tvmaze.com/uploads/images/original_untouched/637/1594382.jpg', + published_at: '2026-07-08T12:00:00.000Z', + tags: ['title', 'anime', 'tvmaze', 'genre:romance'], + ...over, + data: { + form: 'series', + year: 2026, + genres: ['Romance'], + imdbId: null, + rating: null, + tmdbId: null, + category: 'anime', + provider: 'tvmaze', + tvmazeId: '93688', + normTitle: 'perfect addiction', + runtimeMin: 5, + backdropUrl: null, + ratingCount: null, + ...data, + }, + }; +} + +/** A Response-like object for a mocked fetch. */ +export function jsonResponse(body: unknown, init: { status?: number; headers?: Record } = {}): Response { + const status = init.status ?? 200; + return { + ok: status >= 200 && status < 300, + status, + headers: new Headers(init.headers ?? {}), + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response; +} diff --git a/src/lib/nichedb/mirror.test.ts b/src/lib/nichedb/mirror.test.ts new file mode 100644 index 00000000..0ca0c60b --- /dev/null +++ b/src/lib/nichedb/mirror.test.ts @@ -0,0 +1,245 @@ +/** + * nichedb -> imdb_* mirror: row mapping and the cursor walk. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { + MIRROR_CURSOR_NAME, + pageToRows, + runMirror, + titleToBasics, + titleToRatings, + type MirrorCursor, + type MirrorRows, + type MirrorStore, +} from './mirror'; +import { imdbTitle, jsonResponse, tmdbTitle, tvmazeTitle } from './fixtures'; +import type { NichedbTitleItem } from './titles'; + +describe('titleToBasics', () => { + it('maps an IMDb film row onto the dump columns', () => { + expect(titleToBasics(imdbTitle())).toEqual({ + tconst: 'tt0133093', + title_type: 'movie', + primary_title: 'The Matrix', + original_title: 'The Matrix', + is_adult: null, + start_year: 1999, + end_year: null, + runtime_minutes: 136, + genres: 'Action,Sci-Fi', + }); + }); + + it('keeps IMDb titleType and endYear for a series', () => { + const row = titleToBasics(imdbTitle( + { external_id: 'imdb:title:tt0039123', title: 'Kraft Television Theatre', tags: ['title', 'tv', 'imdb'] }, + { imdbId: 'tt0039123', form: 'series', titleType: 'tvSeries', year: 1947, endYear: 1958, genres: ['Drama'], runtimeMin: 60, originalTitle: 'Kraft Television Theatre' }, + )); + expect(row).toMatchObject({ tconst: 'tt0039123', title_type: 'tvSeries', start_year: 1947, end_year: 1958, genres: 'Drama', runtime_minutes: 60 }); + }); + + it('derives title_type from form when a non-IMDb row has no titleType', () => { + expect(titleToBasics(tmdbTitle())?.title_type).toBe('movie'); + expect(titleToBasics(tvmazeTitle({}, { imdbId: 'tt16255458' }))?.title_type).toBe('tvSeries'); + }); + + it('takes the tconst from the external id when data.imdbId is missing', () => { + expect(titleToBasics(imdbTitle({}, { imdbId: null }))?.tconst).toBe('tt0133093'); + }); + + it('answers null for a title with no IMDb id, and null fields for missing data', () => { + expect(titleToBasics(tvmazeTitle())).toBeNull(); + const bare = titleToBasics(imdbTitle({}, { year: null, runtimeMin: null, genres: [], originalTitle: null, titleType: null, form: 'unknown' })); + expect(bare).toMatchObject({ start_year: null, runtime_minutes: null, genres: null, original_title: 'The Matrix', title_type: null }); + }); +}); + +describe('titleToRatings', () => { + it('maps rating and votes, rounded to the dump precision', () => { + expect(titleToRatings(imdbTitle())).toEqual({ tconst: 'tt0133093', average_rating: 8.7, num_votes: 2276418 }); + expect(titleToRatings(imdbTitle({}, { rating: 8.257, ratingCount: 10 }))?.average_rating).toBe(8.3); + }); + + it('answers null when unrated', () => { + expect(titleToRatings(imdbTitle({}, { rating: null, ratingCount: null }))).toBeNull(); + expect(titleToRatings(imdbTitle({}, { rating: 7, ratingCount: null }))).toBeNull(); + }); +}); + +describe('pageToRows', () => { + it('writes IMDb rows as basics + ratings and other providers as fill only', () => { + const rows = pageToRows([imdbTitle(), tmdbTitle(), tvmazeTitle(), tvmazeTitle({ id: 9, external_id: 'tvmaze:title:9' }, { imdbId: 'tt16255458', rating: 4.2, ratingCount: 30 })]); + expect(rows.basics.map((r) => r.tconst)).toEqual(['tt0133093']); + expect(rows.basicsFill.map((r) => r.tconst)).toEqual(['tt16255458']); + expect(rows.ratings).toEqual([{ tconst: 'tt0133093', average_rating: 8.7, num_votes: 2276418 }]); + }); + + it('never lets a TMDB rating masquerade as an IMDb one', () => { + const rows = pageToRows([tmdbTitle()]); + expect(rows.ratings).toEqual([]); + expect(rows.basicsFill).toHaveLength(1); + }); + + it('dedupes within a page, IMDb row winning whatever the order', () => { + const a = pageToRows([tmdbTitle(), imdbTitle()]); + const b = pageToRows([imdbTitle(), tmdbTitle()]); + for (const rows of [a, b]) { + expect(rows.basics).toHaveLength(1); + expect(rows.basicsFill).toHaveLength(0); + expect(rows.basics[0].genres).toBe('Action,Sci-Fi'); + } + }); +}); + +function memoryStore(initial: MirrorCursor | null = null) { + let cursor = initial; + const saves: MirrorCursor[] = []; + const writes: MirrorRows[] = []; + const store: MirrorStore = { + loadCursor: vi.fn(async () => cursor), + saveCursor: vi.fn(async (_name: string, c: MirrorCursor) => { cursor = { ...c }; saves.push({ ...c }); }), + writeRows: vi.fn(async (rows: MirrorRows) => { writes.push(rows); }), + }; + return { store, saves, writes, cursor: () => cursor }; +} + +function pageOf(ids: number[]): NichedbTitleItem[] { + return ids.map((id) => imdbTitle({ id, external_id: `imdb:title:tt${String(id).padStart(7, '0')}` }, { imdbId: `tt${String(id).padStart(7, '0')}` })); +} + +/** A fetch that serves id-ordered pages of `limit` from `all`, honouring `after`. */ +function pagedFetch(all: number[], opts: { headers?: Record } = {}) { + const calls: URL[] = []; + const fetchMock = vi.fn(async (input: string) => { + const url = new URL(input); + calls.push(url); + const after = Number(url.searchParams.get('after') ?? 0); + const limit = Number(url.searchParams.get('limit')); + const items = pageOf(all.filter((id) => id > after).slice(0, limit)); + return jsonResponse({ count: items.length, items }, { headers: opts.headers }); + }); + return { fetchMock, calls }; +} + +const noSleep = vi.fn(async () => {}); +const clock = () => new Date('2026-09-10T00:00:00.000Z'); + +describe('runMirror', () => { + it('walks every page, advances the cursor per page, and closes the walk with since=start', async () => { + const { store, saves, writes } = memoryStore(); + const { fetchMock, calls } = pagedFetch([1, 2, 3, 4, 5]); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + expect(result).toMatchObject({ pages: 3, items: 5, basics: 5, ratings: 5, completed: true, stoppedBecause: 'completed' }); + expect(calls.map((u) => u.searchParams.get('after'))).toEqual([null, '2', '4']); + expect(calls.every((u) => !u.searchParams.has('since'))).toBe(true); + expect(writes).toHaveLength(3); + // one save per page, then the closing save + expect(saves.map((s) => s.after_id)).toEqual([2, 4, 5, 0]); + expect(saves.at(-1)).toEqual({ after_id: 0, since: '2026-09-10T00:00:00.000Z' }); + expect(store.saveCursor).toHaveBeenCalledWith(MIRROR_CURSOR_NAME, expect.anything()); + }); + + it('stops at the page cap and leaves a resumable cursor', async () => { + const { store, saves } = memoryStore(); + const { fetchMock } = pagedFetch([1, 2, 3, 4, 5, 6]); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 2, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + expect(result).toMatchObject({ pages: 2, items: 4, completed: false, stoppedBecause: 'page-cap' }); + expect(result.cursor).toEqual({ after_id: 4, since: null }); + expect(saves.at(-1)).toEqual({ after_id: 4, since: null }); + }); + + it('resumes from a stored cursor and sends its since= with every page', async () => { + const { store } = memoryStore({ after_id: 4, since: '2026-09-01T00:00:00.000Z' }); + const { fetchMock, calls } = pagedFetch([1, 2, 3, 4, 5, 6]); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + // [5,6] is a full page, so one more (empty) page closes the walk + expect(calls.map((u) => u.searchParams.get('after'))).toEqual(['4', '6']); + expect(calls.every((u) => u.searchParams.get('since') === '2026-09-01T00:00:00.000Z')).toBe(true); + expect(result).toMatchObject({ pages: 2, items: 2, completed: true }); + // the next walk's watermark is this walk's start, not the old one + expect(result.cursor).toEqual({ after_id: 0, since: '2026-09-10T00:00:00.000Z' }); + }); + + it('a short final page closes the walk in one request when nothing changed', async () => { + const { store, writes } = memoryStore({ after_id: 0, since: '2026-09-09T00:00:00.000Z' }); + const { fetchMock } = pagedFetch([]); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + expect(result).toMatchObject({ pages: 1, items: 0, completed: true }); + expect(writes).toHaveLength(0); + expect(result.cursor.since).toBe('2026-09-10T00:00:00.000Z'); + }); + + it('sleeps the interval between pages, not after the last', async () => { + const { store } = memoryStore(); + const { fetchMock } = pagedFetch([1, 2, 3, 4]); + const sleep = vi.fn(async (_ms: number) => {}); + + await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 7000, pageLimit: 2, sleep, now: clock }); + + // pages: [1,2] full -> sleep; [3,4] full -> sleep; [] short -> done + expect(sleep.mock.calls.map((c) => c[0])).toEqual([7000, 7000]); + }); + + it('waits out a 429 and carries on, keeping the cursor', async () => { + const { store } = memoryStore(); + let n = 0; + const fetchMock = vi.fn(async (input: string) => { + n += 1; + if (n === 2) return jsonResponse({ error: 'slow down' }, { status: 429 }); + const after = Number(new URL(input).searchParams.get('after') ?? 0); + const items = pageOf([1, 2, 3].filter((id) => id > after).slice(0, 2)); + return jsonResponse({ count: items.length, items }); + }); + const sleep = vi.fn(async (_ms: number) => {}); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep, now: clock }); + + expect(result).toMatchObject({ pages: 2, items: 3, completed: true }); + expect(sleep.mock.calls.map((c) => c[0])).toContain(60_000); + }); + + it('gives up after repeated 429s without touching the cursor', async () => { + const { store, saves } = memoryStore({ after_id: 10, since: null }); + const fetchMock = vi.fn(async () => jsonResponse({}, { status: 429 })); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, sleep: noSleep, now: clock, maxRateLimitWaits: 1 }); + + expect(result.stoppedBecause).toBe('rate-limit'); + expect(result.cursor).toEqual({ after_id: 10, since: null }); + expect(saves).toHaveLength(0); + }); + + it('stops on a network failure with the cursor where the last good page left it', async () => { + const { store } = memoryStore(); + let n = 0; + const fetchMock = vi.fn(async () => { + n += 1; + if (n === 2) throw new Error('reset'); + return jsonResponse({ count: 2, items: pageOf([1, 2]) }); + }); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + expect(result).toMatchObject({ pages: 1, stoppedBecause: 'network', completed: false }); + expect(result.cursor).toEqual({ after_id: 2, since: null }); + }); + + it('is idempotent: re-running a completed walk re-fetches nothing new and rewrites the same rows', async () => { + const first = memoryStore(); + const { fetchMock } = pagedFetch([1, 2, 3]); + await runMirror({ store: first.store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + const second = memoryStore({ after_id: 0, since: null }); + await runMirror({ store: second.store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + expect(second.writes).toEqual(first.writes); + }); +}); diff --git a/src/lib/nichedb/mirror.ts b/src/lib/nichedb/mirror.ts new file mode 100644 index 00000000..0dd79df5 --- /dev/null +++ b/src/lib/nichedb/mirror.ts @@ -0,0 +1,237 @@ +/** + * nichedb `screen` -> imdb_title_basics / imdb_title_ratings. + * + * Pure pieces of scripts/mirror-imdb-from-nichedb.ts: the row mapping and the + * cursor walk, with the network, the store and the clock injected so they can be + * exercised against fixtures. + * + * The walk is id-ordered. A run starts from the cursor's `after_id` and sends the + * cursor's `since` with every page; when it reaches the end it resets `after_id` + * to 0 and sets `since` to the moment this walk began, so the next run only asks + * for items nichedb updated after that. A run that stops early (page cap, network + * error, rate limit) leaves `after_id` where it got to and keeps the same `since`, + * so the next run carries on from there. + */ + +import { fetchTitlesPage, imdbIdOf, type NichedbClientOptions, type NichedbTitleItem } from './titles'; + +export const MIRROR_CURSOR_NAME = 'imdb-titles'; + +/** imdb_title_basics row, the columns the dump import fills. */ +export interface ImdbBasicsRow { + tconst: string; + title_type: string | null; + primary_title: string | null; + original_title: string | null; + is_adult: boolean | null; + start_year: number | null; + end_year: number | null; + runtime_minutes: number | null; + genres: string | null; +} + +/** imdb_title_ratings row. */ +export interface ImdbRatingsRow { + tconst: string; + average_rating: number; + num_votes: number; +} + +export interface MirrorCursor { + after_id: number; + since: string | null; +} + +export interface MirrorRows { + /** From IMDb-sourced rows: authoritative, upserted (overwrite). */ + basics: ImdbBasicsRow[]; + /** From other providers that know the tconst: insert-only, fills gaps but never overwrites. */ + basicsFill: ImdbBasicsRow[]; + /** Only IMDb's own rating goes into imdb_title_ratings; TMDB/TVmaze ratings are a different scale. */ + ratings: ImdbRatingsRow[]; +} + +const FORM_TO_TITLE_TYPE: Record = { movie: 'movie', series: 'tvSeries' }; + +function intOrNull(v: unknown): number | null { + if (v == null || v === '') return null; + const n = typeof v === 'number' ? v : Number(v); + return Number.isFinite(n) ? Math.trunc(n) : null; +} + +/** Map one nichedb title to an imdb_title_basics row, or null when it has no tconst. */ +export function titleToBasics(item: NichedbTitleItem): ImdbBasicsRow | null { + const tconst = imdbIdOf(item); + if (!tconst) return null; + const d = item.data ?? {}; + const titleType = (typeof d.titleType === 'string' && d.titleType) || FORM_TO_TITLE_TYPE[d.form ?? ''] || null; + const genres = Array.isArray(d.genres) ? d.genres.filter((g): g is string => typeof g === 'string' && g.length > 0) : []; + return { + tconst, + title_type: titleType, + primary_title: item.title ?? null, + original_title: (typeof d.originalTitle === 'string' && d.originalTitle) || item.title || null, + is_adult: typeof d.isAdult === 'boolean' ? d.isAdult : null, + start_year: intOrNull(d.year), + end_year: intOrNull(d.endYear), + runtime_minutes: intOrNull(d.runtimeMin), + // The dump stores genres as a bare comma list ("Action,Sci-Fi"); readers + // re-space it themselves (enrich.ts: genres.replace(/,/g, ', ')). + genres: genres.length ? genres.join(',') : null, + }; +} + +/** Map an IMDb-sourced nichedb title to an imdb_title_ratings row, or null when unrated. */ +export function titleToRatings(item: NichedbTitleItem): ImdbRatingsRow | null { + const tconst = imdbIdOf(item); + if (!tconst) return null; + const d = item.data ?? {}; + const rating = typeof d.rating === 'number' ? d.rating : Number(d.rating); + const votes = intOrNull(d.ratingCount); + if (!Number.isFinite(rating) || votes == null) return null; + return { tconst, average_rating: Math.round(rating * 10) / 10, num_votes: votes }; +} + +export function isImdbSourced(item: NichedbTitleItem): boolean { + return item.data?.provider === 'imdb' || /^imdb:title:/.test(item.external_id ?? ''); +} + +/** Split a page of titles into the rows to write. Items without a tconst are skipped. */ +export function pageToRows(items: NichedbTitleItem[]): MirrorRows { + const basics = new Map(); + const basicsFill = new Map(); + const ratings = new Map(); + for (const item of items) { + const row = titleToBasics(item); + if (!row) continue; + if (isImdbSourced(item)) { + basics.set(row.tconst, row); + basicsFill.delete(row.tconst); + const r = titleToRatings(item); + if (r) ratings.set(r.tconst, r); + } else if (!basics.has(row.tconst)) { + basicsFill.set(row.tconst, row); + } + } + return { basics: [...basics.values()], basicsFill: [...basicsFill.values()], ratings: [...ratings.values()] }; +} + +export interface MirrorStore { + loadCursor(name: string): Promise; + saveCursor(name: string, cursor: MirrorCursor): Promise; + writeRows(rows: MirrorRows): Promise; +} + +export interface MirrorRunOptions extends NichedbClientOptions { + store: MirrorStore; + /** Pages this run may fetch; 0 = until the walk completes. */ + maxPages: number; + /** Pause between pages. 600 req/h is one every 6 s; leave room for the site's own calls. */ + intervalMs: number; + pageLimit?: number; + /** Restrict the walk, e.g. ['imdb'] for IMDb-sourced rows only. */ + tags?: string[]; + cursorName?: string; + now?: () => Date; + sleep?: (ms: number) => Promise; + log?: (line: string) => void; + /** How many times to wait out a 429 before giving up this run. */ + maxRateLimitWaits?: number; +} + +export interface MirrorRunResult { + pages: number; + items: number; + basics: number; + basicsFill: number; + ratings: number; + completed: boolean; + cursor: MirrorCursor; + stoppedBecause: 'completed' | 'page-cap' | 'network' | 'rate-limit' | 'http'; +} + +/** + * Walk the collection from the stored cursor, writing rows page by page and + * advancing the cursor after each page so a crash loses at most one page of work + * (which the next run re-fetches; the upserts are idempotent). + */ +export async function runMirror(opts: MirrorRunOptions): Promise { + const name = opts.cursorName ?? MIRROR_CURSOR_NAME; + const now = opts.now ?? (() => new Date()); + const sleep = opts.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const log = opts.log ?? (() => {}); + const pageLimit = opts.pageLimit ?? 200; + const maxRateLimitWaits = opts.maxRateLimitWaits ?? 3; + + const startedAt = now().toISOString(); + const cursor: MirrorCursor = (await opts.store.loadCursor(name)) ?? { after_id: 0, since: null }; + log(`cursor ${name}: after_id=${cursor.after_id} since=${cursor.since ?? 'none'} (walk started ${startedAt})`); + + const result: MirrorRunResult = { + pages: 0, items: 0, basics: 0, basicsFill: 0, ratings: 0, + completed: false, cursor, stoppedBecause: 'page-cap', + }; + let rateLimitWaits = 0; + + while (opts.maxPages === 0 || result.pages < opts.maxPages) { + const fetched = await fetchTitlesPage( + { after: cursor.after_id, since: cursor.since, limit: pageLimit, tags: opts.tags }, + opts, + ); + if (!fetched) { + result.stoppedBecause = 'network'; + log('network error; stopping, cursor kept'); + break; + } + if (fetched.status === 429) { + if (rateLimitWaits++ >= maxRateLimitWaits) { + result.stoppedBecause = 'rate-limit'; + log('rate limited too many times; stopping, cursor kept'); + break; + } + log('rate limited (429); waiting 60s'); + await sleep(60_000); + continue; + } + if (fetched.status >= 400) { + result.stoppedBecause = 'http'; + log(`HTTP ${fetched.status}; stopping, cursor kept`); + break; + } + + result.pages += 1; + const items = fetched.page.items; + if (items.length > 0) { + const rows = pageToRows(items); + await opts.store.writeRows(rows); + result.items += items.length; + result.basics += rows.basics.length; + result.basicsFill += rows.basicsFill.length; + result.ratings += rows.ratings.length; + cursor.after_id = items[items.length - 1].id; + await opts.store.saveCursor(name, { ...cursor }); + log(`page ${result.pages}: ${items.length} items -> ${rows.basics.length} basics, ${rows.basicsFill.length} fill, ${rows.ratings.length} ratings; after_id=${cursor.after_id}${fetched.rateRemaining != null ? ` (rate remaining ${fetched.rateRemaining})` : ''}`); + } + + if (items.length < pageLimit) { + // End of the walk: next run asks only for what changed since this one began. + cursor.after_id = 0; + cursor.since = startedAt; + await opts.store.saveCursor(name, { ...cursor }); + result.completed = true; + result.stoppedBecause = 'completed'; + log(`walk complete; next run uses since=${startedAt}`); + break; + } + + if (fetched.rateRemaining != null && fetched.rateRemaining <= 1) { + log('rate budget exhausted; waiting 60s'); + await sleep(60_000); + } else { + await sleep(opts.intervalMs); + } + } + + result.cursor = { ...cursor }; + return result; +} diff --git a/src/lib/nichedb/titles.test.ts b/src/lib/nichedb/titles.test.ts new file mode 100644 index 00000000..1c56169c --- /dev/null +++ b/src/lib/nichedb/titles.test.ts @@ -0,0 +1,189 @@ +/** + * nichedb titles client: the match pick, id extraction, facts, and the wire calls. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { + MATCH_SCORE_FLOOR, + fetchTitlesPage, + hasPresentation, + imdbIdOf, + matchTitle, + pickMatch, + titleByImdbId, + titleFacts, +} from './titles'; +import { imdbTitle, jsonResponse, tmdbTitle, tvmazeTitle } from './fixtures'; + +describe('imdbIdOf', () => { + it('reads data.imdbId first', () => { + expect(imdbIdOf(tmdbTitle())).toBe('tt0133093'); + }); + + it('falls back to the imdb:title external id', () => { + expect(imdbIdOf(imdbTitle({}, { imdbId: null }))).toBe('tt0133093'); + }); + + it('answers null when neither carries a tconst', () => { + expect(imdbIdOf(tvmazeTitle())).toBeNull(); + expect(imdbIdOf(tmdbTitle({}, { imdbId: 'garbage' }))).toBeNull(); + }); +}); + +describe('pickMatch', () => { + it('prefers the row with a poster among exact titles', () => { + const imdb = imdbTitle({ score: 1 }); + const tmdb = tmdbTitle({ score: 1 }); + expect(pickMatch([imdb, tmdb])?.external_id).toBe('tmdb:title:603'); + expect(pickMatch([tmdb, imdb])?.external_id).toBe('tmdb:title:603'); + }); + + it('takes the best score even when a lower one has a poster', () => { + const exact = imdbTitle({ score: 1 }); + const near = tmdbTitle({ id: 1, external_id: 'tmdb:title:1', title: 'The Matrix Reloaded', score: 0.8 }); + expect(pickMatch([near, exact])?.external_id).toBe('imdb:title:tt0133093'); + }); + + it('answers null when nothing reaches the floor', () => { + const low = tmdbTitle({ score: MATCH_SCORE_FLOOR - 0.01 }); + expect(pickMatch([low])).toBeNull(); + expect(pickMatch([])).toBeNull(); + }); + + it('accepts a candidate exactly at the floor', () => { + const at = tmdbTitle({ score: MATCH_SCORE_FLOOR }); + expect(pickMatch([at])).toBe(at); + }); + + it('with an imdbId keeps only rows carrying that tconst, regardless of score', () => { + // Live shape: "Dune" (tmdb, score 1) vs "Dune: Part One" (imdb, score 0.38), same tconst. + const other = tmdbTitle({ id: 2, external_id: 'tmdb:title:2', title: 'Dune', score: 1 }, { imdbId: 'tt1160419' }); + const wanted = imdbTitle({ id: 3, external_id: 'imdb:title:tt0000001', title: 'Dune World', score: 0.45 }, { imdbId: 'tt0000001' }); + expect(pickMatch([other, wanted], { imdbId: 'tt0000001' })?.id).toBe(3); + expect(pickMatch([other, wanted], { imdbId: 'tt9999999' })).toBeNull(); + }); + + it('with an imdbId still prefers the poster row among rows for that tconst', () => { + const imdb = imdbTitle({ score: 0.38, title: 'Dune: Part One' }, { imdbId: 'tt1160419' }); + const tmdb = tmdbTitle({ score: 1, title: 'Dune' }, { imdbId: 'tt1160419' }); + expect(pickMatch([imdb, tmdb], { imdbId: 'tt1160419' })?.external_id).toBe('tmdb:title:603'); + }); +}); + +describe('hasPresentation / titleFacts', () => { + it('an IMDb row has nothing to show, a TMDB row does', () => { + expect(hasPresentation(imdbTitle())).toBe(false); + expect(hasPresentation(tmdbTitle())).toBe(true); + expect(hasPresentation(imdbTitle({ summary: 'A plot.' }))).toBe(true); + }); + + it('normalises a TMDB row', () => { + const f = titleFacts(tmdbTitle()); + expect(f).toMatchObject({ + imdbId: 'tt0133093', + tmdbId: 603, + year: 1999, + form: 'movie', + category: 'film', + rating: 8.257, + ratingCount: 28678, + runtimeMin: 136, + posterUrl: 'https://image.tmdb.org/t/p/w342/aOIuZAjPaRIE6CMzbazvcHuHXDc.jpg', + backdropUrl: 'https://image.tmdb.org/t/p/w780/lrtSb1skJayPydZk0OSMAKjBOVe.jpg', + tagline: 'Believe the unbelievable.', + }); + expect(f.genres).toEqual(['Action', 'Science Fiction']); + expect(f.cast).toBe('Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss, Hugo Weaving, Gloria Foster, Joe Pantoliano, Marcus Chong, Julian Arahanga'); + expect(f.overview).toMatch(/^Set in the 22nd century/); + }); + + it('normalises an IMDb row with nulls where it has nothing', () => { + const f = titleFacts(imdbTitle()); + expect(f).toMatchObject({ tmdbId: null, posterUrl: null, backdropUrl: null, overview: null, tagline: null, cast: null, rating: 8.7 }); + }); +}); + +describe('matchTitle', () => { + it('asks /api/v1/match with the release name and year and picks the poster row', async () => { + const fetchMock = vi.fn(async (_input: string) => + jsonResponse({ q: 'x', parsed: {}, count: 2, items: [imdbTitle({ score: 1 }), tmdbTitle({ score: 1 })] }), + ); + const item = await matchTitle('The.Matrix.1999.1080p.BluRay.x264-GROUP', { year: 1999, fetch: fetchMock }); + expect(item?.external_id).toBe('tmdb:title:603'); + const url = new URL(fetchMock.mock.calls[0][0]); + expect(url.pathname).toBe('/api/v1/match'); + expect(url.searchParams.get('q')).toBe('The.Matrix.1999.1080p.BluRay.x264-GROUP'); + expect(url.searchParams.get('collection')).toBe('screen'); + expect(url.searchParams.get('kind')).toBe('title'); + expect(url.searchParams.get('year')).toBe('1999'); + }); + + it('answers null on a non-2xx or a thrown fetch', async () => { + expect(await matchTitle('x', { fetch: vi.fn(async () => jsonResponse({}, { status: 503 })) })).toBeNull(); + expect(await matchTitle('x', { fetch: vi.fn(async () => { throw new Error('down'); }) })).toBeNull(); + }); + + it('does not call out for an empty name', async () => { + const fetchMock = vi.fn(); + expect(await matchTitle(' ', { fetch: fetchMock })).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('titleByImdbId', () => { + it('matches on the known title and keeps the row carrying the tconst', async () => { + const fetchMock = vi.fn(async (_input: string) => + jsonResponse({ q: 'x', parsed: {}, count: 2, items: [imdbTitle({ score: 1 }), tmdbTitle({ score: 1 })] }), + ); + const item = await titleByImdbId('tt0133093', { title: 'The Matrix', year: 1999 }, { fetch: fetchMock }); + expect(item?.external_id).toBe('tmdb:title:603'); + const url = new URL(fetchMock.mock.calls[0][0]); + expect(url.searchParams.get('q')).toBe('The Matrix'); + expect(url.searchParams.get('year')).toBe('1999'); + }); + + it('answers null when no candidate carries the tconst', async () => { + const fetchMock = vi.fn(async (_input: string) => jsonResponse({ q: 'x', parsed: {}, count: 1, items: [tmdbTitle({ score: 1 })] })); + expect(await titleByImdbId('tt7777777', { title: 'The Matrix' }, { fetch: fetchMock })).toBeNull(); + }); + + it('answers null without a title to ask for, and for a malformed tconst', async () => { + const fetchMock = vi.fn(); + expect(await titleByImdbId('tt0133093', {}, { fetch: fetchMock })).toBeNull(); + expect(await titleByImdbId('nm0000001', { title: 'x' }, { fetch: fetchMock })).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('fetchTitlesPage', () => { + it('builds the id-ordered page query with after, since and tags', async () => { + const fetchMock = vi.fn(async (_input: string) => + jsonResponse({ count: 1, items: [imdbTitle()] }, { headers: { 'x-ratelimit-remaining': '498' } }), + ); + const res = await fetchTitlesPage({ after: 42, since: '2026-09-01T00:00:00.000Z', limit: 200, tags: ['imdb'] }, { fetch: fetchMock }); + expect(res?.status).toBe(200); + expect(res?.rateRemaining).toBe(498); + expect(res?.page.items).toHaveLength(1); + const url = new URL(fetchMock.mock.calls[0][0]); + expect(url.pathname).toBe('/api/v1/items'); + expect(Object.fromEntries(url.searchParams)).toEqual({ + collection: 'screen', kind: 'title', sort: 'id', order: 'asc', limit: '200', + after: '42', since: '2026-09-01T00:00:00.000Z', tags: 'imdb', + }); + }); + + it('omits after and since on a first page', async () => { + const fetchMock = vi.fn(async (_input: string) => jsonResponse({ count: 0, items: [] })); + await fetchTitlesPage({ after: 0, since: null }, { fetch: fetchMock }); + const url = new URL(fetchMock.mock.calls[0][0]); + expect(url.searchParams.has('after')).toBe(false); + expect(url.searchParams.has('since')).toBe(false); + }); + + it('reports the status on a 429 and null on a network failure', async () => { + const limited = await fetchTitlesPage({}, { fetch: vi.fn(async () => jsonResponse({ error: 'slow down' }, { status: 429, headers: { 'x-ratelimit-remaining': '0' } })) }); + expect(limited?.status).toBe(429); + expect(limited?.rateRemaining).toBe(0); + expect(await fetchTitlesPage({}, { fetch: vi.fn(async () => { throw new Error('reset'); }) })).toBeNull(); + }); +}); diff --git a/src/lib/nichedb/titles.ts b/src/lib/nichedb/titles.ts new file mode 100644 index 00000000..02e6e275 --- /dev/null +++ b/src/lib/nichedb/titles.ts @@ -0,0 +1,282 @@ +/** + * nichedb.dev `screen` collection: film / tv / anime titles. + * + * nichedb keeps one row per provider for a title, so a film usually has both an + * `imdb:title:` row (rating, votes, runtime, no poster, no summary) and a + * `tmdb:title:` row (poster, backdrop, summary, tagline, cast). Both carry + * `data.imdbId`, which is how the two are tied together here. + * + * Read API (anonymous, 600 requests/hour per IP): + * GET /api/v1/items?collection=screen&kind=title&sort=id&order=asc&after=&since=&limit=200 + * GET /api/v1/match?q=&collection=screen&kind=title[&year=] + * + * `match` cleans a release name itself (year, S02E03, quality tags, group) and + * answers items with a `score` in 0..1; an exact title is 1.0. + * + * Nothing in here touches the database. The switch that routes bittorrented's + * reads through nichedb is NICHEDB_TITLES=1 (see src/lib/imdb/tmdb.ts). + */ + +export const NICHEDB_BASE_URL = 'https://nichedb.dev'; + +/** Below this `match` score a candidate is not the title the name refers to. */ +export const MATCH_SCORE_FLOOR = 0.5; + +export const DEFAULT_PAGE_LIMIT = 200; + +/** An item of the `screen` collection with kind `title`, as it comes off the wire (0.6.1). */ +export interface NichedbTitleItem { + id: number; + collection: string; + kind: string; + external_id: string; + updated_at: string; + title: string; + summary: string | null; + url: string | null; + image_url: string | null; + published_at: string | null; + tags: string[]; + data: NichedbTitleData; + /** Only on `/api/v1/match` answers. */ + score?: number; +} + +export interface NichedbTitleData { + provider?: string; + category?: 'film' | 'tv' | 'anime' | string; + form?: 'movie' | 'series' | string; + year?: number | null; + endYear?: number | null; + normTitle?: string; + originalTitle?: string | null; + titleType?: string | null; + isAdult?: boolean | null; + imdbId?: string | null; + tmdbId?: string | number | null; + genres?: string[] | null; + rating?: number | null; + ratingCount?: number | null; + popularity?: number | null; + backdropUrl?: string | null; + tagline?: string | null; + trailerUrl?: string | null; + runtimeMin?: number | null; + watch?: string[] | null; + cast?: string[] | null; + [key: string]: unknown; +} + +export interface NichedbItemsPage { + count: number; + items: NichedbTitleItem[]; +} + +export interface NichedbMatchResponse { + q: string; + parsed: { name: string; year: number | null; season: number | null; episode: number | null; kind: string }; + count: number; + items: NichedbTitleItem[]; +} + +export type FetchLike = (input: string, init?: RequestInit) => Promise; + +export interface NichedbClientOptions { + baseUrl?: string; + fetch?: FetchLike; + /** Per-request timeout; the site calls this on a page render. */ + timeoutMs?: number; +} + +function resolveOptions(opts: NichedbClientOptions = {}) { + return { + baseUrl: (opts.baseUrl ?? process.env.NICHEDB_BASE_URL ?? NICHEDB_BASE_URL).replace(/\/$/, ''), + fetch: opts.fetch ?? ((input: string, init?: RequestInit) => fetch(input, init)), + timeoutMs: opts.timeoutMs ?? 8000, + }; +} + +async function getJson(url: string, opts: NichedbClientOptions): Promise { + const { fetch: doFetch, timeoutMs } = resolveOptions(opts); + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null; + try { + const res = await doFetch(url, { + headers: { accept: 'application/json', 'user-agent': 'bittorrented.com (nichedb mirror)' }, + signal: controller?.signal, + }); + if (!res.ok) return null; + return (await res.json()) as T; + } catch { + return null; + } finally { + if (timer) clearTimeout(timer); + } +} + +/** The IMDb tconst of an item: `data.imdbId`, else the `imdb:title:` external id. */ +export function imdbIdOf(item: Pick): string | null { + const fromData = item.data?.imdbId; + if (typeof fromData === 'string' && /^tt\d+$/.test(fromData)) return fromData; + const m = /^imdb:title:(tt\d+)$/.exec(item.external_id ?? ''); + return m ? m[1] : null; +} + +/** What a title page can show from one nichedb row, normalised. */ +export interface NichedbTitleFacts { + imdbId: string | null; + tmdbId: number | null; + title: string; + year: number | null; + form: 'movie' | 'series' | null; + category: 'film' | 'tv' | 'anime' | null; + genres: string[]; + rating: number | null; + ratingCount: number | null; + runtimeMin: number | null; + posterUrl: string | null; + backdropUrl: string | null; + overview: string | null; + tagline: string | null; + cast: string | null; +} + +export function titleFacts(item: NichedbTitleItem): NichedbTitleFacts { + const d = item.data ?? {}; + const tmdbId = d.tmdbId != null && d.tmdbId !== '' ? Number(d.tmdbId) : NaN; + const form = d.form === 'movie' || d.form === 'series' ? d.form : null; + const category = d.category === 'film' || d.category === 'tv' || d.category === 'anime' ? d.category : null; + const cast = Array.isArray(d.cast) ? d.cast.filter((c): c is string => typeof c === 'string' && c.length > 0).slice(0, 8) : []; + return { + imdbId: imdbIdOf(item), + tmdbId: Number.isFinite(tmdbId) && tmdbId > 0 ? tmdbId : null, + title: item.title, + year: typeof d.year === 'number' ? d.year : null, + form, + category, + genres: Array.isArray(d.genres) ? d.genres.filter((g): g is string => typeof g === 'string') : [], + rating: typeof d.rating === 'number' ? d.rating : null, + ratingCount: typeof d.ratingCount === 'number' ? d.ratingCount : null, + runtimeMin: typeof d.runtimeMin === 'number' ? d.runtimeMin : null, + posterUrl: item.image_url || null, + backdropUrl: (typeof d.backdropUrl === 'string' && d.backdropUrl) || null, + overview: item.summary || null, + tagline: (typeof d.tagline === 'string' && d.tagline) || null, + cast: cast.length ? cast.join(', ') : null, + }; +} + +/** Does the item carry anything a torrent page would show beyond what imdb_* already has? */ +export function hasPresentation(item: NichedbTitleItem): boolean { + return Boolean(item.image_url || item.summary || item.data?.backdropUrl); +} + +/** + * Pick the item a `match` answer refers to. + * + * Exact titles (score 1.0) come back once per provider; only the TMDB row has a + * poster, so among the best-scoring candidates the one with an image wins. Anything + * under the floor is not a match at all. + * + * When `imdbId` is given the pick is restricted to rows that carry that tconst, + * whatever their score: the caller already knows which title it wants and only + * needs nichedb's presentation for it. + */ +export function pickMatch( + items: NichedbTitleItem[], + opts: { imdbId?: string | null; floor?: number } = {}, +): NichedbTitleItem | null { + const floor = opts.floor ?? MATCH_SCORE_FLOOR; + let pool = items; + if (opts.imdbId) { + pool = items.filter((i) => imdbIdOf(i) === opts.imdbId); + } else { + pool = items.filter((i) => (i.score ?? 0) >= floor); + } + if (pool.length === 0) return null; + + const best = Math.max(...pool.map((i) => i.score ?? 0)); + const top = pool.filter((i) => (i.score ?? 0) === best); + return top.find((i) => Boolean(i.image_url)) ?? top.find(hasPresentation) ?? top[0]; +} + +/** Raw `match` candidates for a release name, unranked beyond what nichedb did. */ +export async function matchCandidates( + name: string, + opts: { year?: number | null; limit?: number } & NichedbClientOptions = {}, +): Promise { + const q = name.trim(); + if (!q) return []; + const { baseUrl } = resolveOptions(opts); + const params = new URLSearchParams({ q, collection: 'screen', kind: 'title' }); + if (opts.year) params.set('year', String(opts.year)); + if (opts.limit) params.set('limit', String(opts.limit)); + const res = await getJson(`${baseUrl}/api/v1/match?${params}`, opts); + return res?.items ?? []; +} + +/** + * The title a release name refers to, or null when nichedb has no candidate at or + * above the score floor. Prefers the row with a poster among exact titles. + */ +export async function matchTitle( + name: string, + opts: { year?: number | null } & NichedbClientOptions = {}, +): Promise { + const items = await matchCandidates(name, opts); + return pickMatch(items); +} + +/** + * The nichedb row for a known IMDb id. + * + * nichedb has no per-id tag (`tags=imdb:title:` answers nothing) and the + * items endpoint does not filter on external_id, so this goes through `match` on + * the title we already know and keeps only rows carrying that tconst. Without a + * title there is nothing to ask, and it answers null. + */ +export async function titleByImdbId( + tconst: string, + hint: { title?: string | null; year?: number | null } = {}, + opts: NichedbClientOptions = {}, +): Promise { + if (!/^tt\d+$/.test(tconst) || !hint.title?.trim()) return null; + const items = await matchCandidates(hint.title, { ...opts, year: hint.year ?? null, limit: 20 }); + return pickMatch(items, { imdbId: tconst }); +} + +/** One id-ordered page of `screen` titles; `after` is the last id already seen. */ +export async function fetchTitlesPage( + params: { after?: number; since?: string | null; limit?: number; tags?: string[] }, + opts: NichedbClientOptions = {}, +): Promise<{ page: NichedbItemsPage; rateRemaining: number | null; status: number } | null> { + const { baseUrl, fetch: doFetch, timeoutMs } = resolveOptions(opts); + const qs = new URLSearchParams({ + collection: 'screen', + kind: 'title', + sort: 'id', + order: 'asc', + limit: String(params.limit ?? DEFAULT_PAGE_LIMIT), + }); + if (params.after) qs.set('after', String(params.after)); + if (params.since) qs.set('since', params.since); + if (params.tags?.length) qs.set('tags', params.tags.join(',')); + + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), Math.max(timeoutMs, 30000)) : null; + try { + const res = await doFetch(`${baseUrl}/api/v1/items?${qs}`, { + headers: { accept: 'application/json', 'user-agent': 'bittorrented.com (nichedb mirror)' }, + signal: controller?.signal, + }); + const remaining = res.headers?.get?.('x-ratelimit-remaining'); + const rateRemaining = remaining != null && remaining !== '' ? Number(remaining) : null; + if (!res.ok) return { page: { count: 0, items: [] }, rateRemaining, status: res.status }; + const page = (await res.json()) as NichedbItemsPage; + return { page: { count: page.count ?? page.items?.length ?? 0, items: page.items ?? [] }, rateRemaining, status: res.status }; + } catch { + return null; + } finally { + if (timer) clearTimeout(timer); + } +} diff --git a/supabase/migrations/20260910150000_nichedb_mirror_cursor.sql b/supabase/migrations/20260910150000_nichedb_mirror_cursor.sql new file mode 100644 index 00000000..4077d766 --- /dev/null +++ b/supabase/migrations/20260910150000_nichedb_mirror_cursor.sql @@ -0,0 +1,30 @@ +-- Cursor for the nichedb -> imdb_* mirror (scripts/mirror-imdb-from-nichedb.ts). +-- +-- bittorrented used to download the IMDb daily dumps (title.basics, title.ratings, +-- ...) every night and COPY them into the imdb_* tables. nichedb.dev's `screen` +-- collection already carries every IMDb title with a rating, so the mirror walks +-- that collection over its public API instead and upserts imdb_title_basics and +-- imdb_title_ratings. The walk is id-ordered (`sort=id&order=asc&after=`) and +-- capped per run to stay inside nichedb's 600 requests/hour, so it has to remember +-- where it stopped: that is this table. +-- +-- after_id last item id seen on the current walk (0 = start from the beginning) +-- since `since=` sent with every page of the current walk (NULL on the first, +-- full backfill); set to the walk's own start time once it completes, so +-- the next walk only asks for items updated after that +-- +-- One row per mirror name; the IMDb mirror uses 'imdb-titles'. + +CREATE TABLE IF NOT EXISTS nichedb_mirror_cursor ( + name TEXT PRIMARY KEY, + after_id BIGINT NOT NULL DEFAULT 0, + since TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE nichedb_mirror_cursor IS + 'Resume point for scripts/mirror-imdb-from-nichedb.ts: last nichedb item id seen on the current walk and the since= watermark it was started with.'; + +ALTER TABLE nichedb_mirror_cursor ENABLE ROW LEVEL SECURITY; +-- No policies on purpose: only the service role (which bypasses RLS) reads or +-- writes the cursor.