diff --git a/.env.example b/.env.example index 34f23c94..dd822bf8 100644 --- a/.env.example +++ b/.env.example @@ -44,6 +44,14 @@ FANART_TV_API_KEY=your-fanart-api-key TMDB_API_KEY=your-tmdb-api-key MUSICBRAINZ_USER_AGENT=BitTorrented/1.0.0 (https://bittorrented.com) +# nichedb.dev `screen` collection as the title source (default off for both) +# NICHEDB_TITLES=1 torrent pages ask nichedb for poster/backdrop/overview before TMDB +# NICHEDB_MIRROR=1 setup-server.sh schedules scripts/mirror-imdb-from-nichedb.sh at 00:00 +# instead of the IMDb dump download (pnpm mirror:imdb) +# NICHEDB_MIRROR_MAX_PAGES=480 NICHEDB_MIRROR_INTERVAL_MS=7000 NICHEDB_MIRROR_TAGS= +#NICHEDB_TITLES=1 +#NICHEDB_MIRROR=1 + # CoinPayPortal (Crypto Payments) # Get these from your CoinPayPortal dashboard: https://coinpayportal.com/dashboard COINPAYPORTAL_API_KEY=cp_live_your_api_key diff --git a/package.json b/package.json index 80c7cc75..f2479e1d 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "enrich-folders:all": "tsx --env-file=.env scripts/enrich-folder-metadata.ts --all-status", "enrich-folders:dry-run": "tsx --env-file=.env scripts/enrich-folder-metadata.ts --all-status --dry-run", "enrich-folders:force": "tsx --env-file=.env scripts/enrich-folder-metadata.ts --all-status --force", + "mirror:imdb": "tsx --env-file=.env scripts/mirror-imdb-from-nichedb.ts", "iptv-worker": "tsx --env-file=.env workers/iptv-cache/index.ts", "iptv-worker:dev": "tsx --watch --env-file=.env workers/iptv-cache/index.ts", "podcast-worker": "tsx --env-file=.env workers/podcast-notifier/index.ts", diff --git a/scripts/mirror-imdb-from-nichedb.sh b/scripts/mirror-imdb-from-nichedb.sh new file mode 100755 index 00000000..7e246e63 --- /dev/null +++ b/scripts/mirror-imdb-from-nichedb.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Nightly imdb_* refresh from nichedb.dev, in place of the IMDb dump download. +# +# Runs scripts/mirror-imdb-from-nichedb.ts (pnpm mirror:imdb) with a lock so two +# cron runs never overlap. One run walks at most NICHEDB_MIRROR_MAX_PAGES pages +# (default 480, ~56 minutes at 7 s a page) and stores its cursor, so the first +# backfill spreads over a few nights unless kicked off by hand with +# pnpm mirror:imdb -- --all +# +# crontab (set up by setup-server.sh when NICHEDB_MIRROR=1): +# 0 0 * * * /home/ubuntu/src/media-streamer/scripts/mirror-imdb-from-nichedb.sh >> /var/log/imdb-update.log 2>&1 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +LOCK_FILE="/tmp/imdb-update.lock" + +if [ -f "$LOCK_FILE" ]; then + pid=$(cat "$LOCK_FILE") + if kill -0 "$pid" 2>/dev/null; then + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Another IMDb update is running (pid $pid), skipping" + exit 0 + fi +fi +echo $$ > "$LOCK_FILE" +trap 'rm -f "$LOCK_FILE"' EXIT + +# cron's PATH has no pnpm; pick up the user's toolchain. +export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$PATH" + +cd "$PROJECT_DIR" +echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Starting nichedb -> imdb_* mirror" +pnpm mirror:imdb "$@" +echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] nichedb mirror finished" diff --git a/scripts/mirror-imdb-from-nichedb.ts b/scripts/mirror-imdb-from-nichedb.ts new file mode 100755 index 00000000..47db0f67 --- /dev/null +++ b/scripts/mirror-imdb-from-nichedb.ts @@ -0,0 +1,180 @@ +#!/usr/bin/env npx tsx +/** + * Mirror IMDb titles from nichedb.dev into imdb_title_basics / imdb_title_ratings. + * + * Replaces the nightly IMDb dump download (scripts/update-imdb-daily.sh) for the + * two tables the site reads on every torrent page. It walks nichedb's `screen` + * collection over the public API, id-ordered, with a cursor kept in + * nichedb_mirror_cursor so a run can stop at its page cap and the next run + * carries on. Once a walk reaches the end, later walks send `since=` and only see + * titles nichedb updated after the previous walk began. + * + * Usage: + * pnpm mirror:imdb one capped run (cron) + * pnpm mirror:imdb -- --all run until the walk completes (first backfill, ~3-4 h) + * pnpm mirror:imdb -- --max-pages=N --interval-ms=MS --dry-run --reset + * + * Options: + * --all No page cap for this run + * --max-pages=N Pages per run (default NICHEDB_MIRROR_MAX_PAGES or 480) + * --interval-ms=MS Pause between pages (default NICHEDB_MIRROR_INTERVAL_MS or 7000). + * nichedb allows 600 requests/hour per IP; 7 s is ~514/h, leaving + * the rest for the site's own match calls from the same box. + * --tags=a,b Only walk items carrying all these tags (default: none, all titles; + * `imdb` walks the IMDb-sourced rows only) + * --reset Forget the cursor and start a full backfill + * --dry-run Fetch and map, write nothing (cursor included) + * + * Required environment variables: + * SUPABASE_URL (or NEXT_PUBLIC_SUPABASE_URL), SUPABASE_SERVICE_ROLE_KEY + * Optional: + * NICHEDB_BASE_URL (default https://nichedb.dev) + */ + +import { config } from 'dotenv'; +import { createClient } from '@supabase/supabase-js'; + +config(); + +import { + MIRROR_CURSOR_NAME, + runMirror, + type MirrorCursor, + type MirrorRows, + type MirrorStore, +} from '../src/lib/nichedb/mirror'; + +interface Options { + maxPages: number; + intervalMs: number; + tags: string[]; + reset: boolean; + dryRun: boolean; +} + +function parseArgs(): Options { + const opts: Options = { + maxPages: Number(process.env.NICHEDB_MIRROR_MAX_PAGES) || 480, + intervalMs: Number(process.env.NICHEDB_MIRROR_INTERVAL_MS) || 7000, + tags: (process.env.NICHEDB_MIRROR_TAGS ?? '').split(',').filter(Boolean), + reset: false, + dryRun: false, + }; + for (const arg of process.argv.slice(2)) { + if (arg === '--all') opts.maxPages = 0; + else if (arg.startsWith('--max-pages=')) opts.maxPages = Number(arg.slice('--max-pages='.length)) || 0; + else if (arg.startsWith('--interval-ms=')) opts.intervalMs = Number(arg.slice('--interval-ms='.length)) || 0; + else if (arg.startsWith('--tags=')) opts.tags = arg.slice('--tags='.length).split(',').filter(Boolean); + else if (arg === '--reset') opts.reset = true; + else if (arg === '--dry-run') opts.dryRun = true; + else if (arg === '--help' || arg === '-h') { + console.log('Usage: pnpm mirror:imdb -- [--all] [--max-pages=N] [--interval-ms=MS] [--tags=a,b] [--reset] [--dry-run]'); + process.exit(0); + } else { + console.error(`Unknown option: ${arg}`); + process.exit(1); + } + } + return opts; +} + +function stamp(): string { + return new Date().toISOString(); +} + +/** --dry-run without Supabase credentials: walk from the start, keep nothing. */ +function memoryStore(): MirrorStore { + let cursor: MirrorCursor | null = null; + return { + async loadCursor() { return cursor; }, + async saveCursor(_name, c) { cursor = { ...c }; }, + async writeRows() {}, + }; +} + +function makeStore(dryRun: boolean): MirrorStore { + const url = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL; + const key = process.env.SUPABASE_SERVICE_ROLE_KEY; + if (!url || !key) { + if (dryRun) { + console.log('no SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY: dry run with an in-memory cursor'); + return memoryStore(); + } + console.error('ERROR: SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY are required'); + process.exit(1); + } + const supabase = createClient(url, key, { auth: { persistSession: false } }); + + return { + async loadCursor(name: string): Promise { + const { data, error } = await supabase + .from('nichedb_mirror_cursor') + .select('after_id, since') + .eq('name', name) + .maybeSingle(); + if (error) throw new Error(`cursor read failed: ${error.message}`); + if (!data) return null; + return { after_id: Number(data.after_id) || 0, since: data.since ?? null }; + }, + async saveCursor(name: string, cursor: MirrorCursor): Promise { + if (dryRun) return; + const { error } = await supabase + .from('nichedb_mirror_cursor') + .upsert( + { name, after_id: cursor.after_id, since: cursor.since, updated_at: new Date().toISOString() }, + { onConflict: 'name' }, + ); + if (error) throw new Error(`cursor write failed: ${error.message}`); + }, + async writeRows(rows: MirrorRows): Promise { + if (dryRun) return; + if (rows.basics.length) { + const { error } = await supabase + .from('imdb_title_basics') + .upsert(rows.basics, { onConflict: 'tconst' }); + if (error) throw new Error(`imdb_title_basics upsert failed: ${error.message}`); + } + if (rows.basicsFill.length) { + // Rows from TMDB/TVmaze that know a tconst: fill gaps, never overwrite IMDb's own row. + const { error } = await supabase + .from('imdb_title_basics') + .upsert(rows.basicsFill, { onConflict: 'tconst', ignoreDuplicates: true }); + if (error) throw new Error(`imdb_title_basics fill failed: ${error.message}`); + } + if (rows.ratings.length) { + const { error } = await supabase + .from('imdb_title_ratings') + .upsert(rows.ratings, { onConflict: 'tconst' }); + if (error) throw new Error(`imdb_title_ratings upsert failed: ${error.message}`); + } + }, + }; +} + +async function main(): Promise { + const opts = parseArgs(); + const store = makeStore(opts.dryRun); + + console.log(`[${stamp()}] nichedb -> imdb_* mirror starting (maxPages=${opts.maxPages || 'unlimited'}, interval=${opts.intervalMs}ms, tags=${opts.tags.join(',') || 'none'}${opts.dryRun ? ', DRY RUN' : ''})`); + + if (opts.reset) { + console.log(`[${stamp()}] resetting cursor ${MIRROR_CURSOR_NAME}`); + await store.saveCursor(MIRROR_CURSOR_NAME, { after_id: 0, since: null }); + } + + const result = await runMirror({ + store, + maxPages: opts.maxPages, + intervalMs: opts.intervalMs, + tags: opts.tags.length ? opts.tags : undefined, + log: (line) => console.log(`[${stamp()}] ${line}`), + }); + + console.log(`[${stamp()}] done: ${result.pages} pages, ${result.items} items, ${result.basics} basics upserted, ${result.basicsFill} basics filled, ${result.ratings} ratings upserted; ${result.stoppedBecause}; cursor after_id=${result.cursor.after_id} since=${result.cursor.since ?? 'none'}`); + if (!result.completed && result.stoppedBecause !== 'page-cap') process.exitCode = 1; +} + +main().catch((err) => { + console.error(`[${stamp()}] FAILED:`, err instanceof Error ? err.message : err); + process.exit(1); +}); diff --git a/scripts/setup-server.sh b/scripts/setup-server.sh index d1c2fd44..6ef6b57f 100644 --- a/scripts/setup-server.sh +++ b/scripts/setup-server.sh @@ -1032,19 +1032,42 @@ sudo systemctl enable ${IPTV_WORKER_SERVICE} 2>/dev/null || true sudo systemctl enable ${PODCAST_WORKER_SERVICE} 2>/dev/null || true # Set up IMDB dataset daily update cron job (runs at midnight UTC) +# +# Two sources for imdb_title_basics / imdb_title_ratings: +# default update-imdb-daily.sh downloads the IMDb dumps and COPYs all seven tables +# NICHEDB_MIRROR=1 mirror-imdb-from-nichedb.sh walks nichedb.dev's `screen` collection +# over its API (basics + ratings only; crew/episode/akas/principals/ +# name.basics are not refreshed by it) +# Switching flips the 00:00 crontab line; the other script stays in the repo. echo "=== Setting up IMDB dataset daily update cron job ===" -IMDB_UPDATE_SCRIPT="${PROJECT_ROOT}/scripts/update-imdb-daily.sh" +if [ "${NICHEDB_MIRROR:-}" = "1" ]; then + IMDB_UPDATE_SCRIPT="${PROJECT_ROOT}/scripts/mirror-imdb-from-nichedb.sh" + IMDB_UPDATE_MARK="mirror-imdb-from-nichedb" + IMDB_UPDATE_OTHER="update-imdb-daily" + IMDB_UPDATE_LABEL="nichedb -> imdb_* mirror" +else + IMDB_UPDATE_SCRIPT="${PROJECT_ROOT}/scripts/update-imdb-daily.sh" + IMDB_UPDATE_MARK="update-imdb-daily" + IMDB_UPDATE_OTHER="mirror-imdb-from-nichedb" + IMDB_UPDATE_LABEL="IMDB dataset update" +fi IMDB_CRON_JOB="0 0 * * * ${IMDB_UPDATE_SCRIPT} >> /var/log/imdb-update.log 2>&1" if [ -f "${IMDB_UPDATE_SCRIPT}" ]; then chmod +x "${IMDB_UPDATE_SCRIPT}" chmod +x "${PROJECT_ROOT}/scripts/import-imdb.sh" 2>/dev/null || true + chmod +x "${PROJECT_ROOT}/scripts/mirror-imdb-from-nichedb.sh" 2>/dev/null || true sudo touch /var/log/imdb-update.log sudo chown ${VPS_USER}:${VPS_USER} /var/log/imdb-update.log - if crontab -l 2>/dev/null | grep -q "update-imdb-daily"; then - echo " IMDB update cron job already exists" + if crontab -l 2>/dev/null | grep -q "${IMDB_UPDATE_OTHER}"; then + # The other source is scheduled; replace it so only one job runs at 00:00. + (crontab -l 2>/dev/null | grep -v "${IMDB_UPDATE_OTHER}" || true) | crontab - + echo " Removed the ${IMDB_UPDATE_OTHER} cron job" + fi + if crontab -l 2>/dev/null | grep -q "${IMDB_UPDATE_MARK}"; then + echo " ${IMDB_UPDATE_LABEL} cron job already exists" else (crontab -l 2>/dev/null || true; echo "${IMDB_CRON_JOB}") | crontab - - echo "✓ Added cron job: IMDB dataset update at midnight daily" + echo "✓ Added cron job: ${IMDB_UPDATE_LABEL} at midnight daily" fi else echo " WARNING: ${IMDB_UPDATE_SCRIPT} not found, skipping IMDB cron setup" @@ -1139,13 +1162,23 @@ echo " Errors: tail -f ${PODCAST_WORKER_ERROR_LOG}" echo "" echo "Scheduled Tasks:" echo " WebTorrent temp cleanup: Daily at midnight" +if [ "${NICHEDB_MIRROR:-}" = "1" ]; then +echo " IMDB titles: Daily at midnight, mirrored from nichedb.dev (basics + ratings)" +else echo " IMDB dataset update: Daily at midnight (incremental)" +fi echo " Directory: ${WEBTORRENT_TMP_DIR}" echo " View cron jobs: crontab -l" echo "" echo "IMDB Datasets:" +if [ "${NICHEDB_MIRROR:-}" = "1" ]; then +echo " First backfill: pnpm mirror:imdb -- --all (~3-4 h at nichedb's 600 req/h)" +echo " Daily update: ./scripts/mirror-imdb-from-nichedb.sh (automatic via cron)" +echo " Note: crew/episode/akas/principals/name tables are not refreshed by the mirror" +else echo " First import: ./scripts/import-imdb.sh ~/tmp/data" echo " Daily update: ./scripts/update-imdb-daily.sh (automatic via cron)" +fi echo " Logs: /var/log/imdb-update.log" # DHT Services output (only if enabled) diff --git a/src/lib/imdb/tmdb.test.ts b/src/lib/imdb/tmdb.test.ts new file mode 100644 index 00000000..f48a902e --- /dev/null +++ b/src/lib/imdb/tmdb.test.ts @@ -0,0 +1,207 @@ +/** + * fetchTmdbData: cache -> nichedb (NICHEDB_TITLES=1) -> TMDB, and what lands in tmdb_data. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { imdbTitle, jsonResponse, tmdbTitle } from '@/lib/nichedb/fixtures'; + +const supabaseMock = vi.hoisted(() => { + const state = { cached: null as Record | null, upserts: [] as Array<{ table: string; row: Record }> }; + const from = vi.fn((table: string) => ({ + select: () => ({ eq: () => ({ single: async () => ({ data: state.cached, error: null }) }) }), + upsert: async (row: Record) => { state.upserts.push({ table, row }); return { error: null }; }, + })); + return { state, from }; +}); + +vi.mock('@supabase/supabase-js', () => ({ + createClient: () => ({ from: supabaseMock.from }), +})); + +import { fetchTmdbData, fetchNichedbData } from './tmdb'; + +function tmdbFind(movie: Record | null) { + return { movie_results: movie ? [movie] : [], tv_results: [] }; +} + +const TMDB_MOVIE = { id: 603, poster_path: '/p.jpg', backdrop_path: '/b.jpg', overview: 'TMDB overview' }; +const TMDB_DETAIL = { + tagline: 'TMDB tagline', + overview: 'TMDB overview', + credits: { cast: [{ name: 'Keanu Reeves' }], crew: [{ department: 'Writing', name: 'Lana Wachowski' }] }, + release_dates: { results: [{ iso_3166_1: 'US', release_dates: [{ certification: 'R' }] }] }, +}; + +/** A fetch that answers nichedb and TMDB by host and records what it was asked. */ +function routedFetch(routes: { nichedb?: (url: URL) => Response; tmdb?: (url: URL) => Response }) { + const calls: URL[] = []; + const fetchMock = vi.fn(async (input: string) => { + const url = new URL(input); + calls.push(url); + if (url.hostname === 'nichedb.dev') return routes.nichedb?.(url) ?? jsonResponse({ count: 0, items: [] }); + if (url.hostname === 'api.themoviedb.org') { + if (routes.tmdb) return routes.tmdb(url); + if (url.pathname.startsWith('/3/find/')) return jsonResponse(tmdbFind(TMDB_MOVIE)); + if (url.pathname.startsWith('/3/movie/')) return jsonResponse(TMDB_DETAIL); + if (url.pathname.startsWith('/3/search/')) return jsonResponse({ results: [] }); + } + return jsonResponse({}, { status: 404 }); + }); + vi.stubGlobal('fetch', fetchMock); + const hosts = () => calls.map((u) => u.hostname + u.pathname); + return { fetchMock, calls, hosts }; +} + +beforeEach(() => { + supabaseMock.state.cached = null; + supabaseMock.state.upserts = []; + vi.stubEnv('SUPABASE_URL', 'https://example.supabase.co'); + vi.stubEnv('SUPABASE_SERVICE_ROLE_KEY', 'service-role'); + vi.stubEnv('TMDB_API_KEY', 'tmdb-key'); + vi.stubEnv('NICHEDB_TITLES', ''); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +describe('fetchTmdbData with NICHEDB_TITLES off (default)', () => { + it('never talks to nichedb and caches the TMDB answer as before', async () => { + const { hosts } = routedFetch({}); + const data = await fetchTmdbData('tt0133093', 'The.Matrix.1999.1080p'); + + expect(hosts().some((h) => h.startsWith('nichedb.dev'))).toBe(false); + expect(hosts()).toEqual(['api.themoviedb.org/3/find/tt0133093', 'api.themoviedb.org/3/movie/603']); + expect(data).toEqual({ + posterUrl: 'https://image.tmdb.org/t/p/w500/p.jpg', + backdropUrl: 'https://image.tmdb.org/t/p/w1280/b.jpg', + overview: 'TMDB overview', + tagline: 'TMDB tagline', + cast: 'Keanu Reeves', + writers: 'Lana Wachowski', + contentRating: 'R', + tmdbId: 603, + }); + expect(supabaseMock.state.upserts).toEqual([{ table: 'tmdb_data', row: expect.objectContaining({ lookup_key: 'tt0133093', tmdb_id: 603 }) }]); + }); + + it('is EMPTY without a TMDB key', async () => { + vi.stubEnv('TMDB_API_KEY', ''); + const { fetchMock } = routedFetch({}); + const data = await fetchTmdbData('tt0133093', 'The Matrix'); + expect(data.posterUrl).toBeNull(); + expect(data.tmdbId).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('fetchTmdbData with NICHEDB_TITLES=1', () => { + beforeEach(() => vi.stubEnv('NICHEDB_TITLES', '1')); + + it('serves the cache first without any network call', async () => { + supabaseMock.state.cached = { poster_url: 'cached.jpg', backdrop_url: null, overview: 'cached', tagline: null, cast_names: null, writers: null, content_rating: null, tmdb_id: 1 }; + const { fetchMock } = routedFetch({}); + const data = await fetchTmdbData('tt0133093', 'The Matrix'); + expect(data.posterUrl).toBe('cached.jpg'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('takes the nichedb TMDB row, skips TMDB, and writes tmdb_data in the same shape', async () => { + const { hosts, calls } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 2, items: [imdbTitle({ score: 1 }), tmdbTitle({ score: 1 })] }), + }); + const data = await fetchTmdbData('tt0133093', 'The.Matrix.1999.1080p.BluRay.x264-GROUP'); + + expect(hosts()).toEqual(['nichedb.dev/api/v1/match']); + expect(calls[0].searchParams.get('q')).toBe('The.Matrix.1999.1080p.BluRay.x264-GROUP'); + expect(data).toEqual({ + posterUrl: 'https://image.tmdb.org/t/p/w342/aOIuZAjPaRIE6CMzbazvcHuHXDc.jpg', + backdropUrl: 'https://image.tmdb.org/t/p/w780/lrtSb1skJayPydZk0OSMAKjBOVe.jpg', + overview: expect.stringMatching(/^Set in the 22nd century/), + tagline: 'Believe the unbelievable.', + cast: expect.stringMatching(/^Keanu Reeves, Laurence Fishburne/), + writers: null, + contentRating: null, + tmdbId: 603, + }); + expect(supabaseMock.state.upserts).toEqual([{ + table: 'tmdb_data', + row: { + lookup_key: 'tt0133093', + tmdb_id: 603, + poster_url: 'https://image.tmdb.org/t/p/w342/aOIuZAjPaRIE6CMzbazvcHuHXDc.jpg', + backdrop_url: 'https://image.tmdb.org/t/p/w780/lrtSb1skJayPydZk0OSMAKjBOVe.jpg', + overview: expect.any(String), + tagline: 'Believe the unbelievable.', + cast_names: expect.any(String), + writers: null, + content_rating: null, + }, + }]); + }); + + it('falls back to TMDB when nichedb only has the bare IMDb row (nothing to show)', async () => { + const { hosts } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 1, items: [imdbTitle({ score: 1 })] }), + }); + const data = await fetchTmdbData('tt0133093', 'The Matrix 1999'); + expect(hosts()).toEqual(['nichedb.dev/api/v1/match', 'api.themoviedb.org/3/find/tt0133093', 'api.themoviedb.org/3/movie/603']); + expect(data.posterUrl).toBe('https://image.tmdb.org/t/p/w500/p.jpg'); + expect(data.writers).toBe('Lana Wachowski'); + }); + + it('falls back to TMDB when nichedb has a different title for that name (tconst mismatch)', async () => { + const { hosts } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 1, items: [tmdbTitle({ score: 1 }, { imdbId: 'tt9999999' })] }), + }); + await fetchTmdbData('tt0133093', 'The Matrix 1999'); + expect(hosts()[0]).toBe('nichedb.dev/api/v1/match'); + expect(hosts()).toContain('api.themoviedb.org/3/find/tt0133093'); + }); + + it('falls back to TMDB when nichedb errors', async () => { + const { hosts } = routedFetch({ nichedb: () => jsonResponse({ error: 'down' }, { status: 503 }) }); + const data = await fetchTmdbData('tt0133093', 'The Matrix 1999'); + expect(hosts()).toContain('api.themoviedb.org/3/find/tt0133093'); + expect(data.tmdbId).toBe(603); + }); + + it('without an IMDb id uses the best match at or above the floor, poster row preferred', async () => { + const { hosts } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 2, items: [imdbTitle({ score: 1 }), tmdbTitle({ score: 1 })] }), + }); + const data = await fetchTmdbData('', 'The.Matrix.1999.1080p'); + expect(hosts()).toEqual(['nichedb.dev/api/v1/match']); + expect(data.tmdbId).toBe(603); + expect(supabaseMock.state.upserts[0].row.lookup_key).toMatch(/^title:[0-9a-f]{32}$/); + }); + + it('without an IMDb id a below-floor match goes on to the TMDB search', async () => { + const { hosts } = routedFetch({ + nichedb: () => jsonResponse({ q: 'x', parsed: {}, count: 1, items: [tmdbTitle({ score: 0.3 })] }), + tmdb: (url) => url.pathname.startsWith('/3/search/tv') ? jsonResponse({ results: [] }) : url.pathname.startsWith('/3/search/movie') ? jsonResponse({ results: [TMDB_MOVIE] }) : jsonResponse(TMDB_DETAIL), + }); + const data = await fetchTmdbData('', 'The Matrix 1999'); + expect(hosts()[0]).toBe('nichedb.dev/api/v1/match'); + expect(hosts()).toContain('api.themoviedb.org/3/search/movie'); + expect(data.tmdbId).toBe(603); + }); + + it('still asks nichedb without a TMDB key, and is EMPTY (uncached) when nichedb has nothing', async () => { + vi.stubEnv('TMDB_API_KEY', ''); + const { hosts } = routedFetch({}); + const data = await fetchTmdbData('tt0133093', 'The Matrix'); + expect(hosts()).toEqual(['nichedb.dev/api/v1/match']); + expect(data.tmdbId).toBeNull(); + expect(supabaseMock.state.upserts).toEqual([]); + }); +}); + +describe('fetchNichedbData', () => { + it('answers null without a title hint (nothing to ask nichedb for)', async () => { + const { fetchMock } = routedFetch({}); + expect(await fetchNichedbData('tt0133093', '')).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/imdb/tmdb.ts b/src/lib/imdb/tmdb.ts index 26397ede..9bdcb54a 100644 --- a/src/lib/imdb/tmdb.ts +++ b/src/lib/imdb/tmdb.ts @@ -4,10 +4,15 @@ * Falls back to /search if /find returns nothing (common for obscure IMDB entries). * * Results are cached in the tmdb_data table to avoid repeated API calls. + * + * With NICHEDB_TITLES=1 the lookup asks nichedb.dev's `screen` collection first + * (src/lib/nichedb/titles.ts) and only falls through to TMDB when nichedb has + * nothing to show for the title. The cache row is written the same way either way. */ import { createClient } from '@supabase/supabase-js'; import { createHash } from 'crypto'; +import { hasPresentation, matchTitle, titleByImdbId, titleFacts } from '@/lib/nichedb/titles'; export interface TmdbData { posterUrl: string | null; @@ -108,9 +113,49 @@ function cleanTitleForSearch(titleHint: string): string { return cleanTitle; } +/** NICHEDB_TITLES=1 routes lookups through nichedb.dev before TMDB. Default off. */ +export function nichedbTitlesEnabled(): boolean { + return process.env.NICHEDB_TITLES === '1'; +} + +/** + * The nichedb row for this torrent, shaped like a TMDB answer, or null when + * nichedb has nothing to show for it (no poster, no summary, no backdrop), in + * which case the caller goes on to TMDB as it always did. + * + * With an IMDb id the row must carry that tconst; the release name only tells + * nichedb what to search. Without one the best `match` candidate at or above the + * score floor is taken, preferring the row with a poster among exact titles. + */ +export async function fetchNichedbData(imdbId: string, titleHint?: string): Promise { + const name = titleHint?.trim(); + if (!name) return null; + try { + const item = imdbId + ? await titleByImdbId(imdbId, { title: name }) + : await matchTitle(name); + if (!item || !hasPresentation(item)) return null; + const facts = titleFacts(item); + return { + posterUrl: facts.posterUrl, + backdropUrl: facts.backdropUrl, + overview: facts.overview, + tagline: facts.tagline, + cast: facts.cast, + // nichedb carries no writers or certification; the tmdb_data columns stay null. + writers: null, + contentRating: null, + tmdbId: facts.tmdbId, + }; + } catch { + return null; + } +} + export async function fetchTmdbData(imdbId: string, titleHint?: string): Promise { const tmdbKey = process.env.TMDB_API_KEY; - if (!tmdbKey) return EMPTY; + const useNichedb = nichedbTitlesEnabled(); + if (!tmdbKey && !useNichedb) return EMPTY; if (!imdbId && !titleHint) return EMPTY; // Check cache first @@ -118,6 +163,17 @@ export async function fetchTmdbData(imdbId: string, titleHint?: string): Promise const cached = await getCached(cacheKey); if (cached) return cached; + // nichedb first when switched on; the answer lands in tmdb_data exactly as a + // TMDB one would, so every reader of the cache is unchanged. + if (useNichedb) { + const fromNichedb = await fetchNichedbData(imdbId, titleHint); + if (fromNichedb) { + await setCache(cacheKey, fromNichedb); + return fromNichedb; + } + } + if (!tmdbKey) return EMPTY; + try { let tmdbId: number | null = null; let isTV = false; diff --git a/src/lib/nichedb/fixtures.ts b/src/lib/nichedb/fixtures.ts new file mode 100644 index 00000000..5bc9d999 --- /dev/null +++ b/src/lib/nichedb/fixtures.ts @@ -0,0 +1,130 @@ +/** + * nichedb `screen` title items as they come off the wire (0.6.1), for tests. + * Shapes copied from live answers on 2026-09-10. + */ + +import type { NichedbTitleItem } from './titles'; + +export function imdbTitle(over: Partial = {}, data: Record = {}): NichedbTitleItem { + return { + id: 3742539, + collection: 'screen', + kind: 'title', + external_id: 'imdb:title:tt0133093', + updated_at: '2026-09-10T14:05:02.923Z', + title: 'The Matrix', + summary: null, + url: 'https://www.imdb.com/title/tt0133093/', + image_url: null, + published_at: '1999-07-01T12:00:00.000Z', + tags: ['title', 'film', 'imdb', 'genre:action', 'genre:sci-fi'], + ...over, + data: { + form: 'movie', + year: 1999, + watch: [], + genres: ['Action', 'Sci-Fi'], + imdbId: 'tt0133093', + rating: 8.7, + tmdbId: null, + endYear: null, + tagline: null, + category: 'film', + provider: 'imdb', + tvmazeId: null, + anilistId: null, + normTitle: 'the matrix', + titleType: 'movie', + popularity: null, + runtimeMin: 136, + trailerUrl: null, + backdropUrl: null, + ratingCount: 2276418, + originalTitle: 'The Matrix', + ...data, + }, + }; +} + +export function tmdbTitle(over: Partial = {}, data: Record = {}): NichedbTitleItem { + return { + id: 3729335, + collection: 'screen', + kind: 'title', + external_id: 'tmdb:title:603', + updated_at: '2026-09-10T14:03:59.947Z', + title: 'The Matrix', + summary: 'Set in the 22nd century, The Matrix tells the story of a computer hacker who joins a group of underground insurgents fighting the vast and powerful computers who now rule the earth.', + url: 'https://www.themoviedb.org/movie/603', + image_url: 'https://image.tmdb.org/t/p/w342/aOIuZAjPaRIE6CMzbazvcHuHXDc.jpg', + published_at: '1999-03-31T12:00:00.000Z', + tags: ['title', 'film', 'tmdb', 'genre:action', 'genre:science-fiction'], + ...over, + data: { + cast: ['Keanu Reeves', 'Laurence Fishburne', 'Carrie-Anne Moss', 'Hugo Weaving', 'Gloria Foster', 'Joe Pantoliano', 'Marcus Chong', 'Julian Arahanga'], + form: 'movie', + year: 1999, + watch: ['YouTube TV'], + genres: ['Action', 'Science Fiction'], + imdbId: 'tt0133093', + rating: 8.257, + tmdbId: '603', + tagline: 'Believe the unbelievable.', + category: 'film', + provider: 'tmdb', + normTitle: 'the matrix', + popularity: 47.1488, + runtimeMin: 136, + trailerUrl: 'https://www.youtube.com/watch?v=FVI84Dfx2-I', + backdropUrl: 'https://image.tmdb.org/t/p/w780/lrtSb1skJayPydZk0OSMAKjBOVe.jpg', + ratingCount: 28678, + originalTitle: 'The Matrix', + ...data, + }, + }; +} + +export function tvmazeTitle(over: Partial = {}, data: Record = {}): NichedbTitleItem { + return { + id: 3611951, + collection: 'screen', + kind: 'title', + external_id: 'tvmaze:title:93688', + updated_at: '2026-09-10T14:03:28.877Z', + title: 'Perfect Addiction', + summary: 'Akihito is a college student obsessed with good looks.', + url: 'https://www.tvmaze.com/shows/93688/perfect-addiction', + image_url: 'https://static.tvmaze.com/uploads/images/original_untouched/637/1594382.jpg', + published_at: '2026-07-08T12:00:00.000Z', + tags: ['title', 'anime', 'tvmaze', 'genre:romance'], + ...over, + data: { + form: 'series', + year: 2026, + genres: ['Romance'], + imdbId: null, + rating: null, + tmdbId: null, + category: 'anime', + provider: 'tvmaze', + tvmazeId: '93688', + normTitle: 'perfect addiction', + runtimeMin: 5, + backdropUrl: null, + ratingCount: null, + ...data, + }, + }; +} + +/** A Response-like object for a mocked fetch. */ +export function jsonResponse(body: unknown, init: { status?: number; headers?: Record } = {}): Response { + const status = init.status ?? 200; + return { + ok: status >= 200 && status < 300, + status, + headers: new Headers(init.headers ?? {}), + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response; +} diff --git a/src/lib/nichedb/mirror.test.ts b/src/lib/nichedb/mirror.test.ts new file mode 100644 index 00000000..0ca0c60b --- /dev/null +++ b/src/lib/nichedb/mirror.test.ts @@ -0,0 +1,245 @@ +/** + * nichedb -> imdb_* mirror: row mapping and the cursor walk. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { + MIRROR_CURSOR_NAME, + pageToRows, + runMirror, + titleToBasics, + titleToRatings, + type MirrorCursor, + type MirrorRows, + type MirrorStore, +} from './mirror'; +import { imdbTitle, jsonResponse, tmdbTitle, tvmazeTitle } from './fixtures'; +import type { NichedbTitleItem } from './titles'; + +describe('titleToBasics', () => { + it('maps an IMDb film row onto the dump columns', () => { + expect(titleToBasics(imdbTitle())).toEqual({ + tconst: 'tt0133093', + title_type: 'movie', + primary_title: 'The Matrix', + original_title: 'The Matrix', + is_adult: null, + start_year: 1999, + end_year: null, + runtime_minutes: 136, + genres: 'Action,Sci-Fi', + }); + }); + + it('keeps IMDb titleType and endYear for a series', () => { + const row = titleToBasics(imdbTitle( + { external_id: 'imdb:title:tt0039123', title: 'Kraft Television Theatre', tags: ['title', 'tv', 'imdb'] }, + { imdbId: 'tt0039123', form: 'series', titleType: 'tvSeries', year: 1947, endYear: 1958, genres: ['Drama'], runtimeMin: 60, originalTitle: 'Kraft Television Theatre' }, + )); + expect(row).toMatchObject({ tconst: 'tt0039123', title_type: 'tvSeries', start_year: 1947, end_year: 1958, genres: 'Drama', runtime_minutes: 60 }); + }); + + it('derives title_type from form when a non-IMDb row has no titleType', () => { + expect(titleToBasics(tmdbTitle())?.title_type).toBe('movie'); + expect(titleToBasics(tvmazeTitle({}, { imdbId: 'tt16255458' }))?.title_type).toBe('tvSeries'); + }); + + it('takes the tconst from the external id when data.imdbId is missing', () => { + expect(titleToBasics(imdbTitle({}, { imdbId: null }))?.tconst).toBe('tt0133093'); + }); + + it('answers null for a title with no IMDb id, and null fields for missing data', () => { + expect(titleToBasics(tvmazeTitle())).toBeNull(); + const bare = titleToBasics(imdbTitle({}, { year: null, runtimeMin: null, genres: [], originalTitle: null, titleType: null, form: 'unknown' })); + expect(bare).toMatchObject({ start_year: null, runtime_minutes: null, genres: null, original_title: 'The Matrix', title_type: null }); + }); +}); + +describe('titleToRatings', () => { + it('maps rating and votes, rounded to the dump precision', () => { + expect(titleToRatings(imdbTitle())).toEqual({ tconst: 'tt0133093', average_rating: 8.7, num_votes: 2276418 }); + expect(titleToRatings(imdbTitle({}, { rating: 8.257, ratingCount: 10 }))?.average_rating).toBe(8.3); + }); + + it('answers null when unrated', () => { + expect(titleToRatings(imdbTitle({}, { rating: null, ratingCount: null }))).toBeNull(); + expect(titleToRatings(imdbTitle({}, { rating: 7, ratingCount: null }))).toBeNull(); + }); +}); + +describe('pageToRows', () => { + it('writes IMDb rows as basics + ratings and other providers as fill only', () => { + const rows = pageToRows([imdbTitle(), tmdbTitle(), tvmazeTitle(), tvmazeTitle({ id: 9, external_id: 'tvmaze:title:9' }, { imdbId: 'tt16255458', rating: 4.2, ratingCount: 30 })]); + expect(rows.basics.map((r) => r.tconst)).toEqual(['tt0133093']); + expect(rows.basicsFill.map((r) => r.tconst)).toEqual(['tt16255458']); + expect(rows.ratings).toEqual([{ tconst: 'tt0133093', average_rating: 8.7, num_votes: 2276418 }]); + }); + + it('never lets a TMDB rating masquerade as an IMDb one', () => { + const rows = pageToRows([tmdbTitle()]); + expect(rows.ratings).toEqual([]); + expect(rows.basicsFill).toHaveLength(1); + }); + + it('dedupes within a page, IMDb row winning whatever the order', () => { + const a = pageToRows([tmdbTitle(), imdbTitle()]); + const b = pageToRows([imdbTitle(), tmdbTitle()]); + for (const rows of [a, b]) { + expect(rows.basics).toHaveLength(1); + expect(rows.basicsFill).toHaveLength(0); + expect(rows.basics[0].genres).toBe('Action,Sci-Fi'); + } + }); +}); + +function memoryStore(initial: MirrorCursor | null = null) { + let cursor = initial; + const saves: MirrorCursor[] = []; + const writes: MirrorRows[] = []; + const store: MirrorStore = { + loadCursor: vi.fn(async () => cursor), + saveCursor: vi.fn(async (_name: string, c: MirrorCursor) => { cursor = { ...c }; saves.push({ ...c }); }), + writeRows: vi.fn(async (rows: MirrorRows) => { writes.push(rows); }), + }; + return { store, saves, writes, cursor: () => cursor }; +} + +function pageOf(ids: number[]): NichedbTitleItem[] { + return ids.map((id) => imdbTitle({ id, external_id: `imdb:title:tt${String(id).padStart(7, '0')}` }, { imdbId: `tt${String(id).padStart(7, '0')}` })); +} + +/** A fetch that serves id-ordered pages of `limit` from `all`, honouring `after`. */ +function pagedFetch(all: number[], opts: { headers?: Record } = {}) { + const calls: URL[] = []; + const fetchMock = vi.fn(async (input: string) => { + const url = new URL(input); + calls.push(url); + const after = Number(url.searchParams.get('after') ?? 0); + const limit = Number(url.searchParams.get('limit')); + const items = pageOf(all.filter((id) => id > after).slice(0, limit)); + return jsonResponse({ count: items.length, items }, { headers: opts.headers }); + }); + return { fetchMock, calls }; +} + +const noSleep = vi.fn(async () => {}); +const clock = () => new Date('2026-09-10T00:00:00.000Z'); + +describe('runMirror', () => { + it('walks every page, advances the cursor per page, and closes the walk with since=start', async () => { + const { store, saves, writes } = memoryStore(); + const { fetchMock, calls } = pagedFetch([1, 2, 3, 4, 5]); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + expect(result).toMatchObject({ pages: 3, items: 5, basics: 5, ratings: 5, completed: true, stoppedBecause: 'completed' }); + expect(calls.map((u) => u.searchParams.get('after'))).toEqual([null, '2', '4']); + expect(calls.every((u) => !u.searchParams.has('since'))).toBe(true); + expect(writes).toHaveLength(3); + // one save per page, then the closing save + expect(saves.map((s) => s.after_id)).toEqual([2, 4, 5, 0]); + expect(saves.at(-1)).toEqual({ after_id: 0, since: '2026-09-10T00:00:00.000Z' }); + expect(store.saveCursor).toHaveBeenCalledWith(MIRROR_CURSOR_NAME, expect.anything()); + }); + + it('stops at the page cap and leaves a resumable cursor', async () => { + const { store, saves } = memoryStore(); + const { fetchMock } = pagedFetch([1, 2, 3, 4, 5, 6]); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 2, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + expect(result).toMatchObject({ pages: 2, items: 4, completed: false, stoppedBecause: 'page-cap' }); + expect(result.cursor).toEqual({ after_id: 4, since: null }); + expect(saves.at(-1)).toEqual({ after_id: 4, since: null }); + }); + + it('resumes from a stored cursor and sends its since= with every page', async () => { + const { store } = memoryStore({ after_id: 4, since: '2026-09-01T00:00:00.000Z' }); + const { fetchMock, calls } = pagedFetch([1, 2, 3, 4, 5, 6]); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + // [5,6] is a full page, so one more (empty) page closes the walk + expect(calls.map((u) => u.searchParams.get('after'))).toEqual(['4', '6']); + expect(calls.every((u) => u.searchParams.get('since') === '2026-09-01T00:00:00.000Z')).toBe(true); + expect(result).toMatchObject({ pages: 2, items: 2, completed: true }); + // the next walk's watermark is this walk's start, not the old one + expect(result.cursor).toEqual({ after_id: 0, since: '2026-09-10T00:00:00.000Z' }); + }); + + it('a short final page closes the walk in one request when nothing changed', async () => { + const { store, writes } = memoryStore({ after_id: 0, since: '2026-09-09T00:00:00.000Z' }); + const { fetchMock } = pagedFetch([]); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + expect(result).toMatchObject({ pages: 1, items: 0, completed: true }); + expect(writes).toHaveLength(0); + expect(result.cursor.since).toBe('2026-09-10T00:00:00.000Z'); + }); + + it('sleeps the interval between pages, not after the last', async () => { + const { store } = memoryStore(); + const { fetchMock } = pagedFetch([1, 2, 3, 4]); + const sleep = vi.fn(async (_ms: number) => {}); + + await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 7000, pageLimit: 2, sleep, now: clock }); + + // pages: [1,2] full -> sleep; [3,4] full -> sleep; [] short -> done + expect(sleep.mock.calls.map((c) => c[0])).toEqual([7000, 7000]); + }); + + it('waits out a 429 and carries on, keeping the cursor', async () => { + const { store } = memoryStore(); + let n = 0; + const fetchMock = vi.fn(async (input: string) => { + n += 1; + if (n === 2) return jsonResponse({ error: 'slow down' }, { status: 429 }); + const after = Number(new URL(input).searchParams.get('after') ?? 0); + const items = pageOf([1, 2, 3].filter((id) => id > after).slice(0, 2)); + return jsonResponse({ count: items.length, items }); + }); + const sleep = vi.fn(async (_ms: number) => {}); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep, now: clock }); + + expect(result).toMatchObject({ pages: 2, items: 3, completed: true }); + expect(sleep.mock.calls.map((c) => c[0])).toContain(60_000); + }); + + it('gives up after repeated 429s without touching the cursor', async () => { + const { store, saves } = memoryStore({ after_id: 10, since: null }); + const fetchMock = vi.fn(async () => jsonResponse({}, { status: 429 })); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, sleep: noSleep, now: clock, maxRateLimitWaits: 1 }); + + expect(result.stoppedBecause).toBe('rate-limit'); + expect(result.cursor).toEqual({ after_id: 10, since: null }); + expect(saves).toHaveLength(0); + }); + + it('stops on a network failure with the cursor where the last good page left it', async () => { + const { store } = memoryStore(); + let n = 0; + const fetchMock = vi.fn(async () => { + n += 1; + if (n === 2) throw new Error('reset'); + return jsonResponse({ count: 2, items: pageOf([1, 2]) }); + }); + + const result = await runMirror({ store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + expect(result).toMatchObject({ pages: 1, stoppedBecause: 'network', completed: false }); + expect(result.cursor).toEqual({ after_id: 2, since: null }); + }); + + it('is idempotent: re-running a completed walk re-fetches nothing new and rewrites the same rows', async () => { + const first = memoryStore(); + const { fetchMock } = pagedFetch([1, 2, 3]); + await runMirror({ store: first.store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + + const second = memoryStore({ after_id: 0, since: null }); + await runMirror({ store: second.store, fetch: fetchMock, maxPages: 0, intervalMs: 5, pageLimit: 2, sleep: noSleep, now: clock }); + expect(second.writes).toEqual(first.writes); + }); +}); diff --git a/src/lib/nichedb/mirror.ts b/src/lib/nichedb/mirror.ts new file mode 100644 index 00000000..0dd79df5 --- /dev/null +++ b/src/lib/nichedb/mirror.ts @@ -0,0 +1,237 @@ +/** + * nichedb `screen` -> imdb_title_basics / imdb_title_ratings. + * + * Pure pieces of scripts/mirror-imdb-from-nichedb.ts: the row mapping and the + * cursor walk, with the network, the store and the clock injected so they can be + * exercised against fixtures. + * + * The walk is id-ordered. A run starts from the cursor's `after_id` and sends the + * cursor's `since` with every page; when it reaches the end it resets `after_id` + * to 0 and sets `since` to the moment this walk began, so the next run only asks + * for items nichedb updated after that. A run that stops early (page cap, network + * error, rate limit) leaves `after_id` where it got to and keeps the same `since`, + * so the next run carries on from there. + */ + +import { fetchTitlesPage, imdbIdOf, type NichedbClientOptions, type NichedbTitleItem } from './titles'; + +export const MIRROR_CURSOR_NAME = 'imdb-titles'; + +/** imdb_title_basics row, the columns the dump import fills. */ +export interface ImdbBasicsRow { + tconst: string; + title_type: string | null; + primary_title: string | null; + original_title: string | null; + is_adult: boolean | null; + start_year: number | null; + end_year: number | null; + runtime_minutes: number | null; + genres: string | null; +} + +/** imdb_title_ratings row. */ +export interface ImdbRatingsRow { + tconst: string; + average_rating: number; + num_votes: number; +} + +export interface MirrorCursor { + after_id: number; + since: string | null; +} + +export interface MirrorRows { + /** From IMDb-sourced rows: authoritative, upserted (overwrite). */ + basics: ImdbBasicsRow[]; + /** From other providers that know the tconst: insert-only, fills gaps but never overwrites. */ + basicsFill: ImdbBasicsRow[]; + /** Only IMDb's own rating goes into imdb_title_ratings; TMDB/TVmaze ratings are a different scale. */ + ratings: ImdbRatingsRow[]; +} + +const FORM_TO_TITLE_TYPE: Record = { movie: 'movie', series: 'tvSeries' }; + +function intOrNull(v: unknown): number | null { + if (v == null || v === '') return null; + const n = typeof v === 'number' ? v : Number(v); + return Number.isFinite(n) ? Math.trunc(n) : null; +} + +/** Map one nichedb title to an imdb_title_basics row, or null when it has no tconst. */ +export function titleToBasics(item: NichedbTitleItem): ImdbBasicsRow | null { + const tconst = imdbIdOf(item); + if (!tconst) return null; + const d = item.data ?? {}; + const titleType = (typeof d.titleType === 'string' && d.titleType) || FORM_TO_TITLE_TYPE[d.form ?? ''] || null; + const genres = Array.isArray(d.genres) ? d.genres.filter((g): g is string => typeof g === 'string' && g.length > 0) : []; + return { + tconst, + title_type: titleType, + primary_title: item.title ?? null, + original_title: (typeof d.originalTitle === 'string' && d.originalTitle) || item.title || null, + is_adult: typeof d.isAdult === 'boolean' ? d.isAdult : null, + start_year: intOrNull(d.year), + end_year: intOrNull(d.endYear), + runtime_minutes: intOrNull(d.runtimeMin), + // The dump stores genres as a bare comma list ("Action,Sci-Fi"); readers + // re-space it themselves (enrich.ts: genres.replace(/,/g, ', ')). + genres: genres.length ? genres.join(',') : null, + }; +} + +/** Map an IMDb-sourced nichedb title to an imdb_title_ratings row, or null when unrated. */ +export function titleToRatings(item: NichedbTitleItem): ImdbRatingsRow | null { + const tconst = imdbIdOf(item); + if (!tconst) return null; + const d = item.data ?? {}; + const rating = typeof d.rating === 'number' ? d.rating : Number(d.rating); + const votes = intOrNull(d.ratingCount); + if (!Number.isFinite(rating) || votes == null) return null; + return { tconst, average_rating: Math.round(rating * 10) / 10, num_votes: votes }; +} + +export function isImdbSourced(item: NichedbTitleItem): boolean { + return item.data?.provider === 'imdb' || /^imdb:title:/.test(item.external_id ?? ''); +} + +/** Split a page of titles into the rows to write. Items without a tconst are skipped. */ +export function pageToRows(items: NichedbTitleItem[]): MirrorRows { + const basics = new Map(); + const basicsFill = new Map(); + const ratings = new Map(); + for (const item of items) { + const row = titleToBasics(item); + if (!row) continue; + if (isImdbSourced(item)) { + basics.set(row.tconst, row); + basicsFill.delete(row.tconst); + const r = titleToRatings(item); + if (r) ratings.set(r.tconst, r); + } else if (!basics.has(row.tconst)) { + basicsFill.set(row.tconst, row); + } + } + return { basics: [...basics.values()], basicsFill: [...basicsFill.values()], ratings: [...ratings.values()] }; +} + +export interface MirrorStore { + loadCursor(name: string): Promise; + saveCursor(name: string, cursor: MirrorCursor): Promise; + writeRows(rows: MirrorRows): Promise; +} + +export interface MirrorRunOptions extends NichedbClientOptions { + store: MirrorStore; + /** Pages this run may fetch; 0 = until the walk completes. */ + maxPages: number; + /** Pause between pages. 600 req/h is one every 6 s; leave room for the site's own calls. */ + intervalMs: number; + pageLimit?: number; + /** Restrict the walk, e.g. ['imdb'] for IMDb-sourced rows only. */ + tags?: string[]; + cursorName?: string; + now?: () => Date; + sleep?: (ms: number) => Promise; + log?: (line: string) => void; + /** How many times to wait out a 429 before giving up this run. */ + maxRateLimitWaits?: number; +} + +export interface MirrorRunResult { + pages: number; + items: number; + basics: number; + basicsFill: number; + ratings: number; + completed: boolean; + cursor: MirrorCursor; + stoppedBecause: 'completed' | 'page-cap' | 'network' | 'rate-limit' | 'http'; +} + +/** + * Walk the collection from the stored cursor, writing rows page by page and + * advancing the cursor after each page so a crash loses at most one page of work + * (which the next run re-fetches; the upserts are idempotent). + */ +export async function runMirror(opts: MirrorRunOptions): Promise { + const name = opts.cursorName ?? MIRROR_CURSOR_NAME; + const now = opts.now ?? (() => new Date()); + const sleep = opts.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const log = opts.log ?? (() => {}); + const pageLimit = opts.pageLimit ?? 200; + const maxRateLimitWaits = opts.maxRateLimitWaits ?? 3; + + const startedAt = now().toISOString(); + const cursor: MirrorCursor = (await opts.store.loadCursor(name)) ?? { after_id: 0, since: null }; + log(`cursor ${name}: after_id=${cursor.after_id} since=${cursor.since ?? 'none'} (walk started ${startedAt})`); + + const result: MirrorRunResult = { + pages: 0, items: 0, basics: 0, basicsFill: 0, ratings: 0, + completed: false, cursor, stoppedBecause: 'page-cap', + }; + let rateLimitWaits = 0; + + while (opts.maxPages === 0 || result.pages < opts.maxPages) { + const fetched = await fetchTitlesPage( + { after: cursor.after_id, since: cursor.since, limit: pageLimit, tags: opts.tags }, + opts, + ); + if (!fetched) { + result.stoppedBecause = 'network'; + log('network error; stopping, cursor kept'); + break; + } + if (fetched.status === 429) { + if (rateLimitWaits++ >= maxRateLimitWaits) { + result.stoppedBecause = 'rate-limit'; + log('rate limited too many times; stopping, cursor kept'); + break; + } + log('rate limited (429); waiting 60s'); + await sleep(60_000); + continue; + } + if (fetched.status >= 400) { + result.stoppedBecause = 'http'; + log(`HTTP ${fetched.status}; stopping, cursor kept`); + break; + } + + result.pages += 1; + const items = fetched.page.items; + if (items.length > 0) { + const rows = pageToRows(items); + await opts.store.writeRows(rows); + result.items += items.length; + result.basics += rows.basics.length; + result.basicsFill += rows.basicsFill.length; + result.ratings += rows.ratings.length; + cursor.after_id = items[items.length - 1].id; + await opts.store.saveCursor(name, { ...cursor }); + log(`page ${result.pages}: ${items.length} items -> ${rows.basics.length} basics, ${rows.basicsFill.length} fill, ${rows.ratings.length} ratings; after_id=${cursor.after_id}${fetched.rateRemaining != null ? ` (rate remaining ${fetched.rateRemaining})` : ''}`); + } + + if (items.length < pageLimit) { + // End of the walk: next run asks only for what changed since this one began. + cursor.after_id = 0; + cursor.since = startedAt; + await opts.store.saveCursor(name, { ...cursor }); + result.completed = true; + result.stoppedBecause = 'completed'; + log(`walk complete; next run uses since=${startedAt}`); + break; + } + + if (fetched.rateRemaining != null && fetched.rateRemaining <= 1) { + log('rate budget exhausted; waiting 60s'); + await sleep(60_000); + } else { + await sleep(opts.intervalMs); + } + } + + result.cursor = { ...cursor }; + return result; +} diff --git a/src/lib/nichedb/titles.test.ts b/src/lib/nichedb/titles.test.ts new file mode 100644 index 00000000..1c56169c --- /dev/null +++ b/src/lib/nichedb/titles.test.ts @@ -0,0 +1,189 @@ +/** + * nichedb titles client: the match pick, id extraction, facts, and the wire calls. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { + MATCH_SCORE_FLOOR, + fetchTitlesPage, + hasPresentation, + imdbIdOf, + matchTitle, + pickMatch, + titleByImdbId, + titleFacts, +} from './titles'; +import { imdbTitle, jsonResponse, tmdbTitle, tvmazeTitle } from './fixtures'; + +describe('imdbIdOf', () => { + it('reads data.imdbId first', () => { + expect(imdbIdOf(tmdbTitle())).toBe('tt0133093'); + }); + + it('falls back to the imdb:title external id', () => { + expect(imdbIdOf(imdbTitle({}, { imdbId: null }))).toBe('tt0133093'); + }); + + it('answers null when neither carries a tconst', () => { + expect(imdbIdOf(tvmazeTitle())).toBeNull(); + expect(imdbIdOf(tmdbTitle({}, { imdbId: 'garbage' }))).toBeNull(); + }); +}); + +describe('pickMatch', () => { + it('prefers the row with a poster among exact titles', () => { + const imdb = imdbTitle({ score: 1 }); + const tmdb = tmdbTitle({ score: 1 }); + expect(pickMatch([imdb, tmdb])?.external_id).toBe('tmdb:title:603'); + expect(pickMatch([tmdb, imdb])?.external_id).toBe('tmdb:title:603'); + }); + + it('takes the best score even when a lower one has a poster', () => { + const exact = imdbTitle({ score: 1 }); + const near = tmdbTitle({ id: 1, external_id: 'tmdb:title:1', title: 'The Matrix Reloaded', score: 0.8 }); + expect(pickMatch([near, exact])?.external_id).toBe('imdb:title:tt0133093'); + }); + + it('answers null when nothing reaches the floor', () => { + const low = tmdbTitle({ score: MATCH_SCORE_FLOOR - 0.01 }); + expect(pickMatch([low])).toBeNull(); + expect(pickMatch([])).toBeNull(); + }); + + it('accepts a candidate exactly at the floor', () => { + const at = tmdbTitle({ score: MATCH_SCORE_FLOOR }); + expect(pickMatch([at])).toBe(at); + }); + + it('with an imdbId keeps only rows carrying that tconst, regardless of score', () => { + // Live shape: "Dune" (tmdb, score 1) vs "Dune: Part One" (imdb, score 0.38), same tconst. + const other = tmdbTitle({ id: 2, external_id: 'tmdb:title:2', title: 'Dune', score: 1 }, { imdbId: 'tt1160419' }); + const wanted = imdbTitle({ id: 3, external_id: 'imdb:title:tt0000001', title: 'Dune World', score: 0.45 }, { imdbId: 'tt0000001' }); + expect(pickMatch([other, wanted], { imdbId: 'tt0000001' })?.id).toBe(3); + expect(pickMatch([other, wanted], { imdbId: 'tt9999999' })).toBeNull(); + }); + + it('with an imdbId still prefers the poster row among rows for that tconst', () => { + const imdb = imdbTitle({ score: 0.38, title: 'Dune: Part One' }, { imdbId: 'tt1160419' }); + const tmdb = tmdbTitle({ score: 1, title: 'Dune' }, { imdbId: 'tt1160419' }); + expect(pickMatch([imdb, tmdb], { imdbId: 'tt1160419' })?.external_id).toBe('tmdb:title:603'); + }); +}); + +describe('hasPresentation / titleFacts', () => { + it('an IMDb row has nothing to show, a TMDB row does', () => { + expect(hasPresentation(imdbTitle())).toBe(false); + expect(hasPresentation(tmdbTitle())).toBe(true); + expect(hasPresentation(imdbTitle({ summary: 'A plot.' }))).toBe(true); + }); + + it('normalises a TMDB row', () => { + const f = titleFacts(tmdbTitle()); + expect(f).toMatchObject({ + imdbId: 'tt0133093', + tmdbId: 603, + year: 1999, + form: 'movie', + category: 'film', + rating: 8.257, + ratingCount: 28678, + runtimeMin: 136, + posterUrl: 'https://image.tmdb.org/t/p/w342/aOIuZAjPaRIE6CMzbazvcHuHXDc.jpg', + backdropUrl: 'https://image.tmdb.org/t/p/w780/lrtSb1skJayPydZk0OSMAKjBOVe.jpg', + tagline: 'Believe the unbelievable.', + }); + expect(f.genres).toEqual(['Action', 'Science Fiction']); + expect(f.cast).toBe('Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss, Hugo Weaving, Gloria Foster, Joe Pantoliano, Marcus Chong, Julian Arahanga'); + expect(f.overview).toMatch(/^Set in the 22nd century/); + }); + + it('normalises an IMDb row with nulls where it has nothing', () => { + const f = titleFacts(imdbTitle()); + expect(f).toMatchObject({ tmdbId: null, posterUrl: null, backdropUrl: null, overview: null, tagline: null, cast: null, rating: 8.7 }); + }); +}); + +describe('matchTitle', () => { + it('asks /api/v1/match with the release name and year and picks the poster row', async () => { + const fetchMock = vi.fn(async (_input: string) => + jsonResponse({ q: 'x', parsed: {}, count: 2, items: [imdbTitle({ score: 1 }), tmdbTitle({ score: 1 })] }), + ); + const item = await matchTitle('The.Matrix.1999.1080p.BluRay.x264-GROUP', { year: 1999, fetch: fetchMock }); + expect(item?.external_id).toBe('tmdb:title:603'); + const url = new URL(fetchMock.mock.calls[0][0]); + expect(url.pathname).toBe('/api/v1/match'); + expect(url.searchParams.get('q')).toBe('The.Matrix.1999.1080p.BluRay.x264-GROUP'); + expect(url.searchParams.get('collection')).toBe('screen'); + expect(url.searchParams.get('kind')).toBe('title'); + expect(url.searchParams.get('year')).toBe('1999'); + }); + + it('answers null on a non-2xx or a thrown fetch', async () => { + expect(await matchTitle('x', { fetch: vi.fn(async () => jsonResponse({}, { status: 503 })) })).toBeNull(); + expect(await matchTitle('x', { fetch: vi.fn(async () => { throw new Error('down'); }) })).toBeNull(); + }); + + it('does not call out for an empty name', async () => { + const fetchMock = vi.fn(); + expect(await matchTitle(' ', { fetch: fetchMock })).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('titleByImdbId', () => { + it('matches on the known title and keeps the row carrying the tconst', async () => { + const fetchMock = vi.fn(async (_input: string) => + jsonResponse({ q: 'x', parsed: {}, count: 2, items: [imdbTitle({ score: 1 }), tmdbTitle({ score: 1 })] }), + ); + const item = await titleByImdbId('tt0133093', { title: 'The Matrix', year: 1999 }, { fetch: fetchMock }); + expect(item?.external_id).toBe('tmdb:title:603'); + const url = new URL(fetchMock.mock.calls[0][0]); + expect(url.searchParams.get('q')).toBe('The Matrix'); + expect(url.searchParams.get('year')).toBe('1999'); + }); + + it('answers null when no candidate carries the tconst', async () => { + const fetchMock = vi.fn(async (_input: string) => jsonResponse({ q: 'x', parsed: {}, count: 1, items: [tmdbTitle({ score: 1 })] })); + expect(await titleByImdbId('tt7777777', { title: 'The Matrix' }, { fetch: fetchMock })).toBeNull(); + }); + + it('answers null without a title to ask for, and for a malformed tconst', async () => { + const fetchMock = vi.fn(); + expect(await titleByImdbId('tt0133093', {}, { fetch: fetchMock })).toBeNull(); + expect(await titleByImdbId('nm0000001', { title: 'x' }, { fetch: fetchMock })).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('fetchTitlesPage', () => { + it('builds the id-ordered page query with after, since and tags', async () => { + const fetchMock = vi.fn(async (_input: string) => + jsonResponse({ count: 1, items: [imdbTitle()] }, { headers: { 'x-ratelimit-remaining': '498' } }), + ); + const res = await fetchTitlesPage({ after: 42, since: '2026-09-01T00:00:00.000Z', limit: 200, tags: ['imdb'] }, { fetch: fetchMock }); + expect(res?.status).toBe(200); + expect(res?.rateRemaining).toBe(498); + expect(res?.page.items).toHaveLength(1); + const url = new URL(fetchMock.mock.calls[0][0]); + expect(url.pathname).toBe('/api/v1/items'); + expect(Object.fromEntries(url.searchParams)).toEqual({ + collection: 'screen', kind: 'title', sort: 'id', order: 'asc', limit: '200', + after: '42', since: '2026-09-01T00:00:00.000Z', tags: 'imdb', + }); + }); + + it('omits after and since on a first page', async () => { + const fetchMock = vi.fn(async (_input: string) => jsonResponse({ count: 0, items: [] })); + await fetchTitlesPage({ after: 0, since: null }, { fetch: fetchMock }); + const url = new URL(fetchMock.mock.calls[0][0]); + expect(url.searchParams.has('after')).toBe(false); + expect(url.searchParams.has('since')).toBe(false); + }); + + it('reports the status on a 429 and null on a network failure', async () => { + const limited = await fetchTitlesPage({}, { fetch: vi.fn(async () => jsonResponse({ error: 'slow down' }, { status: 429, headers: { 'x-ratelimit-remaining': '0' } })) }); + expect(limited?.status).toBe(429); + expect(limited?.rateRemaining).toBe(0); + expect(await fetchTitlesPage({}, { fetch: vi.fn(async () => { throw new Error('reset'); }) })).toBeNull(); + }); +}); diff --git a/src/lib/nichedb/titles.ts b/src/lib/nichedb/titles.ts new file mode 100644 index 00000000..02e6e275 --- /dev/null +++ b/src/lib/nichedb/titles.ts @@ -0,0 +1,282 @@ +/** + * nichedb.dev `screen` collection: film / tv / anime titles. + * + * nichedb keeps one row per provider for a title, so a film usually has both an + * `imdb:title:` row (rating, votes, runtime, no poster, no summary) and a + * `tmdb:title:` row (poster, backdrop, summary, tagline, cast). Both carry + * `data.imdbId`, which is how the two are tied together here. + * + * Read API (anonymous, 600 requests/hour per IP): + * GET /api/v1/items?collection=screen&kind=title&sort=id&order=asc&after=&since=&limit=200 + * GET /api/v1/match?q=&collection=screen&kind=title[&year=] + * + * `match` cleans a release name itself (year, S02E03, quality tags, group) and + * answers items with a `score` in 0..1; an exact title is 1.0. + * + * Nothing in here touches the database. The switch that routes bittorrented's + * reads through nichedb is NICHEDB_TITLES=1 (see src/lib/imdb/tmdb.ts). + */ + +export const NICHEDB_BASE_URL = 'https://nichedb.dev'; + +/** Below this `match` score a candidate is not the title the name refers to. */ +export const MATCH_SCORE_FLOOR = 0.5; + +export const DEFAULT_PAGE_LIMIT = 200; + +/** An item of the `screen` collection with kind `title`, as it comes off the wire (0.6.1). */ +export interface NichedbTitleItem { + id: number; + collection: string; + kind: string; + external_id: string; + updated_at: string; + title: string; + summary: string | null; + url: string | null; + image_url: string | null; + published_at: string | null; + tags: string[]; + data: NichedbTitleData; + /** Only on `/api/v1/match` answers. */ + score?: number; +} + +export interface NichedbTitleData { + provider?: string; + category?: 'film' | 'tv' | 'anime' | string; + form?: 'movie' | 'series' | string; + year?: number | null; + endYear?: number | null; + normTitle?: string; + originalTitle?: string | null; + titleType?: string | null; + isAdult?: boolean | null; + imdbId?: string | null; + tmdbId?: string | number | null; + genres?: string[] | null; + rating?: number | null; + ratingCount?: number | null; + popularity?: number | null; + backdropUrl?: string | null; + tagline?: string | null; + trailerUrl?: string | null; + runtimeMin?: number | null; + watch?: string[] | null; + cast?: string[] | null; + [key: string]: unknown; +} + +export interface NichedbItemsPage { + count: number; + items: NichedbTitleItem[]; +} + +export interface NichedbMatchResponse { + q: string; + parsed: { name: string; year: number | null; season: number | null; episode: number | null; kind: string }; + count: number; + items: NichedbTitleItem[]; +} + +export type FetchLike = (input: string, init?: RequestInit) => Promise; + +export interface NichedbClientOptions { + baseUrl?: string; + fetch?: FetchLike; + /** Per-request timeout; the site calls this on a page render. */ + timeoutMs?: number; +} + +function resolveOptions(opts: NichedbClientOptions = {}) { + return { + baseUrl: (opts.baseUrl ?? process.env.NICHEDB_BASE_URL ?? NICHEDB_BASE_URL).replace(/\/$/, ''), + fetch: opts.fetch ?? ((input: string, init?: RequestInit) => fetch(input, init)), + timeoutMs: opts.timeoutMs ?? 8000, + }; +} + +async function getJson(url: string, opts: NichedbClientOptions): Promise { + const { fetch: doFetch, timeoutMs } = resolveOptions(opts); + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null; + try { + const res = await doFetch(url, { + headers: { accept: 'application/json', 'user-agent': 'bittorrented.com (nichedb mirror)' }, + signal: controller?.signal, + }); + if (!res.ok) return null; + return (await res.json()) as T; + } catch { + return null; + } finally { + if (timer) clearTimeout(timer); + } +} + +/** The IMDb tconst of an item: `data.imdbId`, else the `imdb:title:` external id. */ +export function imdbIdOf(item: Pick): string | null { + const fromData = item.data?.imdbId; + if (typeof fromData === 'string' && /^tt\d+$/.test(fromData)) return fromData; + const m = /^imdb:title:(tt\d+)$/.exec(item.external_id ?? ''); + return m ? m[1] : null; +} + +/** What a title page can show from one nichedb row, normalised. */ +export interface NichedbTitleFacts { + imdbId: string | null; + tmdbId: number | null; + title: string; + year: number | null; + form: 'movie' | 'series' | null; + category: 'film' | 'tv' | 'anime' | null; + genres: string[]; + rating: number | null; + ratingCount: number | null; + runtimeMin: number | null; + posterUrl: string | null; + backdropUrl: string | null; + overview: string | null; + tagline: string | null; + cast: string | null; +} + +export function titleFacts(item: NichedbTitleItem): NichedbTitleFacts { + const d = item.data ?? {}; + const tmdbId = d.tmdbId != null && d.tmdbId !== '' ? Number(d.tmdbId) : NaN; + const form = d.form === 'movie' || d.form === 'series' ? d.form : null; + const category = d.category === 'film' || d.category === 'tv' || d.category === 'anime' ? d.category : null; + const cast = Array.isArray(d.cast) ? d.cast.filter((c): c is string => typeof c === 'string' && c.length > 0).slice(0, 8) : []; + return { + imdbId: imdbIdOf(item), + tmdbId: Number.isFinite(tmdbId) && tmdbId > 0 ? tmdbId : null, + title: item.title, + year: typeof d.year === 'number' ? d.year : null, + form, + category, + genres: Array.isArray(d.genres) ? d.genres.filter((g): g is string => typeof g === 'string') : [], + rating: typeof d.rating === 'number' ? d.rating : null, + ratingCount: typeof d.ratingCount === 'number' ? d.ratingCount : null, + runtimeMin: typeof d.runtimeMin === 'number' ? d.runtimeMin : null, + posterUrl: item.image_url || null, + backdropUrl: (typeof d.backdropUrl === 'string' && d.backdropUrl) || null, + overview: item.summary || null, + tagline: (typeof d.tagline === 'string' && d.tagline) || null, + cast: cast.length ? cast.join(', ') : null, + }; +} + +/** Does the item carry anything a torrent page would show beyond what imdb_* already has? */ +export function hasPresentation(item: NichedbTitleItem): boolean { + return Boolean(item.image_url || item.summary || item.data?.backdropUrl); +} + +/** + * Pick the item a `match` answer refers to. + * + * Exact titles (score 1.0) come back once per provider; only the TMDB row has a + * poster, so among the best-scoring candidates the one with an image wins. Anything + * under the floor is not a match at all. + * + * When `imdbId` is given the pick is restricted to rows that carry that tconst, + * whatever their score: the caller already knows which title it wants and only + * needs nichedb's presentation for it. + */ +export function pickMatch( + items: NichedbTitleItem[], + opts: { imdbId?: string | null; floor?: number } = {}, +): NichedbTitleItem | null { + const floor = opts.floor ?? MATCH_SCORE_FLOOR; + let pool = items; + if (opts.imdbId) { + pool = items.filter((i) => imdbIdOf(i) === opts.imdbId); + } else { + pool = items.filter((i) => (i.score ?? 0) >= floor); + } + if (pool.length === 0) return null; + + const best = Math.max(...pool.map((i) => i.score ?? 0)); + const top = pool.filter((i) => (i.score ?? 0) === best); + return top.find((i) => Boolean(i.image_url)) ?? top.find(hasPresentation) ?? top[0]; +} + +/** Raw `match` candidates for a release name, unranked beyond what nichedb did. */ +export async function matchCandidates( + name: string, + opts: { year?: number | null; limit?: number } & NichedbClientOptions = {}, +): Promise { + const q = name.trim(); + if (!q) return []; + const { baseUrl } = resolveOptions(opts); + const params = new URLSearchParams({ q, collection: 'screen', kind: 'title' }); + if (opts.year) params.set('year', String(opts.year)); + if (opts.limit) params.set('limit', String(opts.limit)); + const res = await getJson(`${baseUrl}/api/v1/match?${params}`, opts); + return res?.items ?? []; +} + +/** + * The title a release name refers to, or null when nichedb has no candidate at or + * above the score floor. Prefers the row with a poster among exact titles. + */ +export async function matchTitle( + name: string, + opts: { year?: number | null } & NichedbClientOptions = {}, +): Promise { + const items = await matchCandidates(name, opts); + return pickMatch(items); +} + +/** + * The nichedb row for a known IMDb id. + * + * nichedb has no per-id tag (`tags=imdb:title:` answers nothing) and the + * items endpoint does not filter on external_id, so this goes through `match` on + * the title we already know and keeps only rows carrying that tconst. Without a + * title there is nothing to ask, and it answers null. + */ +export async function titleByImdbId( + tconst: string, + hint: { title?: string | null; year?: number | null } = {}, + opts: NichedbClientOptions = {}, +): Promise { + if (!/^tt\d+$/.test(tconst) || !hint.title?.trim()) return null; + const items = await matchCandidates(hint.title, { ...opts, year: hint.year ?? null, limit: 20 }); + return pickMatch(items, { imdbId: tconst }); +} + +/** One id-ordered page of `screen` titles; `after` is the last id already seen. */ +export async function fetchTitlesPage( + params: { after?: number; since?: string | null; limit?: number; tags?: string[] }, + opts: NichedbClientOptions = {}, +): Promise<{ page: NichedbItemsPage; rateRemaining: number | null; status: number } | null> { + const { baseUrl, fetch: doFetch, timeoutMs } = resolveOptions(opts); + const qs = new URLSearchParams({ + collection: 'screen', + kind: 'title', + sort: 'id', + order: 'asc', + limit: String(params.limit ?? DEFAULT_PAGE_LIMIT), + }); + if (params.after) qs.set('after', String(params.after)); + if (params.since) qs.set('since', params.since); + if (params.tags?.length) qs.set('tags', params.tags.join(',')); + + const controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + const timer = controller ? setTimeout(() => controller.abort(), Math.max(timeoutMs, 30000)) : null; + try { + const res = await doFetch(`${baseUrl}/api/v1/items?${qs}`, { + headers: { accept: 'application/json', 'user-agent': 'bittorrented.com (nichedb mirror)' }, + signal: controller?.signal, + }); + const remaining = res.headers?.get?.('x-ratelimit-remaining'); + const rateRemaining = remaining != null && remaining !== '' ? Number(remaining) : null; + if (!res.ok) return { page: { count: 0, items: [] }, rateRemaining, status: res.status }; + const page = (await res.json()) as NichedbItemsPage; + return { page: { count: page.count ?? page.items?.length ?? 0, items: page.items ?? [] }, rateRemaining, status: res.status }; + } catch { + return null; + } finally { + if (timer) clearTimeout(timer); + } +} diff --git a/supabase/migrations/20260910150000_nichedb_mirror_cursor.sql b/supabase/migrations/20260910150000_nichedb_mirror_cursor.sql new file mode 100644 index 00000000..4077d766 --- /dev/null +++ b/supabase/migrations/20260910150000_nichedb_mirror_cursor.sql @@ -0,0 +1,30 @@ +-- Cursor for the nichedb -> imdb_* mirror (scripts/mirror-imdb-from-nichedb.ts). +-- +-- bittorrented used to download the IMDb daily dumps (title.basics, title.ratings, +-- ...) every night and COPY them into the imdb_* tables. nichedb.dev's `screen` +-- collection already carries every IMDb title with a rating, so the mirror walks +-- that collection over its public API instead and upserts imdb_title_basics and +-- imdb_title_ratings. The walk is id-ordered (`sort=id&order=asc&after=`) and +-- capped per run to stay inside nichedb's 600 requests/hour, so it has to remember +-- where it stopped: that is this table. +-- +-- after_id last item id seen on the current walk (0 = start from the beginning) +-- since `since=` sent with every page of the current walk (NULL on the first, +-- full backfill); set to the walk's own start time once it completes, so +-- the next walk only asks for items updated after that +-- +-- One row per mirror name; the IMDb mirror uses 'imdb-titles'. + +CREATE TABLE IF NOT EXISTS nichedb_mirror_cursor ( + name TEXT PRIMARY KEY, + after_id BIGINT NOT NULL DEFAULT 0, + since TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE nichedb_mirror_cursor IS + 'Resume point for scripts/mirror-imdb-from-nichedb.ts: last nichedb item id seen on the current walk and the since= watermark it was started with.'; + +ALTER TABLE nichedb_mirror_cursor ENABLE ROW LEVEL SECURITY; +-- No policies on purpose: only the service role (which bypasses RLS) reads or +-- writes the cursor.