The IMDb tables and a torrent's title can come from nichedb

.env.example

@@ -44,6 +44,14 @@ FANART_TV_API_KEY=your-fanart-api-key
 TMDB_API_KEY=your-tmdb-api-key
 MUSICBRAINZ_USER_AGENT=BitTorrented/1.0.0 (https://bittorrented.com)
 
+# nichedb.dev `screen` collection as the title source (default off for both)
+# NICHEDB_TITLES=1   torrent pages ask nichedb for poster/backdrop/overview before TMDB
+# NICHEDB_MIRROR=1   setup-server.sh schedules scripts/mirror-imdb-from-nichedb.sh at 00:00
+#                    instead of the IMDb dump download (pnpm mirror:imdb)
+# NICHEDB_MIRROR_MAX_PAGES=480  NICHEDB_MIRROR_INTERVAL_MS=7000  NICHEDB_MIRROR_TAGS=
+#NICHEDB_TITLES=1
+#NICHEDB_MIRROR=1
+
 # CoinPayPortal (Crypto Payments)
 # Get these from your CoinPayPortal dashboard: https://coinpayportal.com/dashboard
 COINPAYPORTAL_API_KEY=cp_live_your_api_key

package.json

@@ -26,6 +26,7 @@
     "enrich-folders:all": "tsx --env-file=.env scripts/enrich-folder-metadata.ts --all-status",
     "enrich-folders:dry-run": "tsx --env-file=.env scripts/enrich-folder-metadata.ts --all-status --dry-run",
     "enrich-folders:force": "tsx --env-file=.env scripts/enrich-folder-metadata.ts --all-status --force",
+    "mirror:imdb": "tsx --env-file=.env scripts/mirror-imdb-from-nichedb.ts",
     "iptv-worker": "tsx --env-file=.env workers/iptv-cache/index.ts",
     "iptv-worker:dev": "tsx --watch --env-file=.env workers/iptv-cache/index.ts",
     "podcast-worker": "tsx --env-file=.env workers/podcast-notifier/index.ts",

scripts/mirror-imdb-from-nichedb.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Nightly imdb_* refresh from nichedb.dev, in place of the IMDb dump download.
+#
+# Runs scripts/mirror-imdb-from-nichedb.ts (pnpm mirror:imdb) with a lock so two
+# cron runs never overlap. One run walks at most NICHEDB_MIRROR_MAX_PAGES pages
+# (default 480, ~56 minutes at 7 s a page) and stores its cursor, so the first
+# backfill spreads over a few nights unless kicked off by hand with
+#   pnpm mirror:imdb -- --all
+#
+# crontab (set up by setup-server.sh when NICHEDB_MIRROR=1):
+# 0 0 * * * /home/ubuntu/src/media-streamer/scripts/mirror-imdb-from-nichedb.sh >> /var/log/imdb-update.log 2>&1
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
+LOCK_FILE="/tmp/imdb-update.lock"
+
+if [ -f "$LOCK_FILE" ]; then
+  pid=$(cat "$LOCK_FILE")
+  if kill -0 "$pid" 2>/dev/null; then
+    echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Another IMDb update is running (pid $pid), skipping"
+    exit 0
+  fi
+fi
+echo $$ > "$LOCK_FILE"
+trap 'rm -f "$LOCK_FILE"' EXIT
+
+# cron's PATH has no pnpm; pick up the user's toolchain.
+export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$PATH"
+
+cd "$PROJECT_DIR"
+echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] Starting nichedb -> imdb_* mirror"
+pnpm mirror:imdb "$@"
+echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] nichedb mirror finished"

scripts/mirror-imdb-from-nichedb.ts

@@ -0,0 +1,180 @@
+#!/usr/bin/env npx tsx
+/**
+ * Mirror IMDb titles from nichedb.dev into imdb_title_basics / imdb_title_ratings.
+ *
+ * Replaces the nightly IMDb dump download (scripts/update-imdb-daily.sh) for the
+ * two tables the site reads on every torrent page. It walks nichedb's `screen`
+ * collection over the public API, id-ordered, with a cursor kept in
+ * nichedb_mirror_cursor so a run can stop at its page cap and the next run
+ * carries on. Once a walk reaches the end, later walks send `since=` and only see
+ * titles nichedb updated after the previous walk began.
+ *
+ * Usage:
+ *   pnpm mirror:imdb                  one capped run (cron)
+ *   pnpm mirror:imdb -- --all         run until the walk completes (first backfill, ~3-4 h)
+ *   pnpm mirror:imdb -- --max-pages=N --interval-ms=MS --dry-run --reset
+ *
+ * Options:
+ *   --all             No page cap for this run
+ *   --max-pages=N     Pages per run (default NICHEDB_MIRROR_MAX_PAGES or 480)
+ *   --interval-ms=MS  Pause between pages (default NICHEDB_MIRROR_INTERVAL_MS or 7000).
+ *                     nichedb allows 600 requests/hour per IP; 7 s is ~514/h, leaving
+ *                     the rest for the site's own match calls from the same box.
+ *   --tags=a,b        Only walk items carrying all these tags (default: none, all titles;
+ *                     `imdb` walks the IMDb-sourced rows only)
+ *   --reset           Forget the cursor and start a full backfill
+ *   --dry-run         Fetch and map, write nothing (cursor included)
+ *
+ * Required environment variables:
+ *   SUPABASE_URL (or NEXT_PUBLIC_SUPABASE_URL), SUPABASE_SERVICE_ROLE_KEY
+ * Optional:
+ *   NICHEDB_BASE_URL (default https://nichedb.dev)
+ */
+
+import { config } from 'dotenv';
+import { createClient } from '@supabase/supabase-js';
+
+config();
+
+import {
+  MIRROR_CURSOR_NAME,
+  runMirror,
+  type MirrorCursor,
+  type MirrorRows,
+  type MirrorStore,
+} from '../src/lib/nichedb/mirror';
+
+interface Options {
+  maxPages: number;
+  intervalMs: number;
+  tags: string[];
+  reset: boolean;
+  dryRun: boolean;
+}
+
+function parseArgs(): Options {
+  const opts: Options = {
+    maxPages: Number(process.env.NICHEDB_MIRROR_MAX_PAGES) || 480,
+    intervalMs: Number(process.env.NICHEDB_MIRROR_INTERVAL_MS) || 7000,
+    tags: (process.env.NICHEDB_MIRROR_TAGS ?? '').split(',').filter(Boolean),
+    reset: false,
+    dryRun: false,
+  };
+  for (const arg of process.argv.slice(2)) {
+    if (arg === '--all') opts.maxPages = 0;
+    else if (arg.startsWith('--max-pages=')) opts.maxPages = Number(arg.slice('--max-pages='.length)) || 0;
+    else if (arg.startsWith('--interval-ms=')) opts.intervalMs = Number(arg.slice('--interval-ms='.length)) || 0;
+    else if (arg.startsWith('--tags=')) opts.tags = arg.slice('--tags='.length).split(',').filter(Boolean);
+    else if (arg === '--reset') opts.reset = true;
+    else if (arg === '--dry-run') opts.dryRun = true;
+    else if (arg === '--help' || arg === '-h') {
+      console.log('Usage: pnpm mirror:imdb -- [--all] [--max-pages=N] [--interval-ms=MS] [--tags=a,b] [--reset] [--dry-run]');
+      process.exit(0);
+    } else {
+      console.error(`Unknown option: ${arg}`);
+      process.exit(1);
+    }
+  }
+  return opts;
+}
+
+function stamp(): string {
+  return new Date().toISOString();
+}
+
+/** --dry-run without Supabase credentials: walk from the start, keep nothing. */
+function memoryStore(): MirrorStore {
+  let cursor: MirrorCursor | null = null;
+  return {
+    async loadCursor() { return cursor; },
+    async saveCursor(_name, c) { cursor = { ...c }; },
+    async writeRows() {},
+  };
+}
+
+function makeStore(dryRun: boolean): MirrorStore {
+  const url = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL;
+  const key = process.env.SUPABASE_SERVICE_ROLE_KEY;
+  if (!url || !key) {
+    if (dryRun) {
+      console.log('no SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY: dry run with an in-memory cursor');
+      return memoryStore();
+    }
+    console.error('ERROR: SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY are required');
+    process.exit(1);
+  }
+  const supabase = createClient(url, key, { auth: { persistSession: false } });
+
+  return {
+    async loadCursor(name: string): Promise<MirrorCursor | null> {
+      const { data, error } = await supabase
+        .from('nichedb_mirror_cursor')
+        .select('after_id, since')
+        .eq('name', name)
+        .maybeSingle();
+      if (error) throw new Error(`cursor read failed: ${error.message}`);
+      if (!data) return null;
+      return { after_id: Number(data.after_id) || 0, since: data.since ?? null };
+    },
+    async saveCursor(name: string, cursor: MirrorCursor): Promise<void> {
+      if (dryRun) return;
+      const { error } = await supabase
+        .from('nichedb_mirror_cursor')
+        .upsert(
+          { name, after_id: cursor.after_id, since: cursor.since, updated_at: new Date().toISOString() },
+          { onConflict: 'name' },
+        );
+      if (error) throw new Error(`cursor write failed: ${error.message}`);
+    },
+    async writeRows(rows: MirrorRows): Promise<void> {
+      if (dryRun) return;
+      if (rows.basics.length) {
+        const { error } = await supabase
+          .from('imdb_title_basics')
+          .upsert(rows.basics, { onConflict: 'tconst' });
+        if (error) throw new Error(`imdb_title_basics upsert failed: ${error.message}`);
+      }
+      if (rows.basicsFill.length) {
+        // Rows from TMDB/TVmaze that know a tconst: fill gaps, never overwrite IMDb's own row.
+        const { error } = await supabase
+          .from('imdb_title_basics')
+          .upsert(rows.basicsFill, { onConflict: 'tconst', ignoreDuplicates: true });
+        if (error) throw new Error(`imdb_title_basics fill failed: ${error.message}`);
+      }
+      if (rows.ratings.length) {
+        const { error } = await supabase
+          .from('imdb_title_ratings')
+          .upsert(rows.ratings, { onConflict: 'tconst' });
+        if (error) throw new Error(`imdb_title_ratings upsert failed: ${error.message}`);
+      }
+    },
+  };
+}
+
+async function main(): Promise<void> {
+  const opts = parseArgs();
+  const store = makeStore(opts.dryRun);
+
+  console.log(`[${stamp()}] nichedb -> imdb_* mirror starting (maxPages=${opts.maxPages || 'unlimited'}, interval=${opts.intervalMs}ms, tags=${opts.tags.join(',') || 'none'}${opts.dryRun ? ', DRY RUN' : ''})`);
+
+  if (opts.reset) {
+    console.log(`[${stamp()}] resetting cursor ${MIRROR_CURSOR_NAME}`);
+    await store.saveCursor(MIRROR_CURSOR_NAME, { after_id: 0, since: null });
+  }
+
+  const result = await runMirror({
+    store,
+    maxPages: opts.maxPages,
+    intervalMs: opts.intervalMs,
+    tags: opts.tags.length ? opts.tags : undefined,
+    log: (line) => console.log(`[${stamp()}] ${line}`),
+  });
+
+  console.log(`[${stamp()}] done: ${result.pages} pages, ${result.items} items, ${result.basics} basics upserted, ${result.basicsFill} basics filled, ${result.ratings} ratings upserted; ${result.stoppedBecause}; cursor after_id=${result.cursor.after_id} since=${result.cursor.since ?? 'none'}`);
+  if (!result.completed && result.stoppedBecause !== 'page-cap') process.exitCode = 1;
+}
+
+main().catch((err) => {
+  console.error(`[${stamp()}] FAILED:`, err instanceof Error ? err.message : err);
+  process.exit(1);
+});

scripts/setup-server.sh

@@ -1032,19 +1032,42 @@ sudo systemctl enable ${IPTV_WORKER_SERVICE} 2>/dev/null || true
 sudo systemctl enable ${PODCAST_WORKER_SERVICE} 2>/dev/null || true
 
 # Set up IMDB dataset daily update cron job (runs at midnight UTC)
+#
+# Two sources for imdb_title_basics / imdb_title_ratings:
+#   default        update-imdb-daily.sh downloads the IMDb dumps and COPYs all seven tables
+#   NICHEDB_MIRROR=1  mirror-imdb-from-nichedb.sh walks nichedb.dev's `screen` collection
+#                     over its API (basics + ratings only; crew/episode/akas/principals/
+#                     name.basics are not refreshed by it)
+# Switching flips the 00:00 crontab line; the other script stays in the repo.
 echo "=== Setting up IMDB dataset daily update cron job ==="
-IMDB_UPDATE_SCRIPT="${PROJECT_ROOT}/scripts/update-imdb-daily.sh"
+if [ "${NICHEDB_MIRROR:-}" = "1" ]; then
+    IMDB_UPDATE_SCRIPT="${PROJECT_ROOT}/scripts/mirror-imdb-from-nichedb.sh"
+    IMDB_UPDATE_MARK="mirror-imdb-from-nichedb"
+    IMDB_UPDATE_OTHER="update-imdb-daily"
+    IMDB_UPDATE_LABEL="nichedb -> imdb_* mirror"
+else
+    IMDB_UPDATE_SCRIPT="${PROJECT_ROOT}/scripts/update-imdb-daily.sh"
+    IMDB_UPDATE_MARK="update-imdb-daily"
+    IMDB_UPDATE_OTHER="mirror-imdb-from-nichedb"
+    IMDB_UPDATE_LABEL="IMDB dataset update"
+fi
 IMDB_CRON_JOB="0 0 * * * ${IMDB_UPDATE_SCRIPT} >> /var/log/imdb-update.log 2>&1"
 if [ -f "${IMDB_UPDATE_SCRIPT}" ]; then
     chmod +x "${IMDB_UPDATE_SCRIPT}"
     chmod +x "${PROJECT_ROOT}/scripts/import-imdb.sh" 2>/dev/null || true
+    chmod +x "${PROJECT_ROOT}/scripts/mirror-imdb-from-nichedb.sh" 2>/dev/null || true
     sudo touch /var/log/imdb-update.log
     sudo chown ${VPS_USER}:${VPS_USER} /var/log/imdb-update.log
-    if crontab -l 2>/dev/null | grep -q "update-imdb-daily"; then
-        echo "  IMDB update cron job already exists"
+    if crontab -l 2>/dev/null | grep -q "${IMDB_UPDATE_OTHER}"; then
+        # The other source is scheduled; replace it so only one job runs at 00:00.
+        (crontab -l 2>/dev/null | grep -v "${IMDB_UPDATE_OTHER}" || true) | crontab -
+        echo "  Removed the ${IMDB_UPDATE_OTHER} cron job"
+    fi
+    if crontab -l 2>/dev/null | grep -q "${IMDB_UPDATE_MARK}"; then
+        echo "  ${IMDB_UPDATE_LABEL} cron job already exists"
     else
         (crontab -l 2>/dev/null || true; echo "${IMDB_CRON_JOB}") | crontab -
-        echo "✓ Added cron job: IMDB dataset update at midnight daily"
+        echo "✓ Added cron job: ${IMDB_UPDATE_LABEL} at midnight daily"
     fi
 else
     echo "  WARNING: ${IMDB_UPDATE_SCRIPT} not found, skipping IMDB cron setup"
@@ -1139,13 +1162,23 @@ echo "  Errors:  tail -f ${PODCAST_WORKER_ERROR_LOG}"
 echo ""
 echo "Scheduled Tasks:"
 echo "  WebTorrent temp cleanup: Daily at midnight"
+if [ "${NICHEDB_MIRROR:-}" = "1" ]; then
+echo "  IMDB titles: Daily at midnight, mirrored from nichedb.dev (basics + ratings)"
+else
 echo "  IMDB dataset update: Daily at midnight (incremental)"
+fi
 echo "  Directory: ${WEBTORRENT_TMP_DIR}"
 echo "  View cron jobs: crontab -l"
 echo ""
 echo "IMDB Datasets:"
+if [ "${NICHEDB_MIRROR:-}" = "1" ]; then
+echo "  First backfill: pnpm mirror:imdb -- --all   (~3-4 h at nichedb's 600 req/h)"
+echo "  Daily update: ./scripts/mirror-imdb-from-nichedb.sh (automatic via cron)"
+echo "  Note: crew/episode/akas/principals/name tables are not refreshed by the mirror"
+else
 echo "  First import: ./scripts/import-imdb.sh ~/tmp/data"
 echo "  Daily update: ./scripts/update-imdb-daily.sh (automatic via cron)"
+fi
 echo "  Logs: /var/log/imdb-update.log"
 
 # DHT Services output (only if enabled)