diff --git a/README.md b/README.md
index 0b1a253..481b0f6 100644
--- a/README.md
+++ b/README.md
@@ -295,3 +295,7 @@ The rules:
## License
MIT
+
+### Geographic filtering and scanner context
+
+Use `?lat=41.88&long=-87.62&radius=10` across located collections, searches and feeds. [Geographic queries](docs/geographic-queries.md) documents the API, CLI/MCP flags, permissioned scanner catalogs, coverage semantics and migration.
diff --git a/apps/cli/src/index.js b/apps/cli/src/index.js
index fe3c150..655353f 100644
--- a/apps/cli/src/index.js
+++ b/apps/cli/src/index.js
@@ -16,6 +16,7 @@ import { join } from 'node:path';
import { createInterface } from 'node:readline';
export const VERSION = '0.26.0';
+const GEO_FLAGS = ['lat', 'long', 'radius', 'unit', 'bbox', 'sort', 'offset'];
const DEFAULT_API = process.env.NICHEDB_API ?? 'https://nichedb.dev';
const CONFIG_DIR = join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'nichedb');
const CONFIG_FILE = join(CONFIG_DIR, 'config.json');
@@ -94,7 +95,7 @@ export const COMMANDS = [
{
name: 'recent',
usage:
- 'recent [--collection …] [--source …] [--kind …] [--tags a,b] [--from …] [--to …] [--since …] [--sort id|published|updated]',
+ 'recent [--collection …] [--source …] [--kind …] [--tags a,b] [--from …] [--to …] [--since …] [--sort id|published|updated|distance]',
summary:
'Newest items across a collection or source, or a window of them, or what changed since.',
options: [
@@ -104,8 +105,11 @@ export const COMMANDS = [
'--tags a,b (every one must be on the item)',
'--from / --to (ISO, on published_at)',
'--since (ISO, on updated_at)',
- '--sort id|published|updated',
+ '--sort id|published|updated|distance',
'--order asc|desc',
+ '--lat / --long / --radius / --unit km|mi',
+ '--bbox west,south,east,north',
+ '--offset (distance pagination)',
'--limit',
'--json',
'--urls',
@@ -597,6 +601,9 @@ export async function run(
tags: csv(flags.tags),
q: flags.q,
upcoming: Boolean(flags.upcoming),
+ ...Object.fromEntries(
+ GEO_FLAGS.filter((k) => flags[k] !== undefined).map((k) => [k, flags[k]]),
+ ),
public: !flags.private,
});
out(
@@ -618,6 +625,7 @@ export async function run(
const [slug] = rest;
if (!slug) throw new Error('items
{feed.description}
: null}
{describeQuery(query)} · {feed.follower_count} following ·{' '}
- RSS · JSON ·{' '}
+ RSS ·{' '}
+ JSON ·{' '}
API
{canEdit ? (
<>
@@ -288,12 +318,20 @@ export const FeedPage = ({
) : null}
+ Geographic context, not verified links to these radio transmissions. Source locations
+ may be approximate or anonymised.
+
+ Coverage: {item.data?.coverage_basis ?? 'unknown'}. Most recent available reports; an
+ empty list does not mean no crime.
+ Reported incidents in scanner coverage
+
{JSON.stringify(item.data, null, 2)}
diff --git a/docs/geographic-queries.md b/docs/geographic-queries.md new file mode 100644 index 0000000..0f722c1 --- /dev/null +++ b/docs/geographic-queries.md @@ -0,0 +1,173 @@ +# Geographic queries and scanner context + +Location filtering is shared by collections, item search, matches, upcoming +items and saved feeds. It is opt-in: requests without geographic parameters +retain their existing results and ordering. + +```http +GET /api/v1/items?collection=crime&lat=41.88&long=-87.62&radius=10&unit=km +GET /api/v1/items?lat=41.88&long=-87.62&sort=distance&limit=50&offset=50 +GET /api/v1/search?q=flood&lat=41.88&long=-87.62&radius=100 +GET /api/v1/items?bbox=-88,41,-87,42 +GET /api/v1/feeds/my-local-feed/items?lat=41.88&long=-87.62&radius=2 +GET /crimes?lat=41.88&long=-87.62 +GET /scanners?lat=41.88&long=-87.62 +``` + +`/crimes` redirects to the existing `/c/crime` collection. `/scanners` +redirects there with `kind=scanner-stream`. Both preserve GPS parameters. +All `/c/:slug` pages and `/f/:slug` pages, RSS and JSON feeds support the same +filters. Page caches and pagination links include the location parameters. + +| Parameter | Contract | +| --- | --- | +| `lat`, `long` | Both required together. Finite decimal numbers, latitude −90…90 and longitude −180…180. Zero is valid. | +| `radius` | Default 10 in the selected unit; positive, at least 0.001 units, at most 1000 km equivalent. | +| `unit` | `km` (default) or `mi`. One mile is 1609.344 metres. | +| `bbox` | `west,south,east,north`. Alternative to a center/radius; west greater than east crosses the antimeridian. | +| `sort=distance` | Requires a center. Nearest first regardless of `order`, with existing tie-breakers. | +| `offset` | For distance-sorted item and feed pages, default 0, capped at 10000. Do not combine distance sorting with `before`/`after`. | + +Invalid combinations return HTTP 400 (MCP reports a tool error). Records with +unknown or invalid locations are excluded from geographic reads. Text-only +city/state metadata is preserved, but is not guessed into coordinates. The API +includes `distance_m` for radius queries; it is absent for unfiltered/bbox reads. +Distances are to a point or to the nearest coverage boundary (zero inside). + +Filtering happens in PostgreSQL **before** result limits. A GiST expression +index on a geographic bounding box narrows radius candidates; a spherical +point/circle distance then applies the radius. Polygon containment honours +holes and supports local polygons crossing the antimeridian. Polygon edge +distances are approximate: longitude/latitude segments are subdivided at +0.1-degree intervals and projected locally before measuring spherical distance. +These are geographic discovery distances, not surveyed distances. Use local +jurisdiction polygons rather than polygons enclosing a pole or most of Earth. + +For coverage shapes, `bbox` means **bounding-envelope overlap**, not exact +polygon intersection. This deliberately includes candidates whose polygon has +a hole or concavity within the viewport. Antimeridian-crossing coverage envelopes +are widened to all longitudes; radius queries refine them with distance. + +## Supported stored metadata + +No re-ingestion is needed. The database recognizes coordinate pairs in +`data.location`, `data.place`, `data.position`, or `data` itself: + +- `lat` or `latitude` paired with `long`, `lon`, `lng` or `longitude`. +- GeoJSON `Point`, `Polygon`, or `MultiPolygon` in those objects or `data.geometry`. +- Explicit `data.coverage`, which takes precedence over every point, including + receiver coordinates. Malformed explicit coverage never falls back to a point. + +GeoJSON coordinates use **[longitude, latitude]**. Approximate circular coverage +uses this extension: + +```json +{"type":"Circle","coordinates":[-87.62,41.88],"radius_m":15000} +``` + +The original payload remains intact, including upstream source, timestamps, +anonymisation notes and precision. Coordinates published as strings are accepted; +empty strings, nulls, nonnumeric values and out-of-range coordinates are unknown. +Adapter-specific sentinels (for example Seattle's −1/−1) remain the adapter's +responsibility because those can be legitimate coordinates elsewhere. + +## Saved feeds, CLI and MCP + +Create a geographic feed by supplying `lat`, `long`, `radius`, `unit`, `bbox` +and optional `sort` alongside existing filters in a feed's `query` (or the +existing flat API body). URL geographic filters **intersect** the saved scope; +they do not replace it. Delivery scanning always walks ascending item IDs, +even when the displayed feed is distance-sorted, so notifications cannot skip +rows by advancing a cursor in distance order. Web editing preserves the saved +geographic fields. + +```sh +nichedb recent --collection crime --lat 41.88 --long -87.62 --radius 10 --sort distance --json +nichedb search theft --lat 41.88 --long -87.62 --radius 5 --json +nichedb feed create --collection crime --name 'Local reports' --lat 41.88 --long -87.62 --radius 5 +``` + +The `items`, `recent`, `search`, `match`, and `upcoming` CLI commands forward +geographic flags. The corresponding MCP tools expose the same fields; +`create_feed` also saves them. Offset pagination is for a live view and is not +a snapshot: concurrent arrivals or location changes can shift pages. Use the +existing ID/update cursors for synchronization with the default ordering. + +## Permissioned scanner catalogs + +Add a `scanner-directory` source in the `crime` collection with `config.url` +pointing to a JSON catalog you have permission to index. There is no automatic +Broadcastify/OpenMHz scrape, API subscription or background audio download. +The catalog can be an array or `{ "feeds": [...] }`, at most 10000 entries. +Each row needs `id`, `name`, `provider`, `access_terms`, and `player_url`. + +```json +{ + "feeds": [{ + "id": "community-dispatch", + "name": "Community dispatch", + "provider": "Example volunteer operator", + "agency": "Example public safety dispatch", + "jurisdiction": "Example county", + "service_type": "public-safety", + "player_url": "https://scanner.example.org/listen", + "access_terms": "Operator permission to index player and coverage metadata", + "stream_reuse_allowed": false, + "coverage": { + "type": "Circle", + "coordinates": [-87.62, 41.88], + "radius_m": 15000 + }, + "coverage_basis": "approximate-radius", + "location_precision": "approximate service area", + "location_source": "operator", + "updated_at": "2026-09-13T00:00:00Z", + "last_checked": "2026-09-13T00:00:00Z" + }] +} +``` + +`stream_url` is retained only if the operator explicitly declares +`stream_reuse_allowed: true`; this declaration must reflect actual permission. +Otherwise only the player link is published. An authorized stream has a +user-operated HTML audio player with `preload="none"`. `last_checked` is the +catalog's observation, not a fabricated claim that NicheDB tested the audio. +Invalid/absent coverage stays unknown. Polygon rings must close, and an entry +may contain at most 2000 polygon vertices. Coverage/metadata source fields +remain separate from the scanner's audio provider. + +## Crime context + +```http +GET /api/v1/items/123/nearby-crime?from=2026-08-01&to=2026-09-01 +GET /api/v1/items/123/nearby-crime?lat=41.88&long=-87.62&radius=2 +``` + +The item must be a `scanner-stream`. This returns recent `crime-report` items +whose points fall inside its coverage, optionally intersected with a GPS search +and a half-open `published_at` window (`from` inclusive, `to` exclusive). +State-level `crime-estimate` rows are not individual incidents and are excluded. +Unknown coverage returns an empty list. A polygon's holes are excluded. +Restricted early-access crime collections are excluded from this context. +The response carries `relationship: "within-coverage"`, coverage basis, +source-linked items, timestamps and original location-precision metadata. +Scanner item pages show this context with date filters. + +This is a spatial association, not a link between an incident and a radio call. +Historical/month-precision reports retain their original dates; the enrichment +does not label them live. Empty results can mean unknown coverage, absent source +data or a date window with no indexed reports, not absence of crime. + +## Deployment and verification + +Migration `0023_geographic_queries.sql` uses standard PostgreSQL functions and +GiST, with no PostGIS extension or container change. The expression index is +built over existing items in the migration transaction. On a large production +table, schedule its initial build for a maintenance window: normal `CREATE INDEX` +blocks writes while building. Measure its duration on a production-sized copy. +Malformed coordinate fields cannot abort the migration or later ingestion. + +Tests execute the actual query functions against PGlite/PostgreSQL, including +migration application, index eligibility, mixed collections, radius boundaries, +unit conversion, antimeridian/polar points, polygon holes, saved-feed intersection, +notification cursor order, pagination and scanner context. diff --git a/packages/adapters/src/index.js b/packages/adapters/src/index.js index 42f2c32..308d0cc 100644 --- a/packages/adapters/src/index.js +++ b/packages/adapters/src/index.js @@ -97,6 +97,7 @@ import { rogueIncidents, rogueResearch } from './rogueaitracker.js'; import { rssamplifier } from './rssamplifier.js'; import { saasrow } from './saasrow.js'; import { scalewayInstances } from './scaleway.js'; +import { scannerDirectory } from './scanners.js'; import { scryfallCards, scryfallSets } from './scryfall.js'; import { slickdeals } from './slickdeals.js'; import { socrataCrime } from './socratacrime.js'; @@ -161,6 +162,7 @@ export const ADAPTERS = [ nasdaqHalts, ecbFxRates, socrataCrime, + scannerDirectory, ukPoliceCrime, fbiCrimeEstimates, usaspendingAwards, diff --git a/packages/adapters/src/scanners.js b/packages/adapters/src/scanners.js new file mode 100644 index 0000000..e48c677 --- /dev/null +++ b/packages/adapters/src/scanners.js @@ -0,0 +1,112 @@ +import { defineAdapter } from '@nichedb/core/adapter'; + +function publicUrl(value) { + try { + const u = new URL(value); + return ['http:', 'https:'].includes(u.protocol) && !u.username && !u.password ? u.href : null; + } catch { + return null; + } +} +const point = (p) => + Array.isArray(p) && + p.length >= 2 && + typeof p[0] === 'number' && + typeof p[1] === 'number' && + Number.isFinite(p[0]) && + Number.isFinite(p[1]) && + Math.abs(p[0]) <= 180 && + Math.abs(p[1]) <= 90; +export function validCoverage(g) { + if (!g || typeof g !== 'object') return false; + if (g.type === 'Circle') + return ( + point(g.coordinates) && Number.isFinite(g.radius_m) && g.radius_m > 0 && g.radius_m <= 1000000 + ); + const polys = + g.type === 'Polygon' ? [g.coordinates] : g.type === 'MultiPolygon' ? g.coordinates : null; + let vertices = 0; + return ( + Array.isArray(polys) && + polys.length > 0 && + polys.length <= 100 && + polys.every( + (poly) => + Array.isArray(poly) && + poly.length > 0 && + poly.every((ring) => { + if (!Array.isArray(ring) || ring.length < 4 || !ring.every(point)) return false; + vertices += ring.length; + return vertices <= 2000 && ring[0][0] === ring.at(-1)[0] && ring[0][1] === ring.at(-1)[1]; + }), + ) + ); +} +export function toItem(row) { + const playerUrl = publicUrl(row?.player_url); + if (!row?.id || !row.name || !playerUrl || !row.provider || !row.access_terms) return null; + // No ownership/permission inference from a publicly reachable audio URL. + const streamUrl = row.stream_reuse_allowed === true ? publicUrl(row.stream_url) : null; + const coverage = validCoverage(row.coverage) ? row.coverage : null; + return { + externalId: String(row.id), + kind: 'scanner-stream', + title: String(row.name), + url: playerUrl, + summary: row.description ?? 'Scanner traffic. Radio reports are not confirmed crimes.', + publishedAt: row.updated_at ?? null, + timeKnown: Boolean(row.updated_at), + tags: ['scanner', row.service_type ?? 'public-safety', row.country, row.state, row.city].filter( + Boolean, + ), + data: { + provider: row.provider, + source_id: String(row.id), + agency: row.agency ?? null, + jurisdiction: row.jurisdiction ?? null, + service_type: row.service_type ?? 'public-safety', + player_url: playerUrl, + stream_url: streamUrl, + access_terms: row.access_terms, + stream_reuse_allowed: Boolean(streamUrl), + // Null coverage means unknown, never a receiver's location masquerading as coverage. + coverage, + coverage_basis: coverage + ? (row.coverage_basis ?? + (coverage.type === 'Circle' ? 'approximate-radius' : 'provider-boundary')) + : 'unknown', + location_precision: row.location_precision ?? null, + location_source: row.location_source ?? row.provider, + location_updated_at: row.updated_at ?? null, + country: row.country ?? null, + state: row.state ?? null, + city: row.city ?? null, + last_checked: row.last_checked ?? null, + }, + }; +} +export const scannerDirectory = defineAdapter({ + name: 'scanner-directory', + title: 'Permissioned scanner directory', + collection: 'crime', + description: + 'An operator-supplied JSON catalog of scanner player links, permitted streams and coverage areas. Configure a catalog you may index; no third-party directory is enabled automatically.', + kinds: ['scanner-stream'], + cadenceMinutes: 60, + configFields: [ + { key: 'url', label: 'Permissioned JSON catalog URL', type: 'text', required: true }, + ], + async pull({ config, http }) { + const url = publicUrl(config.url); + if (!url) throw new Error('A valid HTTP(S) catalog URL is required'); + const body = await http.json(url); + const rows = Array.isArray(body) ? body : body?.feeds; + if (!Array.isArray(rows) || rows.length > 10000) + throw new Error('Catalog must contain at most 10000 feeds'); + const items = rows.map(toItem).filter(Boolean); + return { + items, + note: `${items.length} scanner entries; ${rows.length - items.length} invalid entries skipped`, + }; + }, +}); diff --git a/packages/core/package.json b/packages/core/package.json index e43ed82..9d6beff 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -11,7 +11,8 @@ "./opensite": "./src/opensite.js", "./profiles": "./src/profiles.js", "./dump": "./src/dump.js", - "./data-dumps": "./src/data-dumps.js" + "./data-dumps": "./src/data-dumps.js", + "./geo": "./src/geo.js" }, "dependencies": { "@nichedb/adapters": "workspace:*", diff --git a/packages/core/src/geo.js b/packages/core/src/geo.js new file mode 100644 index 0000000..9d13a6a --- /dev/null +++ b/packages/core/src/geo.js @@ -0,0 +1,69 @@ +/** Shared public contract. Does not depend on a web framework or a database. */ +export class GeoQueryError extends Error { + status = 400; + toolError = true; +} +const fail = (message) => { + throw new GeoQueryError(message); +}; +const present = (v) => v !== undefined && v !== null; +function number(v, name, min, max) { + if (!['number', 'string'].includes(typeof v) || String(v).trim() === '') + fail(`${name} must be a number`); + const n = Number(v); + if (!Number.isFinite(n) || n < min || n > max) fail(`${name} must be between ${min} and ${max}`); + return n; +} +export const GEO_KEYS = ['lat', 'long', 'radius', 'unit', 'bbox']; +export function parseGeoQuery(raw = {}) { + const center = present(raw.lat) || present(raw.long); + const box = present(raw.bbox); + if (!center && !box) { + if (present(raw.radius) || present(raw.unit) || raw.sort === 'distance') + fail('lat and long are required for radius, unit or sort=distance'); + return null; + } + if (box) { + if (center || present(raw.radius) || present(raw.unit) || raw.sort === 'distance') + fail('bbox cannot be combined with lat, long, radius, unit or sort=distance'); + const b = Array.isArray(raw.bbox) ? raw.bbox : String(raw.bbox).split(','); + if (b.length !== 4) fail('bbox must be west,south,east,north'); + const bbox = b.map((v, i) => + number(v, 'bbox coordinate', i % 2 ? -90 : -180, i % 2 ? 90 : 180), + ); + if (bbox[1] > bbox[3]) fail('bbox south must not exceed north'); + return { bbox }; + } + if (!present(raw.lat) || !present(raw.long)) fail('lat and long must be supplied together'); + const lat = number(raw.lat, 'lat', -90, 90); + const long = number(raw.long, 'long', -180, 180); + const unit = raw.unit ?? 'km'; + if (!['km', 'mi'].includes(unit)) fail('unit must be km or mi'); + const radius = number(raw.radius ?? 10, 'radius', 0.001, unit === 'km' ? 1000 : 1000 / 1.609344); + return { lat, long, radius, unit }; +} +export function geoQueryFields(raw = {}) { + const geo = parseGeoQuery(raw); + return { ...geo, ...(raw.sort === 'distance' ? { sort: 'distance' } : {}) }; +} +export const geoSchema = { + lat: { + type: 'number', + minimum: -90, + maximum: 90, + description: 'Query center latitude; requires long', + }, + long: { + type: 'number', + minimum: -180, + maximum: 180, + description: 'Query center longitude; requires lat', + }, + radius: { type: 'number', exclusiveMinimum: 0, description: 'Default 10; maximum 1000 km' }, + unit: { type: 'string', enum: ['km', 'mi'], description: 'Default km' }, + bbox: { + type: 'string', + description: 'west,south,east,north; west > east crosses the antimeridian', + }, + sort: { type: 'string', enum: ['id', 'published', 'updated', 'distance'] }, +}; diff --git a/packages/db/migrations/0023_geographic_queries.sql b/packages/db/migrations/0023_geographic_queries.sql new file mode 100644 index 0000000..3055d0d --- /dev/null +++ b/packages/db/migrations/0023_geographic_queries.sql @@ -0,0 +1,145 @@ +-- Geographic reads without a PostGIS dependency. Expression index also covers +-- existing rows; no re-ingestion or stored-column table rewrite is required. +create function ndb_coord(v text, lim double precision) returns double precision +language plpgsql immutable parallel safe as $$ +declare n double precision; +begin + if v is null or btrim(v) = '' then return null; end if; + n := v::double precision; + if n between -lim and lim then return n; end if; + return null; +exception when invalid_text_representation or numeric_value_out_of_range then return null; +end $$; + +-- Coverage takes precedence over a receiver/site point. Unknown or malformed +-- coverage must not silently fall back to a receiver's physical location. +create function ndb_geo_shape(d jsonb) returns jsonb +language plpgsql immutable parallel safe as $$ +declare p jsonb; x double precision; y double precision; +begin + if d ? 'coverage' and d->'coverage' <> 'null'::jsonb then return d->'coverage'; end if; + foreach p in array array[d->'location', d->'place', d->'position', d->'geometry', d] loop + if p->>'type' in ('Point', 'Polygon', 'MultiPolygon', 'Circle') then return p; end if; + y := ndb_coord(coalesce(p->>'lat', p->>'latitude'), 90); + x := ndb_coord(coalesce(p->>'long', p->>'lon', p->>'lng', p->>'longitude'), 180); + if x is not null and y is not null then + return jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(x, y)); + end if; + end loop; + return null; +end $$; + +create function ndb_distance(x1 double precision, y1 double precision, x2 double precision, y2 double precision) +returns double precision language sql immutable strict parallel safe as $$ + select 12742017.6 * asin(sqrt(least(1.0, greatest(0.0, + power(sin(radians(y2-y1)/2),2) + cos(radians(y1))*cos(radians(y2))*power(sin(radians(x2-x1)/2),2))))); +$$; + +-- A conservative longitude/latitude envelope. A crossing or polar envelope is +-- widened to the full longitude range so the index never loses a match. +create function ndb_radius_box(x double precision, y double precision, radius_m double precision) +returns box language plpgsql immutable strict parallel safe as $$ +declare dy double precision := degrees(radius_m / 6371008.8); dx double precision; +begin + if abs(y) + dy >= 90 then dx := 180; + else dx := degrees(asin(least(1.0, sin(radius_m / 6371008.8) / cos(radians(y))))); end if; + if x-dx < -180 or x+dx > 180 then return box(point(-180, greatest(-90,y-dy)), point(180,least(90,y+dy))); end if; + return box(point(x-dx, greatest(-90,y-dy)), point(x+dx,least(90,y+dy))); +end $$; + +create function ndb_geo_box(d jsonb) returns box +language plpgsql immutable parallel safe as $$ +declare g jsonb := ndb_geo_shape(d); p jsonb; ring jsonb; poly jsonb; polys jsonb; + x double precision; y double precision; r double precision; + west double precision := 180; east double precision := -180; + south double precision := 90; north double precision := -90; +begin + if g->>'type' in ('Point','Circle') then + x := ndb_coord(g#>>'{coordinates,0}',180); y := ndb_coord(g#>>'{coordinates,1}',90); + if x is null or y is null then return null; end if; + if g->>'type' = 'Point' then return box(point(x,y),point(x,y)); end if; + r := ndb_coord(g->>'radius_m',1000000); + if r is null or r < 0 then return null; end if; + return ndb_radius_box(x,y,r); + end if; + if g->>'type' not in ('Polygon','MultiPolygon') then return null; end if; + polys := case when g->>'type' = 'Polygon' then jsonb_build_array(g->'coordinates') else g->'coordinates' end; + for poly in select value from jsonb_array_elements(polys) loop + for ring in select value from jsonb_array_elements(poly) loop + if jsonb_array_length(ring) < 4 or ring->0 <> ring->(jsonb_array_length(ring)-1) then return null; end if; + for p in select value from jsonb_array_elements(ring) loop + x := ndb_coord(p->>0,180); y := ndb_coord(p->>1,90); + if x is null or y is null then return null; end if; + west := least(west,x); east := greatest(east,x); south := least(south,y); north := greatest(north,y); + end loop; + end loop; + end loop; + if east < west then return null; end if; + if east-west > 180 then west := -180; east := 180; end if; + -- Polygon edges follow GeoJSON's straight longitude/latitude segments. + return box(point(west,south),point(east,north)); +exception when data_exception then return null; +end $$; + +-- Local GeoJSON polygon distance: containment honours holes; edges are +-- subdivided before spherical distance so longitude/latitude segments follow +-- the same path as map renderers (rather than a single great-circle arc). +create function ndb_polygon_distance(poly jsonb, x double precision, y double precision) +returns double precision language plpgsql immutable parallel safe as $$ +declare ring jsonb; p jsonb; pts text; px double precision; py double precision; + firstx double precision; prevx double precision; prevy double precision; + qx double precision; inside boolean := false; hole boolean := false; idx integer := 0; + best double precision := 'Infinity'; dx double precision; dy double precision; + t double precision; n integer; j integer; ax double precision; ay double precision; + bx double precision; byy double precision; scale double precision; +begin + for ring in select value from jsonb_array_elements(poly) loop + pts := ''; prevx := null; firstx := null; + for p in select value from jsonb_array_elements(ring) loop + px := (p->>0)::double precision; py := (p->>1)::double precision; + if firstx is null then firstx := px; end if; + if prevx is not null then + while px-prevx > 180 loop px := px-360; end loop; + while px-prevx < -180 loop px := px+360; end loop; + n := greatest(1,ceil(greatest(abs(px-prevx),abs(py-prevy))*10)::integer); + for j in 0..n-1 loop + ax := prevx+(px-prevx)*j/n; ay := prevy+(py-prevy)*j/n; + bx := prevx+(px-prevx)*(j+1)/n; byy := prevy+(py-prevy)*(j+1)/n; + qx := x + 360*round((ax-x)/360); + scale := cos(radians(y)); dx := (bx-ax)*scale; dy := byy-ay; + t := case when dx*dx+dy*dy=0 then 0 else greatest(0,least(1,((qx-ax)*scale*dx+(y-ay)*dy)/(dx*dx+dy*dy))) end; + best := least(best,ndb_distance(x,y,ax+(bx-ax)*t,ay+(byy-ay)*t)); + end loop; + end if; + pts := pts || case when pts = '' then '' else ',' end || '(' || px || ',' || py || ')'; + prevx := px; prevy := py; + end loop; + qx := x + 360*round((firstx-x)/360); + if idx=0 then inside := ('('||pts||')')::polygon @> point(qx,y); + elsif ('('||pts||')')::polygon @> point(qx,y) then hole := true; end if; + idx := idx+1; + end loop; + if inside and not hole then return 0; end if; + return best; +end $$; + +create function ndb_geo_distance(d jsonb, x double precision, y double precision) +returns double precision language plpgsql immutable strict parallel safe as $$ +declare g jsonb := ndb_geo_shape(d); poly jsonb; best double precision := 'Infinity'; +begin + if ndb_geo_box(d) is null then return null; end if; + case g->>'type' + when 'Point' then return ndb_distance(x,y,(g#>>'{coordinates,0}')::double precision,(g#>>'{coordinates,1}')::double precision); + when 'Circle' then return greatest(0,ndb_distance(x,y,(g#>>'{coordinates,0}')::double precision,(g#>>'{coordinates,1}')::double precision)-(g->>'radius_m')::double precision); + when 'Polygon' then return ndb_polygon_distance(g->'coordinates',x,y); + when 'MultiPolygon' then + for poly in select value from jsonb_array_elements(g->'coordinates') loop + best := least(best,ndb_polygon_distance(poly,x,y)); + end loop; + return best; + else return null; + end case; +exception when data_exception then return null; +end $$; + +create index items_geo_box_idx on items using gist (ndb_geo_box(data)) where ndb_geo_box(data) is not null; diff --git a/packages/db/src/geo.js b/packages/db/src/geo.js new file mode 100644 index 0000000..2ae20d0 --- /dev/null +++ b/packages/db/src/geo.js @@ -0,0 +1,24 @@ +import { parseGeoQuery } from '@nichedb/core/geo'; + +/** SQL fragments share validation and an indexed envelope prefilter. */ +export function geoSql(sql, raw = {}) { + const geo = parseGeoQuery(raw); + if (!geo) return { where: sql`true`, distance: sql`null::double precision`, geo: null }; + if (geo.bbox) { + const [w, s, e, n] = geo.bbox; + const where = + w <= e + ? sql`ndb_geo_box(i.data) && box(point(${w},${s}),point(${e},${n}))` + : sql`(ndb_geo_box(i.data) && box(point(${w},${s}),point(180,${n})) or ndb_geo_box(i.data) && box(point(-180,${s}),point(${e},${n})))`; + return { where, distance: sql`null::double precision`, geo }; + } + const radiusM = geo.radius * (geo.unit === 'mi' ? 1609.344 : 1000); + const distance = sql`ndb_geo_distance(i.data,${geo.long}::double precision,${geo.lat}::double precision)`; + return { + geo, + distance, + where: sql`(ndb_geo_box(i.data) is not null + and ndb_geo_box(i.data) && ndb_radius_box(${geo.long}::double precision,${geo.lat}::double precision,${radiusM}::double precision) + and ${distance} <= ${radiusM})`, + }; +} diff --git a/packages/db/src/matchups.js b/packages/db/src/matchups.js index a394338..6ced40d 100644 --- a/packages/db/src/matchups.js +++ b/packages/db/src/matchups.js @@ -1,3 +1,4 @@ +import { geoSql } from './geo.js'; import { sql as defaultSql } from './index.js'; /** @@ -227,9 +228,17 @@ export function fixtureWindow({ date = null, now = new Date() } = {}) { */ export async function matchFixtures( teams, - { league = null, collectionId = null, date = null, now = new Date(), limit = 5 } = {}, + { + league = null, + collectionId = null, + date = null, + now = new Date(), + limit = 5, + ...location + } = {}, { sql = defaultSql } = {}, ) { + const geo = geoSql(sql, location); const sides = (teams ?? []).map((t) => String(t ?? '').trim()).filter(Boolean); if (sides.length !== 2) return []; const { from, to, reference } = fixtureWindow({ date, now }); @@ -239,7 +248,7 @@ export async function matchFixtures( const [a2, b2] = sides.map(expandSide); const rows = await sql` select i.*, s.slug as source_slug, s.name as source_name, s.adapter, - c.slug as collection_slug, c.name as collection_name + c.slug as collection_slug, c.name as collection_name, ${geo.distance} as distance_m from items i join sources s on s.id = i.source_id join collections c on c.id = i.collection_id where i.kind = 'fixture' and (${collectionId === null} or i.collection_id = ${collectionId}) @@ -255,7 +264,8 @@ export async function matchFixtures( or lower(coalesce(i.data->'home'->>'name', '')) in (${a1}, ${b1}, ${a2}, ${b2}) or lower(coalesce(i.data->'away'->>'name', '')) in (${a1}, ${b1}, ${a2}, ${b2}) ) - order by i.published_at asc + and ${geo.where} + order by case when ${location.sort === 'distance'} then ${geo.distance} end asc, i.published_at asc limit 200 `; const scored = []; @@ -265,7 +275,10 @@ export async function matchFixtures( } scored.sort( (x, y) => - y.score - x.score || x.distance_hours - y.distance_hours || Number(y.id) - Number(x.id), + (location.sort === 'distance' ? x.distance_m - y.distance_m : 0) || + y.score - x.score || + x.distance_hours - y.distance_hours || + Number(y.id) - Number(x.id), ); return scored.slice(0, Math.min(Math.max(1, limit), 50)); } diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index 557000b..235b2d0 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -1,3 +1,5 @@ +import { GeoQueryError, geoQueryFields } from '@nichedb/core/geo'; +import { geoSql } from './geo.js'; import { sql } from './index.js'; import { matchFixtures } from './matchups.js'; @@ -676,11 +678,13 @@ export async function previousItemData({ sourceId, externalIds }) { return out; } -const itemColumns = sql` +const itemSelection = (db) => db` i.*, s.slug as source_slug, s.name as source_name, s.adapter, c.slug as collection_slug, c.name as collection_name, c.early_access `; +const itemColumns = itemSelection(sql); + export async function getItem(id) { const [row] = await sql` select ${itemColumns} @@ -713,13 +717,20 @@ export async function recentItems({ beforeId = null, afterId = null, limit = 50, + offset = 0, + db = sql, + ...location } = {}) { + const geo = geoSql(db, { ...location, sort }); + const byDistance = sort === 'distance'; + if (byDistance && (beforeId !== null || afterId !== null)) + throw new GeoQueryError('sort=distance uses offset, not before/after'); const wantedTags = (tags ?? []).map((t) => String(t).trim().toLowerCase()).filter(Boolean); const byPublished = sort === 'published'; const byUpdated = sort === 'updated'; const asc = order === 'asc'; - return sql` - select ${itemColumns} + return db` + select ${itemSelection(db)}, ${geo.distance} as distance_m from items i join sources s on s.id = i.source_id join collections c on c.id = i.collection_id where (${collectionId === null} or i.collection_id = ${collectionId}) and (${sourceId === null} or i.source_id = ${sourceId}) @@ -730,14 +741,17 @@ export async function recentItems({ and (${since === null} or i.updated_at >= ${since}) and (${beforeId === null} or i.id < ${beforeId ?? 0}) and (${afterId === null} or i.id > ${afterId ?? 0}) + and ${geo.where} order by + case when ${byDistance} then ${geo.distance} end asc, case when ${byPublished && asc} then i.published_at end asc nulls last, case when ${byPublished && !asc} then i.published_at end desc nulls last, case when ${byUpdated && asc} then i.updated_at end asc, case when ${byUpdated && !asc} then i.updated_at end desc, - case when ${!byPublished && !byUpdated && asc} then i.id end asc, + case when ${!byDistance && !byPublished && !byUpdated && asc} then i.id end asc, i.id desc limit ${Math.min(Math.max(1, limit), 500)} + offset ${byDistance ? Math.min(Math.max(0, Math.floor(Number(offset) || 0)), 10000) : 0} `; } @@ -771,18 +785,25 @@ export async function matchItems( date = null, fallback = null, now = new Date(), + db = sql, + ...location } = {}, ) { + const geo = geoSql(db, location); let t = String(term ?? '').trim(); if (!t) return []; if (teams?.length === 2 && (kind === null || kind === 'fixture')) { - const fixtures = await matchFixtures(teams, { league, collectionId, date, now, limit }); + const fixtures = await matchFixtures( + teams, + { league, collectionId, date, now, limit, ...location }, + { sql: db }, + ); if (fixtures.length || kind === 'fixture') return fixtures; t = String(fallback ?? '').trim() || t; } const wantedTags = (tags ?? []).map((x) => String(x).trim().toLowerCase()).filter(Boolean); - return sql` - select ${itemColumns}, + return db` + select ${itemSelection(db)}, ${geo.distance} as distance_m, similarity(i.title, ${t}) as score from items i join sources s on s.id = i.source_id join collections c on c.id = i.collection_id where (${collectionId === null} or i.collection_id = ${collectionId}) @@ -790,7 +811,9 @@ export async function matchItems( and (${wantedTags.length === 0} or i.tags @> ${pgArray(wantedTags)}::text[]) and (${year === null} or (i.data->>'year')::int between ${(year ?? 0) - 1} and ${(year ?? 0) + 1}) and (i.title % ${t} or i.title ilike ${`%${t}%`}) + and ${geo.where} order by + case when ${location.sort === 'distance'} then ${geo.distance} end asc, (case when lower(i.title) = lower(${t}) then 1 else 0 end) desc, (case when ${year !== null} and (i.data->>'year')::int = ${year ?? 0} then 1 else 0 end) desc, score desc, @@ -804,14 +827,22 @@ export async function matchItems( } /** What is coming: items dated in the future, soonest first. */ -export async function upcomingItems({ collectionId = null, days = 30, limit = 100 } = {}) { - return sql` - select ${itemColumns} +export async function upcomingItems({ + collectionId = null, + days = 30, + limit = 100, + db = sql, + ...location +} = {}) { + const geo = geoSql(db, location); + return db` + select ${itemSelection(db)}, ${geo.distance} as distance_m from items i join sources s on s.id = i.source_id join collections c on c.id = i.collection_id where (${collectionId === null} or i.collection_id = ${collectionId}) and i.published_at > now() and i.published_at < now() + (${`${days} days`})::interval - order by i.published_at asc + and ${geo.where} + order by case when ${location.sort === 'distance'} then ${geo.distance} end asc, i.published_at asc limit ${Math.min(Math.max(1, limit), 500)} `; } @@ -820,17 +851,22 @@ export async function upcomingItems({ collectionId = null, days = 30, limit = 10 * Full-text over title, summary and tags, with a trigram fallback so a query for * a fragment of a package name still lands. */ -export async function searchItems(term, { collectionId = null, kind = null, limit = 30 } = {}) { +export async function searchItems( + term, + { collectionId = null, kind = null, limit = 30, db = sql, ...location } = {}, +) { + const geo = geoSql(db, location); const t = String(term ?? '').trim(); if (!t) return []; - return sql` - select ${itemColumns}, + return db` + select ${itemSelection(db)}, ${geo.distance} as distance_m, ts_rank(i.search, websearch_to_tsquery('simple', ${t})) as rank from items i join sources s on s.id = i.source_id join collections c on c.id = i.collection_id where (${collectionId === null} or i.collection_id = ${collectionId}) and (${kind === null} or i.kind = ${kind}) and (i.search @@ websearch_to_tsquery('simple', ${t}) or i.title ilike ${`%${t}%`}) - order by rank desc, i.id desc + and ${geo.where} + order by case when ${location.sort === 'distance'} then ${geo.distance} end asc, rank desc, i.id desc limit ${Math.min(Math.max(1, limit), 200)} `; } @@ -940,6 +976,7 @@ export function feedQuery(feed) { .map((s) => String(s).trim()) .filter(Boolean); return { + ...geoQueryFields(q), sources: arr(q.sources), kinds: arr(q.kinds), tags: arr(q.tags), @@ -956,10 +993,21 @@ export function feedQuery(feed) { * `afterId` is the delivery scanner's cursor (ascending); `beforeId` is the * page's (descending). Upcoming feeds order by date rather than arrival. */ -export async function feedItems(feed, { afterId = null, beforeId = null, limit = 50 } = {}) { +export async function feedItems( + feed, + { afterId = null, beforeId = null, limit = 50, offset = 0, db = sql, ...location } = {}, +) { const fq = feedQuery(feed); - return sql` - select ${itemColumns} + const savedGeo = geoSql(db, fq); + const requestGeo = geoSql(db, location); + // URL filters narrow a saved feed; they cannot replace its geographic scope. + const geo = requestGeo.geo && !requestGeo.geo.bbox ? requestGeo : savedGeo; + // The delivery scanner always walks id order, even for a distance-sorted feed. + const byDistance = afterId === null && (location.sort ?? fq.sort) === 'distance'; + if (byDistance && beforeId !== null) + throw new GeoQueryError('sort=distance uses offset, not before'); + return db` + select ${itemSelection(db)}, ${geo.distance} as distance_m from items i join sources s on s.id = i.source_id join collections c on c.id = i.collection_id where i.collection_id = ${feed.collection_id} and (${fq.sources.length === 0} or s.slug = any(${pgArray(fq.sources)}::text[])) @@ -970,11 +1018,14 @@ export async function feedItems(feed, { afterId = null, beforeId = null, limit = and (${!fq.upcoming} or i.published_at > now()) and (${afterId === null} or i.id > ${afterId ?? 0}) and (${beforeId === null} or i.id < ${beforeId ?? 0}) + and ${savedGeo.where} and ${requestGeo.where} order by - case when ${fq.upcoming} then i.published_at end asc, + case when ${byDistance} then ${geo.distance} end asc, + case when ${fq.upcoming && afterId === null} then i.published_at end asc, case when ${afterId !== null} then i.id end asc, i.id desc limit ${Math.min(Math.max(1, limit), 500)} + offset ${byDistance ? Math.min(Math.max(0, Math.floor(Number(offset) || 0)), 10000) : 0} `; } @@ -1297,3 +1348,28 @@ export async function itemsForHost({ collectionId, host, limit = 50 }) { limit ${Math.max(1, Math.min(Number(limit) || 50, 200))} `; } + +/** Reported incidents within scanner coverage (or an explicit GPS search). */ +export async function nearbyCrime( + scanner, + { from = null, to = null, limit = 20, db = sql, ...location } = {}, +) { + const geo = geoSql(db, location); + const coverage = JSON.stringify(scanner.data ?? {}); + return db` + select ${itemSelection(db)}, ${geo.distance} as distance_m + from items i join sources s on s.id = i.source_id join collections c on c.id = i.collection_id + where c.slug = 'crime' and not c.early_access and i.kind = 'crime-report' + and (${from === null} or i.published_at >= ${from}) + and (${to === null} or i.published_at < ${to}) + and ndb_geo_shape(i.data)->>'type' = 'Point' + and ndb_geo_box(i.data) && ndb_geo_box(${coverage}::jsonb) + and ndb_geo_distance(${coverage}::jsonb, + (ndb_geo_shape(i.data)#>>'{coordinates,0}')::double precision, + (ndb_geo_shape(i.data)#>>'{coordinates,1}')::double precision) = 0 + and ${geo.where} + order by case when ${location.sort === 'distance'} then ${geo.distance} end asc, + i.published_at desc nulls last, i.id desc + limit ${Math.min(Math.max(1, limit), 100)} + `; +} diff --git a/test/geo-surfaces.test.js b/test/geo-surfaces.test.js new file mode 100644 index 0000000..552d951 --- /dev/null +++ b/test/geo-surfaces.test.js @@ -0,0 +1,116 @@ +import { describe, expect, test } from 'bun:test'; +import { run } from '../apps/cli/src/index.js'; +import { scannerContextOptions } from '../apps/web/src/lib/scanner-context.js'; + +process.env.DATABASE_URL ??= 'postgres://localhost/unused'; +const { TOOLS } = await import('../apps/web/src/lib/mcp/tools.js'); +const { CollectionPage } = await import('../apps/web/src/views/pages.jsx'); +const { FeedForm } = await import('../apps/web/src/views/admin.jsx'); +const { Pager } = await import('../apps/web/src/views/components.jsx'); +const geo = { lat: 0, long: 0, radius: 10, unit: 'km', sort: 'distance' }; + +describe('geographic public surfaces', () => { + test('CLI forwards zero coordinates, signed longitude, bbox and offset', async () => { + const calls = []; + const fetchImpl = async (url) => { + calls.push(new URL(url)); + return new Response(JSON.stringify({ items: [] })); + }; + for (const args of [['recent'], ['search', 'theft'], ['upcoming'], ['items', 'local']]) { + await run( + [ + ...args, + '--api', + 'https://example.test', + '--lat', + '0', + '--long', + '-87.62', + '--radius', + '5', + '--sort', + 'distance', + '--offset', + '2', + ], + { fetchImpl }, + ); + const qs = calls.at(-1).searchParams; + expect(qs.get('lat')).toBe('0'); + expect(qs.get('long')).toBe('-87.62'); + expect(qs.get('offset')).toBe('2'); + } + await run(['recent', '--api', 'https://example.test', '--bbox', '170,-10,-170,10'], { + fetchImpl, + }); + expect(calls.at(-1).searchParams.get('bbox')).toBe('170,-10,-170,10'); + }); + test('MCP advertises GPS fields and returns actionable validation errors', async () => { + for (const name of [ + 'recent_items', + 'feed_items', + 'search_items', + 'match_items', + 'upcoming', + 'create_feed', + ]) { + const tool = TOOLS.find((t) => t.name === name); + for (const key of ['lat', 'long', 'radius', 'unit', 'bbox', 'sort']) + expect(tool.inputSchema.properties[key]).toBeTruthy(); + } + await expect( + TOOLS.find((t) => t.name === 'recent_items').run({ lat: 0 }), + ).rejects.toMatchObject({ status: 400, toolError: true }); + }); + test('collection facets and distance pagination preserve GPS', async () => { + const html = await CollectionPage({ + collection: { slug: 'crime', name: 'Crime' }, + stats: { items: 1, sources: 1, feeds: 0 }, + sources: [], + feeds: [], + latest: [{ id: 3, title: 'Report', kind: 'crime-report', tags: [], data: {} }], + upcoming: [], + kinds: [ + { kind: 'crime-report', n: 1 }, + { kind: 'scanner-stream', n: 1 }, + ], + tags: [{ tag: 'theft' }], + geo, + offset: 50, + }).toString(); + expect(html).toContain('lat=0&long=0'); + expect(html).toContain('kind=scanner-stream'); + expect(html).toContain('offset=51'); + expect(html).not.toContain('before='); + }); + test('ordinary pagination still uses item IDs', async () => { + const html = await Pager({ items: [{ id: 8 }], base: '/c/crime?lat=0&long=0' }).toString(); + expect(html).toContain('before=8'); + }); + test('feed edit form does not erase a saved geographic query', async () => { + const html = await FeedForm({ + collections: [], + sources: [], + kinds: [], + enrichers: [], + collection: { slug: 'crime' }, + values: { ...geo, name: 'Local' }, + editing: { slug: 'local' }, + preview: [], + }).toString(); + expect(html).toContain('name="lat" value="0"'); + expect(html).toContain('name="long" value="0"'); + expect(html).toContain('name="sort" value="distance"'); + }); + test('date context validates order and half-open window; blank form fields are optional', () => { + expect(scannerContextOptions({ from: '2026-08-01', to: '2026-09-01' })).toEqual({ + from: '2026-08-01T00:00:00.000Z', + to: '2026-09-01T00:00:00.000Z', + }); + expect(scannerContextOptions({ from: '', to: '' })).toEqual({ from: null, to: null }); + expect(() => scannerContextOptions({ from: 'bad' })).toThrow('ISO date'); + expect(() => scannerContextOptions({ from: '2026-09-01', to: '2026-08-01' })).toThrow( + 'precede', + ); + }); +}); diff --git a/test/geo.test.js b/test/geo.test.js new file mode 100644 index 0000000..2c21707 --- /dev/null +++ b/test/geo.test.js @@ -0,0 +1,359 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { readdir, readFile } from 'node:fs/promises'; +import { PGlite } from '@electric-sql/pglite'; +import { citext } from '@electric-sql/pglite/contrib/citext'; +import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; +import { toItem, validCoverage } from '../packages/adapters/src/scanners.js'; +import { geoQueryFields, parseGeoQuery } from '../packages/core/src/geo.js'; + +process.env.DATABASE_URL ??= 'postgres://localhost/unused'; +const q = await import('../packages/db/src/queries.js'); +const { normaliseQuery } = await import('../apps/web/src/lib/service.js'); +const { itemOut } = await import('../apps/web/src/lib/serialize.js'); +function pgliteSql(db) { + class Fragment { + constructor(strings, values) { + this.strings = strings; + this.values = values; + } + compile(params) { + let text = ''; + this.strings.forEach((s, i) => { + text += s; + if (i >= this.values.length) return; + const v = this.values[i]; + if (v instanceof Fragment) text += v.compile(params); + else { + params.push(v); + text += `$${params.length}`; + } + }); + return text; + } + // biome-ignore lint/suspicious/noThenProperty: a query is awaited, as Bun's is + then(resolve, reject) { + const params = []; + const text = this.compile(params); + return db + .query(text, params) + .then((r) => r.rows) + .then(resolve, reject); + } + } + return (strings, ...values) => new Fragment(strings, values); +} + +let db, sql, crime, weather, source; +const center = { lat: 41.88, long: -87.62 }; +const circle = { type: 'Circle', coordinates: [-87.62, 41.88], radius_m: 3000 }; +const polygon = { + type: 'Polygon', + coordinates: [ + [ + [-87.64, 41.86], + [-87.6, 41.86], + [-87.6, 41.9], + [-87.64, 41.9], + [-87.64, 41.86], + ], + ], +}; +const near = { place: { lat: 41.881, lon: -87.62 } }; +const far = { place: { lat: 42.88, lon: -87.62 } }; +const one = async (text, values = []) => (await db.query(text, values)).rows[0]; +let seq = 0; +async function item( + data, + { collection = crime, kind = 'crime-report', at = '2026-08-01', title = 'Test report' } = {}, +) { + return Number( + ( + await one( + `insert into items (collection_id,source_id,external_id,kind,title,data,published_at) values ($1,$2,$3,$4,$5,$6,$7) returning id`, + [collection, source, String(++seq), kind, title, JSON.stringify(data), at], + ) + ).id, + ); +} +beforeAll(async () => { + db = await new PGlite({ extensions: { citext, pg_trgm } }); + const dir = new URL('../packages/db/migrations/', import.meta.url); + for (const f of (await readdir(dir)).filter((f) => f.endsWith('.sql')).sort()) + await db.exec(await readFile(new URL(f, dir), 'utf8')); + sql = pgliteSql(db); + crime = Number( + (await one("insert into collections(slug,name) values ('crime','Crime') returning id")).id, + ); + weather = Number((await one("select id from collections where slug='weather'")).id); + source = Number( + ( + await one( + "insert into sources(collection_id,adapter,slug,name) values ($1,'scanner-directory','test-geo','Test') returning id", + [crime], + ) + ).id, + ); +}, 60000); +afterAll(async () => { + await db?.close(); +}); + +describe('geographic contract', () => { + test('zero coordinates, defaults and saved feed roundtrip', () => { + expect(parseGeoQuery({ lat: '0', long: '0' })).toEqual({ + lat: 0, + long: 0, + radius: 10, + unit: 'km', + }); + expect(parseGeoQuery({})).toBeNull(); + const raw = { ...center, radius: 2, unit: 'mi', sort: 'distance', tags: ['theft'] }; + const saved = normaliseQuery(raw); + expect(q.feedQuery({ query: JSON.stringify(saved) })).toMatchObject(raw); + expect(itemOut({ distance_m: 0 }, 'https://test').distance_m).toBe(0); + }); + test('invalid or ambiguous requests fail instead of returning unfiltered data', () => { + for (const raw of [ + { lat: 1 }, + { long: 1 }, + { lat: '', long: 0 }, + { lat: true, long: 0 }, + { lat: 91, long: 0 }, + { lat: 0, long: 181 }, + { lat: 'NaN', long: 0 }, + { lat: 0, long: 0, radius: 0 }, + { lat: 0, long: 0, radius: 1001 }, + { lat: 0, long: 0, unit: 'm' }, + { radius: 10 }, + { unit: 'km' }, + { sort: 'distance' }, + { bbox: '1,2,3' }, + { bbox: '1,4,3,2' }, + { bbox: '1,2,3,4', lat: 0, long: 0 }, + ]) { + expect(() => parseGeoQuery(raw)).toThrow(); + } + expect(geoQueryFields({ bbox: '170,-10,-170,10' })).toEqual({ bbox: [170, -10, -170, 10] }); + }); +}); + +describe('database geo filtering', () => { + test('existing coordinate shapes, malformed values and unknown locations', async () => { + for (const d of [ + near, + { lat: '41.881', longitude: '-87.62' }, + { location: { latitude: 41.881, lng: -87.62 } }, + { geometry: { type: 'Point', coordinates: [-87.62, 41.881] } }, + ]) { + const row = await one('select ndb_geo_distance($1::jsonb,-87.62,41.88) as d', [ + JSON.stringify(d), + ]); + expect(row.d).toBeCloseTo(111.195, 2); + } + for (const d of [ + {}, + { place: { lat: null, lon: null } }, + { lat: '', lon: ' ' }, + { lat: 'REDACTED', lon: 2 }, + { lat: 100, lon: 0 }, + { lat: 'Infinity', lon: 0 }, + { geometry: { type: 'Point', coordinates: ['bad', 0] } }, + { coverage: { type: 'Unknown' }, ...near }, + ]) { + expect((await one('select ndb_geo_box($1::jsonb) as b', [JSON.stringify(d)])).b).toBeNull(); + } + }); + test('radius filters run before limits across collections; distances sort and paginate', async () => { + const nearId = await item(near); + const closeId = await item({ lat: 41.88, lon: -87.62 }, { collection: weather }); + for (let i = 0; i < 4; i++) await item(far); + await item({}); + const options = { ...center, radius: 1, sort: 'distance', db: sql, limit: 1 }; + expect((await q.recentItems(options)).map((r) => Number(r.id))).toEqual([closeId]); + expect((await q.recentItems({ ...options, offset: 1 })).map((r) => Number(r.id))).toEqual([ + nearId, + ]); + expect( + (await q.recentItems({ ...options, collectionId: crime })).map((r) => Number(r.id)), + ).toEqual([nearId]); + expect((await q.recentItems({ db: sql, limit: 1 }))[0].data).toEqual({}); + await expect(q.recentItems({ ...options, beforeId: 100 })).rejects.toThrow('offset'); + }); + test('miles conversion and exact radius excludes bbox corner', async () => { + const edge = await item({ lat: 41.9, lon: -87.62 }); + expect( + (await q.recentItems({ ...center, radius: 2, unit: 'km', db: sql })).some( + (r) => Number(r.id) === edge, + ), + ).toBe(false); + expect( + (await q.recentItems({ ...center, radius: 2, unit: 'mi', db: sql })).some( + (r) => Number(r.id) === edge, + ), + ).toBe(true); + const corner = await item({ lat: 41.888, lon: -87.61 }); + expect( + (await q.recentItems({ ...center, radius: 1, db: sql })).some((r) => Number(r.id) === corner), + ).toBe(false); + }); + test('antimeridian and polar radius searches', async () => { + const dateline = await item({ lat: 0, lon: -179.99 }); + expect( + (await q.recentItems({ lat: 0, long: 179.99, radius: 5, db: sql })).map((r) => Number(r.id)), + ).toContain(dateline); + expect( + (await q.recentItems({ bbox: '179,-1,-179,1', db: sql })).map((r) => Number(r.id)), + ).toContain(dateline); + const pole = await item({ lat: 89.99, lon: 170 }); + expect( + (await q.recentItems({ lat: 89.99, long: -10, radius: 5, db: sql })).map((r) => Number(r.id)), + ).toContain(pole); + }); + test('coverage supersedes a distant receiver, holes and multiple polygons work', async () => { + const scanner = await item( + { coverage: circle, place: { lat: 0, lon: 0 } }, + { kind: 'scanner-stream' }, + ); + expect( + (await q.recentItems({ ...center, radius: 1, kind: 'scanner-stream', db: sql })).map((r) => + Number(r.id), + ), + ).toContain(scanner); + expect( + ( + await one('select ndb_geo_distance($1::jsonb,-87.62,41.88) as d', [ + JSON.stringify({ coverage: polygon }), + ]) + ).d, + ).toBe(0); + const hole = [ + [-87.625, 41.875], + [-87.615, 41.875], + [-87.615, 41.885], + [-87.625, 41.885], + [-87.625, 41.875], + ]; + const hollow = { type: 'Polygon', coordinates: [...polygon.coordinates, hole] }; + expect( + ( + await one('select ndb_geo_distance($1::jsonb,-87.62,41.88) as d', [ + JSON.stringify({ coverage: hollow }), + ]) + ).d, + ).toBeGreaterThan(400); + const multi = { type: 'MultiPolygon', coordinates: [polygon.coordinates] }; + expect( + ( + await one('select ndb_geo_distance($1::jsonb,-87.62,41.88) as d', [ + JSON.stringify({ coverage: multi }), + ]) + ).d, + ).toBe(0); + }); + test('crossing polygon containment does not match the opposite hemisphere', async () => { + const coverage = { + type: 'Polygon', + coordinates: [ + [ + [179, -1], + [-179, -1], + [-179, 1], + [179, 1], + [179, -1], + ], + ], + }; + expect( + (await one('select ndb_geo_distance($1::jsonb,180,0) as d', [JSON.stringify({ coverage })])) + .d, + ).toBe(0); + expect( + (await one('select ndb_geo_distance($1::jsonb,0,0) as d', [JSON.stringify({ coverage })])).d, + ).toBeGreaterThan(19000000); + }); + test('saved feed geography is intersected; notifications preserve id cursor order', async () => { + const a = await item({ lat: 41.881, lon: -87.62 }, { at: '2099-01-03' }); + const b = await item({ lat: 41.88, lon: -87.62 }, { at: '2099-01-01' }); + const feed = { + collection_id: crime, + query: { ...center, radius: 1, sort: 'distance', upcoming: true }, + }; + const delivered = await q.feedItems(feed, { db: sql, afterId: a - 1 }); + expect(delivered.map((r) => Number(r.id))).toEqual([a, b]); + expect((await q.feedItems(feed, { db: sql }))[0].id).toBe(b); + expect(await q.feedItems(feed, { db: sql, lat: 0, long: 0, radius: 10 })).toEqual([]); + await expect(q.feedItems(feed, { db: sql, beforeId: b })).rejects.toThrow('offset'); + }); + test('search/upcoming/match use the same geographic filter', async () => { + const id = await item(near, { at: '2099-01-01', title: 'Unique geographic marker' }); + await item(far, { at: '2099-01-01', title: 'Unique geographic marker' }); + for (const rows of [ + await q.searchItems('Unique geographic', { ...center, radius: 1, db: sql }), + await q.upcomingItems({ ...center, radius: 1, days: 40000, db: sql }), + await q.matchItems('Unique geographic marker', { ...center, radius: 1, db: sql }), + ]) { + expect(rows.map((r) => Number(r.id))).toContain(id); + expect(rows.every((r) => r.distance_m <= 1000)).toBe(true); + } + }); + test('crime context uses coverage, date window, incident kinds and location precision', async () => { + const id = await item({ ...near, locationBasis: 'anonymised-map-point' }, { at: '2026-09-01' }); + await item(near, { kind: 'crime-estimate', at: '2026-09-01' }); + await item(far, { at: '2026-09-01' }); + const rows = await q.nearbyCrime( + { data: { coverage: circle } }, + { db: sql, from: '2026-09-01', to: '2026-09-02' }, + ); + expect(rows.map((r) => Number(r.id))).toEqual([id]); + expect(rows[0].data.locationBasis).toBe('anonymised-map-point'); + expect(await q.nearbyCrime({ data: {} }, { db: sql })).toEqual([]); + }); + test('expression index is available to geographic queries', async () => { + await db.exec('set enable_seqscan=off'); + const rows = ( + await db.query( + 'explain select id from items where ndb_geo_box(data) is not null and ndb_geo_box(data) && ndb_radius_box(-87.62,41.88,1000)', + ) + ).rows; + expect(JSON.stringify(rows)).toContain('items_geo_box_idx'); + await db.exec('reset enable_seqscan'); + }); +}); + +describe('permissioned scanner adapter', () => { + const row = { + id: 'scanner-1', + name: 'Community dispatch', + provider: 'Operator', + access_terms: 'Permission to index', + player_url: 'https://example.com/listen', + coverage: circle, + }; + test('streams require explicit permission; player links and source attribution remain', () => { + const i = toItem({ ...row, stream_url: 'https://example.com/audio.mp3' }); + expect(i.data.stream_url).toBeNull(); + expect(i.data.coverage_basis).toBe('approximate-radius'); + expect( + toItem({ ...row, stream_url: 'https://example.com/audio.mp3', stream_reuse_allowed: true }) + .data.stream_url, + ).toBe('https://example.com/audio.mp3'); + expect(toItem({ ...row, player_url: 'javascript:alert(1)' })).toBeNull(); + expect(toItem({ ...row, access_terms: '' })).toBeNull(); + }); + test('invalid coverage stays unknown and receiver coordinates do not substitute', () => { + expect(validCoverage(polygon)).toBe(true); + expect( + validCoverage({ + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 1], + ], + ], + }), + ).toBe(false); + expect(toItem({ ...row, coverage: null, lat: 1, long: 2 }).data.coverage).toBeNull(); + }); +});