Add California police feeds for cities over 50,000 residents

.github/workflows/ci.yml

@@ -25,6 +25,7 @@ jobs:
           bun-version: 1.4.0
       - run: bun install
       - run: bun run lint
+      - run: python3 test/police-source-discovery.test.py
       - run: bun test
         env:
           DATABASE_URL: postgres://unused:unused@localhost:5432/unused

.gitignore

@@ -1,4 +1,5 @@
 node_modules/
+__pycache__/
 .env
 .env.local
 *.log

README.md

@@ -34,7 +34,7 @@ Adapters are one file each in `packages/adapters/src`. One hundred and twenty-ei
 | automotive | `fueleconomy-catalog`, `nhtsa-recalls`, `nhtsa-complaints`, `nhtsa-safety-ratings` | no |
 | markets | `iso-mic-exchanges`, `alpaca-corporate-actions`, `alpaca-news`, `alpaca-assets`, `equity-history`, `nasdaq-halts`, `ecb-fx-rates`, `sec-fundamentals` | Alpaca (`APCA_API_KEY_ID`, `APCA_API_SECRET_KEY`, `APCA_FEED` iex or sip); SEC wants `CONTACT_EMAIL` in the user agent |
 | crypto | `coingecko-assets`, `crypto-pairs` | keyless (`COINGECKO_API_KEY` optional; `CRYPTO_PROXY_URL` for Binance.US from a datacenter) |
-| crime | `socrata-crime`, `uk-police-crime`, `fbi-crime-estimates` | FBI only (free api.data.gov key) |
+| crime | `socrata-crime`, `uk-police-crime`, `fbi-crime-estimates`, `police-updates` (39 California police feeds; [source coverage](docs/police-sources.md)) | FBI only (free api.data.gov key) |
 | public-money | `usaspending-awards`, `ocds-tenders`, `ted-notices` | no |
 | housing | `uk-land-registry`, `freddie-mac-rates`, `building-permits` | no |
 | jobs | `bls-series`, `eurostat`, `warn-layoffs`, `agenticjobs` | no |

apps/web/package.json

@@ -10,6 +10,7 @@
   },
   "dependencies": {
     "@logicsrc/openaccess": "^0.3.0",
+    "@nichedb/adapters": "workspace:*",
     "@nichedb/auth": "workspace:*",
     "@nichedb/config": "workspace:*",
     "@nichedb/core": "workspace:*",

apps/web/src/routes/pages.js

@@ -22,6 +22,7 @@ import {
   Landing,
   SearchPage,
 } from '../views/pages.jsx';
+import { PoliceCoveragePage } from '../views/police-coverage.jsx';
 
 /**
  * What a free reader gets where a members-only collection would be.
@@ -79,6 +80,19 @@ export function registerPages(app) {
   });
   app.get('/rings', (c) => c.redirect('/c/webrings', 301));
 
+  app.get('/c/crime/coverage', async (c) =>
+    c.html(
+      await render(
+        <PoliceCoveragePage
+          user={c.get('user')}
+          search={(c.req.query('q') ?? '').slice(0, 100)}
+          minimum={c.req.query('min') === '100000' ? 100000 : 50000}
+          pending={c.req.query('pending') === '1'}
+        />,
+      ),
+    ),
+  );
+
   app.get('/c/:slug', async (c) => {
     const collection = await q.getCollection(c.req.param('slug'));
     if (!collection) return c.notFound();

apps/web/src/views/components.jsx

@@ -127,7 +127,16 @@ export const ItemRow = ({ item, showSource = true, enrichers = null, reserveThum
         {' · '}
         <span class="kind">{item.kind}</span>
         {item.distance_m != null ? (
-          <span> · {(Number(item.distance_m) / 1000).toFixed(2)} km from point or coverage</span>
+          <span>
+            {' '}
+            · {(Number(item.distance_m) / 1000).toFixed(2)} km{' '}
+            {item.kind === 'police-update'
+              ? 'to publishing city reference point'
+              : 'from point or coverage'}
+          </span>
+        ) : null}
+        {item.kind === 'police-update' ? (
+          <span> · city-level announcement · publication date</span>
         ) : null}
         {item.url ? (
           <>

apps/web/src/views/pages.jsx

@@ -97,6 +97,14 @@ export const CollectionPage = ({
   >
     <h1>{collection.name}</h1>
     <p class="lede">{collection.description}</p>
+    {collection.slug === 'crime' ? (
+      <p>
+        <a href="/c/crime/coverage">California police source coverage</a> ·{' '}
+        <a href={locationLink('/c/crime', geo, { kind: 'police-update' })}>
+          Police updates in this area
+        </a>
+      </p>
+    ) : null}
     <p class="stats">
       <Num n={stats.items} /> items · <Num n={stats.items_today} /> today · {stats.sources} sources
       · {stats.feeds} feeds
@@ -378,6 +386,13 @@ export const ItemPage = ({
         ) : null}
       </p>
       {item.summary ? <p class="lede">{item.summary}</p> : null}
+      {item.kind === 'police-update' ? (
+        <p class="muted">
+          Published by {item.data?.publisher}. This is a police announcement. The date is its
+          publication time; the geographic location is the Census reference point for{' '}
+          {item.data?.jurisdiction}. The incident time and exact location are not supplied here.
+        </p>
+      ) : null}
       <AwardPills counts={awardCounts} />
       <AwardForm
         targetType="item"

apps/web/src/views/police-coverage.jsx

@@ -0,0 +1,111 @@
+import { POLICE_CITIES, POLICE_SCOPE } from '@nichedb/adapters';
+import { Layout } from './Layout.jsx';
+
+export const PoliceCoveragePage = ({ user, search = '', minimum = 50000, pending = false }) => {
+  const eligible = POLICE_CITIES.filter((city) => city.population > minimum);
+  const cities = eligible.filter(
+    (city) =>
+      (!pending || !city.ingestion) &&
+      `${city.name} ${city.references.map((r) => r.url).join(' ')}`
+        .toLowerCase()
+        .includes(search.toLowerCase()),
+  );
+  return (
+    <Layout user={user} title="California police sources" canonical="/c/crime/coverage">
+      <p class="crumb">
+        <a href="/c/crime">Crime</a> › California sources
+      </p>
+      <h1>California police sources</h1>
+      <p class="lede">
+        {eligible.length} cities over {minimum.toLocaleString()} residents.{' '}
+        {eligible.filter((city) => city.ingestion).length} have reviewed public feeds configured for
+        hourly checks. Other cities have source leads to follow while feed coverage is expanded.
+      </p>
+      <p>
+        <a href="/f/police-updates-ca">Read California police updates</a>. Announcements can include
+        traffic advisories, crime releases and community notices. Geographic filters locate the
+        publishing city, and dates show when the announcement was published.
+      </p>
+      <form method="get" action="/c/crime/coverage">
+        <label>
+          City or account <input name="q" value={search} placeholder="San Jose, Fremont…" />
+        </label>{' '}
+        <label>
+          Population{' '}
+          <select name="min">
+            <option value="50000" selected={minimum === 50000}>
+              Over 50,000
+            </option>
+            <option value="100000" selected={minimum === 100000}>
+              Over 100,000
+            </option>
+          </select>
+        </label>{' '}
+        <label>
+          <input type="checkbox" name="pending" value="1" checked={pending} /> Needs a feed
+        </label>{' '}
+        <button type="submit">Filter</button>
+      </form>
+      <p class="muted">
+        {cities.length} cities shown. Population: Census {POLICE_SCOPE.populationYear} estimates.
+      </p>
+      <table>
+        <thead>
+          <tr>
+            <th>City</th>
+            <th>Feed coverage</th>
+            <th>Source links</th>
+          </tr>
+        </thead>
+        <tbody>
+          {cities.map((city) => (
+            <tr key={city.geoid}>
+              <td>
+                <strong>{city.name}</strong>
+                <br />
+                <span class="muted small">{city.population.toLocaleString()} residents</span>
+              </td>
+              <td>
+                {city.ingestion ? (
+                  <>
+                    <a href={`/s/police-ca-${city.slug}`}>
+                      Reviewed {city.ingestion.format === 'nixle' ? 'Nixle' : 'RSS'} feed
+                    </a>
+                    <br />
+                    <span class="muted small">
+                      Hourly when enabled · checked {city.ingestion.checkedAt.slice(0, 10)}
+                    </span>
+                  </>
+                ) : (
+                  <span class="muted">Source leads only</span>
+                )}
+              </td>
+              <td>
+                <ul class="plain">
+                  {city.references.map((reference) => (
+                    <li key={reference.url}>
+                      <a href={reference.url} rel="noopener nofollow">
+                        {reference.kind} ↗
+                      </a>{' '}
+                      <span class="muted small">
+                        {reference.reviewed ? 'reviewed' : 'candidate'}
+                      </span>
+                    </li>
+                  ))}
+                </ul>
+              </td>
+            </tr>
+          ))}
+        </tbody>
+      </table>
+      <p class="small muted">
+        Source leads may need jurisdiction and freshness checks. County sheriff pages may cover
+        multiple cities. Social account links are available to visit; automated social ingestion is
+        not included. Public feeds can change or become unavailable.
+      </p>
+      <p class="small">
+        <a href={POLICE_SCOPE.populationSource}>Census population source</a>
+      </p>
+    </Layout>
+  );
+};