diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 77e28a1..d58a190 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,6 +25,7 @@ jobs:
bun-version: 1.4.0
- run: bun install
- run: bun run lint
+ - run: python3 test/police-source-discovery.test.py
- run: bun test
env:
DATABASE_URL: postgres://unused:unused@localhost:5432/unused
diff --git a/.gitignore b/.gitignore
index cbf27d1..104b6fe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
node_modules/
+__pycache__/
.env
.env.local
*.log
diff --git a/README.md b/README.md
index 481b0f6..23543e0 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ Adapters are one file each in `packages/adapters/src`. One hundred and twenty-ei
| automotive | `fueleconomy-catalog`, `nhtsa-recalls`, `nhtsa-complaints`, `nhtsa-safety-ratings` | no |
| markets | `iso-mic-exchanges`, `alpaca-corporate-actions`, `alpaca-news`, `alpaca-assets`, `equity-history`, `nasdaq-halts`, `ecb-fx-rates`, `sec-fundamentals` | Alpaca (`APCA_API_KEY_ID`, `APCA_API_SECRET_KEY`, `APCA_FEED` iex or sip); SEC wants `CONTACT_EMAIL` in the user agent |
| crypto | `coingecko-assets`, `crypto-pairs` | keyless (`COINGECKO_API_KEY` optional; `CRYPTO_PROXY_URL` for Binance.US from a datacenter) |
-| crime | `socrata-crime`, `uk-police-crime`, `fbi-crime-estimates` | FBI only (free api.data.gov key) |
+| crime | `socrata-crime`, `uk-police-crime`, `fbi-crime-estimates`, `police-updates` (39 California police feeds; [source coverage](docs/police-sources.md)) | FBI only (free api.data.gov key) |
| public-money | `usaspending-awards`, `ocds-tenders`, `ted-notices` | no |
| housing | `uk-land-registry`, `freddie-mac-rates`, `building-permits` | no |
| jobs | `bls-series`, `eurostat`, `warn-layoffs`, `agenticjobs` | no |
diff --git a/apps/web/package.json b/apps/web/package.json
index 9d64407..a4335d5 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -10,6 +10,7 @@
},
"dependencies": {
"@logicsrc/openaccess": "^0.3.0",
+ "@nichedb/adapters": "workspace:*",
"@nichedb/auth": "workspace:*",
"@nichedb/config": "workspace:*",
"@nichedb/core": "workspace:*",
diff --git a/apps/web/src/routes/pages.js b/apps/web/src/routes/pages.js
index 1f34d81..faf16bd 100644
--- a/apps/web/src/routes/pages.js
+++ b/apps/web/src/routes/pages.js
@@ -22,6 +22,7 @@ import {
Landing,
SearchPage,
} from '../views/pages.jsx';
+import { PoliceCoveragePage } from '../views/police-coverage.jsx';
/**
* What a free reader gets where a members-only collection would be.
@@ -79,6 +80,19 @@ export function registerPages(app) {
});
app.get('/rings', (c) => c.redirect('/c/webrings', 301));
+ app.get('/c/crime/coverage', async (c) =>
+ c.html(
+ await render(
+
{collection.description}
+ {collection.slug === 'crime' ? ( ++ California police source coverage ·{' '} + + Police updates in this area + +
+ ) : null}
{item.summary}
: null} + {item.kind === 'police-update' ? ( ++ Published by {item.data?.publisher}. This is a police announcement. The date is its + publication time; the geographic location is the Census reference point for{' '} + {item.data?.jurisdiction}. The incident time and exact location are not supplied here. +
+ ) : null}+ Crime › California sources +
++ {eligible.length} cities over {minimum.toLocaleString()} residents.{' '} + {eligible.filter((city) => city.ingestion).length} have reviewed public feeds configured for + hourly checks. Other cities have source leads to follow while feed coverage is expanded. +
++ Read California police updates. Announcements can include + traffic advisories, crime releases and community notices. Geographic filters locate the + publishing city, and dates show when the announcement was published. +
+ ++ {cities.length} cities shown. Population: Census {POLICE_SCOPE.populationYear} estimates. +
+| City | +Feed coverage | +Source links | +
|---|---|---|
|
+ {city.name}
+ + {city.population.toLocaleString()} residents + |
+
+ {city.ingestion ? (
+ <>
+
+ Reviewed {city.ingestion.format === 'nixle' ? 'Nixle' : 'RSS'} feed
+
+ + + Hourly when enabled · checked {city.ingestion.checkedAt.slice(0, 10)} + + > + ) : ( + Source leads only + )} + |
+
+
|
+
+ Source leads may need jurisdiction and freshness checks. County sheriff pages may cover + multiple cities. Social account links are available to visit; automated social ingestion is + not included. Public feeds can change or become unavailable. +
+ +CA municipal governments with more than 50,000 residents in Census Vintage 2025. Includes functioning towns and townships; overlapping governments retain separate Census IDs. Source discovery does not establish complete crime coverage or live ingestion.
+Start with the area in your NicheDB link. The default view shows municipalities whose Census reference point lies within 100 km of 37.243507, −121.942648. These points locate jurisdictions, not incidents.
+ + + +| Place | Population | Sources and evidence |
|---|
“Reviewed” identifies sources checked during this research. Automated links and indexed results remain candidates for checking department identity, jurisdiction, subject matter, and freshness. RSS calendars, empty feeds, single announcements, and alert signup pages are recorded separately. X and other social account URLs are not working ingestion endpoints.
+Files: channels.csv · inventory.json · summary.json · validation.json
+ \ No newline at end of file diff --git a/docs/data/police-sources/indexed-searches.json b/docs/data/police-sources/indexed-searches.json new file mode 100644 index 0000000..d7ae432 --- /dev/null +++ b/docs/data/police-sources/indexed-searches.json @@ -0,0 +1,3090 @@ +[ + { + "id": "162:0655184", + "name": "Palm Desert", + "state": "CA", + "query": "\"Palm Desert\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Public Safety | City of Palm Desert", + "url": "https://www.palmdesert.gov/community/public-safety", + "evidence_url": "https://www.palmdesert.gov/community/public-safety", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0636448", + "name": "Indio", + "state": "CA", + "query": "\"Indio\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "DOJ CJSC CRIMES Context", + "url": "https://data-openjustice.doj.ca.gov/sites/default/files/dataset/2025-07/Crimes-Clear-Arson%20Context_06162025.pdf", + "evidence_url": "https://data-openjustice.doj.ca.gov/sites/default/files/dataset/2025-07/Crimes-Clear-Arson%20Context_06162025.pdf", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0670224", + "name": "Santee", + "state": "CA", + "query": "\"Santee\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "newsroom", + "title": "News List | San Diego County Sheriff", + "url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list", + "evidence_url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "newsroom", + "title": "News List | San Diego County Sheriff", + "url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-selcat-36/-npage-2", + "evidence_url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-selcat-36/-npage-2", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "newsroom", + "title": "News List | San Diego County Sheriff", + "url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-selectview-1/-arch-1", + "evidence_url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-selectview-1/-arch-1", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0621712", + "name": "El Cajon", + "state": "CA", + "query": "\"El Cajon\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "newsroom", + "title": "Happening Now | El Cajon, CA", + "url": "https://www.elcajon.gov/your-government/departments/police-department/information-you-may-need/police-department-news/happening-now", + "evidence_url": "https://www.elcajon.gov/your-government/departments/police-department/information-you-may-need/police-department-news/happening-now", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "newsroom", + "title": "Happening Now | El Cajon, CA", + "url": "https://www.elcajon.gov/your-government/departments/police-department/information-you-may-need/police-department-news/happening-now/-arch-1", + "evidence_url": "https://www.elcajon.gov/your-government/departments/police-department/information-you-may-need/police-department-news/happening-now/-arch-1", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Welcome to the El Cajon Police Department | El Cajon, CA", + "url": "https://www.elcajon.gov/your-government/departments/police", + "evidence_url": "https://www.elcajon.gov/your-government/departments/police", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0650398", + "name": "National City", + "state": "CA", + "query": "\"National City\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "newsroom", + "title": "Press Releases | National City, CA", + "url": "https://www.nationalcityca.gov/government/police/press-releases", + "evidence_url": "https://www.nationalcityca.gov/government/police/press-releases", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Directory | National City, CA", + "url": "https://www.nationalcityca.gov/government/police/directory", + "evidence_url": "https://www.nationalcityca.gov/government/police/directory", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police | National City, CA", + "url": "https://www.nationalcityca.gov/government/police", + "evidence_url": "https://www.nationalcityca.gov/government/police", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0613392", + "name": "Chula Vista", + "state": "CA", + "query": "\"Chula Vista\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "newsroom", + "title": "News Center | City of Chula Vista", + "url": "https://www.chulavistaca.gov/departments/police-department/about-us/transparency-and-accountability/press-log", + "evidence_url": "https://www.chulavistaca.gov/departments/police-department/about-us/transparency-and-accountability/press-log", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police Department | City of Chula Vista", + "url": "https://www.chulavistaca.gov/departments/police-department", + "evidence_url": "https://www.chulavistaca.gov/departments/police-department", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police Services | City of Chula Vista", + "url": "https://www.chulavistaca.gov/departments/police-department/services", + "evidence_url": "https://www.chulavistaca.gov/departments/police-department/services", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0639496", + "name": "Lake Forest", + "state": "CA", + "query": "\"Lake Forest\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Lake Forest | Orange County California - Sheriff's Department", + "url": "https://ocsheriff.gov/patrol-areas/lake-forest", + "evidence_url": "https://ocsheriff.gov/patrol-areas/lake-forest", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "| Orange County California - Sheriff's Department", + "url": "https://www.ocsheriff.gov/node", + "evidence_url": "https://www.ocsheriff.gov/node", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police Services | Lake Forest, CA - Official Website", + "url": "https://city-lakeforest.com/298/Police-Services.html", + "evidence_url": "https://city-lakeforest.com/298/Police-Services.html", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0648256", + "name": "Mission Viejo", + "state": "CA", + "query": "\"Mission Viejo\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Crime Alerts | City of Mission Viejo", + "url": "https://www.missionviejo.gov/category/crime-alerts", + "evidence_url": "https://www.missionviejo.gov/category/crime-alerts", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "incident-log", + "title": "Crime Alerts/News | City of Mission Viejo", + "url": "https://www.missionviejo.gov/departments/police-services/alert-bulletin-resources/crime-alertsnews", + "evidence_url": "https://www.missionviejo.gov/departments/police-services/alert-bulletin-resources/crime-alertsnews", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police Services | City of Mission Viejo", + "url": "https://www.missionviejo.gov/departments/police-services", + "evidence_url": "https://www.missionviejo.gov/departments/police-services", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0639248", + "name": "Laguna Niguel", + "state": "CA", + "query": "\"Laguna Niguel\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Laguna Niguel | Orange County California - Sheriff's Department", + "url": "https://www.ocsheriff.gov/patrol-areas/laguna-niguel", + "evidence_url": "https://www.ocsheriff.gov/patrol-areas/laguna-niguel", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Patrol Areas | Orange County California - Sheriff's Department", + "url": "https://www.ocsheriff.gov/patrol-areas", + "evidence_url": "https://www.ocsheriff.gov/patrol-areas", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Laguna Niguel Police Services | Orange County California - Sheriff's Department", + "url": "https://www.ocsheriff.gov/location/laguna-niguel-police-services", + "evidence_url": "https://www.ocsheriff.gov/location/laguna-niguel-police-services", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0687042", + "name": "Yucaipa", + "state": "CA", + "query": "\"Yucaipa\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Yucaipa \u2013 San Bernardino County Sheriff's Department", + "url": "https://wp.sbcounty.gov/sheriff/patrol-stations/yucaipa/", + "evidence_url": "https://wp.sbcounty.gov/sheriff/patrol-stations/yucaipa/", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "2024 ANNUAL CRIME REPORT", + "url": "https://wp.sbcounty.gov/sheriff/wp-content/uploads/sites/17/2024-Annual-Crime-Report.pdf", + "evidence_url": "https://wp.sbcounty.gov/sheriff/wp-content/uploads/sites/17/2024-Annual-Crime-Report.pdf", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Yucaipa Police Department \u2013 Dawn Rowe Third District Supervisor", + "url": "https://bosd3.sbcounty.gov/venue/yucaipa-police-department/", + "evidence_url": "https://bosd3.sbcounty.gov/venue/yucaipa-police-department/", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0656700", + "name": "Perris", + "state": "CA", + "query": "\"Perris\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Police | City of Perris, CA", + "url": "https://www.cityofperris.org/departments/public-safety/police", + "evidence_url": "https://www.cityofperris.org/departments/public-safety/police", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Public Safety | City of Perris, CA", + "url": "https://www.cityofperris.org/departments/public-safety", + "evidence_url": "https://www.cityofperris.org/departments/public-safety", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0665084", + "name": "San Clemente", + "state": "CA", + "query": "\"San Clemente\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Public Safety Services | San Clemente, CA", + "url": "https://www.sanclemente.gov/239/Public-Safety-Services", + "evidence_url": "https://www.sanclemente.gov/239/Public-Safety-Services", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police Services (OCSD) | San Clemente, CA", + "url": "https://www.sanclemente.gov/301/Police-Services-OCSD", + "evidence_url": "https://www.sanclemente.gov/301/Police-Services-OCSD", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "San Clemente | Orange County California - Sheriff's Department", + "url": "https://ocsheriff.gov/patrol-areas/san-clemente", + "evidence_url": "https://ocsheriff.gov/patrol-areas/san-clemente", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0646842", + "name": "Menifee", + "state": "CA", + "query": "\"Menifee\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "MENIFEE POLICE DEPARTMENT", + "url": "https://www.cityofmenifee.us/Archive/ViewFile/Item/2998", + "evidence_url": "https://www.cityofmenifee.us/Archive/ViewFile/Item/2998", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0650076", + "name": "Murrieta", + "state": "CA", + "query": "\"Murrieta\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Police Activity Reports | Murrieta, CA", + "url": "https://www.murrietaca.gov/759/Police-Activity-Reports", + "evidence_url": "https://www.murrietaca.gov/759/Police-Activity-Reports", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "newsroom", + "title": "Murrieta Police Press Releases | Murrieta, CA", + "url": "https://www.murrietaca.gov/557/Murrieta-Police-Press-Releases", + "evidence_url": "https://www.murrietaca.gov/557/Murrieta-Police-Press-Releases", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police | Murrieta, CA", + "url": "https://mainstreet.murrietaca.gov/1409/Police", + "evidence_url": "https://mainstreet.murrietaca.gov/1409/Police", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0633182", + "name": "Hemet", + "state": "CA", + "query": "\"Hemet\" California police department official crime reports news alerts", + "channels": [], + "search_note": "Stay Up To Date - Hemet Police Department (https://hemetpd.com/stay-up-to-date)\n\ue200cite\ue202turn373search0\ue201 [wordlim: 200] Crawled: 2 months ago; Sign up wi" + }, + { + "id": "162:0678120", + "name": "Temecula", + "state": "CA", + "query": "\"Temecula\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Temecula Police Department | Temecula CA", + "url": "https://www.temeculaca.gov/196/Police", + "evidence_url": "https://www.temeculaca.gov/196/Police", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Public Safety | Temecula CA", + "url": "https://temeculaca.gov/publicsafety", + "evidence_url": "https://temeculaca.gov/publicsafety", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Temecula Police Stations | Temecula CA", + "url": "https://temeculaca.gov/228/Temecula-Police-Stations", + "evidence_url": "https://temeculaca.gov/228/Temecula-Police-Stations", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0653322", + "name": "Oceanside", + "state": "CA", + "query": "\"Oceanside\" California police department official crime reports news alerts", + "channels": [], + "search_note": "Press Releases | Oceanside, CA - Police (https://www.oceansidepolice.com/about-opd/press-releases)\n\ue200cite\ue202turn377search0\ue201 [wordlim: 200] Crawled: 2 day" + }, + { + "id": "162:0682996", + "name": "Vista", + "state": "CA", + "query": "\"Vista\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "newsroom", + "title": "News List | San Diego County Sheriff", + "url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-selectview-1/-arch-1", + "evidence_url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-selectview-1/-arch-1", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "newsroom", + "title": "News List | San Diego County Sheriff", + "url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-npage-3/-arch-1", + "evidence_url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-npage-3/-arch-1", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Vista Station | Patrol Stations | San Diego County Sheriff", + "url": "https://www.sdsheriff.gov/Home/Components/FacilityDirectory/FacilityDirectory/46/61", + "evidence_url": "https://www.sdsheriff.gov/Home/Components/FacilityDirectory/FacilityDirectory/46/61", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0611194", + "name": "Carlsbad", + "state": "CA", + "query": "\"Carlsbad\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Police Department | Carlsbad, CA", + "url": "https://www.carlsbadca.gov/Departments/police", + "evidence_url": "https://www.carlsbadca.gov/Departments/police", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police Department | Carlsbad, CA", + "url": "https://www.carlsbadca.gov/departments/police/-npage-3", + "evidence_url": "https://www.carlsbadca.gov/departments/police/-npage-3", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police Department | Carlsbad, CA", + "url": "https://www.carlsbadca.gov/departments/police/-npage-2", + "evidence_url": "https://www.carlsbadca.gov/departments/police/-npage-2", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0612048", + "name": "Cathedral City", + "state": "CA", + "query": "\"Cathedral City\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Online Services & Reporting | City of Cathedral City", + "url": "https://www.cathedralcity.gov/departments/police-department/online-services-reporting", + "evidence_url": "https://www.cathedralcity.gov/departments/police-department/online-services-reporting", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Services & Operations | City of Cathedral City", + "url": "https://www.cathedralcity.gov/departments/police-department/services-operations", + "evidence_url": "https://www.cathedralcity.gov/departments/police-department/services-operations", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police Department | City of Cathedral City", + "url": "https://www.cathedralcity.gov/departments/police-department", + "evidence_url": "https://www.cathedralcity.gov/departments/police-department", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0622678", + "name": "Encinitas", + "state": "CA", + "query": "\"Encinitas\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Sheriff Quarterly Presentation | City of Encinitas", + "url": "https://www.encinitasca.gov/government/departments/public-safety/sheriff-quarterly-presentation", + "evidence_url": "https://www.encinitasca.gov/government/departments/public-safety/sheriff-quarterly-presentation", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "newsroom", + "title": "News List | San Diego County Sheriff", + "url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-npage-4", + "evidence_url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-npage-4", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "newsroom", + "title": "News List | San Diego County Sheriff", + "url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-npage-3/-arch-1", + "evidence_url": "https://www.sdsheriff.gov/bureaus/media-relations/news-list/-npage-3/-arch-1", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0658072", + "name": "Pomona", + "state": "CA", + "query": "\"Pomona\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Police Department | City of Pomona, CA", + "url": "https://www.pomonaca.gov/government/departments/police-department?locale=en", + "evidence_url": "https://www.pomonaca.gov/government/departments/police-department?locale=en", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Police Department | City of Pomona, CA", + "url": "https://www.pomonaca.gov/government/departments/police-department?locale=en"=", + "evidence_url": "https://www.pomonaca.gov/government/departments/police-department?locale=en"=", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "City Of Pomona Police Department", + "url": "https://locator.lacounty.gov/lac/Location/3175457/city-of-pomona-police-department", + "evidence_url": "https://locator.lacounty.gov/lac/Location/3175457/city-of-pomona-police-department", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0608786", + "name": "Buena Park", + "state": "CA", + "query": "\"Buena Park\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "OC Crime Stoppers | Orange County California - Sheriff's Department", + "url": "https://www.ocsheriff.gov/occrimestoppers", + "evidence_url": "https://www.ocsheriff.gov/occrimestoppers", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "department-page", + "title": "Buena Park Police Department | State of California - Department of Justice - Office of the Attorney General", + "url": "https://oag.ca.gov/fingerprints/locations/buena-park-police-department", + "evidence_url": "https://oag.ca.gov/fingerprints/locations/buena-park-police-department", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "newsroom", + "title": "California Department of Justice Investigating Buena Park Police Department Officer-Involved Shooting Under AB 1506 | State of California - Department of Justice - Office of the Attorney General", + "url": "https://www.oag.ca.gov/news/press-releases/california-department-justice-investigating-buena-park-police-department-officer", + "evidence_url": "https://www.oag.ca.gov/news/press-releases/california-department-justice-investigating-buena-park-police-department-officer", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + } + ] + }, + { + "id": "162:0619192", + "name": "Diamond Bar", + "state": "CA", + "query": "\"Diamond Bar\" California police department official crime reports news alerts", + "channels": [ + { + "kind": "department-page", + "title": "Public Safety | Diamond Bar, CA", + "url": "https://www.diamondbarca.gov/189/Public-Safety", + "evidence_url": "https://www.diamondbarca.gov/189/Public-Safety", + "verification": "search-discovered-primary-source; review-jurisdiction", + "access": "search-readable; automation-unverified" + }, + { + "kind": "newsroom", + "title": "| Place | Population | Sources and evidence |
|---|
“Reviewed” identifies sources checked during this research. Automated links and indexed results remain candidates for checking department identity, jurisdiction, subject matter, and freshness. RSS calendars, empty feeds, single announcements, and alert signup pages are recorded separately. X and other social account URLs are not working ingestion endpoints.
+Files: channels.csv · inventory.json · summary.json · validation.json
+""" + state = summary.get("state", "CA") + minimum = summary.get("population_min_exclusive", 50000) + page = page.replace("U.S. municipal governments with more than 20,000 residents", f"{state} municipal governments with more than {minimum:,} residents") + page = page.replace("All U.S. governments", "All cities in this state").replace("in the national inventory", "in this state inventory") + return page.replace("DATA", data) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--validate", action="store_true") + parser.add_argument("--state", default="CA") + parser.add_argument("--min-population", type=int, default=50000) + args = parser.parse_args() + inventory = json.loads((OUT / "inventory.json").read_text()) + rows = inventory["places"] + rows = [r for r in rows if (args.state.lower() == "all" or r["state"] == args.state.upper()) and r["population"] > args.min_population] + by_id = {r["id"]: r for r in rows} + searches_path = OUT / "indexed-searches.json" + if searches_path.exists(): + for result in json.loads(searches_path.read_text()): + if result["id"] in by_id: + by_id[result["id"]]["channels"].extend(result["channels"]) + by_id[result["id"]]["indexed_search_query"] = result["query"] + nixle_path = OUT / "nixle-directories.json" + if nixle_path.exists(): + for result in json.loads(nixle_path.read_text()): + if result["id"] in by_id: + by_id[result["id"]]["channels"] = [c for c in by_id[result["id"]]["channels"] if not c.get("verification", "").startswith("certified-publisher-in-city-directory")] + by_id[result["id"]]["channels"].extend(result["channels"]) + overrides = json.loads((ROOT / "docs/data/police-source-overrides.json").read_text()) + for row in rows: + row["channels"].extend(overrides.get(row["state"] + ":" + row["name"], {}).get("channels", [])) + row["channels"] = clean_channels(row["channels"]) + validations_path = OUT / "validation.json" + validations = json.loads(validations_path.read_text()) if validations_path.exists() else {} + scope_urls = {c["url"] for row in rows for c in row["channels"]} + validations = {u: v for u, v in validations.items() if u in scope_urls} + if args.validate: + todo = {c["url"]: c for row in rows for c in row["channels"] if c["kind"] in ("nixle", "feed-candidate") and c["url"] not in validations} + print(f"Checking {len(todo)} public archive/feed URLs", flush=True) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + for i, audit in enumerate(executor.map(validate_channel, todo.values()), 1): + validations[audit["url"]] = audit + if i % 20 == 0: + validations_path.write_text(json.dumps(validations, indent=2) + "\n") + print(f"Validated {i}/{len(todo)}", flush=True) + validations_path.write_text(json.dumps(validations, indent=2) + "\n") + for row in rows: + for channel in row["channels"]: + if channel["url"] in validations: + channel["validation"] = validations[channel["url"]] + channel["access"] = channel["validation"]["access"] + useful = [c for c in row["channels"] if c["kind"] not in ("department-page", "feed-directory", "calendar-feed", "alert-directory", "announcement")] + row["status"] = "channels-found" if useful else "department-found" if row["channels"] else "unresolved" + summary = discovery.export(rows, OUT, args.min_population, args.state.upper()) + reviewed = json.loads((ROOT / "docs/data/police-ingestion-ca.json").read_text()) if args.state.upper() == "CA" else {} + summary["reviewed_ingestion_cities"] = sum(r["name"] in reviewed for r in rows) + summary["state"] = args.state.upper() + summary["population_min_exclusive"] = args.min_population + summary["unique_channel_urls"] = len({c["url"] for row in rows for c in row["channels"]}) + summary["local_governments_within_100km"] = sum(row["distance_from_link_km"] is not None and row["distance_from_link_km"] <= 100 for row in rows) + summary["public_archives_or_feeds_with_items"] = sum(v["access"] in ("public-nixle-archive", "public-feed-with-items") for v in validations.values()) + (OUT / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + (OUT / "index.html").write_text(render(rows, summary)) + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/check-police-feeds.js b/scripts/check-police-feeds.js new file mode 100644 index 0000000..f524b26 --- /dev/null +++ b/scripts/check-police-feeds.js @@ -0,0 +1,45 @@ +/** Live, bounded smoke check: one detail per city, no database writes or paid APIs. + * bun scripts/check-police-feeds.js [city-slug] + */ +import { policeUpdates } from '../packages/adapters/src/police-updates.js'; +import { makeHttp } from '../packages/core/src/http.js'; + +const http = makeHttp({ userAgent: 'NicheDBPoliceSources/1.0 (+https://nichedb.dev)' }); +const checks = []; +for (const source of policeUpdates.defaultSources) { + if (process.argv[2] && source.config.city !== process.argv[2]) continue; + try { + const result = await policeUpdates.pull({ + config: source.config, + http, + budget: 1, + deadline: Date.now() + 60000, + previous: async () => new Map(), + }); + const latest = result.items[0]; + if (!latest?.publishedAt) throw new Error('No dated announcement returned'); + checks.push({ + city: source.config.city, + ok: true, + items: result.items.length, + sampleUrl: latest.url, + publishedAt: latest.publishedAt, + checkedAt: new Date().toISOString(), + }); + console.log(`${source.config.city}: ${result.items.length} items; ${latest.publishedAt}`); + } catch (error) { + checks.push({ + city: source.config.city, + ok: false, + error: error.message, + checkedAt: new Date().toISOString(), + }); + console.error(`${source.config.city}: ${error.message}`); + } + await Bun.sleep(500); +} +await Bun.write( + new URL('../docs/data/police-sources/ingestion-check.json', import.meta.url), + `${JSON.stringify(checks, null, 2)}\n`, +); +if (!checks.length || checks.some((c) => !c.ok)) process.exitCode = 1; diff --git a/scripts/discover-police-nixle.py b/scripts/discover-police-nixle.py new file mode 100644 index 0000000..f7207d7 --- /dev/null +++ b/scripts/discover-police-nixle.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Find certified police publishers in Nixle's public city directories. + +Run after discover-police-sources.py. Shares its robots checks and per-host +throttle; never guesses account handles. Resumes from the recorded results. +County and station publishers remain candidates until their jurisdiction is +reviewed. A city directory also lists nearby agencies: do not assign those to +the city merely because they appear in the directory. +""" +import concurrent.futures +import html +import importlib.util +import json +import re +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "docs/data/police-sources" +spec = importlib.util.spec_from_file_location("discovery", ROOT / "scripts/discover-police-sources.py") +discovery = importlib.util.module_from_spec(spec) +spec.loader.exec_module(discovery) + + +def city_publisher(label, city): + label = re.sub(r"^(?:SBSD\s*-\s*|LASD\s*-\s*|City of\s+)", "", label, flags=re.I) + # Chino Hills is not Chino, and Madera County is not Madera city. + return bool(re.match(re.escape(city) + r"(?:,\s*CA)?\s+(?:Police\b|Department of Public Safety\b|Sheriff\b|Station\b)", label, re.I)) + + +def publishers(body, city): + found = {} + for path, label in re.findall(r'