From e227517aa1c0fc6487e184e6ac7acb1517222b22 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 13 Sep 2026 20:14:41 +0000 Subject: [PATCH] Add Ruuster saved-search ingestion and back off MusicBrainz throttling --- README.md | 38 +++- packages/adapters/src/index.js | 2 + packages/adapters/src/musicbrainz.js | 14 +- packages/adapters/src/ruuster.js | 303 ++++++++++++++++++++++++++ packages/core/src/http.js | 6 +- packages/core/src/musicbrainz-http.js | 56 +++++ packages/core/src/seed.js | 8 + scripts/scrape-ruuster.js | 32 +++ test/musicbrainz-http.test.js | 128 +++++++++++ test/ruuster.test.js | 233 ++++++++++++++++++++ 10 files changed, 812 insertions(+), 8 deletions(-) create mode 100644 packages/adapters/src/ruuster.js create mode 100644 packages/core/src/musicbrainz-http.js create mode 100644 scripts/scrape-ruuster.js create mode 100644 test/musicbrainz-http.test.js create mode 100644 test/ruuster.test.js diff --git a/README.md b/README.md index 481b0f6..e1614c3 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Adapters are one file each in `packages/adapters/src`. One hundred and twenty-ei | crypto | `coingecko-assets`, `crypto-pairs` | keyless (`COINGECKO_API_KEY` optional; `CRYPTO_PROXY_URL` for Binance.US from a datacenter) | | crime | `socrata-crime`, `uk-police-crime`, `fbi-crime-estimates` | FBI only (free api.data.gov key) | | public-money | `usaspending-awards`, `ocds-tenders`, `ted-notices` | no | -| housing | `uk-land-registry`, `freddie-mac-rates`, `building-permits` | no | +| housing | `uk-land-registry`, `freddie-mac-rates`, `building-permits`, `ruuster` (agent saved searches) | no | | jobs | `bls-series`, `eurostat`, `warn-layoffs`, `agenticjobs` | no | | ai-incidents | `rogue-ai-incidents`, `rogue-ai-research`, `aiid-reports` | no | | news | `newsfeed`, `gdelt`, `rssamplifier`, `brisk`, `news-channels` | no | @@ -99,6 +99,42 @@ Ten of the sites we run publish a public feed or API of their own, and each is r saasrow's `/api/v1/listings` is per-account and needs a key, so only the public products directory is read. A submission aiornot carries on more than one feed is stored once and tagged with each feed. tsbb's cross-board `/api/v1/latest` does not say which forum a topic is in, which is why the walk is per forum. +## Ruuster housing searches + +`ruuster-san-jose-homes` reads the Real Estate Experts / Talar Davoudi saved +search hourly and feeds `/f/san-jose-homes`. The search asks for San Jose houses +that are Active or Coming Soon, with 2+ bedrooms, 1+ bathrooms, 750+ square feet, +a 4,500+ square foot lot and a build year of 2000 or later. Ruuster calls the +lot filter `lotSizeAcresMin` but accepts **square feet** there; property records +return **acres**, and NicheDB keeps both units. Results and dimensions are +upstream observations; conflicting MLS values are not silently corrected. + +Create another `ruuster` source with its `savedSearchUrl`, `pages` (20 by +default, ten records per page) and `currency` (USD or CAD). Tracking parameters +are removed and repeated status filters are combined. Public detail requests +add photos, property facts and MLS attribution. Syndicated copies are joined +by MLS listing number and address. Hidden-address and deleted records are +omitted. The worker's detail budget and deadline are respected, with a cursor +for unfinished pages; completed walks start from page one on the next refresh. + +Items retain the last status observed while matching the search. A property +disappearing from a search is not evidence that it sold, and this source does +not revisit every historical listing after it leaves the search. + +Run `bun run ingest ruuster-san-jose-homes` against the configured database, or +export without a database with `bun scripts/scrape-ruuster.js > properties.json`. +The export also accepts a saved-search URL and an optional new output filename +as positional arguments. Ruuster's terms restrict automated access without +written consent, and its IDX notice limits reuse; deploying this adapter does +not grant redistribution rights: https://realestateexperts.ruuster.com/terms-of-service. + +MusicBrainz web-service requests share a process-wide queue with at least 1.1 +seconds between requests. A 429 or 503 retries up to three times with 5, 15 and +45 second backoff, honoring longer `Retry-After` values, including HTTP dates. +A zero retry hint cannot cause an immediate retry. This queue covers one +worker process; deployments sharing an egress IP across multiple processes +need coordination across those processes as well. + ## Enrichment After ingest, every item is enriched by the enrichers that apply to it (`packages/enrichers/src`), and the results live on the item under `enrichment.`: diff --git a/packages/adapters/src/index.js b/packages/adapters/src/index.js index 0dd58a2..9beed16 100644 --- a/packages/adapters/src/index.js +++ b/packages/adapters/src/index.js @@ -96,6 +96,7 @@ import { pypi } from './pypi.js'; import { redditDeals } from './redditdeals.js'; import { rogueIncidents, rogueResearch } from './rogueaitracker.js'; import { rssamplifier } from './rssamplifier.js'; +import { ruuster } from './ruuster.js'; import { saasrow } from './saasrow.js'; import { scalewayInstances } from './scaleway.js'; import { scannerDirectory } from './scanners.js'; @@ -220,6 +221,7 @@ export const ADAPTERS = [ landRegistrySales, freddieMacRates, buildingPermits, + ruuster, fueleconomyCatalog, nhtsaRecalls, nhtsaComplaints, diff --git a/packages/adapters/src/musicbrainz.js b/packages/adapters/src/musicbrainz.js index f1d440d..4be3439 100644 --- a/packages/adapters/src/musicbrainz.js +++ b/packages/adapters/src/musicbrainz.js @@ -78,7 +78,7 @@ export const musicbrainz = defineAdapter({ ); for (const r of res.releases ?? []) { let cover = covers[r.id] ?? null; - if (cover === null && spent < budget) { + if (cover === null && spent < budget && Date.now() + 10_000 < deadline) { spent++; const head = await http .request(`https://coverartarchive.org/release/${r.id}/front-250`, { @@ -86,11 +86,13 @@ export const musicbrainz = defineAdapter({ timeoutMs: 10_000, }) .catch(() => null); - cover = - head && (head.ok || head.status === 307) - ? `https://coverartarchive.org/release/${r.id}/front-250` - : false; - covers[r.id] = cover; + if (head && (head.ok || head.status === 307)) { + cover = `https://coverartarchive.org/release/${r.id}/front-250`; + covers[r.id] = cover; + } else if (head?.status === 404) { + cover = false; + covers[r.id] = false; + } } items.push(toItem(r, cover || null)); } diff --git a/packages/adapters/src/ruuster.js b/packages/adapters/src/ruuster.js new file mode 100644 index 0000000..96ffbff --- /dev/null +++ b/packages/adapters/src/ruuster.js @@ -0,0 +1,303 @@ +import { createHash } from 'node:crypto'; +import { defineAdapter, looseDate, slugify, stripHtml } from '@nichedb/core/adapter'; + +// These are the saved search's values, including Ruuster's misleadingly named +// lotSizeAcresMin: the search accepts square feet, while records return acres. +const sanJoseParams = new URLSearchParams({ + address: JSON.stringify({ + value: 'ChIJ9T_5iuTKj4ARe3GfygqMnbk', + label: 'San Jose, CA', + data: [ + { offset: 0, value: 'San Jose' }, + { offset: 10, value: 'CA' }, + ], + source: 'googleAutocomplete', + }), + priceMin: '0', + priceMax: '0', + type: 'house', + bedroomsMin: '2', + bathroomsMin: '1', + squareMin: '750', + lotSizeAcresMin: '4500', + yearBuiltMin: '2000', +}); +sanJoseParams.append('status', 'Active'); +sanJoseParams.append('status', 'ComingSoon'); +export const SAN_JOSE_SEARCH = `https://realestateexperts.ruuster.com/agent/talar-davoudi/listings?${sanJoseParams}`; + +const UI_PARAMS = new Set([ + 'timestamp', + 'view', + 'page', + 'slug', + 'hash', + 'forcedRegistration', + 'registrationPopupAfter', + 'redirectUrl', + 'customCrmTags', + 'isEmbedded', + 'isLikesView', + 'shouldFetchFilterOptions', + 'isNonBoundsAddressChanged', + 'isNeededSendToCRM', + 'utm_source', + 'utm_medium', + 'utm_campaign', + 'utm_content', + 'utm_term', +]); + +const clean = (v) => (typeof v === 'string' ? stripHtml(v) || null : null); +const numeric = (v) => + v !== null && v !== undefined && v !== '' && Number.isFinite(Number(v)) ? Number(v) : null; +const bounded = (v, fallback, min, max) => { + const n = Number(v); + return Number.isFinite(n) ? Math.min(max, Math.max(min, Math.floor(n))) : fallback; +}; +const digest = (v) => createHash('sha256').update(v).digest('hex'); +const urlOrNull = (v) => (typeof v === 'string' && /^https?:\/\//i.test(v) ? v : null); + +/** Turn an agent's browser search into the same public GET the page uses. */ +export function parseSearch(value) { + let url; + try { + url = new URL(value); + } catch { + throw new Error('ruuster needs a savedSearchUrl from a Ruuster agent listing search'); + } + const match = url.pathname.match(/^\/agent\/([a-z0-9-]+)\/listings\/?$/i); + if ( + url.protocol !== 'https:' || + !/^[a-z0-9-]+\.ruuster\.com$/i.test(url.hostname) || + url.username || + url.password || + url.port || + !match + ) { + throw new Error('savedSearchUrl must be https://.ruuster.com/agent//listings'); + } + const params = new URLSearchParams(); + for (const [key, value] of url.searchParams) { + if (UI_PARAMS.has(key) || key.startsWith('utm_')) continue; + if (!params.getAll(key).includes(value)) params.append(key, value); + } + params.sort(); + const searchUrl = `${url.origin}/agent/${match[1]}/listings?${params}`; + params.set('slug', match[1]); + params.set('shouldFetchFilterOptions', 'false'); + params.set('isNeededSendToCRM', 'false'); + return { origin: url.origin, agentSlug: match[1], params, searchUrl, key: digest(searchUrl) }; +} + +// A listing is syndicated through several MLS feeds with different Ruuster +// UUIDs. The MLS listing number plus address joins those copies without joining +// unrelated properties whose local MLS happens to reuse the same number. +export function listingKey(row) { + const number = clean(row?.mlsRecordId); + // Some feeds omit the street suffix ("1674 Husted" / "1674 Husted Ave"). + // Keep the street name, number and unit, and join only within the same MLS id. + const street = clean(row?.firstAddress) + ?.toLowerCase() + .replace( + /\b(?:avenue|ave|street|st|road|rd|drive|dr|court|ct|lane|ln|way|boulevard|blvd|place|pl|circle|cir|terrace|ter)\.?\b(?=\s*(?:#|unit\b|apt\b|$))/g, + '', + ); + const address = [street, clean(row?.secondAddress)].filter(Boolean).join(' '); + if (number && address) { + const normal = address.toLowerCase().replace(/[^a-z0-9]/g, ''); + return `mls:${number}:${digest(normal).slice(0, 20)}`; + } + return clean(row?.id) ? `ruuster:${row.id}` : null; +} + +export function toItem(row, { origin, agentSlug, currency = 'USD' }, externalId = listingKey(row)) { + if (!row?.id || !externalId || row.isDeleted || row.isHideAddress) return null; + const address = [clean(row.firstAddress), clean(row.secondAddress)].filter(Boolean).join(', '); + if (!address) return null; + const photos = [ + ...new Set( + (Array.isArray(row.media) ? row.media : []) + .map((m) => urlOrNull(typeof m === 'string' ? m : m?.imagePath)) + .filter(Boolean), + ), + ].slice(0, 100); + const city = clean(row.city); + const state = clean(row.state); + const status = clean(row.status); + const propertyType = clean(row.unifiedSubtype) ?? clean(row.subType) ?? clean(row.type); + const lotSizeAcres = numeric(row.lotSizeAcres); + const coordinates = row.location?.coordinates; + // Ruuster's response uses [latitude, longitude], despite calling this a Point. + const latitude = Array.isArray(coordinates) ? numeric(coordinates[0]) : null; + const longitude = Array.isArray(coordinates) ? numeric(coordinates[1]) : null; + const located = + latitude !== null && + longitude !== null && + Math.abs(latitude) <= 90 && + Math.abs(longitude) <= 180; + const attribution = row.mlsComplianceInfo ?? {}; + const date = looseDate(row.onMarketTimestamp ?? row.originalEntryTimestamp ?? row.onMarketDate); + + return { + externalId, + kind: 'property-listing', + title: address, + summary: clean(row.description), + url: `${origin}/agent/${agentSlug}/listings/${encodeURIComponent(row.id)}`, + imageUrl: photos[0] ?? null, + ...date, + tags: [ + 'housing', + 'ruuster', + city ? `city:${slugify(city)}` : null, + state ? `state:${slugify(state)}` : null, + status ? `status:${slugify(status)}` : null, + propertyType ? `property-type:${slugify(propertyType)}` : null, + ].filter(Boolean), + data: { + provider: 'ruuster', + ruusterId: row.id, + mlsRecordId: clean(row.mlsRecordId), + mlsId: clean(row.mlsId), + address, + city, + state, + postalCode: clean(row.postalCode), + latitude: located ? latitude : null, + longitude: located ? longitude : null, + price: numeric(row.price), + previousPrice: numeric(row.oldPrice), + currency, + bedrooms: numeric(row.bedrooms), + bathrooms: numeric(row.bathrooms), + squareFeet: numeric(row.square), + lotSizeAcres, + lotSizeSquareFeet: lotSizeAcres === null ? null : Math.round(lotSizeAcres * 43560), + yearBuilt: numeric(row.yearBuilt), + propertyType, + status, + mlsStatus: clean(row.mlsStatus), + photos, + isRental: row.isRental === true, + newConstruction: row.newConstruction === true, + garageSpaces: numeric(row.garageSpaces), + parkingSpaces: numeric(row.parkingSpaces), + associationFee: numeric(row.associationFee), + daysOnMarket: numeric(row.daysOnMarket), + updatedAt: clean(row.modificationTimestamp), + statusChangedAt: clean(row.statusChangeTimestamp), + priceChangedAt: clean(row.priceChangeTimestamp), + openHouses: (Array.isArray(row.openHouse) ? row.openHouse : []).map((h) => ({ + startsAt: clean(h.OpenHouseStartTime), + endsAt: clean(h.OpenHouseEndTime), + })), + attribution: { + agent: clean(attribution.agentName), + brokerage: clean(attribution.brokerage) ?? clean(row.listOfficeName), + mls: clean(attribution.mlsName) ?? clean(row.mls?.originalName), + source: clean(attribution.mlsSource), + listingNumber: clean(attribution.listingNumber) ?? clean(row.mlsRecordId), + contact: clean(attribution.attributionContact) ?? clean(row.AttributionContact), + }, + }, + }; +} + +export const ruuster = defineAdapter({ + name: 'ruuster', + title: 'Ruuster property listings', + collection: 'housing', + description: + 'Properties matching a Ruuster agent saved search, with prices, addresses, photos, property details and MLS attribution. Syndicated copies are combined by MLS listing number and address.', + docs: 'https://realestateexperts.ruuster.com/agent/talar-davoudi/listings', + kinds: ['property-listing'], + cadenceMinutes: 60, + configFields: [ + { key: 'savedSearchUrl', label: 'Saved search URL', type: 'text', required: true }, + { + key: 'pages', + label: 'Pages per run', + type: 'number', + help: 'Ten upstream records per page. Interrupted walks resume next run.', + }, + { key: 'currency', label: 'Listing currency', type: 'select', options: ['USD', 'CAD'] }, + ], + defaults: { pages: 20, currency: 'USD' }, + defaultSources: [ + { + slug: 'ruuster-san-jose-homes', + name: 'San Jose homes: Ruuster saved search', + config: { savedSearchUrl: SAN_JOSE_SEARCH, pages: 20, currency: 'USD' }, + }, + ], + async pull({ config, cursor = {}, http, budget = 150, deadline = Infinity, log = () => {} }) { + const search = parseSearch(config.savedSearchUrl); + const currency = config.currency ?? 'USD'; + if (!['USD', 'CAD'].includes(currency)) throw new Error('ruuster currency must be USD or CAD'); + const pages = bounded(config.pages ?? 20, 20, 1, 100); + const detailBudget = bounded(budget, 150, 0, 1000); + const resume = cursor.searchKey === search.key; + let page = resume ? bounded(cursor.page ?? 1, 1, 1, 100000) : 1; + let offset = resume ? bounded(cursor.offset ?? 0, 0, 0, 10) : 0; + const seen = new Set(resume && Array.isArray(cursor.seen) ? cursor.seen : []); + const items = []; + let total = null; + let details = 0; + let lastRequest = 0; + let complete = false; + + const get = async (url) => { + const wait = Math.max(0, lastRequest + 250 - Date.now()); + if (wait) await new Promise((resolve) => setTimeout(resolve, wait)); + lastRequest = Date.now(); + return http.json(url, { timeoutMs: Math.min(20000, Math.max(1, deadline - Date.now())) }); + }; + const hasTime = () => Date.now() + 1000 < deadline; + + for (let n = 0; n < pages && hasTime(); n++) { + if (details >= detailBudget) break; + search.params.set('page', String(page)); + const doc = await get(`${search.origin}/api/listings?${search.params}`); + if (!Array.isArray(doc?.records) || !Number.isInteger(doc.totalCount) || doc.totalCount < 0) { + throw new Error('ruuster returned an invalid listing page'); + } + total = doc.totalCount; + const rows = doc.records; + // Ruuster currently fixes the page size at ten; failing explicitly is + // safer than silently skipping inventory if its pagination changes. + if (rows.length > 10 || (rows.length < 10 && (page - 1) * 10 + rows.length < total)) { + throw new Error('ruuster returned an inconsistent listing page'); + } + while (offset < rows.length) { + const card = rows[offset]; + const key = listingKey(card); + if (!card?.id || !key) throw new Error('ruuster returned a listing without an identity'); + if (!seen.has(key)) { + if (details >= detailBudget || !hasTime()) break; + const detail = await get(`${search.origin}/api/listings/${encodeURIComponent(card.id)}`); + if (!detail || detail.id !== card.id) + throw new Error('ruuster returned invalid listing details'); + details++; + const item = toItem({ ...card, ...detail }, { ...search, currency }, key); + if (item) items.push(item); + seen.add(key); + } + offset++; + } + if (offset < rows.length) break; + complete = rows.length === 0 || (page - 1) * 10 + rows.length >= total; + page++; + offset = 0; + if (complete) break; + } + const note = `${items.length} properties fetched; ${seen.size} distinct listings in this walk${total === null ? '' : ` / ${total} upstream records`}; ${complete ? 'complete' : `resume page ${page}, row ${offset + 1}`}`; + log(note); + return { + items, + cursor: complete ? {} : { searchKey: search.key, page, offset, seen: [...seen] }, + note, + ...(complete ? {} : { nextInMinutes: 1 }), + }; + }, +}); diff --git a/packages/core/src/http.js b/packages/core/src/http.js index 814d309..cd24ede 100644 --- a/packages/core/src/http.js +++ b/packages/core/src/http.js @@ -9,6 +9,7 @@ import { mkdir, open, stat, unlink } from 'node:fs/promises'; import { dirname } from 'node:path'; +import { requestMusicBrainz, retryDelay } from './musicbrainz-http.js'; export function makeHttp({ userAgent, log = () => {} }) { async function request(url, { headers = {}, timeoutMs = 30_000, method = 'GET', body } = {}) { @@ -20,10 +21,13 @@ export function makeHttp({ userAgent, log = () => {} }) { signal: AbortSignal.timeout(timeoutMs), redirect: 'follow', }); + if (new URL(url).hostname === 'musicbrainz.org') return requestMusicBrainz(doFetch, log); let res = await doFetch(); if (res.status === 429 || res.status === 503) { - const wait = Math.min(Number(res.headers.get('retry-after') ?? 5) * 1000, 60_000); + const wait = retryDelay(res.headers.get('retry-after')); + if (wait > 60_000) return res; log(`${res.status} from ${new URL(url).host}, waiting ${wait}ms`); + await res.body?.cancel().catch(() => {}); await Bun.sleep(wait); res = await doFetch(); } diff --git a/packages/core/src/musicbrainz-http.js b/packages/core/src/musicbrainz-http.js new file mode 100644 index 0000000..a67317a --- /dev/null +++ b/packages/core/src/musicbrainz-http.js @@ -0,0 +1,56 @@ +const INTERVAL_MS = 1100; +const MAX_WAIT_MS = 60000; + +/** Retry-After may be seconds or an HTTP date. Zero never removes our backoff. */ +export function retryDelay(value, attempt = 0, now = Date.now()) { + const fallback = 5000 * 3 ** attempt; + const seconds = value?.trim() ? Number(value) : NaN; + const requested = Number.isFinite(seconds) + ? seconds * 1000 + : value + ? Date.parse(value) - now + : NaN; + return Number.isFinite(requested) ? Math.max(fallback, requested) : fallback; +} + +/** + * Shared by every HTTP client in the process, including manual source runs. + * MusicBrainz limits the IP, so a sleep inside each adapter alone is insufficient. + * Serialize retries too: concurrent callers must not start their own retry storms. + */ +export function createMusicBrainzRequest({ now = Date.now, sleep = (ms) => Bun.sleep(ms) } = {}) { + let tail = Promise.resolve(); + let readyAt = 0; + return (fetchOnce, log = () => {}) => { + const run = async () => { + for (let attempt = 0; attempt < 4; attempt++) { + const wait = Math.max(0, readyAt - now()); + if (wait > MAX_WAIT_MS) { + throw new Error( + `MusicBrainz requested a cooldown until ${new Date(readyAt).toISOString()}`, + ); + } + if (wait) await sleep(wait); + let res; + try { + res = await fetchOnce(); + } finally { + readyAt = now() + INTERVAL_MS; + } + if (res.status !== 429 && res.status !== 503) return res; + const delay = retryDelay(res.headers.get('retry-after'), attempt, now()); + readyAt = now() + delay; + // Long upstream cooldowns are respected without occupying a worker for + // minutes or retrying before the server said it was ready. + if (attempt === 3 || delay > MAX_WAIT_MS) return res; + await res.body?.cancel().catch(() => {}); + log(`${res.status} from musicbrainz.org, waiting ${delay}ms (retry ${attempt + 1}/3)`); + } + }; + const result = tail.then(run); + tail = result.catch(() => {}); + return result; + }; +} + +export const requestMusicBrainz = createMusicBrainzRequest(); diff --git a/packages/core/src/seed.js b/packages/core/src/seed.js index 330226c..731f75a 100644 --- a/packages/core/src/seed.js +++ b/packages/core/src/seed.js @@ -707,6 +707,14 @@ export const DEFAULT_FEEDS = [ /* Housing. The events first, then the series: a sale and a permit are things that happened, and an index is a summary of many of them. */ + { + collection: 'housing', + slug: 'san-jose-homes', + name: 'San Jose homes', + description: + 'Homes found by the Ruuster saved search: 2+ bedrooms, 1+ bathrooms, 750+ square feet, 4,500+ square foot lots, built in 2000 or later. Listings retain their last observed status.', + query: { sources: ['ruuster-san-jose-homes'], kinds: ['property-listing'] }, + }, { collection: 'housing', slug: 'property-sales', diff --git a/scripts/scrape-ruuster.js b/scripts/scrape-ruuster.js new file mode 100644 index 0000000..1961066 --- /dev/null +++ b/scripts/scrape-ruuster.js @@ -0,0 +1,32 @@ +import { writeFile } from 'node:fs/promises'; +import { ruuster, SAN_JOSE_SEARCH } from '../packages/adapters/src/ruuster.js'; +import { normaliseItem } from '../packages/core/src/adapter.js'; +import { makeHttp } from '../packages/core/src/http.js'; + +// Preview/export without a database. Normal ingestion uses: +// bun run ingest ruuster-san-jose-homes +const [savedSearchUrl = SAN_JOSE_SEARCH, output] = process.argv.slice(2); +const http = makeHttp({ userAgent: 'niche-db/0.26 (+https://nichedb.dev)', log: console.error }); +const items = new Map(); +let cursor = {}; +for (let run = 0; run < 100; run++) { + const result = await ruuster.pull({ + config: { ...ruuster.defaults, savedSearchUrl }, + cursor, + http, + budget: 150, + deadline: Date.now() + 5 * 60000, + log: console.error, + }); + for (const raw of result.items) { + const item = normaliseItem(raw); + if (item) items.set(item.externalId, item); + } + cursor = result.cursor; + if (!cursor.page) break; +} +if (cursor.page) throw new Error('Ruuster export did not finish within 100 runs'); +const json = `${JSON.stringify([...items.values()], null, 2)}\n`; +if (output) await writeFile(output, json, { flag: 'wx' }); +else process.stdout.write(json); +console.error(`Exported ${items.size} unique properties${output ? ` to ${output}` : ''}`); diff --git a/test/musicbrainz-http.test.js b/test/musicbrainz-http.test.js new file mode 100644 index 0000000..368939f --- /dev/null +++ b/test/musicbrainz-http.test.js @@ -0,0 +1,128 @@ +import { describe, expect, test } from 'bun:test'; +import { musicbrainz } from '../packages/adapters/src/musicbrainz.js'; +import { createMusicBrainzRequest, retryDelay } from '../packages/core/src/musicbrainz-http.js'; + +function clock() { + let time = 0; + const sleeps = []; + return { + now: () => time, + sleeps, + sleep: async (ms) => { + sleeps.push(ms); + time += ms; + }, + }; +} + +describe('MusicBrainz pacing and retries', () => { + test('zero, missing, malformed and past Retry-After values still back off', () => { + for (const header of ['0', '', null, 'invalid', '-2', 'Thu, 01 Jan 1970 00:00:00 GMT']) { + expect(retryDelay(header, 0, 1000)).toBe(5000); + } + expect(retryDelay('0', 1)).toBe(15000); + expect(retryDelay('0', 2)).toBe(45000); + expect(retryDelay('30')).toBe(30000); + expect(retryDelay('Thu, 01 Jan 1970 00:01:00 GMT', 0, 1000)).toBe(59000); + }); + test('concurrent clients share one request pace', async () => { + const time = clock(); + const request = createMusicBrainzRequest(time); + const starts = []; + const fetch = async () => { + starts.push(time.now()); + return new Response('{}'); + }; + await Promise.all([request(fetch), request(fetch), request(fetch)]); + expect(starts).toEqual([0, 1100, 2200]); + }); + test('retries 503s with increasing waits and returns the successful response', async () => { + const time = clock(); + const request = createMusicBrainzRequest(time); + let calls = 0; + const result = await request( + async () => + new Response('{}', { + status: ++calls < 4 ? 503 : 200, + headers: { 'Retry-After': '0' }, + }), + ); + expect(calls).toBe(4); + expect(result.status).toBe(200); + expect(time.sleeps).toEqual([5000, 15000, 45000]); + }); + test('a second caller waits through the first caller’s cooldown', async () => { + const time = clock(); + const request = createMusicBrainzRequest(time); + const starts = []; + let calls = 0; + const first = request(async () => { + starts.push(['first', time.now()]); + return new Response('{}', { status: ++calls === 1 ? 429 : 200 }); + }); + const second = request(async () => { + starts.push(['second', time.now()]); + return new Response('{}'); + }); + await Promise.all([first, second]); + expect(starts).toEqual([ + ['first', 0], + ['first', 5000], + ['second', 6100], + ]); + }); + test('does not retry permanent errors and a network failure does not poison the queue', async () => { + const request = createMusicBrainzRequest(clock()); + let calls = 0; + expect( + ( + await request(async () => { + calls++; + return new Response('', { status: 400 }); + }) + ).status, + ).toBe(400); + expect(calls).toBe(1); + await expect( + request(async () => { + throw new Error('network'); + }), + ).rejects.toThrow('network'); + expect((await request(async () => new Response('{}'))).ok).toBe(true); + }); + test('long Retry-After values stop the retry instead of retrying early', async () => { + const time = clock(); + const request = createMusicBrainzRequest(time); + let calls = 0; + const fetch = async () => { + calls++; + return new Response('', { status: 503, headers: { 'Retry-After': '600' } }); + }; + expect((await request(fetch)).status).toBe(503); + await expect(request(fetch)).rejects.toThrow('cooldown'); + expect(calls).toBe(1); + expect(time.sleeps).toEqual([]); + }); +}); + +test('temporary cover-art failures are not cached as permanently missing', async () => { + const ctx = { + config: { days: 90, pages: 1 }, + cursor: {}, + budget: 2, + deadline: Date.now() + 60000, + log: () => {}, + http: { + json: async () => ({ + releases: [ + { id: 'transient', title: 'Example' }, + { id: 'missing', title: 'Another' }, + ], + }), + request: async (url) => new Response('', { status: url.includes('transient') ? 503 : 404 }), + }, + }; + const result = await musicbrainz.pull(ctx); + expect(result.items).toHaveLength(2); + expect(result.cursor.covers).toEqual({ missing: false }); +}); diff --git a/test/ruuster.test.js b/test/ruuster.test.js new file mode 100644 index 0000000..1b1e27c --- /dev/null +++ b/test/ruuster.test.js @@ -0,0 +1,233 @@ +import { describe, expect, test } from 'bun:test'; +import { adapterByName } from '../packages/adapters/src/index.js'; +import { + listingKey, + parseSearch, + ruuster, + SAN_JOSE_SEARCH, + toItem, +} from '../packages/adapters/src/ruuster.js'; +import { normaliseItem } from '../packages/core/src/adapter.js'; + +const search = parseSearch(SAN_JOSE_SEARCH); +const property = (n = 1, extra = {}) => ({ + id: `uuid-${n}`, + mlsRecordId: `ML${n}`, + mlsId: 'mls-a', + firstAddress: `${n} Example Lane`, + secondAddress: 'San Jose, CA 95118', + city: 'San Jose', + state: 'CA', + postalCode: '95118', + bedrooms: 3, + bathrooms: 2, + square: 1600, + lotSizeAcres: 0.25, + yearBuilt: 2005, + price: 1500000, + status: 'Active', + unifiedSubtype: 'house', + onMarketTimestamp: '2026-09-10T15:19:04.000Z', + location: { coordinates: [37.26, -121.88] }, + media: [{ imagePath: 'https://images.example/home.jpg' }], + mlsComplianceInfo: { + agentName: 'Example Agent', + brokerage: 'Example Realty', + mlsName: 'Example MLS', + }, + ...extra, +}); +function context(rows, overrides = {}) { + const calls = []; + return { + calls, + config: { ...ruuster.defaults, savedSearchUrl: SAN_JOSE_SEARCH }, + cursor: {}, + budget: 150, + deadline: Date.now() + 60000, + http: { + json: async (url) => { + calls.push(url); + const u = new URL(url); + if (u.pathname === '/api/listings') { + const page = Number(u.searchParams.get('page')); + return { records: rows.slice((page - 1) * 10, page * 10), totalCount: rows.length }; + } + return rows.find((r) => u.pathname.endsWith(`/${r.id}`)); + }, + }, + ...overrides, + }; +} + +describe('Ruuster searches and properties', () => { + test('preserves filters and multiple statuses while removing tracking and CRM actions', () => { + const parsed = parseSearch( + `${SAN_JOSE_SEARCH}&status=Active&utm_source=email×tamp=123&isNeededSendToCRM=true&page=9&view=list`, + ); + expect(parsed.params.getAll('status')).toEqual(['Active', 'ComingSoon']); + expect(parsed.params.get('lotSizeAcresMin')).toBe('4500'); + expect(parsed.params.get('yearBuiltMin')).toBe('2000'); + expect(parsed.params.get('priceMax')).toBe('0'); + expect(parsed.params.get('slug')).toBe('talar-davoudi'); + expect(parsed.params.get('isNeededSendToCRM')).toBe('false'); + for (const key of ['utm_source', 'timestamp', 'page', 'view']) + expect(parsed.params.has(key)).toBe(false); + expect(parsed.key).toBe(search.key); + }); + test('rejects unrelated hosts, credentials, ports, protocols and detail URLs', () => { + for (const value of [ + 'nope', + 'http://example.ruuster.com/agent/name/listings', + 'https://ruuster.com.evil.test/agent/name/listings', + 'https://localhost/agent/name/listings', + 'https://user:pass@example.ruuster.com/agent/name/listings', + 'https://example.ruuster.com:8443/agent/name/listings', + `${SAN_JOSE_SEARCH.split('?')[0]}/uuid`, + ]) { + expect(() => parseSearch(value)).toThrow(); + } + }); + test('maps units, dates, media, location and attribution into housing items', () => { + const item = normaliseItem(toItem(property(), search)); + expect(item.kind).toBe('property-listing'); + expect(item.data).toMatchObject({ + price: 1500000, + currency: 'USD', + squareFeet: 1600, + lotSizeAcres: 0.25, + lotSizeSquareFeet: 10890, + yearBuilt: 2005, + latitude: 37.26, + longitude: -121.88, + attribution: { agent: 'Example Agent', brokerage: 'Example Realty', mls: 'Example MLS' }, + }); + expect(item.imageUrl).toBe('https://images.example/home.jpg'); + expect(item.publishedAt.toISOString()).toBe('2026-09-10T15:19:04.000Z'); + expect(item.tags).toContain('city:san-jose'); + }); + test('syndicated copies share an identity; unrelated properties and relistings do not', () => { + const row = property(); + expect(listingKey({ ...row, id: 'other-uuid', mlsId: 'other-mls' })).toBe(listingKey(row)); + expect(listingKey({ ...row, firstAddress: '1 Example' })).toBe(listingKey(row)); + expect(listingKey({ ...row, firstAddress: '1 Example Lane Unit 2' })).not.toBe(listingKey(row)); + expect(listingKey({ ...row, firstAddress: '999 Different St' })).not.toBe(listingKey(row)); + expect(listingKey({ ...row, mlsRecordId: 'NEW-LISTING' })).not.toBe(listingKey(row)); + }); + test('does not publish hidden addresses or deleted listings', () => { + expect(toItem(property(1, { isHideAddress: true }), search)).toBeNull(); + expect(toItem(property(1, { isDeleted: true }), search)).toBeNull(); + }); + test('unknown values stay unknown and label updates do not become publication dates', () => { + const item = toItem( + property(1, { + onMarketTimestamp: null, + labelUpdatedAt: '2026-09-13', + lotSizeAcres: null, + bedrooms: null, + price: 0, + location: { coordinates: [] }, + media: ['javascript:bad'], + }), + search, + ); + expect(item.publishedAt).toBeNull(); + expect(item.data.lotSizeSquareFeet).toBeNull(); + expect(item.data.bedrooms).toBeNull(); + expect(item.data.price).toBe(0); + expect(item.data.latitude).toBeNull(); + expect(item.imageUrl).toBeNull(); + }); + test('refreshing a property changes its price and status without creating a new identity', () => { + const before = normaliseItem(toItem(property(), search)); + const after = normaliseItem(toItem(property(1, { price: 1400000, status: 'Pending' }), search)); + expect(after.externalId).toBe(before.externalId); + expect(after.contentHash).not.toBe(before.contentHash); + expect(after.tags).toContain('status:pending'); + }); +}); + +describe('Ruuster ingestion', () => { + test('walks pages and fetches each syndicated property once', async () => { + const first = property(); + const rows = [ + ...Array.from({ length: 10 }, (_, n) => ({ ...first, id: `copy-${n}` })), + property(2), + ]; + const ctx = context(rows); + const result = await ruuster.pull(ctx); + expect(result.items).toHaveLength(2); + expect(ctx.calls.filter((u) => new URL(u).pathname === '/api/listings')).toHaveLength(2); + expect(ctx.calls).toHaveLength(4); + expect(result.cursor).toEqual({}); + expect(result.nextInMinutes).toBeUndefined(); + }); + test('resumes inside a page when the detail budget runs out, without skipping a property', async () => { + const rows = [property(1), property(2), property(3)]; + const first = await ruuster.pull(context(rows, { budget: 1 })); + expect(first.items).toHaveLength(1); + expect(first.cursor.offset).toBe(1); + expect(first.nextInMinutes).toBe(1); + const second = await ruuster.pull(context(rows, { cursor: first.cursor })); + expect(second.items.map((i) => i.title)).toEqual([ + toItem(rows[1], search).title, + toItem(rows[2], search).title, + ]); + expect(second.cursor).toEqual({}); + }); + test('page limits resume the next page and remember syndicated copies across runs', async () => { + const row = property(); + const rows = [ + ...Array.from({ length: 10 }, (_, n) => ({ ...row, id: `copy-${n}` })), + row, + property(2), + ]; + const ctx = context(rows); + ctx.config.pages = 1; + const first = await ruuster.pull(ctx); + expect(first.cursor.page).toBe(2); + const second = await ruuster.pull(context(rows, { cursor: first.cursor })); + expect(second.items).toHaveLength(1); + expect(second.items[0].data.mlsRecordId).toBe('ML2'); + }); + test('a changed search restarts the walk', async () => { + const ctx = context([property()], { + cursor: { searchKey: 'old-search', page: 99, seen: [listingKey(property())] }, + }); + const result = await ruuster.pull(ctx); + expect(result.items).toHaveLength(1); + expect(new URL(ctx.calls[0]).searchParams.get('page')).toBe('1'); + }); + test('deadline and zero budget make no upstream requests', async () => { + for (const override of [{ budget: 0 }, { deadline: Date.now() - 1 }]) { + const ctx = context([property()], override); + const result = await ruuster.pull(ctx); + expect(ctx.calls).toHaveLength(0); + expect(result.cursor.page).toBe(1); + } + }); + test('invalid pages and failed details fail rather than advancing a successful cursor', async () => { + await expect( + ruuster.pull(context([], { http: { json: async () => ({ error: 'login required' }) } })), + ).rejects.toThrow('invalid listing page'); + await expect( + ruuster.pull( + context([property()], { + http: { + json: async (url) => { + if (new URL(url).pathname === '/api/listings') + return { records: [property()], totalCount: 1 }; + throw new Error('503'); + }, + }, + }), + ), + ).rejects.toThrow('503'); + }); + test('empty searches finish and the adapter is registered', async () => { + const result = await ruuster.pull(context([])); + expect(result.items).toEqual([]); + expect(result.cursor).toEqual({}); + expect(adapterByName('ruuster').collection).toBe('housing'); + }); +});