From 71a2fbba8d22b43a9ddfd8ecf54f57dc06e8d3b2 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 13 Sep 2026 16:32:06 +0000 Subject: [PATCH] fix: classify Workspace and storage storefront products --- docs/hosting-verification-2026-09-13.md | 78 +++++++++++++++++++ packages/adapters/src/storefront-parsers.js | 4 +- .../storefront-capconnect-products.json | 24 ++++++ test/storefront.test.js | 35 +++++++++ 4 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 docs/hosting-verification-2026-09-13.md create mode 100644 packages/adapters/test/fixtures/storefront-capconnect-products.json diff --git a/docs/hosting-verification-2026-09-13.md b/docs/hosting-verification-2026-09-13.md new file mode 100644 index 0000000..e05995c --- /dev/null +++ b/docs/hosting-verification-2026-09-13.md @@ -0,0 +1,78 @@ +# Hosting verification, September 13, 2026 + +Verified the deployed [Hosting collection](https://nichedb.dev/c/hosting) and +public APIs at about 16:27 UTC. The original directory and import work is +delivered: [PR 64](https://github.com/profullstack/niche-db/pull/64) and +[PR 65](https://github.com/profullstack/niche-db/pull/65) are merged. The checked +source revision was `b68afe6`. + +## Published data + +The collection page and [source API](https://nichedb.dev/api/v1/sources?collection=hosting) +agree on 1,400 items. All nine enabled sources have a successful run and a daily +refresh scheduled for September 14. + +| Source | Items | +| --- | ---: | +| FindHost providers | 185 | +| Vultr plans | 174 | +| Linode types | 75 | +| Scaleway instances | 135 | +| OVH VPS, US | 175 | +| OVH VPS, EU | 70 | +| LowEndBox | 20 | +| Storefronts | 552 | +| BuyVPS | 14 | + +Reading the public storefront API through its three pages returned 552 distinct +external IDs from 13 WHMCS hosts: 507 plans and 45 addons. The latest recorded +storefront run finished at 03:14 UTC, updating 112 existing rows and adding eight. +All 185 provider records carry `FindHost, findhost.app, CC BY 4.0` attribution. +Sample Vultr and DigitalOcean provider records contain Wikipedia, OpenGraph, and +developer-tool enrichment; DigitalOcean also has the company-ticker enrichment. + +## Reading and feed checks + +The collection, item API, source API, and each of the six feed APIs and RSS URLs +returned HTTP 200. Counts below use a 200-item API request; the plans result is +bounded by that limit. + +| Feed | API items | Check | +| --- | ---: | --- | +| [Providers](https://nichedb.dev/f/hosting-providers) | 185 | Provider rows only; attribution retained | +| [Plans](https://nichedb.dev/f/vps-plans) | 200 | Plan rows only | +| [Under 5](https://nichedb.dev/f/vps-under-5) | 27 | Every price is monthly and below 5 in its stated currency | +| [Bare metal](https://nichedb.dev/f/bare-metal-plans) | 23 | Every offer has kind `bare-metal` | +| [Deals](https://nichedb.dev/f/hosting-deals) | 7 | Deal rows only | +| [Providers with a CLI](https://nichedb.dev/f/hosting-cli) | 47 | Provider rows only | + +RSS responses parsed as XML. They contain a promotional item in addition to the +data items; their default data limit is 100. + +## Classification correction + +The live [Google Workspace seat](https://nichedb.dev/i/10035891) and +[online-storage product](https://nichedb.dev/i/10035895) both had the fallback +`vps` offer kind. Their stored public raw records reproduce the problem without +network access. Recognize Google Workspace/G Suite as addons and the French +`stockage` group as storage. + +Replaying all 552 public storefront records through the corrected mapper changes +exactly eight classifications: four Workspace products become addons, and four +storage products become storage plans. All 552 external IDs stay identical, so +the ordinary source upsert can correct the rows without creating duplicates. +The fixture regression failed before the change and passed after it. The focused +hosting, storefront, keyed-source, enrichment, and BuyVPS checks pass (77 tests). +The full suite passes: 1,575 tests across 91 files. Repository-wide Biome passes +with existing warnings. + +## Limits and deployment status + +Verification used public GET requests and local fixture tests. It did not trigger +an import, production write, credential lookup, or paid request. No ServerHunter +source was added. Hetzner, DigitalOcean, UpCloud, and OpenServer remain disabled +with zero rows; their optional credentials or descriptors were not supplied. + +The classification correction is not deployed. The eight live records therefore +still need the reviewed change deployed and an ordinary storefront refresh. +The next daily refresh is scheduled; a second daily run has not yet occurred. diff --git a/packages/adapters/src/storefront-parsers.js b/packages/adapters/src/storefront-parsers.js index d3e4e30..370a1c3 100644 --- a/packages/adapters/src/storefront-parsers.js +++ b/packages/adapters/src/storefront-parsers.js @@ -324,7 +324,7 @@ function diskType(word) { * is read whole, and left out of every plan feed. */ export const ADDON_RE = - /microsoft ?365|office ?365|\bm365\b|exchange online|\bteams\b|audio conferencing|business voice|\blicen[cs]e|\bssl\b|certificate|domain (?:registration|name|transfer|renewal)|\bdomains?\b|\bseo\b|site ?builder|website builder|antivirus|spam (?:filter|expert)|\bcopilot\b|\bvisio\b|\bproject plan\b|\bpower bi\b|\bdynamics\b|\bwindows (?:10|11)\b/i; + /microsoft ?365|office ?365|\bm365\b|\bgoogle[\s-]+workspace\b|\bg[\s-]+suite\b|exchange online|\bteams\b|audio conferencing|business voice|\blicen[cs]e|\bssl\b|certificate|domain (?:registration|name|transfer|renewal)|\bdomains?\b|\bseo\b|site ?builder|website builder|antivirus|spam (?:filter|expert)|\bcopilot\b|\bvisio\b|\bproject plan\b|\bpower bi\b|\bdynamics\b|\bwindows (?:10|11)\b/i; export function kindOf(...names) { const s = names.filter(Boolean).join(' ').toLowerCase(); @@ -334,7 +334,7 @@ export function kindOf(...names) { if (/bare.?metal/.test(s)) return 'bare-metal'; if (/dedicated/.test(s)) return 'dedicated'; if (/gpu/.test(s)) return 'gpu'; - if (/storage|backup|object/.test(s)) return 'storage'; + if (/storage|stockage|backup|object/.test(s)) return 'storage'; if (/managed|cpanel|plesk|wordpress|directadmin/.test(s) && !/vps|vds|kvm|cloud|server/.test(s)) return 'managed'; if (/shared|reseller|web ?hosting|cpanel hosting|email/.test(s) && !/vps|vds|kvm|server/.test(s)) diff --git a/packages/adapters/test/fixtures/storefront-capconnect-products.json b/packages/adapters/test/fixtures/storefront-capconnect-products.json new file mode 100644 index 0000000..e48c9f3 --- /dev/null +++ b/packages/adapters/test/fixtures/storefront-capconnect-products.json @@ -0,0 +1,24 @@ +[ + { + "id": "business-starter", + "url": "https://client.capconnect.com/index.php/store/google-workspace/business-starter", + "name": "Business Starter", + "cycle": { "cycle": "annuel", "divisor": 1, "interval": "year" }, + "group": "google-workspace", + "price": { "amount": 864, "currency": "MAD" }, + "stock": "unknown", + "description": "G SUITE by Google Cloud 1 Email Account Upgradable with additional email accounts 30GB Storage per Account Email, Chat, Productivity Apps, Cloud Storage and other Collaboration tools powered by Google Cloud", + "startingFrom": false + }, + { + "id": "1tb", + "url": "https://client.capconnect.com/index.php/store/stockage-en-ligne/1tb", + "name": "1TB", + "cycle": { "cycle": "mensuel", "divisor": 1, "interval": "month" }, + "group": "stockage-en-ligne", + "price": { "amount": 90, "currency": "MAD" }, + "stock": "unknown", + "description": "Storage 1 TB Unlimited Traffic Supported Access Protocols - HTTPS, FTP, FTPS, SFTP, SCP - WebDAV, Samba/CIFS - BorgBackup rsync via SSH Manage with App 10 Concurrent connections Useable as network drive Upgrade/Downgrade Support Technique", + "startingFrom": false + } +] diff --git a/test/storefront.test.js b/test/storefront.test.js index 633cba7..03873dc 100644 --- a/test/storefront.test.js +++ b/test/storefront.test.js @@ -183,6 +183,41 @@ describe('reading prices, cycles and specs', () => { }); describe('what the first live run got wrong', () => { + test('Google Workspace seats and French storage products keep their actual kinds', async () => { + // Public raw products from /api/v1/items on 2026-09-13, before correction: + // /i/10035891 and /i/10035895 both carried kind:vps. + const products = JSON.parse(await fixture('storefront-capconnect-products.json')); + const items = products.map((product) => + productItem(product, { + platform: 'whmcs', + domain: 'capconnect.com', + provider: 'capconnect', + providerName: 'Cap Connect', + fetchedAt: '2026-09-13T03:13:46.701Z', + }), + ); + const [workspace, storage] = items; + expect(workspace.externalId).toBe('storefront:whmcs:capconnect.com:business-starter'); + expect(workspace.kind).toBe('addon'); + expect(workspace.data.offer.kind).toBeNull(); + expect(workspace.tags).not.toContain('plan'); + expect(workspace.tags).not.toContain('kind:vps'); + expect(workspace.data.offer.price).toMatchObject({ + amount: 864, + currency: 'MAD', + interval: 'year', + }); + expect(storage.externalId).toBe('storefront:whmcs:capconnect.com:1tb'); + expect(storage.kind).toBe('plan'); + expect(storage.data.offer.kind).toBe('storage'); + expect(storage.tags).toContain('kind:storage'); + expect(storage.tags).not.toContain('kind:vps'); + expect(storage.data.offer.storage).toEqual([{ type: null, size_gb: 1024 }]); + expect(kindOf('Google Workspace', 'Business Standard')).toBe('addon'); + expect(kindOf('G-Suite', 'Business Starter')).toBe('addon'); + expect(kindOf('VPS', 'KVM VPS with Google Workspace')).toBe('vps'); + }); + test('a currency written as a suffix word, or only in the page selector, and never a guessed USD', () => { expect(parsePrice('120.00dhs')).toEqual({ amount: 120, currency: 'MAD' }); expect(parsePrice('199 KSh')).toEqual({ amount: 199, currency: 'KES' });