From 130b1c9425c1591bcff53322b99823e436202f8e Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 09:19:32 -0300 Subject: [PATCH 01/77] docs: design workspace follower analytics --- ...-23-workspace-follower-analytics-design.md | 410 ++++++++++++++++++ 1 file changed, 410 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md new file mode 100644 index 000000000..3fc13d55e --- /dev/null +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -0,0 +1,410 @@ +# Workspace follower analytics — design + +**Status:** written design awaiting approval. Nothing implemented. + +## Objective + +Replace the request-time, per-social-account analytics experience with the +first workspace-level historical metric: follower count. + +After the daily collection pipeline begins producing local snapshots, the page +must answer three questions without querying social APIs at request time: + +1. How many followers did this workspace have at the end of the selected + period? +2. How did each connected social account's follower count change over that + period? +3. Which accounts gained or lost followers? + +Success means `/analytics` renders without making social API calls, daily data +collection is resilient to transient failures and rate limits, and one broken +platform cannot block another account's data. + +## Scope boundary + +Analytics are always scoped to the **current workspace**. There is no +cross-workspace or user-global total. + +The workspace is the tenancy and aggregation boundary. A social account is a +dimension inside that workspace so two accounts on the same network remain +separate chart series. + +Every query and collection write must validate the relationship between the +workspace and social account. The design must continue to support multiple +accounts of the same network. + +## First-version scope + +### Included platforms + +| Platform | Value represented | Precision / caveat | +| --- | --- | --- | +| TikTok | Profile followers | Exact value exposed by the user stats API | +| Instagram | Professional-account followers | Includes direct Instagram login and Instagram through Facebook | +| Facebook | Page followers | Page follower total, not daily follows gained | +| Threads | Profile followers | Account insight | +| X | Profile followers | Public user metric; subject to the application's X access and billing limits | +| Pinterest | Account followers | Account `follower_count` | +| YouTube | Channel subscribers | YouTube may round the public subscriber count for larger channels | +| Bluesky | Profile followers | Profile `followersCount` | +| Mastodon | Account followers | Account `followers_count` from the connected instance | +| Telegram | Chat/channel members | Member count, treated as the Telegram follower equivalent | +| Discord | Server members | Approximate member count, treated as the Discord follower equivalent | +| Google Business Profile | Location followers | Total follower count for the connected location | + +The value is labelled using the platform's native meaning where needed in +tooltips, but all values participate in the workspace's top-level follower +total. + +### Explicit exclusions + +Both LinkedIn identity types are excluded from follower analytics v1: + +- LinkedIn personal profile +- LinkedIn Page + +Neither receives follower collection jobs, appears in the follower charts, nor +contributes to the workspace total. Existing LinkedIn publishing and existing +post analytics remain untouched. + +LinkedIn personal follower analytics requires `r_member_profileAnalytics`, +which is provisioned through the vetted Community Management API product. That +product must initially be the only product on a separate LinkedIn developer +application. LinkedIn support can be reconsidered after TryPost receives the +required product approval; it is not part of this delivery. + +## User experience + +### Workspace total + +The page displays a follower-total summary above the chart. It sums one daily +follower value per included social account for the selected range's end date. + +- An account contributes at most once. +- Two accounts on the same network both contribute. +- An account with no value for the end date does not silently contribute an + older, unclassified value. +- A carried-forward value created by the daily fallback does contribute. +- An account that was disconnected or deactivated before that date does not + receive a snapshot for the date and therefore does not contribute. + +### Follower chart + +One follower widget presents the same workspace dataset in three modes: + +- **Line:** daily follower count per social account across the selected range. +- **Bar:** follower count per social account on the selected end date. +- **Growth:** net change per social account between its first and last available + values inside the selected range. Positive and negative changes share a zero + axis. + +Series and rows use the social account's platform icon, display name or +username, and stable social-account identity. They are not collapsed by +network. + +The initial display mode is Line. Changing modes is client-side because all +three views derive from the same response dataset. + +### Date range + +The existing analytics range date picker remains the page filter. + +- `minDate` is the earliest follower snapshot available in the workspace. +- `maxDate` is the latest follower snapshot available in the workspace. +- The picker cannot select a range wholly outside those bounds. +- All chart modes and the total use the same selected range. +- A social account connected after the selected start date begins when its own + data begins; no pre-connection values are invented. +- With no follower snapshots, the picker is disabled and the page shows a + collection-pending empty state. + +Historical data for a disconnected or deactivated account is retained. Its +line ends on the last day for which it was eligible; it remains visible when +the selected range overlaps that history. + +## Collection architecture + +```text +Laravel scheduler (daily, UTC) + -> dispatch-only collection command + -> one queued job per eligible social account + -> platform follower collector + -> normalized follower observation + -> persistence boundary + +End-of-day finalizer + -> identifies eligible accounts without a successful observation + -> carries forward the most recent known value when one exists +``` + +### Scheduler and dispatcher + +The scheduled command starts at `02:00 UTC`, runs with +`withoutOverlapping()` and `onOneServer()`, reads eligible accounts in bounded +chunks, and only dispatches jobs. It never calls a social API itself. + +An account is eligible when it: + +- belongs to a workspace; +- uses an included platform; +- is active; +- is connected and has the platform metadata required by its collector. + +Each account gets an independent job on a dedicated analytics queue. The job's +logical uniqueness key is follower metric + social account + UTC observation +date. Re-dispatching the same logical job is safe and cannot create a second +daily value. + +Connecting a supported account dispatches an immediate first collection so the +workspace does not wait for the next daily sweep. This initial job follows the +same idempotency and retry policy as the scheduled job. + +### Collector contract + +Each platform-specific collector has one responsibility: fetch the current +follower-equivalent value for one social account and return a normalized +observation. A collector does not authorize workspace access, aggregate totals, +or know the database schema. + +The normalized result contains, at minimum: + +- metric identity (`followers` at this stage); +- integer value; +- observation date in UTC; +- actual versus carried-forward provenance; +- exact versus approximate precision; +- platform response timestamp when the API provides one. + +This contract is intentionally independent of the physical persistence model +so future metrics can reuse the collection pipeline after their data shapes are +known. + +### Retry policy + +Transient HTTP failures, connection failures, server errors, and rate limits +must retry far apart within the same UTC day. The target attempt windows are: + +- 02:00 +- 06:00 +- 10:00 +- 14:00 +- 18:00 +- 22:00 + +The actual delayed execution may occur later under queue load. When a platform +returns a longer valid retry time, the job respects that time instead of the +four-hour default, provided the attempt still belongs to the observation day. + +Permanent authentication or permission rejection is not retried six times as +a transient error. It goes through the existing account-health handling and +does not write zero as a follower value. + +An attempt exits without writing when another attempt has already persisted an +actual observation for the account and date. + +### End-of-day fallback + +After the final attempt window, a finalizer covers eligible accounts that had +no successful API observation that day: + +- If an earlier valid follower value exists, copy it into the current date and + mark it as carried forward / estimated. +- If the account has never produced a valid follower value, no value can be + invented; it remains unavailable until a collection succeeds. +- A disconnected or deactivated account is not eligible for carry-forward. +- A carried-forward value may itself be carried into a later unavailable day, + while retaining provenance that the newest value is not a fresh API + observation. + +This keeps charts and workspace totals continuous during a platform outage +without misclassifying a repeated value as a successful API fetch. + +## Persistence decision gate + +This specification deliberately defines the **logical data requirements** but +does not choose a physical table design. + +The user intends to add many account-, post-, and workspace-level analytics +metrics. Choosing a generic metrics table, metric-specific tables, JSON +snapshots, or a hybrid before that catalog exists would prematurely constrain +dimensions, indexes, retention, and aggregation. + +Before any analytics migration or model is implemented, a follow-up design +must inventory each planned metric with: + +- entity level: workspace, social account, or post; +- value type and unit; +- snapshot, interval, delta, or lifetime semantics; +- supported dimensions; +- collection frequency and retention; +- exact, approximate, or estimated provenance; +- availability and historical limits per platform. + +That follow-up design selects the physical schema and proves it on both +PostgreSQL and MySQL. The implementation plan for this feature must not include +a persistence migration until that decision is approved. + +Regardless of the final schema, persistence must support: + +- workspace-scoped queries; +- social-account breakdown; +- one effective follower value per account and UTC date; +- idempotent writes; +- actual versus carried-forward provenance; +- exact versus approximate precision; +- earliest/latest available workspace dates; +- retaining history after an account is disconnected; +- efficient aggregation at a selected end date. + +Workspace totals are derived from account observations and are not stored as a +second source of truth. + +## Read path + +`/analytics` reads only local persisted data. It performs no request-time +social API calls for the follower widget. + +The server response supplies: + +- the workspace's available date bounds; +- the effective selected range after validation; +- the follower total at the range end; +- one daily series per social account; +- account identity and platform presentation metadata; +- actual/carried-forward and exact/approximate provenance required for + truthful tooltips. + +The frontend derives Bar and Growth from this normalized response instead of +requesting separate endpoints. Large date ranges may later be downsampled, but +daily resolution is the source resolution and is sufficient for this first +version. + +## Failure handling and observability + +Failures are isolated per social account. One platform outage cannot prevent +other account jobs from succeeding. + +Operational visibility must distinguish: + +- successful actual observation; +- transient failure awaiting retry; +- rate-limited attempt and next eligible attempt time; +- permanent authentication/permission rejection; +- successful carried-forward fallback; +- unavailable account with no historical value; +- finalizer failure. + +Logs include workspace, social account, platform, observation date, attempt, +and error category, but never access tokens or raw sensitive responses. + +The system must make it possible to alert on workspaces that repeatedly rely on +carried-forward values, even though defining an alerting product is outside +this first delivery. + +## Account lifecycle + +- **Connected:** dispatch an immediate first collection. +- **Active and connected:** participate in the daily sweep. +- **Deactivated:** stop new collection and fallback; preserve history; exclude + from totals after its last eligible date. +- **Disconnected/deleted:** stop collection and fallback; preserve historical + observations even if the account row is later removed. The physical schema + design must decide how to retain enough immutable identity for this. +- **Reconnected as the same persisted identity:** resume collection without + rewriting earlier observations. +- **New identity:** begins a new series even when its username matches an older + disconnected account. + +## Security and privacy + +- Analytics authorization follows the current workspace membership and policy + model. +- A social account id from another workspace must never affect collection or + read results. +- API tokens remain on `social_accounts` and are never copied into analytics + storage or job logs. +- Platform data-retention terms must be checked as each collector is + implemented; the physical schema review must record any network-specific + retention constraint. + +## Testing strategy + +### Collector contract tests + +Each included platform needs tests for: + +- successful exact or approximate follower parsing; +- missing/null metric handling; +- malformed response handling; +- rate-limit classification; +- transient server/connection classification; +- permanent authentication/permission classification; +- no accidental conversion of failure or null to zero. + +HTTP calls are faked. Tests must not call live social APIs. + +### Queue and scheduling tests + +- The daily command dispatches one job per eligible account and none for + LinkedIn, inactive, disconnected, or unsupported accounts. +- Jobs are isolated and idempotent by account, metric, and date. +- Retry delays cover the same UTC day and stop after a successful observation. +- Platform-provided retry timing is respected. +- A permanent authentication failure does not follow the transient retry loop. +- Immediate collection is dispatched after a supported account is connected. +- `withoutOverlapping()` and `onOneServer()` remain present on the schedule. + +### Fallback tests + +- The finalizer carries forward the last known value after all daily attempts + fail. +- Carried-forward provenance is preserved. +- No historical value means no fabricated snapshot. +- Deactivated and disconnected accounts are not carried forward. +- A successful observation is never overwritten by the finalizer. + +### Read and UI tests + +- Every response is workspace-scoped. +- The total sums each eligible account once on the selected end date. +- Multiple accounts on one network remain separate. +- Date bounds reflect the workspace's actual stored history. +- Line, Bar, and Growth derive the expected values from the same dataset. +- Growth handles negative values and a zero baseline. +- A later-connected account does not receive invented earlier points. +- Historical series remain available after disconnect/deactivation. +- No-data workspaces receive the collection-pending state. +- LinkedIn personal and LinkedIn Page never appear or contribute. + +Database-dependent tests run on PostgreSQL and MySQL after the persistence +design is approved and implemented. + +## Considered and rejected + +- **Calling social APIs from `/analytics`.** Slow, rate-limit prone, impossible + to trend reliably, and couples page availability to every provider. +- **One queued job per workspace.** A single slow or broken account delays the + whole workspace and makes retries unnecessarily broad. +- **Fast retry loops.** Follower totals tolerate delay; four-hour spacing gives + providers time to recover and protects API quotas. +- **Writing zero on failure.** Produces false losses and corrupts totals. +- **Silently using an old observation without provenance.** Keeps the UI full + but makes stale data indistinguishable from measured data. +- **Deleting history when an account disconnects.** Removes valid workspace + history and breaks historical comparisons. +- **Persisting workspace totals.** Duplicates account facts and risks drift. +- **Choosing the final table structure now.** The wider metric catalog is not + yet known, so the choice would be speculative. +- **Including either LinkedIn identity in v1.** Personal analytics require a + separately vetted product, and the product decision for this release is to + exclude the network consistently. + +## Delivery gates + +1. This written design must be reviewed and approved. +2. The broader metric catalog must be supplied and its persistence design + approved. +3. Only then can the Superpowers implementation-plan stage define migrations, + concrete classes, and ordered implementation tasks. +4. Implementation begins only after that written plan is reviewed and its + execution method is selected. From 72f04882b6af774afe8d6079d184c537436a5597 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 09:20:27 -0300 Subject: [PATCH 02/77] docs: plan LinkedIn follower analytics v2 --- ...-23-workspace-follower-analytics-design.md | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index 3fc13d55e..4b52751bb 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -70,8 +70,34 @@ post analytics remain untouched. LinkedIn personal follower analytics requires `r_member_profileAnalytics`, which is provisioned through the vetted Community Management API product. That product must initially be the only product on a separate LinkedIn developer -application. LinkedIn support can be reconsidered after TryPost receives the -required product approval; it is not part of this delivery. +application. It is not part of this delivery. + +### Planned v2: LinkedIn + +Follower analytics for both LinkedIn identity types are planned for v2: + +- LinkedIn personal profile follower count; +- LinkedIn Page follower count. + +The v2 keeps the network consistent by introducing both identity types +together. LinkedIn Page data is already technically accessible through the +current application scopes, but personal-profile data remains gated by +Community Management API approval and `r_member_profileAnalytics`. + +Before v2 implementation, TryPost must: + +1. create a separate LinkedIn developer application with no other provisioned + products; +2. request and receive Community Management API access; +3. confirm the production credential arrangement with LinkedIn after approval; +4. add the newly provisioned analytics scope to the appropriate OAuth flow; +5. require affected LinkedIn accounts to reconnect so their tokens contain the + approved scope; +6. verify the current LinkedIn API version and data-retention requirements. + +If Community Management API access is not approved, LinkedIn personal cannot +enter v2. Shipping LinkedIn Page alone would then require a new explicit +product decision rather than happening implicitly. ## User experience @@ -397,7 +423,8 @@ design is approved and implemented. yet known, so the choice would be speculative. - **Including either LinkedIn identity in v1.** Personal analytics require a separately vetted product, and the product decision for this release is to - exclude the network consistently. + defer the whole network to the planned v2 rather than ship partial LinkedIn + support. ## Delivery gates @@ -408,3 +435,5 @@ design is approved and implemented. concrete classes, and ordered implementation tasks. 4. Implementation begins only after that written plan is reviewed and its execution method is selected. +5. LinkedIn follower analytics receives a separate v2 implementation plan + after the external Community Management API dependency is resolved. From d701cd5e0ad7ba2f8cbf6936f58a5bde2141b2b9 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 09:24:22 -0300 Subject: [PATCH 03/77] docs: add workspace post analytics design --- ...-23-workspace-follower-analytics-design.md | 127 +++++++++++++++--- 1 file changed, 110 insertions(+), 17 deletions(-) diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index 4b52751bb..b67af2555 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -1,24 +1,28 @@ -# Workspace follower analytics — design +# Workspace follower and post analytics — design **Status:** written design awaiting approval. Nothing implemented. ## Objective -Replace the request-time, per-social-account analytics experience with the -first workspace-level historical metric: follower count. +Replace the request-time, per-social-account analytics experience with an +initial workspace-level analytics view covering follower history and posts +successfully published through TryPost. After the daily collection pipeline begins producing local snapshots, the page -must answer three questions without querying social APIs at request time: +must answer four questions without querying social APIs at request time: 1. How many followers did this workspace have at the end of the selected period? 2. How did each connected social account's follower count change over that period? 3. Which accounts gained or lost followers? +4. How many posts did each social account successfully publish during the + selected period, and how was that volume distributed over time? -Success means `/analytics` renders without making social API calls, daily data -collection is resilient to transient failures and rate limits, and one broken -platform cannot block another account's data. +Success means `/analytics` renders without making social API calls, daily +follower collection is resilient to transient failures and rate limits, one +broken platform cannot block another account's data, and post volume is derived +from the local publication history. ## Scope boundary @@ -56,7 +60,7 @@ The value is labelled using the platform's native meaning where needed in tooltips, but all values participate in the workspace's top-level follower total. -### Explicit exclusions +### Explicit follower exclusions Both LinkedIn identity types are excluded from follower analytics v1: @@ -65,7 +69,9 @@ Both LinkedIn identity types are excluded from follower analytics v1: Neither receives follower collection jobs, appears in the follower charts, nor contributes to the workspace total. Existing LinkedIn publishing and existing -post analytics remain untouched. +post analytics remain untouched. Successfully published LinkedIn destinations +do appear in the Posts widget because that metric comes from TryPost's local +publication records and requires no LinkedIn analytics permission. LinkedIn personal follower analytics requires `r_member_profileAnalytics`, which is provisioned through the vetted Community Management API product. That @@ -131,18 +137,63 @@ network. The initial display mode is Line. Changing modes is client-side because all three views derive from the same response dataset. +### Posts chart + +A second widget shows the number of destinations successfully published through +TryPost during the selected range. It uses two modes: + +- **Bar:** horizontal total per social account across the entire selected + range. +- **Stacked Bar:** publication count over time, with one colored segment per + social account in each time bucket. + +The initial display mode is Stacked Bar. The time bucket is selected +automatically from the inclusive range length: + +- up to 14 days: one bucket per day; +- 15 through 90 days: one bucket per week; +- more than 90 days: one bucket per calendar month. + +The first and last weekly or monthly buckets may be partial when the selected +range begins or ends inside that period. Empty buckets are returned with zero +values so the time axis remains continuous. + +One successful destination counts as one post for that social account. For +example, one TryPost post successfully delivered to Instagram and X contributes +one count to each account. The metric is based on the destination publication +record, not the parent post, so a partially successful multi-network post counts +only its successful destinations. + +The Posts widget includes every supported publishing platform, including both +LinkedIn identity types. It includes posts published from any TryPost entry +point, such as the app, API, MCP, or repurpose flows, when they share the normal +publication records. It excludes drafts, scheduled posts that have not yet +published, failed or rejected destinations, and posts created directly on a +social network outside TryPost. + +A retry that eventually succeeds counts once because the destination record is +counted once. Historical publications remain facts even if an account is later +deactivated or disconnected. Account snapshot metadata stored with the +destination is used for historical presentation when the live social-account +row is no longer available. + ### Date range -The existing analytics range date picker remains the page filter. +The existing analytics range date picker remains the shared page filter for the +follower total, follower chart, and Posts widget. -- `minDate` is the earliest follower snapshot available in the workspace. -- `maxDate` is the latest follower snapshot available in the workspace. +- `minDate` is the earliest follower snapshot or successful TryPost publication + available in the workspace. +- `maxDate` is the latest follower snapshot or successful TryPost publication + available in the workspace. - The picker cannot select a range wholly outside those bounds. - All chart modes and the total use the same selected range. - A social account connected after the selected start date begins when its own data begins; no pre-connection values are invented. -- With no follower snapshots, the picker is disabled and the page shows a - collection-pending empty state. +- A widget shows its own empty state when the selected range contains no data + for that metric. +- With neither follower snapshots nor successful publications, the picker is + disabled and the page shows an analytics-empty state. Historical data for a disconnected or deactivated account is retained. Its line ends on the last day for which it was eligible; it remains visible when @@ -285,10 +336,17 @@ Regardless of the final schema, persistence must support: Workspace totals are derived from account observations and are not stored as a second source of truth. +The Posts widget does not require a new analytics snapshot or collection table. +Its source of truth is the existing destination publication history. A counted +row must belong to a post in the current workspace, have the published status, +and have a `published_at` timestamp inside the selected range. The concrete +query must use the existing enum/status conventions and work on PostgreSQL and +MySQL. + ## Read path `/analytics` reads only local persisted data. It performs no request-time -social API calls for the follower widget. +social API calls for either widget. The server response supplies: @@ -298,13 +356,21 @@ The server response supplies: - one daily series per social account; - account identity and platform presentation metadata; - actual/carried-forward and exact/approximate provenance required for - truthful tooltips. + truthful tooltips; +- successful publication totals per social account; +- zero-filled publication buckets and per-account values for the automatically + selected daily, weekly, or monthly resolution. The frontend derives Bar and Growth from this normalized response instead of requesting separate endpoints. Large date ranges may later be downsampled, but daily resolution is the source resolution and is sufficient for this first version. +The server aggregates the Posts dataset at the chosen bucket resolution and +returns both bucketed and range-total values. Publication rows are always +filtered through their parent post's `workspace_id`; a social-account id from +the request is never trusted as the tenancy boundary. + ## Failure handling and observability Failures are isolated per social account. One platform outage cannot prevent @@ -400,7 +466,22 @@ HTTP calls are faked. Tests must not call live social APIs. - A later-connected account does not receive invented earlier points. - Historical series remain available after disconnect/deactivation. - No-data workspaces receive the collection-pending state. -- LinkedIn personal and LinkedIn Page never appear or contribute. +- LinkedIn personal and LinkedIn Page never appear or contribute to follower + analytics v1. +- The Posts Bar mode counts one successful destination per social account in + the selected range. +- The Posts Stacked Bar mode selects daily, weekly, and monthly buckets at the + documented range thresholds and zero-fills missing buckets. +- A multi-network post contributes once to every successful destination and + nothing to failed, rejected, pending, or future-scheduled destinations. +- A destination that succeeds after retries counts only once. +- Direct/native social-network posts are absent because no TryPost publication + record exists for them. +- LinkedIn personal and LinkedIn Page publications appear in the Posts widget + even though both remain excluded from follower analytics v1. +- Post aggregation is workspace-scoped through the parent post. +- Historical publications retain presentable account information after the + social account is disconnected or deleted. Database-dependent tests run on PostgreSQL and MySQL after the persistence design is approved and implemented. @@ -425,6 +506,18 @@ design is approved and implemented. separately vetted product, and the product decision for this release is to defer the whole network to the planned v2 rather than ship partial LinkedIn support. +- **Fetching post counts from social APIs.** The v1 metric represents successful + TryPost deliveries, which already have a reliable local destination record; + provider analytics would add permissions, rate limits, inconsistent history, + and native posts outside the agreed definition. +- **Counting parent posts.** One parent can target several accounts and can + partially fail, so the successful destination is the only accurate unit. +- **Persisting daily post-count snapshots.** Publication rows are immutable + facts that can be aggregated for the selected range without introducing a + second source of truth. +- **Using one fixed Posts bucket size.** A fixed daily view becomes noisy over + long ranges, while a fixed weekly or monthly view hides useful short-range + detail. ## Delivery gates From f99fcefa020db77f646af97ba1a59cabb2b1437f Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 09:48:07 -0300 Subject: [PATCH 04/77] docs: expand workspace analytics reporting design --- ...-23-workspace-follower-analytics-design.md | 233 +++++++++++++++++- 1 file changed, 223 insertions(+), 10 deletions(-) diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index b67af2555..42295004d 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -9,7 +9,7 @@ initial workspace-level analytics view covering follower history and posts successfully published through TryPost. After the daily collection pipeline begins producing local snapshots, the page -must answer four questions without querying social APIs at request time: +must answer six questions without querying social APIs at request time: 1. How many followers did this workspace have at the end of the selected period? @@ -18,11 +18,15 @@ must answer four questions without querying social APIs at request time: 3. Which accounts gained or lost followers? 4. How many posts did each social account successfully publish during the selected period, and how was that volume distributed over time? +5. How did publication volume, reactions, comments, and engagement compare + with the immediately preceding equivalent period? +6. Which destination publications and social accounts performed best? Success means `/analytics` renders without making social API calls, daily follower collection is resilient to transient failures and rate limits, one broken platform cannot block another account's data, and post volume is derived -from the local publication history. +from the local publication history. Post-performance metrics are also collected +ahead of page requests and retained locally. ## Scope boundary @@ -109,8 +113,8 @@ product decision rather than happening implicitly. ### Workspace total -The page displays a follower-total summary above the chart. It sums one daily -follower value per included social account for the selected range's end date. +The Total Followers card inside Summary sums one daily follower value per +included social account for the selected range's end date. - An account contributes at most once. - Two accounts on the same network both contribute. @@ -177,10 +181,110 @@ deactivated or disconnected. Account snapshot metadata stored with the destination is used for historical presentation when the live social-account row is no longer available. +### Summary + +The page includes one workspace-level Summary block with exactly five cards: + +- **Posts:** successful destination publications whose `published_at` falls + inside the selected range. +- **Total Followers:** the follower total at the selected range's end date, + using the same eligibility rules as the follower widget. +- **Reactions:** the sum of the latest stored reactions for successful + destination publications inside the selected range. +- **Comments:** the sum of the latest stored comments for successful + destination publications inside the selected range. +- **Engagement Rate:** pooled engagement divided by pooled exposure for the + eligible destination publications inside the selected range. + +Cross-network labels are normalized for comparison. Reactions include native +likes, favorites, and reactions. Comments include native comments and replies +when the platform exposes replies as its comment-equivalent metric. The +underlying native name remains available in the post detail and tooltip. + +Engagement follows the Buffer-style model approved for this design. Each +platform collector normalizes the interactions that its API treats as +engagement, such as reactions, comments, reposts/shares, saves, and clicks when +available. Exposure uses the platform-appropriate impressions, reach, or views +denominator. The workspace rate is calculated from the pooled numerator and +pooled denominator, rather than averaging post percentages, so a low-exposure +post does not weigh the same as a high-exposure post. + +A destination without a supported or valid exposure denominator is excluded +from Engagement Rate only. Its supported reactions and comments still +contribute to those cards. Unsupported metrics render as unavailable and are +never converted to zero. + +### Period comparison + +Summary and Performance compare the selected inclusive range with the +immediately preceding range of equal length. For example, a 30-day selection +compares against the preceding 30 days. The comparison period is calculated +automatically and is not a second user-selectable range. + +- Posts, Reactions, and Comments show percentage change. +- Engagement Rate shows the relative percentage change between the two pooled + rates. +- Total Followers shows the absolute follower change between the two period-end + totals, matching the reference design. +- When the previous value is zero or unavailable, the UI shows a neutral + unavailable/new-data state instead of infinity or a fabricated percentage. +- Partial historical coverage is disclosed in the tooltip and is not presented + as a complete comparison. + +### Top 5 Posts + +The page includes one Top 5 Posts block with a two-option toggle: + +- **Reactions** is the initial ranking; +- **Comments** ranks the same eligible dataset by normalized comments. + +The ranking unit is the successful destination publication, not the parent +post. A parent sent to multiple social accounts may therefore appear more than +once when more than one destination qualifies. Only destinations published +inside the selected range participate. + +Each card shows rank, normalized metric value, platform/account identity, +publication date, content type, excerpt, thumbnail when available, and actions +to open the TryPost post or its public social URL when supported. Ties are +resolved by newest `published_at` and then by stable destination id so the order +does not jump between requests. + +A destination whose selected ranking metric is unsupported is excluded from +that ranking. Fewer than five cards are shown when fewer than five eligible +destinations have a real value. An empty state replaces the list when none do. + +### Performance + +The page includes one Performance table with one row per social account that +has a successful destination publication in the selected range. Multiple +accounts on the same network remain separate rows. + +The fixed first-version columns are: + +- Channel; +- Posts; +- Reactions; +- Comments; +- Engagement Rate. + +Posts use the local successful-destination count. The other columns aggregate +the latest stored post-performance observations using the same normalization +and pooled-rate rules as Summary. Each supported numeric column can be sorted, +and its current value includes the equivalent-period comparison when a valid +comparison exists. + +When a network or content type does not expose a metric, the cell shows an +unavailable marker rather than zero. Historical account snapshot metadata keeps +rows presentable after an account is disconnected or deleted. + +These are exactly the three additional reporting blocks in v1: Summary, Top 5 +Posts, and Performance. More cards, ranking modes, or configurable Performance +columns require a later product decision. + ### Date range The existing analytics range date picker remains the shared page filter for the -follower total, follower chart, and Posts widget. +Summary, follower chart, Posts widget, Top 5 Posts, and Performance. - `minDate` is the earliest follower snapshot or successful TryPost publication available in the workspace. @@ -203,12 +307,18 @@ the selected range overlaps that history. ```text Laravel scheduler (daily, UTC) - -> dispatch-only collection command + -> follower dispatch-only command -> one queued job per eligible social account -> platform follower collector -> normalized follower observation -> persistence boundary + -> post-performance dispatch-only command + -> one queued job per eligible destination publication + -> platform post-metrics collector or trusted local metric source + -> normalized post-performance observation + -> persistence boundary + End-of-day finalizer -> identifies eligible accounts without a successful observation -> carries forward the most recent known value when one exists @@ -296,6 +406,47 @@ no successful API observation that day: This keeps charts and workspace totals continuous during a platform outage without misclassifying a repeated value as a successful API fetch. +## Post-performance collection + +Reactions, comments, engagement inputs, and Top 5 rankings must not trigger +social API calls while `/analytics` is rendering. They are refreshed in daily +queued jobs and stored behind the same persistence decision gate as follower +observations. + +The daily dispatcher selects successful destination publications that have a +platform post id, a connected account with the required access, and remain +inside their refresh window: + +- X destinations: through 20 days after publication; +- every other supported destination: through 30 days after publication. + +There is no free-versus-paid retention rule in TryPost. All workspaces use the +same collection windows. The windows limit external API work only; all values +already collected are retained permanently. + +Each eligible destination gets an independent queued job so one failing API or +post cannot block another. The logical uniqueness key is post-performance + +destination + UTC collection date. The job normalizes only metrics genuinely +returned for that network and content type, preserving unsupported separately +from a measured zero. + +Post-performance values are cumulative totals for that destination as of the +collection timestamp. Summary, Top 5 Posts, and Performance use the latest +stored observation for each destination selected by its publication date; they +do not add daily snapshots together. + +The normal daily run collects once per UTC day. The final eligible day performs +one final collection before the destination becomes inactive for scheduled +refresh. Transient and rate-limit failures use the same widely spaced, same-day +retry approach as follower collection. If the final-day collection fails, the +latest successful observation remains available with its collection timestamp; +the system does not replace it with zero. + +Metrics already maintained from trusted local events, such as webhook-backed +reaction metadata, may be normalized from that local source without making a +redundant provider request. Networks without post analytics still contribute +their locally known Posts count but show other values as unavailable. + ## Persistence decision gate This specification deliberately defines the **logical data requirements** but @@ -317,6 +468,12 @@ must inventory each planned metric with: - exact, approximate, or estimated provenance; - availability and historical limits per platform. +The newly approved post-performance catalog for this design consists of +normalized reactions, normalized comments, normalized engagement numerator, +exposure denominator and kind, provider collection timestamp, and availability +status per destination. It does not remove the gate: the user may supply more +metrics before the physical schema is selected. + That follow-up design selects the physical schema and proves it on both PostgreSQL and MySQL. The implementation plan for this feature must not include a persistence migration until that decision is approved. @@ -331,7 +488,12 @@ Regardless of the final schema, persistence must support: - exact versus approximate precision; - earliest/latest available workspace dates; - retaining history after an account is disconnected; -- efficient aggregation at a selected end date. +- efficient aggregation at a selected end date; +- latest supported post-performance values per destination; +- permanent retention after a destination leaves its refresh window; +- unsupported versus measured-zero post metrics; +- provider and collection timestamps needed to disclose freshness; +- efficient workspace, publication-range, account, and ranking aggregations. Workspace totals are derived from account observations and are not stored as a second source of truth. @@ -346,7 +508,7 @@ MySQL. ## Read path `/analytics` reads only local persisted data. It performs no request-time -social API calls for either widget. +social API calls for any analytics block. The server response supplies: @@ -359,7 +521,12 @@ The server response supplies: truthful tooltips; - successful publication totals per social account; - zero-filled publication buckets and per-account values for the automatically - selected daily, weekly, or monthly resolution. + selected daily, weekly, or monthly resolution; +- current and previous-period Summary values; +- the two deterministic Top 5 rankings; +- Performance rows and comparisons per social account; +- freshness and availability metadata needed for tooltips and unavailable + states. The frontend derives Bar and Growth from this normalized response instead of requesting separate endpoints. Large date ranges may later be downsampled, but @@ -384,7 +551,12 @@ Operational visibility must distinguish: - permanent authentication/permission rejection; - successful carried-forward fallback; - unavailable account with no historical value; -- finalizer failure. +- finalizer failure; +- post-performance collection success; +- post-performance metric unsupported; +- post-performance retry or permanent collection failure; +- post-performance destination leaving its refresh window with a final stored + value. Logs include workspace, social account, platform, observation date, attempt, and error category, but never access tokens or raw sensitive responses. @@ -435,6 +607,14 @@ Each included platform needs tests for: HTTP calls are faked. Tests must not call live social APIs. +Post-performance collector tests additionally cover: + +- native-to-normalized reaction and comment names; +- cumulative metrics stored as one observation rather than summed across days; +- engagement numerator and exposure denominator mapping; +- content-type-specific metric availability; +- unsupported, missing, malformed, and measured-zero distinctions. + ### Queue and scheduling tests - The daily command dispatches one job per eligible account and none for @@ -445,6 +625,14 @@ HTTP calls are faked. Tests must not call live social APIs. - A permanent authentication failure does not follow the transient retry loop. - Immediate collection is dispatched after a supported account is connected. - `withoutOverlapping()` and `onOneServer()` remain present on the schedule. +- Post-performance jobs are dispatched only for successful destinations with a + usable platform id and access. +- X destinations remain eligible through day 20; other supported destinations + remain eligible through day 30. +- The final eligible day receives a final collection and older destinations no + longer create provider jobs. +- Collection-window expiry never deletes an already stored value. +- Unsupported metrics remain distinct from measured zero. ### Fallback tests @@ -482,6 +670,18 @@ HTTP calls are faked. Tests must not call live social APIs. - Post aggregation is workspace-scoped through the parent post. - Historical publications retain presentable account information after the social account is disconnected or deleted. +- Summary contains exactly Posts, Total Followers, Reactions, Comments, and + Engagement Rate. +- Summary compares against the immediately preceding inclusive range of equal + length and handles zero, unavailable, and partial previous data safely. +- Engagement Rate pools normalized engagement and exposure rather than + averaging per-post percentages. +- Posts without a valid exposure denominator are excluded only from the rate. +- Top 5 ranks destination publications deterministically by Reactions or + Comments and excludes unsupported values. +- Performance returns one row per social account, keeps duplicate-network + accounts separate, supports sorting, and uses the same aggregation rules as + Summary. Database-dependent tests run on PostgreSQL and MySQL after the persistence design is approved and implemented. @@ -518,6 +718,19 @@ design is approved and implemented. - **Using one fixed Posts bucket size.** A fixed daily view becomes noisy over long ranges, while a fixed weekly or monthly view hides useful short-range detail. +- **Refreshing every historical post forever.** Engagement changes slow after + publication, while an unbounded daily job set would continually increase API + cost and rate-limit pressure. The last stored result remains available after + the 20/30-day refresh window closes. +- **Applying plan-based analytics retention.** TryPost has no free analytics + tier in this design; collection and permanent local retention are consistent + for every workspace. +- **Averaging individual engagement rates.** It overweights posts with little + exposure. Pooling the engagement and exposure totals produces a weighted + workspace/account rate. +- **Treating unsupported metrics as zero.** Zero means the provider measured no + activity; unsupported means no measurement was available and must remain + visibly different. ## Delivery gates From 09bd5fec9680697bc136f68a083aa215cd1dab0d Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 10:04:55 -0300 Subject: [PATCH 05/77] docs: plan persisted per-post analytics --- ...-23-workspace-follower-analytics-design.md | 164 ++++++++++++++++-- 1 file changed, 151 insertions(+), 13 deletions(-) diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index 42295004d..b0b1d19c6 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -21,6 +21,8 @@ must answer six questions without querying social APIs at request time: 5. How did publication volume, reactions, comments, and engagement compare with the immediately preceding equivalent period? 6. Which destination publications and social accounts performed best? +7. Which detailed metrics, including video-retention metrics where available, + explain the performance of an individual published destination? Success means `/analytics` renders without making social API calls, daily follower collection is resilient to transient failures and rate limits, one @@ -72,10 +74,12 @@ Both LinkedIn identity types are excluded from follower analytics v1: - LinkedIn Page Neither receives follower collection jobs, appears in the follower charts, nor -contributes to the workspace total. Existing LinkedIn publishing and existing -post analytics remain untouched. Successfully published LinkedIn destinations -do appear in the Posts widget because that metric comes from TryPost's local -publication records and requires no LinkedIn analytics permission. +contributes to the workspace total. LinkedIn publishing remains untouched, but +supported LinkedIn post analytics participate in the same database-backed post +metrics migration as the other networks. Successfully published LinkedIn +destinations also appear in the Posts widget because that metric comes from +TryPost's local publication records and requires no LinkedIn follower-analytics +permission. LinkedIn personal follower analytics requires `r_member_profileAnalytics`, which is provisioned through the vetted Community Management API product. That @@ -281,6 +285,90 @@ These are exactly the three additional reporting blocks in v1: Summary, Top 5 Posts, and Performance. More cards, ranking modes, or configurable Performance columns require a later product decision. +### Individual post analytics + +The existing analytics area inside each published post is part of this same +delivery. It must stop fetching provider metrics during the page request and +must stop treating the five-minute Redis entry as the metric source. + +The post-performance pipeline collects through queued jobs and persists through +one observation writer. The individual post page, REST API, MCP, Summary, Top 5 +Posts, and Performance all read the same latest persisted observation for each +destination. Redis is not a source of truth for post analytics; a +database-query cache may be added later only if profiling proves it useful. + +The individual post page is richer than the cross-network reporting blocks. It +shows every persisted metric supported by that platform and content type, +grouped into common engagement, exposure, and video-retention sections. It also +shows when the metrics were last collected and whether the value is actual, +estimated, stale after a failed refresh, experimental, or unsupported. + +The response contract uses stable metric keys and explicit units. Translated +labels are presentation only and are never stored as metric identity. An +unsupported metric is omitted or marked unavailable; an API error must not +replace the most recent successful value with zero. + +### Video metric catalog + +The initial content-type analysis establishes the following catalog. It is the +minimum that the platform collectors should request and persist when supported +by the connected account, login type, API version, and media type. + +| Content type | Metrics for the individual post page | Current TryPost gap | +| --- | --- | --- | +| Instagram feed | Views, reach, likes/reactions, comments, shares, saves, reposts, total interactions, follows, profile visits, and profile activity | The current collector omits views, reposts, follows, profile visits, and profile activity | +| Instagram Reel | Views, reach, likes/reactions, comments, shares, saves, reposts, total interactions, total watch time, average watch time, and skip rate when returned | The current collector already has views/reach/basic engagement but omits interactions, reposts, watch-time metrics, and skip rate | +| Instagram Story | Views, reach, replies, shares, reposts, follows, profile visits/activity, link clicks, and navigation breakdown | The current collector only requests views, reach, and replies | +| YouTube Short | Views, engaged views, watch time, average view duration, average percentage viewed, likes, comments, shares, subscribers gained, and subscribers lost | The current collector already has views, watch time, average duration, likes, comments, and shares, but omits engaged views, average percentage viewed, and subscriber change | +| TikTok video | Views, likes, comments, and shares | The current Display API collector already exposes the complete performance set available to this integration; video duration is metadata, not watch time | + +Instagram Reel total watch time is displayed in minutes and average watch time +in seconds, matching the reference UI, while persistence retains the canonical +unit needed to avoid rounding loss. Metrics that Meta marks estimated or in +development, currently including Reel reach, watch time, views, total +interactions, and skip rate as applicable, preserve that precision/stability +metadata for tooltips. + +Meta documents that Instagram insight values can lag by up to 48 hours. A +successful response with an absent or not-yet-populated metric is therefore not +converted to measured zero. The read model keeps the last successful value and +exposes its collection time so the UI can distinguish fresh, delayed, and stale +data. Provider retention does not control TryPost retention: once collected, +the observation remains stored under TryPost's permanent-history policy. + +Instagram Reel engagement rate uses the normalized interactions divided by +reach when both are available. This matches the reference behavior and avoids +using repeated views as though they were unique people. + +The Meta collector must parse both `values[].value` and `total_value.value`, and +must preserve requested breakdowns such as Story navigation actions. It splits +incompatible or experimental metric families into separate provider requests: +one rejected metric must not blank every otherwise supported metric for the +post. + +Instagram cross-posted and Facebook-only view metrics are conditional: they are +stored and displayed only when the Reel was actually shared or recommended to +Facebook and the API returns them. They do not replace Instagram views. + +The approved TikTok Display API does not expose total watch time, average watch +time, completion rate, or retention. Those values must remain unavailable +rather than being inferred from view count and video duration. + +YouTube's per-video report already supports the retention metrics needed for +Shorts. The collector expands its current query rather than introducing a +second Shorts-specific API path. + +Instagram Stories require an exception to the normal 30-day refresh window: +their media insights are generally available for only 24 hours. A queued +collection is scheduled during the Story lifetime, the `story_insights` webhook +is enabled when the integration supports it, and a final collection runs +shortly before expiry. Webhook deliveries and scheduled jobs persist through the +same idempotent observation writer. The normal once-daily sweep alone is +insufficient because it can miss the availability window. Stored Story metrics +remain available after the provider stops serving them. Privacy-threshold or +"not enough viewers" responses mean unavailable, not measured zero and not a +reason to erase a previous observation. + ### Date range The existing analytics range date picker remains the shared page filter for the @@ -319,6 +407,9 @@ Laravel scheduler (daily, UTC) -> normalized post-performance observation -> persistence boundary + -> Instagram Story lifecycle jobs and `story_insights` webhook + -> same idempotent post-performance observation writer + End-of-day finalizer -> identifies eligible accounts without a successful observation -> carries forward the most recent known value when one exists @@ -408,10 +499,11 @@ without misclassifying a repeated value as a successful API fetch. ## Post-performance collection -Reactions, comments, engagement inputs, and Top 5 rankings must not trigger -social API calls while `/analytics` is rendering. They are refreshed in daily -queued jobs and stored behind the same persistence decision gate as follower -observations. +The complete supported post metric catalog, including reactions, comments, +exposure, engagement inputs, and video retention, must not trigger social API +calls while `/analytics` or an individual post is rendering. Metrics are +refreshed in queued jobs and stored behind the same persistence decision gate as +follower observations. The daily dispatcher selects successful destination publications that have a platform post id, a connected account with the required access, and remain @@ -420,6 +512,9 @@ inside their refresh window: - X destinations: through 20 days after publication; - every other supported destination: through 30 days after publication. +Instagram Stories use their separately documented within-24-hours schedule +instead of the 30-day sweep. + There is no free-versus-paid retention rule in TryPost. All workspaces use the same collection windows. The windows limit external API work only; all values already collected are retained permanently. @@ -468,11 +563,15 @@ must inventory each planned metric with: - exact, approximate, or estimated provenance; - availability and historical limits per platform. -The newly approved post-performance catalog for this design consists of -normalized reactions, normalized comments, normalized engagement numerator, -exposure denominator and kind, provider collection timestamp, and availability -status per destination. It does not remove the gate: the user may supply more -metrics before the physical schema is selected. +The currently specified post-performance catalog includes both cross-network +fields and content-specific detail. Cross-network fields are normalized +reactions, normalized comments, normalized engagement numerator, exposure +denominator and kind, provider collection timestamp, and availability status +per destination. +The full observation additionally retains stable metric key, numeric value, +unit, content type, precision/stability flags, and provider metric identity for +every supported native metric described by the catalog. It does not remove the +gate: the user may supply more metrics before the physical schema is selected. That follow-up design selects the physical schema and proves it on both PostgreSQL and MySQL. The implementation plan for this feature must not include @@ -492,6 +591,8 @@ Regardless of the final schema, persistence must support: - latest supported post-performance values per destination; - permanent retention after a destination leaves its refresh window; - unsupported versus measured-zero post metrics; +- stable metric keys and units independent of the active UI locale; +- content-type-specific metrics without sparse schema assumptions; - provider and collection timestamps needed to disclose freshness; - efficient workspace, publication-range, account, and ranking aggregations. @@ -525,6 +626,8 @@ The server response supplies: - current and previous-period Summary values; - the two deterministic Top 5 rankings; - Performance rows and comparisons per social account; +- the complete latest metric set for each destination on the individual post + page, REST API, and MCP; - freshness and availability metadata needed for tooltips and unavailable states. @@ -613,6 +716,10 @@ Post-performance collector tests additionally cover: - cumulative metrics stored as one observation rather than summed across days; - engagement numerator and exposure denominator mapping; - content-type-specific metric availability; +- Instagram Reel watch-time units and experimental/estimated flags; +- YouTube Short watch time, average duration, average percentage viewed, and + subscriber-change mapping; +- TikTok never fabricating unsupported retention metrics; - unsupported, missing, malformed, and measured-zero distinctions. ### Queue and scheduling tests @@ -633,6 +740,8 @@ Post-performance collector tests additionally cover: longer create provider jobs. - Collection-window expiry never deletes an already stored value. - Unsupported metrics remain distinct from measured zero. +- Instagram Story jobs collect while insights are available and perform a + final pre-expiry collection even when the normal daily sweep would miss it. ### Fallback tests @@ -682,6 +791,11 @@ Post-performance collector tests additionally cover: - Performance returns one row per social account, keeps duplicate-network accounts separate, supports sorting, and uses the same aggregation rules as Summary. +- The individual post page, REST API, and MCP return the same persisted latest + observation and make no provider request during reads. +- The individual post page shows the content-type-specific catalog, canonical + units, freshness, and metric stability/provenance. +- Expired Redis entries cannot remove or change persisted post analytics. Database-dependent tests run on PostgreSQL and MySQL after the persistence design is approved and implemented. @@ -731,6 +845,30 @@ design is approved and implemented. - **Treating unsupported metrics as zero.** Zero means the provider measured no activity; unsupported means no measurement was available and must remain visibly different. +- **Keeping the individual post page on request-time API calls and Redis.** It + would give the post page a different source and freshness model from Summary, + Top 5 Posts, Performance, REST, and MCP. All consumers must converge on the + persisted observation. +- **Reducing post persistence to the five cross-network fields.** That would + discard high-value, content-specific metrics such as Reel/Short watch time + and make the individual post page less useful than the provider data already + available to TryPost. +- **Inferring TikTok retention from duration and views.** Video length describes + the asset, not how long viewers watched it; the approved integration exposes + no retention metric. + +## External references checked + +- Meta Instagram Media Insights, updated September 11, 2026: + +- YouTube Analytics metrics and channel report combinations: + and + +- TikTok Display API video query and Video Object fields: + and + +- Buffer Insights metric presentation and per-post behavior: + ## Delivery gates From c3bcc7841981e53b485123c15008dd50334510ef Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 10:06:41 -0300 Subject: [PATCH 06/77] docs: add Pinterest post metric catalog --- ...-23-workspace-follower-analytics-design.md | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index b0b1d19c6..d6b641798 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -308,7 +308,7 @@ labels are presentation only and are never stored as metric identity. An unsupported metric is omitted or marked unavailable; an API error must not replace the most recent successful value with zero. -### Video metric catalog +### Content-specific metric catalog The initial content-type analysis establishes the following catalog. It is the minimum that the platform collectors should request and persist when supported @@ -321,6 +321,8 @@ by the connected account, login type, API version, and media type. | Instagram Story | Views, reach, replies, shares, reposts, follows, profile visits/activity, link clicks, and navigation breakdown | The current collector only requests views, reach, and replies | | YouTube Short | Views, engaged views, watch time, average view duration, average percentage viewed, likes, comments, shares, subscribers gained, and subscribers lost | The current collector already has views, watch time, average duration, likes, comments, and shares, but omits engaged views, average percentage viewed, and subscriber change | | TikTok video | Views, likes, comments, and shares | The current Display API collector already exposes the complete performance set available to this integration; video duration is metadata, not watch time | +| Pinterest image Pin | Impressions, saves, comments, reactions, engagements, engagement rate, save rate, Pin clicks/rate, outbound clicks/rate, profile visits, follows, total audience, and engaged audience | The current collector has impressions, saves, Pin clicks, and outbound clicks, but omits the remaining native and lifetime metrics | +| Pinterest video Pin | Every applicable image-Pin metric plus video views, average video play time, 10-second plays, plays to 95%, and total play time | The current collector only adds basic video views and omits the richer video-retention metrics | Instagram Reel total watch time is displayed in minutes and average watch time in seconds, matching the reference UI, while persistence retains the canonical @@ -358,6 +360,21 @@ YouTube's per-video report already supports the retention metrics needed for Shorts. The collector expands its current query rather than introducing a second Shorts-specific API path. +Pinterest uses two complementary sources. The Pin Analytics endpoint provides +date-range metrics, including impressions, saves, engagement, clicks, rates, +and video performance. `GET /pins/{pin_id}?pin_metrics=true` provides rolling +and lifetime Pin metrics, including total comments and total reactions. The +collector combines both into one idempotent observation while retaining each +metric's time basis (`range`, `rolling_90_day`, or `lifetime`). A lifetime value +must never be presented as though it occurred entirely inside the selected +dashboard range. + +Pinterest's native engagement rate remains the provider-defined engagements +divided by impressions. Saves, comments, and reactions are displayed as their +own metrics; comments and reactions are not silently added to the provider's +engagement numerator unless Pinterest includes them in the returned native +definition. This keeps the reference labels without changing their meaning. + Instagram Stories require an exception to the normal 30-day refresh window: their media insights are generally available for only 24 hours. A queued collection is scheduled during the Story lifetime, the `story_insights` webhook @@ -720,6 +737,10 @@ Post-performance collector tests additionally cover: - YouTube Short watch time, average duration, average percentage viewed, and subscriber-change mapping; - TikTok never fabricating unsupported retention metrics; +- Pinterest saves, comments, reactions, impressions, and native engagement + rate retain their distinct metric identities and time bases; +- Pinterest video Pins map average play time, 10-second plays, 95% plays, and + total play time with explicit units; - unsupported, missing, malformed, and measured-zero distinctions. ### Queue and scheduling tests @@ -867,6 +888,13 @@ design is approved and implemented. - TikTok Display API video query and Video Object fields: and +- Pinterest organic reporting and metric definitions: + + and + +- Pinterest's official generated API client, including `pin_metrics` lifetime + comments/reactions and Pin Analytics parameters: + - Buffer Insights metric presentation and per-post behavior: From 6c04f14dac6feaf64521cf50034b64887104f99f Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 10:07:38 -0300 Subject: [PATCH 07/77] docs: clarify YouTube post analytics --- ...026-09-23-workspace-follower-analytics-design.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index d6b641798..dd898a236 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -319,7 +319,7 @@ by the connected account, login type, API version, and media type. | Instagram feed | Views, reach, likes/reactions, comments, shares, saves, reposts, total interactions, follows, profile visits, and profile activity | The current collector omits views, reposts, follows, profile visits, and profile activity | | Instagram Reel | Views, reach, likes/reactions, comments, shares, saves, reposts, total interactions, total watch time, average watch time, and skip rate when returned | The current collector already has views/reach/basic engagement but omits interactions, reposts, watch-time metrics, and skip rate | | Instagram Story | Views, reach, replies, shares, reposts, follows, profile visits/activity, link clicks, and navigation breakdown | The current collector only requests views, reach, and replies | -| YouTube Short | Views, engaged views, watch time, average view duration, average percentage viewed, likes, comments, shares, subscribers gained, and subscribers lost | The current collector already has views, watch time, average duration, likes, comments, and shares, but omits engaged views, average percentage viewed, and subscriber change | +| YouTube video or Short | Video views, engaged views, watch time, average view duration, average percentage viewed, likes/reactions, comments, shares, subscribers gained, and subscribers lost | The current collector already has views, watch time, average duration, likes, comments, and shares, but omits engaged views, average percentage viewed, and subscriber change | | TikTok video | Views, likes, comments, and shares | The current Display API collector already exposes the complete performance set available to this integration; video duration is metadata, not watch time | | Pinterest image Pin | Impressions, saves, comments, reactions, engagements, engagement rate, save rate, Pin clicks/rate, outbound clicks/rate, profile visits, follows, total audience, and engaged audience | The current collector has impressions, saves, Pin clicks, and outbound clicks, but omits the remaining native and lifetime metrics | | Pinterest video Pin | Every applicable image-Pin metric plus video views, average video play time, 10-second plays, plays to 95%, and total play time | The current collector only adds basic video views and omits the richer video-retention metrics | @@ -360,6 +360,14 @@ YouTube's per-video report already supports the retention metrics needed for Shorts. The collector expands its current query rather than introducing a second Shorts-specific API path. +On the individual YouTube post, the cross-network `Reactions` label maps to the +native `likes` metric, `Comments` maps to `comments`, and `Video Views` maps to +`views`. The YouTube Analytics API does not return a native per-video engagement +rate, so that native field is unavailable rather than fabricated. A normalized +TryPost engagement rate used by Summary or Performance is a separately labelled +derived value with its numerator and `views` denominator preserved; it must not +be represented as a provider-returned YouTube metric. + Pinterest uses two complementary sources. The Pin Analytics endpoint provides date-range metrics, including impressions, saves, engagement, clicks, rates, and video performance. `GET /pins/{pin_id}?pin_metrics=true` provides rolling @@ -736,6 +744,9 @@ Post-performance collector tests additionally cover: - Instagram Reel watch-time units and experimental/estimated flags; - YouTube Short watch time, average duration, average percentage viewed, and subscriber-change mapping; +- YouTube post labels map likes to Reactions and views to Video Views while the + unavailable native engagement rate remains distinct from TryPost's derived + normalized rate; - TikTok never fabricating unsupported retention metrics; - Pinterest saves, comments, reactions, impressions, and native engagement rate retain their distinct metric identities and time bases; From 2673847bdb8ff297d32ab8f4cb5796cafac181c2 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 10:10:57 -0300 Subject: [PATCH 08/77] docs: audit Buffer per-post metrics --- ...-23-workspace-follower-analytics-design.md | 76 ++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index dd898a236..8ef34ed9f 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -308,6 +308,74 @@ labels are presentation only and are never stored as metric identity. An unsupported metric is omitted or marked unavailable; an API error must not replace the most recent successful value with zero. +### Buffer per-post reference audit + +Buffer's Sent-post and Insights documentation provides the following UX and +normalization reference for individual posts. It is a discovery catalog, not +proof that TryPost's credentials, scopes, account type, or current API version +can retrieve every value. Each collector still requires verification against +the network's official API documentation before implementation. + +| Channel | Per-post metrics exposed or named by Buffer | +| --- | --- | +| Instagram Professional | Reactions/likes, comments, reposts/shares, views or impressions, reach, saves, follows, and engagement rate; availability varies between Feed, Reel, and Story | +| Facebook Page | Reactions, comments, shares/reposts, clicks, reach, views, impressions where still returned, and engagement rate; Group analytics are excluded | +| X/Twitter | Reactions/likes, replies/comments, reposts, quotes, clicks, impressions on the Sent surface, and a Buffer-derived engagement rate | +| LinkedIn Page | Reactions, comments, reposts/shares, impressions, engagement rate, and for video: views, total watch time in minutes, and unique viewers | +| LinkedIn personal profile | Reactions, comments, reach, impressions, video views, and engagement rate; reliable repost counts are not available | +| Pinterest business | Reactions, comments, saves, clicks, impressions, views, and engagement rate where Buffer has a valid exposure value | +| Mastodon | Favorites/reactions, replies/comments, and reblogs/reposts | +| TikTok | Reactions/likes, comments, shares/reposts, views, reach, and engagement rate | +| YouTube | Reactions/likes, comments, video views, shares in aggregate reporting, and a derived engagement rate where Buffer can calculate one | +| Threads | Reactions/likes, comments/replies, reposts, quotes, views, and engagement rate | +| Bluesky | Reactions/likes, replies/comments, reposts, quotes, and a derived engagement rate | + +Buffer does not provide this per-post reference for Telegram, Discord, or +Google Business Profile, and it does not expose post analytics for Instagram +Personal accounts or Facebook Groups. Those absences do not remove TryPost +capabilities: Telegram webhook reactions, Discord reactions/thread replies, and +any official Google Business post data are evaluated from their own official +APIs and existing local event sources. + +The Buffer product surfaces are not internally identical. Sent posts, the new +Insights product, and the retiring Analyze product can expose different metrics +and historical windows. For example, Buffer documents X impressions in Sent +posts while also saying its Insights visibility view has no X impressions or +views. TryPost records the provider metric identity, source, time basis, and +formula so a value is never promoted merely because another Buffer surface +lists it. + +Buffer's normalization vocabulary is useful and is adopted for cross-network +presentation only: + +- `Reactions` covers native likes, favorites, and reactions; +- `Comments` covers comments and reply-equivalents; +- `Reposts` covers retweets, reblogs, reshares, and reposts; +- native names and raw metric identities remain visible in post detail; +- engagement rate must disclose whether it is provider-returned or derived, + plus its interaction numerator and exposure denominator. + +Compared with the existing request-time TryPost collectors, the audit produces +the following implementation inventory. A Buffer-only metric is a candidate to +verify, not permission to invent or request an undocumented field. + +| Channel | Existing TryPost per-post collector | Candidate gap to verify | +| --- | --- | --- | +| Facebook | Feed: impressions, reach, likes, clicks; Story: impressions, reach, interactions, reactions, replies, shares; Reel/video: plays, reactions, interactions | Feed comments/shares and richer Reel actions exposed by current Meta APIs | +| Instagram | Reach, views, likes, comments, shares, saves, replies, or total interactions depending on content type | The expanded Feed/Reel/Story catalog below | +| X/Twitter | Impressions, likes, retweets, replies, quotes, and bookmarks | Click metrics and their access level | +| LinkedIn Page | Impressions, clicks, likes, comments, and shares | Provider engagement rate plus video views, watch time, and unique viewers | +| LinkedIn personal profile | Likes and comments | Impressions, reach, reliable video views, and derived engagement inputs under the app's approved scopes | +| Pinterest | Impressions, saves, Pin clicks, outbound clicks, and video views | Lifetime reactions/comments, rates, audience values, and video-retention metrics described below | +| Mastodon | Favorites, replies, and reblogs | No Buffer-identified basic metric gap | +| TikTok | Views, likes, comments, and shares | Reach is a Buffer candidate but is unavailable in TryPost's currently approved Display API fields | +| YouTube | Views, total watch time, average view duration, likes, comments, and shares | Engaged views, average percentage viewed, and subscriber change described below | +| Threads | Views, likes, replies, reposts, and quotes | No Buffer-identified basic metric gap | +| Bluesky | Likes, replies, reposts, and quotes | No Buffer-identified basic metric gap | +| Telegram | Locally persisted webhook reactions, plus chat/channel member count shown alongside the post today | Buffer has no comparison; keep account members separate from post performance | +| Discord | Message reactions and thread replies, plus server member count shown alongside the post today | Buffer has no comparison; keep account members separate from post performance | +| Google Business Profile | No individual-post collector | Buffer has no post-analytics reference; remain unsupported until official APIs prove a post-level metric | + ### Content-specific metric catalog The initial content-type analysis establishes the following catalog. It is the @@ -906,8 +974,12 @@ design is approved and implemented. - Pinterest's official generated API client, including `pin_metrics` lifetime comments/reactions and Pin Analytics parameters: -- Buffer Insights metric presentation and per-post behavior: - +- Buffer sent-post metric matrix, Insights behavior, and documented + cross-surface/provider differences: + , + , + and + ## Delivery gates From e757ad388eeaf98fe9b700b2477190dc3609a347 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 10:15:07 -0300 Subject: [PATCH 09/77] docs: narrow analytics v1 platform scope --- ...-23-workspace-follower-analytics-design.md | 145 ++++++++++-------- 1 file changed, 85 insertions(+), 60 deletions(-) diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index 8ef34ed9f..d7e651196 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -58,40 +58,42 @@ accounts of the same network. | YouTube | Channel subscribers | YouTube may round the public subscriber count for larger channels | | Bluesky | Profile followers | Profile `followersCount` | | Mastodon | Account followers | Account `followers_count` from the connected instance | -| Telegram | Chat/channel members | Member count, treated as the Telegram follower equivalent | -| Discord | Server members | Approximate member count, treated as the Discord follower equivalent | -| Google Business Profile | Location followers | Total follower count for the connected location | The value is labelled using the platform's native meaning where needed in tooltips, but all values participate in the workspace's top-level follower total. -### Explicit follower exclusions +### Explicit v1 platform exclusions -Both LinkedIn identity types are excluded from follower analytics v1: +The following platforms are excluded from every analytics surface in v1: - LinkedIn personal profile - LinkedIn Page +- Telegram +- Discord +- Google Business Profile -Neither receives follower collection jobs, appears in the follower charts, nor -contributes to the workspace total. LinkedIn publishing remains untouched, but -supported LinkedIn post analytics participate in the same database-backed post -metrics migration as the other networks. Successfully published LinkedIn -destinations also appear in the Posts widget because that metric comes from -TryPost's local publication records and requires no LinkedIn follower-analytics -permission. +They receive no follower or post-performance collection jobs, appear in no +follower or Posts chart, contribute to no Summary, Top 5, or Performance value, +and expose no individual-post analytics block in this delivery. Their successful +publication records remain intact but are filtered out of `/analytics`. +Publishing, comments/community features, and account connection behavior remain +unchanged. LinkedIn personal follower analytics requires `r_member_profileAnalytics`, which is provisioned through the vetted Community Management API product. That product must initially be the only product on a separate LinkedIn developer -application. It is not part of this delivery. +application. LinkedIn post metrics are also deferred so the network enters the +new analytics architecture as one coherent v2 rather than partially in v1. ### Planned v2: LinkedIn -Follower analytics for both LinkedIn identity types are planned for v2: +Analytics for both LinkedIn identity types are planned for v2: - LinkedIn personal profile follower count; -- LinkedIn Page follower count. +- LinkedIn Page follower count; +- database-backed post metrics for personal profiles and Pages; +- inclusion in Posts, Summary, Top 5, Performance, and individual-post detail. The v2 keeps the network consistent by introducing both identity types together. LinkedIn Page data is already technically accessible through the @@ -107,11 +109,18 @@ Before v2 implementation, TryPost must: 4. add the newly provisioned analytics scope to the appropriate OAuth flow; 5. require affected LinkedIn accounts to reconnect so their tokens contain the approved scope; -6. verify the current LinkedIn API version and data-retention requirements. +6. verify the current LinkedIn follower, post, video, and engagement metric + endpoints, API version, scopes, and data-retention requirements; +7. create a separate v2 implementation plan for collection, persistence, and + read-path inclusion. -If Community Management API access is not approved, LinkedIn personal cannot -enter v2. Shipping LinkedIn Page alone would then require a new explicit -product decision rather than happening implicitly. +If Community Management API access is not approved, LinkedIn cannot enter v2. +Shipping LinkedIn Page alone would then require a new explicit product decision +rather than happening implicitly. + +Telegram, Discord, and Google Business Profile have no planned analytics v2 in +this design. Adding any of them later requires a new product decision and +official API capability review. ## User experience @@ -172,12 +181,14 @@ one count to each account. The metric is based on the destination publication record, not the parent post, so a partially successful multi-network post counts only its successful destinations. -The Posts widget includes every supported publishing platform, including both -LinkedIn identity types. It includes posts published from any TryPost entry -point, such as the app, API, MCP, or repurpose flows, when they share the normal -publication records. It excludes drafts, scheduled posts that have not yet -published, failed or rejected destinations, and posts created directly on a -social network outside TryPost. +The Posts widget includes only the platforms included in analytics v1. It +excludes LinkedIn personal profiles, LinkedIn Pages, Telegram, Discord, and +Google Business Profile even when their TryPost destination publication +succeeded. For included platforms, it counts posts published from any TryPost +entry point, such as the app, API, MCP, or repurpose flows, when they share the +normal publication records. It excludes drafts, scheduled posts that have not +yet published, failed or rejected destinations, and posts created directly on +a social network outside TryPost. A retry that eventually succeeds counts once because the destination record is counted once. Historical publications remain facts even if an account is later @@ -287,9 +298,11 @@ columns require a later product decision. ### Individual post analytics -The existing analytics area inside each published post is part of this same -delivery. It must stop fetching provider metrics during the page request and -must stop treating the five-minute Redis entry as the metric source. +For platforms included in analytics v1, the existing analytics area inside each +published post is part of this same delivery. It must stop fetching provider +metrics during the page request and must stop treating the five-minute Redis +entry as the metric source. Destinations on excluded platforms show no analytics +block. The post-performance pipeline collects through queued jobs and persists through one observation writer. The individual post page, REST API, MCP, Summary, Top 5 @@ -332,10 +345,10 @@ the network's official API documentation before implementation. Buffer does not provide this per-post reference for Telegram, Discord, or Google Business Profile, and it does not expose post analytics for Instagram -Personal accounts or Facebook Groups. Those absences do not remove TryPost -capabilities: Telegram webhook reactions, Discord reactions/thread replies, and -any official Google Business post data are evaluated from their own official -APIs and existing local event sources. +Personal accounts or Facebook Groups. Together with the product decision for +this release, that absence keeps Telegram, Discord, and Google Business Profile +outside analytics v1. Existing locally available reactions, replies, or account +counts for those networks are not promoted into the new analytics surfaces. The Buffer product surfaces are not internally identical. Sent posts, the new Insights product, and the retiring Analyze product can expose different metrics @@ -364,17 +377,19 @@ verify, not permission to invent or request an undocumented field. | Facebook | Feed: impressions, reach, likes, clicks; Story: impressions, reach, interactions, reactions, replies, shares; Reel/video: plays, reactions, interactions | Feed comments/shares and richer Reel actions exposed by current Meta APIs | | Instagram | Reach, views, likes, comments, shares, saves, replies, or total interactions depending on content type | The expanded Feed/Reel/Story catalog below | | X/Twitter | Impressions, likes, retweets, replies, quotes, and bookmarks | Click metrics and their access level | -| LinkedIn Page | Impressions, clicks, likes, comments, and shares | Provider engagement rate plus video views, watch time, and unique viewers | -| LinkedIn personal profile | Likes and comments | Impressions, reach, reliable video views, and derived engagement inputs under the app's approved scopes | +| LinkedIn Page | Impressions, clicks, likes, comments, and shares | Deferred to v2: provider engagement rate plus video views, watch time, and unique viewers | +| LinkedIn personal profile | Likes and comments | Deferred to v2: impressions, reach, reliable video views, and derived engagement inputs under newly approved scopes | | Pinterest | Impressions, saves, Pin clicks, outbound clicks, and video views | Lifetime reactions/comments, rates, audience values, and video-retention metrics described below | | Mastodon | Favorites, replies, and reblogs | No Buffer-identified basic metric gap | | TikTok | Views, likes, comments, and shares | Reach is a Buffer candidate but is unavailable in TryPost's currently approved Display API fields | | YouTube | Views, total watch time, average view duration, likes, comments, and shares | Engaged views, average percentage viewed, and subscriber change described below | | Threads | Views, likes, replies, reposts, and quotes | No Buffer-identified basic metric gap | | Bluesky | Likes, replies, reposts, and quotes | No Buffer-identified basic metric gap | -| Telegram | Locally persisted webhook reactions, plus chat/channel member count shown alongside the post today | Buffer has no comparison; keep account members separate from post performance | -| Discord | Message reactions and thread replies, plus server member count shown alongside the post today | Buffer has no comparison; keep account members separate from post performance | -| Google Business Profile | No individual-post collector | Buffer has no post-analytics reference; remain unsupported until official APIs prove a post-level metric | + +LinkedIn rows above are v2 research only. They are not part of the v1 collector +or persistence scope. Telegram, Discord, and Google Business Profile are omitted +from the implementation inventory because they are excluded from analytics v1 +and have no planned analytics v2 in this design. ### Content-specific metric catalog @@ -468,9 +483,9 @@ The existing analytics range date picker remains the shared page filter for the Summary, follower chart, Posts widget, Top 5 Posts, and Performance. - `minDate` is the earliest follower snapshot or successful TryPost publication - available in the workspace. + on an included v1 platform available in the workspace. - `maxDate` is the latest follower snapshot or successful TryPost publication - available in the workspace. + on an included v1 platform available in the workspace. - The picker cannot select a range wholly outside those bounds. - All chart modes and the total use the same selected range. - A social account connected after the selected start date begins when its own @@ -599,8 +614,10 @@ refreshed in queued jobs and stored behind the same persistence decision gate as follower observations. The daily dispatcher selects successful destination publications that have a -platform post id, a connected account with the required access, and remain -inside their refresh window: +platform post id, use a platform included in analytics v1, have a connected +account with the required access, and remain inside their refresh window. +LinkedIn personal profiles, LinkedIn Pages, Telegram, Discord, and Google +Business Profile are never selected: - X destinations: through 20 days after publication; - every other supported destination: through 30 days after publication. @@ -632,8 +649,9 @@ the system does not replace it with zero. Metrics already maintained from trusted local events, such as webhook-backed reaction metadata, may be normalized from that local source without making a -redundant provider request. Networks without post analytics still contribute -their locally known Posts count but show other values as unavailable. +redundant provider request. An included v1 platform without a supported post +metric may still contribute its locally known Posts count and show the metric as +unavailable. An excluded v1 platform contributes neither posts nor metrics. ## Persistence decision gate @@ -695,9 +713,9 @@ second source of truth. The Posts widget does not require a new analytics snapshot or collection table. Its source of truth is the existing destination publication history. A counted row must belong to a post in the current workspace, have the published status, -and have a `published_at` timestamp inside the selected range. The concrete -query must use the existing enum/status conventions and work on PostgreSQL and -MySQL. +use a platform included in analytics v1, and have a `published_at` timestamp +inside the selected range. The concrete query must use the existing enum/status +conventions and work on PostgreSQL and MySQL. ## Read path @@ -825,7 +843,8 @@ Post-performance collector tests additionally cover: ### Queue and scheduling tests - The daily command dispatches one job per eligible account and none for - LinkedIn, inactive, disconnected, or unsupported accounts. + LinkedIn, Telegram, Discord, Google Business Profile, inactive, disconnected, + or unsupported accounts. - Jobs are isolated and idempotent by account, metric, and date. - Retry delays cover the same UTC day and stop after a successful observation. - Platform-provided retry timing is respected. @@ -833,7 +852,9 @@ Post-performance collector tests additionally cover: - Immediate collection is dispatched after a supported account is connected. - `withoutOverlapping()` and `onOneServer()` remain present on the schedule. - Post-performance jobs are dispatched only for successful destinations with a - usable platform id and access. + usable platform id and access on an included v1 platform. +- No post-performance job is dispatched for LinkedIn personal profiles, + LinkedIn Pages, Telegram, Discord, or Google Business Profile. - X destinations remain eligible through day 20; other supported destinations remain eligible through day 30. - The final eligible day receives a final collection and older destinations no @@ -863,19 +884,20 @@ Post-performance collector tests additionally cover: - A later-connected account does not receive invented earlier points. - Historical series remain available after disconnect/deactivation. - No-data workspaces receive the collection-pending state. -- LinkedIn personal and LinkedIn Page never appear or contribute to follower - analytics v1. +- LinkedIn personal profiles, LinkedIn Pages, Telegram, Discord, and Google + Business Profile never appear or contribute anywhere in analytics v1. - The Posts Bar mode counts one successful destination per social account in the selected range. - The Posts Stacked Bar mode selects daily, weekly, and monthly buckets at the documented range thresholds and zero-fills missing buckets. -- A multi-network post contributes once to every successful destination and - nothing to failed, rejected, pending, or future-scheduled destinations. +- A multi-network post contributes once to every successful destination on an + included v1 platform and nothing to excluded, failed, rejected, pending, or + future-scheduled destinations. - A destination that succeeds after retries counts only once. - Direct/native social-network posts are absent because no TryPost publication record exists for them. -- LinkedIn personal and LinkedIn Page publications appear in the Posts widget - even though both remain excluded from follower analytics v1. +- Publications for every excluded v1 platform are absent from both Posts widget + modes, Summary, Top 5, and Performance. - Post aggregation is workspace-scoped through the parent post. - Historical publications retain presentable account information after the social account is disconnected or deleted. @@ -893,6 +915,8 @@ Post-performance collector tests additionally cover: Summary. - The individual post page, REST API, and MCP return the same persisted latest observation and make no provider request during reads. +- The individual post page exposes no analytics block for a destination on an + excluded v1 platform. - The individual post page shows the content-type-specific catalog, canonical units, freshness, and metric stability/provenance. - Expired Redis entries cannot remove or change persisted post analytics. @@ -916,10 +940,11 @@ design is approved and implemented. - **Persisting workspace totals.** Duplicates account facts and risks drift. - **Choosing the final table structure now.** The wider metric catalog is not yet known, so the choice would be speculative. -- **Including either LinkedIn identity in v1.** Personal analytics require a - separately vetted product, and the product decision for this release is to - defer the whole network to the planned v2 rather than ship partial LinkedIn - support. +- **Including LinkedIn, Telegram, Discord, or Google Business Profile in v1.** + LinkedIn requires the separately vetted product for a coherent implementation; + the other three are outside the chosen product scope and lack a Buffer + per-post analytics reference. LinkedIn is deferred as a whole to v2; the + others require a new future product decision. - **Fetching post counts from social APIs.** The v1 metric represents successful TryPost deliveries, which already have a reliable local destination record; provider analytics would add permissions, rate limits, inconsistent history, @@ -990,5 +1015,5 @@ design is approved and implemented. concrete classes, and ordered implementation tasks. 4. Implementation begins only after that written plan is reviewed and its execution method is selected. -5. LinkedIn follower analytics receives a separate v2 implementation plan - after the external Community Management API dependency is resolved. +5. LinkedIn follower and post analytics receive a separate v2 implementation + plan after the external Community Management API dependency is resolved. From 9b4559ba16474b52278bf16c7a596cd96ff7dcb0 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 10:23:13 -0300 Subject: [PATCH 10/77] docs: plan native post history import --- ...-23-workspace-follower-analytics-design.md | 408 +++++++++++++----- 1 file changed, 300 insertions(+), 108 deletions(-) diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index d7e651196..96e821138 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -5,19 +5,21 @@ ## Objective Replace the request-time, per-social-account analytics experience with an -initial workspace-level analytics view covering follower history and posts -successfully published through TryPost. +initial workspace-level analytics view covering follower history, posts +successfully published through TryPost, and native posts imported from connected +social accounts. After the daily collection pipeline begins producing local snapshots, the page -must answer six questions without querying social APIs at request time: +must answer seven questions without querying social APIs at request time: 1. How many followers did this workspace have at the end of the selected period? 2. How did each connected social account's follower count change over that period? 3. Which accounts gained or lost followers? -4. How many posts did each social account successfully publish during the - selected period, and how was that volume distributed over time? +4. How many posts did each social account publish during the selected period, + whether through TryPost or natively, and how was that volume distributed over + time? 5. How did publication volume, reactions, comments, and engagement compare with the immediately preceding equivalent period? 6. Which destination publications and social accounts performed best? @@ -27,8 +29,8 @@ must answer six questions without querying social APIs at request time: Success means `/analytics` renders without making social API calls, daily follower collection is resilient to transient failures and rate limits, one broken platform cannot block another account's data, and post volume is derived -from the local publication history. Post-performance metrics are also collected -ahead of page requests and retained locally. +from a local unified publication catalog. Post history and performance metrics +are imported or collected ahead of page requests and retained locally. ## Scope boundary @@ -154,10 +156,82 @@ network. The initial display mode is Line. Changing modes is client-side because all three views derive from the same response dataset. +### Native post history import + +Connecting an account on a platform included in analytics v1 dispatches a +non-blocking native-history backfill. The product target is every owned post +published during the preceding 365 days. The importer paginates until it reaches +that cutoff, exhausts the provider feed, or encounters a documented provider +limit. It never claims a complete year when the API returned less. + +The account connection succeeds before the backfill finishes. Analytics shows +an import-progress state with the oldest covered publication date, latest +successful sync time, and whether coverage is complete, provider-limited, +partially failed, or still running. Imported results become visible +incrementally; one slow account cannot delay another. + +After the initial backfill, a daily queued discovery job imports newly published +native posts. This is separate from the follower and metric collectors. It uses +a per-account high-water mark plus provider cursor checkpoints, overlaps the +last completed window to tolerate late provider results, and relies on an +idempotent key of workspace + social account + platform + native post id. +Reconnects of the same platform identity resume the catalog rather than creating +a second history. + +Feature rollout also dispatches the same resumable backfill for every already +connected, active account on an included v1 platform. Rollout work is chunked +and rate limited; it does not require users to disconnect and reconnect to seed +their analytics. + +Each imported publication retains, when available: + +- workspace and social-account ownership; +- native post id and platform; +- provider publication timestamp; +- content type; +- permalink; +- caption or textual excerpt; +- preview/thumbnail reference and enough immutable presentation metadata to + render a historical card; +- discovery and last-sync timestamps; +- origin (`trypost` or `native_import`); +- provider coverage and availability state. + +An imported post is an analytics record, not a draft or published `Post` owned +by the TryPost publishing workflow. Importing it must not enable editing, +deletion, retry, repurpose processing, or publishing lifecycle actions. The +physical persistence design may share a catalog with TryPost destinations, but +it must not manufacture `posts` or `post_platforms` rows whose states imply that +TryPost published the content. + +When discovery returns a native id already present on a TryPost destination, +the records reconcile into one analytics publication and `trypost` origin wins. +This prevents one post from being counted twice. The same reconciliation runs +when a delayed publish result gains its provider id after native discovery. + +The current Repurpose source fetchers prove that Instagram and Facebook media +can already be discovered with the connected account. The analytics importer +may extract and share their low-level provider clients and response parsers, but +it does not reuse `PollRepurposeSource`, `RepurposeItem`, media-download rules, +or activation watermarks. Those components fetch only selected video formats, +currently request one page of 25 items, and have different lifecycle semantics. + +For a newly imported publication inside the normal post-metric refresh window, +the importer dispatches the ordinary post-performance job. For an older post in +the 365-day backfill, it dispatches one rate-limited baseline metric collection +and then retains the result without enrolling the post in perpetual daily +refresh. A provider that no longer exposes metrics leaves the post visible with +an explicit unavailable state. + +Post-history backfill does not fabricate follower history. Follower charts begin +at the first real follower observation unless the provider has a separately +documented historical follower endpoint. + ### Posts chart -A second widget shows the number of destinations successfully published through -TryPost during the selected range. It uses two modes: +A second widget shows the number of analytics publications in the selected +range, combining successful TryPost destinations with imported native posts. It +uses two modes: - **Bar:** horizontal total per social account across the entire selected range. @@ -175,20 +249,20 @@ The first and last weekly or monthly buckets may be partial when the selected range begins or ends inside that period. Empty buckets are returned with zero values so the time axis remains continuous. -One successful destination counts as one post for that social account. For +One analytics publication counts as one post for its social account. For example, one TryPost post successfully delivered to Instagram and X contributes -one count to each account. The metric is based on the destination publication -record, not the parent post, so a partially successful multi-network post counts -only its successful destinations. +one count to each account, while one natively published Instagram post adds one +Instagram count. The metric is based on the reconciled destination/native +publication, not the parent TryPost post, so a partially successful +multi-network post counts only its successful destinations. The Posts widget includes only the platforms included in analytics v1. It excludes LinkedIn personal profiles, LinkedIn Pages, Telegram, Discord, and Google Business Profile even when their TryPost destination publication succeeded. For included platforms, it counts posts published from any TryPost -entry point, such as the app, API, MCP, or repurpose flows, when they share the -normal publication records. It excludes drafts, scheduled posts that have not -yet published, failed or rejected destinations, and posts created directly on -a social network outside TryPost. +entry point, such as the app, API, MCP, or repurpose flows, plus native posts +discovered through the connected account. It excludes drafts, scheduled posts +that have not yet published, and failed or rejected destinations. A retry that eventually succeeds counts once because the destination record is counted once. Historical publications remain facts even if an account is later @@ -200,16 +274,16 @@ row is no longer available. The page includes one workspace-level Summary block with exactly five cards: -- **Posts:** successful destination publications whose `published_at` falls - inside the selected range. +- **Posts:** reconciled TryPost and imported native analytics publications whose + provider publication timestamp falls inside the selected range. - **Total Followers:** the follower total at the selected range's end date, using the same eligibility rules as the follower widget. -- **Reactions:** the sum of the latest stored reactions for successful - destination publications inside the selected range. -- **Comments:** the sum of the latest stored comments for successful - destination publications inside the selected range. +- **Reactions:** the sum of the latest stored reactions for eligible analytics + publications inside the selected range. +- **Comments:** the sum of the latest stored comments for eligible analytics + publications inside the selected range. - **Engagement Rate:** pooled engagement divided by pooled exposure for the - eligible destination publications inside the selected range. + eligible analytics publications inside the selected range. Cross-network labels are normalized for comparison. Reactions include native likes, favorites, and reactions. Comments include native comments and replies @@ -253,16 +327,17 @@ The page includes one Top 5 Posts block with a two-option toggle: - **Reactions** is the initial ranking; - **Comments** ranks the same eligible dataset by normalized comments. -The ranking unit is the successful destination publication, not the parent -post. A parent sent to multiple social accounts may therefore appear more than -once when more than one destination qualifies. Only destinations published -inside the selected range participate. +The ranking unit is the reconciled analytics publication, not the parent post. +A parent sent to multiple social accounts may therefore appear more than once +when more than one destination qualifies. Imported native posts participate as +their own publication. Only publications inside the selected range participate. Each card shows rank, normalized metric value, platform/account identity, -publication date, content type, excerpt, thumbnail when available, and actions -to open the TryPost post or its public social URL when supported. Ties are -resolved by newest `published_at` and then by stable destination id so the order -does not jump between requests. +publication date, content type, excerpt, thumbnail when available, publication +origin, and actions to open the TryPost post or its public social URL when +supported. An imported native post has no edit action or fake TryPost post link. +Ties are resolved by newest provider publication timestamp and then by stable +analytics-publication id so the order does not jump between requests. A destination whose selected ranking metric is unsupported is excluded from that ranking. Fewer than five cards are shown when fewer than five eligible @@ -270,9 +345,9 @@ destinations have a real value. An empty state replaces the list when none do. ### Performance -The page includes one Performance table with one row per social account that -has a successful destination publication in the selected range. Multiple -accounts on the same network remain separate rows. +The page includes one Performance table with one row per social account that has +an eligible TryPost or imported native publication in the selected range. +Multiple accounts on the same network remain separate rows. The fixed first-version columns are: @@ -282,11 +357,11 @@ The fixed first-version columns are: - Comments; - Engagement Rate. -Posts use the local successful-destination count. The other columns aggregate -the latest stored post-performance observations using the same normalization -and pooled-rate rules as Summary. Each supported numeric column can be sorted, -and its current value includes the equivalent-period comparison when a valid -comparison exists. +Posts use the local reconciled analytics-publication count. The other columns +aggregate the latest stored post-performance observations using the same +normalization and pooled-rate rules as Summary. Each supported numeric column +can be sorted, and its current value includes the equivalent-period comparison +when a valid comparison exists. When a network or content type does not expose a metric, the cell shows an unavailable marker rather than zero. Historical account snapshot metadata keeps @@ -304,10 +379,17 @@ metrics during the page request and must stop treating the five-minute Redis entry as the metric source. Destinations on excluded platforms show no analytics block. +Imported native posts open a read-only analytics detail using the same metric +components and observation contract. Every detail view displays an origin label: +`Published via TryPost` for a matched TryPost destination, or `Published on +` for an imported native publication. Origin is stored data, not +inferred from the presence of a local caption or URL. + The post-performance pipeline collects through queued jobs and persists through one observation writer. The individual post page, REST API, MCP, Summary, Top 5 Posts, and Performance all read the same latest persisted observation for each -destination. Redis is not a source of truth for post analytics; a +reconciled analytics publication. Redis is not a source of truth for post +analytics; a database-query cache may be added later only if profiling proves it useful. The individual post page is richer than the cross-network reporting blocks. It @@ -482,18 +564,18 @@ reason to erase a previous observation. The existing analytics range date picker remains the shared page filter for the Summary, follower chart, Posts widget, Top 5 Posts, and Performance. -- `minDate` is the earliest follower snapshot or successful TryPost publication - on an included v1 platform available in the workspace. -- `maxDate` is the latest follower snapshot or successful TryPost publication - on an included v1 platform available in the workspace. +- `minDate` is the earliest follower snapshot or reconciled TryPost/native + analytics publication on an included v1 platform available in the workspace. +- `maxDate` is the latest follower snapshot or reconciled TryPost/native + analytics publication on an included v1 platform available in the workspace. - The picker cannot select a range wholly outside those bounds. - All chart modes and the total use the same selected range. - A social account connected after the selected start date begins when its own data begins; no pre-connection values are invented. - A widget shows its own empty state when the selected range contains no data for that metric. -- With neither follower snapshots nor successful publications, the picker is - disabled and the page shows an analytics-empty state. +- With neither follower snapshots nor analytics publications, the picker is + disabled and the page shows an analytics-empty/import-pending state. Historical data for a disconnected or deactivated account is retained. Its line ends on the last day for which it was eligible; it remains visible when @@ -503,6 +585,12 @@ the selected range overlaps that history. ```text Laravel scheduler (daily, UTC) + -> native-post discovery dispatch-only command + -> one queued incremental discovery job per eligible social account + -> platform-owned-post paginator + -> reconciled analytics publication catalog + -> post-performance jobs for new publications + -> follower dispatch-only command -> one queued job per eligible social account -> platform follower collector @@ -510,7 +598,7 @@ Laravel scheduler (daily, UTC) -> persistence boundary -> post-performance dispatch-only command - -> one queued job per eligible destination publication + -> one queued job per eligible reconciled analytics publication -> platform post-metrics collector or trusted local metric source -> normalized post-performance observation -> persistence boundary @@ -521,6 +609,10 @@ Laravel scheduler (daily, UTC) End-of-day finalizer -> identifies eligible accounts without a successful observation -> carries forward the most recent known value when one exists + +Account connection + -> immediate follower collection + -> resumable native-post backfill through the preceding 365 days ``` ### Scheduler and dispatcher @@ -541,9 +633,12 @@ logical uniqueness key is follower metric + social account + UTC observation date. Re-dispatching the same logical job is safe and cannot create a second daily value. -Connecting a supported account dispatches an immediate first collection so the -workspace does not wait for the next daily sweep. This initial job follows the -same idempotency and retry policy as the scheduled job. +Connecting a supported account dispatches an immediate first follower +collection and the resumable native-history backfill so the workspace does not +wait for the next daily sweep. These jobs follow the same isolation, +idempotency, and widely spaced retry principles as their scheduled counterparts. +Backfill jobs run in bounded pages and re-dispatch the next page rather than +holding one worker for the entire year. ### Collector contract @@ -613,11 +708,11 @@ calls while `/analytics` or an individual post is rendering. Metrics are refreshed in queued jobs and stored behind the same persistence decision gate as follower observations. -The daily dispatcher selects successful destination publications that have a -platform post id, use a platform included in analytics v1, have a connected -account with the required access, and remain inside their refresh window. -LinkedIn personal profiles, LinkedIn Pages, Telegram, Discord, and Google -Business Profile are never selected: +The daily dispatcher selects reconciled TryPost/native analytics publications +that have a native post id, use a platform included in analytics v1, have a +connected account with the required access, and remain inside their refresh +window. LinkedIn personal profiles, LinkedIn Pages, Telegram, Discord, and +Google Business Profile are never selected: - X destinations: through 20 days after publication; - every other supported destination: through 30 days after publication. @@ -629,16 +724,16 @@ There is no free-versus-paid retention rule in TryPost. All workspaces use the same collection windows. The windows limit external API work only; all values already collected are retained permanently. -Each eligible destination gets an independent queued job so one failing API or -post cannot block another. The logical uniqueness key is post-performance + -destination + UTC collection date. The job normalizes only metrics genuinely -returned for that network and content type, preserving unsupported separately -from a measured zero. +Each eligible analytics publication gets an independent queued job so one +failing API or post cannot block another. The logical uniqueness key is +post-performance + analytics publication + UTC collection date. The job +normalizes only metrics genuinely returned for that network and content type, +preserving unsupported separately from a measured zero. -Post-performance values are cumulative totals for that destination as of the -collection timestamp. Summary, Top 5 Posts, and Performance use the latest -stored observation for each destination selected by its publication date; they -do not add daily snapshots together. +Post-performance values are cumulative totals for that analytics publication as +of the collection timestamp. Summary, Top 5 Posts, and Performance use the +latest stored observation for each publication selected by its provider +publication date; they do not add daily snapshots together. The normal daily run collects once per UTC day. The final eligible day performs one final collection before the destination becomes inactive for scheduled @@ -705,17 +800,23 @@ Regardless of the final schema, persistence must support: - stable metric keys and units independent of the active UI locale; - content-type-specific metrics without sparse schema assumptions; - provider and collection timestamps needed to disclose freshness; +- a reconciled analytics-publication identity spanning TryPost destinations and + imported native posts without duplicating a native post id; +- publication origin, provider publication time, content excerpt, permalink, + content type, presentation metadata, and import coverage state; +- resumable per-account provider cursor/high-water checkpoints; - efficient workspace, publication-range, account, and ranking aggregations. Workspace totals are derived from account observations and are not stored as a second source of truth. -The Posts widget does not require a new analytics snapshot or collection table. -Its source of truth is the existing destination publication history. A counted -row must belong to a post in the current workspace, have the published status, -use a platform included in analytics v1, and have a `published_at` timestamp -inside the selected range. The concrete query must use the existing enum/status -conventions and work on PostgreSQL and MySQL. +The Posts widget does not require daily count snapshots. Its source of truth is +the reconciled analytics-publication catalog: existing successful TryPost +destinations plus imported native posts. A counted record must belong to the +current workspace, use a platform included in analytics v1, have a provider +publication timestamp inside the selected range, and represent one unique +social-account/native-id pair. The concrete persistence and reconciliation +queries must work on PostgreSQL and MySQL. ## Read path @@ -731,14 +832,19 @@ The server response supplies: - account identity and platform presentation metadata; - actual/carried-forward and exact/approximate provenance required for truthful tooltips; -- successful publication totals per social account; +- reconciled TryPost/native publication totals per social account; - zero-filled publication buckets and per-account values for the automatically selected daily, weekly, or monthly resolution; - current and previous-period Summary values; - the two deterministic Top 5 rankings; - Performance rows and comparisons per social account; -- the complete latest metric set for each destination on the individual post - page, REST API, and MCP; +- the complete latest metric set for each analytics publication on the + individual post page, REST API, and MCP; +- publication origin, provider publication time, permalink, content type, and + presentation metadata required by native-import detail cards; +- native-history coverage per social account, including progress, oldest + covered publication date, last successful sync, and complete, + provider-limited, partial-failure, or running state; - freshness and availability metadata needed for tooltips and unavailable states. @@ -748,9 +854,10 @@ daily resolution is the source resolution and is sufficient for this first version. The server aggregates the Posts dataset at the chosen bucket resolution and -returns both bucketed and range-total values. Publication rows are always -filtered through their parent post's `workspace_id`; a social-account id from -the request is never trusted as the tenancy boundary. +returns both bucketed and range-total values. Every analytics-publication row +has its own immutable workspace ownership, including imported native posts +that have no parent TryPost post. A social-account id from the request is never +trusted as the tenancy boundary. ## Failure handling and observability @@ -769,8 +876,14 @@ Operational visibility must distinguish: - post-performance collection success; - post-performance metric unsupported; - post-performance retry or permanent collection failure; -- post-performance destination leaving its refresh window with a final stored +- post-performance publication leaving its refresh window with a final stored value. +- native-history backfill start, page progress, completion, and oldest covered + publication date; +- native-history backfill or incremental-discovery retry and permanent + failure; +- provider-limited history distinguished from a complete 365-day import; +- native publication reconciliation and duplicate suppression. Logs include workspace, social account, platform, observation date, attempt, and error category, but never access tokens or raw sensitive responses. @@ -782,14 +895,20 @@ this first delivery. ## Account lifecycle - **Connected:** dispatch an immediate first collection. -- **Active and connected:** participate in the daily sweep. +- **Connected on an included v1 platform:** also dispatch the resumable native + post-history backfill without delaying the connection response. +- **Active and connected:** participate in the daily follower, + post-performance, and native-post discovery sweeps. - **Deactivated:** stop new collection and fallback; preserve history; exclude - from totals after its last eligible date. + from follower totals after its last eligible date; preserve imported and + TryPost publication history for ranges in which it exists. - **Disconnected/deleted:** stop collection and fallback; preserve historical - observations even if the account row is later removed. The physical schema - design must decide how to retain enough immutable identity for this. + observations and imported publications even if the account row is later + removed. The physical schema design must decide how to retain enough + immutable identity for this. - **Reconnected as the same persisted identity:** resume collection without - rewriting earlier observations. + rewriting earlier observations and resume native discovery from its + checkpoint with an overlap window. - **New identity:** begins a new series even when its username matches an older disconnected account. @@ -850,9 +969,26 @@ Post-performance collector tests additionally cover: - Platform-provided retry timing is respected. - A permanent authentication failure does not follow the transient retry loop. - Immediate collection is dispatched after a supported account is connected. +- Connecting an included v1 account dispatches a native-history backfill and + returns without waiting for that backfill to finish. +- Rollout dispatches bounded backfills for existing eligible accounts without + requiring reconnection and without placing every workspace in one job. +- No native-history backfill or incremental-discovery job is dispatched for + LinkedIn personal profiles, LinkedIn Pages, Telegram, Discord, or Google + Business Profile. +- Native-history pagination stops at the 365-day cutoff, provider exhaustion, + or a documented provider limit and records which condition ended the import. +- A bounded page can re-dispatch continuation work without holding one worker + for the entire backfill. +- Cursor and high-water checkpoints resume safely after transient failure and + after reconnecting the same platform identity. +- Daily native discovery overlaps the last completed window and remains + idempotent when a provider returns the same page or a late post twice. +- Backfill and discovery failures for one social account do not block any other + account. - `withoutOverlapping()` and `onOneServer()` remain present on the schedule. -- Post-performance jobs are dispatched only for successful destinations with a - usable platform id and access on an included v1 platform. +- Post-performance jobs are dispatched only for reconciled analytics + publications with a usable native id and access on an included v1 platform. - No post-performance job is dispatched for LinkedIn personal profiles, LinkedIn Pages, Telegram, Discord, or Google Business Profile. - X destinations remain eligible through day 20; other supported destinations @@ -860,6 +996,10 @@ Post-performance collector tests additionally cover: - The final eligible day receives a final collection and older destinations no longer create provider jobs. - Collection-window expiry never deletes an already stored value. +- A newly imported publication still inside the refresh window joins the + normal daily metric collection. +- An older backfilled publication receives at most the planned baseline metric + collection and is not enrolled in perpetual refresh. - Unsupported metrics remain distinct from measured zero. - Instagram Story jobs collect while insights are available and perform a final pre-expiry collection even when the normal daily sweep would miss it. @@ -872,6 +1012,25 @@ Post-performance collector tests additionally cover: - No historical value means no fabricated snapshot. - Deactivated and disconnected accounts are not carried forward. - A successful observation is never overwritten by the finalizer. +- Importing historical posts never creates historical follower observations or + follower carry-forward values before the first real collection. + +### Native publication reconciliation tests + +- Repeated provider pages create one analytics publication for each unique + workspace, social account, platform, and native post id. +- A native id matching an existing successful TryPost destination reconciles + to one analytics publication and retains `trypost` as its origin. +- A TryPost destination that receives its native id after discovery reconciles + with the imported publication rather than creating a duplicate. +- Reconnecting the same identity resumes the existing catalog; a genuinely new + platform identity starts a separate catalog even when the username matches. +- Imported publications never create fake `Post` or `PostPlatform` lifecycle + records and cannot be edited, deleted, retried, or published from TryPost. +- The provider publication timestamp, rather than discovery time, controls + range inclusion and aggregation. +- Expired or unavailable preview media falls back to a stable placeholder + without removing the imported publication or its metrics. ### Read and UI tests @@ -886,19 +1045,24 @@ Post-performance collector tests additionally cover: - No-data workspaces receive the collection-pending state. - LinkedIn personal profiles, LinkedIn Pages, Telegram, Discord, and Google Business Profile never appear or contribute anywhere in analytics v1. -- The Posts Bar mode counts one successful destination per social account in - the selected range. +- The Posts Bar mode counts one reconciled TryPost/native analytics publication + per social account in the selected range. - The Posts Stacked Bar mode selects daily, weekly, and monthly buckets at the documented range thresholds and zero-fills missing buckets. - A multi-network post contributes once to every successful destination on an included v1 platform and nothing to excluded, failed, rejected, pending, or future-scheduled destinations. - A destination that succeeds after retries counts only once. -- Direct/native social-network posts are absent because no TryPost publication - record exists for them. +- Direct/native social-network posts are included after discovery even though + no TryPost publication record exists for them. +- Native imports appear incrementally while backfill is running, and the UI + exposes oldest-covered date, last sync, and complete, provider-limited, + partial-failure, or running coverage state without promising unavailable + history. - Publications for every excluded v1 platform are absent from both Posts widget modes, Summary, Top 5, and Performance. -- Post aggregation is workspace-scoped through the parent post. +- Post aggregation is workspace-scoped through immutable analytics-publication + ownership for both TryPost and native imports. - Historical publications retain presentable account information after the social account is disconnected or deleted. - Summary contains exactly Posts, Total Followers, Reactions, Comments, and @@ -908,17 +1072,25 @@ Post-performance collector tests additionally cover: - Engagement Rate pools normalized engagement and exposure rather than averaging per-post percentages. - Posts without a valid exposure denominator are excluded only from the rate. -- Top 5 ranks destination publications deterministically by Reactions or - Comments and excludes unsupported values. +- Top 5 ranks reconciled analytics publications deterministically by Reactions + or Comments, includes eligible native imports, and excludes unsupported + values. +- Imported Top 5 and detail cards show `Published on `, while + reconciled TryPost publications show `Published via TryPost`; origin is read + from persisted provenance rather than inferred from missing relations. +- Imported publication cards expose no fake TryPost edit, retry, or publishing + action. - Performance returns one row per social account, keeps duplicate-network - accounts separate, supports sorting, and uses the same aggregation rules as - Summary. + accounts separate, includes reconciled native publications, supports sorting, + and uses the same aggregation rules as Summary. - The individual post page, REST API, and MCP return the same persisted latest observation and make no provider request during reads. - The individual post page exposes no analytics block for a destination on an excluded v1 platform. - The individual post page shows the content-type-specific catalog, canonical units, freshness, and metric stability/provenance. +- Date-picker bounds expand to the earliest eligible imported provider + publication date as backfill progresses. - Expired Redis entries cannot remove or change persisted post analytics. Database-dependent tests run on PostgreSQL and MySQL after the persistence @@ -945,21 +1117,35 @@ design is approved and implemented. the other three are outside the chosen product scope and lack a Buffer per-post analytics reference. LinkedIn is deferred as a whole to v2; the others require a new future product decision. -- **Fetching post counts from social APIs.** The v1 metric represents successful - TryPost deliveries, which already have a reliable local destination record; - provider analytics would add permissions, rate limits, inconsistent history, - and native posts outside the agreed definition. +- **Keeping Posts limited to TryPost deliveries.** It would make a newly + connected workspace look empty and omit the user's best historical content. + A native-history importer gives Summary, Top 5, Performance, and individual + post analytics useful data immediately while preserving publication origin. - **Counting parent posts.** One parent can target several accounts and can partially fail, so the successful destination is the only accurate unit. - **Persisting daily post-count snapshots.** Publication rows are immutable - facts that can be aggregated for the selected range without introducing a - second source of truth. + facts in the reconciled catalog and can be aggregated for the selected range + without introducing a second source of truth. +- **Reusing `PollRepurposeSource` and `RepurposeItem` for analytics.** Repurpose + imports selected media for a publishing workflow, currently reads one page + of 25, and has activation-watermark and media-download semantics that do not + represent a complete, read-only analytics catalog. Only suitable low-level + provider clients and parsers may be extracted and shared. +- **Creating fake `Post` or `PostPlatform` rows for native content.** Those + records imply TryPost publishing ownership and would expose invalid edit, + retry, delete, and repurpose actions. Native content remains an analytics + publication with explicit origin. +- **Claiming one year of coverage unconditionally.** Providers can impose + shallower history, pagination, permission, or metric-retention limits. The + importer targets 365 days but reports the actual oldest covered date and a + provider-limited or partial state when needed. - **Using one fixed Posts bucket size.** A fixed daily view becomes noisy over long ranges, while a fixed weekly or monthly view hides useful short-range detail. -- **Refreshing every historical post forever.** Engagement changes slow after - publication, while an unbounded daily job set would continually increase API - cost and rate-limit pressure. The last stored result remains available after +- **Refreshing every imported historical post forever.** Engagement changes + slow after publication, while an unbounded daily job set would continually + increase API cost and rate-limit pressure. Older backfilled posts receive a + bounded baseline collection; the last stored result remains available after the 20/30-day refresh window closes. - **Applying plan-based analytics retention.** TryPost has no free analytics tier in this design; collection and permanent local retention are consistent @@ -1009,11 +1195,17 @@ design is approved and implemented. ## Delivery gates 1. This written design must be reviewed and approved. -2. The broader metric catalog must be supplied and its persistence design - approved. -3. Only then can the Superpowers implementation-plan stage define migrations, +2. Before promising native-history import for a v1 platform, verify in that + platform's current official documentation the owned-post enumeration + endpoint, pagination, scopes, accessible content types, history depth, + metric-retention limits, and preview-media expiry. Record any shallower + provider limit in the coverage contract instead of weakening it silently. +3. The broader metric catalog must be supplied and its persistence design + approved, including the reconciled TryPost/native publication identity and + resumable import checkpoints. +4. Only then can the Superpowers implementation-plan stage define migrations, concrete classes, and ordered implementation tasks. -4. Implementation begins only after that written plan is reviewed and its +5. Implementation begins only after that written plan is reviewed and its execution method is selected. -5. LinkedIn follower and post analytics receive a separate v2 implementation +6. LinkedIn follower and post analytics receive a separate v2 implementation plan after the external Community Management API dependency is resolved. From 0b70cd3cfb659744ee1e03cb7db8fab50293f1f7 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 10:51:46 -0300 Subject: [PATCH 11/77] docs: plan workspace analytics backfill --- ...2026-09-23-workspace-analytics-backfill.md | 1122 +++++++++++++++++ ...-23-workspace-follower-analytics-design.md | 183 ++- 2 files changed, 1254 insertions(+), 51 deletions(-) create mode 100644 docs/superpowers/plans/2026-09-23-workspace-analytics-backfill.md diff --git a/docs/superpowers/plans/2026-09-23-workspace-analytics-backfill.md b/docs/superpowers/plans/2026-09-23-workspace-analytics-backfill.md new file mode 100644 index 000000000..527065888 --- /dev/null +++ b/docs/superpowers/plans/2026-09-23-workspace-analytics-backfill.md @@ -0,0 +1,1122 @@ +# Workspace Analytics and Native Backfill Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace request-time social analytics with workspace-scoped, database-backed follower history, reconciled TryPost/external publication history, persisted post metrics, and the Summary, Followers, Posts, Top 5 Posts, Performance, and individual-publication views. + +**Architecture:** Four tables separate daily account facts, publication identity, daily cumulative publication metrics, and durable synchronization state. Every provider call runs in an isolated queued job; page, API, and MCP reads use local query services only. Provider adapters normalize platform responses into stable DTOs, while a hybrid scalar-plus-JSON snapshot keeps cross-network queries portable across PostgreSQL and MySQL and preserves content-specific metrics. + +**Tech Stack:** PHP 8.5, Laravel 13.24, Horizon 5.47, PostgreSQL and MySQL, Inertia 3.3, Vue 3.5, Tailwind CSS 4, Pest 5, Pest Browser 5. + +**Spec:** `docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md` + +## Global Constraints + +- Execute all tasks on one branch named `feat/workspace-analytics-backfill`. +- Analytics tenancy and aggregation are always scoped to the current workspace; never trust a request-provided social-account id as tenancy proof. +- Multiple accounts on the same network remain separate series and Performance rows through `social_account_id` while connected and `social_account_key` historically; reconnecting the same workspace + network + provider user id reuses its historical key. +- V1 includes TikTok, Instagram, Instagram through Facebook, Facebook Pages, Threads, X, Pinterest, YouTube, Bluesky, and Mastodon. +- V1 excludes LinkedIn profile, LinkedIn Page, Telegram, Discord, and Google Business Profile from every analytics read, collection job, summary, chart, and post-detail block. +- LinkedIn profile and Page analytics remain a separate V2 after Community Management API approval. +- `origin=trypost` means a matching TryPost destination proves ownership; every other discovered publication is `origin=external` and renders `Published on `. +- Target 365 days of owned-publication history, but persist and expose actual coverage when a provider is shallower, partial, or permission-limited. +- Never fabricate follower history, historical post-metric snapshots, unsupported metrics, zero values after provider failure, or a native/manual origin the provider cannot prove. +- Followers retry at widely spaced same-day windows and carry the last value forward only after the day is exhausted; provider `Retry-After` wins when valid. +- `/analytics`, post detail, REST, and MCP make no social-provider calls and never use Redis as the analytics source of truth. +- Common aggregate metrics are nullable scalar columns; content-specific metrics use stable enum-backed JSON keys with value, unit, time basis, precision, availability, and provider identity. +- Use string columns plus PHP backed enums; do not use database-native enum types. +- Every query, migration, unique constraint, and test must work on PostgreSQL and MySQL. +- Do not add a charting dependency; use focused Vue/SVG/CSS components and existing UI primitives. +- Do not alter unrelated `package-lock.json` or `ANALYTIC.md` changes already present in the worktree. +- Generate Laravel files with `php artisan make:* --no-interaction`, use Pest TDD, run `vendor/bin/pint --dirty --format agent` after PHP edits, and commit after each task. + +## Review Focus + +- Two Instagram accounts in one workspace must remain independent in totals, charts, publication counts, and sorting; Task 12 adds a cross-network-duplicate account test. +- Deleting a live social account must null the foreign key without erasing historical identity or presentation; Tasks 1 and 2 test `social_account_key` and snapshot retention. +- Provider null/missing metrics must stay unavailable while a measured numeric zero remains zero; Tasks 3 and 10 add explicit parser and writer tests. +- Concurrent TryPost sync and external discovery of the same provider post id must converge to one publication with `trypost` origin; Task 5 tests both arrival orders. +- A failed paginated backfill must resume from the last committed cursor and disclose partial/provider-limited coverage instead of restarting or claiming 365 days; Task 9 tests checkpoint, retry, and completion conditions. + +--- + +## File Structure + +The implementation introduces these focused areas: + +- `app/Enums/Analytics/*`: stable persisted states, metric keys, units, and provenance. +- `app/Models/Analytics*`: four persistence boundaries and their relationships. +- `app/Dto/Analytics/*`: provider-independent account, publication, page, and metric results. +- `app/Contracts/Analytics/*`: follower, history, and publication-metric collector contracts. +- `app/Services/Analytics/Collectors/*`: one provider adapter per concern; no authorization or database writes. +- `app/Actions/Analytics/*`: idempotent writers and publication reconciliation. +- `app/Jobs/Analytics/*`: one bounded piece of external or local synchronization per job. +- `app/Console/Commands/Analytics/*`: chunked dispatchers and rollout entry points; commands never call providers. +- `app/Queries/Analytics/*`: workspace-only local read models for dashboard and publication detail. +- `resources/js/components/analytics/workspace/*`: reusable dashboard cards and dependency-free SVG/CSS charts. +- `tests/Feature/Analytics/*`, `tests/Unit/Analytics/*`, and `tests/Browser/WorkspaceAnalyticsTest.php`: provider contracts, persistence, queue behavior, read paths, and UI coverage. + +### Task 1: Create the four-table analytics schema and persisted enums + +**Files:** +- Create: `database/migrations/2026_09_23_103300_create_analytics_account_daily_snapshots_table.php` +- Create: `database/migrations/2026_09_23_103301_create_analytics_publications_table.php` +- Create: `database/migrations/2026_09_23_103302_create_analytics_publication_daily_snapshots_table.php` +- Create: `database/migrations/2026_09_23_103303_create_analytics_sync_states_table.php` +- Create: `app/Enums/Analytics/ObservationProvenance.php` +- Create: `app/Enums/Analytics/MetricPrecision.php` +- Create: `app/Enums/Analytics/PublicationOrigin.php` +- Create: `app/Enums/Analytics/PublicationAvailability.php` +- Create: `app/Enums/Analytics/PublicationContentType.php` +- Create: `app/Enums/Analytics/ExposureKind.php` +- Create: `app/Enums/Analytics/MetricUnit.php` +- Create: `app/Enums/Analytics/MetricTimeBasis.php` +- Create: `app/Enums/Analytics/MetricAvailability.php` +- Create: `app/Enums/Analytics/MetricKey.php` +- Create: `app/Enums/Analytics/SyncCollector.php` +- Create: `app/Enums/Analytics/SyncStatus.php` +- Test: `tests/Feature/Analytics/AnalyticsSchemaTest.php` + +**Interfaces:** +- Consumes: existing UUID workspace, social-account, and post-platform keys; `App\Enums\SocialAccount\Platform`. +- Produces: the four tables and enum values consumed by every later task. + +- [ ] **Step 1: Generate the migrations and failing schema test** + +Run: + +```bash +php artisan make:migration create_analytics_account_daily_snapshots_table --no-interaction +php artisan make:migration create_analytics_publications_table --no-interaction +php artisan make:migration create_analytics_publication_daily_snapshots_table --no-interaction +php artisan make:migration create_analytics_sync_states_table --no-interaction +php artisan make:test --pest Analytics/AnalyticsSchemaTest --no-interaction +``` + +Add assertions that all four tables exist, that account and publication rows accept two distinct account UUIDs on the same platform, and that deleting `social_accounts` nulls live foreign keys while `social_account_key`, platform, username, and historical facts remain. + +```php +expect(Schema::hasColumns('analytics_account_daily_snapshots', [ + 'workspace_id', 'social_account_id', 'social_account_key', 'platform', + 'network', 'platform_user_id', + 'snapshot_date', 'followers_count', 'metrics', 'provenance', 'precision', + 'provider_observed_at', 'collected_at', +]))->toBeTrue(); +``` + +- [ ] **Step 2: Run the schema test and verify it fails** + +Run: `php artisan test --compact tests/Feature/Analytics/AnalyticsSchemaTest.php` + +Expected: FAIL because the migrations and enum classes are empty or incomplete. + +- [ ] **Step 3: Define the exact persisted enum vocabulary** + +Use backed string enums. The cases and values are: + +```php +enum ObservationProvenance: string { case Actual = 'actual'; case CarriedForward = 'carried_forward'; } +enum MetricPrecision: string { case Exact = 'exact'; case Approximate = 'approximate'; case Estimated = 'estimated'; case Experimental = 'experimental'; } +enum PublicationOrigin: string { case TryPost = 'trypost'; case External = 'external'; } +enum PublicationAvailability: string { case Available = 'available'; case Deleted = 'deleted'; case Unavailable = 'unavailable'; } +enum ExposureKind: string { case Reach = 'reach'; case Impressions = 'impressions'; case Views = 'views'; } +enum MetricUnit: string { case Count = 'count'; case Seconds = 'seconds'; case Percent = 'percent'; } +enum MetricTimeBasis: string { case Lifetime = 'lifetime'; case Range = 'range'; case Rolling90Days = 'rolling_90_days'; case Snapshot = 'snapshot'; } +enum MetricAvailability: string { case Available = 'available'; case Unsupported = 'unsupported'; case Delayed = 'delayed'; case PrivacyLimited = 'privacy_limited'; } +enum SyncCollector: string { case AccountDaily = 'account_daily'; case Publications = 'publications'; case PublicationMetrics = 'publication_metrics'; } +enum SyncStatus: string { case Pending = 'pending'; case Running = 'running'; case Complete = 'complete'; case Partial = 'partial'; case ProviderLimited = 'provider_limited'; case Failed = 'failed'; } +``` + +`PublicationContentType` must contain `Text`, `Image`, `Carousel`, `Video`, `Reel`, `Story`, `Short`, `Link`, `Poll`, and `Unknown`. `MetricKey` must contain every metric in the spec catalog, including normalized reactions/comments/shares/saves/views/impressions/reach, watch-time metrics, clicks, video quartiles, follows, profile activity, Story navigation, Pinterest audience metrics, and YouTube subscriber gains/losses. + +- [ ] **Step 4: Implement portable migrations and indexes** + +Use UUID primary keys and explicit foreign keys. The account table unique key is `workspace_id, social_account_key, snapshot_date`; publication identity is `workspace_id, social_account_key, network, provider_post_id`; publication snapshots are unique on `analytics_publication_id, snapshot_date`; sync states are unique on `workspace_id, social_account_key, collector`. Account snapshots, publications, and sync states also store `network` and `platform_user_id` so the same identity can recover its historical key after deletion/reconnection. + +Add these query indexes: + +```php +$table->index(['workspace_id', 'snapshot_date']); +$table->index(['workspace_id', 'social_account_key', 'snapshot_date']); +$table->index(['workspace_id', 'provider_published_at']); +$table->index(['workspace_id', 'social_account_key', 'provider_published_at']); +$table->index(['analytics_publication_id', 'collected_at']); +$table->index(['workspace_id', 'collector', 'status']); +``` + +`social_account_id` and `post_platform_id` use `nullOnDelete()`. `workspace_id` uses `cascadeOnDelete()`. Keep provider ids as strings, metric counters as nullable big integers, rates/precise values as nullable decimals, timestamps below the MySQL 2038 ceiling, and JSON object assertions order-independent. + +- [ ] **Step 5: Run schema tests on the configured database** + +Run: `php artisan test --compact tests/Feature/Analytics/AnalyticsSchemaTest.php` + +Expected: PASS. + +- [ ] **Step 6: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Enums/Analytics database/migrations tests/Feature/Analytics/AnalyticsSchemaTest.php +git commit -m "feat: add workspace analytics schema" +``` + +### Task 2: Add analytics models, factories, DTOs, and idempotent writers + +**Files:** +- Create: `app/Models/AnalyticsAccountDailySnapshot.php` +- Create: `app/Models/AnalyticsPublication.php` +- Create: `app/Models/AnalyticsPublicationDailySnapshot.php` +- Create: `app/Models/AnalyticsSyncState.php` +- Create: `database/factories/AnalyticsAccountDailySnapshotFactory.php` +- Create: `database/factories/AnalyticsPublicationFactory.php` +- Create: `database/factories/AnalyticsPublicationDailySnapshotFactory.php` +- Create: `database/factories/AnalyticsSyncStateFactory.php` +- Create: `app/Dto/Analytics/AccountDailyObservation.php` +- Create: `app/Dto/Analytics/MetricValue.php` +- Create: `app/Dto/Analytics/PublicationMetricObservation.php` +- Create: `app/Actions/Analytics/WriteAccountDailySnapshot.php` +- Create: `app/Actions/Analytics/WritePublicationDailySnapshot.php` +- Create: `app/Actions/Analytics/ResolveAnalyticsAccountKey.php` +- Test: `tests/Feature/Analytics/AnalyticsObservationWriterTest.php` + +**Interfaces:** +- Consumes: Task 1 tables and enums. +- Produces: `ResolveAnalyticsAccountKey::for(SocialAccount $account): string`, `WriteAccountDailySnapshot::handle(SocialAccount $account, AccountDailyObservation $observation): AnalyticsAccountDailySnapshot`, and `WritePublicationDailySnapshot::handle(AnalyticsPublication $publication, PublicationMetricObservation $observation): AnalyticsPublicationDailySnapshot`. + +- [ ] **Step 1: Generate models, factories, actions, and the failing writer test** + +Run: + +```bash +php artisan make:model AnalyticsAccountDailySnapshot --factory --no-interaction +php artisan make:model AnalyticsPublication --factory --no-interaction +php artisan make:model AnalyticsPublicationDailySnapshot --factory --no-interaction +php artisan make:model AnalyticsSyncState --factory --no-interaction +php artisan make:class Dto/Analytics/AccountDailyObservation --no-interaction +php artisan make:class Dto/Analytics/MetricValue --no-interaction +php artisan make:class Dto/Analytics/PublicationMetricObservation --no-interaction +php artisan make:class Actions/Analytics/ResolveAnalyticsAccountKey --no-interaction +php artisan make:class Actions/Analytics/WriteAccountDailySnapshot --no-interaction +php artisan make:class Actions/Analytics/WritePublicationDailySnapshot --no-interaction +php artisan make:test --pest Analytics/AnalyticsObservationWriterTest --no-interaction +``` + +The test must prove same-day writes update one row, next-day writes create history, null stays null, numeric zero stays zero, deleting the social account preserves snapshots through the immutable key, reconnecting the same workspace + network + `platform_user_id` reuses that key, and a different provider identity receives a new key even if the username is identical. + +```php +$writer->handle($account, new AccountDailyObservation( + date: CarbonImmutable::parse('2026-09-23', 'UTC'), + followers: 0, + provenance: ObservationProvenance::Actual, + precision: MetricPrecision::Exact, + providerObservedAt: null, +)); + +expect(AnalyticsAccountDailySnapshot::count())->toBe(1) + ->and(AnalyticsAccountDailySnapshot::first()->followers_count)->toBe(0); +``` + +- [ ] **Step 2: Run the writer test and verify it fails** + +Run: `php artisan test --compact tests/Feature/Analytics/AnalyticsObservationWriterTest.php` + +Expected: FAIL because the models and writers do not exist. + +- [ ] **Step 3: Implement typed DTOs and model relationships** + +`MetricValue` is the JSON boundary: + +```php +final readonly class MetricValue +{ + public function __construct( + public MetricKey $key, + public int|float|null $value, + public MetricUnit $unit, + public MetricTimeBasis $timeBasis, + public MetricPrecision $precision, + public MetricAvailability $availability, + public ?string $providerMetric = null, + ) {} +} +``` + +Models use `HasUuids`, `HasFactory`, explicit `$fillable`, enum/date/array casts, and typed `belongsTo`/`hasMany` relationships. Add relationships from `Workspace`, `SocialAccount`, and `PostPlatform` only when a later query uses them. + +- [ ] **Step 4: Implement transactional upsert writers** + +Use the unique business keys rather than process-local locks. `ResolveAnalyticsAccountKey` searches historical account snapshots, publications, or sync state by workspace + `Platform::network()` + `platform_user_id`, and otherwise returns the current social-account UUID. Snapshot presentation fields come from the account at write time. Publication metrics map stable scalar fields and serialize `MetricValue` entries keyed by `MetricKey::value`; a missing value never overwrites the latest successful scalar with zero. + +```php +return AnalyticsAccountDailySnapshot::query()->updateOrCreate( + [ + 'workspace_id' => $account->workspace_id, + 'social_account_key' => $this->accountKeys->for($account), + 'snapshot_date' => $observation->date->toDateString(), + ], + $this->attributes($account, $observation), +); +``` + +- [ ] **Step 5: Run writer tests** + +Run: `php artisan test --compact tests/Feature/Analytics/AnalyticsObservationWriterTest.php` + +Expected: PASS. + +- [ ] **Step 6: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Models app/Dto/Analytics app/Actions/Analytics database/factories tests/Feature/Analytics/AnalyticsObservationWriterTest.php +git commit -m "feat: persist analytics observations" +``` + +### Task 3: Normalize follower collection for every included platform + +**Files:** +- Create: `app/Contracts/Analytics/FollowerCollector.php` +- Create: `app/Exceptions/Analytics/AnalyticsCollectionException.php` +- Create: `app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php` +- Create: `app/Services/Analytics/Collectors/Followers/InstagramFollowerCollector.php` +- Create: `app/Services/Analytics/Collectors/Followers/FacebookFollowerCollector.php` +- Create: `app/Services/Analytics/Collectors/Followers/ThreadsFollowerCollector.php` +- Create: `app/Services/Analytics/Collectors/Followers/XFollowerCollector.php` +- Create: `app/Services/Analytics/Collectors/Followers/PinterestFollowerCollector.php` +- Create: `app/Services/Analytics/Collectors/Followers/YouTubeFollowerCollector.php` +- Create: `app/Services/Analytics/Collectors/Followers/TikTokFollowerCollector.php` +- Create: `app/Services/Analytics/Collectors/Followers/BlueskyFollowerCollector.php` +- Create: `app/Services/Analytics/Collectors/Followers/MastodonFollowerCollector.php` +- Test: `tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php` + +**Interfaces:** +- Consumes: `SocialAccount` and Task 2 `AccountDailyObservation`. +- Produces: `FollowerCollector::collect(SocialAccount $account, CarbonImmutable $date): AccountDailyObservation` and `FollowerCollectorFactory::for(Platform $platform): FollowerCollector`. + +- [ ] **Step 1: Write the contract and failing provider dataset** + +```php +interface FollowerCollector +{ + public function collect(SocialAccount $account, CarbonImmutable $date): AccountDailyObservation; +} +``` + +Use one Pest dataset that fakes and verifies these read paths and canonical fields: + +| Platform | Read path | Field | +| --- | --- | --- | +| Instagram variants | Graph user/profile or account insights supported by login type | `followers_count` | +| Facebook Page | Graph Page | `followers_count` | +| Threads | user insights | `followers_count` | +| X | `/2/users/{id}?user.fields=public_metrics` | `public_metrics.followers_count` | +| Pinterest | `/v5/user_account` | `follower_count` | +| YouTube | `channels.list(part=statistics)` | `subscriberCount`, approximate; null if hidden | +| TikTok | `/v2/user/info/?fields=follower_count` | `follower_count` | +| Bluesky | `app.bsky.actor.getProfile` | `followersCount` | +| Mastodon | `/api/v1/accounts/{id}` | `followers_count` | + +The test must also assert excluded platforms make no request and that missing fields throw an unavailable collection result rather than returning zero. + +- [ ] **Step 2: Run the collector test and verify it fails** + +Run: `php artisan test --compact tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php` + +Expected: FAIL because the collector factory is missing. + +- [ ] **Step 3: Implement failure classification and provider collectors** + +`AnalyticsCollectionException` carries `transient`, `rate_limited`, `authentication`, `permission`, `unsupported`, or `malformed`, plus nullable provider retry time. Reuse existing token refresh and Graph error classification where available; never log response bodies containing tokens. + +The factory has an explicit match for the ten included platform values and throws for LinkedIn, Telegram, Discord, and Google Business. Instagram direct and Facebook-login variants share the collector class but branch on the existing platform value. + +- [ ] **Step 4: Run collector tests** + +Run: `php artisan test --compact tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php` + +Expected: PASS with `Http::assertSent` endpoint and field verification for every platform. + +- [ ] **Step 5: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Contracts/Analytics app/Exceptions/Analytics app/Services/Analytics/Collectors/Followers tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php +git commit -m "feat: collect normalized follower snapshots" +``` + +### Task 4: Queue daily followers, same-day retries, and carry-forward + +**Files:** +- Create: `app/Jobs/Analytics/CollectAccountDailySnapshot.php` +- Create: `app/Jobs/Analytics/FinalizeAccountDailySnapshots.php` +- Create: `app/Console/Commands/Analytics/DispatchAccountDailyAnalytics.php` +- Modify: `routes/console.php` +- Modify: `app/Observers/SocialAccountObserver.php` +- Test: `tests/Feature/Analytics/AccountDailyJobsTest.php` +- Test: `tests/Feature/Analytics/AnalyticsScheduleTest.php` +- Modify: `tests/Feature/Observers/SocialAccountObserverTest.php` + +**Interfaces:** +- Consumes: Task 3 collectors and Task 2 account writer. +- Produces: one actual or carried-forward row per eligible account/day and immediate collection after connection. + +- [ ] **Step 1: Generate jobs/command and write failing dispatch tests** + +Test included/excluded platforms, inactive/disconnected accounts, duplicate-network accounts, immediate post-commit dispatch, `analytics` queue selection, and scheduler guards. + +```php +Bus::assertDispatched(CollectAccountDailySnapshot::class, + fn ($job) => $job->socialAccountId === $instagram->id + && $job->observationDate === '2026-09-23'); +Bus::assertNotDispatched(CollectAccountDailySnapshot::class, + fn ($job) => $job->socialAccountId === $linkedin->id); +``` + +- [ ] **Step 2: Run queue tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/AccountDailyJobsTest.php tests/Feature/Analytics/AnalyticsScheduleTest.php tests/Feature/Observers/SocialAccountObserverTest.php` + +Expected: FAIL because jobs and schedules are absent. + +- [ ] **Step 3: Implement bounded dispatch and job isolation** + +The command uses `lazyById(200)` and dispatches IDs only. The job re-queries the social account, revalidates active/connected/included status, uses `WithoutOverlapping` keyed by account/date, and exits if an actual row already exists. + +On a transient/rate-limit exception, release near the next `06:00`, `10:00`, `14:00`, `18:00`, or `22:00` UTC window, honoring a later provider time inside the same UTC day. Authentication/permission errors use existing account-health handling and do not write a value. Set job `retryUntil()` to the end of its observation day. + +- [ ] **Step 4: Implement end-of-day fallback** + +`FinalizeAccountDailySnapshots` iterates eligible accounts without an actual row. It copies the last non-null follower count into the current date with `CarriedForward`; it writes nothing when history is absent and never overwrites an actual row. + +- [ ] **Step 5: Schedule and observer integration** + +Schedule the dispatch command at `02:00` UTC and finalizer after the last retry window, both with `withoutOverlapping()` and `onOneServer()`. Dispatch initial collection `afterCommit()` when an included account becomes connected; observers must never throw during delete/reconnect. + +- [ ] **Step 6: Run queue/schedule tests** + +Run: `php artisan test --compact tests/Feature/Analytics/AccountDailyJobsTest.php tests/Feature/Analytics/AnalyticsScheduleTest.php tests/Feature/Observers/SocialAccountObserverTest.php` + +Expected: PASS. + +- [ ] **Step 7: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Jobs/Analytics app/Console/Commands/Analytics routes/console.php app/Observers/SocialAccountObserver.php tests/Feature/Analytics tests/Feature/Observers/SocialAccountObserverTest.php +git commit -m "feat: schedule resilient follower analytics" +``` + +### Task 5: Reconcile TryPost and external publications into one catalog + +**Files:** +- Create: `app/Dto/Analytics/DiscoveredPublication.php` +- Create: `app/Actions/Analytics/UpsertAnalyticsPublication.php` +- Create: `app/Actions/Analytics/SyncTryPostPublication.php` +- Create: `app/Jobs/Analytics/SyncTryPostPublication.php` +- Modify: `app/Observers/PostPlatformObserver.php` +- Create: `tests/Feature/Analytics/PublicationReconciliationTest.php` +- Modify: `tests/Feature/Observers/PostPlatformObserverTest.php` + +**Interfaces:** +- Consumes: Task 2 `AnalyticsPublication` and existing published `PostPlatform`. +- Produces: `UpsertAnalyticsPublication::external(SocialAccount $account, DiscoveredPublication $publication): AnalyticsPublication` and `SyncTryPostPublication::handle(PostPlatform $postPlatform): AnalyticsPublication`. + +- [ ] **Step 1: Write failing reconciliation tests for both arrival orders** + +Test external-first/TryPost-second, TryPost-first/external-second, duplicate provider pages, same provider id on two social accounts, and workspace isolation. + +```php +expect(AnalyticsPublication::query()->where('provider_post_id', 'remote-1')->count())->toBe(1) + ->and(AnalyticsPublication::first()->origin)->toBe(PublicationOrigin::TryPost) + ->and(AnalyticsPublication::first()->post_platform_id)->toBe($postPlatform->id); +``` + +- [ ] **Step 2: Run reconciliation tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/PublicationReconciliationTest.php tests/Feature/Observers/PostPlatformObserverTest.php` + +Expected: FAIL because catalog actions/jobs are absent. + +- [ ] **Step 3: Implement the discovery DTO and transactional upsert** + +`DiscoveredPublication` carries provider id, publication time, normalized/provider content type, permalink, excerpt, preview metadata, and provider metadata. Resolve the historical account key first, then lock the provider identity row using workspace + key + normalized network + provider post id. `trypost` origin wins; provider publication time never becomes discovery time; presentation snapshots update only with non-null values. + +- [ ] **Step 4: Dispatch catalog sync after a destination becomes published** + +Extend `PostPlatformObserver` independently of the PostHog flag: whenever status changes to `Published` and `platform_post_id` is present on an included platform, dispatch `App\Jobs\Analytics\SyncTryPostPublication` with the post-platform id and `afterCommit()`. + +- [ ] **Step 5: Run reconciliation/observer tests** + +Run: `php artisan test --compact tests/Feature/Analytics/PublicationReconciliationTest.php tests/Feature/Observers/PostPlatformObserverTest.php` + +Expected: PASS. + +- [ ] **Step 6: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Dto/Analytics app/Actions/Analytics app/Jobs/Analytics/SyncTryPostPublication.php app/Observers/PostPlatformObserver.php tests/Feature/Analytics/PublicationReconciliationTest.php tests/Feature/Observers/PostPlatformObserverTest.php +git commit -m "feat: reconcile analytics publications" +``` + +### Task 6: Add owned-publication collector contracts and Meta/Threads adapters + +**Files:** +- Create: `app/Contracts/Analytics/PublicationHistoryCollector.php` +- Create: `app/Dto/Analytics/PublicationPage.php` +- Create: `app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php` +- Create: `app/Services/Analytics/Collectors/Publications/InstagramPublicationCollector.php` +- Create: `app/Services/Analytics/Collectors/Publications/FacebookPublicationCollector.php` +- Create: `app/Services/Analytics/Collectors/Publications/ThreadsPublicationCollector.php` +- Test: `tests/Feature/Analytics/Collectors/MetaPublicationCollectorsTest.php` + +**Interfaces:** +- Consumes: Task 5 `DiscoveredPublication`. +- Produces: `PublicationHistoryCollector::page(SocialAccount $account, ?string $cursor, CarbonImmutable $cutoff): PublicationPage`. + +- [ ] **Step 1: Define the page contract and failing cursor tests** + +```php +final readonly class PublicationPage +{ + /** @param list $publications */ + public function __construct( + public array $publications, + public ?string $nextCursor, + public bool $providerExhausted, + public bool $providerLimited = false, + ) {} +} +``` + +Tests must prove: Instagram paginates `/media` for feed/carousel/Reels but does not claim expired Stories; Facebook uses Page-owned published posts plus required video/Reel hydration without visitor posts; Threads paginates owned posts; timestamps stop at but do not cross the cutoff; preview failure does not drop the publication. + +- [ ] **Step 2: Run Meta collector tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/Collectors/MetaPublicationCollectorsTest.php` + +Expected: FAIL because collectors are absent. + +- [ ] **Step 3: Implement one-page Meta adapters** + +Each call reads exactly one provider page and returns the provider cursor without dispatching or persisting. Share only a low-level Graph client/error parser with Repurpose; do not call `PollRepurposeSource`, create `RepurposeItem`, or download media. Parse Instagram direct and Facebook-login field differences explicitly. + +- [ ] **Step 4: Run Meta collector tests** + +Run: `php artisan test --compact tests/Feature/Analytics/Collectors/MetaPublicationCollectorsTest.php` + +Expected: PASS. + +- [ ] **Step 5: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Contracts/Analytics app/Dto/Analytics/PublicationPage.php app/Services/Analytics/Collectors/Publications tests/Feature/Analytics/Collectors/MetaPublicationCollectorsTest.php +git commit -m "feat: discover Meta analytics publications" +``` + +### Task 7: Add X, Pinterest, YouTube, and TikTok publication adapters + +**Files:** +- Create: `app/Services/Analytics/Collectors/Publications/XPublicationCollector.php` +- Create: `app/Services/Analytics/Collectors/Publications/PinterestPublicationCollector.php` +- Create: `app/Services/Analytics/Collectors/Publications/YouTubePublicationCollector.php` +- Create: `app/Services/Analytics/Collectors/Publications/TikTokPublicationCollector.php` +- Modify: `app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php` +- Test: `tests/Feature/Analytics/Collectors/MediaPublicationCollectorsTest.php` + +**Interfaces:** +- Consumes/produces: Task 6 history contract and page DTO. + +- [ ] **Step 1: Write failing endpoint, pagination, and capability tests** + +Cover X `users/:id/tweets` next tokens and paid-read fields; Pinterest `/v5/pins` bookmarks; YouTube uploads-playlist page tokens followed by batched `videos.list`; TikTok `video.list` cursor with maximum 20. TikTok without `video.list` must return provider-limited coverage, not fail account connection. YouTube imports all uploads as `Video` unless the provider gives an authoritative type; do not infer Shorts from duration or aspect ratio. + +- [ ] **Step 2: Run collector tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/Collectors/MediaPublicationCollectorsTest.php` + +Expected: FAIL because collectors are absent. + +- [ ] **Step 3: Implement bounded provider pages and ephemeral preview handling** + +Never persist TikTok cover URLs as durable truth: store them as provider preview metadata with `expires_at`, and let UI fallback when expired. Pinterest records provider metric time-basis metadata. X requests only fields required by the catalog to control read cost. + +- [ ] **Step 4: Run collector tests** + +Run: `php artisan test --compact tests/Feature/Analytics/Collectors/MediaPublicationCollectorsTest.php` + +Expected: PASS. + +- [ ] **Step 5: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Services/Analytics/Collectors/Publications tests/Feature/Analytics/Collectors/MediaPublicationCollectorsTest.php +git commit -m "feat: discover media network publications" +``` + +### Task 8: Add Bluesky and Mastodon publication adapters + +**Files:** +- Create: `app/Services/Analytics/Collectors/Publications/BlueskyPublicationCollector.php` +- Create: `app/Services/Analytics/Collectors/Publications/MastodonPublicationCollector.php` +- Modify: `app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php` +- Modify: `app/Http/Controllers/Auth/MastodonController.php` +- Test: `tests/Feature/Analytics/Collectors/OpenPublicationCollectorsTest.php` +- Modify: `tests/Feature/Auth/MastodonOAuthTest.php` + +**Interfaces:** +- Consumes/produces: Task 6 history contract and page DTO. + +- [ ] **Step 1: Write failing Bluesky repository and Mastodon scope tests** + +Bluesky must page `com.atproto.repo.listRecords` for `app.bsky.feed.post` and hydrate batches for public counts instead of trusting `getAuthorFeed` completeness. Mastodon must page `/api/v1/accounts/{id}/statuses` with `max_id`; private/complete history requires `read:statuses`. Existing accounts without that scope remain public-history/partial rather than failing. + +- [ ] **Step 2: Run tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/Collectors/OpenPublicationCollectorsTest.php tests/Feature/Auth/MastodonOAuthTest.php` + +Expected: FAIL because collectors and the scope change are absent. + +- [ ] **Step 3: Implement adapters and request `read:statuses` for new Mastodon connections** + +Keep instance URLs account-specific and validate them through the existing connection flow. Mark existing insufficient-scope imports `Partial` with a reconnect hint; never silently claim complete private history. + +- [ ] **Step 4: Run tests** + +Run: `php artisan test --compact tests/Feature/Analytics/Collectors/OpenPublicationCollectorsTest.php tests/Feature/Auth/MastodonOAuthTest.php` + +Expected: PASS. + +- [ ] **Step 5: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Services/Analytics/Collectors/Publications app/Http/Controllers/Auth/MastodonController.php tests/Feature/Analytics/Collectors/OpenPublicationCollectorsTest.php tests/Feature/Auth/MastodonOAuthTest.php +git commit -m "feat: discover open network publications" +``` + +### Task 9: Run resumable account backfill and daily publication discovery entirely through jobs + +**Files:** +- Create: `app/Jobs/Analytics/BootstrapAccountAnalytics.php` +- Create: `app/Jobs/Analytics/BackfillAccountPublications.php` +- Create: `app/Jobs/Analytics/DiscoverAccountPublications.php` +- Create: `app/Jobs/Analytics/BackfillTryPostPublications.php` +- Create: `app/Console/Commands/Analytics/DispatchPublicationDiscovery.php` +- Create: `app/Console/Commands/Analytics/BackfillExistingAnalytics.php` +- Create: `app/Actions/Analytics/AdvanceAnalyticsSyncState.php` +- Modify: `app/Observers/SocialAccountObserver.php` +- Modify: `routes/console.php` +- Test: `tests/Feature/Analytics/PublicationBackfillJobsTest.php` +- Test: `tests/Feature/Analytics/BackfillExistingAnalyticsCommandTest.php` + +**Interfaces:** +- Consumes: Tasks 5–8 catalog action, collectors, and sync-state model. +- Produces: resumable 365-day initial import, local TryPost catalog seeding, and overlapping daily discovery. + +- [ ] **Step 1: Write failing job-chain and resume tests** + +Test one provider page per execution, cursor committed only after publication upserts, continuation dispatch after commit, duplicate job idempotency, failure resume, 365-day stop, exhausted stop, provider-limited stop, overlap window, existing-account rollout chunking, and per-account isolation. + +```php +Bus::assertDispatched(BackfillAccountPublications::class, + fn ($job) => $job->socialAccountId === $account->id); +expect($state->fresh()->cursor)->toBe('provider-next-page') + ->and($state->fresh()->status)->toBe(SyncStatus::Running); +``` + +- [ ] **Step 2: Run backfill tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/PublicationBackfillJobsTest.php tests/Feature/Analytics/BackfillExistingAnalyticsCommandTest.php` + +Expected: FAIL because jobs and command are absent. + +- [ ] **Step 3: Implement sync-state locking and bounded jobs** + +Jobs carry account/state ids only. Inside a transaction, lock the sync state, read its cursor/cutoff, fetch one page outside the transaction, then lock again, upsert the page, and advance the cursor if it still matches. A stale duplicate job exits without moving the cursor backward. + +Set `target_since` to connection-time minus 365 days for initial history. Daily discovery uses the high-water mark minus a fixed overlap window and the same provider identity unique key. Persist `oldest_reached_at`, `last_success_at`, and truthful final status. + +- [ ] **Step 4: Implement rollout and local TryPost backfill** + +`analytics:backfill-existing` uses `lazyById(100)` to dispatch `BootstrapAccountAnalytics` for active included accounts and `BackfillTryPostPublications` for published included destinations. The command itself performs no provider calls and accepts an optional workspace id for controlled rollout. + +- [ ] **Step 5: Connect observer and schedule** + +On included account creation/reconnection, dispatch `BootstrapAccountAnalytics` after commit. Schedule daily discovery dispatch with `withoutOverlapping()` and `onOneServer()`. + +- [ ] **Step 6: Run backfill tests** + +Run: `php artisan test --compact tests/Feature/Analytics/PublicationBackfillJobsTest.php tests/Feature/Analytics/BackfillExistingAnalyticsCommandTest.php tests/Feature/Observers/SocialAccountObserverTest.php` + +Expected: PASS. + +- [ ] **Step 7: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Jobs/Analytics app/Console/Commands/Analytics app/Actions/Analytics app/Observers/SocialAccountObserver.php routes/console.php tests/Feature/Analytics tests/Feature/Observers/SocialAccountObserverTest.php +git commit -m "feat: backfill native analytics through jobs" +``` + +### Task 10: Normalize and persist post metrics for all included networks + +**Files:** +- Create: `app/Contracts/Analytics/PublicationMetricsCollector.php` +- Create: `app/Services/Analytics/Collectors/Metrics/PublicationMetricsCollectorFactory.php` +- Create: one `*PublicationMetricsCollector.php` in that directory for Instagram, Facebook, Threads, X, Pinterest, YouTube, TikTok, Bluesky, and Mastodon +- Refactor: `app/Services/Social/InstagramAnalytics.php` +- Refactor: `app/Services/Social/FacebookAnalytics.php` +- Refactor: `app/Services/Social/ThreadsAnalytics.php` +- Refactor: `app/Services/Social/XAnalytics.php` +- Refactor: `app/Services/Social/PinterestAnalytics.php` +- Refactor: `app/Services/Social/YouTubeAnalytics.php` +- Refactor: `app/Services/Social/TikTokAnalytics.php` +- Refactor: `app/Services/Social/BlueskyAnalytics.php` +- Refactor: `app/Services/Social/MastodonAnalytics.php` +- Test: `tests/Feature/Analytics/Collectors/PublicationMetricsCollectorsTest.php` + +**Interfaces:** +- Consumes: `AnalyticsPublication` and Task 2 metric DTOs. +- Produces: `PublicationMetricsCollector::collect(AnalyticsPublication $publication, CarbonImmutable $date): PublicationMetricObservation`. + +- [ ] **Step 1: Write failing normalized-metric datasets** + +The dataset must pin the exact mapping from the spec: + +- Instagram feed/Reel/Story common engagement, reach/views, Reel watch time/average/skip, and Story navigation; parse both `values[].value` and `total_value.value`. +- Facebook Page publication reactions, comments, shares, impressions/reach, and video metrics when supported. +- Threads views, likes, replies, reposts, and quotes. +- X impressions, likes, reposts, replies, quotes, bookmarks, and 30-day-only private/video fields when available. +- Pinterest image/video Pin metrics with lifetime/range/rolling basis retained and batch-ready IDs. +- YouTube views, engaged views, watch time, average duration/percentage, likes, comments, shares, subscriber gains/losses. +- TikTok views, likes, comments, shares and no fabricated retention. +- Bluesky likes, replies, reposts, quotes. +- Mastodon favourites, replies, reblogs. + +For every provider, include measured zero, omitted, unsupported, malformed, rate-limited, and stale-value-preservation cases. + +- [ ] **Step 2: Run metric collector tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/Collectors/PublicationMetricsCollectorsTest.php` + +Expected: FAIL because normalized collectors are absent. + +- [ ] **Step 3: Implement stable normalized observations** + +```php +interface PublicationMetricsCollector +{ + public function collect( + AnalyticsPublication $publication, + CarbonImmutable $date, + ): PublicationMetricObservation; +} +``` + +Split Meta metric families that cannot share one request. Store watch time canonically in seconds. Compute normalized engagement numerator from supported interaction components and preserve `exposure_count` plus `ExposureKind`; do not store a provider engagement rate as if it were the normalized TryPost rate. + +Refactor existing service methods to share low-level authenticated requests/parsers where safe, but do not return translated labels to persistence. Excluded providers remain callable by legacy code until Task 13 removes their analytics read paths, but the new factory never returns them. + +- [ ] **Step 4: Run metric collector and existing provider tests** + +Run: + +```bash +php artisan test --compact tests/Feature/Analytics/Collectors/PublicationMetricsCollectorsTest.php tests/Feature/Services/Social tests/Feature/XAnalyticsTest.php tests/Feature/YouTubeAnalyticsTest.php +``` + +Expected: PASS. + +- [ ] **Step 5: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Contracts/Analytics app/Services/Analytics/Collectors/Metrics app/Services/Social tests/Feature/Analytics/Collectors/PublicationMetricsCollectorsTest.php tests/Feature/Services/Social +git commit -m "feat: normalize publication analytics metrics" +``` + +### Task 11: Queue metric refresh, baseline backfill, and Story lifecycle collection + +**Files:** +- Create: `app/Jobs/Analytics/CollectPublicationMetrics.php` +- Create: `app/Console/Commands/Analytics/DispatchPublicationMetrics.php` +- Create: `app/Jobs/Analytics/ScheduleInstagramStoryMetrics.php` +- Modify: `app/Jobs/Analytics/BackfillAccountPublications.php` +- Modify: `app/Jobs/Analytics/DiscoverAccountPublications.php` +- Modify: `routes/console.php` +- Test: `tests/Feature/Analytics/PublicationMetricsJobsTest.php` + +**Interfaces:** +- Consumes: Task 10 collector factory and Task 2 publication writer. +- Produces: latest persisted metrics with 20-day X, 30-day other-network, one-time old backfill baseline, and sub-day Story collection. + +- [ ] **Step 1: Write failing eligibility and retry tests** + +Test included/excluded platforms, usable provider ids, active access, X day 20/day 21, other day 30/day 31, final-day collection, old imported baseline exactly once, no perpetual old refresh, same-day idempotency, latest-value preservation on failure, batching eligibility, and Story immediate/pre-expiry runs. + +- [ ] **Step 2: Run job tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/PublicationMetricsJobsTest.php` + +Expected: FAIL because jobs are absent. + +- [ ] **Step 3: Implement metric job and dispatcher** + +The job re-queries publication and live account, skips stale/excluded rows, collects, then writes one daily snapshot. Use the same classified same-day retry policy as followers. Provider-specific internal batching may claim several publication ids, but one failed batch must be split or classified without erasing successful values. + +- [ ] **Step 4: Implement import handoff and Story schedule** + +New publications inside the refresh window dispatch normal collection. Older backfill rows dispatch one baseline job recorded by sync metadata. Instagram Stories dispatch immediately, at configured within-lifetime checkpoints, and once shortly before expiry; all writes converge on the daily writer. + +- [ ] **Step 5: Schedule daily metric dispatch and run tests** + +Run: `php artisan test --compact tests/Feature/Analytics/PublicationMetricsJobsTest.php tests/Feature/Analytics/AnalyticsScheduleTest.php` + +Expected: PASS. + +- [ ] **Step 6: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Jobs/Analytics app/Console/Commands/Analytics routes/console.php tests/Feature/Analytics +git commit -m "feat: schedule persisted publication metrics" +``` + +### Task 12: Build the portable workspace analytics read model + +**Files:** +- Create: `app/Dto/Analytics/DateRange.php` +- Create: `app/Queries/Analytics/WorkspaceAnalyticsQuery.php` +- Create: `app/Queries/Analytics/PublicationAnalyticsQuery.php` +- Create: `app/Support/Analytics/PeriodBuckets.php` +- Test: `tests/Feature/Analytics/WorkspaceAnalyticsQueryTest.php` + +**Interfaces:** +- Consumes: all four analytics models. +- Produces: `WorkspaceAnalyticsQuery::for(Workspace $workspace, DateRange $range): array` and `PublicationAnalyticsQuery::latestForPostPlatform(PostPlatform $postPlatform): array`. + +- [ ] **Step 1: Write failing query tests covering every dashboard block** + +Create two Instagram accounts and one X account in the same workspace plus a foreign-workspace account. Assert: + +- min/max date bounds from snapshots or publications; +- range validation and equal-length previous range; +- end-date follower total, per-account Line/Bar/Growth, carry-forward provenance; +- Posts Bar totals and daily/weekly/monthly zero-filled buckets at 14/15/90/91-day boundaries; +- exactly five Summary values; +- pooled engagement `sum(numerator) / sum(denominator)`, excluding only invalid denominators; +- deterministic Top 5 ties by publication time then id; +- Performance rows per social account, including two separate Instagram rows; +- historical rows after live account deletion; +- no excluded platform or foreign-workspace contribution; +- unavailable/null distinct from zero. + +- [ ] **Step 2: Run read-model tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/WorkspaceAnalyticsQueryTest.php` + +Expected: FAIL because query services are absent. + +- [ ] **Step 3: Implement date/bucket value objects and indexed queries** + +Avoid JSON predicates for dashboard aggregation. Select latest publication snapshot through portable subqueries keyed by publication id and maximum collected time/date. Use scalar nullable columns and Eloquent/query builder without `ILIKE`, driver-specific date truncation, or database-specific JSON functions. Generate calendar bucket boundaries in PHP and group fetched aggregate rows into those boundaries. + +The response shape is stable: + +```php +[ + 'bounds' => ['min' => '2025-09-23', 'max' => '2026-09-23'], + 'range' => ['start' => '2026-08-25', 'end' => '2026-09-23'], + 'summary' => [...], + 'followers' => ['total' => 15200, 'accounts' => [...], 'series' => [...]], + 'posts' => ['resolution' => 'weekly', 'accounts' => [...], 'buckets' => [...]], + 'top_posts' => ['reactions' => [...], 'comments' => [...]], + 'performance' => [...], + 'coverage' => [...], +]; +``` + +- [ ] **Step 4: Run query tests and inspect query count** + +Run: `php artisan test --compact tests/Feature/Analytics/WorkspaceAnalyticsQueryTest.php` + +Expected: PASS with a fixed query count that does not grow with account/publication count. + +- [ ] **Step 5: Run the same test suite against PostgreSQL and MySQL** + +Run the repository's configured PostgreSQL and MySQL CI/database commands. Expected: identical values and ordering on both engines; JSON assertions use recursive equality. + +- [ ] **Step 6: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Dto/Analytics/DateRange.php app/Queries/Analytics app/Support/Analytics tests/Feature/Analytics/WorkspaceAnalyticsQueryTest.php +git commit -m "feat: query workspace analytics reports" +``` + +### Task 13: Replace analytics web, post, REST, and MCP read paths with persisted data + +**Files:** +- Modify: `app/Http/Controllers/App/AnalyticsController.php` +- Modify: `routes/app.php` +- Modify: `app/Http/Controllers/App/PostController.php` +- Modify: `app/Services/Post/PostMetricsFetcher.php` +- Modify: `app/Mcp/Tools/Post/GetPostMetricsTool.php` +- Modify: `app/Http/Controllers/Api/PostController.php` +- Modify: `app/Http/Resources/Api/PostMetricsResource.php` +- Test: `tests/Feature/Analytics/AnalyticsControllerTest.php` +- Test: `tests/Feature/Analytics/PersistedPostMetricsReadTest.php` +- Modify: `tests/Feature/AnalyticsResilienceTest.php` + +**Interfaces:** +- Consumes: Task 12 queries. +- Produces: one workspace analytics Inertia response and one persisted publication-detail contract shared by web/API/MCP. + +- [ ] **Step 1: Write failing no-provider-read tests** + +Seed analytics rows, call `/analytics`, post metrics JSON, REST, and MCP, then assert response values and `Http::assertNothingSent()`. Assert an account id from another workspace cannot affect results. Assert LinkedIn, Telegram, Discord, and Google Business expose no V1 block. + +- [ ] **Step 2: Run controller/read tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/AnalyticsControllerTest.php tests/Feature/Analytics/PersistedPostMetricsReadTest.php tests/Feature/AnalyticsResilienceTest.php` + +Expected: FAIL because current controllers call provider services and Redis-backed `PostMetricsFetcher`. + +- [ ] **Step 3: Make `AnalyticsController@index` the only dashboard read endpoint** + +Validate `start`/`end` as dates, clamp them to available bounds, authorize the current workspace, and pass the Task 12 report to `Inertia::render('analytics/Index', ...)`. Remove the per-account `show` route and provider dispatch after all frontend callers are removed. + +- [ ] **Step 4: Convert `PostMetricsFetcher` into a persisted read facade** + +Remove `Cache::remember` and all social-service dependencies. It delegates to `PublicationAnalyticsQuery`, returns canonical metric keys/labels/units/freshness/origin, and preserves its web/API/MCP callers until their response types are updated together. + +- [ ] **Step 5: Run all analytics read tests** + +Run: `php artisan test --compact tests/Feature/Analytics/AnalyticsControllerTest.php tests/Feature/Analytics/PersistedPostMetricsReadTest.php tests/Feature/AnalyticsResilienceTest.php tests/Feature/Mcp` + +Expected: PASS and no provider HTTP requests. + +- [ ] **Step 6: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Http/Controllers app/Services/Post/PostMetricsFetcher.php app/Mcp routes/app.php tests/Feature/Analytics tests/Feature/AnalyticsResilienceTest.php tests/Feature/Mcp +git commit -m "feat: read analytics exclusively from database" +``` + +### Task 14: Build the workspace analytics dashboard + +**Files:** +- Replace: `resources/js/pages/analytics/Index.vue` +- Create: `resources/js/components/analytics/workspace/types.ts` +- Create: `resources/js/components/analytics/workspace/AnalyticsSection.vue` +- Create: `resources/js/components/analytics/workspace/SummaryCards.vue` +- Create: `resources/js/components/analytics/workspace/FollowersChart.vue` +- Create: `resources/js/components/analytics/workspace/PostsChart.vue` +- Create: `resources/js/components/analytics/workspace/TopPosts.vue` +- Create: `resources/js/components/analytics/workspace/PerformanceTable.vue` +- Create: `resources/js/components/analytics/workspace/ImportCoverage.vue` +- Create: `resources/js/components/analytics/workspace/AccountIdentity.vue` +- Create: `resources/js/components/analytics/workspace/charts/LineChart.vue` +- Create: `resources/js/components/analytics/workspace/charts/HorizontalBarChart.vue` +- Create: `resources/js/components/analytics/workspace/charts/StackedBarChart.vue` +- Modify: `lang/en/analytics.php` +- Modify: every locale counterpart required by localization parity +- Test: `tests/Browser/WorkspaceAnalyticsTest.php` + +**Interfaces:** +- Consumes: Task 13 Inertia report shape. +- Produces: responsive workspace dashboard matching the reference behavior without provider calls. + +- [ ] **Step 1: Write the failing browser test** + +The test seeds two Instagram accounts and one X account, visits `/analytics`, asserts exactly five Summary cards, separate account labels, Line/Bar/Growth switching, Posts Bar/Stacked Bar switching, Top 5 Reactions/Comments switching, Performance rows, range navigation, origin labels, coverage state, empty state, and no JavaScript errors/console logs. + +```php +$page = visit(route('app.analytics')); +$page->assertSee('Total Followers') + ->assertSee('@first · Instagram') + ->assertSee('@second · Instagram') + ->click('Growth') + ->assertSee('-20') + ->assertNoJavaScriptErrors() + ->assertNoConsoleLogs(); +``` + +- [ ] **Step 2: Run the browser test and verify it fails** + +Run: `php artisan test --compact tests/Browser/WorkspaceAnalyticsTest.php` + +Expected: FAIL because the workspace components are absent. + +- [ ] **Step 3: Implement typed dashboard composition and date filter** + +Use a single root element, existing `DateRangePicker`, and an Inertia GET visit preserving state/scroll. Set picker min/max from report bounds and disable it in the no-data/import-pending state. All chart-mode toggles are client-side because the response contains every required series. + +- [ ] **Step 4: Implement dependency-free visualizations** + +`LineChart.vue` computes SVG points from daily values and leaves gaps where no observation exists. `HorizontalBarChart.vue` supports positive/negative Growth around a zero axis. `StackedBarChart.vue` renders zero-filled daily/weekly/monthly buckets. Use stable account colors based on account order/id, platform icons, semantic buttons, keyboard focus, tooltips for precision/freshness, and horizontal scrolling on narrow screens. + +- [ ] **Step 5: Implement reporting blocks and translations** + +Summary contains only Posts, Total Followers, Reactions, Comments, and Engagement Rate. Top 5 cards show destination origin and only valid actions. Performance sorting is local over the returned rows. Unsupported renders an em dash, never `0`; measured zero renders `0`. + +- [ ] **Step 6: Run browser and frontend checks** + +```bash +php artisan test --compact tests/Browser/WorkspaceAnalyticsTest.php +npm run lint +npx vue-tsc --noEmit +npm run build +``` + +Expected: all pass. + +- [ ] **Step 7: Commit** + +```bash +git add resources/js/pages/analytics resources/js/components/analytics/workspace lang tests/Browser/WorkspaceAnalyticsTest.php +git commit -m "feat: add workspace analytics dashboard" +``` + +### Task 15: Replace individual-post analytics UI and add external publication detail + +**Files:** +- Modify: `resources/js/components/posts/PostPlatformMetrics.vue` +- Modify: `resources/js/pages/posts/Show.vue` +- Create: `resources/js/components/analytics/workspace/PublicationMetrics.vue` +- Create: `resources/js/pages/analytics/Publications/Show.vue` +- Create: `app/Http/Controllers/App/AnalyticsPublicationController.php` +- Modify: `routes/app.php` +- Modify: `lang/en/posts.php` +- Modify: locale counterparts required by parity +- Test: `tests/Feature/Analytics/AnalyticsPublicationControllerTest.php` +- Test: `tests/Browser/PublicationAnalyticsTest.php` + +**Interfaces:** +- Consumes: Task 13 persisted detail contract. +- Produces: rich persisted metrics for TryPost destinations and a read-only route for imported external publications. + +- [ ] **Step 1: Write failing authorization and browser tests** + +Assert workspace ownership, imported rows have no edit/retry/delete action, `Published via TryPost` versus `Published on Instagram`, content-specific metric groups, canonical display units, last-collected/stale/estimated labels, excluded-platform absence, and no provider request. + +- [ ] **Step 2: Run tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/AnalyticsPublicationControllerTest.php tests/Browser/PublicationAnalyticsTest.php` + +Expected: FAIL because the page and persisted UI contract are absent. + +- [ ] **Step 3: Implement controller and shared presentation component** + +Authorize through the publication's immutable workspace id. The controller passes publication identity, origin, public URL, preview, account snapshot, and latest metric groups. `PublicationMetrics.vue` renders common engagement, exposure, and video-retention sections and is reused by `PostPlatformMetrics.vue`. + +- [ ] **Step 4: Remove request-time fetching from the post component** + +Pass persisted metrics as page props or load them from the local-only JSON endpoint. Do not keep `onMounted` provider semantics, Redis loading language, or swallowed provider errors. Excluded destinations do not render the block. + +- [ ] **Step 5: Run tests and frontend checks** + +```bash +php artisan test --compact tests/Feature/Analytics/AnalyticsPublicationControllerTest.php tests/Browser/PublicationAnalyticsTest.php tests/Browser/PostShowContentTypeTest.php +npm run lint +npx vue-tsc --noEmit +``` + +Expected: PASS. + +- [ ] **Step 6: Format and commit** + +```bash +vendor/bin/pint --dirty --format agent +git add app/Http/Controllers/App/AnalyticsPublicationController.php routes/app.php resources/js/components/posts resources/js/components/analytics/workspace/PublicationMetrics.vue resources/js/pages/analytics/Publications lang tests/Feature/Analytics tests/Browser +git commit -m "feat: persist individual publication analytics" +``` + +### Task 16: Verify rollout, observability, portability, and remove obsolete analytics code + +**Files:** +- Modify: `config/horizon.php` +- Delete: `resources/js/components/analytics/AnalyticsAccountSelector.vue` +- Delete: `resources/js/components/analytics/FacebookAnalytics.vue` +- Delete: `resources/js/components/analytics/GoogleBusinessAnalytics.vue` +- Delete: `resources/js/components/analytics/InstagramAnalytics.vue` +- Delete: `resources/js/components/analytics/LinkedInPageAnalytics.vue` +- Delete: `resources/js/components/analytics/MetricsGrid.vue` +- Delete: `resources/js/components/analytics/PinterestAnalytics.vue` +- Delete: `resources/js/components/analytics/TelegramAnalytics.vue` +- Delete: `resources/js/components/analytics/ThreadsAnalytics.vue` +- Delete: `resources/js/components/analytics/TikTokAnalytics.vue` +- Delete: `resources/js/components/analytics/XAnalytics.vue` +- Delete: `resources/js/components/analytics/YouTubeAnalytics.vue` +- Delete: `resources/js/components/analytics/types.ts` +- Modify: `docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md` +- Test: `tests/Feature/Analytics/AnalyticsObservabilityTest.php` +- Test: `tests/Feature/LocalizationParityTest.php` + +**Interfaces:** +- Consumes: the completed feature. +- Produces: deployable queue configuration, truthful operational logs, clean code, and verified cross-engine behavior. + +- [ ] **Step 1: Write failing observability and queue configuration tests** + +Assert every log context contains workspace id, social-account key, platform, collector, date/cursor, attempt, and sanitized category but excludes access/refresh tokens and raw sensitive responses. Assert analytics jobs use the `analytics` queue and Horizon supervises it. + +- [ ] **Step 2: Run observability tests and verify they fail** + +Run: `php artisan test --compact tests/Feature/Analytics/AnalyticsObservabilityTest.php` + +Expected: FAIL until queue/log configuration is complete. + +- [ ] **Step 3: Configure the queue and clean obsolete read paths** + +Add the analytics queue to existing Horizon supervisors without changing unrelated queue balancing. Remove old account selector/per-network dashboard components only after `rg` proves no imports. Keep low-level social analytics calls that normalized collectors share; remove translated request-time wrappers only when no publisher, test, API, or MCP path references them. + +- [ ] **Step 4: Run targeted and full verification** + +```bash +vendor/bin/pint --dirty --format agent +php artisan test --compact tests/Feature/Analytics tests/Feature/Services/Social tests/Feature/Observers tests/Feature/Mcp tests/Browser/WorkspaceAnalyticsTest.php tests/Browser/PublicationAnalyticsTest.php tests/Feature/LocalizationParityTest.php +npm run lint +npx vue-tsc --noEmit +npm run build +php artisan test --compact +``` + +Expected: all pass. + +- [ ] **Step 5: Verify PostgreSQL and MySQL** + +Run the full database-dependent analytics suite on both supported engines. Confirm migrations roll up/down, all four unique keys enforce the same identities, nullable booleans/JSON are asserted portably, and aggregate ordering is deterministic. + +- [ ] **Step 6: Perform controlled capability and rollout checks** + +Before dispatching the production rollout: + +1. confirm the production TikTok app has `video.list` and `user.info.stats`; +2. connect/test one Instagram-direct and one Instagram-via-Facebook account; +3. reconnect a Mastodon test account with `read:statuses` and confirm private-history behavior; +4. measure X read cost on one bounded 365-day account before widening rollout; +5. confirm Threads follower insights with a production-approved token; +6. export the selected canary workspace UUID as `ANALYTICS_CANARY_WORKSPACE_ID`, then run `php artisan analytics:backfill-existing --workspace="$ANALYTICS_CANARY_WORKSPACE_ID"`; +7. verify coverage states, then run the command without the workspace filter. + +- [ ] **Step 7: Update spec status and commit** + +Mark implemented gates with the actual provider limitations observed; do not weaken documented coverage silently. + +```bash +git add config .env.example app resources/js docs/superpowers/specs tests +git commit -m "chore: finalize workspace analytics rollout" +``` + +## Self-Review Results + +- **Spec coverage:** Every V1 surface, included/excluded platform, follower fallback, native backfill, reconciliation rule, metric catalog, date range, Summary, Top 5, Performance, individual detail, REST/MCP read path, and LinkedIn V2 boundary maps to Tasks 1–16. +- **Placeholder scan:** The plan contains no forbidden placeholder markers, no unnamed error handling, and no task that delegates unspecified work. Provider mappings and final manual capability gates are explicit. +- **Type consistency:** The four model names, DTO constructors, collector signatures, origin values, sync states, and query method names are introduced once and reused consistently. +- **Review focus:** Each of the five highest-risk inputs is pinned to an explicit test in Tasks 1/2, 3/10, 5, 9, or 12. +- **Scope decomposition:** Backend persistence, account collection, publication discovery, metric collection, read model, dashboard, and detail UI are independent review gates but remain in one plan and one branch because their contracts form one source-of-truth migration. diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index 96e821138..9cd845a22 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -1,6 +1,6 @@ # Workspace follower and post analytics — design -**Status:** written design awaiting approval. Nothing implemented. +**Status:** requirements and persistence design consolidated for implementation planning. Nothing implemented. ## Objective @@ -194,7 +194,7 @@ Each imported publication retains, when available: - preview/thumbnail reference and enough immutable presentation metadata to render a historical card; - discovery and last-sync timestamps; -- origin (`trypost` or `native_import`); +- origin (`trypost` or `external`); - provider coverage and availability state. An imported post is an analytics record, not a draft or published `Post` owned @@ -705,8 +705,8 @@ without misclassifying a repeated value as a successful API fetch. The complete supported post metric catalog, including reactions, comments, exposure, engagement inputs, and video retention, must not trigger social API calls while `/analytics` or an individual post is rendering. Metrics are -refreshed in queued jobs and stored behind the same persistence decision gate as -follower observations. +refreshed in queued jobs and stored through the same idempotent persistence +boundary as follower observations. The daily dispatcher selects reconciled TryPost/native analytics publications that have a native post id, use a platform included in analytics v1, have a @@ -748,40 +748,116 @@ redundant provider request. An included v1 platform without a supported post metric may still contribute its locally known Posts count and show the metric as unavailable. An excluded v1 platform contributes neither posts nor metrics. -## Persistence decision gate - -This specification deliberately defines the **logical data requirements** but -does not choose a physical table design. - -The user intends to add many account-, post-, and workspace-level analytics -metrics. Choosing a generic metrics table, metric-specific tables, JSON -snapshots, or a hybrid before that catalog exists would prematurely constrain -dimensions, indexes, retention, and aggregation. - -Before any analytics migration or model is implemented, a follow-up design -must inventory each planned metric with: - -- entity level: workspace, social account, or post; -- value type and unit; -- snapshot, interval, delta, or lifetime semantics; -- supported dimensions; -- collection frequency and retention; -- exact, approximate, or estimated provenance; -- availability and historical limits per platform. - -The currently specified post-performance catalog includes both cross-network -fields and content-specific detail. Cross-network fields are normalized -reactions, normalized comments, normalized engagement numerator, exposure -denominator and kind, provider collection timestamp, and availability status -per destination. -The full observation additionally retains stable metric key, numeric value, -unit, content type, precision/stability flags, and provider metric identity for -every supported native metric described by the catalog. It does not remove the -gate: the user may supply more metrics before the physical schema is selected. - -That follow-up design selects the physical schema and proves it on both -PostgreSQL and MySQL. The implementation plan for this feature must not include -a persistence migration until that decision is approved. +## Persistence design + +The physical design uses four tables: three analytics fact/catalog tables and +one operational synchronization table. This is the minimum that keeps account +snapshots, publication identity, cumulative publication metrics, and resumable +job state independent. Combining those lifecycles would either lose history, +reintroduce Redis as durable state, or produce a sparse table that cannot be +aggregated portably on both PostgreSQL and MySQL. + +Database enum types are not used. Enum-backed values are stored in string +columns and cast through PHP enums so new providers and metrics do not require +engine-specific enum migrations. + +### Daily account snapshots + +`analytics_account_daily_snapshots` stores one effective observation per +workspace, immutable social-account key, and UTC date. It contains: + +- UUID primary key; +- immutable `workspace_id` ownership; +- nullable `social_account_id` foreign key for the currently connected row; +- non-null `social_account_key`, initially copied from the originating + social-account UUID and retained after that row is deleted; +- provider identity snapshot (`network` plus `platform_user_id`) used to resolve + and reuse the historical key when the same provider identity reconnects; +- platform string cast to the existing `SocialAccount\\Platform` enum; +- immutable account presentation snapshots; +- `snapshot_date`; +- nullable `followers_count` bigint; +- nullable JSON `metrics` for future account-level metrics that are not yet + promoted to first-class aggregate columns; +- actual or carried-forward provenance; +- exact or approximate precision; +- provider observation time and collection time. + +The unique key is workspace + social-account key + snapshot date. Platform is a +dimension, not account identity: two Instagram accounts in one workspace remain +two independent series. Cross-network totals sum the latest eligible row for +each social-account key. `social_account_id` is used while the connection +exists; `social_account_key` and the presentation snapshots preserve truthful +historical series after deletion. A connection/reconnection resolver first +looks for an existing key with the same workspace + network + platform user id; +only a genuinely new identity starts with the current social-account UUID. + +### Reconciled publication catalog + +`analytics_publications` stores one publication per workspace, immutable +social-account key, platform, and provider post id. It contains: + +- UUID primary key and immutable `workspace_id` ownership; +- nullable live `social_account_id` plus non-null historical + `social_account_key`; +- nullable unique `post_platform_id` for a TryPost-owned destination; +- platform, normalized network, provider account id, provider post id, provider + publication time, normalized content type, and provider content type; +- origin `trypost` or `external`; +- permalink, excerpt, preview metadata, and immutable account presentation + snapshots; +- available, deleted, or unavailable state; +- first-seen, last-seen, and provider-sync timestamps; +- JSON provider metadata that is not used for cross-network aggregation. + +`external` means only that TryPost did not publish the record. Provider APIs do +not reliably distinguish a manual native-app post from a post created by +Buffer or another client, so the UI says `Published on ` rather than +claiming it was posted manually. A matching `post_platform_id` proves TryPost +origin and produces `Published via TryPost`. + +The unique provider identity is workspace + social-account key + network + +provider post id. Network, rather than login variant, prevents the same +Instagram media from duplicating when an identity reconnects through direct +Instagram instead of Facebook login. A TryPost destination and external +discovery reconcile onto that identity; `trypost` origin wins and the +publication is counted once. + +### Daily publication snapshots + +`analytics_publication_daily_snapshots` stores at most one cumulative +observation per analytics publication and UTC date. Repeated successful +collections on the same date update that row instead of creating additional +facts. It contains nullable first-class aggregate columns for reactions, +comments, shares, saves, views, impressions, reach, total watch time, and +average watch time, plus normalized engagement numerator, exposure denominator, +and exposure kind. + +The same row has a JSON metric catalog for provider/content-specific values. +Each JSON entry uses a stable enum-backed metric key and retains numeric value, +unit, provider metric identity, lifetime/range/rolling time basis, +exact/estimated/experimental precision, and availability. Cross-network queries +use the first-class columns; the JSON catalog powers the richer individual-post +detail. This hybrid avoids both engine-specific JSON aggregation and an EAV row +explosion. + +Imported historical publications receive a real baseline observation collected +at import time. The system does not fabricate daily metric history between the +publication date and that baseline. + +### Synchronization state + +`analytics_sync_states` stores durable operational state per workspace, +immutable social-account key, and collector. Collector values initially cover +daily account snapshots, owned-publication history/discovery, and publication +metrics. The row retains status, cursor, target cutoff, high-water mark, oldest +and latest provider dates reached, last successful synchronization, next retry, +attempt count, and a sanitized last-error category/message. + +This state does not live in `social_accounts.meta`: collectors advance +independently, need row-level concurrency control, and must survive deletion or +reconnection of the live account row. The unique key is workspace + +social-account key + collector. Regardless of the final schema, persistence must support: @@ -904,8 +980,8 @@ this first delivery. TryPost publication history for ranges in which it exists. - **Disconnected/deleted:** stop collection and fallback; preserve historical observations and imported publications even if the account row is later - removed. The physical schema design must decide how to retain enough - immutable identity for this. + removed. Nullable live foreign keys plus immutable `social_account_key` and + presentation snapshots retain that identity. - **Reconnected as the same persisted identity:** resume collection without rewriting earlier observations and resume native discovery from its checkpoint with an overlap window. @@ -921,8 +997,8 @@ this first delivery. - API tokens remain on `social_accounts` and are never copied into analytics storage or job logs. - Platform data-retention terms must be checked as each collector is - implemented; the physical schema review must record any network-specific - retention constraint. + implemented; every network-specific constraint is recorded in its collector + tests and coverage status. ## Testing strategy @@ -1110,8 +1186,15 @@ design is approved and implemented. - **Deleting history when an account disconnects.** Removes valid workspace history and breaks historical comparisons. - **Persisting workspace totals.** Duplicates account facts and risks drift. -- **Choosing the final table structure now.** The wider metric catalog is not - yet known, so the choice would be speculative. +- **One generic analytics table.** Account snapshots, publication identity, + cumulative publication metrics, and resumable cursors have different + cardinality and lifecycle. Combining them creates sparse rows and weak + constraints. The four-table hybrid is the minimum safe design. +- **JSON-only post metrics.** Cross-network ranking and aggregation would depend + on engine-specific JSON queries. Common aggregate fields are first-class + nullable columns; provider/content-specific metrics remain structured JSON. +- **An EAV row for every metric.** It would multiply row volume and joins for + every post card. One daily publication snapshot keeps the metric set atomic. - **Including LinkedIn, Telegram, Discord, or Google Business Profile in v1.** LinkedIn requires the separately vetted product for a coherent implementation; the other three are outside the chosen product scope and lack a Buffer @@ -1200,12 +1283,10 @@ design is approved and implemented. endpoint, pagination, scopes, accessible content types, history depth, metric-retention limits, and preview-media expiry. Record any shallower provider limit in the coverage contract instead of weakening it silently. -3. The broader metric catalog must be supplied and its persistence design - approved, including the reconciled TryPost/native publication identity and - resumable import checkpoints. -4. Only then can the Superpowers implementation-plan stage define migrations, - concrete classes, and ordered implementation tasks. -5. Implementation begins only after that written plan is reviewed and its +3. The implementation plan must preserve the approved four-table hybrid, + reconciled TryPost/external publication identity, and resumable import + checkpoints. +4. Implementation begins only after that written plan is reviewed and its execution method is selected. -6. LinkedIn follower and post analytics receive a separate v2 implementation +5. LinkedIn follower and post analytics receive a separate v2 implementation plan after the external Community Management API dependency is resolved. From 87700a21554c9e44130b2817e75691c071500162 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 11:09:17 -0300 Subject: [PATCH 12/77] docs: finalize workspace analytics implementation plan --- ANALYTIC.md | 30 +++ ...2026-09-23-workspace-analytics-backfill.md | 249 +++++++++++++++--- ...-23-workspace-follower-analytics-design.md | 166 +++++++++--- 3 files changed, 384 insertions(+), 61 deletions(-) create mode 100644 ANALYTIC.md diff --git a/ANALYTIC.md b/ANALYTIC.md new file mode 100644 index 000000000..f887445ed --- /dev/null +++ b/ANALYTIC.md @@ -0,0 +1,30 @@ +# Analytics por rede social + +Este documento descreve as métricas que o TryPost consegue consultar atualmente por conta e por publicação. + +> Este é um inventário do comportamento atual, anterior ao novo módulo. O desenho aprovado está em `docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md` e o plano executável está em `docs/superpowers/plans/2026-09-23-workspace-analytics-backfill.md`. Na V1 nova, LinkedIn, Telegram, Discord e Google Business Profile ficam fora de todas as superfícies de analytics; LinkedIn fica planejado para V2. + +| Rede / integração | Métricas por conta | Métricas por publicação | Observações | +| --- | --- | --- | --- | +| TikTok | Seguidores; seguindo; curtidas totais; quantidade de vídeos; visualizações, curtidas, comentários e compartilhamentos agregados dos vídeos recentes | Visualizações; curtidas; comentários; compartilhamentos | O agregado da conta considera os 20 vídeos mais recentes. O seletor de período não é aplicado a essa consulta. Posts privados podem não fornecer um ID público consultável. | +| Instagram (conexão direta ou via Facebook) | Alcance; seguidores; curtidas; comentários; compartilhamentos; salvamentos; visualizações; interações | **Feed:** alcance, curtidas, comentários, compartilhamentos, salvamentos e interações.
**Reel:** alcance, curtidas, comentários, compartilhamentos, salvamentos e visualizações.
**Story:** alcance, visualizações e respostas. | Disponível no painel por conta e no detalhe da publicação. | +| Threads | Visualizações; curtidas; respostas; reposts; citações | Visualizações; curtidas; respostas; reposts; citações | Disponível no painel por conta e no detalhe da publicação. | +| Facebook Page | Alcance da página; alcance dos posts; engajamento dos posts; novos seguidores; visualizações da página | **Feed:** impressões, alcance, curtidas e cliques.
**Story:** impressões, alcance, interações, reações, respostas e compartilhamentos.
**Vídeo/Reel:** reproduções, reações e interações. | As métricas disponíveis dependem do tipo e do identificador da publicação. | +| X | Impressões; curtidas; reposts; respostas; citações; bookmarks | Impressões; curtidas; reposts; respostas; citações; bookmarks | A consulta da conta soma as métricas dos posts encontrados no período, com limite de 100 dias e de cinco páginas de resultados. | +| LinkedIn — perfil pessoal | Não disponível no painel por conta | Curtidas; comentários | A API usada pela integração de perfil pessoal não fornece ao TryPost o conjunto completo de analytics disponível para páginas. | +| LinkedIn — página de empresa | Visualizações da página; novos seguidores orgânicos; novos seguidores pagos; impressões; cliques; curtidas; comentários; compartilhamentos | Impressões; cliques; curtidas; comentários; compartilhamentos | Métricas com valor zero podem ser omitidas no painel por conta. | +| Pinterest | Impressões; cliques no Pin; engajamentos; salvamentos; taxa média de clique | Impressões; salvamentos; cliques no Pin; cliques externos; visualizações de vídeo | A consulta por publicação usa uma janela fixa dos últimos 90 dias. | +| YouTube Shorts | Visualizações; minutos assistidos; duração média da visualização; percentual médio assistido; inscritos ganhos; inscritos perdidos; curtidas | Visualizações; minutos assistidos; duração média da visualização; curtidas; comentários; compartilhamentos | As métricas da publicação são consultadas desde a data de publicação até o dia atual. | +| Telegram | Número de inscritos do canal | Número de inscritos do canal; reações separadas por emoji | A Bot API não fornece visualizações das mensagens para bots. As reações são recebidas pelo webhook e armazenadas nos metadados da publicação. | +| Bluesky | Não disponível no painel por conta | Curtidas; reposts; citações; respostas | Atualmente existe apenas analytics por publicação. | +| Mastodon | Não disponível no painel por conta | Favoritos; boosts/reblogs; respostas | Atualmente existe apenas analytics por publicação. | +| Discord | Quantidade aproximada de membros do servidor, disponível no serviço interno, mas ainda não exibida no painel geral | Quantidade aproximada de membros; reações separadas por emoji; respostas na thread | O Discord não fornece impressões, alcance ou visualizações para mensagens de bot. | +| Google Business Profile | Impressões no Search em desktop; impressões no Search em mobile; impressões no Maps em desktop; impressões no Maps em mobile; cliques no site; cliques para ligar; solicitações de rota; conversas; palavras-chave de busca; quando aplicável, agendamentos, pedidos de comida e cliques no cardápio | Não disponível | Palavras-chave são agregadas mensalmente. Contagens de termos com baixo volume podem ser estimadas. Agendamentos e métricas de comida com valor zero são ocultados. | + +## Disponibilidade atual + +O painel geral de analytics permite selecionar contas de TikTok, Instagram, Threads, Facebook, X, LinkedIn Page, Pinterest, YouTube, Telegram e Google Business Profile. + +LinkedIn pessoal, Bluesky e Mastodon possuem apenas métricas por publicação. O Discord também possui métricas implementadas por publicação e uma métrica de conta, mas ainda não aparece no painel geral. + +As métricas por publicação só são consultadas quando a publicação está com status `published` e possui um identificador retornado pela plataforma. Esses resultados ficam em cache por cinco minutos. As métricas do painel por conta usam, em geral, o período selecionado e ficam em cache por uma hora em produção. diff --git a/docs/superpowers/plans/2026-09-23-workspace-analytics-backfill.md b/docs/superpowers/plans/2026-09-23-workspace-analytics-backfill.md index 527065888..9b96444ac 100644 --- a/docs/superpowers/plans/2026-09-23-workspace-analytics-backfill.md +++ b/docs/superpowers/plans/2026-09-23-workspace-analytics-backfill.md @@ -4,9 +4,9 @@ **Goal:** Replace request-time social analytics with workspace-scoped, database-backed follower history, reconciled TryPost/external publication history, persisted post metrics, and the Summary, Followers, Posts, Top 5 Posts, Performance, and individual-publication views. -**Architecture:** Four tables separate daily account facts, publication identity, daily cumulative publication metrics, and durable synchronization state. Every provider call runs in an isolated queued job; page, API, and MCP reads use local query services only. Provider adapters normalize platform responses into stable DTOs, while a hybrid scalar-plus-JSON snapshot keeps cross-network queries portable across PostgreSQL and MySQL and preserves content-specific metrics. +**Architecture:** Four tables separate daily account facts, publication identity, daily cumulative publication metrics, and a deliberately small operational checkpoint used only by publication backfill/discovery. Every provider call runs in an isolated queued job; page, API, and MCP reads use local query services only. Provider adapters normalize platform responses into stable DTOs, while a hybrid scalar-plus-JSON snapshot keeps cross-network queries portable across PostgreSQL and MySQL and preserves content-specific metrics. -**Tech Stack:** PHP 8.5, Laravel 13.24, Horizon 5.47, PostgreSQL and MySQL, Inertia 3.3, Vue 3.5, Tailwind CSS 4, Pest 5, Pest Browser 5. +**Tech Stack:** PHP 8.5, Laravel 13.24, Horizon 5.47, PostgreSQL and MySQL, Inertia Vue 3.6, Vue 3.5, Tailwind CSS 4, Pest 5, Pest Browser 5. **Spec:** `docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md` @@ -24,10 +24,13 @@ - Followers retry at widely spaced same-day windows and carry the last value forward only after the day is exhausted; provider `Retry-After` wins when valid. - `/analytics`, post detail, REST, and MCP make no social-provider calls and never use Redis as the analytics source of truth. - Common aggregate metrics are nullable scalar columns; content-specific metrics use stable enum-backed JSON keys with value, unit, time basis, precision, availability, and provider identity. +- `analytics_sync_states` is not a job ledger: only publication backfill/discovery use it. Queue/Horizon owns attempts and delays; follower and publication snapshots prove successful collection. - Use string columns plus PHP backed enums; do not use database-native enum types. - Every query, migration, unique constraint, and test must work on PostgreSQL and MySQL. - Do not add a charting dependency; use focused Vue/SVG/CSS components and existing UI primitives. -- Do not alter unrelated `package-lock.json` or `ANALYTIC.md` changes already present in the worktree. +- Do not alter the unrelated `package-lock.json` change. Preserve the existing + metric inventory in `ANALYTIC.md`; its only planning change is the note that + distinguishes current behavior from this V1 design. - Generate Laravel files with `php artisan make:* --no-interaction`, use Pest TDD, run `vendor/bin/pint --dirty --format agent` after PHP edits, and commit after each task. ## Review Focus @@ -37,6 +40,8 @@ - Provider null/missing metrics must stay unavailable while a measured numeric zero remains zero; Tasks 3 and 10 add explicit parser and writer tests. - Concurrent TryPost sync and external discovery of the same provider post id must converge to one publication with `trypost` origin; Task 5 tests both arrival orders. - A failed paginated backfill must resume from the last committed cursor and disclose partial/provider-limited coverage instead of restarting or claiming 365 days; Task 9 tests checkpoint, retry, and completion conditions. +- A duplicate or stale page job must never move a provider cursor backward; Task 9 uses a captured checkpoint version and row lock around advancement. +- Pre-rollout `post_platforms` whose social account was already deleted cannot be safely assigned by username; Task 9 skips and reports them instead of inventing historical identity. --- @@ -120,10 +125,10 @@ enum MetricPrecision: string { case Exact = 'exact'; case Approximate = 'approxi enum PublicationOrigin: string { case TryPost = 'trypost'; case External = 'external'; } enum PublicationAvailability: string { case Available = 'available'; case Deleted = 'deleted'; case Unavailable = 'unavailable'; } enum ExposureKind: string { case Reach = 'reach'; case Impressions = 'impressions'; case Views = 'views'; } -enum MetricUnit: string { case Count = 'count'; case Seconds = 'seconds'; case Percent = 'percent'; } +enum MetricUnit: string { case Count = 'count'; case Milliseconds = 'milliseconds'; case Percent = 'percent'; } enum MetricTimeBasis: string { case Lifetime = 'lifetime'; case Range = 'range'; case Rolling90Days = 'rolling_90_days'; case Snapshot = 'snapshot'; } -enum MetricAvailability: string { case Available = 'available'; case Unsupported = 'unsupported'; case Delayed = 'delayed'; case PrivacyLimited = 'privacy_limited'; } -enum SyncCollector: string { case AccountDaily = 'account_daily'; case Publications = 'publications'; case PublicationMetrics = 'publication_metrics'; } +enum MetricAvailability: string { case Available = 'available'; case Unsupported = 'unsupported'; case Unavailable = 'unavailable'; case Delayed = 'delayed'; case PrivacyLimited = 'privacy_limited'; } +enum SyncCollector: string { case PublicationBackfill = 'publication_backfill'; case PublicationDiscovery = 'publication_discovery'; } enum SyncStatus: string { case Pending = 'pending'; case Running = 'running'; case Complete = 'complete'; case Partial = 'partial'; case ProviderLimited = 'provider_limited'; case Failed = 'failed'; } ``` @@ -131,7 +136,43 @@ enum SyncStatus: string { case Pending = 'pending'; case Running = 'running'; ca - [ ] **Step 4: Implement portable migrations and indexes** -Use UUID primary keys and explicit foreign keys. The account table unique key is `workspace_id, social_account_key, snapshot_date`; publication identity is `workspace_id, social_account_key, network, provider_post_id`; publication snapshots are unique on `analytics_publication_id, snapshot_date`; sync states are unique on `workspace_id, social_account_key, collector`. Account snapshots, publications, and sync states also store `network` and `platform_user_id` so the same identity can recover its historical key after deletion/reconnection. +Use UUID primary keys, string-backed enum columns, and explicit foreign keys. + +`analytics_account_daily_snapshots` has `workspace_id` with cascade delete; +nullable `social_account_id` with null-on-delete; non-null +`social_account_key`, `network`, `platform_user_id`, and `platform`; account +name/username/avatar snapshots; `snapshot_date`; nullable +`followers_count`; nullable future account `metrics` JSON; `provenance`, +`precision`, nullable `provider_observed_at`, `collected_at`, and timestamps. Its +unique key is `workspace_id, social_account_key, snapshot_date`. + +`analytics_publications` has `workspace_id` with cascade delete; nullable live +`social_account_id` and unique nullable `post_platform_id`, both null-on-delete; +non-null `social_account_key`, `network`, `platform_user_id`, `platform`, +`provider_post_id`, `provider_published_at`, `origin`, `content_type`, and +`availability`; nullable provider content type, permalink, excerpt, preview +metadata, account presentation snapshots, first/last seen times, +provider-synced time, and provider metadata JSON. Its provider identity unique +key is `workspace_id, social_account_key, network, provider_post_id`. + +`analytics_publication_daily_snapshots` has only its UUID, non-null parent +`analytics_publication_id` with cascade delete, `snapshot_date`, `collected_at`, +nullable `provider_observed_at`, nullable metric-catalog JSON, and nullable +portable projections: reactions, comments, shares, saves, views, impressions, +reach, engagement, exposure, exposure kind, total watch milliseconds, and +average watch milliseconds. It deliberately has no duplicate `workspace_id`. +Its unique key is `analytics_publication_id, snapshot_date`. + +`analytics_sync_states` has a non-null `social_account_id` with cascade delete, +collector, status, nullable provider-specific `checkpoint` JSON, +`target_since`, `oldest_reached_at`, `high_watermark_at`, `last_success_at`, +sanitized `last_error_category`, and timestamps. It deliberately has no +workspace/account-history copies, attempt count, retry timestamp, or raw error +message. Its unique key is `social_account_id, collector`. + +Only snapshots and publications store `network` plus `platform_user_id`, because +they are historical identity. Operational sync state is tied to the live row and +is recreated on reconnect. Add these query indexes: @@ -141,10 +182,16 @@ $table->index(['workspace_id', 'social_account_key', 'snapshot_date']); $table->index(['workspace_id', 'provider_published_at']); $table->index(['workspace_id', 'social_account_key', 'provider_published_at']); $table->index(['analytics_publication_id', 'collected_at']); -$table->index(['workspace_id', 'collector', 'status']); +$table->index(['collector', 'status']); ``` -`social_account_id` and `post_platform_id` use `nullOnDelete()`. `workspace_id` uses `cascadeOnDelete()`. Keep provider ids as strings, metric counters as nullable big integers, rates/precise values as nullable decimals, timestamps below the MySQL 2038 ceiling, and JSON object assertions order-independent. +Historical `social_account_id` and `post_platform_id` use `nullOnDelete()`; +sync-state `social_account_id` and every `workspace_id` use +`cascadeOnDelete()`. Keep provider ids as bounded strings, metric counters and +canonical durations as nullable big integers, precise rates as nullable +decimals, timestamps below the MySQL 2038 ceiling, and JSON object assertions +order-independent. Test that a publication snapshot cannot carry a tenant id +different from its parent because no such child column exists. - [ ] **Step 5: Run schema tests on the configured database** @@ -237,6 +284,8 @@ final readonly class MetricValue public MetricPrecision $precision, public MetricAvailability $availability, public ?string $providerMetric = null, + public ?CarbonImmutable $periodStart = null, + public ?CarbonImmutable $periodEnd = null, ) {} } ``` @@ -245,7 +294,21 @@ Models use `HasUuids`, `HasFactory`, explicit `$fillable`, enum/date/array casts - [ ] **Step 4: Implement transactional upsert writers** -Use the unique business keys rather than process-local locks. `ResolveAnalyticsAccountKey` searches historical account snapshots, publications, or sync state by workspace + `Platform::network()` + `platform_user_id`, and otherwise returns the current social-account UUID. Snapshot presentation fields come from the account at write time. Publication metrics map stable scalar fields and serialize `MetricValue` entries keyed by `MetricKey::value`; a missing value never overwrites the latest successful scalar with zero. +Use the unique business keys rather than process-local locks. +`ResolveAnalyticsAccountKey` searches historical account snapshots and +publications by workspace + `Platform::network()` + `platform_user_id`, and +otherwise returns the current social-account UUID. Sync state is operational +and is never an identity source. Snapshot presentation fields come from the +account at write time. + +The publication writer locks the same-day row and atomically merges the metric +catalog and scalar projections. A collector response may update the metrics it +actually observed, but a missing/unsupported/delayed value never blanks a prior +successful same-day value and never becomes zero. The test compares every +scalar projection against its canonical JSON entry so the two representations +cannot drift. A carried-forward follower snapshot retains the original +`provider_observed_at` while recording its new `collected_at`, so staleness is +not hidden. ```php return AnalyticsAccountDailySnapshot::query()->updateOrCreate( @@ -384,13 +447,27 @@ The command uses `lazyById(200)` and dispatches IDs only. The job re-queries the On a transient/rate-limit exception, release near the next `06:00`, `10:00`, `14:00`, `18:00`, or `22:00` UTC window, honoring a later provider time inside the same UTC day. Authentication/permission errors use existing account-health handling and do not write a value. Set job `retryUntil()` to the end of its observation day. +Set `tries = 6` for the initial `02:00` attempt plus the five delayed windows. +Because `release()` consumes an attempt, calculate the next window from the +observation date and current attempt rather than using a fast `backoff()` array. +If `Retry-After` points beyond the UTC day, stop retrying and let the finalizer +decide whether a historical value exists. + - [ ] **Step 4: Implement end-of-day fallback** -`FinalizeAccountDailySnapshots` iterates eligible accounts without an actual row. It copies the last non-null follower count into the current date with `CarriedForward`; it writes nothing when history is absent and never overwrites an actual row. +`FinalizeAccountDailySnapshots` iterates eligible accounts without an actual +row. It copies the last non-null follower count into the current date with +`CarriedForward`, preserves the source row's original `provider_observed_at`, +and records a new `collected_at`. It writes nothing when history is absent and +never overwrites an actual row, including when a late successful job races the +finalizer. - [ ] **Step 5: Schedule and observer integration** -Schedule the dispatch command at `02:00` UTC and finalizer after the last retry window, both with `withoutOverlapping()` and `onOneServer()`. Dispatch initial collection `afterCommit()` when an included account becomes connected; observers must never throw during delete/reconnect. +Schedule the dispatch command at `02:00` UTC and finalizer at `23:30` UTC, +after the last retry window, both with `withoutOverlapping()` and +`onOneServer()`. Dispatch initial collection `afterCommit()` when an included +account becomes connected; observers must never throw during delete/reconnect. - [ ] **Step 6: Run queue/schedule tests** @@ -410,6 +487,7 @@ git commit -m "feat: schedule resilient follower analytics" **Files:** - Create: `app/Dto/Analytics/DiscoveredPublication.php` +- Create: `app/Dto/Analytics/TryPostPublicationIdentity.php` - Create: `app/Actions/Analytics/UpsertAnalyticsPublication.php` - Create: `app/Actions/Analytics/SyncTryPostPublication.php` - Create: `app/Jobs/Analytics/SyncTryPostPublication.php` @@ -423,7 +501,10 @@ git commit -m "feat: schedule resilient follower analytics" - [ ] **Step 1: Write failing reconciliation tests for both arrival orders** -Test external-first/TryPost-second, TryPost-first/external-second, duplicate provider pages, same provider id on two social accounts, and workspace isolation. +Test external-first/TryPost-second, TryPost-first/external-second, duplicate +provider pages, same provider id on two social accounts, workspace isolation, +and deletion of the social account after the job is dispatched but before it +runs. ```php expect(AnalyticsPublication::query()->where('provider_post_id', 'remote-1')->count())->toBe(1) @@ -439,11 +520,36 @@ Expected: FAIL because catalog actions/jobs are absent. - [ ] **Step 3: Implement the discovery DTO and transactional upsert** -`DiscoveredPublication` carries provider id, publication time, normalized/provider content type, permalink, excerpt, preview metadata, and provider metadata. Resolve the historical account key first, then lock the provider identity row using workspace + key + normalized network + provider post id. `trypost` origin wins; provider publication time never becomes discovery time; presentation snapshots update only with non-null values. +`DiscoveredPublication` carries provider id, publication time, +normalized/provider content type, permalink, excerpt, preview metadata, and +provider metadata. Resolve the historical account key first, then lock the +provider identity row using workspace + key + normalized network + provider +post id. `trypost` origin wins; provider publication time never becomes +discovery time; presentation snapshots update only with non-null values. + +A lock cannot protect a row that does not exist yet. Treat the database unique +constraint as the final concurrency arbiter: attempt the insert, catch only the +unique-constraint collision, reload the winning row under lock, and merge. Test +that simultaneous discovery and TryPost sync converge without swallowing any +other database error. + +`TryPostPublicationIdentity` is a token-free primitive snapshot captured while +the live account still exists: workspace id, social-account id, resolved +historical key, normalized network, provider account id, platform, and account +presentation. The queued local-catalog sync receives this DTO plus the +post-platform id. This closes the race where a user deletes the account after +dispatch but before the job runs; the job must not depend on reloading the live +account to establish historical identity. - [ ] **Step 4: Dispatch catalog sync after a destination becomes published** -Extend `PostPlatformObserver` independently of the PostHog flag: whenever status changes to `Published` and `platform_post_id` is present on an included platform, dispatch `App\Jobs\Analytics\SyncTryPostPublication` with the post-platform id and `afterCommit()`. +Extend `PostPlatformObserver` independently of the PostHog flag: whenever +status changes to `Published` and `platform_post_id` is present on an included +platform, resolve the identity snapshot and dispatch +`App\Jobs\Analytics\SyncTryPostPublication` with that snapshot and the +post-platform id using `afterCommit()`. The observer remains non-throwing: a +local sync dispatch failure is reported and repaired by the rollout/daily local +reconciliation command. - [ ] **Step 5: Run reconciliation/observer tests** @@ -618,12 +724,19 @@ git commit -m "feat: discover open network publications" - [ ] **Step 1: Write failing job-chain and resume tests** -Test one provider page per execution, cursor committed only after publication upserts, continuation dispatch after commit, duplicate job idempotency, failure resume, 365-day stop, exhausted stop, provider-limited stop, overlap window, existing-account rollout chunking, and per-account isolation. +Test one provider page per execution, separate backfill/discovery state rows, +cursor committed only after publication upserts, continuation dispatch after +commit, duplicate job idempotency, stale checkpoint version rejection, account +deletion cascading operational state only, reconnect creating fresh state while +reusing historical publication identity, failure resume, 365-day stop, +exhausted stop, provider-limited stop, overlap window, existing-account rollout +chunking, and per-account isolation. Assert daily discovery is suppressed while +backfill is pending/running and enabled after every terminal backfill state. ```php Bus::assertDispatched(BackfillAccountPublications::class, fn ($job) => $job->socialAccountId === $account->id); -expect($state->fresh()->cursor)->toBe('provider-next-page') +expect($state->fresh()->checkpoint['cursor'])->toBe('provider-next-page') ->and($state->fresh()->status)->toBe(SyncStatus::Running); ``` @@ -635,13 +748,48 @@ Expected: FAIL because jobs and command are absent. - [ ] **Step 3: Implement sync-state locking and bounded jobs** -Jobs carry account/state ids only. Inside a transaction, lock the sync state, read its cursor/cutoff, fetch one page outside the transaction, then lock again, upsert the page, and advance the cursor if it still matches. A stale duplicate job exits without moving the cursor backward. - -Set `target_since` to connection-time minus 365 days for initial history. Daily discovery uses the high-water mark minus a fixed overlap window and the same provider identity unique key. Persist `oldest_reached_at`, `last_success_at`, and truthful final status. +Jobs carry account/state ids only and use a provider-specific queue limiter. +Inside a short transaction, lock the sync-state row, capture its checkpoint and +`updated_at` version, and mark it running. Fetch exactly one provider page +outside the transaction. In a second transaction, lock the state again and +upsert the returned publications idempotently. Advance `checkpoint`, coverage, +and high-water fields only if the captured checkpoint/version still matches; +otherwise leave progress untouched. A stale duplicate may safely reconcile +facts but can never move the cursor backward. + +The sync table has no retry counters or timestamps: queue attempts, classified +delays, failed-job storage, and Horizon are authoritative. Persist only the +sanitized latest error category needed to explain coverage in the UI; never a +raw provider body. + +Set `target_since` to the bootstrap time minus 365 days for initial history. +Daily discovery uses the high-water mark minus a fixed overlap window and the +same provider identity unique key. Persist `oldest_reached_at`, +`last_success_at`, and a truthful final status. A provider listing omission is +not proof of deletion: mark a publication deleted/unavailable only on an +explicit provider response for that publication. + +When backfill first becomes terminal, initialize discovery from the newest +provider publication already stored for that account, falling back to the +current time only when the catalog is empty. If a provider invalidates an old +cursor, clear only that cursor and restart from `oldest_reached_at` plus an +overlap window; idempotent publication identity prevents duplicates and the +365-day target remains unchanged. - [ ] **Step 4: Implement rollout and local TryPost backfill** -`analytics:backfill-existing` uses `lazyById(100)` to dispatch `BootstrapAccountAnalytics` for active included accounts and `BackfillTryPostPublications` for published included destinations. The command itself performs no provider calls and accepts an optional workspace id for controlled rollout. +`analytics:backfill-existing` first uses `lazyById(100)` to dispatch +`BackfillTryPostPublications` for published included destinations with a live +social account, then dispatches `BootstrapAccountAnalytics` for active included +accounts. The command itself performs no provider calls and accepts an optional +workspace id for controlled rollout. + +Pre-rollout published destinations whose `social_account_id` is already null +are counted and logged as `historical_identity_unrecoverable`; they are not +merged by username and no synthetic account key is invented. This limitation +applies only to facts orphaned before the analytics catalog exists. The command +is repeatable, and discovery later reconciles any reachable provider post by +its real account identity. - [ ] **Step 5: Connect observer and schedule** @@ -716,7 +864,12 @@ interface PublicationMetricsCollector } ``` -Split Meta metric families that cannot share one request. Store watch time canonically in seconds. Compute normalized engagement numerator from supported interaction components and preserve `exposure_count` plus `ExposureKind`; do not store a provider engagement rate as if it were the normalized TryPost rate. +Split Meta metric families that cannot share one request. Store all durations +canonically as integer milliseconds and convert only at presentation time. +Compute normalized engagement numerator from supported interaction components +and preserve `exposure_count` plus `ExposureKind`; do not store a provider +engagement rate as if it were the normalized TryPost rate. The writer updates +the JSON catalog and every corresponding scalar projection in one transaction. Refactor existing service methods to share low-level authenticated requests/parsers where safe, but do not return translated labels to persistence. Excluded providers remain callable by legacy code until Task 13 removes their analytics read paths, but the new factory never returns them. @@ -765,11 +918,23 @@ Expected: FAIL because jobs are absent. - [ ] **Step 3: Implement metric job and dispatcher** -The job re-queries publication and live account, skips stale/excluded rows, collects, then writes one daily snapshot. Use the same classified same-day retry policy as followers. Provider-specific internal batching may claim several publication ids, but one failed batch must be split or classified without erasing successful values. +The job re-queries publication and live account, skips stale/excluded rows, +collects, then writes one daily snapshot. Use the same classified same-day retry +policy as followers. Default to a bounded batch job per account/provider; use a +single publication for endpoints that do not batch and the documented provider +maximum for Pinterest, TikTok, and YouTube. Persist successful items before +retrying only failed items, so one bad id never discards a whole successful +batch. - [ ] **Step 4: Implement import handoff and Story schedule** -New publications inside the refresh window dispatch normal collection. Older backfill rows dispatch one baseline job recorded by sync metadata. Instagram Stories dispatch immediately, at configured within-lifetime checkpoints, and once shortly before expiry; all writes converge on the daily writer. +New publications inside the refresh window dispatch normal collection. Older +backfill rows dispatch one baseline job only when that publication has no +snapshot; baseline completion is therefore proved by the fact table, not sync +metadata. Instagram Stories dispatch immediately, at configured within-lifetime +checkpoints, and once shortly before expiry; all writes converge on the daily +writer. A delayed insight must not be mistaken for unsupported, and Story jobs +stop after the provider availability window. - [ ] **Step 5: Schedule daily metric dispatch and run tests** @@ -811,6 +976,10 @@ Create two Instagram accounts and one X account in the same workspace plus a for - deterministic Top 5 ties by publication time then id; - Performance rows per social account, including two separate Instagram rows; - historical rows after live account deletion; +- cumulative metrics for posts selected by publication date use their latest + successful observation and are never summed across snapshot dates; +- an imported YouTube upload without authoritative Short metadata is presented + as YouTube Video, not falsely as YouTube Short; - no excluded platform or foreign-workspace contribution; - unavailable/null distinct from zero. @@ -962,7 +1131,12 @@ Use a single root element, existing `DateRangePicker`, and an Inertia GET visit - [ ] **Step 5: Implement reporting blocks and translations** -Summary contains only Posts, Total Followers, Reactions, Comments, and Engagement Rate. Top 5 cards show destination origin and only valid actions. Performance sorting is local over the returned rows. Unsupported renders an em dash, never `0`; measured zero renders `0`. +Summary contains only Posts, Total Followers, Reactions, Comments, and +Engagement Rate. Top 5 cards show destination origin and only valid actions. +Performance sorting is local over the returned rows. Unsupported renders an em +dash, never `0`; measured zero renders `0`. Summary, Top 5, and Performance +tooltips disclose that reactions/comments are the latest cumulative values for +posts published in the selected period, not events that occurred inside it. - [ ] **Step 6: Run browser and frontend checks** @@ -1098,11 +1272,16 @@ Before dispatching the production rollout: 1. confirm the production TikTok app has `video.list` and `user.info.stats`; 2. connect/test one Instagram-direct and one Instagram-via-Facebook account; -3. reconnect a Mastodon test account with `read:statuses` and confirm private-history behavior; -4. measure X read cost on one bounded 365-day account before widening rollout; -5. confirm Threads follower insights with a production-approved token; -6. export the selected canary workspace UUID as `ANALYTICS_CANARY_WORKSPACE_ID`, then run `php artisan analytics:backfill-existing --workspace="$ANALYTICS_CANARY_WORKSPACE_ID"`; -7. verify coverage states, then run the command without the workspace filter. +3. verify Facebook Page posts, videos/Reels, and follower fields with the current Page token; +4. confirm Threads follower insights and owned-post pagination with a production-approved token; +5. measure X follower, owned-post, and metric read cost on one bounded 365-day account before widening rollout; +6. verify Pinterest owned-Pin bookmarks, lifetime/range analytics, and the production app's read scopes; +7. verify the YouTube uploads playlist, batched video details, channel statistics, and Analytics API scopes; +8. confirm Bluesky repository pagination plus public count hydration against a large account; +9. reconnect a Mastodon test account with `read:statuses` and confirm private-history behavior on two different instances; +10. verify follower collection once for every included platform and both Instagram login variants; +11. export the selected canary workspace UUID as `ANALYTICS_CANARY_WORKSPACE_ID`, then run `php artisan analytics:backfill-existing --workspace="$ANALYTICS_CANARY_WORKSPACE_ID"`; +12. verify coverage, orphan-skip, rate-limit, and retry states, then run the command without the workspace filter. - [ ] **Step 7: Update spec status and commit** @@ -1116,6 +1295,16 @@ git commit -m "chore: finalize workspace analytics rollout" ## Self-Review Results - **Spec coverage:** Every V1 surface, included/excluded platform, follower fallback, native backfill, reconciliation rule, metric catalog, date range, Summary, Top 5, Performance, individual detail, REST/MCP read path, and LinkedIn V2 boundary maps to Tasks 1–16. +- **Schema audit:** The four-table design is retained as the minimum safe split. + Publication snapshots no longer duplicate tenant ownership, and sync state is + reduced to two live-account cursor workflows rather than becoming a second + job/fact ledger. +- **Concurrency audit:** Unique constraints arbitrate missing-row races, + same-day metric families merge under a row lock, and paginated jobs advance + only a checkpoint version they actually fetched. +- **Lifecycle audit:** Historical facts survive account/post deletion, sync + checkpoints do not, reconnects reuse identity through provider ids, and + pre-rollout orphan destinations are skipped and disclosed rather than guessed. - **Placeholder scan:** The plan contains no forbidden placeholder markers, no unnamed error handling, and no task that delegates unspecified work. Provider mappings and final manual capability gates are explicit. - **Type consistency:** The four model names, DTO constructors, collector signatures, origin values, sync states, and query method names are introduced once and reused consistently. - **Review focus:** Each of the five highest-risk inputs is pinned to an explicit test in Tasks 1/2, 3/10, 5, 9, or 12. diff --git a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md index 9cd845a22..df8fc6d9b 100644 --- a/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md +++ b/docs/superpowers/specs/2026-09-23-workspace-follower-analytics-design.md @@ -197,6 +197,12 @@ Each imported publication retains, when available: - origin (`trypost` or `external`); - provider coverage and availability state. +The publishing enum labels the integration `YouTube Shorts`, but the owned +uploads API returns every channel upload and does not authoritatively classify +all of them as Shorts. Analytics labels imported unknown uploads as `YouTube` +and content type `Video`; only a TryPost destination or provider field that +proves a Short may render `Short`. + An imported post is an analytics record, not a draft or published `Post` owned by the TryPost publishing workflow. Importing it must not enable editing, deletion, retry, repurpose processing, or publishing lifecycle actions. The @@ -303,6 +309,12 @@ from Engagement Rate only. Its supported reactions and comments still contribute to those cards. Unsupported metrics render as unavailable and are never converted to zero. +Provider post metrics are cumulative. Historical range reports therefore +answer “how have posts published in this period performed as of their latest +collection,” not “how many reactions happened during this period.” Summary, +Top 5, Performance, and comparison tooltips state this explicitly; TryPost does +not infer a daily reaction timeline that providers did not return. + ### Period comparison Summary and Performance compare the selected inclusive range with the @@ -490,8 +502,9 @@ by the connected account, login type, API version, and media type. | Pinterest video Pin | Every applicable image-Pin metric plus video views, average video play time, 10-second plays, plays to 95%, and total play time | The current collector only adds basic video views and omits the richer video-retention metrics | Instagram Reel total watch time is displayed in minutes and average watch time -in seconds, matching the reference UI, while persistence retains the canonical -unit needed to avoid rounding loss. Metrics that Meta marks estimated or in +in seconds, matching the reference UI, while persistence stores both as integer +milliseconds to avoid rounding drift between providers and displays. Metrics +that Meta marks estimated or in development, currently including Reel reach, watch time, views, total interactions, and skip rate as applicable, preserve that precision/stability metadata for tooltips. @@ -761,6 +774,20 @@ Database enum types are not used. Enum-backed values are stored in string columns and cast through PHP enums so new providers and metrics do not require engine-specific enum migrations. +The reviewed four-table split is: + +| Table | Cardinality and responsibility | Why it is separate | +| --- | --- | --- | +| `analytics_account_daily_snapshots` | One account/day follower fact | Time-series values and carry-forward provenance | +| `analytics_publications` | One account/provider-post identity | Reconciles TryPost and externally discovered posts once | +| `analytics_publication_daily_snapshots` | One publication/day cumulative metric set | Atomic metric history and portable ranking projections | +| `analytics_sync_states` | One live account/import collector checkpoint | Cursor/high-water coordination only; deleted with the account | + +This is the smallest design that preserves strict keys for each different +cardinality. It intentionally does not introduce a fifth shadow-account table, +does not store workspace totals, and does not mix operational cursors into fact +rows. + ### Daily account snapshots `analytics_account_daily_snapshots` stores one effective observation per @@ -827,19 +854,27 @@ publication is counted once. `analytics_publication_daily_snapshots` stores at most one cumulative observation per analytics publication and UTC date. Repeated successful -collections on the same date update that row instead of creating additional -facts. It contains nullable first-class aggregate columns for reactions, -comments, shares, saves, views, impressions, reach, total watch time, and -average watch time, plus normalized engagement numerator, exposure denominator, -and exposure kind. +collections on the same date merge into that row under a database row lock +instead of creating additional facts or blanking a metric family collected by +another provider request. It does not duplicate `workspace_id`; workspace +ownership is obtained through the parent analytics publication, preventing an +inconsistent child/parent tenant pair. + +Its portable aggregate projections are nullable big integers for +`reactions_count`, `comments_count`, `shares_count`, `saves_count`, +`views_count`, `impressions_count`, `reach_count`, `engagement_count`, +`exposure_count`, `watch_time_milliseconds`, and +`average_watch_time_milliseconds`, plus nullable `exposure_kind`. A null means +unavailable; a numeric zero means measured zero. Milliseconds are the canonical +duration unit and the UI converts them to minutes or seconds. The same row has a JSON metric catalog for provider/content-specific values. Each JSON entry uses a stable enum-backed metric key and retains numeric value, unit, provider metric identity, lifetime/range/rolling time basis, -exact/estimated/experimental precision, and availability. Cross-network queries -use the first-class columns; the JSON catalog powers the richer individual-post -detail. This hybrid avoids both engine-specific JSON aggregation and an EAV row -explosion. +nullable period start/end for range metrics, exact/estimated/experimental +precision, and availability. Cross-network queries use the first-class columns; +the JSON catalog powers the richer individual-post detail. This hybrid avoids +both engine-specific JSON aggregation and an EAV row explosion. Imported historical publications receive a real baseline observation collected at import time. The system does not fabricate daily metric history between the @@ -847,19 +882,61 @@ publication date and that baseline. ### Synchronization state -`analytics_sync_states` stores durable operational state per workspace, -immutable social-account key, and collector. Collector values initially cover -daily account snapshots, owned-publication history/discovery, and publication -metrics. The row retains status, cursor, target cutoff, high-water mark, oldest -and latest provider dates reached, last successful synchronization, next retry, -attempt count, and a sanitized last-error category/message. - -This state does not live in `social_accounts.meta`: collectors advance -independently, need row-level concurrency control, and must survive deletion or -reconnection of the live account row. The unique key is workspace + -social-account key + collector. - -Regardless of the final schema, persistence must support: +`analytics_sync_states` is deliberately a small operational checkpoint table, +not a general log of every analytics job. It exists only for workflows whose +progress cannot be inferred from fact rows: the finite 365-day publication +backfill and continuing publication discovery. Daily follower success is +represented by an account snapshot, publication-metric success by a publication +snapshot, and attempts/retries by the queue and Horizon; duplicating those in a +sync-state row would create competing sources of truth. + +Each row belongs to one live `social_account_id` with `cascadeOnDelete()` and +one collector (`publication_backfill` or `publication_discovery`). Its columns +are: UUID primary key, non-null social-account foreign key, collector, status, +nullable JSON `checkpoint`, `target_since`, `oldest_reached_at`, +`high_watermark_at`, `last_success_at`, sanitized `last_error_category`, and +timestamps. The unique key is social account + collector. Workspace, platform, +historical account key, next retry, attempt count, and raw error message are not +duplicated here. + +The checkpoint is the one safe JSON boundary for sync control: provider cursor +shapes vary, it is never filtered or aggregated by SQL, and exactly one +collector owns each row. The job locks that row before reading or advancing the +checkpoint. Keeping these rows separate from `social_accounts.meta` prevents +unrelated collectors from overwriting one shared JSON object or locking the +entire account row. + +Operational state does not need to survive deletion. Historical facts remain +in the three analytics tables; deleting the live account removes its obsolete +checkpoint. Reconnecting the same provider identity receives fresh operational +state, while the identity resolver reuses the prior `social_account_key` found +in account snapshots or publications and provider-id uniqueness makes the +restarted import idempotent. + +Backfill and discovery use separate rows and lifecycles. Discovery does not run +while backfill is pending or running. Once backfill reaches a terminal state +(`complete`, `provider_limited`, `partial`, or `failed`), discovery may keep new +content current while a partial backfill is retried independently. + +For every page, a job captures the locked checkpoint and row version, releases +the transaction before the provider request, then locks the row again. It may +upsert publications idempotently, but advances the checkpoint only when the +captured version still matches; a stale duplicate can never move the cursor +backward. + +### Historical identity limitation at rollout + +Existing `post_platforms` rows null `social_account_id` when an account is +deleted and retain only presentation fields, not the provider account id or the +original social-account UUID. Therefore pre-rollout orphan destinations cannot +be assigned to a historical account without guessing from a mutable username. +The local catalog backfill imports only destinations that still have a live +social account and records the omitted-orphan count in rollout logs. It must not +invent an account key or merge rows by username. After rollout, every analytics +publication is written while the account identity is available and remains +historically addressable after later deletion. + +This approved schema must support: - workspace-scoped queries; - social-account breakdown; @@ -982,9 +1059,10 @@ this first delivery. observations and imported publications even if the account row is later removed. Nullable live foreign keys plus immutable `social_account_key` and presentation snapshots retain that identity. -- **Reconnected as the same persisted identity:** resume collection without - rewriting earlier observations and resume native discovery from its - checkpoint with an overlap window. +- **Reconnected as the same provider identity:** reuse the historical + `social_account_key` without rewriting earlier observations, create fresh + operational checkpoints, and restart the idempotent native import. Existing + publications deduplicate by historical account key + provider post id. - **New identity:** begins a new series even when its username matches an older disconnected account. @@ -1056,8 +1134,12 @@ Post-performance collector tests additionally cover: or a documented provider limit and records which condition ended the import. - A bounded page can re-dispatch continuation work without holding one worker for the entire backfill. -- Cursor and high-water checkpoints resume safely after transient failure and - after reconnecting the same platform identity. +- Cursor and high-water checkpoints resume safely after transient failure. +- Deleting an account cascades only its operational checkpoints; reconnecting + the same provider identity creates fresh checkpoints while deduplicating + already imported facts against the reused historical account key. +- Concurrent page jobs compare the captured checkpoint version and cannot move + a cursor backward. - Daily native discovery overlaps the last completed window and remains idempotent when a provider returns the same page or a late post twice. - Backfill and discovery failures for one social account do not block any other @@ -1101,6 +1183,8 @@ Post-performance collector tests additionally cover: with the imported publication rather than creating a duplicate. - Reconnecting the same identity resumes the existing catalog; a genuinely new platform identity starts a separate catalog even when the username matches. +- Pre-rollout TryPost destinations already orphaned from their social account + are reported and skipped instead of being guessed or grouped by username. - Imported publications never create fake `Post` or `PostPlatform` lifecycle records and cannot be edited, deleted, retried, or published from TryPost. - The provider publication timestamp, rather than discovery time, controls @@ -1190,6 +1274,25 @@ design is approved and implemented. cumulative publication metrics, and resumable cursors have different cardinality and lifecycle. Combining them creates sparse rows and weak constraints. The four-table hybrid is the minimum safe design. +- **A fifth `analytics_accounts` dimension table.** It would normalize repeated + identity/presentation columns and is defensible at warehouse scale, but every + fact would then need another join and the application would maintain a shadow + account lifecycle solely for analytics. The current four-table design keeps + the minimum table count while repeating only small immutable snapshots. +- **Putting collector cursors in `social_accounts.meta`.** Multiple collectors + would contend on one account row and could overwrite independent JSON + branches. One checkpoint row per collector gives a narrow lock and a unique + owner without turning sync state into a general job log. +- **Persisting sync state for follower and publication-metric jobs.** Their fact + snapshots already prove successful work, while queue/Horizon records attempts + and failures. Duplicating that status would create drift and extra writes. +- **Preserving sync-state rows after account deletion.** Checkpoints are + operational, not historical facts. Cascading them prevents dead work from + appearing resumable; a reconnect safely restarts against idempotent facts. +- **Guessing pre-rollout orphan identity from username.** Usernames can change + or be reused, and existing orphaned `post_platforms` lack the provider account + id. Skipping and reporting those rows is more truthful than merging unrelated + accounts. - **JSON-only post metrics.** Cross-network ranking and aggregation would depend on engine-specific JSON queries. Common aggregate fields are first-class nullable columns; provider/content-specific metrics remain structured JSON. @@ -1258,8 +1361,9 @@ design is approved and implemented. - YouTube Analytics metrics and channel report combinations: and -- TikTok Display API video query and Video Object fields: - and +- TikTok Display API video list/query and Video Object fields: + , + , and - Pinterest organic reporting and metric definitions: From ae64f83ded817e66abd82aaf48d72bdda69f9031 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 11:24:17 -0300 Subject: [PATCH 13/77] feat: add workspace analytics schema --- app/Enums/Analytics/ExposureKind.php | 12 ++ app/Enums/Analytics/MetricAvailability.php | 14 ++ app/Enums/Analytics/MetricKey.php | 57 ++++++ app/Enums/Analytics/MetricPrecision.php | 13 ++ app/Enums/Analytics/MetricTimeBasis.php | 13 ++ app/Enums/Analytics/MetricUnit.php | 12 ++ app/Enums/Analytics/ObservationProvenance.php | 11 ++ .../Analytics/PublicationAvailability.php | 12 ++ .../Analytics/PublicationContentType.php | 19 ++ app/Enums/Analytics/PublicationOrigin.php | 11 ++ app/Enums/Analytics/SyncCollector.php | 11 ++ app/Enums/Analytics/SyncStatus.php | 15 ++ ...nalytics_account_daily_snapshots_table.php | 62 +++++++ ...34_create_analytics_publications_table.php | 72 ++++++++ ...tics_publication_daily_snapshots_table.php | 57 ++++++ ...036_create_analytics_sync_states_table.php | 46 +++++ .../Feature/Analytics/AnalyticsSchemaTest.php | 170 ++++++++++++++++++ 17 files changed, 607 insertions(+) create mode 100644 app/Enums/Analytics/ExposureKind.php create mode 100644 app/Enums/Analytics/MetricAvailability.php create mode 100644 app/Enums/Analytics/MetricKey.php create mode 100644 app/Enums/Analytics/MetricPrecision.php create mode 100644 app/Enums/Analytics/MetricTimeBasis.php create mode 100644 app/Enums/Analytics/MetricUnit.php create mode 100644 app/Enums/Analytics/ObservationProvenance.php create mode 100644 app/Enums/Analytics/PublicationAvailability.php create mode 100644 app/Enums/Analytics/PublicationContentType.php create mode 100644 app/Enums/Analytics/PublicationOrigin.php create mode 100644 app/Enums/Analytics/SyncCollector.php create mode 100644 app/Enums/Analytics/SyncStatus.php create mode 100644 database/migrations/2026_09_23_142033_create_analytics_account_daily_snapshots_table.php create mode 100644 database/migrations/2026_09_23_142034_create_analytics_publications_table.php create mode 100644 database/migrations/2026_09_23_142035_create_analytics_publication_daily_snapshots_table.php create mode 100644 database/migrations/2026_09_23_142036_create_analytics_sync_states_table.php create mode 100644 tests/Feature/Analytics/AnalyticsSchemaTest.php diff --git a/app/Enums/Analytics/ExposureKind.php b/app/Enums/Analytics/ExposureKind.php new file mode 100644 index 000000000..250a616fc --- /dev/null +++ b/app/Enums/Analytics/ExposureKind.php @@ -0,0 +1,12 @@ +uuid('id')->primary(); + $table->uuid('workspace_id'); + $table->uuid('social_account_id')->nullable(); + $table->uuid('social_account_key'); + $table->string('network', 32); + $table->string('platform_user_id', 191); + $table->string('platform', 32); + $table->string('account_display_name')->nullable(); + $table->string('account_username')->nullable(); + $table->text('account_avatar_url')->nullable(); + $table->date('snapshot_date'); + $table->bigInteger('followers_count')->nullable(); + $table->json('metrics')->nullable(); + $table->string('provenance', 32); + $table->string('precision', 32); + $table->timestamp('provider_observed_at')->nullable(); + $table->timestamp('collected_at'); + $table->timestamps(); + + $table->foreign('workspace_id', 'analytics_account_daily_workspace_fk') + ->references('id')->on('workspaces')->cascadeOnDelete(); + $table->foreign('social_account_id', 'analytics_account_daily_account_fk') + ->references('id')->on('social_accounts')->nullOnDelete(); + $table->unique( + ['workspace_id', 'social_account_key', 'snapshot_date'], + 'analytics_account_daily_identity_unique', + ); + $table->index( + ['workspace_id', 'snapshot_date'], + 'analytics_account_daily_workspace_date_index', + ); + $table->index( + ['workspace_id', 'social_account_key', 'snapshot_date'], + 'analytics_account_daily_account_date_index', + ); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('analytics_account_daily_snapshots'); + } +}; diff --git a/database/migrations/2026_09_23_142034_create_analytics_publications_table.php b/database/migrations/2026_09_23_142034_create_analytics_publications_table.php new file mode 100644 index 000000000..2e442f30b --- /dev/null +++ b/database/migrations/2026_09_23_142034_create_analytics_publications_table.php @@ -0,0 +1,72 @@ +uuid('id')->primary(); + $table->uuid('workspace_id'); + $table->uuid('social_account_id')->nullable(); + $table->uuid('social_account_key'); + $table->uuid('post_platform_id')->nullable(); + $table->string('network', 32); + $table->string('platform_user_id', 191); + $table->string('platform', 32); + $table->string('provider_post_id', 191); + $table->timestamp('provider_published_at'); + $table->string('origin', 32); + $table->string('content_type', 32); + $table->string('availability', 32); + $table->string('provider_content_type', 64)->nullable(); + $table->text('permalink')->nullable(); + $table->text('excerpt')->nullable(); + $table->json('preview_metadata')->nullable(); + $table->string('account_display_name')->nullable(); + $table->string('account_username')->nullable(); + $table->text('account_avatar_url')->nullable(); + $table->timestamp('first_seen_at'); + $table->timestamp('last_seen_at'); + $table->timestamp('provider_synced_at')->nullable(); + $table->json('provider_metadata')->nullable(); + $table->timestamps(); + + $table->foreign('workspace_id', 'analytics_publications_workspace_fk') + ->references('id')->on('workspaces')->cascadeOnDelete(); + $table->foreign('social_account_id', 'analytics_publications_account_fk') + ->references('id')->on('social_accounts')->nullOnDelete(); + $table->foreign('post_platform_id', 'analytics_publications_post_platform_fk') + ->references('id')->on('post_platforms')->nullOnDelete(); + $table->unique('post_platform_id', 'analytics_publications_post_platform_unique'); + $table->unique( + ['workspace_id', 'social_account_key', 'network', 'provider_post_id'], + 'analytics_publications_identity_unique', + ); + $table->index( + ['workspace_id', 'provider_published_at'], + 'analytics_publications_workspace_date_index', + ); + $table->index( + ['workspace_id', 'social_account_key', 'provider_published_at'], + 'analytics_publications_account_date_index', + ); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('analytics_publications'); + } +}; diff --git a/database/migrations/2026_09_23_142035_create_analytics_publication_daily_snapshots_table.php b/database/migrations/2026_09_23_142035_create_analytics_publication_daily_snapshots_table.php new file mode 100644 index 000000000..4330a48bf --- /dev/null +++ b/database/migrations/2026_09_23_142035_create_analytics_publication_daily_snapshots_table.php @@ -0,0 +1,57 @@ +uuid('id')->primary(); + $table->uuid('analytics_publication_id'); + $table->date('snapshot_date'); + $table->timestamp('collected_at'); + $table->timestamp('provider_observed_at')->nullable(); + $table->json('metrics')->nullable(); + $table->bigInteger('reactions_count')->nullable(); + $table->bigInteger('comments_count')->nullable(); + $table->bigInteger('shares_count')->nullable(); + $table->bigInteger('saves_count')->nullable(); + $table->bigInteger('views_count')->nullable(); + $table->bigInteger('impressions_count')->nullable(); + $table->bigInteger('reach_count')->nullable(); + $table->bigInteger('engagement_count')->nullable(); + $table->bigInteger('exposure_count')->nullable(); + $table->string('exposure_kind', 32)->nullable(); + $table->bigInteger('watch_time_milliseconds')->nullable(); + $table->bigInteger('average_watch_time_milliseconds')->nullable(); + $table->timestamps(); + + $table->foreign('analytics_publication_id', 'analytics_publication_daily_parent_fk') + ->references('id')->on('analytics_publications')->cascadeOnDelete(); + $table->unique( + ['analytics_publication_id', 'snapshot_date'], + 'analytics_publication_daily_identity_unique', + ); + $table->index( + ['analytics_publication_id', 'collected_at'], + 'analytics_publication_daily_collected_index', + ); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('analytics_publication_daily_snapshots'); + } +}; diff --git a/database/migrations/2026_09_23_142036_create_analytics_sync_states_table.php b/database/migrations/2026_09_23_142036_create_analytics_sync_states_table.php new file mode 100644 index 000000000..c5f7406fb --- /dev/null +++ b/database/migrations/2026_09_23_142036_create_analytics_sync_states_table.php @@ -0,0 +1,46 @@ +uuid('id')->primary(); + $table->uuid('social_account_id'); + $table->string('collector', 32); + $table->string('status', 32); + $table->json('checkpoint')->nullable(); + $table->timestamp('target_since')->nullable(); + $table->timestamp('oldest_reached_at')->nullable(); + $table->timestamp('high_watermark_at')->nullable(); + $table->timestamp('last_success_at')->nullable(); + $table->string('last_error_category', 64)->nullable(); + $table->timestamps(); + + $table->foreign('social_account_id', 'analytics_sync_states_account_fk') + ->references('id')->on('social_accounts')->cascadeOnDelete(); + $table->unique( + ['social_account_id', 'collector'], + 'analytics_sync_states_account_collector_unique', + ); + $table->index(['collector', 'status'], 'analytics_sync_states_collector_status_index'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('analytics_sync_states'); + } +}; diff --git a/tests/Feature/Analytics/AnalyticsSchemaTest.php b/tests/Feature/Analytics/AnalyticsSchemaTest.php new file mode 100644 index 000000000..8ccfce2a9 --- /dev/null +++ b/tests/Feature/Analytics/AnalyticsSchemaTest.php @@ -0,0 +1,170 @@ +toBeTrue() + ->and(Schema::hasColumns('analytics_publications', [ + 'id', 'workspace_id', 'social_account_id', 'social_account_key', + 'post_platform_id', 'network', 'platform_user_id', 'platform', + 'provider_post_id', 'provider_published_at', 'origin', 'content_type', + 'availability', 'provider_content_type', 'permalink', 'excerpt', + 'preview_metadata', 'account_display_name', 'account_username', + 'account_avatar_url', 'first_seen_at', 'last_seen_at', + 'provider_synced_at', 'provider_metadata', + ]))->toBeTrue() + ->and(Schema::hasColumns('analytics_publication_daily_snapshots', [ + 'id', 'analytics_publication_id', 'snapshot_date', 'collected_at', + 'provider_observed_at', 'metrics', 'reactions_count', 'comments_count', + 'shares_count', 'saves_count', 'views_count', 'impressions_count', + 'reach_count', 'engagement_count', 'exposure_count', 'exposure_kind', + 'watch_time_milliseconds', 'average_watch_time_milliseconds', + ]))->toBeTrue() + ->and(Schema::hasColumn('analytics_publication_daily_snapshots', 'workspace_id'))->toBeFalse() + ->and(Schema::hasColumns('analytics_sync_states', [ + 'id', 'social_account_id', 'collector', 'status', 'checkpoint', + 'target_since', 'oldest_reached_at', 'high_watermark_at', + 'last_success_at', 'last_error_category', + ]))->toBeTrue(); +}); + +test('same-network accounts remain distinct and historical facts survive account deletion', function () { + $workspace = Workspace::factory()->create(); + $firstAccount = SocialAccount::factory()->instagram()->create([ + 'workspace_id' => $workspace->id, + 'platform_user_id' => 'instagram-1', + 'username' => 'first-account', + ]); + $secondAccount = SocialAccount::factory()->instagram()->create([ + 'workspace_id' => $workspace->id, + 'platform_user_id' => 'instagram-2', + 'username' => 'second-account', + ]); + + foreach ([$firstAccount, $secondAccount] as $account) { + DB::table('analytics_account_daily_snapshots')->insert([ + 'id' => Str::uuid()->toString(), + 'workspace_id' => $workspace->id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + 'network' => $account->platform->network(), + 'platform_user_id' => $account->platform_user_id, + 'platform' => $account->platform->value, + 'account_display_name' => $account->display_name, + 'account_username' => $account->username, + 'snapshot_date' => '2026-09-23', + 'followers_count' => 0, + 'provenance' => ObservationProvenance::Actual->value, + 'precision' => MetricPrecision::Exact->value, + 'collected_at' => now(), + 'created_at' => now(), + 'updated_at' => now(), + ]); + + DB::table('analytics_publications')->insert([ + 'id' => Str::uuid()->toString(), + 'workspace_id' => $workspace->id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + 'network' => $account->platform->network(), + 'platform_user_id' => $account->platform_user_id, + 'platform' => $account->platform->value, + 'provider_post_id' => 'same-provider-post-id', + 'provider_published_at' => now(), + 'origin' => PublicationOrigin::External->value, + 'content_type' => PublicationContentType::Image->value, + 'availability' => PublicationAvailability::Available->value, + 'account_display_name' => $account->display_name, + 'account_username' => $account->username, + 'first_seen_at' => now(), + 'last_seen_at' => now(), + 'created_at' => now(), + 'updated_at' => now(), + ]); + } + + DB::table('analytics_sync_states')->insert([ + 'id' => Str::uuid()->toString(), + 'social_account_id' => $firstAccount->id, + 'collector' => SyncCollector::PublicationBackfill->value, + 'status' => SyncStatus::Pending->value, + 'created_at' => now(), + 'updated_at' => now(), + ]); + + expect(DB::table('analytics_account_daily_snapshots')->count())->toBe(2) + ->and(DB::table('analytics_publications')->count())->toBe(2); + + $firstAccount->deleteQuietly(); + + $this->assertDatabaseHas('analytics_account_daily_snapshots', [ + 'social_account_id' => null, + 'social_account_key' => $firstAccount->id, + 'platform' => Platform::Instagram->value, + 'account_username' => 'first-account', + 'followers_count' => 0, + ]); + $this->assertDatabaseHas('analytics_publications', [ + 'social_account_id' => null, + 'social_account_key' => $firstAccount->id, + 'platform' => Platform::Instagram->value, + 'account_username' => 'first-account', + ]); + $this->assertDatabaseMissing('analytics_sync_states', [ + 'social_account_id' => $firstAccount->id, + ]); +}); + +test('analytics enums expose stable persisted values', function (string $enum, array $values) { + expect(enum_exists($enum))->toBeTrue() + ->and(array_column($enum::cases(), 'value'))->toEqual($values); +})->with([ + 'observation provenance' => [ObservationProvenance::class, ['actual', 'carried_forward']], + 'metric precision' => [MetricPrecision::class, ['exact', 'approximate', 'estimated', 'experimental']], + 'publication origin' => [PublicationOrigin::class, ['trypost', 'external']], + 'publication availability' => [PublicationAvailability::class, ['available', 'deleted', 'unavailable']], + 'publication content type' => [PublicationContentType::class, ['text', 'image', 'carousel', 'video', 'reel', 'story', 'short', 'link', 'poll', 'unknown']], + 'exposure kind' => [ExposureKind::class, ['reach', 'impressions', 'views']], + 'metric unit' => [MetricUnit::class, ['count', 'milliseconds', 'percent']], + 'metric time basis' => [MetricTimeBasis::class, ['lifetime', 'range', 'rolling_90_days', 'snapshot']], + 'metric availability' => [MetricAvailability::class, ['available', 'unsupported', 'unavailable', 'delayed', 'privacy_limited']], + 'sync collector' => [SyncCollector::class, ['publication_backfill', 'publication_discovery']], + 'sync status' => [SyncStatus::class, ['pending', 'running', 'complete', 'partial', 'provider_limited', 'failed']], + 'metric key' => [MetricKey::class, [ + 'reactions', 'comments', 'replies', 'shares', 'reposts', 'quotes', 'saves', 'bookmarks', + 'views', 'video_views', 'impressions', 'reach', 'engagements', 'total_interactions', + 'engagement_rate', 'clicks', 'link_clicks', 'pin_clicks', 'pin_click_rate', + 'outbound_clicks', 'outbound_click_rate', 'save_rate', 'follows', 'profile_visits', + 'profile_activity', 'watch_time_milliseconds', 'average_watch_time_milliseconds', + 'average_percentage_viewed', 'skip_rate', 'engaged_views', 'video_views_10_seconds', + 'video_views_95_percent', 'video_quartile_25', 'video_quartile_50', + 'video_quartile_75', 'video_quartile_100', 'total_play_time_milliseconds', + 'average_video_play_time_milliseconds', 'total_audience', 'engaged_audience', + 'subscribers_gained', 'subscribers_lost', 'story_navigation', 'story_taps_forward', + 'story_taps_back', 'story_exits', 'story_swipes_forward', 'unique_viewers', + ]], +]); From 6c8b609849a92713b4fd9751adedfd9f74f5c473 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 11:28:35 -0300 Subject: [PATCH 14/77] feat: persist analytics observations --- .../Analytics/ResolveAnalyticsAccountKey.php | 37 ++++ .../Analytics/WriteAccountDailySnapshot.php | 45 ++++ .../WritePublicationDailySnapshot.php | 154 ++++++++++++++ app/Dto/Analytics/AccountDailyObservation.php | 22 ++ app/Dto/Analytics/MetricValue.php | 53 +++++ .../PublicationMetricObservation.php | 20 ++ app/Models/AnalyticsAccountDailySnapshot.php | 51 +++++ app/Models/AnalyticsPublication.php | 67 ++++++ .../AnalyticsPublicationDailySnapshot.php | 53 +++++ app/Models/AnalyticsSyncState.php | 43 ++++ .../AnalyticsAccountDailySnapshotFactory.php | 45 ++++ ...alyticsPublicationDailySnapshotFactory.php | 31 +++ .../factories/AnalyticsPublicationFactory.php | 53 +++++ .../factories/AnalyticsSyncStateFactory.php | 37 ++++ .../AnalyticsObservationWriterTest.php | 195 ++++++++++++++++++ 15 files changed, 906 insertions(+) create mode 100644 app/Actions/Analytics/ResolveAnalyticsAccountKey.php create mode 100644 app/Actions/Analytics/WriteAccountDailySnapshot.php create mode 100644 app/Actions/Analytics/WritePublicationDailySnapshot.php create mode 100644 app/Dto/Analytics/AccountDailyObservation.php create mode 100644 app/Dto/Analytics/MetricValue.php create mode 100644 app/Dto/Analytics/PublicationMetricObservation.php create mode 100644 app/Models/AnalyticsAccountDailySnapshot.php create mode 100644 app/Models/AnalyticsPublication.php create mode 100644 app/Models/AnalyticsPublicationDailySnapshot.php create mode 100644 app/Models/AnalyticsSyncState.php create mode 100644 database/factories/AnalyticsAccountDailySnapshotFactory.php create mode 100644 database/factories/AnalyticsPublicationDailySnapshotFactory.php create mode 100644 database/factories/AnalyticsPublicationFactory.php create mode 100644 database/factories/AnalyticsSyncStateFactory.php create mode 100644 tests/Feature/Analytics/AnalyticsObservationWriterTest.php diff --git a/app/Actions/Analytics/ResolveAnalyticsAccountKey.php b/app/Actions/Analytics/ResolveAnalyticsAccountKey.php new file mode 100644 index 000000000..04b8a2878 --- /dev/null +++ b/app/Actions/Analytics/ResolveAnalyticsAccountKey.php @@ -0,0 +1,37 @@ + $account->workspace_id, + 'network' => $account->platform->network(), + 'platform_user_id' => $account->platform_user_id, + ]; + + $snapshotKey = AnalyticsAccountDailySnapshot::query() + ->where($identity) + ->latest('snapshot_date') + ->value('social_account_key'); + + if (is_string($snapshotKey)) { + return $snapshotKey; + } + + $publicationKey = AnalyticsPublication::query() + ->where($identity) + ->latest('provider_published_at') + ->value('social_account_key'); + + return is_string($publicationKey) ? $publicationKey : $account->id; + } +} diff --git a/app/Actions/Analytics/WriteAccountDailySnapshot.php b/app/Actions/Analytics/WriteAccountDailySnapshot.php new file mode 100644 index 000000000..c4d1e1101 --- /dev/null +++ b/app/Actions/Analytics/WriteAccountDailySnapshot.php @@ -0,0 +1,45 @@ +updateOrCreate( + [ + 'workspace_id' => $account->workspace_id, + 'social_account_key' => $this->accountKeys->for($account), + 'snapshot_date' => $observation->date->toDateString(), + ], + [ + 'social_account_id' => $account->id, + 'network' => $account->platform->network(), + 'platform_user_id' => $account->platform_user_id, + 'platform' => $account->platform, + 'account_display_name' => $account->display_name, + 'account_username' => $account->username, + 'account_avatar_url' => $account->avatar_url, + 'followers_count' => $observation->followers, + 'metrics' => $observation->metrics ?: null, + 'provenance' => $observation->provenance, + 'precision' => $observation->precision, + 'provider_observed_at' => $observation->providerObservedAt, + 'collected_at' => $observation->collectedAt ?? now(), + ], + ); + }); + } +} diff --git a/app/Actions/Analytics/WritePublicationDailySnapshot.php b/app/Actions/Analytics/WritePublicationDailySnapshot.php new file mode 100644 index 000000000..53d548586 --- /dev/null +++ b/app/Actions/Analytics/WritePublicationDailySnapshot.php @@ -0,0 +1,154 @@ +write($publication, $observation, true); + } catch (UniqueConstraintViolationException) { + return $this->write($publication, $observation, false); + } + } + + private function write( + AnalyticsPublication $publication, + PublicationMetricObservation $observation, + bool $mayCreate, + ): AnalyticsPublicationDailySnapshot { + return DB::transaction(function () use ($publication, $observation, $mayCreate): AnalyticsPublicationDailySnapshot { + $snapshot = AnalyticsPublicationDailySnapshot::query() + ->where('analytics_publication_id', $publication->id) + ->whereDate('snapshot_date', $observation->date->toDateString()) + ->lockForUpdate() + ->first(); + + if (! $snapshot) { + if (! $mayCreate) { + $snapshot = AnalyticsPublicationDailySnapshot::query() + ->where('analytics_publication_id', $publication->id) + ->whereDate('snapshot_date', $observation->date->toDateString()) + ->lockForUpdate() + ->firstOrFail(); + } else { + $snapshot = new AnalyticsPublicationDailySnapshot([ + 'analytics_publication_id' => $publication->id, + 'snapshot_date' => $observation->date->toDateString(), + ]); + } + } + + $metrics = $this->mergeMetrics($snapshot->metrics ?? [], $observation->metrics); + + $snapshot->fill([ + 'collected_at' => $observation->collectedAt ?? now(), + 'provider_observed_at' => $observation->providerObservedAt ?? $snapshot->provider_observed_at, + 'metrics' => $metrics ?: null, + ...$this->scalarProjections($metrics), + ]); + $snapshot->save(); + + return $snapshot->refresh(); + }); + } + + /** + * @param array> $existing + * @param list $incoming + * @return array> + */ + private function mergeMetrics(array $existing, array $incoming): array + { + foreach ($incoming as $metric) { + $key = $metric->key->value; + $current = $existing[$key] ?? null; + $currentIsMeasured = data_get($current, 'availability') === MetricAvailability::Available->value + && is_numeric(data_get($current, 'value')); + $incomingIsMeasured = $metric->availability === MetricAvailability::Available + && $metric->value !== null; + + if ($incomingIsMeasured || ! $currentIsMeasured) { + $existing[$key] = $metric->toArray(); + } + } + + return $existing; + } + + /** + * @param array> $metrics + * @return array + */ + private function scalarProjections(array $metrics): array + { + [$exposureCount, $exposureKind] = $this->exposure($metrics); + + return [ + 'reactions_count' => $this->measuredInteger($metrics, MetricKey::Reactions), + 'comments_count' => $this->measuredInteger($metrics, MetricKey::Comments), + 'shares_count' => $this->measuredInteger($metrics, MetricKey::Shares), + 'saves_count' => $this->measuredInteger($metrics, MetricKey::Saves), + 'views_count' => $this->measuredInteger($metrics, MetricKey::Views), + 'impressions_count' => $this->measuredInteger($metrics, MetricKey::Impressions), + 'reach_count' => $this->measuredInteger($metrics, MetricKey::Reach), + 'engagement_count' => $this->measuredInteger($metrics, MetricKey::Engagements), + 'exposure_count' => $exposureCount, + 'exposure_kind' => $exposureKind, + 'watch_time_milliseconds' => $this->measuredInteger($metrics, MetricKey::WatchTimeMilliseconds), + 'average_watch_time_milliseconds' => $this->measuredInteger($metrics, MetricKey::AverageWatchTimeMilliseconds), + ]; + } + + /** + * @param array> $metrics + * @return array{int|null, string|null} + */ + private function exposure(array $metrics): array + { + foreach ([ + MetricKey::Reach->value => ExposureKind::Reach, + MetricKey::Impressions->value => ExposureKind::Impressions, + MetricKey::Views->value => ExposureKind::Views, + ] as $key => $kind) { + $value = $this->measuredInteger($metrics, MetricKey::from($key)); + + if ($value !== null) { + return [$value, $kind->value]; + } + } + + return [null, null]; + } + + /** @param array> $metrics */ + private function measuredInteger(array $metrics, MetricKey $key): ?int + { + $metric = $metrics[$key->value] ?? null; + $value = data_get($metric, 'value'); + + if ( + data_get($metric, 'availability') !== MetricAvailability::Available->value + || ! is_numeric($value) + ) { + return null; + } + + return (int) $value; + } +} diff --git a/app/Dto/Analytics/AccountDailyObservation.php b/app/Dto/Analytics/AccountDailyObservation.php new file mode 100644 index 000000000..9cf7fa4ec --- /dev/null +++ b/app/Dto/Analytics/AccountDailyObservation.php @@ -0,0 +1,22 @@ + $this->value, + 'unit' => $this->unit->value, + 'time_basis' => $this->timeBasis->value, + 'precision' => $this->precision->value, + 'availability' => $this->availability->value, + 'provider_metric' => $this->providerMetric, + 'period_start' => $this->periodStart?->toIso8601String(), + 'period_end' => $this->periodEnd?->toIso8601String(), + ]; + } +} diff --git a/app/Dto/Analytics/PublicationMetricObservation.php b/app/Dto/Analytics/PublicationMetricObservation.php new file mode 100644 index 000000000..7975c3bc9 --- /dev/null +++ b/app/Dto/Analytics/PublicationMetricObservation.php @@ -0,0 +1,20 @@ + $metrics + */ + public function __construct( + public CarbonImmutable $date, + public array $metrics, + public ?CarbonImmutable $providerObservedAt = null, + public ?CarbonImmutable $collectedAt = null, + ) {} +} diff --git a/app/Models/AnalyticsAccountDailySnapshot.php b/app/Models/AnalyticsAccountDailySnapshot.php new file mode 100644 index 000000000..22eeb24a3 --- /dev/null +++ b/app/Models/AnalyticsAccountDailySnapshot.php @@ -0,0 +1,51 @@ + */ + use HasFactory, HasUuids; + + protected $fillable = [ + 'workspace_id', 'social_account_id', 'social_account_key', 'network', + 'platform_user_id', 'platform', 'account_display_name', 'account_username', + 'account_avatar_url', 'snapshot_date', 'followers_count', 'metrics', + 'provenance', 'precision', 'provider_observed_at', 'collected_at', + ]; + + protected function casts(): array + { + return [ + 'platform' => Platform::class, + 'snapshot_date' => 'immutable_date', + 'followers_count' => 'integer', + 'metrics' => 'array', + 'provenance' => ObservationProvenance::class, + 'precision' => MetricPrecision::class, + 'provider_observed_at' => 'immutable_datetime', + 'collected_at' => 'immutable_datetime', + ]; + } + + public function workspace(): BelongsTo + { + return $this->belongsTo(Workspace::class); + } + + public function socialAccount(): BelongsTo + { + return $this->belongsTo(SocialAccount::class); + } +} diff --git a/app/Models/AnalyticsPublication.php b/app/Models/AnalyticsPublication.php new file mode 100644 index 000000000..824cb1b40 --- /dev/null +++ b/app/Models/AnalyticsPublication.php @@ -0,0 +1,67 @@ + */ + use HasFactory, HasUuids; + + protected $fillable = [ + 'workspace_id', 'social_account_id', 'social_account_key', 'post_platform_id', + 'network', 'platform_user_id', 'platform', 'provider_post_id', + 'provider_published_at', 'origin', 'content_type', 'availability', + 'provider_content_type', 'permalink', 'excerpt', 'preview_metadata', + 'account_display_name', 'account_username', 'account_avatar_url', + 'first_seen_at', 'last_seen_at', 'provider_synced_at', 'provider_metadata', + ]; + + protected function casts(): array + { + return [ + 'platform' => Platform::class, + 'provider_published_at' => 'immutable_datetime', + 'origin' => PublicationOrigin::class, + 'content_type' => PublicationContentType::class, + 'availability' => PublicationAvailability::class, + 'preview_metadata' => 'array', + 'first_seen_at' => 'immutable_datetime', + 'last_seen_at' => 'immutable_datetime', + 'provider_synced_at' => 'immutable_datetime', + 'provider_metadata' => 'array', + ]; + } + + public function workspace(): BelongsTo + { + return $this->belongsTo(Workspace::class); + } + + public function socialAccount(): BelongsTo + { + return $this->belongsTo(SocialAccount::class); + } + + public function postPlatform(): BelongsTo + { + return $this->belongsTo(PostPlatform::class); + } + + public function dailySnapshots(): HasMany + { + return $this->hasMany(AnalyticsPublicationDailySnapshot::class); + } +} diff --git a/app/Models/AnalyticsPublicationDailySnapshot.php b/app/Models/AnalyticsPublicationDailySnapshot.php new file mode 100644 index 000000000..ca791964b --- /dev/null +++ b/app/Models/AnalyticsPublicationDailySnapshot.php @@ -0,0 +1,53 @@ + */ + use HasFactory, HasUuids; + + protected $fillable = [ + 'analytics_publication_id', 'snapshot_date', 'collected_at', + 'provider_observed_at', 'metrics', 'reactions_count', 'comments_count', + 'shares_count', 'saves_count', 'views_count', 'impressions_count', + 'reach_count', 'engagement_count', 'exposure_count', 'exposure_kind', + 'watch_time_milliseconds', 'average_watch_time_milliseconds', + ]; + + protected function casts(): array + { + return [ + 'snapshot_date' => 'immutable_date', + 'collected_at' => 'immutable_datetime', + 'provider_observed_at' => 'immutable_datetime', + 'metrics' => 'array', + 'reactions_count' => 'integer', + 'comments_count' => 'integer', + 'shares_count' => 'integer', + 'saves_count' => 'integer', + 'views_count' => 'integer', + 'impressions_count' => 'integer', + 'reach_count' => 'integer', + 'engagement_count' => 'integer', + 'exposure_count' => 'integer', + 'exposure_kind' => ExposureKind::class, + 'watch_time_milliseconds' => 'integer', + 'average_watch_time_milliseconds' => 'integer', + ]; + } + + public function publication(): BelongsTo + { + return $this->belongsTo(AnalyticsPublication::class, 'analytics_publication_id'); + } +} diff --git a/app/Models/AnalyticsSyncState.php b/app/Models/AnalyticsSyncState.php new file mode 100644 index 000000000..007b9152d --- /dev/null +++ b/app/Models/AnalyticsSyncState.php @@ -0,0 +1,43 @@ + */ + use HasFactory, HasUuids; + + protected $fillable = [ + 'social_account_id', 'collector', 'status', 'checkpoint', 'target_since', + 'oldest_reached_at', 'high_watermark_at', 'last_success_at', + 'last_error_category', + ]; + + protected function casts(): array + { + return [ + 'collector' => SyncCollector::class, + 'status' => SyncStatus::class, + 'checkpoint' => 'array', + 'target_since' => 'immutable_datetime', + 'oldest_reached_at' => 'immutable_datetime', + 'high_watermark_at' => 'immutable_datetime', + 'last_success_at' => 'immutable_datetime', + ]; + } + + public function socialAccount(): BelongsTo + { + return $this->belongsTo(SocialAccount::class); + } +} diff --git a/database/factories/AnalyticsAccountDailySnapshotFactory.php b/database/factories/AnalyticsAccountDailySnapshotFactory.php new file mode 100644 index 000000000..567057c6a --- /dev/null +++ b/database/factories/AnalyticsAccountDailySnapshotFactory.php @@ -0,0 +1,45 @@ + + */ +class AnalyticsAccountDailySnapshotFactory extends Factory +{ + /** + * Define the model's default state. + * + * @return array + */ + public function definition(): array + { + return [ + 'workspace_id' => Workspace::factory(), + 'social_account_id' => null, + 'social_account_key' => fake()->uuid(), + 'network' => Platform::Instagram->network(), + 'platform_user_id' => fake()->uuid(), + 'platform' => Platform::Instagram, + 'account_display_name' => fake()->name(), + 'account_username' => fake()->userName(), + 'account_avatar_url' => fake()->imageUrl(), + 'snapshot_date' => today(), + 'followers_count' => fake()->numberBetween(0, 100000), + 'metrics' => null, + 'provenance' => ObservationProvenance::Actual, + 'precision' => MetricPrecision::Exact, + 'provider_observed_at' => now(), + 'collected_at' => now(), + ]; + } +} diff --git a/database/factories/AnalyticsPublicationDailySnapshotFactory.php b/database/factories/AnalyticsPublicationDailySnapshotFactory.php new file mode 100644 index 000000000..47954f771 --- /dev/null +++ b/database/factories/AnalyticsPublicationDailySnapshotFactory.php @@ -0,0 +1,31 @@ + + */ +class AnalyticsPublicationDailySnapshotFactory extends Factory +{ + /** + * Define the model's default state. + * + * @return array + */ + public function definition(): array + { + return [ + 'analytics_publication_id' => AnalyticsPublication::factory(), + 'snapshot_date' => today(), + 'collected_at' => now(), + 'provider_observed_at' => now(), + 'metrics' => null, + ]; + } +} diff --git a/database/factories/AnalyticsPublicationFactory.php b/database/factories/AnalyticsPublicationFactory.php new file mode 100644 index 000000000..06573fc0e --- /dev/null +++ b/database/factories/AnalyticsPublicationFactory.php @@ -0,0 +1,53 @@ + + */ +class AnalyticsPublicationFactory extends Factory +{ + /** + * Define the model's default state. + * + * @return array + */ + public function definition(): array + { + return [ + 'workspace_id' => Workspace::factory(), + 'social_account_id' => null, + 'social_account_key' => fake()->uuid(), + 'post_platform_id' => null, + 'network' => Platform::Instagram->network(), + 'platform_user_id' => fake()->uuid(), + 'platform' => Platform::Instagram, + 'provider_post_id' => fake()->uuid(), + 'provider_published_at' => now()->subDay(), + 'origin' => PublicationOrigin::External, + 'content_type' => PublicationContentType::Image, + 'availability' => PublicationAvailability::Available, + 'provider_content_type' => 'IMAGE', + 'permalink' => fake()->url(), + 'excerpt' => fake()->sentence(), + 'preview_metadata' => null, + 'account_display_name' => fake()->name(), + 'account_username' => fake()->userName(), + 'account_avatar_url' => fake()->imageUrl(), + 'first_seen_at' => now(), + 'last_seen_at' => now(), + 'provider_synced_at' => now(), + 'provider_metadata' => null, + ]; + } +} diff --git a/database/factories/AnalyticsSyncStateFactory.php b/database/factories/AnalyticsSyncStateFactory.php new file mode 100644 index 000000000..46efd0f3f --- /dev/null +++ b/database/factories/AnalyticsSyncStateFactory.php @@ -0,0 +1,37 @@ + + */ +class AnalyticsSyncStateFactory extends Factory +{ + /** + * Define the model's default state. + * + * @return array + */ + public function definition(): array + { + return [ + 'social_account_id' => SocialAccount::factory(), + 'collector' => SyncCollector::PublicationBackfill, + 'status' => SyncStatus::Pending, + 'checkpoint' => null, + 'target_since' => now()->subYear(), + 'oldest_reached_at' => null, + 'high_watermark_at' => null, + 'last_success_at' => null, + 'last_error_category' => null, + ]; + } +} diff --git a/tests/Feature/Analytics/AnalyticsObservationWriterTest.php b/tests/Feature/Analytics/AnalyticsObservationWriterTest.php new file mode 100644 index 000000000..89cb0b6b1 --- /dev/null +++ b/tests/Feature/Analytics/AnalyticsObservationWriterTest.php @@ -0,0 +1,195 @@ +create(); + $account = SocialAccount::factory()->instagram()->create([ + 'workspace_id' => $workspace->id, + 'platform_user_id' => 'provider-account-1', + 'username' => 'shared-name', + ]); + $writer = app(WriteAccountDailySnapshot::class); + + $writer->handle($account, accountObservation('2026-09-22', 10)); + $writer->handle($account, accountObservation('2026-09-22', 0)); + $writer->handle($account, accountObservation('2026-09-23', null)); + + expect(AnalyticsAccountDailySnapshot::count())->toBe(2) + ->and(AnalyticsAccountDailySnapshot::query() + ->whereDate('snapshot_date', '2026-09-22') + ->value('followers_count'))->toBe(0) + ->and(AnalyticsAccountDailySnapshot::query() + ->whereDate('snapshot_date', '2026-09-23') + ->value('followers_count'))->toBeNull(); + + $historicalKey = $account->id; + $account->deleteQuietly(); + + expect(AnalyticsAccountDailySnapshot::query()->where('social_account_key', $historicalKey)->count())->toBe(2) + ->and(AnalyticsAccountDailySnapshot::query()->where('social_account_key', $historicalKey)->whereNotNull('social_account_id')->exists())->toBeFalse(); + + $reconnected = SocialAccount::factory()->instagram()->create([ + 'workspace_id' => $workspace->id, + 'platform_user_id' => 'provider-account-1', + 'username' => 'renamed-account', + ]); + $writer->handle($reconnected, accountObservation('2026-09-24', 12)); + + $differentIdentity = SocialAccount::factory()->instagram()->create([ + 'workspace_id' => $workspace->id, + 'platform_user_id' => 'provider-account-2', + 'username' => 'shared-name', + ]); + $writer->handle($differentIdentity, accountObservation('2026-09-24', 7)); + + expect(AnalyticsAccountDailySnapshot::query() + ->where('social_account_id', $reconnected->id) + ->value('social_account_key'))->toBe($historicalKey) + ->and(AnalyticsAccountDailySnapshot::query() + ->where('social_account_id', $differentIdentity->id) + ->value('social_account_key'))->toBe($differentIdentity->id); +}); + +test('carried forward observations retain provider time and record a new collection time', function () { + $account = SocialAccount::factory()->x()->create(); + $providerObservedAt = CarbonImmutable::parse('2026-09-22 02:00:00', 'UTC'); + $collectedAt = CarbonImmutable::parse('2026-09-23 23:30:00', 'UTC'); + + $snapshot = app(WriteAccountDailySnapshot::class)->handle( + $account, + new AccountDailyObservation( + date: CarbonImmutable::parse('2026-09-23', 'UTC'), + followers: 42, + provenance: ObservationProvenance::CarriedForward, + precision: MetricPrecision::Exact, + providerObservedAt: $providerObservedAt, + collectedAt: $collectedAt, + ), + ); + + expect($snapshot->provider_observed_at?->toImmutable()->equalTo($providerObservedAt))->toBeTrue() + ->and($snapshot->collected_at->toImmutable()->equalTo($collectedAt))->toBeTrue(); +}); + +test('publication observations merge same-day metrics without erasing successful values', function () { + $account = SocialAccount::factory()->instagram()->create(); + $publication = AnalyticsPublication::query()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + 'network' => Platform::Instagram->network(), + 'platform_user_id' => $account->platform_user_id, + 'platform' => Platform::Instagram, + 'provider_post_id' => 'provider-post-1', + 'provider_published_at' => CarbonImmutable::parse('2026-09-20 10:00:00', 'UTC'), + 'origin' => PublicationOrigin::External, + 'content_type' => PublicationContentType::Reel, + 'availability' => PublicationAvailability::Available, + 'first_seen_at' => CarbonImmutable::parse('2026-09-23 02:00:00', 'UTC'), + 'last_seen_at' => CarbonImmutable::parse('2026-09-23 02:00:00', 'UTC'), + ]); + $writer = app(WritePublicationDailySnapshot::class); + + $writer->handle($publication, publicationObservation('2026-09-23', [ + metric(MetricKey::Reactions, 0), + metric(MetricKey::Comments, null, MetricAvailability::Unavailable), + metric(MetricKey::Shares, 4), + metric(MetricKey::Saves, 2), + metric(MetricKey::Views, 10), + metric(MetricKey::Impressions, 20), + metric(MetricKey::Reach, 8), + metric(MetricKey::Engagements, 6), + metric(MetricKey::WatchTimeMilliseconds, 65000, unit: MetricUnit::Milliseconds), + metric(MetricKey::AverageWatchTimeMilliseconds, 2000, unit: MetricUnit::Milliseconds), + ])); + $writer->handle($publication, publicationObservation('2026-09-23', [ + metric(MetricKey::Reactions, null, MetricAvailability::Delayed), + metric(MetricKey::Comments, 3), + ])); + + $snapshot = AnalyticsPublicationDailySnapshot::sole(); + + expect($snapshot->reactions_count)->toBe(0) + ->and($snapshot->comments_count)->toBe(3) + ->and($snapshot->shares_count)->toBe(4) + ->and($snapshot->saves_count)->toBe(2) + ->and($snapshot->views_count)->toBe(10) + ->and($snapshot->impressions_count)->toBe(20) + ->and($snapshot->reach_count)->toBe(8) + ->and($snapshot->engagement_count)->toBe(6) + ->and($snapshot->exposure_count)->toBe(8) + ->and($snapshot->exposure_kind?->value)->toBe('reach') + ->and($snapshot->watch_time_milliseconds)->toBe(65000) + ->and($snapshot->average_watch_time_milliseconds)->toBe(2000) + ->and(data_get($snapshot->metrics, 'reactions.value'))->toBe(0) + ->and(data_get($snapshot->metrics, 'comments.value'))->toBe(3) + ->and(data_get($snapshot->metrics, 'reactions.availability'))->toBe('available'); + + $writer->handle($publication, publicationObservation('2026-09-24', [ + metric(MetricKey::Views, 11), + ])); + + expect(AnalyticsPublicationDailySnapshot::count())->toBe(2); +}); + +function accountObservation(string $date, ?int $followers): AccountDailyObservation +{ + return new AccountDailyObservation( + date: CarbonImmutable::parse($date, 'UTC'), + followers: $followers, + provenance: ObservationProvenance::Actual, + precision: MetricPrecision::Exact, + providerObservedAt: CarbonImmutable::parse("{$date} 02:00:00", 'UTC'), + collectedAt: CarbonImmutable::parse("{$date} 02:01:00", 'UTC'), + ); +} + +function publicationObservation(string $date, array $metrics): PublicationMetricObservation +{ + return new PublicationMetricObservation( + date: CarbonImmutable::parse($date, 'UTC'), + metrics: $metrics, + providerObservedAt: CarbonImmutable::parse("{$date} 02:00:00", 'UTC'), + collectedAt: CarbonImmutable::parse("{$date} 02:01:00", 'UTC'), + ); +} + +function metric( + MetricKey $key, + int|float|null $value, + MetricAvailability $availability = MetricAvailability::Available, + MetricUnit $unit = MetricUnit::Count, +): MetricValue { + return new MetricValue( + key: $key, + value: $value, + unit: $unit, + timeBasis: MetricTimeBasis::Lifetime, + precision: MetricPrecision::Exact, + availability: $availability, + providerMetric: $key->value, + ); +} From 7e6c5c9c9c43ae6b2980e697194465abf034a362 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 11:32:17 -0300 Subject: [PATCH 15/77] feat: collect normalized follower snapshots --- app/Contracts/Analytics/FollowerCollector.php | 14 +++ .../AnalyticsCollectionException.php | 29 +++++ .../Followers/AbstractFollowerCollector.php | 81 ++++++++++++ .../Followers/BlueskyFollowerCollector.php | 24 ++++ .../Followers/FacebookFollowerCollector.php | 25 ++++ .../Followers/FollowerCollectorFactory.php | 28 +++++ .../Followers/InstagramFollowerCollector.php | 25 ++++ .../Followers/MastodonFollowerCollector.php | 25 ++++ .../Followers/PinterestFollowerCollector.php | 20 +++ .../Followers/ThreadsFollowerCollector.php | 28 +++++ .../Followers/TikTokFollowerCollector.php | 24 ++++ .../Followers/XFollowerCollector.php | 24 ++++ .../Followers/YouTubeFollowerCollector.php | 42 +++++++ .../Collectors/FollowerCollectorsTest.php | 116 ++++++++++++++++++ 14 files changed, 505 insertions(+) create mode 100644 app/Contracts/Analytics/FollowerCollector.php create mode 100644 app/Exceptions/Analytics/AnalyticsCollectionException.php create mode 100644 app/Services/Analytics/Collectors/Followers/AbstractFollowerCollector.php create mode 100644 app/Services/Analytics/Collectors/Followers/BlueskyFollowerCollector.php create mode 100644 app/Services/Analytics/Collectors/Followers/FacebookFollowerCollector.php create mode 100644 app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php create mode 100644 app/Services/Analytics/Collectors/Followers/InstagramFollowerCollector.php create mode 100644 app/Services/Analytics/Collectors/Followers/MastodonFollowerCollector.php create mode 100644 app/Services/Analytics/Collectors/Followers/PinterestFollowerCollector.php create mode 100644 app/Services/Analytics/Collectors/Followers/ThreadsFollowerCollector.php create mode 100644 app/Services/Analytics/Collectors/Followers/TikTokFollowerCollector.php create mode 100644 app/Services/Analytics/Collectors/Followers/XFollowerCollector.php create mode 100644 app/Services/Analytics/Collectors/Followers/YouTubeFollowerCollector.php create mode 100644 tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php diff --git a/app/Contracts/Analytics/FollowerCollector.php b/app/Contracts/Analytics/FollowerCollector.php new file mode 100644 index 000000000..547fcd1e6 --- /dev/null +++ b/app/Contracts/Analytics/FollowerCollector.php @@ -0,0 +1,14 @@ +withToken($account->access_token) + ->timeout(120) + ->get($url, $query); + + if ($response->successful()) { + return $response; + } + + $code = (int) data_get($response->json(), 'error.code', 0); + $isMetaRateLimit = $meta && in_array($code, [4, 17, 32, 80001, 80002], true); + $category = match (true) { + $response->status() === 429, $isMetaRateLimit => 'rate_limited', + $response->status() === 401 => 'authentication', + $response->status() === 403 => 'permission', + $response->serverError(), $meta && in_array($code, [1, 2], true) => 'transient', + default => 'malformed', + }; + + throw new AnalyticsCollectionException( + $category, + "follower collection failed with HTTP {$response->status()}", + $this->retryAt($response), + ); + } + + protected function observation( + CarbonImmutable $date, + mixed $value, + MetricPrecision $precision = MetricPrecision::Exact, + ): AccountDailyObservation { + if (! is_numeric($value)) { + throw AnalyticsCollectionException::malformed('missing follower metric'); + } + + return new AccountDailyObservation( + date: $date, + followers: (int) $value, + provenance: ObservationProvenance::Actual, + precision: $precision, + providerObservedAt: CarbonImmutable::now('UTC'), + collectedAt: CarbonImmutable::now('UTC'), + ); + } + + private function retryAt(Response $response): ?CarbonImmutable + { + $retryAfter = $response->header('Retry-After'); + + if (! is_string($retryAfter) || $retryAfter === '') { + return null; + } + + return ctype_digit($retryAfter) + ? CarbonImmutable::now('UTC')->addSeconds((int) $retryAfter) + : CarbonImmutable::parse($retryAfter)->utc(); + } +} diff --git a/app/Services/Analytics/Collectors/Followers/BlueskyFollowerCollector.php b/app/Services/Analytics/Collectors/Followers/BlueskyFollowerCollector.php new file mode 100644 index 000000000..c9f0de971 --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/BlueskyFollowerCollector.php @@ -0,0 +1,24 @@ +get( + $account, + config('trypost.platforms.bluesky.public_appview').'/xrpc/app.bsky.actor.getProfile', + ['actor' => $account->platform_user_id], + ); + + return $this->observation($date, data_get($response->json(), 'followersCount')); + } +} diff --git a/app/Services/Analytics/Collectors/Followers/FacebookFollowerCollector.php b/app/Services/Analytics/Collectors/Followers/FacebookFollowerCollector.php new file mode 100644 index 000000000..47d498e57 --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/FacebookFollowerCollector.php @@ -0,0 +1,25 @@ +get( + $account, + config('trypost.platforms.facebook.graph_api')."/{$account->platform_user_id}", + ['fields' => 'followers_count'], + meta: true, + ); + + return $this->observation($date, data_get($response->json(), 'followers_count')); + } +} diff --git a/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php b/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php new file mode 100644 index 000000000..1f07cb12c --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php @@ -0,0 +1,28 @@ + app(InstagramFollowerCollector::class), + Platform::Facebook => app(FacebookFollowerCollector::class), + Platform::Threads => app(ThreadsFollowerCollector::class), + Platform::X => app(XFollowerCollector::class), + Platform::Pinterest => app(PinterestFollowerCollector::class), + Platform::YouTube => app(YouTubeFollowerCollector::class), + Platform::TikTok => app(TikTokFollowerCollector::class), + Platform::Bluesky => app(BlueskyFollowerCollector::class), + Platform::Mastodon => app(MastodonFollowerCollector::class), + default => throw AnalyticsCollectionException::unsupported("{$platform->value} follower analytics is excluded"), + }; + } +} diff --git a/app/Services/Analytics/Collectors/Followers/InstagramFollowerCollector.php b/app/Services/Analytics/Collectors/Followers/InstagramFollowerCollector.php new file mode 100644 index 000000000..b330c7875 --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/InstagramFollowerCollector.php @@ -0,0 +1,25 @@ +get( + $account, + "{$account->platform->instagramGraphBaseUrl()}/{$account->platform_user_id}", + ['fields' => 'followers_count'], + meta: true, + ); + + return $this->observation($date, data_get($response->json(), 'followers_count')); + } +} diff --git a/app/Services/Analytics/Collectors/Followers/MastodonFollowerCollector.php b/app/Services/Analytics/Collectors/Followers/MastodonFollowerCollector.php new file mode 100644 index 000000000..b8f47497c --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/MastodonFollowerCollector.php @@ -0,0 +1,25 @@ +meta, + 'instance', + config('trypost.platforms.mastodon.default_instance'), + ), '/'); + $response = $this->get($account, "{$instance}/api/v1/accounts/{$account->platform_user_id}"); + + return $this->observation($date, data_get($response->json(), 'followers_count')); + } +} diff --git a/app/Services/Analytics/Collectors/Followers/PinterestFollowerCollector.php b/app/Services/Analytics/Collectors/Followers/PinterestFollowerCollector.php new file mode 100644 index 000000000..18448ab72 --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/PinterestFollowerCollector.php @@ -0,0 +1,20 @@ +get($account, config('trypost.platforms.pinterest.api').'/user_account'); + + return $this->observation($date, data_get($response->json(), 'follower_count')); + } +} diff --git a/app/Services/Analytics/Collectors/Followers/ThreadsFollowerCollector.php b/app/Services/Analytics/Collectors/Followers/ThreadsFollowerCollector.php new file mode 100644 index 000000000..4e65a6d93 --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/ThreadsFollowerCollector.php @@ -0,0 +1,28 @@ +get( + $account, + config('trypost.platforms.threads.graph_api')."/{$account->platform_user_id}/threads_insights", + ['metric' => 'followers_count'], + meta: true, + ); + + $metric = collect(data_get($response->json(), 'data', [])) + ->firstWhere('name', 'followers_count'); + + return $this->observation($date, data_get($metric, 'total_value.value')); + } +} diff --git a/app/Services/Analytics/Collectors/Followers/TikTokFollowerCollector.php b/app/Services/Analytics/Collectors/Followers/TikTokFollowerCollector.php new file mode 100644 index 000000000..d17da3a6a --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/TikTokFollowerCollector.php @@ -0,0 +1,24 @@ +get( + $account, + config('trypost.platforms.tiktok.api').'/user/info/', + ['fields' => 'follower_count'], + ); + + return $this->observation($date, data_get($response->json(), 'data.user.follower_count')); + } +} diff --git a/app/Services/Analytics/Collectors/Followers/XFollowerCollector.php b/app/Services/Analytics/Collectors/Followers/XFollowerCollector.php new file mode 100644 index 000000000..7004bb897 --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/XFollowerCollector.php @@ -0,0 +1,24 @@ +get( + $account, + config('trypost.platforms.x.api')."/users/{$account->platform_user_id}", + ['user.fields' => 'public_metrics'], + ); + + return $this->observation($date, data_get($response->json(), 'data.public_metrics.followers_count')); + } +} diff --git a/app/Services/Analytics/Collectors/Followers/YouTubeFollowerCollector.php b/app/Services/Analytics/Collectors/Followers/YouTubeFollowerCollector.php new file mode 100644 index 000000000..62b8d6f91 --- /dev/null +++ b/app/Services/Analytics/Collectors/Followers/YouTubeFollowerCollector.php @@ -0,0 +1,42 @@ +get( + $account, + config('trypost.platforms.youtube.data_api').'/channels', + ['part' => 'statistics', 'id' => $account->platform_user_id], + ); + $statistics = data_get($response->json(), 'items.0.statistics'); + + if (data_get($statistics, 'hiddenSubscriberCount') === true) { + return new AccountDailyObservation( + date: $date, + followers: null, + provenance: ObservationProvenance::Actual, + precision: MetricPrecision::Approximate, + providerObservedAt: CarbonImmutable::now('UTC'), + collectedAt: CarbonImmutable::now('UTC'), + ); + } + + return $this->observation( + $date, + data_get($statistics, 'subscriberCount'), + MetricPrecision::Approximate, + ); + } +} diff --git a/tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php b/tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php new file mode 100644 index 000000000..d6f50ac16 --- /dev/null +++ b/tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php @@ -0,0 +1,116 @@ +url(); + + return match (true) { + str_contains($url, 'graph.instagram.com'), + str_contains($url, 'graph.facebook.com') => Http::response(['followers_count' => 123]), + str_contains($url, 'graph.threads.net') => Http::response(['data' => [[ + 'name' => 'followers_count', 'total_value' => ['value' => 123], + ]]]), + str_contains($url, 'api.x.com/2/users/') => Http::response(['data' => ['public_metrics' => ['followers_count' => 123]]]), + str_contains($url, 'api.pinterest.com/v5/user_account') => Http::response(['follower_count' => 123]), + str_contains($url, 'googleapis.com/youtube/v3/channels') => Http::response(['items' => [[ + 'statistics' => ['subscriberCount' => '123', 'hiddenSubscriberCount' => false], + ]]]), + str_contains($url, 'open.tiktokapis.com/v2/user/info') => Http::response(['data' => ['user' => ['follower_count' => 123]]]), + str_contains($url, 'public.api.bsky.app/xrpc/app.bsky.actor.getProfile') => Http::response(['followersCount' => 123]), + str_contains($url, 'mastodon.example/api/v1/accounts/') => Http::response(['followers_count' => 123]), + default => Http::response([], 404), + }; + }); + + $factory = app(FollowerCollectorFactory::class); + $date = CarbonImmutable::parse('2026-09-23', 'UTC'); + $platforms = [ + Platform::Instagram, + Platform::InstagramFacebook, + Platform::Facebook, + Platform::Threads, + Platform::X, + Platform::Pinterest, + Platform::YouTube, + Platform::TikTok, + Platform::Bluesky, + Platform::Mastodon, + ]; + + foreach ($platforms as $platform) { + $account = SocialAccount::factory()->create([ + 'platform' => $platform, + 'platform_user_id' => "provider-{$platform->value}", + 'meta' => $platform === Platform::Mastodon ? ['instance' => 'https://mastodon.example'] : [], + ]); + + $observation = $factory->for($platform)->collect($account, $date); + + expect($observation->followers)->toBe(123) + ->and($observation->date->equalTo($date))->toBeTrue() + ->and($observation->precision)->toBe( + $platform === Platform::YouTube + ? MetricPrecision::Approximate + : MetricPrecision::Exact, + ); + } + + Http::assertSentCount(10); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/2/users/provider-x') + && $request['user.fields'] === 'public_metrics'); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/youtube/v3/channels') + && $request['part'] === 'statistics' + && $request['id'] === 'provider-youtube'); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/v2/user/info/') + && $request['fields'] === 'follower_count'); +}); + +test('missing follower fields are unavailable rather than measured zero', function () { + Http::fake(['*' => Http::response(['data' => ['public_metrics' => []]])]); + $account = SocialAccount::factory()->x()->create(); + + expect(fn () => app(FollowerCollectorFactory::class) + ->for(Platform::X) + ->collect($account, CarbonImmutable::parse('2026-09-23', 'UTC'))) + ->toThrow(AnalyticsCollectionException::class, 'missing follower metric'); +}); + +test('a hidden youtube subscriber total is persisted as unavailable null', function () { + Http::fake(['*' => Http::response(['items' => [[ + 'statistics' => ['hiddenSubscriberCount' => true], + ]]])]); + $account = SocialAccount::factory()->youtube()->create(); + + $observation = app(FollowerCollectorFactory::class) + ->for(Platform::YouTube) + ->collect($account, CarbonImmutable::parse('2026-09-23', 'UTC')); + + expect($observation->followers)->toBeNull() + ->and($observation->precision)->toBe(MetricPrecision::Approximate); +}); + +test('excluded platforms have no collector and make no provider request', function (Platform $platform) { + Http::preventStrayRequests(); + + expect(fn () => app(FollowerCollectorFactory::class)->for($platform)) + ->toThrow(AnalyticsCollectionException::class); + + Http::assertNothingSent(); +})->with([ + Platform::LinkedIn, + Platform::LinkedInPage, + Platform::Telegram, + Platform::Discord, + Platform::GoogleBusiness, +]); From f432293e3cb428004629520168e9c84996f82b4e Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 11:41:14 -0300 Subject: [PATCH 16/77] feat: schedule resilient follower analytics --- .ai/rules/app.md | 9 ++ .ai/rules/index.md | 1 + .../Analytics/WriteAccountDailySnapshot.php | 72 ++++++--- .../DispatchAccountDailyAnalytics.php | 35 +++++ .../Analytics/CollectAccountDailySnapshot.php | 126 +++++++++++++++ .../FinalizeAccountDailySnapshots.php | 83 ++++++++++ app/Models/SocialAccount.php | 5 + app/Observers/SocialAccountObserver.php | 31 ++++ .../Followers/FollowerCollectorFactory.php | 16 ++ routes/console.php | 12 ++ .../Analytics/AccountDailyJobsTest.php | 146 ++++++++++++++++++ .../Analytics/AnalyticsScheduleTest.php | 26 ++++ .../Collectors/FollowerCollectorsTest.php | 5 + .../Observers/SocialAccountObserverTest.php | 22 +++ 14 files changed, 566 insertions(+), 23 deletions(-) create mode 100644 .ai/rules/app.md create mode 100644 app/Console/Commands/Analytics/DispatchAccountDailyAnalytics.php create mode 100644 app/Jobs/Analytics/CollectAccountDailySnapshot.php create mode 100644 app/Jobs/Analytics/FinalizeAccountDailySnapshots.php create mode 100644 tests/Feature/Analytics/AccountDailyJobsTest.php create mode 100644 tests/Feature/Analytics/AnalyticsScheduleTest.php diff --git a/.ai/rules/app.md b/.ai/rules/app.md new file mode 100644 index 000000000..9a36e0d84 --- /dev/null +++ b/.ai/rules/app.md @@ -0,0 +1,9 @@ +--- +paths: + - 'app/**' +--- + +# App + +## Reuse model scopes for canonical state filters +When a model already exposes a scope for a recurring state filter, jobs, commands, services, observers, and controllers must use that scope instead of repeating raw where clauses. Add a descriptive model scope when a canonical state condition will be reused (for example SocialAccount::connected()->active()). diff --git a/.ai/rules/index.md b/.ai/rules/index.md index 09b33beb2..4f0810dfb 100644 --- a/.ai/rules/index.md +++ b/.ai/rules/index.md @@ -4,6 +4,7 @@ Before planning or editing, find the row whose globs match the file's path and r | Applies to | Rule file | | --- | --- | +| app/** | .ai/rules/app.md | | app/Http/Controllers/Auth/GoogleBusinessController.php | .ai/rules/auth.md | | app/Enums/GoogleBusiness/**, app/Jobs/PublishToSocialPlatform.php, app/Jobs/ReconcileGoogleBusinessPost.php, app/Console/Commands/ReconcileGoogleBusinessPosts.php, app/Services/Social/GoogleBusinessPublisher.php, app/Support/PostPlatformMetaRules.php, app/Services/Social/GoogleBusinessAnalytics.php | .ai/rules/google-business.md | | app/Jobs/ReconcileGoogleBusinessPost.php, app/Console/Commands/RecoverStuckPosts.php | .ai/rules/jobs.md | diff --git a/app/Actions/Analytics/WriteAccountDailySnapshot.php b/app/Actions/Analytics/WriteAccountDailySnapshot.php index c4d1e1101..6d361919e 100644 --- a/app/Actions/Analytics/WriteAccountDailySnapshot.php +++ b/app/Actions/Analytics/WriteAccountDailySnapshot.php @@ -5,8 +5,10 @@ namespace App\Actions\Analytics; use App\Dto\Analytics\AccountDailyObservation; +use App\Enums\Analytics\ObservationProvenance; use App\Models\AnalyticsAccountDailySnapshot; use App\Models\SocialAccount; +use Illuminate\Database\UniqueConstraintViolationException; use Illuminate\Support\Facades\DB; class WriteAccountDailySnapshot @@ -17,29 +19,53 @@ public function handle( SocialAccount $account, AccountDailyObservation $observation, ): AnalyticsAccountDailySnapshot { - return DB::transaction(function () use ($account, $observation): AnalyticsAccountDailySnapshot { - return AnalyticsAccountDailySnapshot::query()->updateOrCreate( - [ - 'workspace_id' => $account->workspace_id, - 'social_account_key' => $this->accountKeys->for($account), - 'snapshot_date' => $observation->date->toDateString(), - ], - [ - 'social_account_id' => $account->id, - 'network' => $account->platform->network(), - 'platform_user_id' => $account->platform_user_id, - 'platform' => $account->platform, - 'account_display_name' => $account->display_name, - 'account_username' => $account->username, - 'account_avatar_url' => $account->avatar_url, - 'followers_count' => $observation->followers, - 'metrics' => $observation->metrics ?: null, - 'provenance' => $observation->provenance, - 'precision' => $observation->precision, - 'provider_observed_at' => $observation->providerObservedAt, - 'collected_at' => $observation->collectedAt ?? now(), - ], - ); + try { + return $this->write($account, $observation, true); + } catch (UniqueConstraintViolationException) { + return $this->write($account, $observation, false); + } + } + + private function write( + SocialAccount $account, + AccountDailyObservation $observation, + bool $mayCreate, + ): AnalyticsAccountDailySnapshot { + return DB::transaction(function () use ($account, $observation, $mayCreate): AnalyticsAccountDailySnapshot { + $identity = [ + 'workspace_id' => $account->workspace_id, + 'social_account_key' => $this->accountKeys->for($account), + 'snapshot_date' => $observation->date->toDateString(), + ]; + $snapshot = AnalyticsAccountDailySnapshot::query()->where($identity)->lockForUpdate()->first(); + + if ($snapshot?->provenance === ObservationProvenance::Actual + && $observation->provenance === ObservationProvenance::CarriedForward) { + return $snapshot; + } + + if (! $snapshot && ! $mayCreate) { + $snapshot = AnalyticsAccountDailySnapshot::query()->where($identity)->lockForUpdate()->firstOrFail(); + } + + $snapshot ??= new AnalyticsAccountDailySnapshot($identity); + $snapshot->fill([ + 'social_account_id' => $account->id, + 'network' => $account->platform->network(), + 'platform_user_id' => $account->platform_user_id, + 'platform' => $account->platform, + 'account_display_name' => $account->display_name, + 'account_username' => $account->username, + 'account_avatar_url' => $account->avatar_url, + 'followers_count' => $observation->followers, + 'metrics' => $observation->metrics ?: null, + 'provenance' => $observation->provenance, + 'precision' => $observation->precision, + 'provider_observed_at' => $observation->providerObservedAt, + 'collected_at' => $observation->collectedAt ?? now(), + ])->save(); + + return $snapshot->refresh(); }); } } diff --git a/app/Console/Commands/Analytics/DispatchAccountDailyAnalytics.php b/app/Console/Commands/Analytics/DispatchAccountDailyAnalytics.php new file mode 100644 index 000000000..5d9f3e765 --- /dev/null +++ b/app/Console/Commands/Analytics/DispatchAccountDailyAnalytics.php @@ -0,0 +1,35 @@ +toDateString(); + + SocialAccount::query() + ->connected() + ->active() + ->lazyById(200) + ->each(function (SocialAccount $account) use ($collectors, $date): void { + if ($collectors->supports($account->platform)) { + CollectAccountDailySnapshot::dispatch($account->id, $date); + } + }); + + return self::SUCCESS; + } +} diff --git a/app/Jobs/Analytics/CollectAccountDailySnapshot.php b/app/Jobs/Analytics/CollectAccountDailySnapshot.php new file mode 100644 index 000000000..a769f8c49 --- /dev/null +++ b/app/Jobs/Analytics/CollectAccountDailySnapshot.php @@ -0,0 +1,126 @@ +onQueue('analytics'); + } + + /** @return array */ + public function middleware(): array + { + return [ + (new WithoutOverlapping("analytics-followers:{$this->socialAccountId}:{$this->observationDate}")) + ->releaseAfter(300) + ->expireAfter($this->timeout + 30), + ]; + } + + public function retryUntil(): CarbonImmutable + { + return CarbonImmutable::parse($this->observationDate, 'UTC')->endOfDay(); + } + + public function handle( + FollowerCollectorFactory $collectors, + ResolveAnalyticsAccountKey $accountKeys, + WriteAccountDailySnapshot $writer, + ): void { + $account = SocialAccount::query() + ->connected() + ->active() + ->find($this->socialAccountId); + + if (! $account + || ! $collectors->supports($account->platform)) { + return; + } + + $alreadyCollected = AnalyticsAccountDailySnapshot::query() + ->where('workspace_id', $account->workspace_id) + ->where('social_account_key', $accountKeys->for($account)) + ->whereDate('snapshot_date', $this->observationDate) + ->where('provenance', ObservationProvenance::Actual) + ->exists(); + + if ($alreadyCollected) { + return; + } + + try { + $observation = $collectors->for($account->platform)->collect( + $account, + CarbonImmutable::parse($this->observationDate, 'UTC'), + ); + $writer->handle($account, $observation); + } catch (AnalyticsCollectionException $exception) { + if (in_array($exception->category, ['authentication', 'permission'], true)) { + $account->markAsTokenExpired('Analytics permission or authentication failed.', notify: false); + + return; + } + + if (! in_array($exception->category, ['transient', 'rate_limited'], true) || $this->attempts() >= 6) { + return; + } + + $nextAttempt = $this->nextAttemptAt($exception->retryAt); + + if ($nextAttempt) { + $this->release($nextAttempt); + } + } + } + + private function nextAttemptAt(?CarbonImmutable $providerRetryAt): ?CarbonImmutable + { + $now = CarbonImmutable::now('UTC'); + $endOfDay = CarbonImmutable::parse($this->observationDate, 'UTC')->endOfDay(); + $nextWindow = null; + + foreach ([6, 10, 14, 18, 22] as $hour) { + $window = CarbonImmutable::parse($this->observationDate, 'UTC')->setTime($hour, 0); + + if ($window->greaterThanOrEqualTo($now)) { + $nextWindow = $window; + + break; + } + } + + if (! $nextWindow) { + return null; + } + + $nextAttempt = $providerRetryAt && $providerRetryAt->greaterThan($nextWindow) + ? $providerRetryAt + : $nextWindow; + + return $nextAttempt->lessThanOrEqualTo($endOfDay) ? $nextAttempt : null; + } +} diff --git a/app/Jobs/Analytics/FinalizeAccountDailySnapshots.php b/app/Jobs/Analytics/FinalizeAccountDailySnapshots.php new file mode 100644 index 000000000..b629d2d4c --- /dev/null +++ b/app/Jobs/Analytics/FinalizeAccountDailySnapshots.php @@ -0,0 +1,83 @@ +onQueue('analytics'); + } + + public function handle( + WriteAccountDailySnapshot $writer, + ?FollowerCollectorFactory $collectors = null, + ): void { + $collectors ??= app(FollowerCollectorFactory::class); + $date = CarbonImmutable::parse( + $this->observationDate ?? CarbonImmutable::now('UTC')->toDateString(), + 'UTC', + ); + + SocialAccount::query() + ->connected() + ->active() + ->lazyById(200) + ->each(function (SocialAccount $account) use ($collectors, $date, $writer): void { + if (! $collectors->supports($account->platform)) { + return; + } + + $hasActual = AnalyticsAccountDailySnapshot::query() + ->where('workspace_id', $account->workspace_id) + ->whereDate('snapshot_date', $date->toDateString()) + ->where('social_account_key', app(ResolveAnalyticsAccountKey::class)->for($account)) + ->where('provenance', ObservationProvenance::Actual) + ->exists(); + + if ($hasActual) { + return; + } + + $previous = AnalyticsAccountDailySnapshot::query() + ->where('workspace_id', $account->workspace_id) + ->where('network', $account->platform->network()) + ->where('platform_user_id', $account->platform_user_id) + ->whereDate('snapshot_date', '<', $date->toDateString()) + ->whereNotNull('followers_count') + ->latest('snapshot_date') + ->first(); + + if (! $previous) { + return; + } + + $writer->handle($account, new AccountDailyObservation( + date: $date, + followers: $previous->followers_count, + provenance: ObservationProvenance::CarriedForward, + precision: $previous->precision, + providerObservedAt: $previous->provider_observed_at, + collectedAt: CarbonImmutable::now('UTC'), + metrics: $previous->metrics ?? [], + )); + }); + } +} diff --git a/app/Models/SocialAccount.php b/app/Models/SocialAccount.php index 804836ff2..de7e415c7 100644 --- a/app/Models/SocialAccount.php +++ b/app/Models/SocialAccount.php @@ -426,4 +426,9 @@ public function scopeActive(Builder $query): Builder { return $query->where('is_active', true)->orderBy('platform'); } + + public function scopeConnected(Builder $query): Builder + { + return $query->where('status', Status::Connected); + } } diff --git a/app/Observers/SocialAccountObserver.php b/app/Observers/SocialAccountObserver.php index e7bf28cbf..ddd81ede1 100644 --- a/app/Observers/SocialAccountObserver.php +++ b/app/Observers/SocialAccountObserver.php @@ -5,17 +5,22 @@ namespace App\Observers; use App\Enums\SocialAccount\Status; +use App\Jobs\Analytics\CollectAccountDailySnapshot; use App\Jobs\PostHog\IdentifyConnectedPlatforms; use App\Jobs\PostHog\SyncAccountUsage; use App\Models\SocialAccount; +use App\Services\Analytics\Collectors\Followers\FollowerCollectorFactory; use App\Services\PostHogService; use App\Services\Repurpose\RepurposeAccountSync; +use Carbon\CarbonImmutable; +use Throwable; class SocialAccountObserver { public function created(SocialAccount $socialAccount): void { $this->syncUsageAndIdentify($socialAccount); + $this->dispatchInitialAnalytics($socialAccount); } public function deleted(SocialAccount $socialAccount): void @@ -41,6 +46,10 @@ public function updated(SocialAccount $socialAccount): void if ($wasConnected !== $isConnected) { $this->identifyConnectedPlatforms($socialAccount); + + if ($isConnected) { + $this->dispatchInitialAnalytics($socialAccount); + } } } @@ -68,4 +77,26 @@ private function syncUsage(SocialAccount $socialAccount): void ); } } + + private function dispatchInitialAnalytics(SocialAccount $socialAccount): void + { + try { + $currentAccount = SocialAccount::query() + ->connected() + ->active() + ->find($socialAccount->id); + + if (! $currentAccount + || ! app(FollowerCollectorFactory::class)->supports($currentAccount->platform)) { + return; + } + + CollectAccountDailySnapshot::dispatch( + $currentAccount->id, + CarbonImmutable::now('UTC')->toDateString(), + )->afterCommit(); + } catch (Throwable $exception) { + report($exception); + } + } } diff --git a/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php b/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php index 1f07cb12c..e26489e74 100644 --- a/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php +++ b/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php @@ -10,6 +10,22 @@ class FollowerCollectorFactory { + public function supports(Platform $platform): bool + { + return in_array($platform, [ + Platform::Instagram, + Platform::InstagramFacebook, + Platform::Facebook, + Platform::Threads, + Platform::X, + Platform::Pinterest, + Platform::YouTube, + Platform::TikTok, + Platform::Bluesky, + Platform::Mastodon, + ], true); + } + public function for(Platform $platform): FollowerCollector { return match ($platform) { diff --git a/routes/console.php b/routes/console.php index c0c70120e..c5ebacb97 100644 --- a/routes/console.php +++ b/routes/console.php @@ -2,6 +2,7 @@ declare(strict_types=1); +use App\Console\Commands\Analytics\DispatchAccountDailyAnalytics; use App\Console\Commands\CheckSocialConnections; use App\Console\Commands\CheckUpcomingPostConnections; use App\Console\Commands\ProcessScheduledPosts; @@ -10,6 +11,7 @@ use App\Console\Commands\RecoverStuckPosts; use App\Console\Commands\RefreshExpiringTokens; use App\Console\Commands\Repurpose\PollRepurposes; +use App\Jobs\Analytics\FinalizeAccountDailySnapshots; use Illuminate\Support\Facades\Schedule; Schedule::command(ProcessScheduledPosts::class)->everyMinute()->withoutOverlapping()->onOneServer(); @@ -20,3 +22,13 @@ Schedule::command(ReconcileGoogleBusinessPosts::class)->everyFiveMinutes()->withoutOverlapping()->onOneServer(); Schedule::command(PruneWebhookLogs::class)->daily()->withoutOverlapping()->onOneServer(); Schedule::command(PollRepurposes::class)->everyFiveMinutes()->withoutOverlapping()->onOneServer(); +Schedule::command(DispatchAccountDailyAnalytics::class) + ->dailyAt('02:00') + ->timezone('UTC') + ->withoutOverlapping() + ->onOneServer(); +Schedule::job(new FinalizeAccountDailySnapshots) + ->dailyAt('23:30') + ->timezone('UTC') + ->withoutOverlapping() + ->onOneServer(); diff --git a/tests/Feature/Analytics/AccountDailyJobsTest.php b/tests/Feature/Analytics/AccountDailyJobsTest.php new file mode 100644 index 000000000..e439757a1 --- /dev/null +++ b/tests/Feature/Analytics/AccountDailyJobsTest.php @@ -0,0 +1,146 @@ +create(); + $first = SocialAccount::factory()->instagram()->create(['workspace_id' => $workspace->id]); + $second = SocialAccount::factory()->instagram()->create(['workspace_id' => $workspace->id]); + $linkedin = SocialAccount::factory()->linkedin()->create(['workspace_id' => $workspace->id]); + $inactive = SocialAccount::factory()->x()->create(['workspace_id' => $workspace->id, 'is_active' => false]); + $disconnected = SocialAccount::factory()->x()->disconnected()->create(['workspace_id' => $workspace->id]); + + Artisan::call('analytics:dispatch-account-daily'); + + foreach ([$first, $second] as $account) { + Bus::assertDispatched(CollectAccountDailySnapshot::class, fn ($job): bool => $job->socialAccountId === $account->id + && $job->observationDate === '2026-09-23' + && $job->queue === 'analytics'); + } + + foreach ([$linkedin, $inactive, $disconnected] as $account) { + Bus::assertNotDispatched(CollectAccountDailySnapshot::class, fn ($job): bool => $job->socialAccountId === $account->id); + } +}); + +test('collection job writes once and skips an existing actual observation', function () { + $account = SocialAccount::factory()->x()->create(); + $collector = Mockery::mock(FollowerCollector::class); + $collector->shouldReceive('collect')->once()->andReturn(followerObservation(25)); + $factory = Mockery::mock(FollowerCollectorFactory::class); + $factory->shouldReceive('supports')->twice()->with(Platform::X)->andReturnTrue(); + $factory->shouldReceive('for')->once()->with(Platform::X)->andReturn($collector); + $job = new CollectAccountDailySnapshot($account->id, '2026-09-23'); + + $job->handle($factory, app(ResolveAnalyticsAccountKey::class), app(WriteAccountDailySnapshot::class)); + $job->handle($factory, app(ResolveAnalyticsAccountKey::class), app(WriteAccountDailySnapshot::class)); + + expect(AnalyticsAccountDailySnapshot::count())->toBe(1) + ->and(AnalyticsAccountDailySnapshot::first()->followers_count)->toBe(25); +}); + +test('collection job retries at spaced windows and honors a later provider retry time', function () { + $account = SocialAccount::factory()->x()->create(); + $collector = Mockery::mock(FollowerCollector::class); + $collector->shouldReceive('collect')->twice() + ->andThrowExceptions([ + new AnalyticsCollectionException('transient', 'temporarily unavailable'), + new AnalyticsCollectionException( + 'rate_limited', + 'rate limited', + CarbonImmutable::parse('2026-09-23 07:30:00', 'UTC'), + ), + ]); + $factory = Mockery::mock(FollowerCollectorFactory::class); + $factory->shouldReceive('supports')->twice()->with(Platform::X)->andReturnTrue(); + $factory->shouldReceive('for')->twice()->with(Platform::X)->andReturn($collector); + + $windowJob = (new CollectAccountDailySnapshot($account->id, '2026-09-23')) + ->withFakeQueueInteractions(); + $windowJob->handle($factory, app(ResolveAnalyticsAccountKey::class), app(WriteAccountDailySnapshot::class)); + $windowJob->assertReleased(4 * 60 * 60); + + $providerJob = (new CollectAccountDailySnapshot($account->id, '2026-09-23')) + ->withFakeQueueInteractions(); + $providerJob->handle($factory, app(ResolveAnalyticsAccountKey::class), app(WriteAccountDailySnapshot::class)); + $providerJob->assertReleased((5 * 60 * 60) + (30 * 60)); +}); + +test('collection job stops when provider retry time falls outside the observation day', function () { + $account = SocialAccount::factory()->x()->create(); + $collector = Mockery::mock(FollowerCollector::class); + $collector->shouldReceive('collect')->once()->andThrow(new AnalyticsCollectionException( + 'rate_limited', + 'rate limited', + CarbonImmutable::parse('2026-09-24 01:00:00', 'UTC'), + )); + $factory = Mockery::mock(FollowerCollectorFactory::class); + $factory->shouldReceive('supports')->once()->with(Platform::X)->andReturnTrue(); + $factory->shouldReceive('for')->once()->with(Platform::X)->andReturn($collector); + $job = (new CollectAccountDailySnapshot($account->id, '2026-09-23')) + ->withFakeQueueInteractions(); + + $job->handle($factory, app(ResolveAnalyticsAccountKey::class), app(WriteAccountDailySnapshot::class)); + + $job->assertNotReleased(); +}); + +test('finalizer carries the latest measured total and does not invent missing history', function () { + $workspace = Workspace::factory()->create(); + $withHistory = SocialAccount::factory()->x()->create(['workspace_id' => $workspace->id]); + $withoutHistory = SocialAccount::factory()->instagram()->create(['workspace_id' => $workspace->id]); + app(WriteAccountDailySnapshot::class)->handle($withHistory, new AccountDailyObservation( + date: CarbonImmutable::parse('2026-09-22', 'UTC'), + followers: 50, + provenance: ObservationProvenance::Actual, + precision: MetricPrecision::Exact, + providerObservedAt: CarbonImmutable::parse('2026-09-22 02:00:00', 'UTC'), + )); + + (new FinalizeAccountDailySnapshots('2026-09-23'))->handle(app(WriteAccountDailySnapshot::class)); + + $carried = AnalyticsAccountDailySnapshot::query()->whereDate('snapshot_date', '2026-09-23')->sole(); + expect($carried->social_account_id)->toBe($withHistory->id) + ->and($carried->followers_count)->toBe(50) + ->and($carried->provenance)->toBe(ObservationProvenance::CarriedForward) + ->and($carried->provider_observed_at?->toDateTimeString())->toBe('2026-09-22 02:00:00') + ->and(AnalyticsAccountDailySnapshot::query()->where('social_account_id', $withoutHistory->id)->exists())->toBeFalse(); +}); + +function followerObservation(int $followers): AccountDailyObservation +{ + return new AccountDailyObservation( + date: CarbonImmutable::parse('2026-09-23', 'UTC'), + followers: $followers, + provenance: ObservationProvenance::Actual, + precision: MetricPrecision::Exact, + providerObservedAt: CarbonImmutable::now('UTC'), + ); +} diff --git a/tests/Feature/Analytics/AnalyticsScheduleTest.php b/tests/Feature/Analytics/AnalyticsScheduleTest.php new file mode 100644 index 000000000..a47de2791 --- /dev/null +++ b/tests/Feature/Analytics/AnalyticsScheduleTest.php @@ -0,0 +1,26 @@ +events()); + $collection = $events->first(fn ($event): bool => str_contains( + (string) $event->command, + 'analytics:dispatch-account-daily', + )); + $finalizer = $events->first(fn ($event): bool => $event->description === FinalizeAccountDailySnapshots::class); + + expect($collection)->not->toBeNull() + ->and($collection->expression)->toBe('0 2 * * *') + ->and($collection->timezone)->toBe('UTC') + ->and($collection->withoutOverlapping)->toBeTrue() + ->and($collection->onOneServer)->toBeTrue() + ->and($finalizer)->not->toBeNull() + ->and($finalizer->expression)->toBe('30 23 * * *') + ->and($finalizer->timezone)->toBe('UTC') + ->and($finalizer->withoutOverlapping)->toBeTrue() + ->and($finalizer->onOneServer)->toBeTrue(); +}); diff --git a/tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php b/tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php index d6f50ac16..6acb996c1 100644 --- a/tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php +++ b/tests/Feature/Analytics/Collectors/FollowerCollectorsTest.php @@ -9,8 +9,13 @@ use App\Services\Analytics\Collectors\Followers\FollowerCollectorFactory; use Carbon\CarbonImmutable; use Illuminate\Http\Client\Request; +use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\Http; +beforeEach(function () { + Bus::fake(); +}); + test('included platforms collect follower totals from their canonical read paths', function () { Http::fake(function (Request $request) { $url = $request->url(); diff --git a/tests/Feature/Observers/SocialAccountObserverTest.php b/tests/Feature/Observers/SocialAccountObserverTest.php index bbf697716..7e7aa8fab 100644 --- a/tests/Feature/Observers/SocialAccountObserverTest.php +++ b/tests/Feature/Observers/SocialAccountObserverTest.php @@ -3,6 +3,7 @@ declare(strict_types=1); use App\Enums\SocialAccount\Status; +use App\Jobs\Analytics\CollectAccountDailySnapshot; use App\Jobs\PostHog\IdentifyConnectedPlatforms; use App\Jobs\PostHog\SendEvent; use App\Jobs\PostHog\SyncAccountUsage; @@ -140,3 +141,24 @@ $socialAccount->update(['status' => Status::Disconnected]); } })->throwsNoExceptions(); + +test('connecting an included account dispatches its initial follower collection', function () { + Bus::fake(); + + $socialAccount = SocialAccount::factory()->instagram()->create([ + 'workspace_id' => $this->workspace->id, + ]); + + Bus::assertDispatched(CollectAccountDailySnapshot::class, fn ($job): bool => $job->socialAccountId === $socialAccount->id + && $job->queue === 'analytics'); +}); + +test('connecting an excluded account does not dispatch follower collection', function () { + Bus::fake(); + + SocialAccount::factory()->linkedin()->create([ + 'workspace_id' => $this->workspace->id, + ]); + + Bus::assertNotDispatched(CollectAccountDailySnapshot::class); +}); From b14516d95d1b14e4c14c4d3716d932e0c237eb8c Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 11:47:49 -0300 Subject: [PATCH 17/77] feat: reconcile analytics publications --- .../Analytics/SyncTryPostPublication.php | 84 +++++++ .../Analytics/UpsertAnalyticsPublication.php | 206 ++++++++++++++++++ app/Dto/Analytics/DiscoveredPublication.php | 26 +++ .../Analytics/TryPostPublicationIdentity.php | 38 ++++ app/Jobs/Analytics/SyncTryPostPublication.php | 37 ++++ app/Observers/PostPlatformObserver.php | 37 +++- .../PublicationReconciliationTest.php | 147 +++++++++++++ .../Observers/PostPlatformObserverTest.php | 19 +- 8 files changed, 589 insertions(+), 5 deletions(-) create mode 100644 app/Actions/Analytics/SyncTryPostPublication.php create mode 100644 app/Actions/Analytics/UpsertAnalyticsPublication.php create mode 100644 app/Dto/Analytics/DiscoveredPublication.php create mode 100644 app/Dto/Analytics/TryPostPublicationIdentity.php create mode 100644 app/Jobs/Analytics/SyncTryPostPublication.php create mode 100644 tests/Feature/Analytics/PublicationReconciliationTest.php diff --git a/app/Actions/Analytics/SyncTryPostPublication.php b/app/Actions/Analytics/SyncTryPostPublication.php new file mode 100644 index 000000000..d74e9a24d --- /dev/null +++ b/app/Actions/Analytics/SyncTryPostPublication.php @@ -0,0 +1,84 @@ +loadMissing(['post', 'socialAccount']); + $account = $postPlatform->socialAccount; + + if (! $account) { + throw new \LogicException('A live social account is required to capture publication identity.'); + } + + return $this->fromIdentity( + TryPostPublicationIdentity::fromAccount($account, $this->accountKeys->for($account)), + $postPlatform, + ); + } + + public function fromIdentity( + TryPostPublicationIdentity $identity, + PostPlatform $postPlatform, + ): AnalyticsPublication { + return $this->publications->tryPost( + $identity, + $postPlatform, + $this->normalizedContentType($postPlatform), + $this->excerpt($postPlatform), + ); + } + + private function normalizedContentType(PostPlatform $postPlatform): PublicationContentType + { + $specializedType = match ($postPlatform->content_type) { + ContentType::InstagramReel, ContentType::FacebookReel => PublicationContentType::Reel, + ContentType::InstagramStory, ContentType::FacebookStory => PublicationContentType::Story, + ContentType::YouTubeShort => PublicationContentType::Short, + ContentType::TikTokVideo, ContentType::PinterestVideoPin => PublicationContentType::Video, + ContentType::TikTokPhoto, ContentType::PinterestCarousel => PublicationContentType::Carousel, + default => null, + }; + + if ($specializedType) { + return $specializedType; + } + + $media = $postPlatform->post?->media_items; + + if (! $media || $media->isEmpty()) { + return PublicationContentType::Text; + } + + if ($media->count() > 1) { + return PublicationContentType::Carousel; + } + + return $media->first()->isVideo() + ? PublicationContentType::Video + : PublicationContentType::Image; + } + + private function excerpt(PostPlatform $postPlatform): ?string + { + $content = trim(html_entity_decode(strip_tags((string) $postPlatform->post?->content))); + + return $content === '' ? null : Str::limit($content, 500); + } +} diff --git a/app/Actions/Analytics/UpsertAnalyticsPublication.php b/app/Actions/Analytics/UpsertAnalyticsPublication.php new file mode 100644 index 000000000..901757b40 --- /dev/null +++ b/app/Actions/Analytics/UpsertAnalyticsPublication.php @@ -0,0 +1,206 @@ +accountKeys->for($account), + ); + + return $this->persist( + identity: $identity, + providerPostId: $publication->providerPostId, + providerPublishedAt: $publication->publishedAt, + origin: PublicationOrigin::External, + contentType: $publication->contentType, + providerContentType: $publication->providerContentType, + permalink: $publication->permalink, + excerpt: $publication->excerpt, + previewMetadata: $publication->previewMetadata, + providerMetadata: $publication->providerMetadata, + providerSyncedAt: now(), + ); + } + + public function tryPost( + TryPostPublicationIdentity $identity, + PostPlatform $postPlatform, + PublicationContentType $contentType, + ?string $excerpt, + ): AnalyticsPublication { + return $this->persist( + identity: $identity, + providerPostId: (string) $postPlatform->platform_post_id, + providerPublishedAt: ($postPlatform->published_at ?? $postPlatform->updated_at)->toImmutable(), + origin: PublicationOrigin::TryPost, + contentType: $contentType, + providerContentType: $postPlatform->content_type->value, + permalink: $postPlatform->platform_url, + excerpt: $excerpt, + previewMetadata: null, + providerMetadata: null, + postPlatformId: $postPlatform->id, + ); + } + + /** + * @param array|null $previewMetadata + * @param array|null $providerMetadata + */ + private function persist( + TryPostPublicationIdentity $identity, + string $providerPostId, + \DateTimeInterface $providerPublishedAt, + PublicationOrigin $origin, + PublicationContentType $contentType, + ?string $providerContentType, + ?string $permalink, + ?string $excerpt, + ?array $previewMetadata, + ?array $providerMetadata, + ?string $postPlatformId = null, + ?\DateTimeInterface $providerSyncedAt = null, + ): AnalyticsPublication { + try { + return $this->write( + $identity, + $providerPostId, + $providerPublishedAt, + $origin, + $contentType, + $providerContentType, + $permalink, + $excerpt, + $previewMetadata, + $providerMetadata, + $postPlatformId, + $providerSyncedAt, + true, + ); + } catch (UniqueConstraintViolationException) { + return $this->write( + $identity, + $providerPostId, + $providerPublishedAt, + $origin, + $contentType, + $providerContentType, + $permalink, + $excerpt, + $previewMetadata, + $providerMetadata, + $postPlatformId, + $providerSyncedAt, + false, + ); + } + } + + /** + * @param array|null $previewMetadata + * @param array|null $providerMetadata + */ + private function write( + TryPostPublicationIdentity $identity, + string $providerPostId, + \DateTimeInterface $providerPublishedAt, + PublicationOrigin $origin, + PublicationContentType $contentType, + ?string $providerContentType, + ?string $permalink, + ?string $excerpt, + ?array $previewMetadata, + ?array $providerMetadata, + ?string $postPlatformId, + ?\DateTimeInterface $providerSyncedAt, + bool $mayCreate, + ): AnalyticsPublication { + return DB::transaction(function () use ($contentType, $excerpt, $identity, $mayCreate, $origin, $permalink, $postPlatformId, $previewMetadata, $providerContentType, $providerMetadata, $providerPostId, $providerPublishedAt, $providerSyncedAt): AnalyticsPublication { + $identityFields = [ + 'workspace_id' => $identity->workspaceId, + 'social_account_key' => $identity->socialAccountKey, + 'network' => $identity->network, + 'provider_post_id' => $providerPostId, + ]; + $publication = AnalyticsPublication::query() + ->where($identityFields) + ->lockForUpdate() + ->first(); + + if (! $publication && $postPlatformId) { + $publication = AnalyticsPublication::query() + ->where('post_platform_id', $postPlatformId) + ->lockForUpdate() + ->first(); + } + + if (! $publication && ! $mayCreate) { + $publication = AnalyticsPublication::query() + ->where($identityFields) + ->lockForUpdate() + ->firstOrFail(); + } + + $now = now(); + $publication ??= new AnalyticsPublication([ + ...$identityFields, + 'first_seen_at' => $now, + ]); + + $values = [ + 'social_account_id' => SocialAccount::query()->whereKey($identity->socialAccountId)->value('id'), + 'post_platform_id' => $postPlatformId ?? $publication->post_platform_id, + 'platform_user_id' => $identity->platformUserId, + 'platform' => $identity->platform, + 'provider_published_at' => $providerPublishedAt, + 'origin' => $origin === PublicationOrigin::TryPost + ? PublicationOrigin::TryPost + : ($publication->origin ?? PublicationOrigin::External), + 'content_type' => $contentType, + 'availability' => PublicationAvailability::Available, + 'last_seen_at' => $now, + 'provider_synced_at' => $providerSyncedAt ?? $publication->provider_synced_at, + ]; + + foreach ([ + 'provider_content_type' => $providerContentType, + 'permalink' => $permalink, + 'excerpt' => $excerpt, + 'preview_metadata' => $previewMetadata, + 'provider_metadata' => $providerMetadata, + 'account_display_name' => $identity->accountDisplayName, + 'account_username' => $identity->accountUsername, + 'account_avatar_url' => $identity->accountAvatarUrl, + ] as $key => $value) { + if ($value !== null) { + $values[$key] = $value; + } + } + + $publication->fill($values)->save(); + + return $publication->refresh(); + }); + } +} diff --git a/app/Dto/Analytics/DiscoveredPublication.php b/app/Dto/Analytics/DiscoveredPublication.php new file mode 100644 index 000000000..88ccdb701 --- /dev/null +++ b/app/Dto/Analytics/DiscoveredPublication.php @@ -0,0 +1,26 @@ +|null $previewMetadata + * @param array|null $providerMetadata + */ + public function __construct( + public string $providerPostId, + public CarbonImmutable $publishedAt, + public PublicationContentType $contentType, + public ?string $providerContentType = null, + public ?string $permalink = null, + public ?string $excerpt = null, + public ?array $previewMetadata = null, + public ?array $providerMetadata = null, + ) {} +} diff --git a/app/Dto/Analytics/TryPostPublicationIdentity.php b/app/Dto/Analytics/TryPostPublicationIdentity.php new file mode 100644 index 000000000..c0e5f68d1 --- /dev/null +++ b/app/Dto/Analytics/TryPostPublicationIdentity.php @@ -0,0 +1,38 @@ +workspace_id, + socialAccountId: $account->id, + socialAccountKey: $accountKey, + network: $account->platform->network(), + platformUserId: $account->platform_user_id, + platform: $account->platform, + accountDisplayName: $account->display_name, + accountUsername: $account->username, + accountAvatarUrl: $account->avatar_url, + ); + } +} diff --git a/app/Jobs/Analytics/SyncTryPostPublication.php b/app/Jobs/Analytics/SyncTryPostPublication.php new file mode 100644 index 000000000..a520df276 --- /dev/null +++ b/app/Jobs/Analytics/SyncTryPostPublication.php @@ -0,0 +1,37 @@ +onQueue('analytics'); + } + + public function handle(SyncTryPostPublicationAction $sync): void + { + $postPlatform = PostPlatform::query() + ->published() + ->with('post') + ->find($this->postPlatformId); + + if (! $postPlatform || ! filled($postPlatform->platform_post_id)) { + return; + } + + $sync->fromIdentity($this->identity, $postPlatform); + } +} diff --git a/app/Observers/PostPlatformObserver.php b/app/Observers/PostPlatformObserver.php index b56da0b09..49fba2694 100644 --- a/app/Observers/PostPlatformObserver.php +++ b/app/Observers/PostPlatformObserver.php @@ -4,18 +4,29 @@ namespace App\Observers; +use App\Actions\Analytics\ResolveAnalyticsAccountKey; +use App\Dto\Analytics\TryPostPublicationIdentity; use App\Enums\PostPlatform\Status; +use App\Jobs\Analytics\SyncTryPostPublication; use App\Jobs\PostHog\SyncAccountPublishingActivity; use App\Models\PostPlatform; +use App\Services\Analytics\Collectors\Followers\FollowerCollectorFactory; use App\Services\PostHogService; +use Throwable; class PostPlatformObserver { public function updated(PostPlatform $postPlatform): void { - if (! PostHogService::isEnabled() - || ! $postPlatform->wasChanged('status') - || $postPlatform->status !== Status::Published) { + if (! $postPlatform->wasChanged('status') + || $postPlatform->status !== Status::Published + || ! filled($postPlatform->platform_post_id)) { + return; + } + + $this->dispatchAnalyticsSync($postPlatform); + + if (! PostHogService::isEnabled()) { return; } @@ -31,4 +42,24 @@ public function updated(PostPlatform $postPlatform): void ->delay(now()->addSeconds(SyncAccountPublishingActivity::DEBOUNCE_SECONDS)) ->afterCommit(); } + + private function dispatchAnalyticsSync(PostPlatform $postPlatform): void + { + try { + $account = $postPlatform->loadMissing('socialAccount')->socialAccount; + + if (! $account || ! app(FollowerCollectorFactory::class)->supports($postPlatform->platform)) { + return; + } + + $identity = TryPostPublicationIdentity::fromAccount( + $account, + app(ResolveAnalyticsAccountKey::class)->for($account), + ); + + SyncTryPostPublication::dispatch($identity, $postPlatform->id)->afterCommit(); + } catch (Throwable $exception) { + report($exception); + } + } } diff --git a/tests/Feature/Analytics/PublicationReconciliationTest.php b/tests/Feature/Analytics/PublicationReconciliationTest.php new file mode 100644 index 000000000..ab362345f --- /dev/null +++ b/tests/Feature/Analytics/PublicationReconciliationTest.php @@ -0,0 +1,147 @@ +external($account, discoveredPublication('remote-1')); + app(SyncTryPostPublication::class)->handle($postPlatform); + + $publication = AnalyticsPublication::sole(); + expect($publication->origin)->toBe(PublicationOrigin::TryPost) + ->and($publication->post_platform_id)->toBe($postPlatform->id) + ->and($publication->provider_published_at?->toISOString())->toBe('2026-09-20T12:00:00.000000Z'); +}); + +test('TryPost sync followed by external discovery preserves TryPost ownership', function () { + [$account, $postPlatform] = publicationFixture('remote-1'); + + app(SyncTryPostPublication::class)->handle($postPlatform); + app(UpsertAnalyticsPublication::class)->external($account, discoveredPublication( + providerPostId: 'remote-1', + excerpt: 'Provider copy', + )); + + $publication = AnalyticsPublication::sole(); + expect($publication->origin)->toBe(PublicationOrigin::TryPost) + ->and($publication->post_platform_id)->toBe($postPlatform->id) + ->and($publication->excerpt)->toBe('Provider copy'); +}); + +test('duplicate provider pages are idempotent', function () { + [$account] = publicationFixture(); + $upsert = app(UpsertAnalyticsPublication::class); + + $upsert->external($account, discoveredPublication('remote-1')); + $upsert->external($account, discoveredPublication('remote-1')); + + expect(AnalyticsPublication::count())->toBe(1); +}); + +test('database failures other than identity collisions are not swallowed', function () { + [$account] = publicationFixture(); + + expect(fn () => app(UpsertAnalyticsPublication::class)->external( + $account, + discoveredPublication(str_repeat('x', 192)), + ))->toThrow(QueryException::class); +}); + +test('the same provider post id remains separate by account and workspace', function () { + $workspace = Workspace::factory()->create(); + $firstAccount = SocialAccount::factory()->x()->create(['workspace_id' => $workspace->id]); + $secondAccount = SocialAccount::factory()->x()->create(['workspace_id' => $workspace->id]); + $otherAccount = SocialAccount::factory()->x()->create(); + $upsert = app(UpsertAnalyticsPublication::class); + + foreach ([$firstAccount, $secondAccount, $otherAccount] as $account) { + $upsert->external($account, discoveredPublication('shared-provider-id')); + } + + expect(AnalyticsPublication::count())->toBe(3) + ->and(AnalyticsPublication::query()->where('workspace_id', $workspace->id)->count())->toBe(2); +}); + +test('queued TryPost sync survives social account deletion after dispatch', function () { + [$account, $postPlatform] = publicationFixture(); + $postPlatform->update(['status' => Status::Pending]); + $queuedJob = null; + Queue::fake(); + + $postPlatform->markAsPublished('remote-1', 'https://x.com/example/status/remote-1'); + + Queue::assertPushed(SyncTryPostPublicationJob::class, function (SyncTryPostPublicationJob $job) use (&$queuedJob, $account, $postPlatform): bool { + $queuedJob = $job; + + return $job->postPlatformId === $postPlatform->id + && $job->identity->socialAccountId === $account->id + && $job->queue === 'analytics' + && $job->afterCommit === true; + }); + + $account->delete(); + $queuedJob->handle(app(SyncTryPostPublication::class)); + + $publication = AnalyticsPublication::sole(); + expect($publication->social_account_id)->toBeNull() + ->and($publication->social_account_key)->toBe($account->id) + ->and($publication->origin)->toBe(PublicationOrigin::TryPost) + ->and($publication->post_platform_id)->toBe($postPlatform->id); +}); + +/** @return array{SocialAccount, PostPlatform} */ +function publicationFixture(?string $providerPostId = null): array +{ + $account = SocialAccount::factory()->x()->create(); + $post = Post::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'content' => '

TryPost copy

', + ]); + $postPlatform = PostPlatform::factory()->x()->published()->create([ + 'post_id' => $post->id, + 'social_account_id' => $account->id, + 'platform_post_id' => $providerPostId ?? 'remote-1', + 'platform_url' => 'https://x.com/example/status/remote-1', + 'published_at' => CarbonImmutable::parse('2026-09-20 12:00:00', 'UTC'), + ]); + + return [$account, $postPlatform]; +} + +function discoveredPublication( + string $providerPostId = 'remote-1', + string $excerpt = 'External copy', +): DiscoveredPublication { + return new DiscoveredPublication( + providerPostId: $providerPostId, + publishedAt: CarbonImmutable::parse('2026-09-20 12:00:00', 'UTC'), + contentType: PublicationContentType::Text, + providerContentType: 'tweet', + permalink: "https://x.com/example/status/{$providerPostId}", + excerpt: $excerpt, + previewMetadata: ['thumbnail_url' => 'https://example.com/thumb.jpg'], + providerMetadata: ['source' => 'provider'], + ); +} diff --git a/tests/Feature/Observers/PostPlatformObserverTest.php b/tests/Feature/Observers/PostPlatformObserverTest.php index 66495c7c5..909db86d2 100644 --- a/tests/Feature/Observers/PostPlatformObserverTest.php +++ b/tests/Feature/Observers/PostPlatformObserverTest.php @@ -3,10 +3,12 @@ declare(strict_types=1); use App\Enums\PostPlatform\Status; +use App\Jobs\Analytics\SyncTryPostPublication; use App\Jobs\PostHog\SyncAccountPublishingActivity; use App\Models\Account; use App\Models\Post; use App\Models\PostPlatform; +use App\Models\SocialAccount; use App\Models\User; use App\Models\Workspace; use Illuminate\Support\Facades\Queue; @@ -55,13 +57,26 @@ test('account activity sync is not queued when PostHog is disabled', function () { config(['services.posthog.enabled' => false]); - $postPlatform = PostPlatform::factory()->recycle($this->post)->create(); - Queue::fake(); + $socialAccount = SocialAccount::factory()->x()->create(['workspace_id' => $this->workspace->id]); + $postPlatform = PostPlatform::factory()->x()->recycle($this->post)->create([ + 'social_account_id' => $socialAccount->id, + ]); $postPlatform->markAsPublished('remote-post-id'); Queue::assertNotPushed(SyncAccountPublishingActivity::class); + Queue::assertPushed(SyncTryPostPublication::class); +}); + +test('excluded platform does not queue an analytics publication sync', function () { + config(['services.posthog.enabled' => false]); + $postPlatform = PostPlatform::factory()->linkedin()->recycle($this->post)->create(); + Queue::fake(); + + $postPlatform->markAsPublished('remote-post-id'); + + Queue::assertNotPushed(SyncTryPostPublication::class); }); test('updating an already published platform does not queue another sync', function () { From 63e3cd4d65cfb653cad3158f4899f907637054ed Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 11:52:19 -0300 Subject: [PATCH 18/77] feat: collect Meta publication history --- .../Analytics/PublicationHistoryCollector.php | 18 ++ app/Dto/Analytics/PublicationPage.php | 18 ++ .../AbstractMetaPublicationCollector.php | 95 +++++++++++ .../FacebookPublicationCollector.php | 111 ++++++++++++ .../InstagramPublicationCollector.php | 82 +++++++++ .../PublicationHistoryCollectorFactory.php | 23 +++ .../ThreadsPublicationCollector.php | 74 ++++++++ .../MetaPublicationCollectorsTest.php | 158 ++++++++++++++++++ 8 files changed, 579 insertions(+) create mode 100644 app/Contracts/Analytics/PublicationHistoryCollector.php create mode 100644 app/Dto/Analytics/PublicationPage.php create mode 100644 app/Services/Analytics/Collectors/Publications/AbstractMetaPublicationCollector.php create mode 100644 app/Services/Analytics/Collectors/Publications/FacebookPublicationCollector.php create mode 100644 app/Services/Analytics/Collectors/Publications/InstagramPublicationCollector.php create mode 100644 app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php create mode 100644 app/Services/Analytics/Collectors/Publications/ThreadsPublicationCollector.php create mode 100644 tests/Feature/Analytics/Collectors/MetaPublicationCollectorsTest.php diff --git a/app/Contracts/Analytics/PublicationHistoryCollector.php b/app/Contracts/Analytics/PublicationHistoryCollector.php new file mode 100644 index 000000000..39c801d67 --- /dev/null +++ b/app/Contracts/Analytics/PublicationHistoryCollector.php @@ -0,0 +1,18 @@ + $publications + */ + public function __construct( + public array $publications, + public ?string $nextCursor, + public bool $providerExhausted, + public bool $providerLimited = false, + ) {} +} diff --git a/app/Services/Analytics/Collectors/Publications/AbstractMetaPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/AbstractMetaPublicationCollector.php new file mode 100644 index 000000000..45ff6e6c2 --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/AbstractMetaPublicationCollector.php @@ -0,0 +1,95 @@ + $query + */ + protected function get(SocialAccount $account, string $url, array $query = []): Response + { + $response = Http::acceptJson() + ->withToken($account->access_token) + ->timeout(120) + ->get($url, array_filter($query, fn (mixed $value): bool => $value !== null && $value !== '')); + + if ($response->successful()) { + return $response; + } + + $code = (int) data_get($response->json(), 'error.code', 0); + $category = match (true) { + $response->status() === 429, in_array($code, [4, 17, 32, 80001, 80002], true) => 'rate_limited', + $response->status() === 401, $code === 190 => 'authentication', + $response->status() === 403, in_array($code, [10, 200], true) => 'permission', + $response->serverError(), in_array($code, [1, 2], true) => 'transient', + default => 'malformed', + }; + + throw new AnalyticsCollectionException( + $category, + "publication history collection failed with HTTP {$response->status()}", + $this->retryAt($response), + ); + } + + /** + * @param list $publications + */ + protected function result( + array $publications, + Response $response, + bool $crossedCutoff, + bool $providerLimited = false, + ): PublicationPage { + $nextCursor = data_get($response->json(), 'paging.next') + ? data_get($response->json(), 'paging.cursors.after') + : null; + + return new PublicationPage( + publications: $publications, + nextCursor: $crossedCutoff ? null : (is_string($nextCursor) ? $nextCursor : null), + providerExhausted: $crossedCutoff || ! is_string($nextCursor), + providerLimited: $providerLimited, + ); + } + + protected function publishedAt(mixed $value): ?CarbonImmutable + { + if (! is_string($value) || $value === '') { + return null; + } + + return CarbonImmutable::parse($value)->utc(); + } + + /** @return array|null */ + protected function preview(?string $url): ?array + { + return filled($url) ? ['thumbnail_url' => $url] : null; + } + + private function retryAt(Response $response): ?CarbonImmutable + { + $retryAfter = $response->header('Retry-After'); + + if (! is_string($retryAfter) || $retryAfter === '') { + return null; + } + + return ctype_digit($retryAfter) + ? CarbonImmutable::now('UTC')->addSeconds((int) $retryAfter) + : CarbonImmutable::parse($retryAfter)->utc(); + } +} diff --git a/app/Services/Analytics/Collectors/Publications/FacebookPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/FacebookPublicationCollector.php new file mode 100644 index 000000000..643c5f195 --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/FacebookPublicationCollector.php @@ -0,0 +1,111 @@ +get( + $account, + config('trypost.platforms.facebook.graph_api')."/{$account->platform_user_id}/published_posts", + ['fields' => self::FIELDS, 'limit' => self::PAGE_SIZE, 'after' => $cursor], + ); + $publications = []; + $crossedCutoff = false; + $providerLimited = false; + + foreach ((array) $response->json('data', []) as $row) { + $publishedAt = $this->publishedAt(data_get($row, 'created_time')); + + if (! $publishedAt) { + $providerLimited = true; + + continue; + } + + if ($publishedAt->lessThan($cutoff)) { + $crossedCutoff = true; + + break; + } + + $attachment = (array) data_get($row, 'attachments.data.0', []); + $previewUrl = data_get($attachment, 'media.image.src'); + $permalink = data_get($row, 'permalink_url'); + + if ($this->isVideo($attachment) && filled(data_get($attachment, 'target.id'))) { + $hydrated = $this->hydratePreview($account, (string) data_get($attachment, 'target.id')); + $previewUrl = data_get($hydrated, 'picture') ?: $previewUrl; + $permalink = $permalink ?: data_get($hydrated, 'permalink_url'); + } + + $publications[] = new DiscoveredPublication( + providerPostId: (string) data_get($row, 'id'), + publishedAt: $publishedAt, + contentType: $this->contentType($attachment), + providerContentType: data_get($attachment, 'type') ?: data_get($attachment, 'media_type') ?: data_get($row, 'status_type'), + permalink: $permalink, + excerpt: data_get($row, 'message'), + previewMetadata: $this->preview($previewUrl), + providerMetadata: null, + ); + } + + return $this->result($publications, $response, $crossedCutoff, $providerLimited); + } + + /** @param array $attachment */ + private function contentType(array $attachment): PublicationContentType + { + $mediaType = strtolower((string) data_get($attachment, 'media_type')); + $type = strtolower((string) data_get($attachment, 'type')); + $url = strtolower((string) data_get($attachment, 'url')); + + return match (true) { + count((array) data_get($attachment, 'subattachments.data', [])) > 1 => PublicationContentType::Carousel, + str_contains($type, 'reel'), str_contains($url, '/reel') => PublicationContentType::Reel, + str_contains($mediaType, 'video'), str_contains($type, 'video') => PublicationContentType::Video, + str_contains($mediaType, 'photo'), str_contains($mediaType, 'image') => PublicationContentType::Image, + str_contains($type, 'share'), filled($url) => PublicationContentType::Link, + $attachment === [] => PublicationContentType::Text, + default => PublicationContentType::Unknown, + }; + } + + /** @param array $attachment */ + private function isVideo(array $attachment): bool + { + return in_array($this->contentType($attachment), [PublicationContentType::Video, PublicationContentType::Reel], true); + } + + /** @return array */ + private function hydratePreview(SocialAccount $account, string $videoId): array + { + try { + return (array) $this->get( + $account, + config('trypost.platforms.facebook.graph_api')."/{$videoId}", + ['fields' => 'picture,permalink_url'], + )->json(); + } catch (\Throwable) { + return []; + } + } +} diff --git a/app/Services/Analytics/Collectors/Publications/InstagramPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/InstagramPublicationCollector.php new file mode 100644 index 000000000..ec984390d --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/InstagramPublicationCollector.php @@ -0,0 +1,82 @@ +platform === Platform::InstagramFacebook + ? 'id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,timestamp' + : 'id,caption,media_type,media_url,permalink,thumbnail_url,timestamp'; + $response = $this->get( + $account, + "{$account->platform->instagramGraphBaseUrl()}/{$account->platform_user_id}/media", + ['fields' => $fields, 'limit' => self::PAGE_SIZE, 'after' => $cursor], + ); + $publications = []; + $crossedCutoff = false; + $providerLimited = false; + + foreach ((array) $response->json('data', []) as $row) { + $publishedAt = $this->publishedAt(data_get($row, 'timestamp')); + + if (! $publishedAt) { + $providerLimited = true; + + continue; + } + + if ($publishedAt->lessThan($cutoff)) { + $crossedCutoff = true; + + break; + } + + $publications[] = new DiscoveredPublication( + providerPostId: (string) data_get($row, 'id'), + publishedAt: $publishedAt, + contentType: $this->contentType($row, $account->platform), + providerContentType: data_get($row, 'media_product_type') ?: data_get($row, 'media_type'), + permalink: data_get($row, 'permalink'), + excerpt: data_get($row, 'caption'), + previewMetadata: $this->preview(data_get($row, 'thumbnail_url') ?: data_get($row, 'media_url')), + providerMetadata: null, + ); + } + + return $this->result($publications, $response, $crossedCutoff, $providerLimited); + } + + /** @param array $row */ + private function contentType(array $row, Platform $platform): PublicationContentType + { + $mediaType = strtoupper((string) data_get($row, 'media_type')); + $productType = strtoupper((string) data_get($row, 'media_product_type')); + + return match (true) { + $productType === 'STORY' => PublicationContentType::Story, + $productType === 'REELS' => PublicationContentType::Reel, + $mediaType === 'CAROUSEL_ALBUM' => PublicationContentType::Carousel, + $mediaType === 'VIDEO' && $platform === Platform::Instagram => PublicationContentType::Reel, + $mediaType === 'VIDEO' => PublicationContentType::Video, + $mediaType === 'IMAGE' => PublicationContentType::Image, + default => PublicationContentType::Unknown, + }; + } +} diff --git a/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php b/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php new file mode 100644 index 000000000..f24fd8a28 --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php @@ -0,0 +1,23 @@ +platform) { + Platform::Instagram, Platform::InstagramFacebook => app(InstagramPublicationCollector::class), + Platform::Facebook => app(FacebookPublicationCollector::class), + Platform::Threads => app(ThreadsPublicationCollector::class), + default => throw AnalyticsCollectionException::unsupported("{$account->platform->value} publication history is not handled by a Meta collector"), + }; + } +} diff --git a/app/Services/Analytics/Collectors/Publications/ThreadsPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/ThreadsPublicationCollector.php new file mode 100644 index 000000000..6d77eceb7 --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/ThreadsPublicationCollector.php @@ -0,0 +1,74 @@ +get( + $account, + config('trypost.platforms.threads.graph_api')."/{$account->platform_user_id}/threads", + ['fields' => self::FIELDS, 'limit' => self::PAGE_SIZE, 'after' => $cursor], + ); + $publications = []; + $crossedCutoff = false; + $providerLimited = false; + + foreach ((array) $response->json('data', []) as $row) { + $publishedAt = $this->publishedAt(data_get($row, 'timestamp')); + + if (! $publishedAt) { + $providerLimited = true; + + continue; + } + + if ($publishedAt->lessThan($cutoff)) { + $crossedCutoff = true; + + break; + } + + $publications[] = new DiscoveredPublication( + providerPostId: (string) data_get($row, 'id'), + publishedAt: $publishedAt, + contentType: $this->contentType($row), + providerContentType: data_get($row, 'media_product_type') ?: data_get($row, 'media_type'), + permalink: data_get($row, 'permalink'), + excerpt: data_get($row, 'text'), + previewMetadata: $this->preview(data_get($row, 'thumbnail_url') ?: data_get($row, 'media_url')), + providerMetadata: ['is_quote_post' => (bool) data_get($row, 'is_quote_post', false)], + ); + } + + return $this->result($publications, $response, $crossedCutoff, $providerLimited); + } + + /** @param array $row */ + private function contentType(array $row): PublicationContentType + { + return match (strtoupper((string) data_get($row, 'media_type'))) { + 'VIDEO' => PublicationContentType::Video, + 'IMAGE' => PublicationContentType::Image, + 'CAROUSEL_ALBUM' => PublicationContentType::Carousel, + default => PublicationContentType::Text, + }; + } +} diff --git a/tests/Feature/Analytics/Collectors/MetaPublicationCollectorsTest.php b/tests/Feature/Analytics/Collectors/MetaPublicationCollectorsTest.php new file mode 100644 index 000000000..b27fa8b53 --- /dev/null +++ b/tests/Feature/Analytics/Collectors/MetaPublicationCollectorsTest.php @@ -0,0 +1,158 @@ + Http::response([ + 'data' => [ + ['id' => 'feed-1', 'media_type' => 'IMAGE', 'media_product_type' => 'FEED', 'caption' => 'Feed', 'permalink' => 'https://instagram.com/p/feed', 'timestamp' => '2026-09-20T12:00:00+0000', 'media_url' => 'https://cdn.example/feed.jpg'], + ['id' => 'carousel-1', 'media_type' => 'CAROUSEL_ALBUM', 'media_product_type' => 'FEED', 'caption' => 'Carousel', 'permalink' => 'https://instagram.com/p/carousel', 'timestamp' => '2026-09-19T12:00:00+0000'], + ['id' => 'reel-1', 'media_type' => 'VIDEO', 'media_product_type' => 'REELS', 'caption' => 'Reel', 'permalink' => 'https://instagram.com/reel/1', 'timestamp' => '2026-09-18T12:00:00+0000', 'thumbnail_url' => 'https://cdn.example/reel.jpg'], + ], + 'paging' => ['cursors' => ['after' => 'cursor-2'], 'next' => 'https://graph.instagram.com/next'], + ]), + ]); + $account = SocialAccount::factory()->instagram()->create(); + + $page = app(InstagramPublicationCollector::class)->page( + $account, + 'cursor-1', + CarbonImmutable::parse('2026-09-18 12:00:00', 'UTC'), + ); + + expect($page->publications)->toHaveCount(3) + ->and(array_column($page->publications, 'contentType'))->toBe([ + PublicationContentType::Image, + PublicationContentType::Carousel, + PublicationContentType::Reel, + ]) + ->and($page->nextCursor)->toBe('cursor-2') + ->and($page->providerExhausted)->toBeFalse(); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), "/{$account->platform_user_id}/media") + && ! str_contains($request->url(), '/stories') + && $request['after'] === 'cursor-1'); +}); + +test('instagram stops a page at the cutoff and omits an older provider row', function () { + Http::fake(['*' => Http::response([ + 'data' => [ + ['id' => 'at-cutoff', 'media_type' => 'IMAGE', 'timestamp' => '2026-09-18T12:00:00+0000'], + ['id' => 'too-old', 'media_type' => 'IMAGE', 'timestamp' => '2026-09-18T11:59:59+0000'], + ], + 'paging' => ['cursors' => ['after' => 'ignored'], 'next' => 'https://graph.instagram.com/next'], + ])]); + $account = SocialAccount::factory()->instagram()->create(); + + $page = app(InstagramPublicationCollector::class)->page( + $account, + null, + CarbonImmutable::parse('2026-09-18 12:00:00', 'UTC'), + ); + + expect($page->publications)->toHaveCount(1) + ->and($page->publications[0]->providerPostId)->toBe('at-cutoff') + ->and($page->nextCursor)->toBeNull() + ->and($page->providerExhausted)->toBeTrue(); +}); + +test('facebook reads page-owned published posts and keeps a video when preview hydration fails', function () { + $graph = config('trypost.platforms.facebook.graph_api'); + Http::fake([ + "{$graph}/*/published_posts*" => Http::response([ + 'data' => [ + [ + 'id' => 'page_video', + 'message' => 'Video post', + 'created_time' => '2026-09-20T12:00:00+0000', + 'permalink_url' => 'https://facebook.com/page/videos/123', + 'attachments' => ['data' => [[ + 'media_type' => 'video', + 'target' => ['id' => 'video-123'], + ]]], + ], + [ + 'id' => 'page_photo', + 'message' => 'Photo post', + 'created_time' => '2026-09-19T12:00:00+0000', + 'permalink_url' => 'https://facebook.com/page/posts/456', + 'attachments' => ['data' => [[ + 'media_type' => 'photo', + 'media' => ['image' => ['src' => 'https://cdn.example/photo.jpg']], + ]]], + ], + ], + ]), + "{$graph}/video-123*" => Http::response(['error' => ['code' => 2]], 500), + ]); + $account = SocialAccount::factory()->facebook()->create(); + + $page = app(FacebookPublicationCollector::class)->page( + $account, + null, + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->publications)->toHaveCount(2) + ->and($page->publications[0]->providerPostId)->toBe('page_video') + ->and($page->publications[0]->contentType)->toBe(PublicationContentType::Video) + ->and($page->publications[0]->previewMetadata)->toBeNull() + ->and($page->publications[1]->contentType)->toBe(PublicationContentType::Image) + ->and($page->providerExhausted)->toBeTrue(); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/published_posts')); + Http::assertNotSent(fn (Request $request): bool => str_contains($request->url(), '/feed')); +}); + +test('threads reads one owned post page and returns its cursor', function () { + Http::fake(['*' => Http::response([ + 'data' => [[ + 'id' => 'thread-1', + 'media_type' => 'VIDEO', + 'text' => 'A thread', + 'permalink' => 'https://threads.net/@example/post/1', + 'timestamp' => '2026-09-20T12:00:00+0000', + 'thumbnail_url' => 'https://cdn.example/thread.jpg', + ]], + 'paging' => ['cursors' => ['after' => 'thread-cursor'], 'next' => 'https://graph.threads.net/next'], + ])]); + $account = SocialAccount::factory()->threads()->create(); + + $page = app(ThreadsPublicationCollector::class)->page( + $account, + null, + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->publications)->toHaveCount(1) + ->and($page->publications[0]->contentType)->toBe(PublicationContentType::Video) + ->and($page->nextCursor)->toBe('thread-cursor') + ->and($page->providerExhausted)->toBeFalse(); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), "/{$account->platform_user_id}/threads")); +}); + +test('publication collector factory distinguishes supported Meta platforms', function (Platform $platform, string $collector) { + $account = SocialAccount::factory()->create(['platform' => $platform]); + + expect(app(PublicationHistoryCollectorFactory::class)->for($account))->toBeInstanceOf($collector); +})->with([ + [Platform::Instagram, InstagramPublicationCollector::class], + [Platform::InstagramFacebook, InstagramPublicationCollector::class], + [Platform::Facebook, FacebookPublicationCollector::class], + [Platform::Threads, ThreadsPublicationCollector::class], +]); From 8eb9e3d3dc7d1c7d68ccc85eab7ced39bf99bb6b Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 12:01:41 -0300 Subject: [PATCH 19/77] feat: discover media network publications --- .../Controllers/App/AnalyticsController.php | 2 +- .../AbstractApiPublicationCollector.php | 91 ++++++++ .../PinterestPublicationCollector.php | 76 +++++++ .../PublicationHistoryCollectorFactory.php | 6 +- .../TikTokPublicationCollector.php | 112 +++++++++ .../Publications/XPublicationCollector.php | 93 ++++++++ .../YouTubePublicationCollector.php | 106 +++++++++ .../MediaPublicationCollectorsTest.php | 213 ++++++++++++++++++ 8 files changed, 697 insertions(+), 2 deletions(-) create mode 100644 app/Services/Analytics/Collectors/Publications/AbstractApiPublicationCollector.php create mode 100644 app/Services/Analytics/Collectors/Publications/PinterestPublicationCollector.php create mode 100644 app/Services/Analytics/Collectors/Publications/TikTokPublicationCollector.php create mode 100644 app/Services/Analytics/Collectors/Publications/XPublicationCollector.php create mode 100644 app/Services/Analytics/Collectors/Publications/YouTubePublicationCollector.php create mode 100644 tests/Feature/Analytics/Collectors/MediaPublicationCollectorsTest.php diff --git a/app/Http/Controllers/App/AnalyticsController.php b/app/Http/Controllers/App/AnalyticsController.php index a764e2796..cb2600472 100644 --- a/app/Http/Controllers/App/AnalyticsController.php +++ b/app/Http/Controllers/App/AnalyticsController.php @@ -49,7 +49,7 @@ public function index(Request $request): Response $this->authorize('view', $workspace); $accounts = $workspace->socialAccounts() - ->where('is_active', true) + ->active() ->whereIn('platform', self::SUPPORTED_PLATFORMS) ->get() ->map(fn (SocialAccount $account) => [ diff --git a/app/Services/Analytics/Collectors/Publications/AbstractApiPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/AbstractApiPublicationCollector.php new file mode 100644 index 000000000..0e31aea52 --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/AbstractApiPublicationCollector.php @@ -0,0 +1,91 @@ + $query */ + protected function get(SocialAccount $account, string $url, array $query = []): Response + { + $response = Http::acceptJson() + ->withToken($account->access_token) + ->timeout(120) + ->get($url, array_filter($query, fn (mixed $value): bool => $value !== null && $value !== '')); + + return $this->successfulResponse($response); + } + + /** @param array $payload */ + protected function post(SocialAccount $account, string $url, array $payload): Response + { + $response = Http::acceptJson() + ->asJson() + ->withToken($account->access_token) + ->timeout(120) + ->post($url, $payload); + + return $this->successfulResponse($response); + } + + protected function publishedAt(mixed $value, bool $timestamp = false): ?CarbonImmutable + { + if ($timestamp && is_numeric($value)) { + return CarbonImmutable::createFromTimestampUTC((int) $value); + } + + if (! is_string($value) || $value === '') { + return null; + } + + try { + return CarbonImmutable::parse($value)->utc(); + } catch (Throwable) { + return null; + } + } + + private function successfulResponse(Response $response): Response + { + if ($response->successful()) { + return $response; + } + + $reason = (string) data_get($response->json(), 'error.errors.0.reason', ''); + $category = match (true) { + $response->status() === 429, + in_array($reason, ['quotaExceeded', 'rateLimitExceeded', 'userRateLimitExceeded'], true) => 'rate_limited', + $response->status() === 401 => 'authentication', + $response->status() === 403 => 'permission', + $response->serverError() => 'transient', + default => 'malformed', + }; + + throw new AnalyticsCollectionException( + $category, + "publication history collection failed with HTTP {$response->status()}", + $this->retryAt($response), + ); + } + + private function retryAt(Response $response): ?CarbonImmutable + { + $retryAfter = $response->header('Retry-After'); + + if (! is_string($retryAfter) || $retryAfter === '') { + return null; + } + + return ctype_digit($retryAfter) + ? CarbonImmutable::now('UTC')->addSeconds((int) $retryAfter) + : CarbonImmutable::parse($retryAfter)->utc(); + } +} diff --git a/app/Services/Analytics/Collectors/Publications/PinterestPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/PinterestPublicationCollector.php new file mode 100644 index 000000000..0d0564c54 --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/PinterestPublicationCollector.php @@ -0,0 +1,76 @@ +get($account, config('trypost.platforms.pinterest.api').'/pins', [ + 'page_size' => self::PAGE_SIZE, + 'bookmark' => $cursor, + ]); + $publications = []; + $crossedCutoff = false; + $providerLimited = false; + + foreach ((array) $response->json('items', []) as $row) { + $publishedAt = $this->publishedAt(data_get($row, 'created_at')); + + if (! $publishedAt) { + $providerLimited = true; + + continue; + } + + if ($publishedAt->lessThan($cutoff)) { + $crossedCutoff = true; + + break; + } + + $postId = (string) data_get($row, 'id'); + $providerType = strtolower((string) data_get($row, 'media.media_type')); + $thumbnail = data_get($row, 'media.images.600x.url') + ?: data_get($row, 'media.images.originals.url'); + + $publications[] = new DiscoveredPublication( + providerPostId: $postId, + publishedAt: $publishedAt, + contentType: $this->contentType($providerType), + providerContentType: $providerType ?: null, + permalink: "https://www.pinterest.com/pin/{$postId}/", + excerpt: data_get($row, 'description') ?: data_get($row, 'title'), + previewMetadata: filled($thumbnail) ? ['thumbnail_url' => $thumbnail] : null, + providerMetadata: ['metric_time_basis' => MetricTimeBasis::Lifetime->value], + ); + } + + $nextCursor = data_get($response->json(), 'bookmark'); + $hasNext = is_string($nextCursor) && $nextCursor !== '' && ! $crossedCutoff; + + return new PublicationPage($publications, $hasNext ? $nextCursor : null, ! $hasNext, $providerLimited); + } + + private function contentType(string $providerType): PublicationContentType + { + return match (true) { + str_contains($providerType, 'video') => PublicationContentType::Video, + str_contains($providerType, 'multiple'), str_contains($providerType, 'carousel') => PublicationContentType::Carousel, + str_contains($providerType, 'image') => PublicationContentType::Image, + default => PublicationContentType::Unknown, + }; + } +} diff --git a/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php b/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php index f24fd8a28..81b9a5401 100644 --- a/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php +++ b/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php @@ -17,7 +17,11 @@ public function for(SocialAccount $account): PublicationHistoryCollector Platform::Instagram, Platform::InstagramFacebook => app(InstagramPublicationCollector::class), Platform::Facebook => app(FacebookPublicationCollector::class), Platform::Threads => app(ThreadsPublicationCollector::class), - default => throw AnalyticsCollectionException::unsupported("{$account->platform->value} publication history is not handled by a Meta collector"), + Platform::X => app(XPublicationCollector::class), + Platform::Pinterest => app(PinterestPublicationCollector::class), + Platform::YouTube => app(YouTubePublicationCollector::class), + Platform::TikTok => app(TikTokPublicationCollector::class), + default => throw AnalyticsCollectionException::unsupported("{$account->platform->value} publication history is not supported"), }; } } diff --git a/app/Services/Analytics/Collectors/Publications/TikTokPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/TikTokPublicationCollector.php new file mode 100644 index 000000000..8643cda3f --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/TikTokPublicationCollector.php @@ -0,0 +1,112 @@ +scopes ?? [], true)) { + return new PublicationPage([], null, true, true); + } + + $payload = ['max_count' => self::PAGE_SIZE]; + + if (is_string($cursor) && ctype_digit($cursor)) { + $payload['cursor'] = (int) $cursor; + } + + $response = $this->post( + $account, + config('trypost.platforms.tiktok.api').'/video/list/?fields='.self::FIELDS, + $payload, + ); + $errorCode = data_get($response->json(), 'error.code'); + + if (is_string($errorCode) && ! in_array($errorCode, ['', 'ok'], true)) { + $category = match ($errorCode) { + 'rate_limit_exceeded' => 'rate_limited', + 'internal_error' => 'transient', + default => 'permission', + }; + + throw new AnalyticsCollectionException( + $category, + "TikTok video list failed with {$errorCode}", + ); + } + + $publications = []; + $crossedCutoff = false; + $providerLimited = false; + + foreach ((array) $response->json('data.videos', []) as $row) { + $publishedAt = $this->publishedAt(data_get($row, 'create_time'), timestamp: true); + + if (! $publishedAt) { + $providerLimited = true; + + continue; + } + + if ($publishedAt->lessThan($cutoff)) { + $crossedCutoff = true; + + break; + } + + $postId = (string) data_get($row, 'id'); + $cover = data_get($row, 'cover_image_url'); + + $publications[] = new DiscoveredPublication( + providerPostId: $postId, + publishedAt: $publishedAt, + contentType: PublicationContentType::Video, + providerContentType: 'video', + permalink: data_get($row, 'share_url'), + excerpt: data_get($row, 'video_description') ?: data_get($row, 'title'), + previewMetadata: is_string($cover) && $cover !== '' ? [ + 'thumbnail_url' => $cover, + 'expires_at' => $this->coverExpiresAt($cover), + ] : null, + providerMetadata: ['duration' => data_get($row, 'duration')], + ); + } + + $nextCursor = data_get($response->json(), 'data.cursor'); + $hasNext = (bool) data_get($response->json(), 'data.has_more') + && is_numeric($nextCursor) + && ! $crossedCutoff; + + return new PublicationPage( + $publications, + $hasNext ? (string) $nextCursor : null, + ! $hasNext, + $providerLimited, + ); + } + + private function coverExpiresAt(string $url): ?string + { + parse_str((string) parse_url($url, PHP_URL_QUERY), $query); + $expires = $query['x-expires'] ?? $query['expires'] ?? null; + + return is_scalar($expires) && ctype_digit((string) $expires) + ? CarbonImmutable::createFromTimestampUTC((int) $expires)->toIso8601String() + : null; + } +} diff --git a/app/Services/Analytics/Collectors/Publications/XPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/XPublicationCollector.php new file mode 100644 index 000000000..bd5b32fd3 --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/XPublicationCollector.php @@ -0,0 +1,93 @@ +get( + $account, + config('trypost.platforms.x.api')."/users/{$account->platform_user_id}/tweets", + [ + 'max_results' => self::PAGE_SIZE, + 'pagination_token' => $cursor, + 'tweet.fields' => 'created_at,attachments', + 'expansions' => 'attachments.media_keys', + 'media.fields' => 'media_key,type,preview_image_url,url', + ], + ); + $media = collect((array) $response->json('includes.media', []))->keyBy('media_key'); + $publications = []; + $crossedCutoff = false; + $providerLimited = false; + + foreach ((array) $response->json('data', []) as $row) { + $publishedAt = $this->publishedAt(data_get($row, 'created_at')); + + if (! $publishedAt) { + $providerLimited = true; + + continue; + } + + if ($publishedAt->lessThan($cutoff)) { + $crossedCutoff = true; + + break; + } + + $attachedMedia = collect((array) data_get($row, 'attachments.media_keys', [])) + ->map(fn (mixed $key) => $media->get((string) $key)) + ->filter(fn (mixed $item): bool => is_array($item)) + ->values(); + $types = $attachedMedia->pluck('type')->filter()->unique()->values(); + $postId = (string) data_get($row, 'id'); + $preview = $attachedMedia->first( + fn (array $item): bool => filled(data_get($item, 'preview_image_url')) || filled(data_get($item, 'url')), + ); + + $publications[] = new DiscoveredPublication( + providerPostId: $postId, + publishedAt: $publishedAt, + contentType: $this->contentType($types->all(), $attachedMedia->count()), + providerContentType: $types->implode(','), + permalink: filled($account->username) ? "https://x.com/{$account->username}/status/{$postId}" : null, + excerpt: data_get($row, 'text'), + previewMetadata: is_array($preview) + ? ['thumbnail_url' => data_get($preview, 'preview_image_url') ?: data_get($preview, 'url')] + : null, + ); + } + + $nextCursor = data_get($response->json(), 'meta.next_token'); + $hasNext = is_string($nextCursor) && $nextCursor !== '' && ! $crossedCutoff; + + return new PublicationPage($publications, $hasNext ? $nextCursor : null, ! $hasNext, $providerLimited); + } + + /** @param list $types */ + private function contentType(array $types, int $mediaCount): PublicationContentType + { + if (in_array('video', $types, true) || in_array('animated_gif', $types, true)) { + return PublicationContentType::Video; + } + + if ($mediaCount > 1) { + return PublicationContentType::Carousel; + } + + return in_array('photo', $types, true) ? PublicationContentType::Image : PublicationContentType::Text; + } +} diff --git a/app/Services/Analytics/Collectors/Publications/YouTubePublicationCollector.php b/app/Services/Analytics/Collectors/Publications/YouTubePublicationCollector.php new file mode 100644 index 000000000..9fb6e23ac --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/YouTubePublicationCollector.php @@ -0,0 +1,106 @@ +get($account, "{$api}/channels", [ + 'part' => 'contentDetails', + 'id' => $account->platform_user_id, + 'maxResults' => 1, + ]); + $uploadsPlaylist = data_get($channel->json(), 'items.0.contentDetails.relatedPlaylists.uploads'); + + if (! is_string($uploadsPlaylist) || $uploadsPlaylist === '') { + throw AnalyticsCollectionException::malformed('YouTube channel did not expose an uploads playlist'); + } + + $playlist = $this->get($account, "{$api}/playlistItems", [ + 'part' => 'contentDetails', + 'playlistId' => $uploadsPlaylist, + 'maxResults' => self::PAGE_SIZE, + 'pageToken' => $cursor, + ]); + $videoIds = collect((array) $playlist->json('items', [])) + ->pluck('contentDetails.videoId') + ->filter(fn (mixed $id): bool => is_string($id) && $id !== '') + ->values(); + $videos = collect(); + + if ($videoIds->isNotEmpty()) { + $videosResponse = $this->get($account, "{$api}/videos", [ + 'part' => 'snippet,contentDetails', + 'id' => $videoIds->implode(','), + 'maxResults' => self::PAGE_SIZE, + ]); + $videos = collect((array) $videosResponse->json('items', []))->keyBy('id'); + } + + $publications = []; + $crossedCutoff = false; + $providerLimited = false; + + foreach ($videoIds as $videoId) { + $row = $videos->get($videoId); + + if (! is_array($row)) { + $providerLimited = true; + + continue; + } + + $publishedAt = $this->publishedAt(data_get($row, 'snippet.publishedAt')); + + if (! $publishedAt) { + $providerLimited = true; + + continue; + } + + if ($publishedAt->lessThan($cutoff)) { + $crossedCutoff = true; + + break; + } + + $thumbnail = data_get($row, 'snippet.thumbnails.maxres.url') + ?: data_get($row, 'snippet.thumbnails.high.url') + ?: data_get($row, 'snippet.thumbnails.medium.url') + ?: data_get($row, 'snippet.thumbnails.default.url'); + + $publications[] = new DiscoveredPublication( + providerPostId: (string) $videoId, + publishedAt: $publishedAt, + contentType: PublicationContentType::Video, + providerContentType: 'video', + permalink: "https://www.youtube.com/watch?v={$videoId}", + excerpt: data_get($row, 'snippet.description') ?: data_get($row, 'snippet.title'), + previewMetadata: filled($thumbnail) ? ['thumbnail_url' => $thumbnail] : null, + providerMetadata: [ + 'duration' => data_get($row, 'contentDetails.duration'), + 'short_classification' => 'unknown', + ], + ); + } + + $nextCursor = data_get($playlist->json(), 'nextPageToken'); + $hasNext = is_string($nextCursor) && $nextCursor !== '' && ! $crossedCutoff; + + return new PublicationPage($publications, $hasNext ? $nextCursor : null, ! $hasNext, $providerLimited); + } +} diff --git a/tests/Feature/Analytics/Collectors/MediaPublicationCollectorsTest.php b/tests/Feature/Analytics/Collectors/MediaPublicationCollectorsTest.php new file mode 100644 index 000000000..b2c4599dd --- /dev/null +++ b/tests/Feature/Analytics/Collectors/MediaPublicationCollectorsTest.php @@ -0,0 +1,213 @@ + Http::response([ + 'data' => [[ + 'id' => 'tweet-1', + 'text' => 'A paid read', + 'created_at' => '2026-09-20T12:00:00.000Z', + 'attachments' => ['media_keys' => ['media-1']], + ]], + 'includes' => ['media' => [[ + 'media_key' => 'media-1', + 'type' => 'photo', + 'url' => 'https://cdn.example/tweet.jpg', + ]]], + 'meta' => ['next_token' => 'x-next'], + ])]); + $account = SocialAccount::factory()->create([ + 'platform' => Platform::X, + 'username' => 'example', + ]); + + $page = app(XPublicationCollector::class)->page( + $account, + 'x-cursor', + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->publications)->toHaveCount(1) + ->and($page->publications[0]->contentType)->toBe(PublicationContentType::Image) + ->and($page->publications[0]->permalink)->toBe('https://x.com/example/status/tweet-1') + ->and($page->nextCursor)->toBe('x-next') + ->and($page->providerExhausted)->toBeFalse(); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), "/users/{$account->platform_user_id}/tweets") + && $request['pagination_token'] === 'x-cursor' + && $request['tweet.fields'] === 'created_at,attachments' + && ! str_contains((string) $request['tweet.fields'], 'public_metrics')); +}); + +test('pinterest reads one pin page and records lifetime metric semantics', function () { + Http::fake(['*' => Http::response([ + 'items' => [[ + 'id' => 'pin-1', + 'created_at' => '2026-09-20T12:00:00Z', + 'title' => 'A pin', + 'description' => 'Pin description', + 'media' => [ + 'media_type' => 'video', + 'images' => ['600x' => ['url' => 'https://cdn.example/pin.jpg']], + ], + ]], + 'bookmark' => 'pin-next', + ])]); + $account = SocialAccount::factory()->create(['platform' => Platform::Pinterest]); + + $page = app(PinterestPublicationCollector::class)->page( + $account, + 'pin-cursor', + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->publications)->toHaveCount(1) + ->and($page->publications[0]->contentType)->toBe(PublicationContentType::Video) + ->and($page->publications[0]->providerMetadata)->toMatchArray([ + 'metric_time_basis' => MetricTimeBasis::Lifetime->value, + ]) + ->and($page->nextCursor)->toBe('pin-next'); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/v5/pins') + && $request['bookmark'] === 'pin-cursor'); +}); + +test('youtube pages the uploads playlist and hydrates videos in one batch without inferring shorts', function () { + $api = config('trypost.platforms.youtube.data_api'); + Http::fake([ + "{$api}/channels*" => Http::response([ + 'items' => [['contentDetails' => ['relatedPlaylists' => ['uploads' => 'uploads-1']]]], + ]), + "{$api}/playlistItems*" => Http::response([ + 'items' => [ + ['contentDetails' => ['videoId' => 'video-1']], + ['contentDetails' => ['videoId' => 'video-2']], + ], + 'nextPageToken' => 'youtube-next', + ]), + "{$api}/videos*" => Http::response([ + 'items' => [ + [ + 'id' => 'video-1', + 'snippet' => [ + 'title' => 'Short-shaped upload', + 'description' => 'Still not authoritatively a Short', + 'publishedAt' => '2026-09-20T12:00:00Z', + 'thumbnails' => ['medium' => ['url' => 'https://cdn.example/video-1.jpg']], + ], + 'contentDetails' => ['duration' => 'PT20S'], + ], + [ + 'id' => 'video-2', + 'snippet' => [ + 'title' => 'Long upload', + 'publishedAt' => '2026-09-19T12:00:00Z', + ], + 'contentDetails' => ['duration' => 'PT20M'], + ], + ], + ]), + ]); + $account = SocialAccount::factory()->create(['platform' => Platform::YouTube]); + + $page = app(YouTubePublicationCollector::class)->page( + $account, + 'youtube-cursor', + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->publications)->toHaveCount(2) + ->and(array_column($page->publications, 'contentType'))->toBe([ + PublicationContentType::Video, + PublicationContentType::Video, + ]) + ->and($page->nextCursor)->toBe('youtube-next'); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/playlistItems') + && $request['playlistId'] === 'uploads-1' + && $request['pageToken'] === 'youtube-cursor'); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/videos') + && $request['id'] === 'video-1,video-2'); +}); + +test('tiktok reads at most twenty videos and marks expiring covers', function () { + Http::fake(['*' => Http::response(['data' => [ + 'videos' => [[ + 'id' => 'video-1', + 'title' => 'A TikTok', + 'create_time' => 1_790_000_000, + 'cover_image_url' => 'https://cdn.example/cover.jpg?x-expires=1790003600', + 'share_url' => 'https://www.tiktok.com/@example/video/video-1', + ]], + 'cursor' => 1_790_000_000_000, + 'has_more' => true, + ]])]); + $account = SocialAccount::factory()->create([ + 'platform' => Platform::TikTok, + 'scopes' => ['video.list'], + ]); + + $page = app(TikTokPublicationCollector::class)->page( + $account, + '1789000000000', + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->publications)->toHaveCount(1) + ->and($page->publications[0]->contentType)->toBe(PublicationContentType::Video) + ->and($page->publications[0]->previewMetadata)->toMatchArray([ + 'thumbnail_url' => 'https://cdn.example/cover.jpg?x-expires=1790003600', + 'expires_at' => CarbonImmutable::createFromTimestampUTC(1_790_003_600)->toIso8601String(), + ]) + ->and($page->nextCursor)->toBe('1790000000000'); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/video/list/') + && $request['max_count'] === 20 + && $request['cursor'] === 1_789_000_000_000); +}); + +test('tiktok reports provider limited coverage when video list scope is missing', function () { + Http::fake(); + $account = SocialAccount::factory()->create([ + 'platform' => Platform::TikTok, + 'scopes' => ['video.publish'], + ]); + + $page = app(TikTokPublicationCollector::class)->page( + $account, + null, + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->publications)->toBe([]) + ->and($page->nextCursor)->toBeNull() + ->and($page->providerExhausted)->toBeTrue() + ->and($page->providerLimited)->toBeTrue(); + Http::assertNothingSent(); +}); + +test('publication collector factory supports media networks', function (Platform $platform, string $collector) { + $account = SocialAccount::factory()->create(['platform' => $platform]); + + expect(app(PublicationHistoryCollectorFactory::class)->for($account))->toBeInstanceOf($collector); +})->with([ + [Platform::X, XPublicationCollector::class], + [Platform::Pinterest, PinterestPublicationCollector::class], + [Platform::YouTube, YouTubePublicationCollector::class], + [Platform::TikTok, TikTokPublicationCollector::class], +]); From f6e6166c1407c338979c05170e1312aefbd1efbe Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 12:09:03 -0300 Subject: [PATCH 20/77] feat: discover open network publications --- app/Dto/Analytics/PublicationPage.php | 1 + .../Controllers/Auth/MastodonController.php | 2 +- .../AbstractApiPublicationCollector.php | 26 ++- .../BlueskyPublicationCollector.php | 144 ++++++++++++ .../MastodonPublicationCollector.php | 130 +++++++++++ .../PublicationHistoryCollectorFactory.php | 2 + app/Services/Social/BlueskyLexicon.php | 2 + .../OpenPublicationCollectorsTest.php | 210 ++++++++++++++++++ .../Feature/Social/MastodonControllerTest.php | 10 +- 9 files changed, 513 insertions(+), 14 deletions(-) create mode 100644 app/Services/Analytics/Collectors/Publications/BlueskyPublicationCollector.php create mode 100644 app/Services/Analytics/Collectors/Publications/MastodonPublicationCollector.php create mode 100644 tests/Feature/Analytics/Collectors/OpenPublicationCollectorsTest.php diff --git a/app/Dto/Analytics/PublicationPage.php b/app/Dto/Analytics/PublicationPage.php index c1193f5e9..33a42c9bb 100644 --- a/app/Dto/Analytics/PublicationPage.php +++ b/app/Dto/Analytics/PublicationPage.php @@ -14,5 +14,6 @@ public function __construct( public ?string $nextCursor, public bool $providerExhausted, public bool $providerLimited = false, + public ?string $partialReason = null, ) {} } diff --git a/app/Http/Controllers/Auth/MastodonController.php b/app/Http/Controllers/Auth/MastodonController.php index f31550200..91da1de30 100644 --- a/app/Http/Controllers/Auth/MastodonController.php +++ b/app/Http/Controllers/Auth/MastodonController.php @@ -20,7 +20,7 @@ class MastodonController extends SocialController { protected SocialPlatform $platform = SocialPlatform::Mastodon; - private const SCOPES = 'read:accounts write:statuses write:media'; + private const SCOPES = 'read:accounts read:statuses write:statuses write:media'; /** * Show form to enter Mastodon instance URL diff --git a/app/Services/Analytics/Collectors/Publications/AbstractApiPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/AbstractApiPublicationCollector.php index 0e31aea52..7c0bf4fad 100644 --- a/app/Services/Analytics/Collectors/Publications/AbstractApiPublicationCollector.php +++ b/app/Services/Analytics/Collectors/Publications/AbstractApiPublicationCollector.php @@ -7,6 +7,7 @@ use App\Exceptions\Analytics\AnalyticsCollectionException; use App\Models\SocialAccount; use Carbon\CarbonImmutable; +use Illuminate\Http\Client\PendingRequest; use Illuminate\Http\Client\Response; use Illuminate\Support\Facades\Http; use Throwable; @@ -14,11 +15,13 @@ abstract class AbstractApiPublicationCollector { /** @param array $query */ - protected function get(SocialAccount $account, string $url, array $query = []): Response - { - $response = Http::acceptJson() - ->withToken($account->access_token) - ->timeout(120) + protected function get( + SocialAccount $account, + string $url, + array $query = [], + bool $authenticated = true, + ): Response { + $response = $this->client($account, $authenticated) ->get($url, array_filter($query, fn (mixed $value): bool => $value !== null && $value !== '')); return $this->successfulResponse($response); @@ -27,10 +30,8 @@ protected function get(SocialAccount $account, string $url, array $query = []): /** @param array $payload */ protected function post(SocialAccount $account, string $url, array $payload): Response { - $response = Http::acceptJson() + $response = $this->client($account, true) ->asJson() - ->withToken($account->access_token) - ->timeout(120) ->post($url, $payload); return $this->successfulResponse($response); @@ -53,7 +54,7 @@ protected function publishedAt(mixed $value, bool $timestamp = false): ?CarbonIm } } - private function successfulResponse(Response $response): Response + protected function successfulResponse(Response $response): Response { if ($response->successful()) { return $response; @@ -76,6 +77,13 @@ private function successfulResponse(Response $response): Response ); } + private function client(SocialAccount $account, bool $authenticated): PendingRequest + { + $client = Http::acceptJson()->timeout(120); + + return $authenticated ? $client->withToken($account->access_token) : $client; + } + private function retryAt(Response $response): ?CarbonImmutable { $retryAfter = $response->header('Retry-After'); diff --git a/app/Services/Analytics/Collectors/Publications/BlueskyPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/BlueskyPublicationCollector.php new file mode 100644 index 000000000..a3fe839fb --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/BlueskyPublicationCollector.php @@ -0,0 +1,144 @@ +meta, + 'service', + config('trypost.platforms.bluesky.default_service'), + ), '/'); + $response = $this->get( + $account, + "{$pds}/xrpc/".BlueskyLexicon::LIST_RECORDS, + [ + 'repo' => $account->platform_user_id, + 'collection' => BlueskyLexicon::FEED_POST, + 'limit' => self::PAGE_SIZE, + 'cursor' => $cursor, + 'reverse' => true, + ], + authenticated: false, + ); + $records = collect((array) $response->json('records', [])); + $hydrated = $this->hydrate($account, $records->pluck('uri')->filter()->values()); + $publications = []; + $crossedCutoff = false; + + foreach ($records as $record) { + $publishedAt = $this->publishedAt(data_get($record, 'value.createdAt')); + + if (! $publishedAt) { + continue; + } + + if ($publishedAt->lessThan($cutoff)) { + $crossedCutoff = true; + + break; + } + + $uri = (string) data_get($record, 'uri'); + $postId = basename($uri); + $view = $hydrated->get($uri); + + $publications[] = new DiscoveredPublication( + providerPostId: $postId, + publishedAt: $publishedAt, + contentType: $this->contentType((array) data_get($record, 'value.embed', [])), + providerContentType: data_get($record, 'value.embed.$type'), + permalink: filled($account->username) + ? "https://bsky.app/profile/{$account->username}/post/{$postId}" + : "https://bsky.app/profile/{$account->platform_user_id}/post/{$postId}", + excerpt: data_get($record, 'value.text'), + previewMetadata: $this->preview(is_array($view) ? $view : []), + providerMetadata: $this->providerMetadata(is_array($view) ? $view : []), + ); + } + + $nextCursor = data_get($response->json(), 'cursor'); + $hasNext = is_string($nextCursor) && $nextCursor !== '' && ! $crossedCutoff; + + return new PublicationPage($publications, $hasNext ? $nextCursor : null, ! $hasNext); + } + + /** + * @param Collection $uris + * @return Collection> + */ + private function hydrate(SocialAccount $account, Collection $uris): Collection + { + $appView = rtrim((string) config('trypost.platforms.bluesky.public_appview'), '/'); + $posts = collect(); + + foreach ($uris->chunk(self::HYDRATION_BATCH_SIZE) as $batch) { + $response = $this->get( + $account, + "{$appView}/xrpc/".BlueskyLexicon::GET_POSTS, + ['uris' => $batch->values()->all()], + authenticated: false, + ); + + $posts->push(...(array) $response->json('posts', [])); + } + + return $posts->filter(fn (mixed $post): bool => is_array($post))->keyBy('uri'); + } + + /** @param array $embed */ + private function contentType(array $embed): PublicationContentType + { + $type = (string) data_get($embed, '$type'); + $media = str_ends_with($type, 'recordWithMedia') ? (array) data_get($embed, 'media', []) : $embed; + $mediaType = (string) data_get($media, '$type'); + + return match (true) { + str_ends_with($mediaType, 'video') => PublicationContentType::Video, + str_ends_with($mediaType, 'gallery'), count((array) data_get($media, 'images', [])) > 1 => PublicationContentType::Carousel, + str_ends_with($mediaType, 'images') => PublicationContentType::Image, + str_ends_with($type, 'external') => PublicationContentType::Link, + default => PublicationContentType::Text, + }; + } + + /** @param array $view */ + private function preview(array $view): ?array + { + $thumbnail = data_get($view, 'embed.images.0.thumb') + ?: data_get($view, 'embed.thumbnail') + ?: data_get($view, 'embed.media.images.0.thumb') + ?: data_get($view, 'embed.media.thumbnail'); + + return filled($thumbnail) ? ['thumbnail_url' => $thumbnail] : null; + } + + /** @param array $view */ + private function providerMetadata(array $view): array + { + return [ + 'like_count' => (int) data_get($view, 'likeCount', 0), + 'repost_count' => (int) data_get($view, 'repostCount', 0), + 'reply_count' => (int) data_get($view, 'replyCount', 0), + 'quote_count' => (int) data_get($view, 'quoteCount', 0), + 'public_metrics_available' => $view !== [], + ]; + } +} diff --git a/app/Services/Analytics/Collectors/Publications/MastodonPublicationCollector.php b/app/Services/Analytics/Collectors/Publications/MastodonPublicationCollector.php new file mode 100644 index 000000000..c0b1c1a16 --- /dev/null +++ b/app/Services/Analytics/Collectors/Publications/MastodonPublicationCollector.php @@ -0,0 +1,130 @@ +scopes ?? [])) > 0; + $instance = rtrim((string) data_get( + $account->meta, + 'instance', + config('trypost.platforms.mastodon.default_instance'), + ), '/'); + $response = $this->get( + $account, + "{$instance}/api/v1/accounts/{$account->platform_user_id}/statuses", + [ + 'limit' => self::PAGE_SIZE, + 'exclude_reblogs' => true, + 'max_id' => $cursor, + ], + authenticated: $hasPrivateHistoryScope, + ); + $rows = (array) $response->json(); + $publications = []; + $crossedCutoff = false; + + foreach ($rows as $row) { + $publishedAt = $this->publishedAt(data_get($row, 'created_at')); + + if (! $publishedAt) { + continue; + } + + if ($publishedAt->lessThan($cutoff)) { + $crossedCutoff = true; + + break; + } + + $attachments = (array) data_get($row, 'media_attachments', []); + $thumbnail = data_get($attachments, '0.preview_url') ?: data_get($attachments, '0.url'); + + $publications[] = new DiscoveredPublication( + providerPostId: (string) data_get($row, 'id'), + publishedAt: $publishedAt, + contentType: $this->contentType($row, $attachments), + providerContentType: data_get($attachments, '0.type') ?: (data_get($row, 'poll') ? 'poll' : 'status'), + permalink: data_get($row, 'url'), + excerpt: $this->plainText((string) data_get($row, 'content', '')), + previewMetadata: filled($thumbnail) ? ['thumbnail_url' => $thumbnail] : null, + providerMetadata: [ + 'visibility' => data_get($row, 'visibility'), + 'favourites_count' => (int) data_get($row, 'favourites_count', 0), + 'reblogs_count' => (int) data_get($row, 'reblogs_count', 0), + 'replies_count' => (int) data_get($row, 'replies_count', 0), + 'history_visibility' => $hasPrivateHistoryScope ? 'authorized' : 'public_only', + 'reconnect_required' => ! $hasPrivateHistoryScope, + ], + ); + } + + $nextCursor = $crossedCutoff ? null : $this->nextCursor($response, $rows); + + return new PublicationPage( + publications: $publications, + nextCursor: $nextCursor, + providerExhausted: $crossedCutoff || $nextCursor === null, + partialReason: $hasPrivateHistoryScope ? null : self::PARTIAL_REASON, + ); + } + + /** @param array $row @param array $attachments */ + private function contentType(array $row, array $attachments): PublicationContentType + { + $types = collect($attachments)->pluck('type'); + + return match (true) { + $types->contains(fn (mixed $type): bool => in_array($type, ['video', 'gifv'], true)) => PublicationContentType::Video, + count($attachments) > 1 => PublicationContentType::Carousel, + $types->contains('image') => PublicationContentType::Image, + data_get($row, 'poll') !== null => PublicationContentType::Poll, + filled(data_get($row, 'card.url')) => PublicationContentType::Link, + default => PublicationContentType::Text, + }; + } + + /** @param array $rows */ + private function nextCursor(Response $response, array $rows): ?string + { + $link = $response->header('Link'); + + if (is_string($link) && preg_match('/<([^>]+)>;\s*rel="next"/', $link, $matches) === 1) { + parse_str((string) parse_url($matches[1], PHP_URL_QUERY), $query); + $cursor = $query['max_id'] ?? null; + + if (is_scalar($cursor) && (string) $cursor !== '') { + return (string) $cursor; + } + } + + if (count($rows) < self::PAGE_SIZE) { + return null; + } + + $lastId = data_get($rows, (count($rows) - 1).'.id'); + + return is_scalar($lastId) && (string) $lastId !== '' ? (string) $lastId : null; + } + + private function plainText(string $html): string + { + return trim(html_entity_decode(strip_tags($html), ENT_QUOTES | ENT_HTML5)); + } +} diff --git a/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php b/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php index 81b9a5401..b095c893b 100644 --- a/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php +++ b/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php @@ -21,6 +21,8 @@ public function for(SocialAccount $account): PublicationHistoryCollector Platform::Pinterest => app(PinterestPublicationCollector::class), Platform::YouTube => app(YouTubePublicationCollector::class), Platform::TikTok => app(TikTokPublicationCollector::class), + Platform::Bluesky => app(BlueskyPublicationCollector::class), + Platform::Mastodon => app(MastodonPublicationCollector::class), default => throw AnalyticsCollectionException::unsupported("{$account->platform->value} publication history is not supported"), }; } diff --git a/app/Services/Social/BlueskyLexicon.php b/app/Services/Social/BlueskyLexicon.php index ae56ad848..99ef78c71 100644 --- a/app/Services/Social/BlueskyLexicon.php +++ b/app/Services/Social/BlueskyLexicon.php @@ -15,6 +15,8 @@ final class BlueskyLexicon public const CREATE_RECORD = 'com.atproto.repo.createRecord'; + public const LIST_RECORDS = 'com.atproto.repo.listRecords'; + public const UPLOAD_BLOB = 'com.atproto.repo.uploadBlob'; public const CREATE_SESSION = 'com.atproto.server.createSession'; diff --git a/tests/Feature/Analytics/Collectors/OpenPublicationCollectorsTest.php b/tests/Feature/Analytics/Collectors/OpenPublicationCollectorsTest.php new file mode 100644 index 000000000..d42144e29 --- /dev/null +++ b/tests/Feature/Analytics/Collectors/OpenPublicationCollectorsTest.php @@ -0,0 +1,210 @@ + Http::response([ + 'records' => [[ + 'uri' => $uri, + 'cid' => 'cid-1', + 'value' => [ + '$type' => 'app.bsky.feed.post', + 'text' => 'Hello from the repo', + 'createdAt' => '2026-09-20T12:00:00Z', + 'embed' => [ + '$type' => 'app.bsky.embed.images', + 'images' => [['image' => ['$type' => 'blob']]], + ], + ], + ]], + 'cursor' => 'repo-next', + ]), + "{$appView}/xrpc/app.bsky.feed.getPosts*" => Http::response([ + 'posts' => [[ + 'uri' => $uri, + 'likeCount' => 12, + 'repostCount' => 3, + 'replyCount' => 4, + 'quoteCount' => 2, + 'embed' => ['images' => [['thumb' => 'https://cdn.example/post.jpg']]], + ]], + ]), + ]); + $account = SocialAccount::factory()->bluesky()->create([ + 'platform_user_id' => 'did:plc:alice', + 'username' => 'alice.bsky.social', + 'meta' => ['service' => $pds], + ]); + + $page = app(BlueskyPublicationCollector::class)->page( + $account, + 'repo-cursor', + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->publications)->toHaveCount(1) + ->and($page->publications[0]->providerPostId)->toBe('post-1') + ->and($page->publications[0]->contentType)->toBe(PublicationContentType::Image) + ->and($page->publications[0]->providerMetadata)->toMatchArray([ + 'like_count' => 12, + 'repost_count' => 3, + 'reply_count' => 4, + 'quote_count' => 2, + ]) + ->and($page->nextCursor)->toBe('repo-next'); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/com.atproto.repo.listRecords') + && $request['repo'] === 'did:plc:alice' + && $request['collection'] === 'app.bsky.feed.post' + && $request['reverse'] === true + && $request['cursor'] === 'repo-cursor'); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/app.bsky.feed.getPosts')); +}); + +test('bluesky respects the twenty five uri hydration limit', function () { + $records = collect(range(1, 26))->map(fn (int $number): array => [ + 'uri' => "at://did:plc:alice/app.bsky.feed.post/post-{$number}", + 'cid' => "cid-{$number}", + 'value' => [ + '$type' => 'app.bsky.feed.post', + 'text' => "Post {$number}", + 'createdAt' => '2026-09-20T12:00:00Z', + ], + ])->all(); + Http::fake(function (Request $request) use ($records) { + if (str_contains($request->url(), '/com.atproto.repo.listRecords')) { + return Http::response(['records' => $records]); + } + + return Http::response(['posts' => collect($request['uris'])->map(fn (string $uri): array => [ + 'uri' => $uri, + ])->all()]); + }); + $account = SocialAccount::factory()->bluesky()->create([ + 'platform_user_id' => 'did:plc:alice', + 'meta' => ['service' => 'https://pds.example'], + ]); + + $page = app(BlueskyPublicationCollector::class)->page( + $account, + null, + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + $hydrationRequests = Http::recorded( + fn (Request $request): bool => str_contains($request->url(), '/app.bsky.feed.getPosts'), + )->values(); + + expect($page->publications)->toHaveCount(26) + ->and($hydrationRequests)->toHaveCount(2) + ->and(count($hydrationRequests[0][0]['uris']))->toBe(25) + ->and(count($hydrationRequests[1][0]['uris']))->toBe(1); +}); + +test('mastodon pages account statuses with authenticated private history', function () { + Http::fake(['*' => Http::response([ + [ + 'id' => 'status-2', + 'created_at' => '2026-09-20T12:00:00Z', + 'content' => '

Hello Mastodon

', + 'url' => 'https://mastodon.example/@alice/status-2', + 'visibility' => 'private', + 'favourites_count' => 5, + 'reblogs_count' => 2, + 'replies_count' => 1, + 'media_attachments' => [[ + 'type' => 'image', + 'preview_url' => 'https://cdn.example/status.jpg', + ]], + ], + ], 200, [ + 'Link' => '; rel="next"', + ])]); + $account = SocialAccount::factory()->mastodon()->create([ + 'platform_user_id' => '42', + 'scopes' => ['read:accounts', 'read:statuses', 'write:statuses', 'write:media'], + 'meta' => ['instance' => 'https://mastodon.example'], + ]); + + $page = app(MastodonPublicationCollector::class)->page( + $account, + 'status-3', + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->publications)->toHaveCount(1) + ->and($page->publications[0]->contentType)->toBe(PublicationContentType::Image) + ->and($page->publications[0]->excerpt)->toBe('Hello Mastodon') + ->and($page->nextCursor)->toBe('status-1') + ->and($page->partialReason)->toBeNull(); + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), '/api/v1/accounts/42/statuses') + && $request['limit'] === 40 + && $request['exclude_reblogs'] === true + && $request['max_id'] === 'status-3' + && $request->hasHeader('Authorization', 'Bearer '.$account->access_token)); +}); + +test('mastodon imports public history as partial when an existing token lacks read statuses', function () { + Http::fake(['*' => Http::response([])]); + $account = SocialAccount::factory()->mastodon()->create([ + 'platform_user_id' => '42', + 'scopes' => ['read:accounts', 'write:statuses', 'write:media'], + 'meta' => ['instance' => 'https://mastodon.example'], + ]); + + $page = app(MastodonPublicationCollector::class)->page( + $account, + null, + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->providerExhausted)->toBeTrue() + ->and($page->providerLimited)->toBeFalse() + ->and($page->partialReason)->toBe('mastodon_reconnect_for_private_history'); + Http::assertSent(fn (Request $request): bool => ! $request->hasHeader('Authorization')); +}); + +test('mastodon parent read scope authorizes complete private history', function () { + Http::fake(['*' => Http::response([])]); + $account = SocialAccount::factory()->mastodon()->create([ + 'platform_user_id' => '42', + 'scopes' => ['read', 'write:statuses'], + 'meta' => ['instance' => 'https://mastodon.example'], + ]); + + $page = app(MastodonPublicationCollector::class)->page( + $account, + null, + CarbonImmutable::parse('2026-01-01', 'UTC'), + ); + + expect($page->partialReason)->toBeNull(); + Http::assertSent(fn (Request $request): bool => $request->hasHeader('Authorization', 'Bearer '.$account->access_token)); +}); + +test('publication collector factory supports open networks', function (Platform $platform, string $collector) { + $account = SocialAccount::factory()->create(['platform' => $platform]); + + expect(app(PublicationHistoryCollectorFactory::class)->for($account))->toBeInstanceOf($collector); +})->with([ + [Platform::Bluesky, BlueskyPublicationCollector::class], + [Platform::Mastodon, MastodonPublicationCollector::class], +]); diff --git a/tests/Feature/Social/MastodonControllerTest.php b/tests/Feature/Social/MastodonControllerTest.php index 768c3db9e..0c7212a91 100644 --- a/tests/Feature/Social/MastodonControllerTest.php +++ b/tests/Feature/Social/MastodonControllerTest.php @@ -8,6 +8,7 @@ use App\Models\SocialAccount; use App\Models\User; use App\Models\Workspace; +use Illuminate\Http\Client\Request as ClientRequest; use Illuminate\Support\Facades\Http; use Inertia\Testing\AssertableInertia; @@ -46,6 +47,7 @@ expect(session('mastodon_instance'))->toBe('https://mastodon.social'); expect(session('mastodon_client_id'))->toBe('test-client-id'); expect(session('mastodon_client_secret'))->toBe('test-client-secret'); + Http::assertSent(fn (ClientRequest $request): bool => $request['scopes'] === 'read:accounts read:statuses write:statuses write:media'); }); test('user cannot connect to invalid mastodon instance', function () { @@ -75,7 +77,7 @@ 'https://mastodon.social/oauth/token' => Http::response([ 'access_token' => 'test-access-token', 'token_type' => 'Bearer', - 'scope' => 'read:accounts write:statuses write:media', + 'scope' => 'read:accounts read:statuses write:statuses write:media', 'created_at' => time(), ], 200), 'https://mastodon.social/api/v1/accounts/verify_credentials' => Http::response([ @@ -105,7 +107,7 @@ ]); $account = SocialAccount::where('platform', Platform::Mastodon->value)->first(); - expect($account->scopes)->toBe(['read:accounts', 'write:statuses', 'write:media']); + expect($account->scopes)->toBe(['read:accounts', 'read:statuses', 'write:statuses', 'write:media']); }); test('mastodon callback fails with invalid state', function () { @@ -235,7 +237,7 @@ 'https://mastodon.social/oauth/token' => Http::response([ 'access_token' => 'fresh-access-token', 'token_type' => 'Bearer', - 'scope' => 'read:accounts write:statuses write:media', + 'scope' => 'read:accounts read:statuses write:statuses write:media', 'created_at' => time(), ], 200), 'https://mastodon.social/api/v1/accounts/verify_credentials' => Http::response([ @@ -282,7 +284,7 @@ 'https://mastodon.social/oauth/token' => Http::response([ 'access_token' => 'other-access-token', 'token_type' => 'Bearer', - 'scope' => 'read:accounts write:statuses write:media', + 'scope' => 'read:accounts read:statuses write:statuses write:media', 'created_at' => time(), ], 200), 'https://mastodon.social/api/v1/accounts/verify_credentials' => Http::response([ From 217e266e1f35a5430c1bbfeec8f0381d7457a219 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 12:19:37 -0300 Subject: [PATCH 21/77] feat: backfill native analytics through jobs --- .../Analytics/AdvanceAnalyticsSyncState.php | 201 +++++++++++++ .../Analytics/BackfillExistingAnalytics.php | 59 ++++ .../DispatchPublicationDiscovery.php | 44 +++ app/Enums/SocialAccount/Platform.php | 18 ++ .../Analytics/BackfillAccountPublications.php | 110 ++++++++ .../Analytics/BackfillTryPostPublications.php | 33 +++ .../Analytics/BootstrapAccountAnalytics.php | 73 +++++ .../Analytics/DiscoverAccountPublications.php | 121 ++++++++ app/Models/AnalyticsSyncState.php | 26 ++ app/Models/PostPlatform.php | 5 + app/Models/SocialAccount.php | 10 + app/Observers/SocialAccountObserver.php | 17 +- app/Providers/AppServiceProvider.php | 5 + .../Followers/FollowerCollectorFactory.php | 13 +- .../PublicationHistoryCollectorFactory.php | 5 + routes/console.php | 6 + .../Analytics/AnalyticsScheduleTest.php | 11 +- .../BackfillExistingAnalyticsCommandTest.php | 90 ++++++ .../Analytics/PublicationBackfillJobsTest.php | 264 ++++++++++++++++++ .../Observers/SocialAccountObserverTest.php | 4 + 20 files changed, 1096 insertions(+), 19 deletions(-) create mode 100644 app/Actions/Analytics/AdvanceAnalyticsSyncState.php create mode 100644 app/Console/Commands/Analytics/BackfillExistingAnalytics.php create mode 100644 app/Console/Commands/Analytics/DispatchPublicationDiscovery.php create mode 100644 app/Jobs/Analytics/BackfillAccountPublications.php create mode 100644 app/Jobs/Analytics/BackfillTryPostPublications.php create mode 100644 app/Jobs/Analytics/BootstrapAccountAnalytics.php create mode 100644 app/Jobs/Analytics/DiscoverAccountPublications.php create mode 100644 tests/Feature/Analytics/BackfillExistingAnalyticsCommandTest.php create mode 100644 tests/Feature/Analytics/PublicationBackfillJobsTest.php diff --git a/app/Actions/Analytics/AdvanceAnalyticsSyncState.php b/app/Actions/Analytics/AdvanceAnalyticsSyncState.php new file mode 100644 index 000000000..94024f260 --- /dev/null +++ b/app/Actions/Analytics/AdvanceAnalyticsSyncState.php @@ -0,0 +1,201 @@ +lockForUpdate()->find($stateId); + + if (! $state || ($state->isTerminal() && ! $restartTerminal)) { + return null; + } + + $checkpoint = $state->checkpoint ?? []; + $revision = ((int) ($checkpoint['revision'] ?? 0)) + 1; + $cursor = $restartTerminal && $state->isTerminal() + ? null + : ($checkpoint['cursor'] ?? null); + + $state->update([ + 'status' => SyncStatus::Running, + 'checkpoint' => ['cursor' => $cursor, 'revision' => $revision], + 'last_error_category' => null, + ]); + + $cutoff = $state->collector === SyncCollector::PublicationBackfill + ? ($state->target_since ?? CarbonImmutable::now('UTC')->subDays(365)) + : ($state->high_watermark_at ?? CarbonImmutable::now('UTC'))->subDays(3); + + return [ + 'cursor' => is_string($cursor) && $cursor !== '' ? $cursor : null, + 'revision' => $revision, + 'cutoff' => $cutoff->toImmutable(), + ]; + }); + } + + /** + * Persist page facts even for a stale worker, but only let the worker that + * owns the current revision advance the provider cursor. + * + * @return array{advanced: bool, terminal: bool} + */ + public function handle( + string $stateId, + int $capturedRevision, + SocialAccount $account, + PublicationPage $page, + ): array { + return DB::transaction(function () use ($account, $capturedRevision, $page, $stateId): array { + $state = AnalyticsSyncState::query()->lockForUpdate()->find($stateId); + + if (! $state) { + return ['advanced' => false, 'terminal' => true]; + } + + foreach ($page->publications as $publication) { + $this->publications->external($account, $publication); + } + + $checkpoint = $state->checkpoint ?? []; + + if ((int) ($checkpoint['revision'] ?? 0) !== $capturedRevision) { + return ['advanced' => false, 'terminal' => $state->isTerminal()]; + } + + $publishedAt = collect($page->publications)->pluck('publishedAt'); + $pageOldest = $publishedAt->min(); + $pageNewest = $publishedAt->max(); + $oldest = $this->earlier($state->oldest_reached_at, $pageOldest); + $highWatermark = $this->later($state->high_watermark_at, $pageNewest); + $reachedTarget = $state->collector === SyncCollector::PublicationBackfill + && $oldest + && $state->target_since + && $oldest->lessThanOrEqualTo($state->target_since); + + $status = match (true) { + $page->providerLimited => SyncStatus::ProviderLimited, + filled($page->partialReason) && ($page->providerExhausted || $reachedTarget) => SyncStatus::Partial, + $page->providerExhausted || $reachedTarget => SyncStatus::Complete, + default => SyncStatus::Running, + }; + + $state->update([ + 'status' => $status, + 'checkpoint' => [ + 'cursor' => $status === SyncStatus::Running ? $page->nextCursor : null, + 'revision' => $capturedRevision, + ], + 'oldest_reached_at' => $oldest, + 'high_watermark_at' => $highWatermark, + 'last_success_at' => CarbonImmutable::now('UTC'), + 'last_error_category' => $page->providerLimited + ? 'provider_limited' + : $page->partialReason, + ]); + + if ($state->collector === SyncCollector::PublicationBackfill && $status !== SyncStatus::Running) { + $this->initializeDiscovery($account, $highWatermark); + } + + return ['advanced' => true, 'terminal' => $status !== SyncStatus::Running]; + }); + } + + public function recordFailure(string $stateId, int $capturedRevision, string $category, bool $terminal): void + { + DB::transaction(function () use ($capturedRevision, $category, $stateId, $terminal): void { + $state = AnalyticsSyncState::query()->lockForUpdate()->find($stateId); + + if (! $state || (int) data_get($state->checkpoint, 'revision', 0) !== $capturedRevision) { + return; + } + + $state->update([ + 'status' => $terminal ? SyncStatus::Failed : SyncStatus::Running, + 'last_error_category' => mb_substr($category, 0, 64), + ]); + }); + } + + public function resetInvalidCursor(string $stateId, int $capturedRevision): bool + { + return DB::transaction(function () use ($capturedRevision, $stateId): bool { + $state = AnalyticsSyncState::query()->lockForUpdate()->find($stateId); + + if (! $state || (int) data_get($state->checkpoint, 'revision', 0) !== $capturedRevision) { + return false; + } + + $state->update([ + 'status' => SyncStatus::Pending, + 'checkpoint' => ['cursor' => null, 'revision' => $capturedRevision], + 'last_error_category' => 'invalid_cursor', + ]); + + return true; + }); + } + + private function initializeDiscovery(SocialAccount $account, ?CarbonImmutable $highWatermark): void + { + $latest = AnalyticsPublication::query() + ->where('social_account_id', $account->id) + ->max('provider_published_at'); + $initialHighWatermark = $highWatermark + ?? ($latest ? CarbonImmutable::parse($latest, 'UTC') : CarbonImmutable::now('UTC')); + + $state = AnalyticsSyncState::query()->firstOrCreate([ + 'social_account_id' => $account->id, + 'collector' => SyncCollector::PublicationDiscovery, + ], [ + 'status' => SyncStatus::Pending, + 'checkpoint' => ['cursor' => null, 'revision' => 0], + ]); + + if (! $state->high_watermark_at || $initialHighWatermark->greaterThan($state->high_watermark_at)) { + $state->update(['high_watermark_at' => $initialHighWatermark]); + } + } + + private function earlier(?CarbonImmutable $current, mixed $candidate): ?CarbonImmutable + { + if (! $candidate) { + return $current; + } + + $candidate = CarbonImmutable::parse($candidate, 'UTC'); + + return ! $current || $candidate->lessThan($current) ? $candidate : $current; + } + + private function later(?CarbonImmutable $current, mixed $candidate): ?CarbonImmutable + { + if (! $candidate) { + return $current; + } + + $candidate = CarbonImmutable::parse($candidate, 'UTC'); + + return ! $current || $candidate->greaterThan($current) ? $candidate : $current; + } +} diff --git a/app/Console/Commands/Analytics/BackfillExistingAnalytics.php b/app/Console/Commands/Analytics/BackfillExistingAnalytics.php new file mode 100644 index 000000000..c356eac39 --- /dev/null +++ b/app/Console/Commands/Analytics/BackfillExistingAnalytics.php @@ -0,0 +1,59 @@ +option('workspace'); + $destinations = PostPlatform::query() + ->published() + ->includedInAnalytics() + ->whereNotNull('social_account_id') + ->when($workspaceId, fn ($query) => $query->whereHas( + 'post', + fn ($post) => $post->where('workspace_id', $workspaceId), + )); + + $destinations->lazyById(100) + ->chunk(100) + ->each(fn ($chunk) => BackfillTryPostPublications::dispatch( + $chunk->pluck('id')->map(fn ($id): string => (string) $id)->values()->all(), + )); + + $orphaned = PostPlatform::query() + ->published() + ->includedInAnalytics() + ->whereNull('social_account_id') + ->when($workspaceId, fn ($query) => $query->whereHas( + 'post', + fn ($post) => $post->where('workspace_id', $workspaceId), + )) + ->count(); + + SocialAccount::query() + ->connected() + ->active() + ->includedInAnalytics() + ->when($workspaceId, fn ($query) => $query->where('workspace_id', $workspaceId)) + ->lazyById(100) + ->each(fn (SocialAccount $account) => BootstrapAccountAnalytics::dispatch($account->id)); + + $this->info("historical_identity_unrecoverable={$orphaned}"); + + return self::SUCCESS; + } +} diff --git a/app/Console/Commands/Analytics/DispatchPublicationDiscovery.php b/app/Console/Commands/Analytics/DispatchPublicationDiscovery.php new file mode 100644 index 000000000..383e6c466 --- /dev/null +++ b/app/Console/Commands/Analytics/DispatchPublicationDiscovery.php @@ -0,0 +1,44 @@ +connected() + ->active() + ->includedInAnalytics() + ->lazyById(100) + ->each(function (SocialAccount $account): void { + $backfillIsTerminal = AnalyticsSyncState::query() + ->where('social_account_id', $account->id) + ->forCollector(SyncCollector::PublicationBackfill) + ->terminal() + ->exists(); + $discovery = AnalyticsSyncState::query() + ->where('social_account_id', $account->id) + ->forCollector(SyncCollector::PublicationDiscovery) + ->first(); + + if ($backfillIsTerminal && $discovery) { + DiscoverAccountPublications::dispatch($account->id, $discovery->id); + } + }); + + return self::SUCCESS; + } +} diff --git a/app/Enums/SocialAccount/Platform.php b/app/Enums/SocialAccount/Platform.php index c87258188..1c6b06dfa 100644 --- a/app/Enums/SocialAccount/Platform.php +++ b/app/Enums/SocialAccount/Platform.php @@ -34,6 +34,24 @@ public function network(): string }; } + public function isIncludedInAnalytics(): bool + { + return match ($this) { + self::LinkedIn, self::LinkedInPage, self::Telegram, + self::Discord, self::GoogleBusiness => false, + default => true, + }; + } + + /** @return list */ + public static function analyticsValues(): array + { + return array_values(array_map( + fn (self $platform): string => $platform->value, + array_filter(self::cases(), fn (self $platform): bool => $platform->isIncludedInAnalytics()), + )); + } + /** * @return array */ diff --git a/app/Jobs/Analytics/BackfillAccountPublications.php b/app/Jobs/Analytics/BackfillAccountPublications.php new file mode 100644 index 000000000..586cc7ce2 --- /dev/null +++ b/app/Jobs/Analytics/BackfillAccountPublications.php @@ -0,0 +1,110 @@ +onQueue('analytics'); + } + + /** @return list */ + public function middleware(): array + { + return [ + new RateLimited('analytics-publications'), + (new WithoutOverlapping("analytics-publication-backfill:{$this->socialAccountId}")) + ->releaseAfter(300) + ->expireAfter($this->timeout + 30), + ]; + } + + public function providerRateLimitKey(): string + { + $account = SocialAccount::query()->find($this->socialAccountId); + + return $account ? $account->platform->network() : 'missing'; + } + + public function backoff(): array + { + return [300, 3600, 7200, 10800, 14400]; + } + + public function handle( + AdvanceAnalyticsSyncState $sync, + PublicationHistoryCollectorFactory $collectors, + ): void { + $account = SocialAccount::query() + ->connected() + ->active() + ->includedInAnalytics() + ->find($this->socialAccountId); + $capture = $sync->begin($this->syncStateId); + + if (! $account || ! $capture) { + return; + } + + try { + $page = $collectors->for($account)->page($account, $capture['cursor'], $capture['cutoff']); + $result = $sync->handle($this->syncStateId, $capture['revision'], $account, $page); + } catch (AnalyticsCollectionException $exception) { + if ($exception->category === 'invalid_cursor') { + if ($sync->resetInvalidCursor($this->syncStateId, $capture['revision'])) { + self::dispatch($account->id, $this->syncStateId)->afterCommit(); + } + + return; + } + + $transient = in_array($exception->category, ['transient', 'rate_limited'], true); + $sync->recordFailure($this->syncStateId, $capture['revision'], $exception->category, ! $transient); + + if ($transient) { + throw $exception; + } + + return; + } + + if ($result['advanced'] && ! $result['terminal']) { + self::dispatch($account->id, $this->syncStateId)->afterCommit(); + } + } + + public function failed(?Throwable $exception): void + { + $state = AnalyticsSyncState::query()->find($this->syncStateId); + + if ($state && ! $state->isTerminal()) { + $state->update([ + 'status' => SyncStatus::Failed, + 'last_error_category' => 'queue_failed', + ]); + } + } +} diff --git a/app/Jobs/Analytics/BackfillTryPostPublications.php b/app/Jobs/Analytics/BackfillTryPostPublications.php new file mode 100644 index 000000000..58bd6094e --- /dev/null +++ b/app/Jobs/Analytics/BackfillTryPostPublications.php @@ -0,0 +1,33 @@ + $postPlatformIds */ + public function __construct(public array $postPlatformIds) + { + $this->onQueue('analytics'); + } + + public function handle(SyncTryPostPublication $sync): void + { + PostPlatform::query() + ->published() + ->includedInAnalytics() + ->whereIn('id', $this->postPlatformIds) + ->whereNotNull('social_account_id') + ->with(['post', 'socialAccount']) + ->get() + ->each(fn (PostPlatform $postPlatform) => $sync->handle($postPlatform)); + } +} diff --git a/app/Jobs/Analytics/BootstrapAccountAnalytics.php b/app/Jobs/Analytics/BootstrapAccountAnalytics.php new file mode 100644 index 000000000..7266b8dca --- /dev/null +++ b/app/Jobs/Analytics/BootstrapAccountAnalytics.php @@ -0,0 +1,73 @@ +onQueue('analytics'); + } + + public function handle(): void + { + $this->handleFor($this->socialAccountId); + } + + public function handleFor(string $socialAccountId): void + { + $account = SocialAccount::query() + ->connected() + ->active() + ->includedInAnalytics() + ->find($socialAccountId); + + if (! $account || ! app(PublicationHistoryCollectorFactory::class)->supports($account->platform)) { + return; + } + + $backfill = AnalyticsSyncState::query()->firstOrCreate([ + 'social_account_id' => $account->id, + 'collector' => SyncCollector::PublicationBackfill, + ], [ + 'status' => SyncStatus::Pending, + 'checkpoint' => ['cursor' => null, 'revision' => 0], + 'target_since' => CarbonImmutable::now('UTC')->subDays(365), + ]); + + AnalyticsSyncState::query()->firstOrCreate([ + 'social_account_id' => $account->id, + 'collector' => SyncCollector::PublicationDiscovery, + ], [ + 'status' => SyncStatus::Pending, + 'checkpoint' => ['cursor' => null, 'revision' => 0], + ]); + + if (in_array($backfill->status, [SyncStatus::Partial, SyncStatus::Failed], true)) { + $backfill->update([ + 'status' => SyncStatus::Pending, + 'checkpoint' => [ + 'cursor' => null, + 'revision' => (int) data_get($backfill->checkpoint, 'revision', 0), + ], + ]); + } + + if (! $backfill->isTerminal()) { + BackfillAccountPublications::dispatch($account->id, $backfill->id)->afterCommit(); + } + } +} diff --git a/app/Jobs/Analytics/DiscoverAccountPublications.php b/app/Jobs/Analytics/DiscoverAccountPublications.php new file mode 100644 index 000000000..7669fa446 --- /dev/null +++ b/app/Jobs/Analytics/DiscoverAccountPublications.php @@ -0,0 +1,121 @@ +onQueue('analytics'); + } + + /** @return list */ + public function middleware(): array + { + return [ + new RateLimited('analytics-publications'), + (new WithoutOverlapping("analytics-publication-discovery:{$this->socialAccountId}")) + ->releaseAfter(300) + ->expireAfter($this->timeout + 30), + ]; + } + + public function providerRateLimitKey(): string + { + $account = SocialAccount::query()->find($this->socialAccountId); + + return $account ? $account->platform->network() : 'missing'; + } + + public function backoff(): array + { + return [300, 3600, 7200, 10800, 14400]; + } + + public function handle( + AdvanceAnalyticsSyncState $sync, + PublicationHistoryCollectorFactory $collectors, + ): void { + $account = SocialAccount::query() + ->connected() + ->active() + ->includedInAnalytics() + ->find($this->socialAccountId); + + if (! $account || ! AnalyticsSyncState::query() + ->where('social_account_id', $account->id) + ->forCollector(SyncCollector::PublicationBackfill) + ->terminal() + ->exists()) { + return; + } + + $state = AnalyticsSyncState::query()->find($this->syncStateId); + $capture = $sync->begin($this->syncStateId, restartTerminal: $state?->isTerminal() ?? false); + + if (! $capture) { + return; + } + + try { + $page = $collectors->for($account)->page($account, $capture['cursor'], $capture['cutoff']); + $result = $sync->handle($this->syncStateId, $capture['revision'], $account, $page); + } catch (AnalyticsCollectionException $exception) { + if ($exception->category === 'invalid_cursor') { + if ($sync->resetInvalidCursor($this->syncStateId, $capture['revision'])) { + self::dispatch($account->id, $this->syncStateId)->afterCommit(); + } + + return; + } + + $transient = in_array($exception->category, ['transient', 'rate_limited'], true); + $sync->recordFailure($this->syncStateId, $capture['revision'], $exception->category, ! $transient); + + if ($transient) { + throw $exception; + } + + return; + } + + if ($result['advanced'] && ! $result['terminal']) { + self::dispatch($account->id, $this->syncStateId)->afterCommit(); + } + } + + public function failed(?Throwable $exception): void + { + $state = AnalyticsSyncState::query()->find($this->syncStateId); + + if ($state && ! $state->isTerminal()) { + $state->update([ + 'status' => SyncStatus::Failed, + 'last_error_category' => 'queue_failed', + ]); + } + } +} diff --git a/app/Models/AnalyticsSyncState.php b/app/Models/AnalyticsSyncState.php index 007b9152d..2e987aad7 100644 --- a/app/Models/AnalyticsSyncState.php +++ b/app/Models/AnalyticsSyncState.php @@ -7,6 +7,7 @@ use App\Enums\Analytics\SyncCollector; use App\Enums\Analytics\SyncStatus; use Database\Factories\AnalyticsSyncStateFactory; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Concerns\HasUuids; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; @@ -40,4 +41,29 @@ public function socialAccount(): BelongsTo { return $this->belongsTo(SocialAccount::class); } + + public function scopeForCollector(Builder $query, SyncCollector $collector): Builder + { + return $query->where('collector', $collector); + } + + public function scopeTerminal(Builder $query): Builder + { + return $query->whereIn('status', [ + SyncStatus::Complete, + SyncStatus::Partial, + SyncStatus::ProviderLimited, + SyncStatus::Failed, + ]); + } + + public function isTerminal(): bool + { + return in_array($this->status, [ + SyncStatus::Complete, + SyncStatus::Partial, + SyncStatus::ProviderLimited, + SyncStatus::Failed, + ], true); + } } diff --git a/app/Models/PostPlatform.php b/app/Models/PostPlatform.php index c0b0ffbf4..304e46934 100644 --- a/app/Models/PostPlatform.php +++ b/app/Models/PostPlatform.php @@ -90,6 +90,11 @@ public function scopePublished(Builder $query): Builder return $query->where('post_platforms.status', Status::Published); } + public function scopeIncludedInAnalytics(Builder $query): Builder + { + return $query->whereIn('post_platforms.platform', SocialPlatform::analyticsValues()); + } + /** * Get display name, falling back to snapshot if account was deleted. */ diff --git a/app/Models/SocialAccount.php b/app/Models/SocialAccount.php index de7e415c7..fe5042892 100644 --- a/app/Models/SocialAccount.php +++ b/app/Models/SocialAccount.php @@ -215,6 +215,11 @@ public function postPlatforms(): HasMany return $this->hasMany(PostPlatform::class); } + public function analyticsSyncStates(): HasMany + { + return $this->hasMany(AnalyticsSyncState::class); + } + protected function isTokenExpired(): Attribute { return Attribute::make( @@ -431,4 +436,9 @@ public function scopeConnected(Builder $query): Builder { return $query->where('status', Status::Connected); } + + public function scopeIncludedInAnalytics(Builder $query): Builder + { + return $query->whereIn('platform', SocialPlatform::analyticsValues()); + } } diff --git a/app/Observers/SocialAccountObserver.php b/app/Observers/SocialAccountObserver.php index ddd81ede1..e892f7b34 100644 --- a/app/Observers/SocialAccountObserver.php +++ b/app/Observers/SocialAccountObserver.php @@ -5,6 +5,7 @@ namespace App\Observers; use App\Enums\SocialAccount\Status; +use App\Jobs\Analytics\BootstrapAccountAnalytics; use App\Jobs\Analytics\CollectAccountDailySnapshot; use App\Jobs\PostHog\IdentifyConnectedPlatforms; use App\Jobs\PostHog\SyncAccountUsage; @@ -84,17 +85,21 @@ private function dispatchInitialAnalytics(SocialAccount $socialAccount): void $currentAccount = SocialAccount::query() ->connected() ->active() + ->includedInAnalytics() ->find($socialAccount->id); - if (! $currentAccount - || ! app(FollowerCollectorFactory::class)->supports($currentAccount->platform)) { + if (! $currentAccount) { return; } - CollectAccountDailySnapshot::dispatch( - $currentAccount->id, - CarbonImmutable::now('UTC')->toDateString(), - )->afterCommit(); + if (app(FollowerCollectorFactory::class)->supports($currentAccount->platform)) { + CollectAccountDailySnapshot::dispatch( + $currentAccount->id, + CarbonImmutable::now('UTC')->toDateString(), + )->afterCommit(); + } + + BootstrapAccountAnalytics::dispatch($currentAccount->id)->afterCommit(); } catch (Throwable $exception) { report($exception); } diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 040f423b7..0d36c96ea 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -146,6 +146,11 @@ protected function configureRateLimiting(): void fn (Request $request): Limit => Limit::perMinute(30)->by($request->ip()), ); + RateLimiter::for( + 'analytics-publications', + fn (object $job): Limit => Limit::perMinute(30)->by($job->providerRateLimitKey()), + ); + // Signed media uploads (api.uploads.store). MCP hosts share egress IPs // across tenants — key by workspace_id from the signed URL, with a high // IP backstop so one client cannot flood every workspace. diff --git a/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php b/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php index e26489e74..96fa1ae9f 100644 --- a/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php +++ b/app/Services/Analytics/Collectors/Followers/FollowerCollectorFactory.php @@ -12,18 +12,7 @@ class FollowerCollectorFactory { public function supports(Platform $platform): bool { - return in_array($platform, [ - Platform::Instagram, - Platform::InstagramFacebook, - Platform::Facebook, - Platform::Threads, - Platform::X, - Platform::Pinterest, - Platform::YouTube, - Platform::TikTok, - Platform::Bluesky, - Platform::Mastodon, - ], true); + return $platform->isIncludedInAnalytics(); } public function for(Platform $platform): FollowerCollector diff --git a/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php b/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php index b095c893b..282d14214 100644 --- a/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php +++ b/app/Services/Analytics/Collectors/Publications/PublicationHistoryCollectorFactory.php @@ -11,6 +11,11 @@ class PublicationHistoryCollectorFactory { + public function supports(Platform $platform): bool + { + return $platform->isIncludedInAnalytics(); + } + public function for(SocialAccount $account): PublicationHistoryCollector { return match ($account->platform) { diff --git a/routes/console.php b/routes/console.php index c5ebacb97..6101cd5f2 100644 --- a/routes/console.php +++ b/routes/console.php @@ -3,6 +3,7 @@ declare(strict_types=1); use App\Console\Commands\Analytics\DispatchAccountDailyAnalytics; +use App\Console\Commands\Analytics\DispatchPublicationDiscovery; use App\Console\Commands\CheckSocialConnections; use App\Console\Commands\CheckUpcomingPostConnections; use App\Console\Commands\ProcessScheduledPosts; @@ -27,6 +28,11 @@ ->timezone('UTC') ->withoutOverlapping() ->onOneServer(); +Schedule::command(DispatchPublicationDiscovery::class) + ->dailyAt('03:00') + ->timezone('UTC') + ->withoutOverlapping() + ->onOneServer(); Schedule::job(new FinalizeAccountDailySnapshots) ->dailyAt('23:30') ->timezone('UTC') diff --git a/tests/Feature/Analytics/AnalyticsScheduleTest.php b/tests/Feature/Analytics/AnalyticsScheduleTest.php index a47de2791..6062a7517 100644 --- a/tests/Feature/Analytics/AnalyticsScheduleTest.php +++ b/tests/Feature/Analytics/AnalyticsScheduleTest.php @@ -12,6 +12,10 @@ 'analytics:dispatch-account-daily', )); $finalizer = $events->first(fn ($event): bool => $event->description === FinalizeAccountDailySnapshots::class); + $discovery = $events->first(fn ($event): bool => str_contains( + (string) $event->command, + 'analytics:dispatch-publication-discovery', + )); expect($collection)->not->toBeNull() ->and($collection->expression)->toBe('0 2 * * *') @@ -22,5 +26,10 @@ ->and($finalizer->expression)->toBe('30 23 * * *') ->and($finalizer->timezone)->toBe('UTC') ->and($finalizer->withoutOverlapping)->toBeTrue() - ->and($finalizer->onOneServer)->toBeTrue(); + ->and($finalizer->onOneServer)->toBeTrue() + ->and($discovery)->not->toBeNull() + ->and($discovery->expression)->toBe('0 3 * * *') + ->and($discovery->timezone)->toBe('UTC') + ->and($discovery->withoutOverlapping)->toBeTrue() + ->and($discovery->onOneServer)->toBeTrue(); }); diff --git a/tests/Feature/Analytics/BackfillExistingAnalyticsCommandTest.php b/tests/Feature/Analytics/BackfillExistingAnalyticsCommandTest.php new file mode 100644 index 000000000..596628d76 --- /dev/null +++ b/tests/Feature/Analytics/BackfillExistingAnalyticsCommandTest.php @@ -0,0 +1,90 @@ +create(); + $account = SocialAccount::factory()->instagram()->create(['workspace_id' => $workspace->id]); + $published = PostPlatform::factory()->instagram()->published()->create([ + 'social_account_id' => $account->id, + 'platform' => $account->platform, + ]); + $published->post->update(['workspace_id' => $workspace->id]); + $failed = PostPlatform::factory()->instagram()->failed()->create([ + 'social_account_id' => $account->id, + 'platform' => $account->platform, + ]); + + app()->call([new BackfillTryPostPublications([$published->id, $failed->id]), 'handle']); + + expect(AnalyticsPublication::query()->where('post_platform_id', $published->id)->exists())->toBeTrue() + ->and(AnalyticsPublication::query()->where('post_platform_id', $failed->id)->exists())->toBeFalse(); +}); + +test('rollout command dispatches bounded jobs only for eligible scoped accounts and destinations', function () { + Bus::fake(); + $workspace = Workspace::factory()->create(); + $eligible = SocialAccount::factory()->instagram()->create([ + 'workspace_id' => $workspace->id, + 'is_active' => true, + 'status' => Status::Connected, + ]); + SocialAccount::factory()->linkedin()->create([ + 'workspace_id' => $workspace->id, + 'is_active' => true, + 'status' => Status::Connected, + ]); + SocialAccount::factory()->instagram()->create([ + 'workspace_id' => $workspace->id, + 'is_active' => false, + 'status' => Status::Connected, + ]); + $published = PostPlatform::factory()->instagram()->published()->create([ + 'social_account_id' => $eligible->id, + 'platform' => $eligible->platform, + ]); + $published->post->update(['workspace_id' => $workspace->id]); + PostPlatform::factory()->instagram()->create([ + 'social_account_id' => $eligible->id, + 'platform' => $eligible->platform, + 'status' => PostPlatformStatus::Failed, + ]); + Bus::fake(); + + expect(SocialAccount::query()->connected()->active()->includedInAnalytics() + ->where('workspace_id', $workspace->id)->pluck('id')->all())->toBe([$eligible->id]); + + $this->artisan('analytics:backfill-existing', ['--workspace' => $workspace->id]) + ->assertSuccessful(); + + Bus::assertDispatched(BootstrapAccountAnalytics::class, fn ($job): bool => $job->socialAccountId === $eligible->id); + Bus::assertDispatched(BackfillTryPostPublications::class, fn ($job): bool => $job->postPlatformIds === [$published->id]); + Bus::assertDispatchedTimes(BootstrapAccountAnalytics::class, 1); + Bus::assertDispatchedTimes(BackfillTryPostPublications::class, 1); +}); + +test('rollout reports orphaned historical destinations without inventing an identity', function () { + Bus::fake(); + $workspace = Workspace::factory()->create(); + $destination = PostPlatform::factory()->instagram()->published()->create(); + $destination->post->update(['workspace_id' => $workspace->id]); + $destination->updateQuietly(['social_account_id' => null]); + Bus::fake(); + + $this->artisan('analytics:backfill-existing', ['--workspace' => $workspace->id]) + ->expectsOutputToContain('historical_identity_unrecoverable=1') + ->assertSuccessful(); + + Bus::assertNotDispatched(BackfillTryPostPublications::class); +}); diff --git a/tests/Feature/Analytics/PublicationBackfillJobsTest.php b/tests/Feature/Analytics/PublicationBackfillJobsTest.php new file mode 100644 index 000000000..980389592 --- /dev/null +++ b/tests/Feature/Analytics/PublicationBackfillJobsTest.php @@ -0,0 +1,264 @@ +shouldReceive('page')->once()->andReturn($page); + + $factory = Mockery::mock(PublicationHistoryCollectorFactory::class); + $factory->shouldReceive('for')->once()->andReturn($collector); + app()->instance(PublicationHistoryCollectorFactory::class, $factory); +} + +test('publication jobs are provider limited and account overlap protected', function () { + $account = SocialAccount::factory()->instagram()->create(); + $backfill = new BackfillAccountPublications($account->id, fake()->uuid()); + $discovery = new DiscoverAccountPublications($account->id, fake()->uuid()); + + expect($backfill->providerRateLimitKey())->toBe('instagram') + ->and($backfill->middleware()[0])->toBeInstanceOf(RateLimited::class) + ->and($backfill->middleware()[1])->toBeInstanceOf(WithoutOverlapping::class) + ->and($discovery->middleware()[0])->toBeInstanceOf(RateLimited::class) + ->and($discovery->middleware()[1])->toBeInstanceOf(WithoutOverlapping::class); +}); + +test('bootstrap creates separate backfill and discovery states and dispatches the first page', function () { + Bus::fake(); + CarbonImmutable::setTestNow('2026-09-23 10:00:00 UTC'); + $account = SocialAccount::factory()->instagram()->create(['is_active' => true]); + + (new BootstrapAccountAnalytics($account->id))->handleFor($account->id); + + $backfill = AnalyticsSyncState::query()->where('social_account_id', $account->id) + ->where('collector', SyncCollector::PublicationBackfill)->firstOrFail(); + $discovery = AnalyticsSyncState::query()->where('social_account_id', $account->id) + ->where('collector', SyncCollector::PublicationDiscovery)->firstOrFail(); + + expect($backfill->status)->toBe(SyncStatus::Pending) + ->and($backfill->target_since?->toDateString())->toBe('2025-09-23') + ->and($discovery->status)->toBe(SyncStatus::Pending); + Bus::assertDispatched(BackfillAccountPublications::class, fn ($job): bool => $job->socialAccountId === $account->id && $job->syncStateId === $backfill->id); +}); + +test('a backfill job persists one page then advances its cursor and dispatches continuation', function () { + Bus::fake(); + $account = SocialAccount::factory()->instagram()->create(['is_active' => true]); + $state = AnalyticsSyncState::factory()->create([ + 'social_account_id' => $account->id, + 'checkpoint' => ['cursor' => null, 'revision' => 0], + 'target_since' => CarbonImmutable::parse('2025-09-23', 'UTC'), + ]); + bindPublicationPage(new PublicationPage([ + new DiscoveredPublication( + providerPostId: 'native-1', + publishedAt: CarbonImmutable::parse('2026-08-01', 'UTC'), + contentType: PublicationContentType::Image, + ), + ], 'next-page', false)); + + app()->call([new BackfillAccountPublications($account->id, $state->id), 'handle']); + + expect(AnalyticsPublication::query()->where('provider_post_id', 'native-1')->exists())->toBeTrue() + ->and($state->fresh()->checkpoint)->toMatchArray(['cursor' => 'next-page', 'revision' => 1]) + ->and($state->fresh()->status)->toBe(SyncStatus::Running); + Bus::assertDispatched(BackfillAccountPublications::class, fn ($job): bool => $job->socialAccountId === $account->id && $job->syncStateId === $state->id); +}); + +test('backfill records truthful terminal coverage and initializes discovery high water', function (PublicationPage $page, SyncStatus $expectedStatus, ?string $reason) { + Bus::fake(); + $account = SocialAccount::factory()->mastodon()->create(['is_active' => true]); + $state = AnalyticsSyncState::factory()->create([ + 'social_account_id' => $account->id, + 'checkpoint' => ['cursor' => null, 'revision' => 0], + 'target_since' => CarbonImmutable::parse('2025-09-23', 'UTC'), + ]); + bindPublicationPage($page); + + app()->call([new BackfillAccountPublications($account->id, $state->id), 'handle']); + + expect($state->fresh()->status)->toBe($expectedStatus) + ->and($state->fresh()->last_error_category)->toBe($reason); + $discovery = AnalyticsSyncState::query()->where('social_account_id', $account->id) + ->where('collector', SyncCollector::PublicationDiscovery)->firstOrFail(); + expect($discovery->high_watermark_at)->not->toBeNull(); + Bus::assertNotDispatched(BackfillAccountPublications::class); +})->with([ + 'exhausted' => [new PublicationPage([], null, true), SyncStatus::Complete, null], + 'provider limited' => [new PublicationPage([], null, true, true), SyncStatus::ProviderLimited, 'provider_limited'], + 'partial permission' => [new PublicationPage([], null, true, false, 'mastodon_reconnect_for_private_history'), SyncStatus::Partial, 'mastodon_reconnect_for_private_history'], +]); + +test('reaching the 365 day target stops pagination even when the provider has another cursor', function () { + Bus::fake(); + $account = SocialAccount::factory()->instagram()->create(['is_active' => true]); + $target = CarbonImmutable::parse('2025-09-23', 'UTC'); + $state = AnalyticsSyncState::factory()->create([ + 'social_account_id' => $account->id, + 'checkpoint' => ['cursor' => 'older', 'revision' => 0], + 'target_since' => $target, + ]); + bindPublicationPage(new PublicationPage([ + new DiscoveredPublication('boundary-post', $target, PublicationContentType::Image), + ], 'even-older', false)); + + app()->call([new BackfillAccountPublications($account->id, $state->id), 'handle']); + + expect($state->fresh()->status)->toBe(SyncStatus::Complete) + ->and(data_get($state->fresh()->checkpoint, 'cursor'))->toBeNull(); + Bus::assertNotDispatched(BackfillAccountPublications::class); +}); + +test('a stale page can reconcile facts but cannot move the current cursor backwards', function () { + $account = SocialAccount::factory()->instagram()->create(['is_active' => true]); + $state = AnalyticsSyncState::factory()->create([ + 'social_account_id' => $account->id, + 'checkpoint' => ['cursor' => 'page-a', 'revision' => 0], + ]); + $sync = app(AdvanceAnalyticsSyncState::class); + $first = $sync->begin($state->id); + $second = $sync->begin($state->id); + + $result = $sync->handle($state->id, $first['revision'], $account, new PublicationPage([ + new DiscoveredPublication( + 'stale-fact', + CarbonImmutable::parse('2026-09-01', 'UTC'), + PublicationContentType::Text, + ), + ], 'stale-next', false)); + + expect($result['advanced'])->toBeFalse() + ->and($state->fresh()->checkpoint)->toMatchArray([ + 'cursor' => 'page-a', + 'revision' => $second['revision'], + ]) + ->and(AnalyticsPublication::query()->where('provider_post_id', 'stale-fact')->exists())->toBeTrue(); +}); + +test('a transient failure preserves the cursor for a later queue attempt', function () { + Bus::fake(); + $account = SocialAccount::factory()->instagram()->create(['is_active' => true]); + $state = AnalyticsSyncState::factory()->create([ + 'social_account_id' => $account->id, + 'checkpoint' => ['cursor' => 'resume-here', 'revision' => 0], + ]); + $collector = Mockery::mock(PublicationHistoryCollector::class); + $collector->shouldReceive('page')->once()->withArgs( + fn (SocialAccount $received, ?string $cursor): bool => $received->is($account) && $cursor === 'resume-here', + )->andThrow(new AnalyticsCollectionException('transient', 'temporary')); + $factory = Mockery::mock(PublicationHistoryCollectorFactory::class); + $factory->shouldReceive('for')->once()->andReturn($collector); + app()->instance(PublicationHistoryCollectorFactory::class, $factory); + + expect(SocialAccount::query()->connected()->active()->includedInAnalytics()->find($account->id)) + ->not->toBeNull() + ->and($state->fresh()->status)->toBe(SyncStatus::Pending); + + expect(fn () => app()->call([new BackfillAccountPublications($account->id, $state->id), 'handle'])) + ->toThrow(AnalyticsCollectionException::class); + + expect(data_get($state->fresh()->checkpoint, 'cursor'))->toBe('resume-here') + ->and($state->fresh()->status)->toBe(SyncStatus::Running) + ->and($state->fresh()->last_error_category)->toBe('transient'); +}); + +test('an invalid provider cursor clears only the cursor and restarts the bounded backfill', function () { + Bus::fake(); + $account = SocialAccount::factory()->instagram()->create(['is_active' => true]); + $target = CarbonImmutable::parse('2025-09-23', 'UTC'); + $oldest = CarbonImmutable::parse('2026-01-10', 'UTC'); + $state = AnalyticsSyncState::factory()->create([ + 'social_account_id' => $account->id, + 'checkpoint' => ['cursor' => 'expired-cursor', 'revision' => 4], + 'target_since' => $target, + 'oldest_reached_at' => $oldest, + ]); + $collector = Mockery::mock(PublicationHistoryCollector::class); + $collector->shouldReceive('page')->once() + ->andThrow(new AnalyticsCollectionException('invalid_cursor', 'expired')); + $factory = Mockery::mock(PublicationHistoryCollectorFactory::class); + $factory->shouldReceive('for')->once()->andReturn($collector); + app()->instance(PublicationHistoryCollectorFactory::class, $factory); + + app()->call([new BackfillAccountPublications($account->id, $state->id), 'handle']); + + expect($state->fresh()->status)->toBe(SyncStatus::Pending) + ->and($state->fresh()->target_since?->equalTo($target))->toBeTrue() + ->and($state->fresh()->oldest_reached_at?->equalTo($oldest))->toBeTrue() + ->and(data_get($state->fresh()->checkpoint, 'cursor'))->toBeNull(); + Bus::assertDispatched(BackfillAccountPublications::class); +}); + +test('daily discovery is suppressed during backfill and resumes after terminal state', function () { + Bus::fake(); + $account = SocialAccount::factory()->instagram()->create(['is_active' => true]); + AnalyticsSyncState::factory()->create([ + 'social_account_id' => $account->id, + 'collector' => SyncCollector::PublicationBackfill, + 'status' => SyncStatus::Running, + ]); + $discovery = AnalyticsSyncState::factory()->create([ + 'social_account_id' => $account->id, + 'collector' => SyncCollector::PublicationDiscovery, + 'status' => SyncStatus::Pending, + 'high_watermark_at' => CarbonImmutable::parse('2026-09-20 12:00:00', 'UTC'), + ]); + + $job = new DiscoverAccountPublications($account->id, $discovery->id); + app()->call([$job, 'handle']); + expect($discovery->fresh()->status)->toBe(SyncStatus::Pending); + + AnalyticsSyncState::query()->where('social_account_id', $account->id) + ->where('collector', SyncCollector::PublicationBackfill) + ->update(['status' => SyncStatus::Complete]); + $collector = Mockery::mock(PublicationHistoryCollector::class); + $collector->shouldReceive('page')->once()->withArgs( + fn (SocialAccount $received, ?string $cursor, CarbonImmutable $cutoff): bool => $received->is($account) + && $cursor === null + && $cutoff->equalTo(CarbonImmutable::parse('2026-09-17 12:00:00', 'UTC')), + )->andReturn(new PublicationPage([], null, true)); + $factory = Mockery::mock(PublicationHistoryCollectorFactory::class); + $factory->shouldReceive('for')->once()->andReturn($collector); + app()->instance(PublicationHistoryCollectorFactory::class, $factory); + app()->call([$job, 'handle']); + + expect($discovery->fresh()->status)->toBe(SyncStatus::Complete) + ->and($discovery->fresh()->last_success_at)->not->toBeNull(); +}); + +test('deleting an account cascades operational states but retains publication history', function () { + $account = SocialAccount::factory()->instagram()->create(); + $state = AnalyticsSyncState::factory()->create(['social_account_id' => $account->id]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + ]); + + $account->delete(); + + expect(AnalyticsSyncState::query()->whereKey($state->id)->exists())->toBeFalse() + ->and($publication->fresh())->not->toBeNull() + ->and($publication->fresh()->social_account_id)->toBeNull(); +}); diff --git a/tests/Feature/Observers/SocialAccountObserverTest.php b/tests/Feature/Observers/SocialAccountObserverTest.php index 7e7aa8fab..1b27a9552 100644 --- a/tests/Feature/Observers/SocialAccountObserverTest.php +++ b/tests/Feature/Observers/SocialAccountObserverTest.php @@ -3,6 +3,7 @@ declare(strict_types=1); use App\Enums\SocialAccount\Status; +use App\Jobs\Analytics\BootstrapAccountAnalytics; use App\Jobs\Analytics\CollectAccountDailySnapshot; use App\Jobs\PostHog\IdentifyConnectedPlatforms; use App\Jobs\PostHog\SendEvent; @@ -151,6 +152,8 @@ Bus::assertDispatched(CollectAccountDailySnapshot::class, fn ($job): bool => $job->socialAccountId === $socialAccount->id && $job->queue === 'analytics'); + Bus::assertDispatched(BootstrapAccountAnalytics::class, fn ($job): bool => $job->socialAccountId === $socialAccount->id + && $job->queue === 'analytics'); }); test('connecting an excluded account does not dispatch follower collection', function () { @@ -161,4 +164,5 @@ ]); Bus::assertNotDispatched(CollectAccountDailySnapshot::class); + Bus::assertNotDispatched(BootstrapAccountAnalytics::class); }); From b6cf945d3bace39a8f06697e495ee862a7098591 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 12:35:48 -0300 Subject: [PATCH 22/77] feat: normalize publication analytics metrics --- .../Analytics/PublicationMetricsCollector.php | 17 ++ .../AbstractPublicationMetricsCollector.php | 204 +++++++++++++++ .../BlueskyPublicationMetricsCollector.php | 35 +++ .../FacebookPublicationMetricsCollector.php | 79 ++++++ .../InstagramPublicationMetricsCollector.php | 85 ++++++ .../MastodonPublicationMetricsCollector.php | 29 ++ .../PinterestPublicationMetricsCollector.php | 73 ++++++ .../PublicationMetricsCollectorFactory.php | 28 ++ .../ThreadsPublicationMetricsCollector.php | 39 +++ .../TikTokPublicationMetricsCollector.php | 60 +++++ .../Metrics/XPublicationMetricsCollector.php | 48 ++++ .../YouTubePublicationMetricsCollector.php | 60 +++++ .../PublicationMetricsCollectorsTest.php | 247 ++++++++++++++++++ 13 files changed, 1004 insertions(+) create mode 100644 app/Contracts/Analytics/PublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/AbstractPublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/BlueskyPublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/FacebookPublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/InstagramPublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/MastodonPublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/PinterestPublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/PublicationMetricsCollectorFactory.php create mode 100644 app/Services/Analytics/Collectors/Metrics/ThreadsPublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/TikTokPublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/XPublicationMetricsCollector.php create mode 100644 app/Services/Analytics/Collectors/Metrics/YouTubePublicationMetricsCollector.php create mode 100644 tests/Feature/Analytics/Collectors/PublicationMetricsCollectorsTest.php diff --git a/app/Contracts/Analytics/PublicationMetricsCollector.php b/app/Contracts/Analytics/PublicationMetricsCollector.php new file mode 100644 index 000000000..fb245f13b --- /dev/null +++ b/app/Contracts/Analytics/PublicationMetricsCollector.php @@ -0,0 +1,17 @@ +json('error.code', 0); + + if (in_array($errorCode, [4, 17, 32, 80001, 80002], true)) { + throw new AnalyticsCollectionException( + 'rate_limited', + 'Publication metrics provider rate limited the request.', + $this->retryAfter($response), + ); + } + + if (in_array($errorCode, [1, 2], true)) { + throw new AnalyticsCollectionException('transient', 'Publication metrics provider is temporarily unavailable.'); + } + + return parent::successfulResponse($response); + } + + private function retryAfter(Response $response): ?CarbonImmutable + { + $header = $response->header('Retry-After'); + + if (! is_string($header) || $header === '') { + return null; + } + + if (ctype_digit($header)) { + return CarbonImmutable::now('UTC')->addSeconds((int) $header); + } + + try { + return CarbonImmutable::parse($header)->utc(); + } catch (\Throwable) { + return null; + } + } + + protected function account(AnalyticsPublication $publication): SocialAccount + { + $account = $publication->socialAccount; + + if (! $account) { + throw AnalyticsCollectionException::unsupported('Publication has no live social account.'); + } + + return $account; + } + + /** @param list $metrics */ + protected function observation(CarbonImmutable $date, array $metrics): PublicationMetricObservation + { + if ($metrics === []) { + throw AnalyticsCollectionException::malformed('Publication metrics response contained no supported measurements.'); + } + + return new PublicationMetricObservation($date, $metrics, collectedAt: CarbonImmutable::now('UTC')); + } + + protected function count( + MetricKey $key, + array $source, + string $field, + MetricTimeBasis $timeBasis = MetricTimeBasis::Lifetime, + ): ?MetricValue { + if (! array_key_exists($field, $source) || ! is_numeric($source[$field])) { + return null; + } + + return new MetricValue( + key: $key, + value: (int) $source[$field], + unit: MetricUnit::Count, + timeBasis: $timeBasis, + precision: MetricPrecision::Exact, + availability: MetricAvailability::Available, + providerMetric: $field, + ); + } + + protected function decimal( + MetricKey $key, + array $source, + string $field, + MetricUnit $unit, + float $multiplier = 1, + MetricTimeBasis $timeBasis = MetricTimeBasis::Lifetime, + ): ?MetricValue { + if (! array_key_exists($field, $source) || ! is_numeric($source[$field])) { + return null; + } + + return new MetricValue( + key: $key, + value: $unit === MetricUnit::Milliseconds + ? (int) round((float) $source[$field] * $multiplier) + : (float) $source[$field] * $multiplier, + unit: $unit, + timeBasis: $timeBasis, + precision: MetricPrecision::Exact, + availability: MetricAvailability::Available, + providerMetric: $field, + ); + } + + /** @return array */ + protected function insights(array $data): array + { + $values = []; + + foreach ($data as $item) { + if (! is_array($item) || ! is_string($item['name'] ?? null)) { + continue; + } + + $value = data_get($item, 'total_value.value') ?? data_get($item, 'values.0.value'); + + if (is_numeric($value)) { + $values[$item['name']] = $value + 0; + } elseif (is_array($value) && $value !== []) { + $numbers = array_filter($value, 'is_numeric'); + + if (count($numbers) === count($value)) { + $values[$item['name']] = array_sum($numbers); + } + } + } + + return $values; + } + + /** @param list $metrics */ + protected function withEngagements(array $metrics): array + { + foreach ($metrics as $metric) { + if ($metric->key === MetricKey::Engagements) { + return $metrics; + } + + if ($metric->key === MetricKey::TotalInteractions) { + $metrics[] = new MetricValue( + key: MetricKey::Engagements, + value: $metric->value, + unit: MetricUnit::Count, + timeBasis: $metric->timeBasis, + precision: $metric->precision, + availability: $metric->availability, + providerMetric: $metric->providerMetric, + ); + + return $metrics; + } + } + + $interactionKeys = [ + MetricKey::Reactions, MetricKey::Comments, MetricKey::Shares, + MetricKey::Saves, MetricKey::Quotes, MetricKey::Bookmarks, + MetricKey::Clicks, MetricKey::LinkClicks, + ]; + $present = array_filter($metrics, fn (MetricValue $metric): bool => in_array($metric->key, $interactionKeys, true)); + + if ($present !== []) { + $metrics[] = new MetricValue( + key: MetricKey::Engagements, + value: array_sum(array_map(fn (MetricValue $metric): int => (int) $metric->value, $present)), + unit: MetricUnit::Count, + timeBasis: MetricTimeBasis::Lifetime, + precision: MetricPrecision::Exact, + availability: MetricAvailability::Available, + providerMetric: 'derived_interactions', + ); + } + + return $metrics; + } + + /** @param array $metrics @return list */ + protected function present(array $metrics): array + { + return array_values(array_filter($metrics, fn (?MetricValue $metric): bool => $metric !== null)); + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/BlueskyPublicationMetricsCollector.php b/app/Services/Analytics/Collectors/Metrics/BlueskyPublicationMetricsCollector.php new file mode 100644 index 000000000..15eeb659d --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/BlueskyPublicationMetricsCollector.php @@ -0,0 +1,35 @@ +account($publication); + $uri = "at://{$account->platform_user_id}/".BlueskyLexicon::FEED_POST."/{$publication->provider_post_id}"; + $response = $this->get($account, + rtrim((string) config('trypost.platforms.bluesky.public_appview'), '/').'/xrpc/'.BlueskyLexicon::GET_POSTS, + ['uris' => [$uri]], + authenticated: false, + ); + + $post = (array) $response->json('posts.0', []); + + return $this->observation($date, $this->withEngagements($this->present([ + $this->count(MetricKey::Reactions, $post, 'likeCount'), + $this->count(MetricKey::Comments, $post, 'replyCount'), + $this->count(MetricKey::Shares, $post, 'repostCount'), + $this->count(MetricKey::Quotes, $post, 'quoteCount'), + ]))); + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/FacebookPublicationMetricsCollector.php b/app/Services/Analytics/Collectors/Metrics/FacebookPublicationMetricsCollector.php new file mode 100644 index 000000000..e84e983b1 --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/FacebookPublicationMetricsCollector.php @@ -0,0 +1,79 @@ +account($publication); + $isStory = $publication->content_type === PublicationContentType::Story; + $isVideo = ! $isStory && ! str_contains($publication->provider_post_id, '_'); + $edge = $isVideo ? 'video_insights' : 'insights'; + $fields = match (true) { + $isStory => ['page_story_impressions_by_story_id', 'page_story_impressions_by_story_id_unique', 'story_interaction', 'pages_fb_story_thread_lightweight_reactions', 'pages_fb_story_replies', 'pages_fb_story_shares'], + $isVideo => ['fb_reels_total_plays', 'post_video_likes_by_reaction_type', 'post_video_social_actions'], + default => ['post_media_view', 'post_total_media_view_unique', 'post_reactions_like_total', 'post_clicks'], + }; + $response = $this->get($account, + rtrim((string) config('trypost.platforms.facebook.graph_api'), '/')."/{$publication->provider_post_id}/{$edge}", + ['metric' => implode(',', $fields), 'period' => 'lifetime', 'access_token' => $account->access_token], + ); + $items = $response->json('data'); + + if (! is_array($items)) { + throw AnalyticsCollectionException::malformed('Facebook insights response lacks data.'); + } + + $values = $this->insights($items); + + if (! $isStory) { + $details = $this->get($account, + rtrim((string) config('trypost.platforms.facebook.graph_api'), '/')."/{$publication->provider_post_id}", + [ + 'fields' => 'reactions.limit(0).summary(true),comments.limit(0).summary(true),shares', + 'access_token' => $account->access_token, + ], + )->json(); + + if (is_array($details)) { + foreach ([ + 'reactions.summary.total_count' => 'reactions_count', + 'comments.summary.total_count' => 'comments_count', + 'shares.count' => 'shares_count', + ] as $source => $target) { + $value = data_get($details, $source); + + if (is_numeric($value)) { + $values[$target] = (int) $value; + } + } + } + } + + return $this->observation($date, $this->withEngagements($this->present([ + $this->count(MetricKey::Impressions, $values, $isStory ? 'page_story_impressions_by_story_id' : 'post_media_view'), + $this->count(MetricKey::Reach, $values, $isStory ? 'page_story_impressions_by_story_id_unique' : 'post_total_media_view_unique'), + $this->count(MetricKey::Views, $values, 'fb_reels_total_plays'), + $this->count(MetricKey::Reactions, $values, 'reactions_count') ?? $this->count(MetricKey::Reactions, $values, match (true) { + $isStory => 'pages_fb_story_thread_lightweight_reactions', + $isVideo => 'post_video_likes_by_reaction_type', + default => 'post_reactions_like_total', + }), + $this->count(MetricKey::Comments, $values, 'comments_count') ?? $this->count(MetricKey::Comments, $values, 'pages_fb_story_replies'), + $this->count(MetricKey::Shares, $values, 'shares_count') ?? $this->count(MetricKey::Shares, $values, 'pages_fb_story_shares'), + $this->count(MetricKey::Clicks, $values, 'post_clicks'), + $this->count(MetricKey::TotalInteractions, $values, $isStory ? 'story_interaction' : 'post_video_social_actions'), + ]))); + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/InstagramPublicationMetricsCollector.php b/app/Services/Analytics/Collectors/Metrics/InstagramPublicationMetricsCollector.php new file mode 100644 index 000000000..442b3d824 --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/InstagramPublicationMetricsCollector.php @@ -0,0 +1,85 @@ +account($publication); + $isStory = $publication->content_type === PublicationContentType::Story; + $isReel = $publication->content_type === PublicationContentType::Reel; + $fields = $isStory + ? ['reach', 'views', 'replies'] + : ['reach', 'views', 'likes', 'comments', 'shares', 'saved']; + $url = $account->platform->instagramGraphBaseUrl()."/{$publication->provider_post_id}/insights"; + $response = $this->get($account, $url, ['metric' => implode(',', $fields), 'access_token' => $account->access_token]); + $items = $response->json('data'); + + if (! is_array($items)) { + throw AnalyticsCollectionException::malformed('Instagram insights response lacks data.'); + } + + $values = $this->insights($items); + $extras = $isStory + ? $this->optionalInsights($account, $url, ['navigation', 'taps_forward', 'taps_back', 'exits']) + : $this->optionalInsights($account, $url, ['total_interactions', 'reposts', 'follows', 'profile_visits', 'profile_activity']); + $values = array_merge($values, $extras); + $metrics = $this->present([ + $this->count(MetricKey::Reach, $values, 'reach'), + $this->count(MetricKey::Views, $values, 'views'), + $this->count(MetricKey::Reactions, $values, 'likes'), + $this->count(MetricKey::Comments, $values, $isStory ? 'replies' : 'comments'), + $this->count(MetricKey::Shares, $values, 'shares'), + $this->count(MetricKey::Saves, $values, 'saved'), + $this->count(MetricKey::Reposts, $values, 'reposts'), + $this->count(MetricKey::TotalInteractions, $values, 'total_interactions'), + $this->count(MetricKey::Follows, $values, 'follows'), + $this->count(MetricKey::ProfileVisits, $values, 'profile_visits'), + $this->count(MetricKey::ProfileActivity, $values, 'profile_activity'), + $this->count(MetricKey::StoryNavigation, $values, 'navigation'), + $this->count(MetricKey::StoryTapsForward, $values, 'taps_forward'), + $this->count(MetricKey::StoryTapsBack, $values, 'taps_back'), + $this->count(MetricKey::StoryExits, $values, 'exits'), + ]); + + if ($isReel) { + $reelValues = $this->optionalInsights($account, $url, ['ig_reels_video_view_total_time', 'ig_reels_avg_watch_time']); + $metrics = array_merge($metrics, $this->present([ + $this->decimal(MetricKey::WatchTimeMilliseconds, $reelValues, 'ig_reels_video_view_total_time', MetricUnit::Milliseconds), + $this->decimal(MetricKey::AverageWatchTimeMilliseconds, $reelValues, 'ig_reels_avg_watch_time', MetricUnit::Milliseconds), + ])); + } + + return $this->observation($date, $this->withEngagements($metrics)); + } + + /** @param list $fields @return array */ + private function optionalInsights(SocialAccount $account, string $url, array $fields): array + { + try { + $response = $this->get($account, $url, [ + 'metric' => implode(',', $fields), + 'access_token' => $account->access_token, + ]); + } catch (AnalyticsCollectionException) { + return []; + } + + $data = $response->json('data'); + + return is_array($data) ? $this->insights($data) : []; + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/MastodonPublicationMetricsCollector.php b/app/Services/Analytics/Collectors/Metrics/MastodonPublicationMetricsCollector.php new file mode 100644 index 000000000..9401b367b --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/MastodonPublicationMetricsCollector.php @@ -0,0 +1,29 @@ +account($publication); + $instance = rtrim((string) data_get($account->meta, 'instance', config('trypost.platforms.mastodon.default_instance')), '/'); + $response = $this->get($account, "{$instance}/api/v1/statuses/{$publication->provider_post_id}"); + + $status = (array) $response->json(); + + return $this->observation($date, $this->withEngagements($this->present([ + $this->count(MetricKey::Reactions, $status, 'favourites_count'), + $this->count(MetricKey::Comments, $status, 'replies_count'), + $this->count(MetricKey::Shares, $status, 'reblogs_count'), + ]))); + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/PinterestPublicationMetricsCollector.php b/app/Services/Analytics/Collectors/Metrics/PinterestPublicationMetricsCollector.php new file mode 100644 index 000000000..3e3c370ff --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/PinterestPublicationMetricsCollector.php @@ -0,0 +1,73 @@ +account($publication); + $isVideo = in_array($publication->content_type, [PublicationContentType::Video, PublicationContentType::Short], true); + $fields = ['IMPRESSION', 'SAVE', 'PIN_CLICK', 'OUTBOUND_CLICK', 'ENGAGEMENT', 'ENGAGEMENT_RATE', 'SAVE_RATE', 'PIN_CLICK_RATE', 'OUTBOUND_CLICK_RATE']; + + if ($isVideo) { + $fields = array_merge($fields, ['VIDEO_MRC_VIEW', 'VIDEO_AVG_WATCH_TIME', 'VIDEO_10S_VIEW', 'QUARTILE_95_PERCENT_VIEW', 'VIDEO_V50_WATCH_TIME']); + } + + $response = $this->get($account, + rtrim((string) config('trypost.platforms.pinterest.api'), '/')."/pins/{$publication->provider_post_id}/analytics", + [ + 'start_date' => $date->subDays(89)->toDateString(), + 'end_date' => $date->toDateString(), + 'metric_types' => implode(',', $fields), + ], + ); + $values = $response->json('all.summary_metrics'); + + if (! is_array($values)) { + throw AnalyticsCollectionException::malformed('Pinterest Pin analytics response lacks summary metrics.'); + } + + $basis = MetricTimeBasis::Rolling90Days; + $metrics = $this->present([ + $this->count(MetricKey::Impressions, $values, 'IMPRESSION', $basis), + $this->count(MetricKey::Saves, $values, 'SAVE', $basis), + $this->count(MetricKey::PinClicks, $values, 'PIN_CLICK', $basis), + $this->count(MetricKey::OutboundClicks, $values, 'OUTBOUND_CLICK', $basis), + $this->count(MetricKey::Engagements, $values, 'ENGAGEMENT', $basis), + $this->decimal(MetricKey::EngagementRate, $values, 'ENGAGEMENT_RATE', MetricUnit::Percent, 100, $basis), + $this->decimal(MetricKey::SaveRate, $values, 'SAVE_RATE', MetricUnit::Percent, 100, $basis), + $this->decimal(MetricKey::PinClickRate, $values, 'PIN_CLICK_RATE', MetricUnit::Percent, 100, $basis), + $this->decimal(MetricKey::OutboundClickRate, $values, 'OUTBOUND_CLICK_RATE', MetricUnit::Percent, 100, $basis), + $this->count(MetricKey::VideoViews, $values, 'VIDEO_MRC_VIEW', $basis), + $this->decimal(MetricKey::AverageVideoPlayTimeMilliseconds, $values, 'VIDEO_AVG_WATCH_TIME', MetricUnit::Milliseconds, timeBasis: $basis), + $this->count(MetricKey::VideoViews10Seconds, $values, 'VIDEO_10S_VIEW', $basis), + $this->count(MetricKey::VideoViews95Percent, $values, 'QUARTILE_95_PERCENT_VIEW', $basis), + $this->decimal(MetricKey::TotalPlayTimeMilliseconds, $values, 'VIDEO_V50_WATCH_TIME', MetricUnit::Milliseconds, timeBasis: $basis), + ]); + $details = $this->get($account, + rtrim((string) config('trypost.platforms.pinterest.api'), '/')."/pins/{$publication->provider_post_id}", + ['pin_metrics' => 'true'], + )->json(); + $lifetime = (array) (data_get($details, 'pin_metrics.all.lifetime_metrics') + ?? data_get($details, 'pin_metrics.lifetime_metrics', [])); + $metrics = array_merge($metrics, $this->present([ + $this->count(MetricKey::Comments, $lifetime, 'TOTAL_COMMENTS'), + $this->count(MetricKey::Reactions, $lifetime, 'TOTAL_REACTIONS'), + ])); + + return $this->observation($date, $metrics); + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/PublicationMetricsCollectorFactory.php b/app/Services/Analytics/Collectors/Metrics/PublicationMetricsCollectorFactory.php new file mode 100644 index 000000000..1def35a37 --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/PublicationMetricsCollectorFactory.php @@ -0,0 +1,28 @@ + app(InstagramPublicationMetricsCollector::class), + Platform::Facebook => app(FacebookPublicationMetricsCollector::class), + Platform::Threads => app(ThreadsPublicationMetricsCollector::class), + Platform::X => app(XPublicationMetricsCollector::class), + Platform::Pinterest => app(PinterestPublicationMetricsCollector::class), + Platform::YouTube => app(YouTubePublicationMetricsCollector::class), + Platform::TikTok => app(TikTokPublicationMetricsCollector::class), + Platform::Bluesky => app(BlueskyPublicationMetricsCollector::class), + Platform::Mastodon => app(MastodonPublicationMetricsCollector::class), + default => throw AnalyticsCollectionException::unsupported("{$platform->value} publication metrics are excluded"), + }; + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/ThreadsPublicationMetricsCollector.php b/app/Services/Analytics/Collectors/Metrics/ThreadsPublicationMetricsCollector.php new file mode 100644 index 000000000..b39c398e7 --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/ThreadsPublicationMetricsCollector.php @@ -0,0 +1,39 @@ +account($publication); + $response = $this->get($account, + rtrim((string) config('trypost.platforms.threads.graph_api'), '/')."/{$publication->provider_post_id}/insights", + ['metric' => 'views,likes,replies,reposts,quotes'], + ); + $items = $response->json('data'); + + if (! is_array($items)) { + throw AnalyticsCollectionException::malformed('Threads insights response lacks data.'); + } + + $values = $this->insights($items); + + return $this->observation($date, $this->withEngagements($this->present([ + $this->count(MetricKey::Views, $values, 'views'), + $this->count(MetricKey::Reactions, $values, 'likes'), + $this->count(MetricKey::Comments, $values, 'replies'), + $this->count(MetricKey::Shares, $values, 'reposts'), + $this->count(MetricKey::Quotes, $values, 'quotes'), + ]))); + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/TikTokPublicationMetricsCollector.php b/app/Services/Analytics/Collectors/Metrics/TikTokPublicationMetricsCollector.php new file mode 100644 index 000000000..76c29cded --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/TikTokPublicationMetricsCollector.php @@ -0,0 +1,60 @@ +account($publication); + $videoId = $publication->provider_post_id; + + if (! ctype_digit($videoId)) { + $status = $this->post($account, + rtrim((string) config('trypost.platforms.tiktok.api'), '/').'/post/publish/status/fetch/', + ['publish_id' => $videoId], + ); + $videoId = (string) $status->json('data.publicaly_available_post_id.0', ''); + + if (! ctype_digit($videoId)) { + throw new AnalyticsCollectionException('delayed', 'TikTok publication has no public video id yet.'); + } + } + + $response = $this->post($account, + rtrim((string) config('trypost.platforms.tiktok.api'), '/').'/video/query/?fields=id,view_count,like_count,comment_count,share_count', + ['filters' => ['video_ids' => [$videoId]]], + ); + $errorCode = $response->json('error.code'); + + if (is_string($errorCode) && ! in_array($errorCode, ['', 'ok'], true)) { + throw new AnalyticsCollectionException( + $errorCode === 'rate_limit_exceeded' ? 'rate_limited' : 'permission', + 'TikTok video metrics query rejected the request.', + ); + } + + $video = collect((array) $response->json('data.videos', [])) + ->first(fn (mixed $item): bool => is_array($item) && (string) ($item['id'] ?? '') === $videoId); + + if (! is_array($video)) { + throw AnalyticsCollectionException::malformed('TikTok video query did not return the requested video.'); + } + + return $this->observation($date, $this->withEngagements($this->present([ + $this->count(MetricKey::Views, $video, 'view_count'), + $this->count(MetricKey::Reactions, $video, 'like_count'), + $this->count(MetricKey::Comments, $video, 'comment_count'), + $this->count(MetricKey::Shares, $video, 'share_count'), + ]))); + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/XPublicationMetricsCollector.php b/app/Services/Analytics/Collectors/Metrics/XPublicationMetricsCollector.php new file mode 100644 index 000000000..e3fe2d3e5 --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/XPublicationMetricsCollector.php @@ -0,0 +1,48 @@ +account($publication); + $fields = ['public_metrics']; + + if ($publication->provider_published_at->greaterThan(CarbonImmutable::now('UTC')->subDays(30))) { + $fields[] = 'non_public_metrics'; + } + + $response = $this->get($account, + rtrim((string) config('trypost.platforms.x.api'), '/')."/tweets/{$publication->provider_post_id}", + ['tweet.fields' => implode(',', $fields)], + ); + $tweet = $response->json('data'); + + if (! is_array($tweet)) { + throw AnalyticsCollectionException::malformed('X post response lacks data.'); + } + + $public = (array) ($tweet['public_metrics'] ?? []); + $private = (array) ($tweet['non_public_metrics'] ?? []); + + return $this->observation($date, $this->withEngagements($this->present([ + $this->count(MetricKey::Impressions, $public, 'impression_count') ?? $this->count(MetricKey::Impressions, $private, 'impression_count'), + $this->count(MetricKey::Reactions, $public, 'like_count'), + $this->count(MetricKey::Comments, $public, 'reply_count'), + $this->count(MetricKey::Shares, $public, 'retweet_count'), + $this->count(MetricKey::Quotes, $public, 'quote_count'), + $this->count(MetricKey::Bookmarks, $public, 'bookmark_count'), + $this->count(MetricKey::LinkClicks, $private, 'url_link_clicks'), + ]))); + } +} diff --git a/app/Services/Analytics/Collectors/Metrics/YouTubePublicationMetricsCollector.php b/app/Services/Analytics/Collectors/Metrics/YouTubePublicationMetricsCollector.php new file mode 100644 index 000000000..d3ab39b5f --- /dev/null +++ b/app/Services/Analytics/Collectors/Metrics/YouTubePublicationMetricsCollector.php @@ -0,0 +1,60 @@ +account($publication); + $response = $this->get($account, + rtrim((string) config('trypost.platforms.youtube.analytics_api'), '/').'/reports', + [ + 'ids' => 'channel==MINE', + 'startDate' => $publication->provider_published_at->toDateString(), + 'endDate' => $date->toDateString(), + 'metrics' => 'views,engagedViews,estimatedMinutesWatched,averageViewDuration,averageViewPercentage,likes,comments,shares,subscribersGained,subscribersLost', + 'filters' => "video=={$publication->provider_post_id}", + ], + ); + $headers = $response->json('columnHeaders'); + $row = $response->json('rows.0'); + + if (! is_array($headers)) { + throw AnalyticsCollectionException::malformed('YouTube Analytics response lacks column headers.'); + } + + $values = []; + + if (is_array($row)) { + foreach ($headers as $index => $header) { + if (is_string(data_get($header, 'name')) && array_key_exists($index, $row)) { + $values[$header['name']] = $row[$index]; + } + } + } + + return $this->observation($date, $this->withEngagements($this->present([ + $this->count(MetricKey::Views, $values, 'views'), + $this->count(MetricKey::EngagedViews, $values, 'engagedViews'), + $this->decimal(MetricKey::WatchTimeMilliseconds, $values, 'estimatedMinutesWatched', MetricUnit::Milliseconds, 60000), + $this->decimal(MetricKey::AverageWatchTimeMilliseconds, $values, 'averageViewDuration', MetricUnit::Milliseconds, 1000), + $this->decimal(MetricKey::AveragePercentageViewed, $values, 'averageViewPercentage', MetricUnit::Percent), + $this->count(MetricKey::Reactions, $values, 'likes'), + $this->count(MetricKey::Comments, $values, 'comments'), + $this->count(MetricKey::Shares, $values, 'shares'), + $this->count(MetricKey::SubscribersGained, $values, 'subscribersGained'), + $this->count(MetricKey::SubscribersLost, $values, 'subscribersLost'), + ]))); + } +} diff --git a/tests/Feature/Analytics/Collectors/PublicationMetricsCollectorsTest.php b/tests/Feature/Analytics/Collectors/PublicationMetricsCollectorsTest.php new file mode 100644 index 000000000..06ee4e5d8 --- /dev/null +++ b/tests/Feature/Analytics/Collectors/PublicationMetricsCollectorsTest.php @@ -0,0 +1,247 @@ +for($platform))->toBeInstanceOf($collector); +})->with([ + [Platform::Instagram, InstagramPublicationMetricsCollector::class], + [Platform::InstagramFacebook, InstagramPublicationMetricsCollector::class], + [Platform::Facebook, FacebookPublicationMetricsCollector::class], + [Platform::Threads, ThreadsPublicationMetricsCollector::class], + [Platform::X, XPublicationMetricsCollector::class], + [Platform::Pinterest, PinterestPublicationMetricsCollector::class], + [Platform::YouTube, YouTubePublicationMetricsCollector::class], + [Platform::TikTok, TikTokPublicationMetricsCollector::class], + [Platform::Bluesky, BlueskyPublicationMetricsCollector::class], + [Platform::Mastodon, MastodonPublicationMetricsCollector::class], +]); + +test('bluesky normalizes measured zero and does not fabricate omitted counts', function () { + Http::fake(['*' => Http::response(['posts' => [[ + 'likeCount' => 0, + 'replyCount' => 3, + 'repostCount' => 2, + ]]])]); + $account = SocialAccount::factory()->bluesky()->create(); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'platform' => Platform::Bluesky, + 'platform_user_id' => $account->platform_user_id, + 'provider_post_id' => 'record-key', + ]); + + $observation = app(BlueskyPublicationMetricsCollector::class)->collect( + $publication, + CarbonImmutable::parse('2026-09-23', 'UTC'), + ); + $metrics = collect($observation->metrics)->keyBy(fn ($metric) => $metric->key->value); + + expect($metrics[MetricKey::Reactions->value]->value)->toBe(0) + ->and($metrics[MetricKey::Comments->value]->value)->toBe(3) + ->and($metrics[MetricKey::Shares->value]->value)->toBe(2) + ->and($metrics->has(MetricKey::Quotes->value))->toBeFalse(); +}); + +test('provider metric responses normalize measured values without inventing omitted metrics', function ( + Platform $platform, + array $response, + array $expected, + PublicationContentType $contentType = PublicationContentType::Image, +) { + Http::fake(['*' => Http::response($response)]); + $account = SocialAccount::factory()->create(['platform' => $platform]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'platform' => $platform, + 'platform_user_id' => $account->platform_user_id, + 'provider_post_id' => $platform === Platform::Facebook ? 'page_123456789' : '123456789', + 'content_type' => $contentType, + ]); + + $observation = app(PublicationMetricsCollectorFactory::class) + ->for($platform) + ->collect($publication, CarbonImmutable::parse('2026-09-23', 'UTC')); + $actual = collect($observation->metrics)->mapWithKeys(fn ($metric) => [$metric->key->value => $metric->value])->all(); + + expect($actual)->toBe($expected); +})->with([ + 'instagram' => [Platform::Instagram, ['data' => [ + ['name' => 'reach', 'values' => [['value' => 100]]], + ['name' => 'likes', 'total_value' => ['value' => 0]], + ['name' => 'comments', 'values' => [['value' => 2]]], + ['name' => 'saved', 'values' => [['value' => 1]]], + ]], ['reach' => 100, 'reactions' => 0, 'comments' => 2, 'saves' => 1, 'engagements' => 3]], + 'facebook feed' => [Platform::Facebook, ['data' => [ + ['name' => 'post_media_view', 'values' => [['value' => 120]]], + ['name' => 'post_reactions_like_total', 'values' => [['value' => 0]]], + ]], ['impressions' => 120, 'reactions' => 0, 'engagements' => 0]], + 'threads' => [Platform::Threads, ['data' => [ + ['name' => 'views', 'total_value' => ['value' => 40]], + ['name' => 'likes', 'values' => [['value' => 0]]], + ['name' => 'reposts', 'values' => [['value' => 3]]], + ]], ['views' => 40, 'reactions' => 0, 'shares' => 3, 'engagements' => 3]], + 'x' => [Platform::X, ['data' => ['public_metrics' => [ + 'impression_count' => 200, 'like_count' => 0, 'retweet_count' => 3, + 'bookmark_count' => 4, + ]]], ['impressions' => 200, 'reactions' => 0, 'shares' => 3, 'bookmarks' => 4, 'engagements' => 7]], + 'pinterest' => [Platform::Pinterest, ['all' => ['summary_metrics' => [ + 'IMPRESSION' => 30, 'SAVE' => 0, 'PIN_CLICK' => 2, + ]]], ['impressions' => 30, 'saves' => 0, 'pin_clicks' => 2]], + 'youtube' => [Platform::YouTube, [ + 'columnHeaders' => [['name' => 'views'], ['name' => 'estimatedMinutesWatched'], ['name' => 'averageViewDuration'], ['name' => 'likes']], + 'rows' => [[231, 2.5, 23.4, 0]], + ], ['views' => 231, 'watch_time_milliseconds' => 150000, 'average_watch_time_milliseconds' => 23400, 'reactions' => 0, 'engagements' => 0]], + 'tiktok' => [Platform::TikTok, ['data' => ['videos' => [[ + 'id' => '123456789', 'view_count' => 400, 'like_count' => 0, 'comment_count' => 3, + ]]]], ['views' => 400, 'reactions' => 0, 'comments' => 3, 'engagements' => 3]], + 'mastodon' => [Platform::Mastodon, [ + 'favourites_count' => 0, 'replies_count' => 2, 'reblogs_count' => 1, + ], ['reactions' => 0, 'comments' => 2, 'shares' => 1, 'engagements' => 3]], +]); + +test('pinterest preserves its rolling window basis', function () { + Http::fake(['*' => Http::response(['all' => ['summary_metrics' => ['SAVE' => 0]]])]); + $account = SocialAccount::factory()->create(['platform' => Platform::Pinterest]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'platform' => Platform::Pinterest, + 'platform_user_id' => $account->platform_user_id, + ]); + + $metric = app(PinterestPublicationMetricsCollector::class) + ->collect($publication, CarbonImmutable::parse('2026-09-23', 'UTC'))->metrics[0]; + + expect($metric->timeBasis)->toBe(MetricTimeBasis::Rolling90Days) + ->and($metric->value)->toBe(0); +}); + +test('a rate-limited metric request throws a retryable collection exception', function () { + Http::fake(['*' => Http::response(['error' => 'rate limited'], 429, ['Retry-After' => '120'])]); + $account = SocialAccount::factory()->create(['platform' => Platform::Mastodon]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'platform' => Platform::Mastodon, + 'platform_user_id' => $account->platform_user_id, + ]); + + try { + app(MastodonPublicationMetricsCollector::class)->collect($publication, CarbonImmutable::today('UTC')); + test()->fail('Expected rate limit classification.'); + } catch (AnalyticsCollectionException $exception) { + expect($exception->category)->toBe('rate_limited') + ->and($exception->retryAt)->not->toBeNull(); + } +}); + +test('instagram reels collect watch duration in milliseconds without losing engagement', function () { + $account = SocialAccount::factory()->create(['platform' => Platform::Instagram]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'platform' => Platform::Instagram, + 'platform_user_id' => $account->platform_user_id, + 'content_type' => PublicationContentType::Reel, + ]); + Http::fake(['*' => Http::sequence() + ->push(['data' => [ + ['name' => 'reach', 'values' => [['value' => 80]]], + ['name' => 'likes', 'total_value' => ['value' => 5]], + ]]) + ->push(['data' => []]) + ->push(['data' => [ + ['name' => 'ig_reels_video_view_total_time', 'total_value' => ['value' => 185000]], + ['name' => 'ig_reels_avg_watch_time', 'values' => [['value' => 23000]]], + ]])]); + + $metrics = collect(app(InstagramPublicationMetricsCollector::class) + ->collect($publication, CarbonImmutable::today('UTC'))->metrics) + ->mapWithKeys(fn ($metric) => [$metric->key->value => $metric->value]); + + expect($metrics->all())->toBe([ + 'reach' => 80, + 'reactions' => 5, + 'watch_time_milliseconds' => 185000, + 'average_watch_time_milliseconds' => 23000, + 'engagements' => 5, + ]); + Http::assertSentCount(3); +}); + +test('X requests only public fields outside the private-metric window', function () { + Http::fake(['*' => Http::response(['data' => ['public_metrics' => ['like_count' => 0]]])]); + $account = SocialAccount::factory()->create(['platform' => Platform::X]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'platform' => Platform::X, + 'platform_user_id' => $account->platform_user_id, + 'provider_published_at' => CarbonImmutable::now('UTC')->subDays(40), + ]); + + app(XPublicationMetricsCollector::class)->collect($publication, CarbonImmutable::today('UTC')); + + Http::assertSent(fn (Request $request): bool => str_contains($request->url(), 'tweet.fields=public_metrics')); +}); + +test('Meta rate limiting in HTTP 400 is classified as retryable', function () { + Http::fake(['*' => Http::response(['error' => ['code' => 80002]], 400)]); + $account = SocialAccount::factory()->create(['platform' => Platform::InstagramFacebook]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'platform' => Platform::InstagramFacebook, + 'platform_user_id' => $account->platform_user_id, + ]); + + expect(fn () => app(InstagramPublicationMetricsCollector::class)->collect($publication, CarbonImmutable::today('UTC'))) + ->toThrow(fn (AnalyticsCollectionException $exception): bool => $exception->category === 'rate_limited'); +}); + +test('TikTok resolves a TryPost publish id before collecting the public video', function () { + $account = SocialAccount::factory()->create(['platform' => Platform::TikTok]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'platform' => Platform::TikTok, + 'platform_user_id' => $account->platform_user_id, + 'provider_post_id' => 'v_pub_abc', + ]); + Http::fake(['*' => Http::sequence() + ->push(['data' => ['publicaly_available_post_id' => ['123456789']]]) + ->push(['error' => ['code' => 'ok'], 'data' => ['videos' => [[ + 'id' => '123456789', 'view_count' => 12, 'like_count' => 0, + ]]]])]); + + $metrics = collect(app(TikTokPublicationMetricsCollector::class) + ->collect($publication, CarbonImmutable::today('UTC'))->metrics) + ->mapWithKeys(fn ($metric) => [$metric->key->value => $metric->value]); + + expect($metrics->all())->toBe(['views' => 12, 'reactions' => 0, 'engagements' => 0]); + Http::assertSentCount(2); +}); From b230490e006f6e8ea4a3ce9d6bfda1b57b9cc6df Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 12:43:45 -0300 Subject: [PATCH 23/77] feat: schedule persisted publication metrics --- .../QueuePublicationMetricsForPage.php | 66 +++++++ .../Analytics/DispatchPublicationMetrics.php | 54 ++++++ .../Analytics/BackfillAccountPublications.php | 4 + .../Analytics/CollectPublicationMetrics.php | 171 ++++++++++++++++++ .../Analytics/DiscoverAccountPublications.php | 4 + .../ScheduleInstagramStoryMetrics.php | 60 ++++++ app/Jobs/Analytics/SyncTryPostPublication.php | 6 +- app/Models/AnalyticsPublication.php | 6 + routes/console.php | 6 + .../Analytics/AnalyticsScheduleTest.php | 9 + .../Analytics/PublicationMetricsJobsTest.php | 171 ++++++++++++++++++ .../PublicationReconciliationTest.php | 2 +- 12 files changed, 556 insertions(+), 3 deletions(-) create mode 100644 app/Actions/Analytics/QueuePublicationMetricsForPage.php create mode 100644 app/Console/Commands/Analytics/DispatchPublicationMetrics.php create mode 100644 app/Jobs/Analytics/CollectPublicationMetrics.php create mode 100644 app/Jobs/Analytics/ScheduleInstagramStoryMetrics.php create mode 100644 tests/Feature/Analytics/PublicationMetricsJobsTest.php diff --git a/app/Actions/Analytics/QueuePublicationMetricsForPage.php b/app/Actions/Analytics/QueuePublicationMetricsForPage.php new file mode 100644 index 000000000..7b3998e13 --- /dev/null +++ b/app/Actions/Analytics/QueuePublicationMetricsForPage.php @@ -0,0 +1,66 @@ + $item->providerPostId, $page->publications); + + if ($providerIds === []) { + return; + } + + AnalyticsPublication::query() + ->available() + ->where('social_account_id', $account->id) + ->whereIn('provider_post_id', $providerIds) + ->each(function (AnalyticsPublication $publication): void { + $this->queue($publication); + }); + } + + public function queue(AnalyticsPublication $publication): void + { + if (! in_array($publication->platform->value, Platform::analyticsValues(), true)) { + return; + } + + $now = CarbonImmutable::now('UTC'); + $isStory = $publication->content_type === PublicationContentType::Story + && in_array($publication->platform, [Platform::Instagram, Platform::InstagramFacebook], true); + + if ($isStory) { + if ($publication->provider_published_at->addDay()->greaterThan($now)) { + ScheduleInstagramStoryMetrics::dispatch($publication->id)->afterCommit(); + } + + return; + } + + $days = $publication->platform === Platform::X ? 20 : 30; + $recent = $publication->provider_published_at->greaterThanOrEqualTo($now->subDays($days)->startOfDay()); + + if (! $recent && $publication->dailySnapshots()->exists()) { + return; + } + + CollectPublicationMetrics::dispatch( + [$publication->id], + $now->toDateString(), + ! $recent, + )->afterCommit(); + } +} diff --git a/app/Console/Commands/Analytics/DispatchPublicationMetrics.php b/app/Console/Commands/Analytics/DispatchPublicationMetrics.php new file mode 100644 index 000000000..aa59418bb --- /dev/null +++ b/app/Console/Commands/Analytics/DispatchPublicationMetrics.php @@ -0,0 +1,54 @@ +connected() + ->active() + ->includedInAnalytics() + ->lazyById(100) + ->each(function (SocialAccount $account) use ($now): void { + $days = $account->platform === Platform::X ? 20 : 30; + $batchSize = match ($account->platform) { + Platform::Pinterest => 100, + Platform::TikTok => 20, + Platform::YouTube => 50, + default => 1, + }; + + AnalyticsPublication::query() + ->available() + ->where('social_account_id', $account->id) + ->where('provider_published_at', '>=', $now->subDays($days)->startOfDay()) + ->lazyById($batchSize) + ->chunk($batchSize) + ->each(function ($publications) use ($now): void { + CollectPublicationMetrics::dispatch( + $publications->pluck('id')->all(), + $now->toDateString(), + ); + }); + }); + + return self::SUCCESS; + } +} diff --git a/app/Jobs/Analytics/BackfillAccountPublications.php b/app/Jobs/Analytics/BackfillAccountPublications.php index 586cc7ce2..ceecc2f68 100644 --- a/app/Jobs/Analytics/BackfillAccountPublications.php +++ b/app/Jobs/Analytics/BackfillAccountPublications.php @@ -5,6 +5,7 @@ namespace App\Jobs\Analytics; use App\Actions\Analytics\AdvanceAnalyticsSyncState; +use App\Actions\Analytics\QueuePublicationMetricsForPage; use App\Enums\Analytics\SyncStatus; use App\Exceptions\Analytics\AnalyticsCollectionException; use App\Models\AnalyticsSyncState; @@ -57,6 +58,7 @@ public function backoff(): array public function handle( AdvanceAnalyticsSyncState $sync, PublicationHistoryCollectorFactory $collectors, + QueuePublicationMetricsForPage $metrics, ): void { $account = SocialAccount::query() ->connected() @@ -91,6 +93,8 @@ public function handle( return; } + $metrics->handle($account, $page); + if ($result['advanced'] && ! $result['terminal']) { self::dispatch($account->id, $this->syncStateId)->afterCommit(); } diff --git a/app/Jobs/Analytics/CollectPublicationMetrics.php b/app/Jobs/Analytics/CollectPublicationMetrics.php new file mode 100644 index 000000000..b8a349762 --- /dev/null +++ b/app/Jobs/Analytics/CollectPublicationMetrics.php @@ -0,0 +1,171 @@ + $publicationIds */ + public function __construct( + public array $publicationIds, + public string $observationDate, + public bool $baseline = false, + public bool $refreshSameDay = false, + public int $retryNumber = 0, + ) { + $this->onQueue('analytics'); + } + + /** @return list */ + public function middleware(): array + { + return [ + new RateLimited('analytics-publications'), + (new WithoutOverlapping('analytics-metrics:'.md5(implode(',', $this->publicationIds)).":{$this->observationDate}")) + ->releaseAfter(300) + ->expireAfter($this->timeout + 30), + ]; + } + + public function providerRateLimitKey(): string + { + $publication = AnalyticsPublication::query()->find($this->publicationIds[0] ?? ''); + + return $publication?->platform->network() ?? 'missing'; + } + + public function retryUntil(): CarbonImmutable + { + return CarbonImmutable::parse($this->observationDate, 'UTC')->endOfDay(); + } + + public function handle( + PublicationMetricsCollectorFactory $collectors, + WritePublicationDailySnapshot $writer, + ): void { + $date = CarbonImmutable::parse($this->observationDate, 'UTC'); + + foreach ($this->publicationIds as $publicationId) { + $publication = AnalyticsPublication::query()->available()->find($publicationId); + $account = $publication ? SocialAccount::query() + ->connected() + ->active() + ->includedInAnalytics() + ->find($publication->social_account_id) : null; + + if (! $publication || ! $account || ! $this->eligible($publication, $account, $date)) { + continue; + } + + $publication->setRelation('socialAccount', $account); + + try { + $observation = $collectors->for($publication->platform)->collect($publication, $date); + $writer->handle($publication, $observation); + } catch (AnalyticsCollectionException $exception) { + $this->retry($publication, $date, $exception->category, $exception->retryAt); + } catch (ConnectionException) { + $this->retry($publication, $date, 'transient', null); + } + } + } + + private function eligible(AnalyticsPublication $publication, SocialAccount $account, CarbonImmutable $date): bool + { + if ($account->workspace_id !== $publication->workspace_id + || $account->platform !== $publication->platform + || $account->platform_user_id !== $publication->platform_user_id) { + return false; + } + + $isStory = $publication->content_type === PublicationContentType::Story + && in_array($publication->platform, [Platform::Instagram, Platform::InstagramFacebook], true); + + if ($isStory && CarbonImmutable::now('UTC')->greaterThanOrEqualTo($publication->provider_published_at->addDay())) { + return false; + } + + if (! $this->baseline && ! $isStory) { + $ageLimit = $publication->platform === Platform::X ? 20 : 30; + + if ($publication->provider_published_at->lessThan($date->subDays($ageLimit)->startOfDay())) { + return false; + } + } + + return $this->refreshSameDay || ! AnalyticsPublicationDailySnapshot::query() + ->where('analytics_publication_id', $publication->id) + ->whereDate('snapshot_date', $this->observationDate) + ->exists(); + } + + private function retry( + AnalyticsPublication $publication, + CarbonImmutable $date, + string $category, + ?CarbonImmutable $providerRetryAt, + ): void { + if (! in_array($category, ['rate_limited', 'transient', 'delayed'], true) || $this->retryNumber >= 5) { + return; + } + + $now = CarbonImmutable::now('UTC'); + $isStory = $publication->content_type === PublicationContentType::Story; + $next = $isStory ? $now->addMinutes(30) : null; + + if (! $next) { + foreach ([6, 10, 14, 18, 22] as $hour) { + $window = $date->setTime($hour, 0); + + if ($window->greaterThan($now)) { + $next = $window; + + break; + } + } + } + + if (! $next) { + return; + } + + if ($providerRetryAt && $providerRetryAt->greaterThan($next)) { + $next = $providerRetryAt; + } + + if ($next->greaterThan($date->endOfDay()) + || ($isStory && $next->greaterThanOrEqualTo($publication->provider_published_at->addDay()))) { + return; + } + + self::dispatch( + [$publication->id], + $this->observationDate, + $this->baseline, + $this->refreshSameDay, + $this->retryNumber + 1, + )->delay($next)->afterCommit(); + } +} diff --git a/app/Jobs/Analytics/DiscoverAccountPublications.php b/app/Jobs/Analytics/DiscoverAccountPublications.php index 7669fa446..26d5bc0bc 100644 --- a/app/Jobs/Analytics/DiscoverAccountPublications.php +++ b/app/Jobs/Analytics/DiscoverAccountPublications.php @@ -5,6 +5,7 @@ namespace App\Jobs\Analytics; use App\Actions\Analytics\AdvanceAnalyticsSyncState; +use App\Actions\Analytics\QueuePublicationMetricsForPage; use App\Enums\Analytics\SyncCollector; use App\Enums\Analytics\SyncStatus; use App\Exceptions\Analytics\AnalyticsCollectionException; @@ -58,6 +59,7 @@ public function backoff(): array public function handle( AdvanceAnalyticsSyncState $sync, PublicationHistoryCollectorFactory $collectors, + QueuePublicationMetricsForPage $metrics, ): void { $account = SocialAccount::query() ->connected() @@ -102,6 +104,8 @@ public function handle( return; } + $metrics->handle($account, $page); + if ($result['advanced'] && ! $result['terminal']) { self::dispatch($account->id, $this->syncStateId)->afterCommit(); } diff --git a/app/Jobs/Analytics/ScheduleInstagramStoryMetrics.php b/app/Jobs/Analytics/ScheduleInstagramStoryMetrics.php new file mode 100644 index 000000000..b4bb6c5c9 --- /dev/null +++ b/app/Jobs/Analytics/ScheduleInstagramStoryMetrics.php @@ -0,0 +1,60 @@ +onQueue('analytics'); + } + + public function handle(): void + { + $publication = AnalyticsPublication::query()->available()->find($this->publicationId); + + if (! $publication + || $publication->content_type !== PublicationContentType::Story + || ! in_array($publication->platform, [Platform::Instagram, Platform::InstagramFacebook], true)) { + return; + } + + $now = CarbonImmutable::now('UTC'); + $publishedAt = $publication->provider_published_at; + $expiresAt = $publishedAt->addDay(); + + if ($now->greaterThanOrEqualTo($expiresAt)) { + return; + } + + $checkpoints = [ + $now, + $publishedAt->addHours(6), + $expiresAt->subMinutes(30), + ]; + + foreach ($checkpoints as $checkpoint) { + if ($checkpoint->lessThan($now) || $checkpoint->greaterThanOrEqualTo($expiresAt)) { + continue; + } + + CollectPublicationMetrics::dispatch( + [$publication->id], + $checkpoint->toDateString(), + false, + true, + )->delay($checkpoint)->afterCommit(); + } + } +} diff --git a/app/Jobs/Analytics/SyncTryPostPublication.php b/app/Jobs/Analytics/SyncTryPostPublication.php index a520df276..8f619188d 100644 --- a/app/Jobs/Analytics/SyncTryPostPublication.php +++ b/app/Jobs/Analytics/SyncTryPostPublication.php @@ -4,6 +4,7 @@ namespace App\Jobs\Analytics; +use App\Actions\Analytics\QueuePublicationMetricsForPage; use App\Actions\Analytics\SyncTryPostPublication as SyncTryPostPublicationAction; use App\Dto\Analytics\TryPostPublicationIdentity; use App\Models\PostPlatform; @@ -21,7 +22,7 @@ public function __construct( $this->onQueue('analytics'); } - public function handle(SyncTryPostPublicationAction $sync): void + public function handle(SyncTryPostPublicationAction $sync, QueuePublicationMetricsForPage $metrics): void { $postPlatform = PostPlatform::query() ->published() @@ -32,6 +33,7 @@ public function handle(SyncTryPostPublicationAction $sync): void return; } - $sync->fromIdentity($this->identity, $postPlatform); + $publication = $sync->fromIdentity($this->identity, $postPlatform); + $metrics->queue($publication); } } diff --git a/app/Models/AnalyticsPublication.php b/app/Models/AnalyticsPublication.php index 824cb1b40..29a0128f1 100644 --- a/app/Models/AnalyticsPublication.php +++ b/app/Models/AnalyticsPublication.php @@ -9,6 +9,7 @@ use App\Enums\Analytics\PublicationOrigin; use App\Enums\SocialAccount\Platform; use Database\Factories\AnalyticsPublicationFactory; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Concerns\HasUuids; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; @@ -64,4 +65,9 @@ public function dailySnapshots(): HasMany { return $this->hasMany(AnalyticsPublicationDailySnapshot::class); } + + public function scopeAvailable(Builder $query): Builder + { + return $query->where('availability', PublicationAvailability::Available); + } } diff --git a/routes/console.php b/routes/console.php index 6101cd5f2..5c594e217 100644 --- a/routes/console.php +++ b/routes/console.php @@ -4,6 +4,7 @@ use App\Console\Commands\Analytics\DispatchAccountDailyAnalytics; use App\Console\Commands\Analytics\DispatchPublicationDiscovery; +use App\Console\Commands\Analytics\DispatchPublicationMetrics; use App\Console\Commands\CheckSocialConnections; use App\Console\Commands\CheckUpcomingPostConnections; use App\Console\Commands\ProcessScheduledPosts; @@ -33,6 +34,11 @@ ->timezone('UTC') ->withoutOverlapping() ->onOneServer(); +Schedule::command(DispatchPublicationMetrics::class) + ->dailyAt('04:00') + ->timezone('UTC') + ->withoutOverlapping() + ->onOneServer(); Schedule::job(new FinalizeAccountDailySnapshots) ->dailyAt('23:30') ->timezone('UTC') diff --git a/tests/Feature/Analytics/AnalyticsScheduleTest.php b/tests/Feature/Analytics/AnalyticsScheduleTest.php index 6062a7517..d68393fd4 100644 --- a/tests/Feature/Analytics/AnalyticsScheduleTest.php +++ b/tests/Feature/Analytics/AnalyticsScheduleTest.php @@ -16,6 +16,10 @@ (string) $event->command, 'analytics:dispatch-publication-discovery', )); + $metrics = $events->first(fn ($event): bool => str_contains( + (string) $event->command, + 'analytics:dispatch-publication-metrics', + )); expect($collection)->not->toBeNull() ->and($collection->expression)->toBe('0 2 * * *') @@ -32,4 +36,9 @@ ->and($discovery->timezone)->toBe('UTC') ->and($discovery->withoutOverlapping)->toBeTrue() ->and($discovery->onOneServer)->toBeTrue(); + expect($metrics)->not->toBeNull() + ->and($metrics->expression)->toBe('0 4 * * *') + ->and($metrics->timezone)->toBe('UTC') + ->and($metrics->withoutOverlapping)->toBeTrue() + ->and($metrics->onOneServer)->toBeTrue(); }); diff --git a/tests/Feature/Analytics/PublicationMetricsJobsTest.php b/tests/Feature/Analytics/PublicationMetricsJobsTest.php new file mode 100644 index 000000000..3ef6d11db --- /dev/null +++ b/tests/Feature/Analytics/PublicationMetricsJobsTest.php @@ -0,0 +1,171 @@ +create(['platform' => $platform]); + + return AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + 'platform' => $platform, + 'network' => $platform->network(), + 'platform_user_id' => $account->platform_user_id, + 'provider_published_at' => $publishedAt, + ]); +} + +test('metric job writes a daily snapshot once and never calls providers for excluded networks', function () { + $date = CarbonImmutable::parse('2026-09-23 12:00:00', 'UTC'); + CarbonImmutable::setTestNow($date); + $included = metricJobPublication(Platform::Mastodon, $date->subDay()); + $excluded = metricJobPublication(Platform::LinkedIn, $date->subDay()); + Http::fake(['*' => Http::response(['favourites_count' => 0, 'replies_count' => 2])]); + + $job = new CollectPublicationMetrics([$included->id, $excluded->id], $date->toDateString()); + app()->call([$job, 'handle']); + app()->call([$job, 'handle']); + + expect(AnalyticsPublicationDailySnapshot::query()->where('analytics_publication_id', $included->id)->count())->toBe(1) + ->and(AnalyticsPublicationDailySnapshot::query()->where('analytics_publication_id', $excluded->id)->count())->toBe(0); + Http::assertSentCount(1); +}); + +test('regular collection respects the X and non-X refresh windows', function (Platform $platform, int $age, bool $eligible) { + $date = CarbonImmutable::parse('2026-09-23 12:00:00', 'UTC'); + CarbonImmutable::setTestNow($date); + $publication = metricJobPublication($platform, $date->subDays($age)); + Http::fake(['*' => Http::response($platform === Platform::X + ? ['data' => ['public_metrics' => ['like_count' => 1]]] + : ['favourites_count' => 1])]); + + app()->call([(new CollectPublicationMetrics([$publication->id], $date->toDateString())), 'handle']); + + expect(AnalyticsPublicationDailySnapshot::query()->where('analytics_publication_id', $publication->id)->exists()) + ->toBe($eligible, "{$platform->value} at {$age} days"); +})->with([ + [Platform::X, 20, true], + [Platform::X, 21, false], + [Platform::Mastodon, 30, true], + [Platform::Mastodon, 31, false], +]); + +test('an old imported publication receives one baseline measurement only', function () { + $date = CarbonImmutable::parse('2026-09-23 12:00:00', 'UTC'); + CarbonImmutable::setTestNow($date); + $publication = metricJobPublication(Platform::Mastodon, $date->subDays(180)); + Http::fake(['*' => Http::response(['favourites_count' => 5])]); + $job = new CollectPublicationMetrics([$publication->id], $date->toDateString(), baseline: true); + + app()->call([$job, 'handle']); + app()->call([$job, 'handle']); + + expect(AnalyticsPublicationDailySnapshot::query()->where('analytics_publication_id', $publication->id)->count())->toBe(1); + Http::assertSentCount(1); +}); + +test('story scheduler queues bounded in-lifetime checks', function () { + $date = CarbonImmutable::parse('2026-09-23 12:00:00', 'UTC'); + CarbonImmutable::setTestNow($date); + $publication = metricJobPublication(Platform::Instagram, $date); + $publication->update(['content_type' => PublicationContentType::Story]); + Bus::fake([CollectPublicationMetrics::class]); + + app()->call([(new ScheduleInstagramStoryMetrics($publication->id)), 'handle']); + + Bus::assertDispatched(CollectPublicationMetrics::class, 3); +}); + +test('transient metric failure preserves the last measured value and queues only the failed item', function () { + $date = CarbonImmutable::parse('2026-09-23 12:00:00', 'UTC'); + CarbonImmutable::setTestNow($date); + $publication = metricJobPublication(Platform::Mastodon, $date->subDay()); + AnalyticsPublicationDailySnapshot::factory()->create([ + 'analytics_publication_id' => $publication->id, + 'snapshot_date' => $date->subDay()->toDateString(), + 'reactions_count' => 6, + ]); + Http::fake(['*' => Http::response(['error' => 'rate limit'], 429)]); + Bus::fake([CollectPublicationMetrics::class]); + + app()->call([(new CollectPublicationMetrics([$publication->id], $date->toDateString())), 'handle']); + + expect(AnalyticsPublicationDailySnapshot::query()->where('analytics_publication_id', $publication->id)->count())->toBe(1) + ->and($publication->dailySnapshots()->first()->reactions_count)->toBe(6); + Bus::assertDispatched(CollectPublicationMetrics::class, fn (CollectPublicationMetrics $job): bool => $job->publicationIds === [$publication->id] && $job->retryNumber === 1); +}); + +test('a disconnected account is skipped even when its publication remains available', function () { + $date = CarbonImmutable::parse('2026-09-23 12:00:00', 'UTC'); + CarbonImmutable::setTestNow($date); + $publication = metricJobPublication(Platform::Mastodon, $date->subDay()); + $publication->socialAccount->update(['status' => Status::Disconnected]); + Http::fake(['*' => Http::response(['favourites_count' => 3])]); + + app()->call([(new CollectPublicationMetrics([$publication->id], $date->toDateString())), 'handle']); + + expect($publication->dailySnapshots()->exists())->toBeFalse(); + Http::assertNothingSent(); +}); + +test('an old discovered post queues a baseline only until it has one snapshot', function () { + $date = CarbonImmutable::parse('2026-09-23 12:00:00', 'UTC'); + CarbonImmutable::setTestNow($date); + $publication = metricJobPublication(Platform::Mastodon, $date->subDays(180)); + Bus::fake([CollectPublicationMetrics::class]); + + app(QueuePublicationMetricsForPage::class)->queue($publication); + + Bus::assertDispatched(CollectPublicationMetrics::class, fn (CollectPublicationMetrics $job): bool => $job->baseline); + AnalyticsPublicationDailySnapshot::factory()->create(['analytics_publication_id' => $publication->id]); + app(QueuePublicationMetricsForPage::class)->queue($publication); + Bus::assertDispatched(CollectPublicationMetrics::class, 1); +}); + +test('daily dispatcher batches per connected account and excludes old or v2 publications', function () { + $date = CarbonImmutable::parse('2026-09-23 12:00:00', 'UTC'); + CarbonImmutable::setTestNow($date); + $first = metricJobPublication(Platform::TikTok, $date->subDay()); + + for ($number = 0; $number < 20; $number++) { + AnalyticsPublication::factory()->create([ + 'workspace_id' => $first->workspace_id, + 'social_account_id' => $first->social_account_id, + 'social_account_key' => $first->social_account_key, + 'platform' => Platform::TikTok, + 'network' => Platform::TikTok->network(), + 'platform_user_id' => $first->platform_user_id, + 'provider_published_at' => $date->subDay(), + ]); + } + + metricJobPublication(Platform::LinkedIn, $date->subDay()); + metricJobPublication(Platform::Mastodon, $date->subDays(31)); + Bus::fake([CollectPublicationMetrics::class]); + + $this->artisan('analytics:dispatch-publication-metrics')->assertExitCode(0); + + Bus::assertDispatched(CollectPublicationMetrics::class, 2); +}); diff --git a/tests/Feature/Analytics/PublicationReconciliationTest.php b/tests/Feature/Analytics/PublicationReconciliationTest.php index ab362345f..11883fd17 100644 --- a/tests/Feature/Analytics/PublicationReconciliationTest.php +++ b/tests/Feature/Analytics/PublicationReconciliationTest.php @@ -102,7 +102,7 @@ }); $account->delete(); - $queuedJob->handle(app(SyncTryPostPublication::class)); + app()->call([$queuedJob, 'handle']); $publication = AnalyticsPublication::sole(); expect($publication->social_account_id)->toBeNull() From d272178fc57202622816155681dae9a648ad7bd6 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 12:50:02 -0300 Subject: [PATCH 24/77] feat: query workspace analytics reports --- app/Dto/Analytics/DateRange.php | 43 +++ .../Analytics/PublicationAnalyticsQuery.php | 101 ++++++ .../Analytics/WorkspaceAnalyticsQuery.php | 331 ++++++++++++++++++ app/Support/Analytics/PeriodBuckets.php | 54 +++ .../Analytics/WorkspaceAnalyticsQueryTest.php | 265 ++++++++++++++ 5 files changed, 794 insertions(+) create mode 100644 app/Dto/Analytics/DateRange.php create mode 100644 app/Queries/Analytics/PublicationAnalyticsQuery.php create mode 100644 app/Queries/Analytics/WorkspaceAnalyticsQuery.php create mode 100644 app/Support/Analytics/PeriodBuckets.php create mode 100644 tests/Feature/Analytics/WorkspaceAnalyticsQueryTest.php diff --git a/app/Dto/Analytics/DateRange.php b/app/Dto/Analytics/DateRange.php new file mode 100644 index 000000000..f0ac51291 --- /dev/null +++ b/app/Dto/Analytics/DateRange.php @@ -0,0 +1,43 @@ +start = $start->utc()->startOfDay(); + $this->end = $end->utc()->startOfDay(); + + if ($this->start->greaterThan($this->end)) { + throw new InvalidArgumentException('The analytics date range must start before it ends.'); + } + } + + public function days(): int + { + return (int) $this->start->diffInDays($this->end) + 1; + } + + public function previous(): self + { + $end = $this->start->subDay(); + + return new self($end->subDays($this->days() - 1), $end); + } + + /** @return array{start: string, end: string} */ + public function toArray(): array + { + return ['start' => $this->start->toDateString(), 'end' => $this->end->toDateString()]; + } +} diff --git a/app/Queries/Analytics/PublicationAnalyticsQuery.php b/app/Queries/Analytics/PublicationAnalyticsQuery.php new file mode 100644 index 000000000..7f1b55bb2 --- /dev/null +++ b/app/Queries/Analytics/PublicationAnalyticsQuery.php @@ -0,0 +1,101 @@ + */ + public function latestForPostPlatform(PostPlatform $postPlatform): array + { + if (! in_array($postPlatform->platform->value, Platform::analyticsValues(), true)) { + return $this->unavailable('platform_not_supported'); + } + + if ($postPlatform->status !== Status::Published || ! $postPlatform->platform_post_id) { + return $this->unavailable('not_published'); + } + + $publication = AnalyticsPublication::query() + ->where('workspace_id', $postPlatform->post->workspace_id) + ->where('post_platform_id', $postPlatform->id) + ->whereIn('platform', Platform::analyticsValues()) + ->first(); + + return $publication ? $this->latestForPublication($publication) : $this->unavailable('not_collected'); + } + + /** @return array */ + public function latestForPublication(AnalyticsPublication $publication): array + { + if (! in_array($publication->platform->value, Platform::analyticsValues(), true)) { + return $this->unavailable('platform_not_supported'); + } + + $snapshot = $publication->dailySnapshots()->orderByDesc('snapshot_date')->first(); + + return [ + 'available' => true, + 'reason' => null, + 'publication' => [ + 'id' => $publication->id, + 'post_platform_id' => $publication->post_platform_id, + 'social_account_key' => $publication->social_account_key, + 'platform' => $publication->platform->value, + 'origin' => $publication->origin->value, + 'content_type' => $publication->content_type->value, + 'availability' => $publication->availability->value, + 'provider_published_at' => $publication->provider_published_at?->toIso8601String(), + 'permalink' => $publication->permalink, + 'excerpt' => $publication->excerpt, + 'preview_metadata' => $publication->preview_metadata, + 'account_display_name' => $publication->account_display_name, + 'account_username' => $publication->account_username, + 'account_avatar_url' => $publication->account_avatar_url, + ], + 'snapshot' => $snapshot ? $this->snapshot($snapshot) : null, + 'metrics' => $snapshot?->metrics ?? [], + ]; + } + + /** @return array */ + private function snapshot(AnalyticsPublicationDailySnapshot $snapshot): array + { + return [ + 'date' => $snapshot->snapshot_date->toDateString(), + 'collected_at' => $snapshot->collected_at?->toIso8601String(), + 'provider_observed_at' => $snapshot->provider_observed_at?->toIso8601String(), + 'reactions_count' => $snapshot->reactions_count, + 'comments_count' => $snapshot->comments_count, + 'shares_count' => $snapshot->shares_count, + 'saves_count' => $snapshot->saves_count, + 'views_count' => $snapshot->views_count, + 'impressions_count' => $snapshot->impressions_count, + 'reach_count' => $snapshot->reach_count, + 'engagement_count' => $snapshot->engagement_count, + 'exposure_count' => $snapshot->exposure_count, + 'exposure_kind' => $snapshot->exposure_kind?->value, + 'watch_time_milliseconds' => $snapshot->watch_time_milliseconds, + 'average_watch_time_milliseconds' => $snapshot->average_watch_time_milliseconds, + ]; + } + + /** @return array{available: false, reason: string, publication: null, snapshot: null, metrics: array} */ + private function unavailable(string $reason): array + { + return [ + 'available' => false, + 'reason' => $reason, + 'publication' => null, + 'snapshot' => null, + 'metrics' => [], + ]; + } +} diff --git a/app/Queries/Analytics/WorkspaceAnalyticsQuery.php b/app/Queries/Analytics/WorkspaceAnalyticsQuery.php new file mode 100644 index 000000000..6f48bb7b1 --- /dev/null +++ b/app/Queries/Analytics/WorkspaceAnalyticsQuery.php @@ -0,0 +1,331 @@ + */ + public function for(Workspace $workspace, DateRange $range): array + { + $previous = $range->previous(); + $publications = $this->publications($workspace, $previous->start, $range->end); + $followers = $this->followerRows($workspace, $previous->end, $range); + $currentPublications = $publications->filter(fn (object $row): bool => $this->inRange($row->provider_published_at, $range)); + $previousPublications = $publications->filter(fn (object $row): bool => $this->inRange($row->provider_published_at, $previous)); + $currentFollowers = $this->followerTotal($followers, $range->end); + $previousFollowers = $this->followerTotal($followers, $previous->end); + $current = $this->totals($currentPublications); + $prior = $this->totals($previousPublications); + + return [ + 'bounds' => $this->bounds($workspace), + 'range' => $range->toArray(), + 'previous_range' => $previous->toArray(), + 'summary' => [ + 'posts' => $this->comparison($current['posts'], $prior['posts']), + 'followers' => [ + 'value' => $currentFollowers, + 'previous' => $previousFollowers, + 'change' => $currentFollowers !== null && $previousFollowers !== null + ? $currentFollowers - $previousFollowers : null, + ], + 'reactions' => $this->comparison($current['reactions'], $prior['reactions']), + 'comments' => $this->comparison($current['comments'], $prior['comments']), + 'engagement_rate' => $this->comparison($current['engagement_rate'], $prior['engagement_rate']), + ], + 'followers' => $this->followers($followers, $range, $currentFollowers), + 'posts' => $this->posts($currentPublications, $range), + 'top_posts' => [ + 'reactions' => $this->top($currentPublications, 'reactions_count'), + 'comments' => $this->top($currentPublications, 'comments_count'), + ], + 'performance' => $this->performance($currentPublications, $previousPublications), + 'coverage' => $this->coverage($workspace), + ]; + } + + private function publications(Workspace $workspace, CarbonImmutable $start, CarbonImmutable $end): Collection + { + $latest = DB::table('analytics_publication_daily_snapshots as daily') + ->join('analytics_publications as parent', 'parent.id', '=', 'daily.analytics_publication_id') + ->where('parent.workspace_id', $workspace->id) + ->select('daily.analytics_publication_id') + ->selectRaw('MAX(daily.snapshot_date) as latest_date') + ->groupBy('daily.analytics_publication_id'); + + return DB::table('analytics_publications as publication') + ->leftJoinSub($latest, 'latest', 'latest.analytics_publication_id', '=', 'publication.id') + ->leftJoin('analytics_publication_daily_snapshots as metric', function ($join): void { + $join->on('metric.analytics_publication_id', '=', 'publication.id') + ->on('metric.snapshot_date', '=', 'latest.latest_date'); + }) + ->where('publication.workspace_id', $workspace->id) + ->whereIn('publication.platform', Platform::analyticsValues()) + ->whereBetween('publication.provider_published_at', [$start->startOfDay(), $end->endOfDay()]) + ->select([ + 'publication.id', 'publication.social_account_key', 'publication.social_account_id', + 'publication.post_platform_id', 'publication.platform', 'publication.network', + 'publication.account_display_name', 'publication.account_username', + 'publication.account_avatar_url', 'publication.provider_post_id', + 'publication.provider_published_at', 'publication.origin', 'publication.content_type', + 'publication.permalink', 'publication.excerpt', 'publication.preview_metadata', + 'metric.reactions_count', 'metric.comments_count', 'metric.shares_count', + 'metric.saves_count', 'metric.views_count', 'metric.impressions_count', + 'metric.reach_count', 'metric.engagement_count', 'metric.exposure_count', + 'metric.exposure_kind', 'metric.collected_at', + ]) + ->get(); + } + + private function followerRows(Workspace $workspace, CarbonImmutable $previousEnd, DateRange $range): Collection + { + return DB::table('analytics_account_daily_snapshots') + ->where('workspace_id', $workspace->id) + ->whereIn('platform', Platform::analyticsValues()) + ->where(function ($query) use ($previousEnd, $range): void { + $query->whereDate('snapshot_date', $previousEnd->toDateString()) + ->orWhereBetween('snapshot_date', [$range->start->toDateString(), $range->end->toDateString()]); + }) + ->select([ + 'social_account_key', 'social_account_id', 'platform', 'network', + 'account_display_name', 'account_username', 'account_avatar_url', + 'snapshot_date', 'followers_count', 'provenance', 'precision', 'collected_at', + ]) + ->orderBy('snapshot_date') + ->get(); + } + + /** @return array{min: ?string, max: ?string} */ + private function bounds(Workspace $workspace): array + { + $accounts = DB::table('analytics_account_daily_snapshots') + ->where('workspace_id', $workspace->id) + ->whereIn('platform', Platform::analyticsValues()) + ->selectRaw('MIN(snapshot_date) as earliest, MAX(snapshot_date) as latest') + ->first(); + $publications = DB::table('analytics_publications') + ->where('workspace_id', $workspace->id) + ->whereIn('platform', Platform::analyticsValues()) + ->selectRaw('MIN(provider_published_at) as earliest, MAX(provider_published_at) as latest') + ->first(); + $minimum = array_filter([$accounts?->earliest, $publications?->earliest]); + $maximum = array_filter([$accounts?->latest, $publications?->latest]); + + return [ + 'min' => $minimum ? CarbonImmutable::parse(min($minimum), 'UTC')->toDateString() : null, + 'max' => $maximum ? CarbonImmutable::parse(max($maximum), 'UTC')->toDateString() : null, + ]; + } + + private function followerTotal(Collection $rows, CarbonImmutable $date): ?int + { + $onDate = $rows->filter(fn (object $row): bool => substr((string) $row->snapshot_date, 0, 10) === $date->toDateString() + && $row->followers_count !== null); + + return $onDate->isEmpty() ? null : (int) $onDate->sum('followers_count'); + } + + /** @return array */ + private function followers(Collection $rows, DateRange $range, ?int $total): array + { + $current = $rows->filter(fn (object $row): bool => $this->inRange($row->snapshot_date, $range)); + $byAccount = $current->groupBy('social_account_key'); + $accounts = []; + + foreach ($byAccount as $key => $values) { + $first = $values->first(); + $last = $values->last(); + $end = $values->first(fn (object $row): bool => substr((string) $row->snapshot_date, 0, 10) === $range->end->toDateString()); + $accounts[] = [ + 'social_account_key' => $key, + 'social_account_id' => $last->social_account_id, + 'platform' => $last->platform, + 'network' => $last->network, + 'name' => $last->account_display_name, + 'username' => $last->account_username, + 'avatar_url' => $last->account_avatar_url, + 'value' => $end?->followers_count === null ? null : (int) $end->followers_count, + 'growth' => $values->count() > 1 && $first->followers_count !== null && $last->followers_count !== null + ? (int) $last->followers_count - (int) $first->followers_count : null, + 'provenance' => $end?->provenance, + ]; + } + + usort($accounts, fn (array $a, array $b): int => [$a['platform'], $a['username'], $a['social_account_key']] + <=> [$b['platform'], $b['username'], $b['social_account_key']]); + $series = []; + $byDate = $current->groupBy(fn (object $row): string => substr((string) $row->snapshot_date, 0, 10)); + + for ($day = $range->start; $day->lessThanOrEqualTo($range->end); $day = $day->addDay()) { + $date = $day->toDateString(); + $values = array_fill_keys(array_column($accounts, 'social_account_key'), null); + + foreach ($byDate->get($date, collect()) as $row) { + $values[$row->social_account_key] = $row->followers_count === null ? null : (int) $row->followers_count; + } + + $series[] = ['date' => $date, 'accounts' => $values]; + } + + return ['total' => $total, 'accounts' => $accounts, 'series' => $series]; + } + + /** @return array */ + private function posts(Collection $rows, DateRange $range): array + { + $accounts = $rows->groupBy('social_account_key')->map(function (Collection $items, string $key): array { + $row = $items->first(); + + return [ + 'social_account_key' => $key, + 'platform' => $row->platform, + 'name' => $row->account_display_name, + 'username' => $row->account_username, + 'avatar_url' => $row->account_avatar_url, + 'count' => $items->count(), + ]; + })->values()->all(); + $buckets = $this->buckets->for($range); + + foreach ($buckets as &$bucket) { + $counts = array_fill_keys(array_column($accounts, 'social_account_key'), 0); + + foreach ($rows as $row) { + $date = substr((string) $row->provider_published_at, 0, 10); + + if ($date >= $bucket['start'] && $date <= $bucket['end']) { + $counts[$row->social_account_key]++; + } + } + + $bucket['accounts'] = $counts; + $bucket['total'] = array_sum($counts); + } + + return [ + 'resolution' => $this->buckets->resolution($range), + 'accounts' => $accounts, + 'buckets' => $buckets, + ]; + } + + /** @return array */ + private function totals(Collection $rows): array + { + $reactions = $rows->filter(fn (object $row): bool => $row->reactions_count !== null); + $comments = $rows->filter(fn (object $row): bool => $row->comments_count !== null); + $rateRows = $rows->filter(fn (object $row): bool => $row->engagement_count !== null + && $row->exposure_count !== null && (int) $row->exposure_count > 0); + $exposure = (int) $rateRows->sum('exposure_count'); + + return [ + 'posts' => $rows->count(), + 'reactions' => $reactions->isEmpty() ? null : (int) $reactions->sum('reactions_count'), + 'comments' => $comments->isEmpty() ? null : (int) $comments->sum('comments_count'), + 'engagement_rate' => $exposure === 0 ? null : round(((int) $rateRows->sum('engagement_count')) / $exposure * 100, 2), + ]; + } + + /** @return array{value: int|float|null, previous: int|float|null, change: ?float} */ + private function comparison(int|float|null $current, int|float|null $previous): array + { + return [ + 'value' => $current, + 'previous' => $previous, + 'change' => $current !== null && $previous !== null && $previous != 0 + ? round(($current - $previous) / $previous * 100, 2) : null, + ]; + } + + /** @return list> */ + private function top(Collection $rows, string $metric): array + { + $eligible = $rows->filter(fn (object $row): bool => $row->{$metric} !== null)->all(); + usort($eligible, fn (object $a, object $b): int => ((int) $b->{$metric} <=> (int) $a->{$metric}) + ?: strcmp((string) $b->provider_published_at, (string) $a->provider_published_at) + ?: strcmp((string) $a->id, (string) $b->id)); + + return array_map(function (object $row): array { + return [ + 'id' => $row->id, + 'post_platform_id' => $row->post_platform_id, + 'social_account_key' => $row->social_account_key, + 'platform' => $row->platform, + 'name' => $row->account_display_name, + 'username' => $row->account_username, + 'origin' => $row->origin, + 'content_type' => $row->content_type, + 'published_at' => $row->provider_published_at, + 'permalink' => $row->permalink, + 'excerpt' => $row->excerpt, + 'preview_metadata' => $row->preview_metadata ? json_decode((string) $row->preview_metadata, true) : null, + 'reactions' => $row->reactions_count === null ? null : (int) $row->reactions_count, + 'comments' => $row->comments_count === null ? null : (int) $row->comments_count, + ]; + }, array_slice($eligible, 0, 5)); + } + + /** @return list> */ + private function performance(Collection $current, Collection $previous): array + { + $previousByAccount = $previous->groupBy('social_account_key'); + $rows = []; + + foreach ($current->groupBy('social_account_key') as $key => $items) { + $representative = $items->first(); + $totals = $this->totals($items); + $prior = $this->totals($previousByAccount->get($key, collect())); + $rows[] = [ + 'social_account_key' => $key, + 'platform' => $representative->platform, + 'name' => $representative->account_display_name, + 'username' => $representative->account_username, + 'avatar_url' => $representative->account_avatar_url, + 'posts' => $this->comparison($totals['posts'], $prior['posts']), + 'reactions' => $this->comparison($totals['reactions'], $prior['reactions']), + 'comments' => $this->comparison($totals['comments'], $prior['comments']), + 'engagement_rate' => $this->comparison($totals['engagement_rate'], $prior['engagement_rate']), + ]; + } + + usort($rows, fn (array $a, array $b): int => [$a['platform'], $a['username'], $a['social_account_key']] + <=> [$b['platform'], $b['username'], $b['social_account_key']]); + + return $rows; + } + + /** @return list */ + private function coverage(Workspace $workspace): array + { + return DB::table('analytics_sync_states as state') + ->join('social_accounts as account', 'account.id', '=', 'state.social_account_id') + ->where('account.workspace_id', $workspace->id) + ->whereIn('account.platform', Platform::analyticsValues()) + ->select([ + 'state.social_account_id', 'state.collector', 'state.status', + 'state.target_since', 'state.oldest_reached_at', 'state.high_watermark_at', + 'state.last_success_at', 'state.last_error_category', + ]) + ->get() + ->all(); + } + + private function inRange(string $date, DateRange $range): bool + { + $day = substr($date, 0, 10); + + return $day >= $range->start->toDateString() && $day <= $range->end->toDateString(); + } +} diff --git a/app/Support/Analytics/PeriodBuckets.php b/app/Support/Analytics/PeriodBuckets.php new file mode 100644 index 000000000..a7f94b877 --- /dev/null +++ b/app/Support/Analytics/PeriodBuckets.php @@ -0,0 +1,54 @@ +days() <= 14 => 'daily', + $range->days() <= 90 => 'weekly', + default => 'monthly', + }; + } + + /** @return list */ + public function for(DateRange $range): array + { + $resolution = $this->resolution($range); + $cursor = $range->start; + $buckets = []; + + while ($cursor->lessThanOrEqualTo($range->end)) { + $boundary = match ($resolution) { + 'weekly' => $cursor->endOfWeek(), + 'monthly' => $cursor->endOfMonth(), + default => $cursor, + }; + $last = $boundary->lessThan($range->end) ? $boundary : $range->end; + $buckets[] = [ + 'start' => $cursor->toDateString(), + 'end' => $last->toDateString(), + 'label' => $resolution === 'monthly' + ? $cursor->format('M Y') + : $this->label($cursor, $last), + ]; + $cursor = $last->addDay()->startOfDay(); + } + + return $buckets; + } + + private function label(CarbonImmutable $start, CarbonImmutable $end): string + { + return $start->isSameDay($end) + ? $start->format('M j') + : $start->format('M j').' – '.$end->format('M j'); + } +} diff --git a/tests/Feature/Analytics/WorkspaceAnalyticsQueryTest.php b/tests/Feature/Analytics/WorkspaceAnalyticsQueryTest.php new file mode 100644 index 000000000..f34fd7ce6 --- /dev/null +++ b/tests/Feature/Analytics/WorkspaceAnalyticsQueryTest.php @@ -0,0 +1,265 @@ +create(['workspace_id' => $workspace->id, 'platform' => $platform]); +} + +function analyticsReportFollower(SocialAccount $account, string $date, int $followers, ObservationProvenance $provenance = ObservationProvenance::Actual): void +{ + AnalyticsAccountDailySnapshot::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + 'network' => $account->platform->network(), + 'platform_user_id' => $account->platform_user_id, + 'platform' => $account->platform, + 'snapshot_date' => $date, + 'followers_count' => $followers, + 'provenance' => $provenance, + ]); +} + +function analyticsReportPublication(SocialAccount $account, string $publishedAt, ?int $reactions, ?int $comments, ?int $engagement, ?int $exposure): AnalyticsPublication +{ + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $account->workspace_id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + 'network' => $account->platform->network(), + 'platform_user_id' => $account->platform_user_id, + 'platform' => $account->platform, + 'provider_published_at' => CarbonImmutable::parse($publishedAt, 'UTC'), + ]); + AnalyticsPublicationDailySnapshot::factory()->create([ + 'analytics_publication_id' => $publication->id, + 'snapshot_date' => '2026-09-12', + 'reactions_count' => $reactions, + 'comments_count' => $comments, + 'engagement_count' => $engagement, + 'exposure_count' => $exposure, + ]); + + return $publication; +} + +test('workspace report keeps accounts separate and aggregates only latest normalized facts', function () { + $workspace = Workspace::factory()->create(); + $instagramA = analyticsReportAccount($workspace, Platform::Instagram); + $instagramB = analyticsReportAccount($workspace, Platform::Instagram); + $x = analyticsReportAccount($workspace, Platform::X); + $linkedIn = analyticsReportAccount($workspace, Platform::LinkedIn); + $foreign = analyticsReportAccount(Workspace::factory()->create(), Platform::Instagram); + + foreach ([[$instagramA, 95], [$instagramB, 19], [$x, 45]] as [$account, $followers]) { + analyticsReportFollower($account, '2026-08-31', $followers); + } + analyticsReportFollower($instagramA, '2026-09-10', 100); + analyticsReportFollower($instagramB, '2026-09-10', 20, ObservationProvenance::CarriedForward); + analyticsReportFollower($x, '2026-09-10', 50); + analyticsReportFollower($linkedIn, '2026-09-10', 1000); + analyticsReportFollower($foreign, '2026-09-10', 999); + + $first = analyticsReportPublication($instagramA, '2026-09-05 10:00:00', 10, 2, 12, 100); + AnalyticsPublicationDailySnapshot::factory()->create([ + 'analytics_publication_id' => $first->id, + 'snapshot_date' => '2026-09-11', + 'reactions_count' => 999, + 'comments_count' => 999, + 'engagement_count' => 999, + 'exposure_count' => 999, + ]); + analyticsReportPublication($instagramB, '2026-09-06 10:00:00', 5, 1, 6, 900); + analyticsReportPublication($x, '2026-09-07 10:00:00', null, 0, null, null); + analyticsReportPublication($instagramA, '2026-08-25 10:00:00', 2, 1, 3, 30); + analyticsReportPublication($linkedIn, '2026-09-05 10:00:00', 900, 900, 900, 1); + analyticsReportPublication($foreign, '2026-09-05 10:00:00', 900, 900, 900, 1); + + $report = app(WorkspaceAnalyticsQuery::class)->for( + $workspace, + new DateRange(CarbonImmutable::parse('2026-09-01', 'UTC'), CarbonImmutable::parse('2026-09-10', 'UTC')), + ); + + expect(array_keys($report['summary']))->toBe(['posts', 'followers', 'reactions', 'comments', 'engagement_rate']) + ->and($report['summary']['posts']['value'])->toBe(3) + ->and($report['summary']['posts']['change'])->toBe(200.0) + ->and($report['summary']['followers']['value'])->toBe(170) + ->and($report['summary']['followers']['change'])->toBe(11) + ->and($report['summary']['reactions']['value'])->toBe(15) + ->and($report['summary']['comments']['value'])->toBe(3) + ->and($report['summary']['engagement_rate']['value'])->toBe(1.8) + ->and($report['bounds'])->toBe(['min' => '2026-08-25', 'max' => '2026-09-10']) + ->and($report['posts']['resolution'])->toBe('daily') + ->and(count($report['posts']['buckets']))->toBe(10) + ->and(count($report['performance']))->toBe(3) + ->and(count($report['followers']['accounts']))->toBe(3) + ->and($report['followers']['total'])->toBe(170) + ->and($report['top_posts']['reactions'][0]['social_account_key'])->toBe($instagramA->id) + ->and($report['top_posts']['reactions'][0]['reactions'])->toBe(10); + + $performanceKeys = array_column($report['performance'], 'social_account_key'); + expect($performanceKeys)->toContain($instagramA->id, $instagramB->id, $x->id); +}); + +test('range boundaries and bucket resolutions are deterministic', function (int $days, string $resolution) { + $range = new DateRange( + CarbonImmutable::parse('2026-09-23', 'UTC')->subDays($days - 1), + CarbonImmutable::parse('2026-09-23', 'UTC'), + ); + + expect(app(PeriodBuckets::class)->resolution($range))->toBe($resolution) + ->and($range->previous()->days())->toBe($days); +})->with([[14, 'daily'], [15, 'weekly'], [90, 'weekly'], [91, 'monthly']]); + +test('invalid reversed date range is rejected', function () { + expect(fn () => new DateRange( + CarbonImmutable::parse('2026-09-10', 'UTC'), + CarbonImmutable::parse('2026-09-01', 'UTC'), + ))->toThrow(InvalidArgumentException::class); +}); + +test('a deleted social account retains its historical performance row', function () { + $workspace = Workspace::factory()->create(); + $account = analyticsReportAccount($workspace, Platform::Instagram); + analyticsReportPublication($account, '2026-09-05 10:00:00', 0, null, 0, 100); + $account->delete(); + + $report = app(WorkspaceAnalyticsQuery::class)->for( + $workspace, + new DateRange(CarbonImmutable::parse('2026-09-01', 'UTC'), CarbonImmutable::parse('2026-09-10', 'UTC')), + ); + + expect($report['performance'][0]['social_account_key'])->toBe($account->id) + ->and($report['summary']['reactions']['value'])->toBe(0) + ->and($report['summary']['comments']['value'])->toBeNull(); +}); + +test('publication detail uses the latest persisted metric snapshot without provider reads', function () { + Http::fake(); + $workspace = Workspace::factory()->create(); + $account = analyticsReportAccount($workspace, Platform::Instagram); + $post = Post::factory()->create(['workspace_id' => $workspace->id]); + $destination = PostPlatform::factory()->instagram()->published()->create([ + 'post_id' => $post->id, + 'social_account_id' => $account->id, + ]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $workspace->id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + 'post_platform_id' => $destination->id, + 'platform' => Platform::Instagram, + 'content_type' => PublicationContentType::Reel, + ]); + AnalyticsPublicationDailySnapshot::factory()->create([ + 'analytics_publication_id' => $publication->id, + 'snapshot_date' => '2026-09-10', + 'reactions_count' => 50, + 'metrics' => ['reactions' => ['value' => 50, 'unit' => 'count', 'availability' => 'available']], + ]); + AnalyticsPublicationDailySnapshot::factory()->create([ + 'analytics_publication_id' => $publication->id, + 'snapshot_date' => '2026-09-11', + 'reactions_count' => 0, + 'watch_time_milliseconds' => 185000, + 'metrics' => [ + 'reactions' => ['value' => 0, 'unit' => 'count', 'availability' => 'available'], + 'watch_time_milliseconds' => ['value' => 185000, 'unit' => 'milliseconds', 'availability' => 'available'], + ], + ]); + + $detail = app(PublicationAnalyticsQuery::class)->latestForPostPlatform($destination); + + expect($detail['available'])->toBeTrue() + ->and($detail['publication']['content_type'])->toBe('reel') + ->and($detail['snapshot']['reactions_count'])->toBe(0) + ->and($detail['snapshot']['watch_time_milliseconds'])->toBe(185000) + ->and($detail['metrics']['watch_time_milliseconds']['unit'])->toBe('milliseconds'); + Http::assertNothingSent(); +}); + +test('publication detail remains scoped to the post workspace and excludes unsupported networks', function () { + $workspace = Workspace::factory()->create(); + $foreign = analyticsReportAccount(Workspace::factory()->create(), Platform::Instagram); + $post = Post::factory()->create(['workspace_id' => $workspace->id]); + $destination = PostPlatform::factory()->instagram()->published()->create([ + 'post_id' => $post->id, + 'social_account_id' => $foreign->id, + ]); + AnalyticsPublication::factory()->create([ + 'workspace_id' => $foreign->workspace_id, + 'post_platform_id' => $destination->id, + 'platform' => Platform::Instagram, + ]); + + expect(app(PublicationAnalyticsQuery::class)->latestForPostPlatform($destination)['available'])->toBeFalse(); + + $destination->update(['platform' => Platform::LinkedIn]); + expect(app(PublicationAnalyticsQuery::class)->latestForPostPlatform($destination)['reason'])->toBe('platform_not_supported'); +}); + +test('external YouTube upload stays a video and not a short without authoritative metadata', function () { + $workspace = Workspace::factory()->create(); + $account = analyticsReportAccount($workspace, Platform::YouTube); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $workspace->id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + 'platform' => Platform::YouTube, + 'content_type' => PublicationContentType::Video, + ]); + + $detail = app(PublicationAnalyticsQuery::class)->latestForPublication($publication); + + expect($detail['publication']['content_type'])->toBe('video') + ->and($detail['snapshot'])->toBeNull(); +}); + +test('dashboard query count is independent of the number of account rows', function () { + $workspace = Workspace::factory()->create(); + $range = new DateRange(CarbonImmutable::parse('2026-09-01', 'UTC'), CarbonImmutable::parse('2026-09-10', 'UTC')); + $queries = []; + DB::listen(function ($query) use (&$queries): void { + $queries[] = $query->sql; + }); + app(WorkspaceAnalyticsQuery::class)->for($workspace, $range); + $baseCount = count($queries); + $queries = []; + + foreach (range(1, 3) as $index) { + $account = analyticsReportAccount($workspace, Platform::Instagram); + analyticsReportFollower($account, '2026-09-10', $index); + analyticsReportPublication($account, '2026-09-05 10:00:00', $index, 0, $index, 100); + } + $queries = []; + app(WorkspaceAnalyticsQuery::class)->for($workspace, $range); + + expect(count($queries))->toBe($baseCount); +}); From 0d27a163e90bdd2845c10dd5e6f4735b4cbe12cd Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 12:56:07 -0300 Subject: [PATCH 25/77] feat: read analytics exclusively from database --- .../Controllers/App/AnalyticsController.php | 133 ++++-------------- .../Resources/Api/PostMetricsResource.php | 3 +- app/Mcp/Tools/Post/GetPostMetricsTool.php | 2 +- .../Analytics/PublicationAnalyticsQuery.php | 8 +- .../Analytics/WorkspaceAnalyticsQuery.php | 4 +- app/Services/Post/PostMetricsFetcher.php | 80 +++-------- routes/app.php | 1 - .../Analytics/AnalyticsControllerTest.php | 85 +++++++++++ .../PersistedPostMetricsReadTest.php | 86 +++++++++++ tests/Feature/AnalyticsResilienceTest.php | 55 +++----- .../Social/GoogleBusinessAnalyticsTest.php | 42 ++---- .../Services/Social/TikTokAnalyticsTest.php | 29 ++-- tests/Feature/YouTubeAnalyticsTest.php | 51 ++++--- 13 files changed, 296 insertions(+), 283 deletions(-) create mode 100644 tests/Feature/Analytics/AnalyticsControllerTest.php create mode 100644 tests/Feature/Analytics/PersistedPostMetricsReadTest.php diff --git a/app/Http/Controllers/App/AnalyticsController.php b/app/Http/Controllers/App/AnalyticsController.php index cb2600472..84d83282c 100644 --- a/app/Http/Controllers/App/AnalyticsController.php +++ b/app/Http/Controllers/App/AnalyticsController.php @@ -4,133 +4,52 @@ namespace App\Http\Controllers\App; -use App\Enums\SocialAccount\Platform; -use App\Exceptions\PlatformUnavailableException; +use App\Dto\Analytics\DateRange; use App\Http\Controllers\Controller; -use App\Models\SocialAccount; -use App\Services\Social\FacebookAnalytics; -use App\Services\Social\GoogleBusinessAnalytics; -use App\Services\Social\InstagramAnalytics; -use App\Services\Social\LinkedInPageAnalytics; -use App\Services\Social\PinterestAnalytics; -use App\Services\Social\Telegram\TelegramAnalytics; -use App\Services\Social\ThreadsAnalytics; -use App\Services\Social\TikTokAnalytics; -use App\Services\Social\XAnalytics; -use App\Services\Social\YouTubeAnalytics; -use Illuminate\Http\Client\ConnectionException; -use Illuminate\Http\JsonResponse; +use App\Queries\Analytics\WorkspaceAnalyticsQuery; +use Carbon\CarbonImmutable; use Illuminate\Http\Request; -use Illuminate\Support\Carbon; use Inertia\Inertia; use Inertia\Response; -use Symfony\Component\HttpFoundation\Response as HttpResponse; class AnalyticsController extends Controller { - private const SUPPORTED_PLATFORMS = [ - Platform::TikTok, - Platform::Instagram, - Platform::InstagramFacebook, - Platform::Threads, - Platform::Facebook, - Platform::X, - Platform::LinkedInPage, - Platform::Pinterest, - Platform::YouTube, - Platform::Telegram, - Platform::GoogleBusiness, - ]; - - public function index(Request $request): Response + public function index(Request $request, WorkspaceAnalyticsQuery $analytics): Response { $workspace = $request->user()->currentWorkspace; $this->authorize('view', $workspace); - $accounts = $workspace->socialAccounts() - ->active() - ->whereIn('platform', self::SUPPORTED_PLATFORMS) - ->get() - ->map(fn (SocialAccount $account) => [ - 'id' => $account->id, - 'platform' => $account->platform->value, - 'username' => $account->username, - 'display_label' => $account->display_label, - 'avatar_url' => $account->avatar_url, - ]); - - return Inertia::render('analytics/Index', [ - 'accounts' => $accounts, + $validated = $request->validate([ + 'start' => ['sometimes', 'required', 'date_format:Y-m-d'], + 'end' => ['sometimes', 'required', 'date_format:Y-m-d', 'after_or_equal:start'], ]); - } - - public function show(Request $request, SocialAccount $account): JsonResponse - { - $workspace = $request->user()->currentWorkspace; + $bounds = $analytics->boundsFor($workspace); + $today = CarbonImmutable::today('UTC'); + $end = $bounds['max'] ? CarbonImmutable::parse($bounds['max'], 'UTC') : $today; + $start = $end->subDays(29); - if ($account->workspace_id !== $workspace->id) { - abort(HttpResponse::HTTP_FORBIDDEN); + if (isset($validated['start'])) { + $start = CarbonImmutable::parse($validated['start'], 'UTC'); } - $since = $request->has('since') ? Carbon::parse($request->input('since')) : null; - $until = $request->has('until') ? Carbon::parse($request->input('until')) : null; - - $metrics = $this->metricsFor($account, $since, $until); - - // Google aggregates search keywords by month, so they cannot be folded - // into the daily metric cards and travel as their own list. - if ($account->platform === Platform::GoogleBusiness) { - return response()->json([ - 'metrics' => $metrics, - 'keywords' => $this->searchKeywordsFor($account, $since, $until), - ]); + if (isset($validated['end'])) { + $end = CarbonImmutable::parse($validated['end'], 'UTC'); } - return response()->json(['metrics' => $metrics]); - } - - /** - * @return array - */ - private function searchKeywordsFor(SocialAccount $account, ?Carbon $since, ?Carbon $until): array - { - try { - return app(GoogleBusinessAnalytics::class)->getSearchKeywords($account, $since, $until); - } catch (PlatformUnavailableException|ConnectionException $e) { - report($e); - - return []; + if ($bounds['min'] !== null && $bounds['max'] !== null) { + $minimum = CarbonImmutable::parse($bounds['min'], 'UTC'); + $maximum = CarbonImmutable::parse($bounds['max'], 'UTC'); + $start = $start->lessThan($minimum) ? $minimum : ($start->greaterThan($maximum) ? $maximum : $start); + $end = $end->lessThan($minimum) ? $minimum : ($end->greaterThan($maximum) ? $maximum : $end); } - } - /** - * An unreachable platform is not a server error — empty numbers beat a 500 - * on a page the user just opened. Narrow on purpose: catching Throwable - * would render a defect as "this account has no activity". - * - * @return array - */ - private function metricsFor(SocialAccount $account, ?Carbon $since, ?Carbon $until): array - { - try { - return match ($account->platform) { - Platform::TikTok => app(TikTokAnalytics::class)->getMetrics($account), - Platform::Instagram, Platform::InstagramFacebook => app(InstagramAnalytics::class)->getMetrics($account, $since, $until), - Platform::Threads => app(ThreadsAnalytics::class)->getMetrics($account, $since, $until), - Platform::Facebook => app(FacebookAnalytics::class)->getMetrics($account, $since, $until), - Platform::X => app(XAnalytics::class)->getMetrics($account, $since, $until), - Platform::LinkedInPage => app(LinkedInPageAnalytics::class)->getMetrics($account, $since, $until), - Platform::Pinterest => app(PinterestAnalytics::class)->getMetrics($account, $since, $until), - Platform::YouTube => app(YouTubeAnalytics::class)->getMetrics($account, $since, $until), - Platform::Telegram => app(TelegramAnalytics::class)->getMetrics($account), - Platform::GoogleBusiness => app(GoogleBusinessAnalytics::class)->getMetrics($account, $since, $until), - default => [], - }; - } catch (PlatformUnavailableException|ConnectionException $e) { - report($e); - - return []; + if ($start->greaterThan($end)) { + $start = $end; } + + return Inertia::render('analytics/Index', [ + 'report' => $analytics->for($workspace, new DateRange($start, $end)), + ]); } } diff --git a/app/Http/Resources/Api/PostMetricsResource.php b/app/Http/Resources/Api/PostMetricsResource.php index 6724a00fc..fb26b85a8 100644 --- a/app/Http/Resources/Api/PostMetricsResource.php +++ b/app/Http/Resources/Api/PostMetricsResource.php @@ -9,8 +9,7 @@ use Illuminate\Http\Resources\Json\JsonResource; /** - * Wraps a Post with its per-platform engagement metrics. The actual fetching - * (with cache + per-platform dispatch) is delegated to PostMetricsFetcher. + * Wraps a Post with its persisted per-platform engagement metrics. */ class PostMetricsResource extends JsonResource { diff --git a/app/Mcp/Tools/Post/GetPostMetricsTool.php b/app/Mcp/Tools/Post/GetPostMetricsTool.php index b6ceb14f4..157a92fb4 100644 --- a/app/Mcp/Tools/Post/GetPostMetricsTool.php +++ b/app/Mcp/Tools/Post/GetPostMetricsTool.php @@ -15,7 +15,7 @@ use Laravel\Mcp\Server\Tools\Annotations\IsReadOnly; #[IsReadOnly] -#[Description('Fetch engagement metrics (likes, comments, shares, etc.) for a published post across all platforms it was posted to. Returns "unsupported" entries for platforms that do not expose post-level metrics or for unpublished platforms.')] +#[Description('Read the latest saved engagement metrics for a post across its published platforms. Values may lag the provider until the next analytics job. Returns "unsupported" for excluded or unpublished platforms.')] class GetPostMetricsTool extends Tool { public function handle(Request $request): Response|ResponseFactory diff --git a/app/Queries/Analytics/PublicationAnalyticsQuery.php b/app/Queries/Analytics/PublicationAnalyticsQuery.php index 7f1b55bb2..ed0471ff3 100644 --- a/app/Queries/Analytics/PublicationAnalyticsQuery.php +++ b/app/Queries/Analytics/PublicationAnalyticsQuery.php @@ -15,14 +15,14 @@ class PublicationAnalyticsQuery /** @return array */ public function latestForPostPlatform(PostPlatform $postPlatform): array { - if (! in_array($postPlatform->platform->value, Platform::analyticsValues(), true)) { - return $this->unavailable('platform_not_supported'); - } - if ($postPlatform->status !== Status::Published || ! $postPlatform->platform_post_id) { return $this->unavailable('not_published'); } + if (! in_array($postPlatform->platform->value, Platform::analyticsValues(), true)) { + return $this->unavailable('platform_not_supported'); + } + $publication = AnalyticsPublication::query() ->where('workspace_id', $postPlatform->post->workspace_id) ->where('post_platform_id', $postPlatform->id) diff --git a/app/Queries/Analytics/WorkspaceAnalyticsQuery.php b/app/Queries/Analytics/WorkspaceAnalyticsQuery.php index 6f48bb7b1..1cc51c705 100644 --- a/app/Queries/Analytics/WorkspaceAnalyticsQuery.php +++ b/app/Queries/Analytics/WorkspaceAnalyticsQuery.php @@ -30,7 +30,7 @@ public function for(Workspace $workspace, DateRange $range): array $prior = $this->totals($previousPublications); return [ - 'bounds' => $this->bounds($workspace), + 'bounds' => $this->boundsFor($workspace), 'range' => $range->toArray(), 'previous_range' => $previous->toArray(), 'summary' => [ @@ -108,7 +108,7 @@ private function followerRows(Workspace $workspace, CarbonImmutable $previousEnd } /** @return array{min: ?string, max: ?string} */ - private function bounds(Workspace $workspace): array + public function boundsFor(Workspace $workspace): array { $accounts = DB::table('analytics_account_daily_snapshots') ->where('workspace_id', $workspace->id) diff --git a/app/Services/Post/PostMetricsFetcher.php b/app/Services/Post/PostMetricsFetcher.php index dec6438a9..4ee46255b 100644 --- a/app/Services/Post/PostMetricsFetcher.php +++ b/app/Services/Post/PostMetricsFetcher.php @@ -4,85 +4,43 @@ namespace App\Services\Post; -use App\Enums\SocialAccount\Platform; use App\Models\Post; use App\Models\PostPlatform; -use App\Services\Social\BlueskyAnalytics; -use App\Services\Social\Discord\DiscordAnalytics; -use App\Services\Social\FacebookAnalytics; -use App\Services\Social\InstagramAnalytics; -use App\Services\Social\LinkedInAnalytics; -use App\Services\Social\LinkedInPageAnalytics; -use App\Services\Social\MastodonAnalytics; -use App\Services\Social\PinterestAnalytics; -use App\Services\Social\Telegram\TelegramAnalytics; -use App\Services\Social\ThreadsAnalytics; -use App\Services\Social\TikTokAnalytics; -use App\Services\Social\XAnalytics; -use App\Services\Social\YouTubeAnalytics; +use App\Queries\Analytics\PublicationAnalyticsQuery; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Cache; /** - * Fetches per-platform post metrics. Used by the web controller, the REST - * API controller, and the MCP `GetPostMetricsTool` so that the per-platform - * dispatch and 5-minute cache stay in one place. + * Local-only facade shared by web, REST, and MCP post detail reads. */ class PostMetricsFetcher { - /** - * @return Collection|array{unsupported: true, reason: string} - * }> - */ + public function __construct(private readonly PublicationAnalyticsQuery $publications) {} + + /** @return Collection> */ public function forPost(Post $post): Collection { return $post->postPlatforms ->where('enabled', true) ->values() - ->map(function (PostPlatform $pp): array { - $metrics = $this->forPlatform($pp); - - return [ - 'post_platform_id' => $pp->id, - 'platform' => $pp->platform->value, - 'status' => $pp->status->value, - 'platform_post_id' => $pp->platform_post_id, - 'platform_url' => $pp->platform_url, - 'metrics' => $metrics, - ]; - }); + ->map(fn (PostPlatform $destination): array => [ + 'post_platform_id' => $destination->id, + 'platform' => $destination->platform->value, + 'status' => $destination->status->value, + 'platform_post_id' => $destination->platform_post_id, + 'platform_url' => $destination->platform_url, + 'metrics' => $this->forPlatform($destination), + ]); } - /** - * @return array|array{unsupported: true, reason: string} - */ + /** @return array */ public function forPlatform(PostPlatform $postPlatform): array { - if ($postPlatform->status->value !== 'published' || ! $postPlatform->platform_post_id) { - return ['unsupported' => true, 'reason' => 'not_published']; + $detail = $this->publications->latestForPostPlatform($postPlatform); + + if (! $detail['available']) { + return ['unsupported' => true, 'reason' => $detail['reason']]; } - return Cache::remember("post_metrics:{$postPlatform->id}", 300, fn () => match ($postPlatform->platform) { - Platform::X => app(XAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::Bluesky => app(BlueskyAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::Mastodon => app(MastodonAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::Telegram => app(TelegramAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::Discord => app(DiscordAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::Instagram, Platform::InstagramFacebook => app(InstagramAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::Facebook => app(FacebookAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::Threads => app(ThreadsAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::LinkedIn => app(LinkedInAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::LinkedInPage => app(LinkedInPageAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::YouTube => app(YouTubeAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::Pinterest => app(PinterestAnalytics::class)->fetchPostMetrics($postPlatform), - Platform::TikTok => app(TikTokAnalytics::class)->fetchPostMetrics($postPlatform), - default => ['unsupported' => true, 'reason' => 'platform_not_supported'], - }); + return $detail; } } diff --git a/routes/app.php b/routes/app.php index 8ffcbdff7..35656fa24 100644 --- a/routes/app.php +++ b/routes/app.php @@ -189,7 +189,6 @@ // Analytics Route::get('analytics', [AnalyticsController::class, 'index'])->name('app.analytics'); - Route::get('analytics/{account}', [AnalyticsController::class, 'show'])->name('app.analytics.show'); // Calendar Route::get('calendar', [PostController::class, 'calendar'])->name('app.calendar'); diff --git a/tests/Feature/Analytics/AnalyticsControllerTest.php b/tests/Feature/Analytics/AnalyticsControllerTest.php new file mode 100644 index 000000000..192e84720 --- /dev/null +++ b/tests/Feature/Analytics/AnalyticsControllerTest.php @@ -0,0 +1,85 @@ +create(); + $workspace = Workspace::factory()->create(['user_id' => $user->id]); + $user->update(['current_workspace_id' => $workspace->id]); + $account = SocialAccount::factory()->create(['workspace_id' => $workspace->id, 'platform' => Platform::Instagram]); + $foreign = SocialAccount::factory()->create(['workspace_id' => Workspace::factory()->create()->id, 'platform' => Platform::Instagram]); + foreach ([[$account, 12], [$foreign, 999]] as [$socialAccount, $followers]) { + AnalyticsAccountDailySnapshot::factory()->create([ + 'workspace_id' => $socialAccount->workspace_id, + 'social_account_id' => $socialAccount->id, + 'social_account_key' => $socialAccount->id, + 'platform' => $socialAccount->platform, + 'network' => $socialAccount->platform->network(), + 'platform_user_id' => $socialAccount->platform_user_id, + 'snapshot_date' => '2026-09-20', + 'followers_count' => $followers, + ]); + } + Http::fake(); + + $this->actingAs($user) + ->get(route('app.analytics', ['start' => '2026-01-01', 'end' => '2026-12-31'])) + ->assertOk() + ->assertInertia(fn (Assert $page) => $page + ->component('analytics/Index') + ->where('report.range.start', '2026-09-20') + ->where('report.range.end', '2026-09-20') + ->where('report.summary.followers.value', 12) + ->etc()); + + Http::assertNothingSent(); +}); + +test('dashboard rejects invalid dates and never renders an excluded account', function () { + $user = User::factory()->create(); + $workspace = Workspace::factory()->create(['user_id' => $user->id]); + $user->update(['current_workspace_id' => $workspace->id]); + $linkedIn = SocialAccount::factory()->create(['workspace_id' => $workspace->id, 'platform' => Platform::LinkedIn]); + AnalyticsAccountDailySnapshot::factory()->create([ + 'workspace_id' => $workspace->id, + 'social_account_id' => $linkedIn->id, + 'social_account_key' => $linkedIn->id, + 'platform' => Platform::LinkedIn, + 'network' => Platform::LinkedIn->network(), + 'platform_user_id' => $linkedIn->platform_user_id, + 'snapshot_date' => '2026-09-20', + 'followers_count' => 100, + ]); + + $this->actingAs($user) + ->get(route('app.analytics', ['start' => 'nonsense', 'end' => '2026-09-20'])) + ->assertSessionHasErrors('start'); + + $this->actingAs($user) + ->get(route('app.analytics', ['start' => '2026-09-20', 'end' => '2026-09-10'])) + ->assertSessionHasErrors('end'); + + $this->actingAs($user) + ->get(route('app.analytics')) + ->assertOk() + ->assertInertia(fn (Assert $page) => $page + ->component('analytics/Index') + ->where('report.bounds.min', null) + ->where('report.summary.followers.value', null) + ->etc()); +}); diff --git a/tests/Feature/Analytics/PersistedPostMetricsReadTest.php b/tests/Feature/Analytics/PersistedPostMetricsReadTest.php new file mode 100644 index 000000000..434ecbbdd --- /dev/null +++ b/tests/Feature/Analytics/PersistedPostMetricsReadTest.php @@ -0,0 +1,86 @@ +create(['workspace_id' => $workspace->id, 'platform' => Platform::Instagram]); + $post = Post::factory()->published()->create(['workspace_id' => $workspace->id, 'user_id' => $user->id]); + $destination = PostPlatform::factory()->instagram()->published()->create([ + 'post_id' => $post->id, + 'social_account_id' => $account->id, + 'platform_post_id' => '123', + ]); + $publication = AnalyticsPublication::factory()->create([ + 'workspace_id' => $workspace->id, + 'social_account_id' => $account->id, + 'social_account_key' => $account->id, + 'post_platform_id' => $destination->id, + 'platform' => Platform::Instagram, + 'content_type' => PublicationContentType::Reel, + ]); + AnalyticsPublicationDailySnapshot::factory()->create([ + 'analytics_publication_id' => $publication->id, + 'snapshot_date' => '2026-09-20', + 'reactions_count' => 7, + 'watch_time_milliseconds' => 180000, + 'metrics' => [ + 'reactions' => ['value' => 7, 'unit' => 'count', 'availability' => 'available'], + 'watch_time_milliseconds' => ['value' => 180000, 'unit' => 'milliseconds', 'availability' => 'available'], + ], + ]); + Http::fake(); + + $this->actingAs($user) + ->getJson(route('app.posts.platforms.metrics', [$post, $destination])) + ->assertOk() + ->assertJsonPath('publication.content_type', 'reel') + ->assertJsonPath('snapshot.reactions_count', 7) + ->assertJsonPath('metrics.watch_time_milliseconds.value', 180000); + + $this->withHeaders(['Authorization' => 'Bearer '.$access['plain_token']]) + ->getJson(route('api.posts.metrics', $post)) + ->assertOk() + ->assertJsonPath('platforms.0.metrics.metrics.reactions.value', 7); + + expect(app(PostMetricsFetcher::class)->forPlatform($destination)['snapshot']['reactions_count'])->toBe(7); + Http::assertNothingSent(); +}); + +test('excluded destinations expose no analytics and a foreign post cannot be read', function () { + Queue::fake([BootstrapAccountAnalytics::class, CollectAccountDailySnapshot::class]); + $access = createApiTestToken(); + $account = SocialAccount::factory()->create(['workspace_id' => $access['workspace']->id, 'platform' => Platform::LinkedIn]); + $post = Post::factory()->published()->create(['workspace_id' => $access['workspace']->id, 'user_id' => $access['user']->id]); + $destination = PostPlatform::factory()->linkedin()->published()->create([ + 'post_id' => $post->id, + 'social_account_id' => $account->id, + ]); + + $this->actingAs($access['user']) + ->getJson(route('app.posts.platforms.metrics', [$post, $destination])) + ->assertOk() + ->assertJsonPath('unsupported', true) + ->assertJsonPath('reason', 'platform_not_supported'); + + $foreignPost = Post::factory()->published()->create(); + $this->actingAs($access['user']) + ->getJson(route('app.posts.platforms.metrics', [$foreignPost, $destination])) + ->assertNotFound(); +}); diff --git a/tests/Feature/AnalyticsResilienceTest.php b/tests/Feature/AnalyticsResilienceTest.php index a1928d220..56f31e57b 100644 --- a/tests/Feature/AnalyticsResilienceTest.php +++ b/tests/Feature/AnalyticsResilienceTest.php @@ -2,59 +2,38 @@ declare(strict_types=1); -use App\Enums\SocialAccount\Status; -use App\Models\SocialAccount; use App\Models\User; use App\Models\Workspace; -use App\Services\Social\XAnalytics; -use Illuminate\Support\Facades\Cache; +use App\Queries\Analytics\WorkspaceAnalyticsQuery; use Illuminate\Support\Facades\Http; -test('analytics degrades to empty when a refresh is already in flight', function () { +test('workspace analytics remains available when a provider is unavailable', function () { $user = User::factory()->create(); $workspace = Workspace::factory()->create(['user_id' => $user->id]); $user->update(['current_workspace_id' => $workspace->id]); + Http::fake(['*' => Http::response([], 503)]); - $account = SocialAccount::factory()->x()->create([ - 'workspace_id' => $workspace->id, - 'status' => Status::Connected, - 'platform_user_id' => '4242', - // Expired, so the analytics service tries to refresh before reading. - 'token_expires_at' => now()->subMinutes(5), - ]); - - Http::fake(['*' => Http::response(['data' => [], 'meta' => []], 200)]); - - // The scheduled RefreshSocialToken is mid-refresh for this account. - Cache::lock("token_refresh:{$account->id}", 120)->get(); - - $response = $this->actingAs($user)->getJson(route('app.analytics.show', $account)); - - // A transient collision is not a server error. Before the lock started - // reporting itself as transient this returned empty metrics, and a 500 on - // a page the user just opened is a worse answer than no numbers. - $response->assertOk(); - expect($response->json('metrics'))->toBe([]); + $this->actingAs($user) + ->get(route('app.analytics')) + ->assertOk() + ->assertInertia(fn ($page) => $page + ->component('analytics/Index') + ->where('report.summary.followers.value', null) + ->etc()); + + Http::assertNothingSent(); }); -test('a bug in a metrics service is not hidden behind empty numbers', function () { +test('a read-model defect is not hidden behind empty numbers', function () { $user = User::factory()->create(); $workspace = Workspace::factory()->create(['user_id' => $user->id]); $user->update(['current_workspace_id' => $workspace->id]); - $account = SocialAccount::factory()->x()->create([ - 'workspace_id' => $workspace->id, - 'status' => Status::Connected, - 'token_expires_at' => now()->addHours(2), - ]); - - $this->mock(XAnalytics::class) - ->shouldReceive('getMetrics') - ->andThrow(new RuntimeException('a real bug, not the platform being down')); + $this->mock(WorkspaceAnalyticsQuery::class) + ->shouldReceive('boundsFor') + ->andThrow(new RuntimeException('a real report bug')); - // Degrading to [] here would show the user an empty dashboard and leave a - // genuine defect looking like "this account has no activity". $this->actingAs($user) - ->getJson(route('app.analytics.show', $account)) + ->get(route('app.analytics')) ->assertStatus(500); }); diff --git a/tests/Feature/Services/Social/GoogleBusinessAnalyticsTest.php b/tests/Feature/Services/Social/GoogleBusinessAnalyticsTest.php index 26e5c7b2a..78e7e2364 100644 --- a/tests/Feature/Services/Social/GoogleBusinessAnalyticsTest.php +++ b/tests/Feature/Services/Social/GoogleBusinessAnalyticsTest.php @@ -2,7 +2,6 @@ declare(strict_types=1); -use App\Enums\SocialAccount\Platform; use App\Enums\SocialAccount\Status as AccountStatus; use App\Enums\UserWorkspace\Role; use App\Models\Account; @@ -95,41 +94,25 @@ Http::assertNothingSent(); }); -test('google business is listed on the analytics page', function () { +test('google business is excluded from workspace analytics', function () { $response = $this->actingAs($this->user)->get(route('app.analytics')); $response->assertOk(); - $accounts = $response->original->getData()['page']['props']['accounts']; + $report = $response->original->getData()['page']['props']['report']; - expect(collect($accounts)->firstWhere('platform', Platform::GoogleBusiness->value))->not->toBeNull(); + expect($report['bounds']['min'])->toBeNull() + ->and($report['followers']['accounts'])->toBe([]); }); -test('the analytics show endpoint returns google business metrics', function () { - Http::fake([ - config('trypost.platforms.google_business.performance_api').'/*' => Http::response([ - 'multiDailyMetricTimeSeries' => [ - [ - 'dailyMetricTimeSeries' => [ - [ - 'dailyMetric' => 'WEBSITE_CLICKS', - 'timeSeries' => ['datedValues' => [['value' => '7']]], - ], - ], - ], - ], - ], 200), - ]); +test('workspace analytics does not request google business reporting', function () { + Http::fake(); $response = $this->actingAs($this->user) - ->getJson(route('app.analytics.show', $this->socialAccount)); - - $response->assertOk()->assertJsonCount(8, 'metrics'); + ->get(route('app.analytics')); - $metrics = $response->json('metrics'); - - expect(collect($metrics)->pluck('label')->filter())->toHaveCount(8) - ->and(collect($metrics)->firstWhere('label', __('analytics.metrics.website_clicks'))['value'])->toBe(7); + $response->assertOk(); + Http::assertNothingSent(); }); test('caches the metrics so a repeated call within the window does not hit the api again', function () { @@ -291,7 +274,7 @@ }); }); -test('the analytics endpoint returns the search keywords alongside the metrics', function () { +test('the low-level google business service retains search keyword support outside V1 analytics', function () { Http::fake([ config('trypost.platforms.google_business.performance_api').'/*/searchkeywords/*' => Http::response([ 'searchKeywordsCounts' => [['searchKeyword' => 'coffee near me', 'insightsValue' => ['value' => '320']]], @@ -299,8 +282,7 @@ config('trypost.platforms.google_business.performance_api').'/*' => Http::response(['multiDailyMetricTimeSeries' => []]), ]); - $response = $this->actingAs($this->user) - ->getJson(route('app.analytics.show', $this->socialAccount)); + $keywords = $this->analytics->getSearchKeywords($this->socialAccount); - $response->assertOk()->assertJsonPath('keywords.0.keyword', 'coffee near me'); + expect($keywords[0]['keyword'])->toBe('coffee near me'); }); diff --git a/tests/Feature/Services/Social/TikTokAnalyticsTest.php b/tests/Feature/Services/Social/TikTokAnalyticsTest.php index 3fe9fd5f8..b9bb04dad 100644 --- a/tests/Feature/Services/Social/TikTokAnalyticsTest.php +++ b/tests/Feature/Services/Social/TikTokAnalyticsTest.php @@ -5,6 +5,8 @@ use App\Enums\PostPlatform\Status as PostPlatformStatus; use App\Enums\SocialAccount\Platform; use App\Enums\TikTok\PrivacyLevel; +use App\Models\AnalyticsPublication; +use App\Models\AnalyticsPublicationDailySnapshot; use App\Models\Post; use App\Models\PostPlatform; use App\Models\SocialAccount; @@ -280,25 +282,30 @@ function tiktokPostPlatform(?string $platformPostId = '7685359243088103444'): Po Http::assertSentCount(2); }); -test('tiktok analytics forPost returns the backfilled video url alongside the metrics', function () { +test('tiktok post metrics facade returns the saved video url and metrics without provider reads', function () { $videoId = '7685359243088103444'; - $postPlatform = tiktokPostPlatform('v_pub_url~v2-1.backfill'); - $postPlatform->update(['status' => PostPlatformStatus::Published]); - - Http::fake([ - $this->api.'/post/publish/status/fetch/' => Http::response([ - 'data' => ['status' => 'PUBLISH_COMPLETE', 'publicaly_available_post_id' => [$videoId]], - 'error' => ['code' => 'ok'], - ]), - $this->api.'/video/query/*' => Http::response(tiktokVideoQueryResponse($videoId, ['view_count' => 5])), + $postPlatform = tiktokPostPlatform($videoId); + $postPlatform->update([ + 'status' => PostPlatformStatus::Published, + 'platform_url' => "https://www.tiktok.com/@tiktoker/video/{$videoId}", ]); + $publication = AnalyticsPublication::query()->where('post_platform_id', $postPlatform->id)->firstOrFail(); + AnalyticsPublicationDailySnapshot::factory()->create([ + 'analytics_publication_id' => $publication->id, + 'views_count' => 5, + 'metrics' => ['views' => ['value' => 5, 'unit' => 'count', 'availability' => 'available']], + ]); + + Http::fake(); $platforms = app(PostMetricsFetcher::class)->forPost($this->post->fresh()); expect($platforms->first())->toMatchArray([ 'platform_post_id' => $videoId, 'platform_url' => "https://www.tiktok.com/@tiktoker/video/{$videoId}", - ])->and($platforms->first()['metrics'][0])->toBe(['label' => __('analytics.metrics.views'), 'value' => 5]); + ])->and($platforms->first()['metrics']['metrics']['views']['value'])->toBe(5); + + Http::assertNothingSent(); }); test('tiktok analytics reports a missing platform post id as unsupported', function () { diff --git a/tests/Feature/YouTubeAnalyticsTest.php b/tests/Feature/YouTubeAnalyticsTest.php index 57231a368..da8e479e8 100644 --- a/tests/Feature/YouTubeAnalyticsTest.php +++ b/tests/Feature/YouTubeAnalyticsTest.php @@ -7,6 +7,7 @@ use App\Enums\UserWorkspace\Role; use App\Exceptions\TokenExpiredException; use App\Models\Account; +use App\Models\AnalyticsAccountDailySnapshot; use App\Models\SocialAccount; use App\Models\User; use App\Models\Workspace; @@ -196,46 +197,43 @@ $analytics->getMetrics($this->youtubeAccount); })->throws(TokenExpiredException::class); -test('youtube is in supported analytics platforms', function () { +test('youtube follower facts appear in the workspace analytics report', function () { config(['trypost.self_hosted' => true]); + AnalyticsAccountDailySnapshot::factory()->create([ + 'workspace_id' => $this->workspace->id, + 'social_account_id' => $this->youtubeAccount->id, + 'social_account_key' => $this->youtubeAccount->id, + 'platform' => Platform::YouTube, + 'network' => Platform::YouTube->network(), + 'platform_user_id' => $this->youtubeAccount->platform_user_id, + 'snapshot_date' => '2026-09-23', + 'followers_count' => 50, + ]); + $response = $this->actingAs($this->user) ->get(route('app.analytics')); $response->assertOk(); - $accounts = $response->original->getData()['page']['props']['accounts']; - $youtubeAccount = collect($accounts)->firstWhere('platform', Platform::YouTube->value); + $report = $response->original->getData()['page']['props']['report']; - expect($youtubeAccount)->not->toBeNull() - ->and($youtubeAccount['id'])->toBe($this->youtubeAccount->id) - ->and($youtubeAccount)->toHaveKeys(['display_label']); + expect($report['summary']['followers']['value'])->toBe(50) + ->and($report['followers']['accounts'][0]['social_account_key'])->toBe($this->youtubeAccount->id); }); -test('youtube analytics show endpoint returns metrics', function () { +test('youtube analytics dashboard never calls the provider on read', function () { config(['trypost.self_hosted' => true]); - - Http::fake([ - 'https://youtubeanalytics.googleapis.com/v2/reports*' => Http::response([ - 'columnHeaders' => [ - ['name' => 'views'], - ['name' => 'likes'], - ], - 'rows' => [ - [500, 30], - ], - ], 200), - ]); + Http::fake(); $response = $this->actingAs($this->user) - ->getJson(route('app.analytics.show', $this->youtubeAccount)); + ->get(route('app.analytics')); - $response->assertOk() - ->assertJsonStructure(['metrics']) - ->assertJsonCount(2, 'metrics'); + $response->assertOk(); + Http::assertNothingSent(); }); -test('youtube analytics show endpoint rejects other workspace accounts', function () { +test('youtube facts in another workspace do not leak into the report', function () { config(['trypost.self_hosted' => true]); $otherUser = User::factory()->create([]); @@ -244,7 +242,8 @@ $otherUser->update(['current_workspace_id' => $otherWorkspace->id]); $response = $this->actingAs($otherUser) - ->getJson(route('app.analytics.show', $this->youtubeAccount)); + ->get(route('app.analytics')); - $response->assertForbidden(); + $response->assertOk(); + expect($response->original->getData()['page']['props']['report']['summary']['followers']['value'])->toBeNull(); }); From c56c4d9250552f01df89b0abc94be40e565c83c1 Mon Sep 17 00:00:00 2001 From: Paulo Castellano Date: Wed, 23 Sep 2026 13:07:40 -0300 Subject: [PATCH 26/77] feat: add workspace analytics dashboard --- lang/ar/analytics.php | 31 +++ lang/de/analytics.php | 31 +++ lang/el/analytics.php | 31 +++ lang/en/analytics.php | 31 +++ lang/es/analytics.php | 31 +++ lang/fr/analytics.php | 31 +++ lang/it/analytics.php | 31 +++ lang/ja/analytics.php | 31 +++ lang/ko/analytics.php | 31 +++ lang/nl/analytics.php | 31 +++ lang/pl/analytics.php | 31 +++ lang/pt-BR/analytics.php | 31 +++ lang/ru/analytics.php | 31 +++ lang/tr/analytics.php | 31 +++ lang/uk/analytics.php | 31 +++ lang/zh/analytics.php | 31 +++ .../analytics/AnalyticsAccountSelector.vue | 125 ---------- .../analytics/FacebookAnalytics.vue | 61 ----- .../analytics/GoogleBusinessAnalytics.vue | 108 --------- .../analytics/InstagramAnalytics.vue | 61 ----- .../analytics/LinkedInPageAnalytics.vue | 61 ----- .../js/components/analytics/MetricsGrid.vue | 138 ----------- .../analytics/PinterestAnalytics.vue | 61 ----- .../analytics/TelegramAnalytics.vue | 57 ----- .../components/analytics/ThreadsAnalytics.vue | 61 ----- .../components/analytics/TikTokAnalytics.vue | 50 ---- .../js/components/analytics/XAnalytics.vue | 61 ----- .../components/analytics/YouTubeAnalytics.vue | 61 ----- resources/js/components/analytics/types.ts | 7 - .../analytics/workspace/AccountIdentity.vue | 38 +++ .../analytics/workspace/AnalyticsSection.vue | 22 ++ .../analytics/workspace/FollowersChart.vue | 113 +++++++++ .../analytics/workspace/ImportCoverage.vue | 22 ++ .../analytics/workspace/PerformanceTable.vue | 121 ++++++++++ .../analytics/workspace/PostsChart.vue | 84 +++++++ .../analytics/workspace/SummaryCards.vue | 91 ++++++++ .../analytics/workspace/TopPosts.vue | 119 ++++++++++ .../workspace/charts/HorizontalBarChart.vue | 67 ++++++ .../analytics/workspace/charts/LineChart.vue | 148 ++++++++++++ .../workspace/charts/StackedBarChart.vue | 43 ++++ .../components/analytics/workspace/types.ts | 106 +++++++++ .../ui/date-range-picker/DateRangePicker.vue | 18 +- resources/js/lib/googleBusiness.ts | 1 - resources/js/pages/analytics/Index.vue | 218 +++++++----------- resources/js/pages/posts/Show.vue | 2 +- tests/Browser/WorkspaceAnalyticsTest.php | 98 ++++++++ 46 files changed, 1670 insertions(+), 1049 deletions(-) delete mode 100644 resources/js/components/analytics/AnalyticsAccountSelector.vue delete mode 100644 resources/js/components/analytics/FacebookAnalytics.vue delete mode 100644 resources/js/components/analytics/GoogleBusinessAnalytics.vue delete mode 100644 resources/js/components/analytics/InstagramAnalytics.vue delete mode 100644 resources/js/components/analytics/LinkedInPageAnalytics.vue delete mode 100644 resources/js/components/analytics/MetricsGrid.vue delete mode 100644 resources/js/components/analytics/PinterestAnalytics.vue delete mode 100644 resources/js/components/analytics/TelegramAnalytics.vue delete mode 100644 resources/js/components/analytics/ThreadsAnalytics.vue delete mode 100644 resources/js/components/analytics/TikTokAnalytics.vue delete mode 100644 resources/js/components/analytics/XAnalytics.vue delete mode 100644 resources/js/components/analytics/YouTubeAnalytics.vue delete mode 100644 resources/js/components/analytics/types.ts create mode 100644 resources/js/components/analytics/workspace/AccountIdentity.vue create mode 100644 resources/js/components/analytics/workspace/AnalyticsSection.vue create mode 100644 resources/js/components/analytics/workspace/FollowersChart.vue create mode 100644 resources/js/components/analytics/workspace/ImportCoverage.vue create mode 100644 resources/js/components/analytics/workspace/PerformanceTable.vue create mode 100644 resources/js/components/analytics/workspace/PostsChart.vue create mode 100644 resources/js/components/analytics/workspace/SummaryCards.vue create mode 100644 resources/js/components/analytics/workspace/TopPosts.vue create mode 100644 resources/js/components/analytics/workspace/charts/HorizontalBarChart.vue create mode 100644 resources/js/components/analytics/workspace/charts/LineChart.vue create mode 100644 resources/js/components/analytics/workspace/charts/StackedBarChart.vue create mode 100644 resources/js/components/analytics/workspace/types.ts create mode 100644 tests/Browser/WorkspaceAnalyticsTest.php diff --git a/lang/ar/analytics.php b/lang/ar/analytics.php index adee88c5c..9d1e8033f 100644 --- a/lang/ar/analytics.php +++ b/lang/ar/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'لا توجد حسابات متصلة تحتوي على تحليلات.', 'no_accounts_match' => 'لا توجد حسابات مطابقة.', 'search_account' => 'البحث عن حساب…', diff --git a/lang/de/analytics.php b/lang/de/analytics.php index 8d9de7433..b8077ae20 100644 --- a/lang/de/analytics.php +++ b/lang/de/analytics.php @@ -3,6 +3,37 @@ declare(strict_types=1); return [ + 'dashboard' => [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'Keine verbundenen Konten mit Analysedaten.', 'no_accounts_match' => 'Keine passenden Konten.', 'search_account' => 'Konto suchen…', diff --git a/lang/el/analytics.php b/lang/el/analytics.php index af402c594..ac29d1f13 100644 --- a/lang/el/analytics.php +++ b/lang/el/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'Δεν υπάρχουν συνδεδεμένοι λογαριασμοί με στατιστικά.', 'no_accounts_match' => 'Δεν ταιριάζει κανένας λογαριασμός.', 'search_account' => 'Αναζήτηση λογαριασμού…', diff --git a/lang/en/analytics.php b/lang/en/analytics.php index 3212f6cac..a6081e793 100644 --- a/lang/en/analytics.php +++ b/lang/en/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'No connected accounts with analytics.', 'no_accounts_match' => 'No accounts match.', 'search_account' => 'Search account…', diff --git a/lang/es/analytics.php b/lang/es/analytics.php index 5b97ec60c..7378e6bb4 100644 --- a/lang/es/analytics.php +++ b/lang/es/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'No hay cuentas conectadas con analytics.', 'no_accounts_match' => 'Ninguna cuenta coincide.', 'search_account' => 'Buscar cuenta…', diff --git a/lang/fr/analytics.php b/lang/fr/analytics.php index 465dc8f56..3351aa79a 100644 --- a/lang/fr/analytics.php +++ b/lang/fr/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'Aucun compte connecté avec des statistiques.', 'no_accounts_match' => 'Aucun compte correspondant.', 'search_account' => 'Rechercher un compte…', diff --git a/lang/it/analytics.php b/lang/it/analytics.php index b2963aecc..73106386a 100644 --- a/lang/it/analytics.php +++ b/lang/it/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'Nessun account collegato con statistiche.', 'no_accounts_match' => 'Nessun account corrisponde.', 'search_account' => 'Cerca account…', diff --git a/lang/ja/analytics.php b/lang/ja/analytics.php index cbb1c8588..c846b6777 100644 --- a/lang/ja/analytics.php +++ b/lang/ja/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'アナリティクスを利用できる接続済みアカウントがありません。', 'no_accounts_match' => '一致するアカウントがありません。', 'search_account' => 'アカウントを検索…', diff --git a/lang/ko/analytics.php b/lang/ko/analytics.php index 2001a0388..313acc55f 100644 --- a/lang/ko/analytics.php +++ b/lang/ko/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => '분석을 사용할 수 있는 연결된 계정이 없습니다.', 'no_accounts_match' => '일치하는 계정이 없습니다.', 'search_account' => '계정 검색…', diff --git a/lang/nl/analytics.php b/lang/nl/analytics.php index 70e81f62a..1a84ebbab 100644 --- a/lang/nl/analytics.php +++ b/lang/nl/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'Geen gekoppelde accounts met statistieken.', 'no_accounts_match' => 'Geen accounts komen overeen.', 'search_account' => 'Account zoeken…', diff --git a/lang/pl/analytics.php b/lang/pl/analytics.php index eb22d018e..3f3fea4e7 100644 --- a/lang/pl/analytics.php +++ b/lang/pl/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'Brak połączonych kont z analityką.', 'no_accounts_match' => 'Brak pasujących kont.', 'search_account' => 'Szukaj konta…', diff --git a/lang/pt-BR/analytics.php b/lang/pt-BR/analytics.php index 74bd1348e..02eeb727e 100644 --- a/lang/pt-BR/analytics.php +++ b/lang/pt-BR/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Resumo', + 'posts' => 'Posts', + 'total_followers' => 'Total de seguidores', + 'reactions' => 'Reações', + 'comments' => 'Comentários', + 'engagement_rate' => 'Taxa de engajamento', + 'followers' => 'Seguidores', + 'performance' => 'Desempenho', + 'top_posts' => 'Top 5 posts', + 'channel' => 'Canal', + 'workspace_description' => 'Todas as redes do workspace, com cada conta identificada separadamente.', + 'latest_snapshot_hint' => 'As métricas dos posts usam a última medição salva para posts publicados neste período.', + 'followers_chart_mode' => 'Visualização dos seguidores', + 'posts_chart_mode' => 'Visualização dos posts', + 'followers_line_description' => 'Seguidores por conta ao longo do tempo', + 'no_follower_data' => 'Não há histórico de seguidores neste período.', + 'no_post_data' => 'Não há posts publicados neste período.', + 'no_ranked_posts' => 'Não há posts com reações ou comentários medidos neste período.', + 'no_performance' => 'Não há desempenho por canal neste período.', + 'no_excerpt' => 'Prévia de texto indisponível.', + 'published_via_trypost' => 'Publicado pelo TryPost', + 'published_on_network' => 'Publicado na rede social', + 'view_post' => 'Ver post', + 'top_posts_sort' => 'Ordenar posts por', + 'carried_forward' => 'Último valor', + 'carried_forward_hint' => 'A rede ficou indisponível; este é o último número conhecido de seguidores.', + 'import_in_progress' => 'Importando o histórico das contas. Posts antigos aparecerão conforme a coleta avançar.', + 'no_data_title' => 'Estamos preparando seu histórico de analytics', + 'no_data_body' => 'Conecte uma conta compatível ou aguarde a primeira coleta em segundo plano. Os posts históricos aparecerão durante a importação.', + ], 'no_accounts' => 'Nenhuma conta conectada com analytics.', 'no_accounts_match' => 'Nenhuma conta corresponde.', 'search_account' => 'Buscar conta…', diff --git a/lang/ru/analytics.php b/lang/ru/analytics.php index 0abb9aae6..e89cb62af 100644 --- a/lang/ru/analytics.php +++ b/lang/ru/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'Нет подключённых аккаунтов с аналитикой.', 'no_accounts_match' => 'Нет подходящих аккаунтов.', 'search_account' => 'Поиск аккаунта…', diff --git a/lang/tr/analytics.php b/lang/tr/analytics.php index 15ff9f09c..7325517f0 100644 --- a/lang/tr/analytics.php +++ b/lang/tr/analytics.php @@ -3,6 +3,37 @@ declare(strict_types=1); return [ + 'dashboard' => [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'Analitiği olan bağlı hesap yok.', 'no_accounts_match' => 'Eşleşen hesap yok.', 'search_account' => 'Hesap ara…', diff --git a/lang/uk/analytics.php b/lang/uk/analytics.php index 615409b4d..eee597df7 100644 --- a/lang/uk/analytics.php +++ b/lang/uk/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => 'Немає підключених акаунтів з аналітикою.', 'no_accounts_match' => 'Жоден акаунт не відповідає.', 'search_account' => 'Пошук акаунта…', diff --git a/lang/zh/analytics.php b/lang/zh/analytics.php index f8840f2d1..9cdf0dbc5 100644 --- a/lang/zh/analytics.php +++ b/lang/zh/analytics.php @@ -1,6 +1,37 @@ [ + 'summary' => 'Summary', + 'posts' => 'Posts', + 'total_followers' => 'Total Followers', + 'reactions' => 'Reactions', + 'comments' => 'Comments', + 'engagement_rate' => 'Eng. Rate', + 'followers' => 'Followers', + 'performance' => 'Performance', + 'top_posts' => 'Top 5 Posts', + 'channel' => 'Channel', + 'workspace_description' => 'Your connected channels together, with each account kept distinct.', + 'latest_snapshot_hint' => 'Post metrics use the latest saved observation for posts published in this range.', + 'followers_chart_mode' => 'Follower chart mode', + 'posts_chart_mode' => 'Post chart mode', + 'followers_line_description' => 'Followers by account over time', + 'no_follower_data' => 'Follower history is not available for this period.', + 'no_post_data' => 'No published posts in this period.', + 'no_ranked_posts' => 'No posts with measured reactions or comments in this period.', + 'no_performance' => 'No channel performance in this period.', + 'no_excerpt' => 'No text preview available.', + 'published_via_trypost' => 'Published via TryPost', + 'published_on_network' => 'Published on the social network', + 'view_post' => 'View post', + 'top_posts_sort' => 'Rank posts by', + 'carried_forward' => 'Last known', + 'carried_forward_hint' => 'The provider was unavailable; this is the last known follower count.', + 'import_in_progress' => 'Importing account history. Older posts may appear as collection continues.', + 'no_data_title' => 'Your analytics history is being prepared', + 'no_data_body' => 'Connect a supported account or wait for the first background collection. Historical posts appear as they are imported.', + ], 'no_accounts' => '没有可查看分析数据的已连接账号。', 'no_accounts_match' => '没有匹配的账号。', 'search_account' => '搜索账号…', diff --git a/resources/js/components/analytics/AnalyticsAccountSelector.vue b/resources/js/components/analytics/AnalyticsAccountSelector.vue deleted file mode 100644 index aa1125a95..000000000 --- a/resources/js/components/analytics/AnalyticsAccountSelector.vue +++ /dev/null @@ -1,125 +0,0 @@ - - - diff --git a/resources/js/components/analytics/FacebookAnalytics.vue b/resources/js/components/analytics/FacebookAnalytics.vue deleted file mode 100644 index 943e0480f..000000000 --- a/resources/js/components/analytics/FacebookAnalytics.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - diff --git a/resources/js/components/analytics/GoogleBusinessAnalytics.vue b/resources/js/components/analytics/GoogleBusinessAnalytics.vue deleted file mode 100644 index bdbe88692..000000000 --- a/resources/js/components/analytics/GoogleBusinessAnalytics.vue +++ /dev/null @@ -1,108 +0,0 @@ - - - diff --git a/resources/js/components/analytics/InstagramAnalytics.vue b/resources/js/components/analytics/InstagramAnalytics.vue deleted file mode 100644 index 943e0480f..000000000 --- a/resources/js/components/analytics/InstagramAnalytics.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - diff --git a/resources/js/components/analytics/LinkedInPageAnalytics.vue b/resources/js/components/analytics/LinkedInPageAnalytics.vue deleted file mode 100644 index 943e0480f..000000000 --- a/resources/js/components/analytics/LinkedInPageAnalytics.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - diff --git a/resources/js/components/analytics/MetricsGrid.vue b/resources/js/components/analytics/MetricsGrid.vue deleted file mode 100644 index 6b0323fbb..000000000 --- a/resources/js/components/analytics/MetricsGrid.vue +++ /dev/null @@ -1,138 +0,0 @@ - - - diff --git a/resources/js/components/analytics/PinterestAnalytics.vue b/resources/js/components/analytics/PinterestAnalytics.vue deleted file mode 100644 index 943e0480f..000000000 --- a/resources/js/components/analytics/PinterestAnalytics.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - diff --git a/resources/js/components/analytics/TelegramAnalytics.vue b/resources/js/components/analytics/TelegramAnalytics.vue deleted file mode 100644 index 44994bffb..000000000 --- a/resources/js/components/analytics/TelegramAnalytics.vue +++ /dev/null @@ -1,57 +0,0 @@ - - - diff --git a/resources/js/components/analytics/ThreadsAnalytics.vue b/resources/js/components/analytics/ThreadsAnalytics.vue deleted file mode 100644 index 943e0480f..000000000 --- a/resources/js/components/analytics/ThreadsAnalytics.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - diff --git a/resources/js/components/analytics/TikTokAnalytics.vue b/resources/js/components/analytics/TikTokAnalytics.vue deleted file mode 100644 index a8482991f..000000000 --- a/resources/js/components/analytics/TikTokAnalytics.vue +++ /dev/null @@ -1,50 +0,0 @@ - - - diff --git a/resources/js/components/analytics/XAnalytics.vue b/resources/js/components/analytics/XAnalytics.vue deleted file mode 100644 index 943e0480f..000000000 --- a/resources/js/components/analytics/XAnalytics.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - diff --git a/resources/js/components/analytics/YouTubeAnalytics.vue b/resources/js/components/analytics/YouTubeAnalytics.vue deleted file mode 100644 index 943e0480f..000000000 --- a/resources/js/components/analytics/YouTubeAnalytics.vue +++ /dev/null @@ -1,61 +0,0 @@ - - - diff --git a/resources/js/components/analytics/types.ts b/resources/js/components/analytics/types.ts deleted file mode 100644 index 0cf0bb822..000000000 --- a/resources/js/components/analytics/types.ts +++ /dev/null @@ -1,7 +0,0 @@ -export interface AnalyticsAccount { - id: string; - platform: string; - username: string | null; - display_label: string; - avatar_url: string | null; -} diff --git a/resources/js/components/analytics/workspace/AccountIdentity.vue b/resources/js/components/analytics/workspace/AccountIdentity.vue new file mode 100644 index 000000000..4443f7399 --- /dev/null +++ b/resources/js/components/analytics/workspace/AccountIdentity.vue @@ -0,0 +1,38 @@ + + + diff --git a/resources/js/components/analytics/workspace/AnalyticsSection.vue b/resources/js/components/analytics/workspace/AnalyticsSection.vue new file mode 100644 index 000000000..7e9df9b27 --- /dev/null +++ b/resources/js/components/analytics/workspace/AnalyticsSection.vue @@ -0,0 +1,22 @@ + + + diff --git a/resources/js/components/analytics/workspace/FollowersChart.vue b/resources/js/components/analytics/workspace/FollowersChart.vue new file mode 100644 index 000000000..2b229886b --- /dev/null +++ b/resources/js/components/analytics/workspace/FollowersChart.vue @@ -0,0 +1,113 @@ + + + diff --git a/resources/js/components/analytics/workspace/ImportCoverage.vue b/resources/js/components/analytics/workspace/ImportCoverage.vue new file mode 100644 index 000000000..719745b27 --- /dev/null +++ b/resources/js/components/analytics/workspace/ImportCoverage.vue @@ -0,0 +1,22 @@ + + + diff --git a/resources/js/components/analytics/workspace/PerformanceTable.vue b/resources/js/components/analytics/workspace/PerformanceTable.vue new file mode 100644 index 000000000..8cd9764c9 --- /dev/null +++ b/resources/js/components/analytics/workspace/PerformanceTable.vue @@ -0,0 +1,121 @@ + + + diff --git a/resources/js/components/analytics/workspace/PostsChart.vue b/resources/js/components/analytics/workspace/PostsChart.vue new file mode 100644 index 000000000..4023f8b6a --- /dev/null +++ b/resources/js/components/analytics/workspace/PostsChart.vue @@ -0,0 +1,84 @@ + + + diff --git a/resources/js/components/analytics/workspace/SummaryCards.vue b/resources/js/components/analytics/workspace/SummaryCards.vue new file mode 100644 index 000000000..628b4d8a5 --- /dev/null +++ b/resources/js/components/analytics/workspace/SummaryCards.vue @@ -0,0 +1,91 @@ + + + diff --git a/resources/js/components/analytics/workspace/TopPosts.vue b/resources/js/components/analytics/workspace/TopPosts.vue new file mode 100644 index 000000000..ebe4a52d2 --- /dev/null +++ b/resources/js/components/analytics/workspace/TopPosts.vue @@ -0,0 +1,119 @@ + + + diff --git a/resources/js/components/analytics/workspace/charts/HorizontalBarChart.vue b/resources/js/components/analytics/workspace/charts/HorizontalBarChart.vue new file mode 100644 index 000000000..6c45f7273 --- /dev/null +++ b/resources/js/components/analytics/workspace/charts/HorizontalBarChart.vue @@ -0,0 +1,67 @@ + + + diff --git a/resources/js/components/analytics/workspace/charts/LineChart.vue b/resources/js/components/analytics/workspace/charts/LineChart.vue new file mode 100644 index 000000000..26f3f7c3f --- /dev/null +++ b/resources/js/components/analytics/workspace/charts/LineChart.vue @@ -0,0 +1,148 @@ + + + diff --git a/resources/js/components/analytics/workspace/charts/StackedBarChart.vue b/resources/js/components/analytics/workspace/charts/StackedBarChart.vue new file mode 100644 index 000000000..20965c12e --- /dev/null +++ b/resources/js/components/analytics/workspace/charts/StackedBarChart.vue @@ -0,0 +1,43 @@ + + + diff --git a/resources/js/components/analytics/workspace/types.ts b/resources/js/components/analytics/workspace/types.ts new file mode 100644 index 000000000..62c3e7a8e --- /dev/null +++ b/resources/js/components/analytics/workspace/types.ts @@ -0,0 +1,106 @@ +export interface Comparison { + value: number | null; + previous: number | null; + change: number | null; +} + +export interface AccountIdentityData { + social_account_key: string; + platform: string; + name: string | null; + username: string | null; + avatar_url: string | null; +} + +export interface FollowerAccount extends AccountIdentityData { + social_account_id: string | null; + network: string; + value: number | null; + growth: number | null; + provenance: string | null; +} + +export interface PostAccount extends AccountIdentityData { + count: number; +} + +export interface PostBucket { + start: string; + end: string; + label: string; + accounts: Record; + total: number; +} + +export interface TopPost { + id: string; + post_platform_id: string | null; + social_account_key: string; + platform: string; + name: string | null; + username: string | null; + origin: string; + content_type: string; + published_at: string; + permalink: string | null; + excerpt: string | null; + preview_metadata: Record | null; + reactions: number | null; + comments: number | null; +} + +export interface PerformanceRow extends AccountIdentityData { + posts: Comparison; + reactions: Comparison; + comments: Comparison; + engagement_rate: Comparison; +} + +export interface CoverageRow { + social_account_id: string; + collector: string; + status: string; + target_since: string | null; + oldest_reached_at: string | null; + high_watermark_at: string | null; + last_success_at: string | null; + last_error_category: string | null; +} + +export interface WorkspaceAnalyticsReport { + bounds: { min: string | null; max: string | null }; + range: { start: string; end: string }; + previous_range: { start: string; end: string }; + summary: { + posts: Comparison; + followers: Comparison; + reactions: Comparison; + comments: Comparison; + engagement_rate: Comparison; + }; + followers: { + total: number | null; + accounts: FollowerAccount[]; + series: { date: string; accounts: Record }[]; + }; + posts: { + resolution: string; + accounts: PostAccount[]; + buckets: PostBucket[]; + }; + top_posts: { reactions: TopPost[]; comments: TopPost[] }; + performance: PerformanceRow[]; + coverage: CoverageRow[]; +} + +export const accountColors = [ + '#6D43CC', + '#159B92', + '#E9833D', + '#347AB7', + '#C64F7B', + '#74894B', +]; + +export const accountColor = (index: number): string => + accountColors[index % accountColors.length]; diff --git a/resources/js/components/ui/date-range-picker/DateRangePicker.vue b/resources/js/components/ui/date-range-picker/DateRangePicker.vue index e067cfe84..d38376672 100644 --- a/resources/js/components/ui/date-range-picker/DateRangePicker.vue +++ b/resources/js/components/ui/date-range-picker/DateRangePicker.vue @@ -23,6 +23,9 @@ import dayjs from "@/dayjs" const props = defineProps<{ modelValue: { start: Date, end: Date } triggerClass?: string + minDate?: Date + maxDate?: Date + disabled?: boolean }>() const emit = defineEmits<{ @@ -53,6 +56,8 @@ const isUpdating = ref(false) const isOpen = ref(false) const { width } = useWindowSize() const numberOfMonths = computed(() => width.value < 640 ? 1 : 2) +const minimum = computed(() => props.minDate ? toCalendarDate(props.minDate) : undefined) +const maximum = computed(() => props.maxDate ? toCalendarDate(props.maxDate) : undefined) const range = (start: dayjs.Dayjs, end: dayjs.Dayjs) => ({ start: toCalendarDate(start.toDate()), @@ -82,7 +87,15 @@ const presetGroups = computed(() => [ type Preset = { label: string, getValue: () => { start: any, end: any } } const applyPreset = (preset: Preset) => { - value.value = preset.getValue() + const selected = preset.getValue() + const lower = minimum.value?.toString() + const upper = maximum.value?.toString() + const clamp = (date: CalendarDate) => { + if (lower && date.toString() < lower) return minimum.value! + if (upper && date.toString() > upper) return maximum.value! + return date + } + value.value = { start: clamp(selected.start), end: clamp(selected.end) } isOpen.value = false } @@ -122,6 +135,7 @@ watch(