From 0e4dc69ec0d4dd45dd142f8e4ddaf60919d47e0f Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Fri, 10 Jul 2026 13:54:23 +0800 Subject: [PATCH] fix(analytics): normalize PostHog event semantics (#2041) --- .../docs/ai-context/metrics-ownership.md | 18 +- .../product-analytics-dashboard-setup.md | 26 +- .../product-analytics-instrumentation.md | 43 +-- .../posthog-forwarding-and-pageview.md | 2 +- .../verifications/product-analytics-smoke.md | 58 +++- .../services/domain/product-events.test.ts | 4 +- .../src/services/domain/product-events.ts | 8 +- .../src/modules/analytics.test.ts | 48 ++++ apps/ui-server-auth/src/modules/analytics.ts | 21 +- apps/ui-server-auth/src/pages/sign-in.vue | 6 +- docs/.vitepress/modules/posthog.ts | 2 +- .../runtime/chat-orchestrator-runtime.test.ts | 91 +++++- .../src/runtime/chat-orchestrator-runtime.ts | 112 ++++++-- .../stage-pages/src/pages/settings/flux.vue | 14 +- .../src/composables/use-analytics.test.ts | 259 +++++++++--------- .../stage-ui/src/composables/use-analytics.ts | 223 +++++++-------- .../stage-ui/src/stores/analytics/index.ts | 8 +- .../src/stores/analytics/posthog.test.ts | 35 +++ .../stage-ui/src/stores/analytics/posthog.ts | 4 +- .../stage-ui/src/stores/chat.contract.test.ts | 143 +++++++++- packages/stage-ui/src/stores/chat.ts | 89 +++--- packages/stage-ui/src/stores/providers.ts | 4 +- posthog.config.ts | 2 +- 23 files changed, 808 insertions(+), 412 deletions(-) create mode 100644 apps/ui-server-auth/src/modules/analytics.test.ts create mode 100644 packages/stage-ui/src/stores/analytics/posthog.test.ts diff --git a/apps/server/docs/ai-context/metrics-ownership.md b/apps/server/docs/ai-context/metrics-ownership.md index 18099d893..a35810136 100644 --- a/apps/server/docs/ai-context/metrics-ownership.md +++ b/apps/server/docs/ai-context/metrics-ownership.md @@ -112,17 +112,17 @@ ### PostHog(前端 / 外部数据源,产品侧) 已接入: -- 前端 `posthog-js` 通过 `packages/stage-ui/src/stores/analytics/posthog.ts` 初始化,三个 app(web / desktop / pocket)按 `isStageTamagotchi()` 等选 project key -- Server 不接入 `posthog-node`。后端产品事件写 `product_events`,Grafana 展示低基数聚合;需要 PostHog revenue/person analytics 时优先用 PostHog Stripe source connector 或离线导入,不允许在 API 请求路径同步发 PostHog。 +- 前端 `posthog-js` 通过 `packages/stage-ui/src/stores/analytics/posthog.ts` 初始化;web / desktop / pocket / auth / docs 共用一个 project key,以 `app_surface` 区分运行端。 +- Server 先把产品事实写入 `product_events`,再异步 best-effort 转发注册、支付、订阅等白名单业务事实到 PostHog;LLM / TTS per-request 事件不转发,PostHog 失败也不能影响请求主链路。 - 前端 identity:`useSharedAnalyticsStore.initialize()` watch `authStore.isAuthenticated` 自动调 `posthog.identify(user.id)` / `reset()` -- Conversation controls 事件的 `surface` 由 `packages/stage-ui/src/composables/use-analytics.ts` 统一按 runtime 推断,UI 调用点只传业务字段。 +- 平台统一写入 `app_surface`;`entry_surface` 只表示 `settings_flux` 这类业务入口,避免同名字段混用或覆盖 PostHog super property。 已埋点: | 域 | 事件 | 来源 | 落点 | Truth | |---|---|---|---|---| | 付费漏斗 | `pricing_page_viewed` / `plan_selected` / `checkout_started` | 前端 | `packages/stage-pages/src/pages/settings/flux.vue` | PostHog | -| 付费漏斗终点 | `payment_completed` | 后端 webhook | `product_events` + Grafana;PostHog 走 Stripe source connector/离线导入 | Postgres | +| 付费漏斗终点 | `payment_completed` | 后端 webhook | `product_events` + Grafana,并 best-effort 转发 PostHog | Postgres | | Activation / Retention | `first_model_selected` / `model_switched` | 前端(consciousness store watcher) | `packages/stage-ui/src/stores/analytics/index.ts` | PostHog | | Retention | `character_created` | 前端 | `apps/stage-web/src/pages/settings/characters/components/CharacterDialog.vue` | PostHog | | Retention | `chat_session_started` | 前端 | `packages/stage-ui/src/components/scenarios/chat/components/sessions-drawer.vue` | PostHog | @@ -131,13 +131,15 @@ | Conversation controls | `tts_stop_clicked` | 前端 | `packages/stage-layouts/src/composables/useStopSpeakingButton.ts` | PostHog | | Churn | `subscription_cancelled`(带 cancellation_reason) | 外部 Stripe 数据源 | PostHog Stripe source connector | Stripe/Postgres | | 老事件 | `provider_card_clicked` | 前端 | `packages/stage-ui/src/composables/use-analytics.ts` | PostHog | -| 已退役 | `first_message_sent` / `user_signed_up`(前端版) | 无生产者 | wrapper 已删除;激活口径用 `chat_activation_succeeded`,注册事实用服务端转发的 `signup_completed` | PostHog 历史数据仍在 | +| 兼容指标 | `first_message_sent` | 前端 | 仍有生产者,仅供历史 dashboard;新激活口径使用 `chat_activation_succeeded` | PostHog | +| 聊天轮次 | `message_send_started` / `message_sent` / `llm_*` / `message_round` / `message_round_failed` | 前端 core runtime | 以 `conversation_id` / `round_id` / `turn_index` 关联;成功和失败各有唯一终点事件 | PostHog | +| 注册 UI | `signup_form_completed` | auth SPA | 匿名表单完成信号,不计作注册事实 | PostHog | +| 注册事实 | `signup_completed` | Better Auth user create hook | 仅服务端生产,以 Better Auth user id 识别 | Postgres + PostHog | 待埋点(API 已在 `use-analytics.ts` 暴露但调用点未接入): | 域 | 事件 | 状态 | |---|---|---| -| Activation | `user_signed_up` | 等接到 auth callback 完成事件(Better Auth 的 signUp 成功 hook) | | Retention | `voice_mode_activated` | 需要先在 hearing store 加显式 `enableVoiceMode` action — 当前 hearing 没有单一"用户主动启用"那一刻的 trigger,被动监听 + 录音 action 不构成 user intent 信号 | | Feature adoption | `flux_image_generated` | 等图片生成 feature 上线 | @@ -155,7 +157,7 @@ ### 阶段 1(P0 — 付费漏斗 + activation) -所有 surface 共用根目录 `posthog.config.ts`(单一 project key,`surface` super property 区分端)。初始化实况: +所有运行端共用根目录 `posthog.config.ts`(单一 project key,`app_surface` super property 区分端)。初始化实况: ```ts import { DEFAULT_POSTHOG_CONFIG, POSTHOG_PROJECT_KEY } from '../posthog.config' @@ -184,7 +186,7 @@ posthog.init(import.meta.env.VITE_POSTHOG_KEY, { 埋点事件清单(P0): -- 前端:`pricing_page_viewed`、`plan_selected`、`checkout_started`、`signup_completed`(ui-server-auth 邮箱注册路径)、`first_model_selected` +- 前端:`pricing_page_viewed`、`plan_selected`、`checkout_started`、`signup_form_completed`(ui-server-auth 匿名邮箱表单里程碑)、`first_model_selected` - 后端:`product_events` 是事实账本;其中业务事实白名单(`signup_completed`、`payment_completed`、`subscription_started/renewed/cancelled`)由 product-events 服务经 posthog-node 转发一份到 PostHog(`apps/server/src/services/domain/product-events.ts`,distinctId = Better Auth user id)。LLM / TTS 等 per-request 事件不转发 PostHog UI 配两个 funnel: diff --git a/apps/server/docs/ai-context/product-analytics-dashboard-setup.md b/apps/server/docs/ai-context/product-analytics-dashboard-setup.md index dadc86833..530dae32e 100644 --- a/apps/server/docs/ai-context/product-analytics-dashboard-setup.md +++ b/apps/server/docs/ai-context/product-analytics-dashboard-setup.md @@ -60,7 +60,7 @@ Steps: Breakdowns: - `provider_mode` -- `surface` +- `app_surface` Filters: @@ -116,6 +116,26 @@ Watch for: - `failure_stage = model_list` - `failure_stage = llm_response` +### Insight 2a: All Message Round Failures + +Type: Trends + +Event: + +- `message_round_failed` + +Breakdowns: + +- `failure_stage` +- `error_code` +- `provider_id` +- `app_surface` + +Watch for: + +- Failures where `turn_index > 1`, which are intentionally outside the activation-failure series. +- Repeated failures for the same `conversation_id` with different `round_id` values. + ### Insight 3: Provider Configuration Health Type: Funnel @@ -213,7 +233,7 @@ Breakdowns: - `stt_provider_id` - `error_code` -- `surface` +- `app_surface` ### Insight 7: Feedback And Bug Reports @@ -229,7 +249,7 @@ Breakdowns: - `category` - `severity` - `entrypoint` -- `surface` +- `app_surface` Watch for: diff --git a/apps/server/docs/ai-context/product-analytics-instrumentation.md b/apps/server/docs/ai-context/product-analytics-instrumentation.md index 53555391f..74d095151 100644 --- a/apps/server/docs/ai-context/product-analytics-instrumentation.md +++ b/apps/server/docs/ai-context/product-analytics-instrumentation.md @@ -127,7 +127,8 @@ Grafana 当前不能回答: | Field | Values | Notes | |---|---|---| -| `surface` | `web` / `desktop` / `mobile` | 所有关键前端事件必带 | +| `app_surface` | `web` / `electron` / `mobile` / `auth` / `docs` / `server` | 所有关键事件的平台 / 运行端 | +| `entry_surface` | `settings_flux` / `onboarding` / `chat_toolbar` 等受控枚举 | 业务入口;不得用于表示运行端 | | `provider_mode` | `official` / `custom` / `unknown` | 官方开箱即用 vs 用户自配置 | | `provider_id` | 白名单 ID | 不传 raw URL / raw key / 用户输入 | | `model_id` | 白名单或归一化后的 ID | 自定义模型用 `is_custom_model = true` | @@ -147,6 +148,7 @@ Grafana 当前不能回答: | `chat_activation_started` | frontend | PostHog | 用户进入首次聊天路径或点击发送第一条消息前 | | `chat_activation_succeeded` | frontend | PostHog | 首次消息完成并看到 assistant response | | `chat_activation_failed` | frontend | PostHog | 首次消息未完成,包含配置、网络、鉴权、余额、模型等失败 | +| `message_round_failed` | frontend | PostHog | 任意用户轮次在 assistant response 完成前失败;成功轮次的唯一终点仍为 `message_round` | | `official_provider_selected` | frontend | PostHog | 官方 Provider 被默认落地或在设置页被手动选择,记录 provider id 与是否自动选择 | | `second_turn_started` | frontend | PostHog | 同一会话开始第二轮对话 | @@ -157,11 +159,13 @@ Grafana 当前不能回答: | `provider_mode` | yes | `official` / `custom` | | `provider_id` | yes | 归一化 ID | | `model_id` | yes | 归一化 ID | -| `surface` | yes | web / desktop / mobile | +| `app_surface` | yes | web / electron / mobile | +| `conversation_id` | yes | 应用会话 ID;同一 conversation 的聊天事件保持一致 | +| `round_id` | yes | 单轮关联 ID,复用该轮 user message id;同一轮所有聊天主链路事件保持一致 | +| `turn_index` | yes | conversation 内从 `1` 开始的用户轮次;`second_turn_started` 固定为 `2` | | `time_to_first_message_ms` | success only | 从 app start 或 onboarding complete 到首次成功 | | `error_code` | failed only | 稳定错误码 | | `failure_stage` | failed only | `provider_config` / `model_list` / `message_send` / `llm_response` / `tts` | -| `turn_index` | second turn only | 固定为 `2`,用于首轮成功后的二轮启动 | | `auto_selected` | official provider only | 官方默认 Provider 自动落地时为 `true` | 推荐看板: @@ -169,6 +173,7 @@ Grafana 当前不能回答: - 新用户 `chat_activation_started -> chat_activation_succeeded` 漏斗。 - 按 `provider_mode` 拆分 activation conversion。 - `chat_activation_failed` 按 `failure_stage` / `provider_id` 排名。 +- `message_round_failed` 按 `turn_index` / `failure_stage` / `provider_id` 排名,用于分析激活后的聊天失败。 异常提醒: @@ -282,14 +287,14 @@ Grafana 当前不能回答: | Field | Required | Notes | |---|---|---| | `stt_provider_id` | yes | 归一化 ID | -| `surface` | yes | web / desktop / mobile | +| `app_surface` | yes | web / electron / mobile | | `duration_ms` | no | 用户按住或录音时长 | | `error_code` | failed only | `permission_denied` / `device_unavailable` / `provider_error` / `timeout` | 推荐看板: - Voice input start -> STT success funnel。 -- Permission denied rate by browser / surface。 +- Permission denied rate by browser / app surface。 - STT failure rate by Provider。 ### Feedback And Bug Reports @@ -312,7 +317,7 @@ Grafana 当前不能回答: | `severity` | yes | `blocker` / `major` / `minor` / `suggestion` | | `user_type` | yes | `new_user` / `paid_user` / `overseas_user` / `developer_user` / `role_chat_user` / `unknown` | | `entrypoint` | yes | `about_update_error` / `community_manual_tag` 等低基数入口 | -| `surface` | in-app | web / desktop / mobile | +| `app_surface` | in-app | web / electron / mobile | | `provider_mode` | no | 可从最近一次配置状态补齐 | | `description_length_bucket` | bug report | `empty` / `short` / `medium` / `long`,不要上传正文 | | `include_triage_context` | bug report | 是否附带页面上下文 | @@ -395,7 +400,7 @@ Grafana 当前不能回答: - `has_voice` - `latency_ms` - `error_code` -- `surface` +- `app_surface` 看板: @@ -420,7 +425,7 @@ Grafana 当前不能回答: - `character_type`: `built_in` / `imported` / `custom` - `has_voice` - `voice_type` -- `surface` +- `app_surface` 看板: @@ -438,7 +443,7 @@ Grafana 当前不能回答: 字段: -- `surface` +- `entry_surface`(付费入口,例如 `settings_flux`;运行端使用 `app_surface`) - `balance_state` - `plan_id` - `currency` @@ -457,6 +462,8 @@ Grafana 当前不能回答: |---|---|---| | `first_message_sent` | 保留历史指标 | 继续用于老 dashboard;新激活口径用 `chat_activation_succeeded` | | `chat_activation_started` / `chat_activation_succeeded` / `chat_activation_failed` | 新核心 activation 口径 | 用来回答“用户能不能正常开始聊天” | +| `message_send_started` / `message_sent` / `llm_*` / `message_round` / `message_round_failed` | 聊天主链路 | 共享 `conversation_id` / `round_id` / `turn_index`;`message_round` 和 `message_round_failed` 分别是单轮成功 / 失败的唯一终点;不再重复发送 `chat_started`、`assistant_response_completed`、`chat_failed` 或 chat 的通用 `feature_used` 别名 | +| `signup_form_completed` / `signup_completed` | UI 里程碑 / 注册事实 | 前者可匿名;后者只由服务端按 Better Auth user id 发送,禁止复用同名客户端事件 | | `provider_card_clicked` | 保留入口点击 | 不等于配置成功;成功 / 失败看 `provider_config_succeeded` / `provider_config_failed` | | `first_model_selected` / `model_switched` | 保留模型选择行为 | 配置链路和模型列表健康看 `model_list_loaded` / `model_list_failed` | | `stt_started` / `stt_succeeded` / `stt_failed` | 保留 STT Provider 结果 | 权限和设备问题用新增 `microphone_*` / `audio_device_unavailable` 拆开 | @@ -495,7 +502,8 @@ PostHog 线上已经能看到 `model` / `model_id` 存在自由文本风险。 | `voice_pack_id` | medium | 只进 PostHog / Postgres,不进 Prometheus label | | `error_code` | low | enum | | `source` | low | enum | -| `surface` | low | enum | +| `app_surface` | low | runtime enum | +| `entry_surface` | low | 低基数业务入口 enum;不得复用为 runtime | Prometheus label 不放 `user_id`、`session_id`、`voice_pack_id`、自定义模型名、自定义音色名。 @@ -673,7 +681,7 @@ Prometheus label 不放 `user_id`、`session_id`、`voice_pack_id`、自定义 - Top failing providers: - Top failing error codes: -- Rage-click pages / surfaces: +- Rage-click pages / app surfaces: - Discord / QQ feedback categories: - performance: - config: @@ -709,7 +717,7 @@ Prometheus label 不放 `user_id`、`session_id`、`voice_pack_id`、自定义 给负责上手体验的人看: - Funnel:`app_loaded -> chat_activation_started -> provider_config_succeeded -> model_list_loaded -> chat_activation_succeeded` -- Breakdown:`provider_mode`、`surface`、`region` +- Breakdown:`provider_mode`、`app_surface`、`region` - Table:Top `provider_config_failed` by `provider_id` / `error_code` - Timeseries:`time_to_first_message_ms` p50 / p95 @@ -729,7 +737,7 @@ Prometheus label 不放 `user_id`、`session_id`、`voice_pack_id`、自定义 - Grafana:5xx、LLM latency、provider failure、TTS blocked - PostHog:rage-click trend、failed frontend events -- Table:Top error_code by surface / provider +- Table:Top error_code by app surface / provider - Community tags:Discord / QQ 反馈分类趋势 ### 分析方法 @@ -763,7 +771,7 @@ Prometheus label 不放 `user_id`、`session_id`、`voice_pack_id`、自定义 - `chat_activation_failed` by `failure_stage` - `provider_config_failed` by `error_code` - `model_list_failed` by `provider_id` -- `$rageclick` by page / surface +- `$rageclick` by page / `app_surface` - Discord / QQ `category = bug` 的高频词 日报只报异常;周报把异常和社区反馈合并成“优先修复建议”。 @@ -863,11 +871,14 @@ Prometheus label 不放 `user_id`、`session_id`、`voice_pack_id`、自定义 10. 建异常检查:activation、provider config、model list、STT、TTS blocked、bug report。 11. 建周报模板:自动填指标,社区负责人补充 Discord / QQ 反馈解释和下周建议。 -### 当前接入状态(2026-06-30) +### 当前接入状态(2026-07-10) 已接入代码: -- Chat activation:`chat_activation_started`、`chat_activation_succeeded`、`chat_activation_failed`、`second_turn_started`;官方 Provider 选择事件为 `official_provider_selected`,实际聊天使用口径看 activation events 的 `provider_mode = official`。 +- Chat activation:`chat_activation_started`、`chat_activation_succeeded`、`chat_activation_failed` 只覆盖每个 conversation 首次 assistant response 之前的尝试;后续轮次继续发 message / latency events,第二轮单独发 `second_turn_started`。 +- Chat correlation:每次发送以 user message id 作为 `round_id`;activation、message、LLM latency、render、`message_round` 和 `message_round_failed` 事件共享 `conversation_id`、`round_id`、`turn_index`。 +- Identity:匿名 auth SPA 发 `signup_form_completed`;只有服务端 Better Auth user create hook 发 identified `signup_completed`。平台统一使用 `app_surface`,业务入口统一使用 `entry_surface`。 +- Chat event reuse:主链路使用 `message_send_started`、`message_sent`、`llm_*`、`message_round`、`message_round_failed`;每轮成功 / 失败各自只有一个终点事件,不再发送 `chat_started`、`assistant_response_completed`、`chat_failed` 和 chat 的通用 `feature_used` 别名。 - Model list:`model_list_loaded`、`model_list_failed`。 - Provider config:`provider_config_started`、`provider_config_succeeded`、`provider_config_failed`。 - TTS voice:`tts_provider_selected`、`voice_selected`、`voice_preview_played`、`voice_pack_bound`、`official_tts_exposed`、`official_tts_preview_started`、`official_tts_preview_succeeded`、`official_tts_auto_enabled`。 diff --git a/apps/server/docs/ai-context/verifications/posthog-forwarding-and-pageview.md b/apps/server/docs/ai-context/verifications/posthog-forwarding-and-pageview.md index 40987a8fa..e1687bca3 100644 --- a/apps/server/docs/ai-context/verifications/posthog-forwarding-and-pageview.md +++ b/apps/server/docs/ai-context/verifications/posthog-forwarding-and-pageview.md @@ -16,7 +16,7 @@ Environment: commit 689f02ac4 + 本次工作区改动;posthog-node 5.39.4;po |---|---|---| | 转发白名单与映射 | `pnpm exec vitest run src/services/domain/product-events.test.ts`(apps/server) | 6 passed:`payment_completed` 原名转发、`user_signed_up→signup_completed` 映射、per-request 动作不转发、sink 抛错时 DB 行仍落库且 track 不抛 | | posthog-node 真实传输 | `node posthog-smoke.mjs`(captureImmediate → us.i.posthog.com,生产 project key,事件名 `server_forwarding_smoke_test`) | `captureImmediate resolved in 1380ms` + `shutdown clean` | -| SPA 路由 pageview | `VITE_ENABLE_POSTHOG=true pnpm -F @proj-airi/stage-web dev` + agent-browser 两次 `history.pushState` | 两条 `$pageview`,`$pathname` 分别为 `/settings/flux`、`/settings/airi-card`,`navigation_type: pushState`,携带 `$prev_pageview_duration` 与 `surface: web` super property;批量 POST `us.i.posthog.com/e/` 返回 200 | +| SPA 路由 pageview | `VITE_ENABLE_POSTHOG=true pnpm -F @proj-airi/stage-web dev` + agent-browser 两次 `history.pushState` | 两条 `$pageview`,`$pathname` 分别为 `/settings/flux`、`/settings/airi-card`,`navigation_type: pushState`,携带 `$prev_pageview_duration` 与 `app_surface: web` super property;批量 POST `us.i.posthog.com/e/` 返回 200 | | 服务端 typecheck / lint | `pnpm -F @proj-airi/server typecheck`、eslint 改动文件 | 均通过 | ## 注意事项 diff --git a/apps/server/docs/ai-context/verifications/product-analytics-smoke.md b/apps/server/docs/ai-context/verifications/product-analytics-smoke.md index 35948a7b3..5db13df56 100644 --- a/apps/server/docs/ai-context/verifications/product-analytics-smoke.md +++ b/apps/server/docs/ai-context/verifications/product-analytics-smoke.md @@ -2,7 +2,7 @@ Status: **code-level instrumentation verified; live PostHog dashboard updated; Grafana dashboard updated; alert setup pending** Owner: Community / Product Analytics -Last updated: 2026-07-01 +Last updated: 2026-07-10 Related: - [`product-analytics-instrumentation.md`](../product-analytics-instrumentation.md) - [`product-analytics-dashboard-setup.md`](../product-analytics-dashboard-setup.md) @@ -20,6 +20,7 @@ Related: |---|---| | Frontend analytics API | `packages/stage-ui/src/composables/use-analytics.test.ts` 覆盖 activation、model list、provider config、voice selection、voice input、feedback event API | | Chat activation hooks | `packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts` 覆盖 activation started / succeeded / failed hook | +| Chat round failures | core runtime 与 stage contract tests 覆盖激活前和激活后的 `message_round_failed`,并验证 `conversation_id` / `round_id` / `turn_index` | | Voice input failures | `packages/stage-ui/src/composables/audio/audio-device.test.ts` 与 `packages/stage-ui/src/stores/modules/hearing.analytics.test.ts` 覆盖 permission / device / cancel / STT failed | | Server TTS metadata | `apps/server/src/routes/openai/v1/route.test.ts` 与 `apps/server/src/routes/audio-speech-ws/route.test.ts` 覆盖 REST / WS TTS `voice_id`、`voice_type`、`voice_pack_id` metadata | | Grafana product row | `apps/server/otel/grafana/dashboards/build.test.ts` 覆盖 Product Analytics panels、layout references、PromQL 不包含 high-cardinality voice / user fields | @@ -51,8 +52,10 @@ Required properties: provider_mode = official provider_id = model_id = -surface = web | mobile | electron -turn_index = 2 +app_surface = web | mobile | electron +conversation_id = +round_id = +turn_index = 1 for the first round; 2 for second_turn_started and the second round ``` Fail if: @@ -60,9 +63,52 @@ Fail if: - `chat_activation_started` appears but `chat_activation_succeeded` never appears for a successful chat. - The second message is sent but `second_turn_started` does not appear. - `provider_mode` is missing or always `unknown`. -- `surface` is missing. +- `app_surface` is missing. +- Any chat-chain event is missing `conversation_id`, `round_id`, or `turn_index`. +- Events from one round disagree on `round_id`, or two different rounds reuse the same `round_id`. -### 1b. PostHog: official provider selection +### 1a. PostHog: message round failure + +Action: + +1. Complete a successful first chat round. +2. Force the second round to fail before the assistant response completes. + +Expected PostHog events: + +```text +message_round_failed +``` + +Required properties: + +```text +conversation_id = +round_id = +turn_index = 2 +provider_id = +model_id = +failure_stage = llm_response +error_code = llm_response_failed +app_surface = web | mobile | electron +``` + +Fail if: + +- The failed second round has no `message_round_failed` event. +- The failed round emits `message_round`, `chat_failed`, or `assistant_response_completed` as an alias. +- A new `chat_activation_failed` appears after the conversation already completed its first assistant response. +- Correlation keys disagree with the failed round's preceding message / LLM events. + +### 1b. Signup identity ownership + +1. Complete an email signup in the auth SPA. +2. Confirm the auth SPA emits `signup_form_completed` with `app_surface = auth`. +3. Confirm the Better Auth user-create hook emits exactly one `signup_completed` with `app_surface = server` and the Better Auth user id as `distinctId`. + +Fail if the auth SPA emits `signup_completed`, or if the server event lands on a different PostHog person from later identified onboarding events. + +### 1c. PostHog: official provider selection Action: @@ -219,7 +265,7 @@ paywall_seen Required properties: ```text -surface = settings_flux +entry_surface = settings_flux reason = manual_topup flux_balance_bucket = zero | 1_100 | 101_1000 | 1001_10000 | 10000_plus | unknown ``` diff --git a/apps/server/src/services/domain/product-events.test.ts b/apps/server/src/services/domain/product-events.test.ts index 05526bdbf..c5109ca81 100644 --- a/apps/server/src/services/domain/product-events.test.ts +++ b/apps/server/src/services/domain/product-events.test.ts @@ -162,7 +162,7 @@ describe('productEventService', () => { distinctId: 'user-1', event: 'payment_completed', properties: { - surface: 'server', + app_surface: 'server', feature: 'billing', status: 'succeeded', source: 'stripe.webhook', @@ -174,7 +174,7 @@ describe('productEventService', () => { distinctId: 'user-2', event: 'signup_completed', properties: { - surface: 'server', + app_surface: 'server', feature: 'auth', status: 'succeeded', }, diff --git a/apps/server/src/services/domain/product-events.ts b/apps/server/src/services/domain/product-events.ts index 036238eb0..888acfee8 100644 --- a/apps/server/src/services/domain/product-events.ts +++ b/apps/server/src/services/domain/product-events.ts @@ -82,9 +82,9 @@ export interface ProductEventAggregateRow { * a funnel are forwarded — per-request LLM/TTS volume stays in Postgres and * Grafana where it belongs (see `docs/ai-context/metrics-ownership.md`). * - * `user_signed_up` maps to `signup_completed` because that is the canonical - * activation-funnel step name the browser surfaces emit; the server copy - * covers OAuth signups the auth UI cannot classify client-side. + * `user_signed_up` maps to `signup_completed` because the identified server + * hook is the canonical registration fact for every signup method. Anonymous + * auth UI progress uses `signup_form_completed` and never reuses this name. */ const POSTHOG_FORWARDED_ACTIONS: Partial> = { user_signed_up: 'signup_completed', @@ -175,7 +175,7 @@ export function createProductEventService(db: Database, metrics?: ProductMetrics distinctId: input.userId, event: forwardedEvent, properties: { - surface: 'server', + app_surface: 'server', feature: input.feature, status: input.status, ...(input.source && { source: input.source }), diff --git a/apps/ui-server-auth/src/modules/analytics.test.ts b/apps/ui-server-auth/src/modules/analytics.test.ts new file mode 100644 index 000000000..064e91966 --- /dev/null +++ b/apps/ui-server-auth/src/modules/analytics.test.ts @@ -0,0 +1,48 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { initAuthAnalytics, trackSignupFormCompleted } from './analytics' + +const posthogMocks = vi.hoisted(() => ({ + capture: vi.fn(), + init: vi.fn(), + register: vi.fn(), +})) + +vi.mock('posthog-js', () => ({ + default: posthogMocks, +})) + +vi.mock('../../../../posthog.config', () => ({ + DEFAULT_POSTHOG_CONFIG: {}, + POSTHOG_ENABLED: true, + POSTHOG_PROJECT_KEY: 'test-project-key', +})) + +describe('auth product analytics', () => { + beforeEach(() => { + posthogMocks.capture.mockClear() + posthogMocks.init.mockClear() + posthogMocks.register.mockClear() + }) + + // ROOT CAUSE: + // + // The auth SPA emitted `signup_completed` before it knew the Better Auth + // user id, while the server emitted the same canonical event with that id. + // PostHog therefore counted one email signup as two unrelated persons. + // + // The anonymous UI milestone must use its own name. The identified server + // event remains the only canonical `signup_completed` business fact. + it('keeps anonymous signup UI completion separate from the canonical server signup fact', () => { + expect(initAuthAnalytics()).toBe(true) + expect(posthogMocks.register).toHaveBeenCalledWith({ app_surface: 'auth' }) + + trackSignupFormCompleted({ source: 'email', requires_verification: true }) + + expect(posthogMocks.capture).toHaveBeenCalledWith( + 'signup_form_completed', + { source: 'email', requires_verification: true }, + undefined, + ) + }) +}) diff --git a/apps/ui-server-auth/src/modules/analytics.ts b/apps/ui-server-auth/src/modules/analytics.ts index 6d3d61807..ab2e9c3d5 100644 --- a/apps/ui-server-auth/src/modules/analytics.ts +++ b/apps/ui-server-auth/src/modules/analytics.ts @@ -1,12 +1,11 @@ /** * PostHog product analytics for the auth-only SPA (`apps/ui-server-auth`). * - * This surface is the top of the activation funnel: sign-up, sign-in, email - * verification, password recovery. Events captured here are the funnel - * entry that the in-app surfaces (`signup_completed → onboarding_started → - * first message`) join against, keyed by the Better Auth user id passed to - * {@link identifyAuthUser} — the same id the server uses as `distinctId` - * for its own events, so the person profiles merge. + * This surface captures anonymous auth-UI milestones such as form completion, + * sign-in attempts, email verification, and password recovery. Canonical + * registration facts come from the server as identified `signup_completed` + * events; the auth SPA intentionally uses `signup_form_completed` so an event + * emitted before {@link identifyAuthUser} cannot double-count a new user. * * Unlike the stage apps there is no in-app analytics consent toggle here * (the user isn't signed in yet, so there's no settings store to read). @@ -43,9 +42,9 @@ export function initAuthAnalytics(): boolean { return true posthog.init(POSTHOG_PROJECT_KEY, { ...DEFAULT_POSTHOG_CONFIG }) - // Same single-project setup as the stage apps: the `surface` super + // Same single-project setup as the stage apps: the `app_surface` super // property is how auth traffic is told apart in shared dashboards. - posthog.register({ surface: 'auth' }) + posthog.register({ app_surface: 'auth' }) initialized = true return true } @@ -81,9 +80,9 @@ function capture(event: string, properties: Record, options?: C ) } -/** Activation funnel step 1 — the account now exists (email flow). */ -export function trackSignupCompleted(properties: { source: AuthMethod, requires_verification: boolean }): void { - capture('signup_completed', properties, { beforeNavigation: !properties.requires_verification }) +/** Anonymous email-signup UI milestone; the server owns the registration fact. */ +export function trackSignupFormCompleted(properties: { source: AuthMethod, requires_verification: boolean }): void { + capture('signup_form_completed', properties, { beforeNavigation: !properties.requires_verification }) } /** diff --git a/apps/ui-server-auth/src/pages/sign-in.vue b/apps/ui-server-auth/src/pages/sign-in.vue index 4c79c9d8d..6e18dd1f6 100644 --- a/apps/ui-server-auth/src/pages/sign-in.vue +++ b/apps/ui-server-auth/src/pages/sign-in.vue @@ -12,7 +12,7 @@ import { trackLoginFailed, trackLoginStarted, trackLoginSucceeded, - trackSignupCompleted, + trackSignupFormCompleted, } from '../modules/analytics' import { buildCurrentOriginAuthUiUrl } from '../modules/auth-ui-base' import { @@ -247,7 +247,7 @@ async function handleEmailSignUp(event: Event) { }) if (result.requiresVerification) { - trackSignupCompleted({ source: 'email', requires_verification: true }) + trackSignupFormCompleted({ source: 'email', requires_verification: true }) await router.push({ path: '/verify-email', query: { @@ -260,7 +260,7 @@ async function handleEmailSignUp(event: Event) { // Verification disabled at server config: session is live, fall through // to the OIDC continuation just like sign-in. - trackSignupCompleted({ source: 'email', requires_verification: false }) + trackSignupFormCompleted({ source: 'email', requires_verification: false }) window.location.href = effectiveCallbackURL.value } catch (error) { diff --git a/docs/.vitepress/modules/posthog.ts b/docs/.vitepress/modules/posthog.ts index a77d72bd6..7f06faf38 100644 --- a/docs/.vitepress/modules/posthog.ts +++ b/docs/.vitepress/modules/posthog.ts @@ -8,5 +8,5 @@ if (!import.meta.env.DEV) { }) // Tag docs-site traffic so it can be told apart from the app surfaces // inside the shared project. - posthog.register({ surface: 'docs' }) + posthog.register({ app_surface: 'docs' }) } diff --git a/packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts b/packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts index 738013a10..20b880e6c 100644 --- a/packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts +++ b/packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts @@ -43,6 +43,7 @@ function createHarness() { llmFirstToken: [] as unknown[], assistantResponseRendered: [] as unknown[], messageRound: [] as unknown[], + messageRoundFailed: [] as unknown[], } const stream = vi.fn(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options?: { onStreamEvent?: (event: StreamEvent) => Promise | void @@ -100,6 +101,7 @@ function createHarness() { onLlmFirstToken: event => telemetry.llmFirstToken.push(event), onAssistantResponseRendered: event => telemetry.assistantResponseRendered.push(event), onMessageRound: event => telemetry.messageRound.push(event), + onMessageRoundFailed: event => telemetry.messageRoundFailed.push(event), }) return { @@ -308,42 +310,86 @@ describe('createChatOrchestratorRuntime', () => { }) expect(harness.telemetry.messageSendStarted).toEqual([{ + conversationId: 'session-1', + roundId: 'user-id', source: 'voice', model: 'gpt-test', + turnIndex: 1, }]) expect(harness.telemetry.llmRequestStarted).toEqual([{ + conversationId: 'session-1', + roundId: 'user-id', model: 'gpt-test', provider: 'mock-provider', hasVoice: true, + turnIndex: 1, }]) expect(harness.telemetry.llmFirstToken).toEqual([{ + conversationId: 'session-1', + roundId: 'user-id', model: 'gpt-test', ttfbMs: 100, + turnIndex: 1, }]) expect(harness.telemetry.assistantResponseRendered).toEqual([{ + conversationId: 'session-1', + roundId: 'user-id', model: 'gpt-test', latencyMs: 250, + turnIndex: 1, }]) expect(harness.telemetry.messageRound).toEqual([{ + conversationId: 'session-1', + roundId: 'user-id', durationMs: 360, hasVoice: true, model: 'gpt-test', + turnIndex: 1, }]) expect(harness.telemetry.chatActivationStarted).toEqual([{ + conversationId: 'session-1', model: 'gpt-test', provider: 'mock-provider', - sessionId: 'session-1', + roundId: 'user-id', source: 'voice', + turnIndex: 1, }]) expect(harness.telemetry.chatActivationSucceeded).toEqual([{ + conversationId: 'session-1', durationMs: 360, model: 'gpt-test', provider: 'mock-provider', + roundId: 'user-id', source: 'voice', + turnIndex: 1, }]) expect(harness.telemetry.chatActivationFailed).toEqual([]) }) + // ROOT CAUSE: + // + // Activation callbacks were emitted for every chat round, so production + // `chat_activation_*` volume tracked message traffic instead of the first + // successful assistant response in a conversation. + it('emits activation milestones only until the conversation gets its first assistant response', async () => { + const harness = createHarness() + + await harness.runtime.ingest('first turn', { + model: 'gpt-test', + chatProvider: provider, + }) + await harness.runtime.ingest('second turn', { + model: 'gpt-test', + chatProvider: provider, + }) + + expect(harness.telemetry.chatActivationStarted).toHaveLength(1) + expect(harness.telemetry.chatActivationSucceeded).toHaveLength(1) + expect(harness.telemetry.chatActivationFailed).toHaveLength(0) + expect(harness.telemetry.messageSendStarted).toHaveLength(2) + expect(harness.telemetry.messageRound).toHaveLength(2) + }) + /** * @example * await expect(runtime.ingest('hello', { model, chatProvider })).rejects.toThrow('provider rejected') @@ -358,19 +404,60 @@ describe('createChatOrchestratorRuntime', () => { })).rejects.toThrow('provider rejected') expect(harness.telemetry.chatActivationStarted).toEqual([{ + conversationId: 'session-1', model: 'gpt-test', provider: 'mock-provider', - sessionId: 'session-1', + roundId: 'user-id', source: 'text', + turnIndex: 1, }]) expect(harness.telemetry.chatActivationSucceeded).toEqual([]) expect(harness.telemetry.chatActivationFailed).toEqual([{ + conversationId: 'session-1', errorCode: 'llm_response_failed', failureStage: 'llm_response', model: 'gpt-test', provider: 'mock-provider', + roundId: 'user-id', source: 'text', + turnIndex: 1, }]) + expect(harness.telemetry.messageRoundFailed).toEqual([{ + conversationId: 'session-1', + errorCode: 'llm_response_failed', + failureStage: 'llm_response', + model: 'gpt-test', + provider: 'mock-provider', + roundId: 'user-id', + source: 'text', + turnIndex: 1, + }]) + }) + + it('emits a round failure for later turns without repeating activation failure', async () => { + const harness = createHarness() + + await harness.runtime.ingest('first turn succeeds', { + model: 'gpt-test', + chatProvider: provider, + }) + harness.stream.mockRejectedValueOnce(new Error('later turn rejected')) + + await expect(harness.runtime.ingest('second turn fails', { + model: 'gpt-test', + chatProvider: provider, + })).rejects.toThrow('later turn rejected') + + expect(harness.telemetry.chatActivationFailed).toEqual([]) + expect(harness.telemetry.messageRoundFailed).toEqual([ + expect.objectContaining({ + conversationId: 'session-1', + errorCode: 'llm_response_failed', + failureStage: 'llm_response', + roundId: expect.any(String), + turnIndex: 2, + }), + ]) }) /** diff --git a/packages/core-agent/src/runtime/chat-orchestrator-runtime.ts b/packages/core-agent/src/runtime/chat-orchestrator-runtime.ts index 221f05e31..5ed767960 100644 --- a/packages/core-agent/src/runtime/chat-orchestrator-runtime.ts +++ b/packages/core-agent/src/runtime/chat-orchestrator-runtime.ts @@ -156,6 +156,16 @@ export interface ChatOrchestratorRuntimeState { pendingQueuedSendCount: number } +/** Correlation keys shared by every analytics milestone from one user-to-assistant round. */ +interface ChatRoundCorrelation { + /** Application conversation that owns the round. */ + conversationId: string + /** Stable round key; the runtime reuses the persisted user-message ID. */ + roundId: string + /** One-based user turn position within the conversation. */ + turnIndex: number +} + /** * Dependency surface used by the platform-agnostic chat orchestrator runtime. */ @@ -190,22 +200,21 @@ export interface ChatOrchestratorRuntimeDeps { onSendSettled?: (event: { sessionId: string }) => void /** Called when a send starts and the first assistant placeholder is created. */ onTrackFirstMessage?: () => void - /** Called when a user starts a chat activation attempt. */ - onChatActivationStarted?: (event: { - sessionId: string + /** Called for attempts made before the conversation has its first assistant response. */ + onChatActivationStarted?: (event: ChatRoundCorrelation & { source: 'text' | 'voice' model: string provider: string }) => void - /** Called after one user-to-assistant message round completes successfully. */ - onChatActivationSucceeded?: (event: { + /** Called when the conversation reaches its first successful assistant response. */ + onChatActivationSucceeded?: (event: ChatRoundCorrelation & { source: 'text' | 'voice' model: string provider: string durationMs: number }) => void - /** Called after a chat activation attempt fails before assistant completion. */ - onChatActivationFailed?: (event: { + /** Called when a pre-activation attempt fails before assistant completion. */ + onChatActivationFailed?: (event: ChatRoundCorrelation & { source: 'text' | 'voice' model: string provider: string @@ -213,32 +222,40 @@ export interface ChatOrchestratorRuntimeDeps { errorCode: 'llm_response_failed' }) => void /** Called when a user message send begins. */ - onMessageSendStarted?: (event: { + onMessageSendStarted?: (event: ChatRoundCorrelation & { source: 'text' | 'voice' model: string }) => void /** Called immediately before the provider LLM request starts. */ - onLlmRequestStarted?: (event: { + onLlmRequestStarted?: (event: ChatRoundCorrelation & { model: string provider: string hasVoice: boolean }) => void /** Called when the first text token arrives from the provider stream. */ - onLlmFirstToken?: (event: { + onLlmFirstToken?: (event: ChatRoundCorrelation & { model: string ttfbMs: number }) => void /** Called after the assistant stream is parsed and rendered into runtime state. */ - onAssistantResponseRendered?: (event: { + onAssistantResponseRendered?: (event: ChatRoundCorrelation & { model: string latencyMs: number }) => void /** Called after one user-to-assistant message round completes successfully. */ - onMessageRound?: (event: { + onMessageRound?: (event: ChatRoundCorrelation & { durationMs: number hasVoice: boolean model: string }) => void + /** Called whenever a user-to-assistant round fails before completion. */ + onMessageRoundFailed?: (event: ChatRoundCorrelation & { + source: 'text' | 'voice' + model: string + provider: string + failureStage: 'llm_response' + errorCode: 'llm_response_failed' + }) => void /** Called for context/prompt lifecycle observability. */ onLifecycle?: (record: ChatOrchestratorLifecycleRecord) => void /** Called with the final provider prompt projection. */ @@ -251,6 +268,7 @@ export interface ChatOrchestratorRuntimeDeps { source: 'text' | 'voice' model: string provider: string + roundId: string turnIndex: number }) => void /** Called after the assistant message has been finalized into session history. */ @@ -386,6 +404,14 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) deps.session.ensureSession(sessionId) + const existingSessionMessages = deps.session.getSessionMessages(sessionId) + const turnIndex = existingSessionMessages.filter(message => message.role === 'user').length + 1 + + // Activation measures whether a conversation reaches its first assistant + // response. Later turns still emit message and latency telemetry, but they + // must not inflate the one-time activation milestones. + const isActivationAttempt = !existingSessionMessages.some(message => message.role === 'assistant') + // Datetime is no longer injected through the side-channel context store. // It is applied at message-assembly time (see below) as a system-prompt // date anchor + per-message [HH:MM] prefixes, which is more KV-cache @@ -429,14 +455,25 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) patchForegroundStream(sessionId, buildingMessage) const sendSource = options.input ? 'voice' : 'text' const activeProvider = deps.getActiveProvider?.() ?? '' + // The user message is the durable start of a round, so its ID also serves + // as the correlation key for every telemetry milestone emitted by it. + const roundId = createId() + const correlation: ChatRoundCorrelation = { + conversationId: sessionId, + roundId, + turnIndex, + } deps.onTrackFirstMessage?.() - deps.onChatActivationStarted?.({ - sessionId, - source: sendSource, - model: options.model, - provider: activeProvider, - }) + if (isActivationAttempt) { + deps.onChatActivationStarted?.({ + ...correlation, + source: sendSource, + model: options.model, + provider: activeProvider, + }) + } deps.onMessageSendStarted?.({ + ...correlation, source: sendSource, model: options.model, }) @@ -473,15 +510,13 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) if (shouldAbort()) return - const userMessageId = createId() const userMessage = { role: 'user' as const, content: finalContent, createdAt: sendingCreatedAt, - id: userMessageId, + id: roundId, } deps.session.appendSessionMessage(sessionId, userMessage) - const userTurnIndex = deps.session.getSessionMessages(sessionId).filter(message => message.role === 'user').length // Cloud sync v1: only the raw text part round-trips; image attachments // and other non-text parts stay local. @@ -492,7 +527,8 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) source: sendSource, model: options.model, provider: activeProvider, - turnIndex: userTurnIndex, + roundId, + turnIndex, }) const sessionMessagesForSend = deps.session.getSessionMessages(sessionId) @@ -644,6 +680,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) const llmRequestStartedAt = monotonicNow() let llmFirstTokenEmitted = false deps.onLlmRequestStarted?.({ + ...correlation, model: options.model, provider: deps.getActiveProvider() || 'unknown', hasVoice: !!options.input, @@ -684,6 +721,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) if (!llmFirstTokenEmitted) { llmFirstTokenEmitted = true deps.onLlmFirstToken?.({ + ...correlation, model: options.model, ttfbMs: Math.round(monotonicNow() - llmRequestStartedAt), }) @@ -718,6 +756,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) await parser.end() deps.onAssistantResponseRendered?.({ + ...correlation, model: options.model, latencyMs: Math.round(monotonicNow() - llmRequestStartedAt), }) @@ -751,26 +790,41 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) resetForegroundStream(sessionId) const durationMs = Math.round(monotonicNow() - roundStartedAt) deps.onMessageRound?.({ + ...correlation, durationMs, hasVoice: !!options.input, model: options.model, }) - deps.onChatActivationSucceeded?.({ - durationMs, - source: sendSource, - model: options.model, - provider: activeProvider, - }) + if (isActivationAttempt) { + deps.onChatActivationSucceeded?.({ + ...correlation, + durationMs, + source: sendSource, + model: options.model, + provider: activeProvider, + }) + } } catch (error) { console.error('Error sending message:', error) - deps.onChatActivationFailed?.({ + deps.onMessageRoundFailed?.({ + ...correlation, source: sendSource, model: options.model, provider: activeProvider, failureStage: 'llm_response', errorCode: 'llm_response_failed', }) + if (isActivationAttempt) { + deps.onChatActivationFailed?.({ + ...correlation, + source: sendSource, + model: options.model, + provider: activeProvider, + failureStage: 'llm_response', + errorCode: 'llm_response_failed', + }) + } throw error } finally { diff --git a/packages/stage-pages/src/pages/settings/flux.vue b/packages/stage-pages/src/pages/settings/flux.vue index 551af2b0c..4f34b146c 100644 --- a/packages/stage-pages/src/pages/settings/flux.vue +++ b/packages/stage-pages/src/pages/settings/flux.vue @@ -284,11 +284,11 @@ onMounted(async () => { // PostHog funnel step 1: pricing surface view. Today this is an in-app // settings page (already-authenticated users); when we add a public - // pricing landing page the surface label changes but the event stays the + // pricing landing page the entry-surface label changes but the event stays the // same, so the funnel definition in PostHog doesn't need re-wiring. if (!fluxPurchaseDisabled) { trackPaywallSeen({ - surface: 'settings_flux', + entry_surface: 'settings_flux', reason: 'manual_topup', flux_balance_bucket: fluxBalanceBucket(credits.value), }) @@ -317,7 +317,10 @@ async function handleBuy(stripePriceId: string) { current_plan: 'flux', trigger: 'manual_topup', }) - trackPlanSelected(stripePriceId, { currency: selectedCurrency.value }) + trackPlanSelected(stripePriceId, { + currency: selectedCurrency.value, + entry_surface: 'settings_flux', + }) try { const res = await client.api.v1.stripe.checkout.$post({ json: { stripePriceId, currency: selectedCurrency.value } }) if (!res.ok) { @@ -330,7 +333,10 @@ async function handleBuy(stripePriceId: string) { // PostHog funnel step 3: about to redirect to Stripe. Capture before // the page nav so the event is sent (PostHog's beforeunload handler // would otherwise race the navigation). - trackCheckoutStarted(stripePriceId, { currency: selectedCurrency.value }) + trackCheckoutStarted(stripePriceId, { + currency: selectedCurrency.value, + entry_surface: 'settings_flux', + }) // Electron renderer runs from file:// and cannot navigate to Stripe in-window // (the settings window would load checkout.stripe.com and never come back). // window.open routes through setWindowOpenHandler -> shell.openExternal, so the diff --git a/packages/stage-ui/src/composables/use-analytics.test.ts b/packages/stage-ui/src/composables/use-analytics.test.ts index b5c76dd60..3bf53234b 100644 --- a/packages/stage-ui/src/composables/use-analytics.test.ts +++ b/packages/stage-ui/src/composables/use-analytics.test.ts @@ -70,7 +70,7 @@ describe('useAnalytics conversation product events', () => { analyticsMocks.isPosthogAvailableInBuildMock.mockClear() }) - it('infers the web surface for browser conversation actions', () => { + it('uses app_surface for the web runtime without occupying the event entry surface', () => { const analytics = useAnalytics() analytics.trackTtsStopClicked({ @@ -78,7 +78,7 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenCalledWith('tts_stop_clicked', { - surface: 'web', + app_surface: 'web', reason: 'manual-chat', }) }) @@ -94,7 +94,7 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenCalledWith('chat_session_selected', { - surface: 'mobile', + app_surface: 'mobile', source: 'sessions_drawer', message_count: 4, cloud_synced: true, @@ -118,36 +118,42 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'chat_message_deleted', { - surface: 'electron', + app_surface: 'electron', source: 'history', message_role: 'assistant', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'chat_messages_cleared', { - surface: 'electron', + app_surface: 'electron', source: 'chat_controls', message_count: 3, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'chat_message_retried', { - surface: 'electron', + app_surface: 'electron', source: 'history', }) }) /** * @example - * analytics.trackChatActivationStarted({ provider_mode: 'official', provider_id: 'official-provider', model_id: 'gpt-test', source: 'text' }) - * expect(posthog.capture).toHaveBeenCalledWith('chat_activation_started', expect.objectContaining({ surface: 'web' })) + * analytics.trackChatActivationStarted({ conversation_id: 'session-1', round_id: 'round-1', turn_index: 1, provider_mode: 'official', provider_id: 'official-provider', model_id: 'gpt-test', source: 'text' }) + * expect(posthog.capture).toHaveBeenCalledWith('chat_activation_started', expect.objectContaining({ app_surface: 'web' })) */ it('emits chat activation milestones with inferred surface and normalized fields', () => { const analytics = useAnalytics() analytics.trackChatActivationStarted({ + conversation_id: 'session-1', + round_id: 'round-1', + turn_index: 1, provider_mode: 'official', provider_id: 'official-provider', model_id: 'gpt-test', source: 'text', }) analytics.trackChatActivationSucceeded({ + conversation_id: 'session-1', + round_id: 'round-1', + turn_index: 1, provider_mode: 'official', provider_id: 'official-provider', model_id: 'gpt-test', @@ -155,6 +161,9 @@ describe('useAnalytics conversation product events', () => { source: 'voice', }) analytics.trackChatActivationFailed({ + conversation_id: 'session-1', + round_id: 'round-1', + turn_index: 1, provider_mode: 'custom', provider_id: 'openai-compatible', model_id: 'custom', @@ -164,14 +173,20 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'chat_activation_started', { - surface: 'web', + app_surface: 'web', + conversation_id: 'session-1', + round_id: 'round-1', + turn_index: 1, provider_mode: 'official', provider_id: 'official-provider', model_id: 'gpt-test', source: 'text', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'chat_activation_succeeded', { - surface: 'web', + app_surface: 'web', + conversation_id: 'session-1', + round_id: 'round-1', + turn_index: 1, provider_mode: 'official', provider_id: 'official-provider', model_id: 'gpt-test', @@ -179,7 +194,10 @@ describe('useAnalytics conversation product events', () => { source: 'voice', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'chat_activation_failed', { - surface: 'web', + app_surface: 'web', + conversation_id: 'session-1', + round_id: 'round-1', + turn_index: 1, provider_mode: 'custom', provider_id: 'openai-compatible', model_id: 'custom', @@ -225,13 +243,13 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'tts_provider_selected', { - surface: 'web', + app_surface: 'web', tts_provider_id: 'official-provider', tts_model_id: 'stepfun/tts', source: 'settings', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'voice_selected', { - surface: 'web', + app_surface: 'web', tts_provider_id: 'official-provider', tts_model_id: 'stepfun/tts', voice_id: 'longxiaochun_v2', @@ -239,7 +257,7 @@ describe('useAnalytics conversation product events', () => { source: 'settings', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'voice_preview_played', { - surface: 'web', + app_surface: 'web', tts_provider_id: 'official-provider', tts_model_id: 'stepfun/tts', voice_id: 'longxiaochun_v2', @@ -247,7 +265,7 @@ describe('useAnalytics conversation product events', () => { source: 'manual_preview', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'voice_pack_bound', { - surface: 'web', + app_surface: 'web', tts_provider_id: 'official-provider', tts_model_id: 'stepfun/tts', voice_id: 'longxiaochun_v2', @@ -272,15 +290,17 @@ describe('useAnalytics conversation product events', () => { model_id: 'chat-auto', }) analytics.trackSecondTurnStarted({ + conversation_id: 'session-1', provider_id: 'official-provider', provider_mode: 'official', model_id: 'chat-auto', + round_id: 'round-2', source: 'text', turn_index: 2, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'official_provider_selected', { - surface: 'web', + app_surface: 'web', provider_id: 'official-provider', provider_mode: 'official', source: 'default_auto', @@ -288,10 +308,12 @@ describe('useAnalytics conversation product events', () => { model_id: 'chat-auto', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'second_turn_started', { - surface: 'web', + app_surface: 'web', + conversation_id: 'session-1', provider_id: 'official-provider', provider_mode: 'official', model_id: 'chat-auto', + round_id: 'round-2', source: 'text', turn_index: 2, }) @@ -333,13 +355,13 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'official_tts_exposed', { - surface: 'web', + app_surface: 'web', tts_provider_id: 'official-provider-speech', tts_model_id: 'stepfun/tts', source: 'post_first_chat', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'official_tts_preview_started', { - surface: 'web', + app_surface: 'web', tts_provider_id: 'official-provider-speech', tts_model_id: 'stepfun/tts', voice_id: 'longxiaochun_v2', @@ -347,7 +369,7 @@ describe('useAnalytics conversation product events', () => { source: 'manual_preview', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'official_tts_preview_succeeded', { - surface: 'web', + app_surface: 'web', tts_provider_id: 'official-provider-speech', tts_model_id: 'stepfun/tts', voice_id: 'longxiaochun_v2', @@ -356,7 +378,7 @@ describe('useAnalytics conversation product events', () => { duration_ms: 320, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'official_tts_auto_enabled', { - surface: 'web', + app_surface: 'web', tts_provider_id: 'official-provider-speech', tts_model_id: 'stepfun/tts', source: 'settings', @@ -368,23 +390,55 @@ describe('useAnalytics conversation product events', () => { const analytics = useAnalytics() analytics.trackPaywallSeen({ - surface: 'settings_flux', + entry_surface: 'settings_flux', reason: 'manual_topup', flux_balance_bucket: '1_100', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenCalledWith('paywall_seen', { - surface: 'settings_flux', app_surface: 'web', + entry_surface: 'settings_flux', reason: 'manual_topup', flux_balance_bucket: '1_100', }) }) + it('uses entry_surface across the pricing funnel without emitting surface', () => { + const analytics = useAnalytics() + + analytics.trackPricingViewed('settings_flux', 'one_time') + analytics.trackPlanSelected('price-1', { + currency: 'USD', + entry_surface: 'settings_flux', + }) + analytics.trackCheckoutStarted('price-1', { + currency: 'USD', + entry_surface: 'settings_flux', + }) + + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'pricing_page_viewed', { + entry_surface: 'settings_flux', + plan_period: 'one_time', + }) + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'plan_selected', { + currency: 'USD', + entry_surface: 'settings_flux', + plan_id: 'price-1', + }) + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'checkout_started', { + currency: 'USD', + entry_surface: 'settings_flux', + plan_id: 'price-1', + }, { + send_instantly: true, + transport: 'sendBeacon', + }) + }) + /** * @example * analytics.trackMicrophonePermissionDenied({ stt_provider_id: 'browser-web-speech-api' }) - * expect(posthog.capture).toHaveBeenCalledWith('microphone_permission_denied', expect.objectContaining({ surface: 'web' })) + * expect(posthog.capture).toHaveBeenCalledWith('microphone_permission_denied', expect.objectContaining({ app_surface: 'web' })) */ it('emits voice input friction events with low-cardinality error fields', () => { const analytics = useAnalytics() @@ -409,29 +463,29 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'voice_input_started', { - surface: 'web', + app_surface: 'web', stt_provider_id: 'browser-web-speech-api', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'voice_input_used', { - surface: 'web', + app_surface: 'web', stt_provider_id: 'browser-web-speech-api', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'microphone_permission_requested', { - surface: 'web', + app_surface: 'web', stt_provider_id: 'browser-web-speech-api', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'microphone_permission_denied', { - surface: 'web', + app_surface: 'web', stt_provider_id: 'browser-web-speech-api', error_code: 'permission_denied', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'audio_device_unavailable', { - surface: 'web', + app_surface: 'web', stt_provider_id: 'browser-web-speech-api', error_code: 'device_unavailable', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'voice_input_cancelled', { - surface: 'web', + app_surface: 'web', stt_provider_id: 'browser-web-speech-api', duration_ms: 420, }) @@ -459,14 +513,14 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'model_list_loaded', { - surface: 'web', + app_surface: 'web', provider_id: 'official-provider', provider_mode: 'official', model_count: 3, duration_ms: 25, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'model_list_failed', { - surface: 'web', + app_surface: 'web', provider_id: 'openai-compatible', provider_mode: 'custom', error_code: 'provider_error', @@ -502,20 +556,20 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'provider_config_started', { - surface: 'web', + app_surface: 'web', provider_id: 'openai-compatible', provider_mode: 'custom', step: 'settings_auto_validate', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'provider_config_succeeded', { - surface: 'web', + app_surface: 'web', provider_id: 'official-provider', provider_mode: 'official', step: 'manual_chat_ping', duration_ms: 18, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'provider_config_completed', { - surface: 'web', + app_surface: 'web', provider_id: 'official-provider', provider_mode: 'official', provider_type: 'official', @@ -526,12 +580,12 @@ describe('useAnalytics conversation product events', () => { success: true, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'official_provider_enabled', { - surface: 'web', + app_surface: 'web', provider_name: 'official-provider', entry: 'settings', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'provider_config_failed', { - surface: 'web', + app_surface: 'web', provider_id: 'openai-compatible', provider_mode: 'custom', step: 'settings_auto_validate', @@ -540,14 +594,9 @@ describe('useAnalytics conversation product events', () => { }) }) - it('emits P0 activation, chat, quota, and feature events using canonical names', () => { + it('emits P0 onboarding, message, quota, and feature events using canonical names', () => { const analytics = useAnalytics() - analytics.trackSignupCompleted({ - source: 'google', - locale: 'en', - utm_source: 'launch', - }) analytics.trackOnboardingStarted({ entry: 'app_start', }) @@ -556,41 +605,19 @@ describe('useAnalytics conversation product events', () => { selected_provider_id: 'official-provider', selected_use_case: 'role_chat', }) - analytics.trackChatStarted({ - conversation_id: 'session-1', - provider_type: 'official', - provider_name: 'official-provider', - model: 'gpt-test', - entry: 'chat', - is_paid_user: true, - }) analytics.trackMessageSent({ conversation_id: 'session-1', provider_type: 'official', provider_name: 'official-provider', model: 'gpt-test', message_id: 'message-1', + round_id: 'message-1', + turn_index: 1, message_index: 2, message_length: 24, has_attachment: false, mode: 'text', }) - analytics.trackAssistantResponseCompleted({ - conversation_id: 'session-1', - provider_type: 'official', - provider_name: 'official-provider', - model: 'gpt-test', - latency_ms: 350, - completion_length: 120, - }) - analytics.trackChatFailed({ - conversation_id: 'session-1', - provider_type: 'custom', - provider_name: 'openai-compatible', - model: 'custom', - failure_stage: 'llm_response', - error_code: 'provider_error', - }) analytics.trackQuotaLimitReached({ limit_type: 'flux', current_usage: 0, @@ -609,73 +636,43 @@ describe('useAnalytics conversation product events', () => { success: true, }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'signup_completed', { - source: 'google', - locale: 'en', - utm_source: 'launch', - }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'onboarding_started', { - surface: 'web', + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'onboarding_started', { + app_surface: 'web', entry: 'app_start', }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'onboarding_completed', { - surface: 'web', + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'onboarding_completed', { + app_surface: 'web', selected_provider_type: 'official', selected_provider_id: 'official-provider', selected_use_case: 'role_chat', }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'chat_started', { - surface: 'web', - conversation_id: 'session-1', - provider_type: 'official', - provider_name: 'official-provider', - model: 'gpt-test', - entry: 'chat', - is_paid_user: true, - }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'message_sent', { - surface: 'web', + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'message_sent', { + app_surface: 'web', conversation_id: 'session-1', provider_type: 'official', provider_name: 'official-provider', model: 'gpt-test', message_id: 'message-1', + round_id: 'message-1', + turn_index: 1, message_index: 2, message_length: 24, has_attachment: false, mode: 'text', }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'assistant_response_completed', { - surface: 'web', - conversation_id: 'session-1', - provider_type: 'official', - provider_name: 'official-provider', - model: 'gpt-test', - latency_ms: 350, - completion_length: 120, - }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(7, 'chat_failed', { - surface: 'web', - conversation_id: 'session-1', - provider_type: 'custom', - provider_name: 'openai-compatible', - model: 'custom', - failure_stage: 'llm_response', - error_code: 'provider_error', - }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(8, 'quota_limit_reached', { + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'quota_limit_reached', { limit_type: 'flux', current_usage: 0, limit_value: 0, entry: 'pricing', }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(9, 'upgrade_clicked', { + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'upgrade_clicked', { source_page: 'settings_flux', current_plan: 'flux', trigger: 'manual_topup', }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(10, 'feature_used', { - surface: 'web', + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'feature_used', { + app_surface: 'web', feature_name: 'chat', business_domain: 'conversation', entry: 'chat', @@ -736,43 +733,43 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'conversation_created', { - surface: 'web', + app_surface: 'web', conversation_id: 'session-1', source: 'new_session', character_id: 'character-1', cloud_synced: true, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'conversation_renamed', { - surface: 'web', + app_surface: 'web', conversation_id: 'session-1', source: 'sessions_drawer', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'conversation_shared', { - surface: 'web', + app_surface: 'web', conversation_id: 'session-1', source: 'share_button', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'conversation_deleted', { - surface: 'web', + app_surface: 'web', conversation_id: 'session-1', message_count: 6, cloud_synced: true, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'attachment_uploaded', { - surface: 'web', + app_surface: 'web', attachment_type: 'image', size_bytes: 2048, source: 'chat', success: true, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'preset_used', { - surface: 'web', + app_surface: 'web', preset_id: 'preset-live2d-1', preset_type: 'stage_model', source: 'settings', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(7, 'provider_switched', { - surface: 'web', + app_surface: 'web', from_provider: 'openai-compatible', to_provider: 'official-provider', from_provider_type: 'custom', @@ -780,14 +777,14 @@ describe('useAnalytics conversation product events', () => { reason: 'manual', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(8, 'settings_changed', { - surface: 'web', + app_surface: 'web', setting_name: 'analytics_enabled', previous_value: false, new_value: true, source: 'settings', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(9, 'support_contacted', { - surface: 'web', + app_surface: 'web', channel: 'discord', source: 'settings', category: 'payment', @@ -821,7 +818,7 @@ describe('useAnalytics conversation product events', () => { }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'bug_report_submitted', { - surface: 'web', + app_surface: 'web', source: 'app', category: 'update', severity: 'major', @@ -832,7 +829,7 @@ describe('useAnalytics conversation product events', () => { screenshot_attached: true, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'feedback_submitted', { - surface: 'web', + app_surface: 'web', source: 'discord', category: 'voice_input', severity: 'minor', @@ -851,21 +848,21 @@ describe('useAnalytics conversation product events', () => { analytics.trackAccountDeletionRequested() analytics.trackOauthCallbackFailed({ stage: 'missing_flow_state' }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'password_changed', { surface: 'web' }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'password_reset_requested', { surface: 'web' }) + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'password_changed', { app_surface: 'web' }) + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'password_reset_requested', { app_surface: 'web' }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith( 3, 'oauth_provider_link_started', - { surface: 'web', provider: 'github' }, + { app_surface: 'web', provider: 'github' }, { send_instantly: true, transport: 'sendBeacon' }, ) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'oauth_provider_unlinked', { - surface: 'web', + app_surface: 'web', provider: 'google', }) - expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'account_deletion_requested', { surface: 'web' }) + expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'account_deletion_requested', { app_surface: 'web' }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'oauth_callback_failed', { - surface: 'web', + app_surface: 'web', stage: 'missing_flow_state', }) }) @@ -879,16 +876,16 @@ describe('useAnalytics conversation product events', () => { analytics.trackCharacterUpdated({ character_id: 'character-1' }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'card_edited', { - surface: 'web', + app_surface: 'web', card_id: 'card-1', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'scene_background_set', { - surface: 'web', + app_surface: 'web', source: 'card_gallery', cleared: false, }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'scene_background_set', { - surface: 'web', + app_surface: 'web', source: 'scene_settings', cleared: true, }) @@ -904,11 +901,11 @@ describe('useAnalytics conversation product events', () => { analytics.trackDataAction({ action: 'app_data_cleared' }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'data_action', { - surface: 'web', + app_surface: 'web', action: 'chats_exported', }) expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'data_action', { - surface: 'web', + app_surface: 'web', action: 'app_data_cleared', }) }) diff --git a/packages/stage-ui/src/composables/use-analytics.ts b/packages/stage-ui/src/composables/use-analytics.ts index 08e492955..9f212d8fd 100644 --- a/packages/stage-ui/src/composables/use-analytics.ts +++ b/packages/stage-ui/src/composables/use-analytics.ts @@ -51,7 +51,13 @@ export type OauthCallbackFailureStage | 'parse' | 'relay_unreachable' -interface ChatActivationBaseProperties { +interface ChatRoundCorrelationProperties { + conversation_id: string + round_id: string + turn_index: number +} + +interface ChatActivationBaseProperties extends ChatRoundCorrelationProperties { provider_mode: ProviderMode provider_id: string model_id: string @@ -167,23 +173,23 @@ export function useAnalytics() { * - Any UI surface that shows Flux packages / subscription plans renders. * Current surfaces: `settings_flux` (in-app billing settings). Future * surfaces (a public pricing landing page, an upsell modal) just pass a - * different `surface` so the funnel split stays clean. + * different `entry_surface` so the funnel split stays clean. * * Expects: - * - `surface` is a stable identifier — don't rename without coordinating + * - `entry_surface` is a stable identifier — don't rename without coordinating * PostHog funnel definitions in `docs/ai-context/metrics-ownership.md`. */ - function trackPricingViewed(surface: string, planPeriod?: 'monthly' | 'annual' | 'one_time') { + function trackPricingViewed(entrySurface: string, planPeriod?: 'monthly' | 'annual' | 'one_time') { if (!canCapture()) return - posthog.capture('pricing_page_viewed', { surface, ...(planPeriod && { plan_period: planPeriod }) }) + posthog.capture('pricing_page_viewed', { entry_surface: entrySurface, ...(planPeriod && { plan_period: planPeriod }) }) } /** * Pricing funnel — step 2. Fires when the user picks a plan/package but * hasn't yet kicked off the Stripe checkout redirect. */ - function trackPlanSelected(planId: string, properties?: { price_minor_unit?: number, currency?: string }) { + function trackPlanSelected(planId: string, properties: { entry_surface: string, price_minor_unit?: number, currency?: string }) { if (!canCapture()) return posthog.capture('plan_selected', { plan_id: planId, ...properties }) @@ -206,7 +212,7 @@ export function useAnalytics() { * `apps/server/src/services/domain/product-events.ts`), keyed by the * Better Auth user id. */ - function trackCheckoutStarted(planId: string, properties: { checkout_session_id?: string, price_minor_unit?: number, currency?: string }) { + function trackCheckoutStarted(planId: string, properties: { entry_surface: string, checkout_session_id?: string, price_minor_unit?: number, currency?: string }) { if (!canCapture()) return posthog.capture( @@ -217,14 +223,14 @@ export function useAnalytics() { } function trackPaywallSeen(properties: { - surface: string + entry_surface: string reason: 'manual_topup' | 'insufficient_balance' | 'checkout_recovery' | 'unknown' flux_balance_bucket: FluxBalanceBucket }) { if (!canCapture()) return posthog.capture('paywall_seen', { - surface: properties.surface, + entry_surface: properties.entry_surface, app_surface: getConversationAnalyticsSurface(), reason: properties.reason, flux_balance_bucket: properties.flux_balance_bucket, @@ -243,7 +249,7 @@ export function useAnalytics() { return posthog.capture('oauth_callback_failed', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -253,13 +259,13 @@ export function useAnalytics() { function trackPasswordChanged() { if (!canCapture()) return - posthog.capture('password_changed', { surface: getConversationAnalyticsSurface() }) + posthog.capture('password_changed', { app_surface: getConversationAnalyticsSurface() }) } function trackPasswordResetRequested() { if (!canCapture()) return - posthog.capture('password_reset_requested', { surface: getConversationAnalyticsSurface() }) + posthog.capture('password_reset_requested', { app_surface: getConversationAnalyticsSurface() }) } function trackOauthProviderLinkStarted(properties: { provider: string }) { @@ -272,7 +278,7 @@ export function useAnalytics() { 'oauth_provider_link_started', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }, { send_instantly: true, transport: 'sendBeacon' }, ) @@ -283,7 +289,7 @@ export function useAnalytics() { return posthog.capture('oauth_provider_unlinked', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -295,21 +301,7 @@ export function useAnalytics() { function trackAccountDeletionRequested() { if (!canCapture()) return - posthog.capture('account_deletion_requested', { surface: getConversationAnalyticsSurface() }) - } - - function trackSignupCompleted(properties: { - source: string - referrer?: string - country?: string - locale?: string - utm_source?: string - utm_medium?: string - utm_campaign?: string - }) { - if (!canCapture()) - return - posthog.capture('signup_completed', properties) + posthog.capture('account_deletion_requested', { app_surface: getConversationAnalyticsSurface() }) } function trackOnboardingStarted(properties: { entry: ProductAnalyticsEntry }) { @@ -317,7 +309,7 @@ export function useAnalytics() { return posthog.capture('onboarding_started', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -326,7 +318,7 @@ export function useAnalytics() { return posthog.capture('onboarding_completed', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -357,7 +349,7 @@ export function useAnalytics() { from_model: fromModel, to_model: toModel, reason, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -379,39 +371,55 @@ export function useAnalytics() { // (per-request volume stays in DB/Grafana). These client emits supply the // user-facing latency picture (TTFT, render time) the server cannot see. - function trackMessageSendStarted(properties: { source: 'text' | 'voice', model?: string }) { + function trackMessageSendStarted(properties: ChatRoundCorrelationProperties & { source: 'text' | 'voice', model?: string }) { if (!canCapture()) return posthog.capture('message_send_started', properties) } - function trackLlmRequestStarted(properties: { model: string, provider: string, has_voice: boolean }) { + function trackLlmRequestStarted(properties: ChatRoundCorrelationProperties & { model: string, provider: string, has_voice: boolean }) { if (!canCapture()) return posthog.capture('llm_request_started', properties) } /** First token from a streaming LLM response — perceived responsiveness anchor. */ - function trackLlmFirstToken(properties: { model: string, ttfb_ms: number }) { + function trackLlmFirstToken(properties: ChatRoundCorrelationProperties & { model: string, ttfb_ms: number }) { if (!canCapture()) return posthog.capture('llm_first_token', properties) } /** Stream finished and the UI has fully rendered the assistant message. */ - function trackAssistantResponseRendered(properties: { model: string, latency_ms: number }) { + function trackAssistantResponseRendered(properties: ChatRoundCorrelationProperties & { model: string, latency_ms: number }) { if (!canCapture()) return posthog.capture('assistant_response_rendered', properties) } /** Closing event for one full message round (user send → assistant render). */ - function trackMessageRound(properties: { duration_ms: number, has_voice: boolean, model: string }) { + function trackMessageRound(properties: ChatRoundCorrelationProperties & { duration_ms: number, has_voice: boolean, model: string }) { if (!canCapture()) return posthog.capture('message_round', properties) } + /** Canonical failure event for every user-to-assistant round, including post-activation turns. */ + function trackMessageRoundFailed(properties: ChatRoundCorrelationProperties & { + provider_id: string + model_id: string + source: 'text' | 'voice' + error_code: string + failure_stage: ChatActivationFailureStage + }) { + if (!canCapture()) + return + posthog.capture('message_round_failed', { + ...properties, + app_surface: getConversationAnalyticsSurface(), + }) + } + // ─── Chat activation events ────────────────────────────────────────── function trackChatActivationStarted(properties: ChatActivationBaseProperties) { @@ -419,7 +427,7 @@ export function useAnalytics() { return posthog.capture('chat_activation_started', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -428,7 +436,7 @@ export function useAnalytics() { return posthog.capture('chat_activation_succeeded', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -440,19 +448,7 @@ export function useAnalytics() { return posthog.capture('chat_activation_failed', { ...properties, - surface: getConversationAnalyticsSurface(), - }) - } - - function trackChatStarted(properties: ConversationBaseProperties & { - entry: ProductAnalyticsEntry - is_paid_user?: boolean - }) { - if (!canCapture()) - return - posthog.capture('chat_started', { - ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -467,11 +463,13 @@ export function useAnalytics() { return posthog.capture('official_provider_selected', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } function trackMessageSent(properties: ConversationBaseProperties & { + round_id: string + turn_index: number message_id?: string message_index?: number message_length?: number @@ -482,40 +480,16 @@ export function useAnalytics() { return posthog.capture('message_sent', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } - function trackAssistantResponseCompleted(properties: ConversationBaseProperties & { - latency_ms?: number - completion_length?: number - }) { - if (!canCapture()) - return - posthog.capture('assistant_response_completed', { - ...properties, - surface: getConversationAnalyticsSurface(), - }) - } - - function trackChatFailed(properties: ConversationBaseProperties & { - failure_stage: ChatActivationFailureStage - error_code: string - }) { - if (!canCapture()) - return - posthog.capture('chat_failed', { - ...properties, - surface: getConversationAnalyticsSurface(), - }) - } - - function trackSecondTurnStarted(properties: ChatActivationBaseProperties & { turn_index: number }) { + function trackSecondTurnStarted(properties: ChatActivationBaseProperties) { if (!canCapture()) return posthog.capture('second_turn_started', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -529,7 +503,7 @@ export function useAnalytics() { return posthog.capture('model_list_loaded', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -543,7 +517,7 @@ export function useAnalytics() { return posthog.capture('model_list_failed', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -552,7 +526,7 @@ export function useAnalytics() { return posthog.capture('provider_config_started', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -561,7 +535,7 @@ export function useAnalytics() { return posthog.capture('provider_config_succeeded', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) trackProviderConfigCompleted({ ...properties, @@ -583,7 +557,7 @@ export function useAnalytics() { return posthog.capture('provider_config_failed', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -599,7 +573,7 @@ export function useAnalytics() { provider_type: properties.provider_mode, provider_name: properties.provider_id, entry_page: properties.step, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -611,7 +585,7 @@ export function useAnalytics() { return posthog.capture('official_provider_enabled', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -622,7 +596,7 @@ export function useAnalytics() { return posthog.capture('tts_stop_clicked', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -631,7 +605,7 @@ export function useAnalytics() { return posthog.capture('chat_session_selected', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -640,7 +614,7 @@ export function useAnalytics() { return posthog.capture('chat_message_deleted', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -649,7 +623,7 @@ export function useAnalytics() { return posthog.capture('chat_messages_cleared', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -658,7 +632,7 @@ export function useAnalytics() { return posthog.capture('chat_message_retried', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -672,7 +646,7 @@ export function useAnalytics() { return posthog.capture('conversation_created', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -684,7 +658,7 @@ export function useAnalytics() { return posthog.capture('conversation_renamed', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -696,7 +670,7 @@ export function useAnalytics() { return posthog.capture('conversation_shared', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -709,7 +683,7 @@ export function useAnalytics() { return posthog.capture('conversation_deleted', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -738,11 +712,11 @@ export function useAnalytics() { return posthog.capture('voice_input_started', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) posthog.capture('voice_input_used', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -751,7 +725,7 @@ export function useAnalytics() { return posthog.capture('microphone_permission_requested', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -760,7 +734,7 @@ export function useAnalytics() { return posthog.capture('microphone_permission_denied', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -769,7 +743,7 @@ export function useAnalytics() { return posthog.capture('audio_device_unavailable', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -778,7 +752,7 @@ export function useAnalytics() { return posthog.capture('voice_input_cancelled', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -793,7 +767,7 @@ export function useAnalytics() { return posthog.capture('bug_report_submitted', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -802,7 +776,7 @@ export function useAnalytics() { return posthog.capture('feedback_submitted', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -848,7 +822,7 @@ export function useAnalytics() { return posthog.capture('tts_provider_selected', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -861,7 +835,7 @@ export function useAnalytics() { return posthog.capture('voice_selected', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -874,7 +848,7 @@ export function useAnalytics() { return posthog.capture('voice_preview_played', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -886,7 +860,7 @@ export function useAnalytics() { return posthog.capture('voice_pack_bound', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -900,7 +874,7 @@ export function useAnalytics() { return posthog.capture('attachment_uploaded', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -909,7 +883,7 @@ export function useAnalytics() { return posthog.capture('official_tts_exposed', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -922,7 +896,7 @@ export function useAnalytics() { return posthog.capture('preset_used', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -936,7 +910,7 @@ export function useAnalytics() { return posthog.capture('official_tts_preview_started', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -951,7 +925,7 @@ export function useAnalytics() { return posthog.capture('official_tts_preview_succeeded', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -966,7 +940,7 @@ export function useAnalytics() { return posthog.capture('provider_switched', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -980,7 +954,7 @@ export function useAnalytics() { return posthog.capture('settings_changed', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -993,7 +967,7 @@ export function useAnalytics() { return posthog.capture('support_contacted', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -1005,7 +979,7 @@ export function useAnalytics() { return posthog.capture('official_tts_auto_enabled', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -1029,7 +1003,7 @@ export function useAnalytics() { return posthog.capture('card_edited', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -1039,7 +1013,7 @@ export function useAnalytics() { return posthog.capture('scene_background_set', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -1097,7 +1071,7 @@ export function useAnalytics() { posthog.capture('flux_low_warning_shown', properties) } - function trackFluxTopupClicked(properties: { balance: number, surface: string }) { + function trackFluxTopupClicked(properties: { balance: number, entry_surface: string }) { if (!canCapture()) return posthog.capture('flux_topup_clicked', properties) @@ -1134,7 +1108,7 @@ export function useAnalytics() { return posthog.capture('feature_used', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -1153,7 +1127,7 @@ export function useAnalytics() { return posthog.capture('data_action', { ...properties, - surface: getConversationAnalyticsSurface(), + app_surface: getConversationAnalyticsSurface(), }) } @@ -1243,7 +1217,6 @@ export function useAnalytics() { trackPlanSelected, trackCheckoutStarted, trackPaywallSeen, - trackSignupCompleted, trackOauthCallbackFailed, trackPasswordChanged, trackPasswordResetRequested, @@ -1261,11 +1234,9 @@ export function useAnalytics() { trackLlmRequestStarted, trackLlmFirstToken, trackAssistantResponseRendered, - trackAssistantResponseCompleted, trackMessageRound, - trackChatStarted, + trackMessageRoundFailed, trackMessageSent, - trackChatFailed, trackChatActivationStarted, trackChatActivationSucceeded, trackChatActivationFailed, diff --git a/packages/stage-ui/src/stores/analytics/index.ts b/packages/stage-ui/src/stores/analytics/index.ts index 0a4ead9d7..8e785617d 100644 --- a/packages/stage-ui/src/stores/analytics/index.ts +++ b/packages/stage-ui/src/stores/analytics/index.ts @@ -59,7 +59,7 @@ export const useSharedAnalyticsStore = defineStore('analytics-shared', () => { previous_value: previousEnabled, new_value: enabled, source: 'settings', - surface: analyticsSurface(), + app_surface: analyticsSurface(), }) } @@ -71,7 +71,7 @@ export const useSharedAnalyticsStore = defineStore('analytics-shared', () => { previous_value: previousEnabled, new_value: enabled, source: 'settings', - surface: analyticsSurface(), + app_surface: analyticsSurface(), }) } @@ -189,7 +189,7 @@ export const useSharedAnalyticsStore = defineStore('analytics-shared', () => { from_provider_type: providerMode(prev.provider), to_provider_type: providerMode(next.provider), reason: 'manual', - surface: analyticsSurface(), + app_surface: analyticsSurface(), }) } @@ -204,7 +204,7 @@ export const useSharedAnalyticsStore = defineStore('analytics-shared', () => { to_model: next.model, provider: next.provider, reason: 'manual', - surface: analyticsSurface(), + app_surface: analyticsSurface(), }) } }, diff --git a/packages/stage-ui/src/stores/analytics/posthog.test.ts b/packages/stage-ui/src/stores/analytics/posthog.test.ts new file mode 100644 index 000000000..57bf31533 --- /dev/null +++ b/packages/stage-ui/src/stores/analytics/posthog.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it, vi } from 'vitest' + +import { ensurePosthogInitialized } from './posthog' + +const posthogMocks = vi.hoisted(() => ({ + init: vi.fn(), + register: vi.fn(), +})) + +vi.mock('posthog-js', () => ({ + default: posthogMocks, +})) + +vi.mock('@proj-airi/stage-shared', () => ({ + isStageCapacitor: () => false, + isStageTamagotchi: () => false, +})) + +vi.mock('../../../../../posthog.config', () => ({ + DEFAULT_POSTHOG_CONFIG: {}, + POSTHOG_ENABLED: true, + POSTHOG_PROJECT_KEY: 'test-project-key', +})) + +describe('stage PostHog initialization', () => { + // ROOT CAUSE: + // + // `surface` was registered as the runtime platform but individual events + // also used `surface` for entry points such as `settings_flux`. Event + // properties overwrite super properties, so platform breakdowns drifted. + it('registers the runtime under the dedicated app_surface property', () => { + expect(ensurePosthogInitialized(true)).toBe(true) + expect(posthogMocks.register).toHaveBeenCalledWith({ app_surface: 'web' }) + }) +}) diff --git a/packages/stage-ui/src/stores/analytics/posthog.ts b/packages/stage-ui/src/stores/analytics/posthog.ts index 2a226bdf8..f9eff92f3 100644 --- a/packages/stage-ui/src/stores/analytics/posthog.ts +++ b/packages/stage-ui/src/stores/analytics/posthog.ts @@ -13,7 +13,7 @@ import { let posthogInitialized = false // All AIRI surfaces (web, desktop, mobile) capture into a single PostHog -// project. The platform is carried on every event via the `surface` super +// project. The platform is carried on every event via the `app_surface` super // property (registered at init), so cross-platform funnels live in one // project instead of being split across per-platform projects. function currentSurface(): 'web' | 'mobile' | 'electron' { @@ -43,7 +43,7 @@ export function ensurePosthogInitialized(enabled: boolean): boolean { }) // Tag every event (including autocapture / pageview) with the platform so // the single project can still be broken down by web / desktop / mobile. - posthog.register({ surface: currentSurface() }) + posthog.register({ app_surface: currentSurface() }) posthogInitialized = true return true } diff --git a/packages/stage-ui/src/stores/chat.contract.test.ts b/packages/stage-ui/src/stores/chat.contract.test.ts index 7aa1c9b2f..4004825f5 100644 --- a/packages/stage-ui/src/stores/chat.contract.test.ts +++ b/packages/stage-ui/src/stores/chat.contract.test.ts @@ -39,7 +39,26 @@ const ioTracerMocks = vi.hoisted(() => { const llmStreamMock = vi.fn() const trackFirstMessageMock = vi.fn() -const trackSecondTurnStartedMock = vi.fn() +const chatAnalyticsMocks = vi.hoisted(() => ({ + trackAssistantResponseRendered: vi.fn(), + trackChatActivationFailed: vi.fn(), + trackChatActivationStarted: vi.fn(), + trackChatActivationSucceeded: vi.fn(), + trackLlmFirstToken: vi.fn(), + trackLlmRequestStarted: vi.fn(), + trackMessageRound: vi.fn(), + trackMessageRoundFailed: vi.fn(), + trackMessageSendStarted: vi.fn(), + trackMessageSent: vi.fn(), + trackSecondTurnStarted: vi.fn(), +})) +const trackSecondTurnStartedMock = chatAnalyticsMocks.trackSecondTurnStarted +const redundantChatAnalyticsMocks = vi.hoisted(() => ({ + trackAssistantResponseCompleted: vi.fn(), + trackChatFailed: vi.fn(), + trackChatStarted: vi.fn(), + trackFeatureUsed: vi.fn(), +})) const ingestContextMessageMock = vi.fn() const getContextsSnapshotMock = vi.fn() const createMinecraftContextMock = vi.fn() @@ -65,19 +84,20 @@ vi.mock('pinia', async () => { vi.mock('../composables', () => ({ useAnalytics: () => ({ trackFirstMessage: trackFirstMessageMock, - trackChatFailed: vi.fn(), - trackChatStarted: vi.fn(), - trackMessageSendStarted: vi.fn(), - trackMessageSent: vi.fn(), - trackLlmRequestStarted: vi.fn(), - trackLlmFirstToken: vi.fn(), - trackAssistantResponseRendered: vi.fn(), - trackAssistantResponseCompleted: vi.fn(), - trackMessageRound: vi.fn(), - trackFeatureUsed: vi.fn(), - trackChatActivationStarted: vi.fn(), - trackChatActivationSucceeded: vi.fn(), - trackChatActivationFailed: vi.fn(), + trackChatFailed: redundantChatAnalyticsMocks.trackChatFailed, + trackChatStarted: redundantChatAnalyticsMocks.trackChatStarted, + trackMessageSendStarted: chatAnalyticsMocks.trackMessageSendStarted, + trackMessageSent: chatAnalyticsMocks.trackMessageSent, + trackLlmRequestStarted: chatAnalyticsMocks.trackLlmRequestStarted, + trackLlmFirstToken: chatAnalyticsMocks.trackLlmFirstToken, + trackAssistantResponseRendered: chatAnalyticsMocks.trackAssistantResponseRendered, + trackAssistantResponseCompleted: redundantChatAnalyticsMocks.trackAssistantResponseCompleted, + trackMessageRound: chatAnalyticsMocks.trackMessageRound, + trackMessageRoundFailed: chatAnalyticsMocks.trackMessageRoundFailed, + trackFeatureUsed: redundantChatAnalyticsMocks.trackFeatureUsed, + trackChatActivationStarted: chatAnalyticsMocks.trackChatActivationStarted, + trackChatActivationSucceeded: chatAnalyticsMocks.trackChatActivationSucceeded, + trackChatActivationFailed: chatAnalyticsMocks.trackChatActivationFailed, trackSecondTurnStarted: trackSecondTurnStartedMock, }), })) @@ -166,7 +186,12 @@ describe('chat orchestrator contract', () => { setActivePinia(createPinia()) llmStreamMock.mockReset() trackFirstMessageMock.mockReset() - trackSecondTurnStartedMock.mockReset() + for (const analyticsMock of Object.values(chatAnalyticsMocks)) + analyticsMock.mockReset() + redundantChatAnalyticsMocks.trackAssistantResponseCompleted.mockReset() + redundantChatAnalyticsMocks.trackChatFailed.mockReset() + redundantChatAnalyticsMocks.trackChatStarted.mockReset() + redundantChatAnalyticsMocks.trackFeatureUsed.mockReset() ingestContextMessageMock.mockReset() getContextsSnapshotMock.mockReset() getContextsSnapshotMock.mockReturnValue({}) @@ -190,6 +215,39 @@ describe('chat orchestrator contract', () => { sessionMessages['session-1'] = [{ role: 'system', content: 'system prompt', createdAt: 1, id: 'system' }] }) + it('forwards one correlation identity across every PostHog chat milestone', async () => { + llmStreamMock.mockImplementation(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => { + await options.onStreamEvent({ type: 'text-delta', text: 'ok' }) + await options.onStreamEvent({ type: 'finish', finishReason: 'stop' }) + }) + + const store = useChatOrchestratorStore() + await store.ingest('hello', { + model: 'gpt-test', + chatProvider: provider, + }) + + const messageProperties = chatAnalyticsMocks.trackMessageSent.mock.calls[0]?.[0] + expect(messageProperties).toMatchObject({ + conversation_id: 'session-1', + round_id: messageProperties.message_id, + turn_index: 1, + }) + + const correlation = { + conversation_id: 'session-1', + round_id: messageProperties.round_id, + turn_index: 1, + } + expect(chatAnalyticsMocks.trackMessageSendStarted).toHaveBeenCalledWith(expect.objectContaining(correlation)) + expect(chatAnalyticsMocks.trackLlmRequestStarted).toHaveBeenCalledWith(expect.objectContaining(correlation)) + expect(chatAnalyticsMocks.trackLlmFirstToken).toHaveBeenCalledWith(expect.objectContaining(correlation)) + expect(chatAnalyticsMocks.trackAssistantResponseRendered).toHaveBeenCalledWith(expect.objectContaining(correlation)) + expect(chatAnalyticsMocks.trackMessageRound).toHaveBeenCalledWith(expect.objectContaining(correlation)) + expect(chatAnalyticsMocks.trackChatActivationStarted).toHaveBeenCalledWith(expect.objectContaining(correlation)) + expect(chatAnalyticsMocks.trackChatActivationSucceeded).toHaveBeenCalledWith(expect.objectContaining(correlation)) + }) + it('emits second turn analytics from chat sends', async () => { activeProviderRef.value = 'official-provider' llmStreamMock.mockImplementation(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => { @@ -210,9 +268,64 @@ describe('chat orchestrator contract', () => { expect(trackSecondTurnStartedMock).toHaveBeenCalledTimes(1) expect(trackSecondTurnStartedMock).toHaveBeenCalledWith({ + conversation_id: 'session-1', provider_id: 'official-provider', provider_mode: 'official', model_id: 'chat-auto', + round_id: expect.any(String), + source: 'text', + turn_index: 2, + }) + }) + + // ROOT CAUSE: + // + // One successful send emitted both the canonical message/latency events + // and four generic aliases, multiplying PostHog volume without adding a + // distinct product decision. + it('does not emit redundant generic chat aliases for a successful send', async () => { + llmStreamMock.mockImplementation(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => { + await options.onStreamEvent({ type: 'text-delta', text: 'ok' }) + await options.onStreamEvent({ type: 'finish', finishReason: 'stop' }) + }) + + const store = useChatOrchestratorStore() + await store.ingest('hello', { + model: 'gpt-test', + chatProvider: provider, + }) + + expect(redundantChatAnalyticsMocks.trackChatStarted).not.toHaveBeenCalled() + expect(redundantChatAnalyticsMocks.trackAssistantResponseCompleted).not.toHaveBeenCalled() + expect(redundantChatAnalyticsMocks.trackChatFailed).not.toHaveBeenCalled() + expect(redundantChatAnalyticsMocks.trackFeatureUsed).not.toHaveBeenCalled() + }) + + it('forwards later-turn failures to the canonical round failure event', async () => { + llmStreamMock.mockImplementationOnce(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => { + await options.onStreamEvent({ type: 'text-delta', text: 'ok' }) + await options.onStreamEvent({ type: 'finish', finishReason: 'stop' }) + }) + llmStreamMock.mockRejectedValueOnce(new Error('later turn rejected')) + + const store = useChatOrchestratorStore() + await store.ingest('first turn', { + model: 'gpt-test', + chatProvider: provider, + }) + await expect(store.ingest('second turn', { + model: 'gpt-test', + chatProvider: provider, + })).rejects.toThrow('later turn rejected') + + expect(chatAnalyticsMocks.trackChatActivationFailed).not.toHaveBeenCalled() + expect(chatAnalyticsMocks.trackMessageRoundFailed).toHaveBeenCalledWith({ + conversation_id: 'session-1', + error_code: 'llm_response_failed', + failure_stage: 'llm_response', + model_id: 'gpt-test', + provider_id: 'mock-provider', + round_id: expect.any(String), source: 'text', turn_index: 2, }) diff --git a/packages/stage-ui/src/stores/chat.ts b/packages/stage-ui/src/stores/chat.ts index 08b57ce45..665c57eeb 100644 --- a/packages/stage-ui/src/stores/chat.ts +++ b/packages/stage-ui/src/stores/chat.ts @@ -51,16 +51,13 @@ export const useChatOrchestratorStore = defineStore('chat-orchestrator', () => { const { activeModel, activeProvider } = storeToRefs(consciousnessStore) const { trackFirstMessage, - trackChatFailed, - trackChatStarted, trackMessageSendStarted, trackMessageSent, trackLlmRequestStarted, trackLlmFirstToken, trackAssistantResponseRendered, - trackAssistantResponseCompleted, trackMessageRound, - trackFeatureUsed, + trackMessageRoundFailed, trackChatActivationStarted, trackChatActivationSucceeded, trackChatActivationFailed, @@ -185,109 +182,119 @@ export const useChatOrchestratorStore = defineStore('chat-orchestrator', () => { onStateChange: syncRuntimeState, onSendSettled: settleOwnedActiveTurnSpan, onTrackFirstMessage: trackFirstMessage, - onMessageSendStarted: ({ source, model }) => { + onMessageSendStarted: ({ conversationId, roundId, turnIndex, source, model }) => { lastSendSource = source trackMessageSendStarted({ + conversation_id: conversationId, + round_id: roundId, + turn_index: turnIndex, source, model, }) - trackChatStarted({ - conversation_id: activeSessionId.value || 'unknown', - provider_type: providerMode(activeProvider.value), - provider_name: activeProvider.value || 'unknown', - model: model || 'unknown', - entry: 'chat', - }) }, - onLlmRequestStarted: ({ model, provider, hasVoice }) => trackLlmRequestStarted({ + onLlmRequestStarted: ({ conversationId, roundId, turnIndex, model, provider, hasVoice }) => trackLlmRequestStarted({ + conversation_id: conversationId, + round_id: roundId, + turn_index: turnIndex, model, provider, has_voice: hasVoice, }), - onLlmFirstToken: ({ model, ttfbMs }) => trackLlmFirstToken({ + onLlmFirstToken: ({ conversationId, roundId, turnIndex, model, ttfbMs }) => trackLlmFirstToken({ + conversation_id: conversationId, + round_id: roundId, + turn_index: turnIndex, model, ttfb_ms: ttfbMs, }), - onAssistantResponseRendered: ({ model, latencyMs }) => { + onAssistantResponseRendered: ({ conversationId, roundId, turnIndex, model, latencyMs }) => { trackAssistantResponseRendered({ + conversation_id: conversationId, + round_id: roundId, + turn_index: turnIndex, model, latency_ms: latencyMs, }) - trackAssistantResponseCompleted({ - conversation_id: activeSessionId.value || 'unknown', - provider_type: providerMode(activeProvider.value), - provider_name: activeProvider.value || 'unknown', - model: model || 'unknown', - latency_ms: latencyMs, - }) }, - onMessageRound: ({ durationMs, hasVoice, model }) => trackMessageRound({ + onMessageRound: ({ conversationId, roundId, turnIndex, durationMs, hasVoice, model }) => trackMessageRound({ + conversation_id: conversationId, + round_id: roundId, + turn_index: turnIndex, duration_ms: durationMs, has_voice: hasVoice, model, }), - onChatActivationStarted: ({ model, provider, source }) => { + onMessageRoundFailed: ({ conversationId, roundId, turnIndex, model, provider, errorCode, failureStage, source }) => trackMessageRoundFailed({ + conversation_id: conversationId, + round_id: roundId, + turn_index: turnIndex, + provider_id: provider || 'unknown', + model_id: model || 'unknown', + source, + error_code: errorCode, + failure_stage: failureStage, + }), + onChatActivationStarted: ({ conversationId, roundId, turnIndex, model, provider, source }) => { const mode = providerMode(provider) const providerId = provider || 'unknown' const modelId = model || 'unknown' trackChatActivationStarted({ + conversation_id: conversationId, provider_mode: mode, provider_id: providerId, model_id: modelId, + round_id: roundId, source, + turn_index: turnIndex, }) }, - onChatActivationSucceeded: ({ model, provider, durationMs, source }) => trackChatActivationSucceeded({ + onChatActivationSucceeded: ({ conversationId, roundId, turnIndex, model, provider, durationMs, source }) => trackChatActivationSucceeded({ + conversation_id: conversationId, provider_mode: providerMode(provider), provider_id: provider || 'unknown', model_id: model || 'unknown', + round_id: roundId, time_to_first_message_ms: durationMs, source, + turn_index: turnIndex, }), - onChatActivationFailed: ({ model, provider, errorCode, failureStage, source }) => { + onChatActivationFailed: ({ conversationId, roundId, turnIndex, model, provider, errorCode, failureStage, source }) => { trackChatActivationFailed({ + conversation_id: conversationId, provider_mode: providerMode(provider), provider_id: provider || 'unknown', model_id: model || 'unknown', + round_id: roundId, error_code: errorCode, failure_stage: failureStage, source, - }) - trackChatFailed({ - conversation_id: activeSessionId.value || 'unknown', - provider_type: providerMode(provider), - provider_name: provider || 'unknown', - model: model || 'unknown', - failure_stage: failureStage, - error_code: errorCode, + turn_index: turnIndex, }) }, onLifecycle: record => contextObservability.recordLifecycle(record), onPromptProjection: payload => contextObservability.capturePromptProjection(payload), - onUserMessageAppended: ({ sessionId, message, messageText, source, model, provider, turnIndex }) => { + onUserMessageAppended: ({ sessionId, message, messageText, source, model, provider, roundId, turnIndex }) => { trackMessageSent({ conversation_id: sessionId, provider_type: providerMode(activeProvider.value), provider_name: activeProvider.value || 'unknown', model: activeModel.value || 'unknown', message_id: message.id, + round_id: roundId, + turn_index: turnIndex, message_index: chatSession.getSessionMessages(sessionId).length, message_length: messageText.length, has_attachment: false, mode: lastSendSource, }) - trackFeatureUsed({ - feature_name: 'chat', - business_domain: 'conversation', - entry: 'chat', - success: true, - }) if (turnIndex === 2) { trackSecondTurnStarted({ + conversation_id: sessionId, provider_mode: providerMode(provider), provider_id: provider || 'unknown', model_id: model || 'unknown', + round_id: roundId, source, turn_index: turnIndex, }) diff --git a/packages/stage-ui/src/stores/providers.ts b/packages/stage-ui/src/stores/providers.ts index cb9d34a41..c7e31846a 100644 --- a/packages/stage-ui/src/stores/providers.ts +++ b/packages/stage-ui/src/stores/providers.ts @@ -132,7 +132,7 @@ function trackModelListLoaded(properties: { capturePosthogEvent('model_list_loaded', { ...properties, - surface: analyticsSurface(), + app_surface: analyticsSurface(), }) } @@ -150,7 +150,7 @@ function trackModelListFailed(properties: { capturePosthogEvent('model_list_failed', { ...properties, - surface: analyticsSurface(), + app_surface: analyticsSurface(), }) } diff --git a/posthog.config.ts b/posthog.config.ts index c65392814..7111999c3 100644 --- a/posthog.config.ts +++ b/posthog.config.ts @@ -13,7 +13,7 @@ function isEnvFlagEnabled(value: string | undefined): boolean { export const POSTHOG_ENABLED = isEnvFlagEnabled(import.meta.env.VITE_ENABLE_POSTHOG) // Single PostHog project for every AIRI surface (web / desktop / mobile). -// Platforms are told apart by the `surface` super property set at init, not +// Platforms are told apart by the `app_surface` super property set at init, not // by routing to separate per-platform projects. export const POSTHOG_PROJECT_KEY = import.meta.env.VITE_POSTHOG_PROJECT_KEY