feat(llm-obs): LLM observability dashboard (#27415)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Peter Kirkham <[email protected]>

frontend/src/layout/navigation-3000/navigationLogic.tsx

@@ -1,4 +1,5 @@
 import {
+    IconAI,
     IconCursorClick,
     IconDashboard,
     IconDatabase,
@@ -496,6 +497,15 @@ export const navigation3000Logic = kea<navigation3000LogicType>([
                                   }
                                 : undefined,
                         },
+                        featureFlags[FEATURE_FLAGS.LLM_OBSERVABILITY]
+                            ? {
+                                  identifier: Scene.LLMObservability,
+                                  label: 'LLM observability',
+                                  icon: <IconAI />,
+                                  to: urls.llmObservability(),
+                                  tag: 'beta' as const,
+                              }
+                            : null,
                         {
                             identifier: Scene.Replay,
                             label: 'Session replay',

frontend/src/lib/constants.tsx

@@ -233,11 +233,12 @@ export const FEATURE_FLAGS = {
     WEB_ANALYTICS_CONVERSION_GOAL_FILTERS: 'web-analytics-conversion-goal-filters', // owner: @rafaeelaudibert #team-web-analytics
     CDP_ACTIVITY_LOG_NOTIFICATIONS: 'cdp-activity-log-notifications', // owner: #team-cdp
     COOKIELESS_SERVER_HASH_MODE_SETTING: 'cookieless-server-hash-mode-setting', // owner: @robbie-c #team-web-analytics
-    INSIGHT_COLORS: 'insight-colors', // owner @thmsobrmlr #team-product-analytics
-    WEB_ANALYTICS_FOR_MOBILE: 'web-analytics-for-mobile', // owner @robbie-c #team-web-analytics
-    REPLAY_FLAGS_FILTERS: 'replay-flags-filters', // owner @pauldambra #team-replay
-    REPLAY_LANDING_PAGE: 'replay-landing-page', // owner #team-replay
-    CORE_WEB_VITALS: 'core-web-vitals', // owner @rafaeelaudibert #team-web-analytics
+    INSIGHT_COLORS: 'insight-colors', // owner: @thmsobrmlr #team-product-analytics
+    WEB_ANALYTICS_FOR_MOBILE: 'web-analytics-for-mobile', // owner: @robbie-c #team-web-analytics
+    REPLAY_FLAGS_FILTERS: 'replay-flags-filters', // owner: @pauldambra #team-replay
+    REPLAY_LANDING_PAGE: 'replay-landing-page', // owner :#team-replay
+    CORE_WEB_VITALS: 'core-web-vitals', // owner: @rafaeelaudibert #team-web-analytics
+    LLM_OBSERVABILITY: 'llm-observability', // owner: #team-ai-product-manager
 } as const
 export type FeatureFlagKey = (typeof FEATURE_FLAGS)[keyof typeof FEATURE_FLAGS]
 

frontend/src/scenes/appScenes.ts

@@ -87,4 +87,5 @@ export const appScenes: Record<Scene, () => any> = {
         import('scenes/web-analytics/SessionAttributionExplorer/SessionAttributionExplorerScene'),
     [Scene.MessagingProviders]: () => import('./messaging/Providers'),
     [Scene.MessagingBroadcasts]: () => import('./messaging/Broadcasts'),
+    [Scene.LLMObservability]: () => import('./llm-observability/LLMObservabilityScene'),
 }

frontend/src/scenes/llm-observability/LLMObservabilityScene.tsx

@@ -0,0 +1,68 @@
+import clsx from 'clsx'
+import { BindLogic, useActions, useValues } from 'kea'
+import { DateFilter } from 'lib/components/DateFilter/DateFilter'
+import { SceneExport } from 'scenes/sceneTypes'
+
+import { navigationLogic } from '~/layout/navigation/navigationLogic'
+import { dataNodeCollectionLogic } from '~/queries/nodes/DataNode/dataNodeCollectionLogic'
+import { Query } from '~/queries/Query/Query'
+import { NodeKind } from '~/queries/schema'
+
+import { LLM_OBSERVABILITY_DATA_COLLECTION_NODE_ID, llmObservabilityLogic, QueryTile } from './llmObservabilityLogic'
+
+export const scene: SceneExport = {
+    component: LLMObservabilityScene,
+}
+
+const Filters = (): JSX.Element => {
+    const {
+        dateFilter: { dateTo, dateFrom },
+    } = useValues(llmObservabilityLogic)
+    const { setDates } = useActions(llmObservabilityLogic)
+    const { mobileLayout } = useValues(navigationLogic)
+
+    return (
+        <div
+            className={clsx(
+                'sticky z-20 bg-bg-3000',
+                mobileLayout ? 'top-[var(--breadcrumbs-height-full)]' : 'top-[var(--breadcrumbs-height-compact)]'
+            )}
+        >
+            <div className="border-b py-2 flex flex-row flex-wrap gap-2 md:[&>*]:grow-0 [&>*]:grow">
+                <DateFilter dateFrom={dateFrom} dateTo={dateTo} onChange={setDates} />
+            </div>
+        </div>
+    )
+}
+
+const Tiles = (): JSX.Element => {
+    const { tiles } = useValues(llmObservabilityLogic)
+
+    return (
+        <div className="mt-2 grid grid-cols-1 md:grid-cols-2 xxl:grid-cols-3 gap-4">
+            {tiles.map((tile, i) => (
+                <QueryTileItem key={i} tile={tile} />
+            ))}
+        </div>
+    )
+}
+
+const QueryTileItem = ({ tile }: { tile: QueryTile }): JSX.Element => {
+    const { query, title, layout } = tile
+
+    return (
+        <div className={clsx('col-span-1 row-span-1 flex flex-col', layout?.className)}>
+            {title && <h2 className="mb-1 flex flex-row ml-1">{title}</h2>}
+            <Query query={{ kind: NodeKind.InsightVizNode, source: query }} readOnly />
+        </div>
+    )
+}
+
+export function LLMObservabilityScene(): JSX.Element {
+    return (
+        <BindLogic logic={dataNodeCollectionLogic} props={{ key: LLM_OBSERVABILITY_DATA_COLLECTION_NODE_ID }}>
+            <Filters />
+            <Tiles />
+        </BindLogic>
+    )
+}

frontend/src/scenes/llm-observability/llmObservabilityLogic.tsx

@@ -0,0 +1,93 @@
+import { actions, kea, path, reducers, selectors } from 'kea'
+
+import { NodeKind, TrendsQuery } from '~/queries/schema'
+import { PropertyMathType } from '~/types'
+
+import type { llmObservabilityLogicType } from './llmObservabilityLogicType'
+
+export const LLM_OBSERVABILITY_DATA_COLLECTION_NODE_ID = 'llm-observability-data'
+
+const INITIAL_DATE_FROM = '-30d' as string | null
+const INITIAL_DATE_TO = null as string | null
+
+export interface QueryTile {
+    title: string
+    query: TrendsQuery
+    layout?: {
+        className?: string
+    }
+}
+
+export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
+    path(['scenes', 'llm-observability', 'llmObservabilityLogic']),
+
+    actions({
+        setDates: (dateFrom: string | null, dateTo: string | null) => ({ dateFrom, dateTo }),
+    }),
+
+    reducers({
+        dateFilter: [
+            {
+                dateFrom: INITIAL_DATE_FROM,
+                dateTo: INITIAL_DATE_TO,
+            },
+            {
+                setDates: (_, { dateFrom, dateTo }) => ({ dateFrom, dateTo }),
+            },
+        ],
+    }),
+
+    selectors({
+        tiles: [
+            (s) => [s.dateFilter],
+            (dateFilter): QueryTile[] => [
+                {
+                    title: 'LLM generations',
+                    query: {
+                        kind: NodeKind.TrendsQuery,
+                        series: [
+                            {
+                                event: '$ai_generation',
+                                kind: NodeKind.EventsNode,
+                            },
+                        ],
+                        dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                    },
+                },
+                {
+                    title: 'LLM costs (USD)',
+                    query: {
+                        kind: NodeKind.TrendsQuery,
+                        series: [
+                            {
+                                event: '$ai_generation',
+                                math: PropertyMathType.Sum,
+                                kind: NodeKind.EventsNode,
+                                math_property: '$ai_total_cost_usd',
+                            },
+                        ],
+                        dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                    },
+                },
+                {
+                    title: 'Average latency (ms)',
+                    query: {
+                        kind: NodeKind.TrendsQuery,
+                        series: [
+                            {
+                                event: '$ai_generation',
+                                math: PropertyMathType.Average,
+                                kind: NodeKind.EventsNode,
+                                math_property: '$ai_latency',
+                            },
+                        ],
+                        breakdownFilter: {
+                            breakdown: '$ai_model',
+                        },
+                        dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                    },
+                },
+            ],
+        ],
+    }),
+])

frontend/src/scenes/sceneTypes.ts

@@ -29,6 +29,7 @@ export enum Scene {
     ReplayPlaylist = 'ReplayPlaylist',
     ReplayFilePlayback = 'ReplayFilePlayback',
     CustomCss = 'CustomCss',
+    LLMObservability = 'LLMObservability',
     PersonsManagement = 'PersonsManagement',
     Person = 'Person',
     PipelineNodeNew = 'PipelineNodeNew',

frontend/src/scenes/scenes.ts

@@ -280,6 +280,11 @@ export const sceneConfigurations: Record<Scene, SceneConfig> = {
         name: 'Data warehouse table',
         defaultDocsPath: '/docs/data-warehouse',
     },
+    [Scene.LLMObservability]: {
+        projectBased: true,
+        name: 'LLM observability',
+        layout: 'app-container',
+    },
     [Scene.EarlyAccessFeatures]: {
         projectBased: true,
         defaultDocsPath: '/docs/feature-flags/early-access-feature-management',
@@ -653,4 +658,5 @@ export const routes: Record<string, Scene> = {
     [urls.messagingBroadcasts()]: Scene.MessagingBroadcasts,
     [urls.messagingBroadcast(':id')]: Scene.MessagingBroadcasts,
     [urls.messagingBroadcastNew()]: Scene.MessagingBroadcasts,
+    [urls.llmObservability()]: Scene.LLMObservability,
 }

frontend/src/scenes/urls.ts

@@ -259,4 +259,5 @@ export const urls = {
     insightAlert: (insightShortId: InsightShortId, alertId: AlertType['id']): string =>
         `/insights/${insightShortId}/alerts?alert_id=${alertId}`,
     sessionAttributionExplorer: (): string => '/web/session-attribution-explorer',
+    llmObservability: (): string => '/llm-observability',
 }

posthog/demo/matrix/models.py

@@ -17,9 +17,23 @@
 from urllib.parse import urlparse, parse_qs
 from uuid import UUID
 
+import tiktoken
+
+from posthog.models.utils import uuid7
+
 if TYPE_CHECKING:
     from posthog.demo.matrix.matrix import Cluster, Matrix
 
+llm_encoding = tiktoken.encoding_for_model("gpt-4o")
+
+# Refer to https://github.com/PostHog/posthog-ai-costs-app/tree/main/src/ai-cost-data for missing models
+LLM_COSTS_BY_MODEL = {
+    "gpt-4o": {
+        "prompt_token": 0.000005,
+        "completion_token": 0.000015,
+    },
+}
+
 SP = TypeVar("SP", bound="SimPerson")
 EffectCallback = Callable[[SP], Any]
 EffectCondition = Callable[[SP], bool]
@@ -257,6 +271,48 @@ def capture_pageview(
         self.current_url = current_url
         self.capture(EVENT_PAGEVIEW, properties)
 
+    def capture_ai_generation(
+        self,
+        input: list[dict],
+        output_content: str,
+        latency: float,
+        request_url: str = "https://api.openai.com/v1/chat/completions",
+        provider: str = "openai",
+        model: str = "gpt-4o",
+        trace_id: Optional[str] = None,
+        http_status: int = 200,
+    ):
+        """Capture an AI generation event."""
+        input_tokens = sum(len(llm_encoding.encode(message["content"])) for message in input)
+        output_tokens = len(llm_encoding.encode(output_content))
+        input_cost_usd = input_tokens * LLM_COSTS_BY_MODEL[model]["prompt_token"]
+        output_cost_usd = output_tokens * LLM_COSTS_BY_MODEL[model]["completion_token"]
+        self.capture(
+            "$ai_generation",
+            {
+                "$ai_request_url": request_url,
+                "$ai_provider": provider,
+                "$ai_model": model,
+                "$ai_http_status": http_status,
+                "$ai_input_tokens": input_tokens,
+                "$ai_output_tokens": output_tokens,
+                "$ai_input_cost_usd": input_cost_usd,
+                "$ai_output_cost_usd": output_cost_usd,
+                "$ai_total_cost_usd": input_cost_usd + output_cost_usd,
+                "$ai_input": input,
+                "$ai_output": {
+                    "choices": [
+                        {
+                            "content": output_content,
+                            "role": "assistant",
+                        }
+                    ]
+                },
+                "$ai_latency": latency,
+                "$ai_trace_id": trace_id or str(uuid7()),
+            },
+        )
+
     def identify(self, distinct_id: Optional[str], set_properties: Optional[Properties] = None):
         """Identify person in active client. Similar to JS `posthog.identify()`.
 

posthog/demo/products/__init__.py

@@ -1,3 +1,4 @@
 from .hedgebox import HedgeboxMatrix
+from .spikegpt import SpikeGPTMatrix
 
-__all__ = ["HedgeboxMatrix"]
+__all__ = ["HedgeboxMatrix", "SpikeGPTMatrix"]

posthog/demo/products/spikegpt/__init__.py

@@ -0,0 +1,5 @@
+from .matrix import SpikeGPTMatrix
+
+# This is a simulation of a ChatGPT clone called SpikeGPT
+
+__all__ = ["SpikeGPTMatrix"]