add experiments pattern (#54228)

Co-authored-by: Kevin Heis <[email protected]>

src/events/components/events.ts

@@ -3,6 +3,7 @@ import Cookies from 'src/frame/components/lib/cookies'
 import { parseUserAgent } from './user-agent'
 import { Router } from 'next/router'
 import { isLoggedIn } from 'src/frame/components/hooks/useHasAccount'
+import { getExperimentVariationForContext } from './experiments/experiment'
 import { EventType, EventPropsByType } from '../types'
 
 const COOKIE_NAME = '_docs-events'
@@ -110,6 +111,8 @@ export function sendEvent<T extends EventType>({
       color_mode_preference: getColorModePreference(),
       os_preference: Cookies.get('osPreferred'),
       code_display_preference: Cookies.get('annotate-mode'),
+
+      experiment_variation: getExperimentVariationForContext(getMetaContent('path-language')),
     },
 
     ...props,

src/events/components/experiments/README.md

@@ -0,0 +1,172 @@
+# Experiments
+
+There are times when we want make a change, but aren't sure if it will provide a better user experience.
+
+In these scenarios we can run experiments.
+
+Experiments are A/B tests where A is some version of our site and B is another. When the user requests our site they are randomly served either site A or B.
+
+After the experiment is live, we gather data via [events](../../README.md) that help determine which version of the site we want to stick with.
+
+## TOC
+
+- [Experiments as feature flags](#experiments-as-feature-flags)
+- [Experiment variations](#experiment-variations)
+- [Adding an experiment](#adding-an-experiment)
+- [Implementing an experiment](#implementing-an-experiment)
+- [Toggling an experiment on or off](#toggling-an-experiment-on-or-off)
+- [Tracking results on an experiment](#tracking-results-on-an-experiment)
+  - [Via regular events](#via-regular-events)
+  - [Via the `experiment` event](#via-the-experiment-event)
+
+
+## Experiments as feature flags
+
+An additional benefit of this pattern is that it lets you turn on/off a feature in the UI and toggle it from the developer console. This is useful if you want to ship UI changes in parts, or test something in production before turning it on.
+
+## Experiment variations
+
+To clarify terminology, if a user is shown site A which is the original site _without_ the experiment they will have an `experiment_variation` value of `"control"` to indicate that they are in the control group.
+
+If the user is shown the experiment (site B), they will have an `experiment_variation` value of `"treatment"`
+
+## Adding an experiment
+
+1. Create a `key` for you experiment, e.g. `ai_search_experiment`
+2. Determine how many users will see the experiment. The default is 50% and makes sense for _most_ use cases.
+3. Add your experiment to [experiments.ts](./experiments.ts)
+
+Example,
+
+```typescript
+// Add new experiment key to this list
+export type ExperimentNames = 'example_experiment' | 'ai_search_experiment'
+
+export const EXPERIMENTS = {
+  example_experiment: { ... }
+  ai_search_experiment: {
+    key: 'ai_search_experiment',
+    isActive: true, // Set to false when the experiment is over
+    percentOfUsersToGetExperiment: 10, // Only 10% of users will get the experiment
+    limitToLanguages: ['en'], // Only users with the `en` language will be included in the experiment
+    includeVariationInContext: true, // See note below
+  },
+}
+```
+
+When `includeVariationInContext` is true **all** analytics events sent will include `experiment_variation` in their context. `experiment_variation` will be `"treatment"` or `"control"` depending on which the user was randomly assigned.
+
+> [!IMPORTANT]
+> Since the `experiment_variation` is a single key in the context, **only one experiment** can include their variations in the context e.g. only one value of `includeVariationInContext` can be `true`.
+
+## Implementing an experiment
+
+For example, let's say you are conducting an experiment that changes the search bar.
+
+In the code that displays the search bar, you can use the `shouldShowExperiment` function to determine which version of the code to show the user.
+
+Example:
+
+```typescript
+import { useRouter } from 'next/router'
+import { useShouldShowExperiment } from '@/events/components/experiments/useShouldShowExperiment'
+import { EXPERIMENTS } from '@/events/components/experiments/experiments'
+import { ClassicSearchBar } from "@/search/components/ClassicSearchBar.tsx"
+import { NewSearchBar } from "@/search/components/NewSearchBar.tsx"
+
+export function SearchBar() {
+  const router = useRouter()
+  // Users who were randomly placed in the `treatment` group will be shown the experiment
+  const { shouldShow: shouldShowNewSearch } = useShouldShowExperiment(
+    EXPERIMENTS.ai_search_experiment,
+    router.locale
+  )
+
+  if (shouldShowNewSearch) {
+    return (
+      <NewSearchBar />
+    )
+  }
+  return <ClassicSearchBar />
+}
+```
+
+> [!NOTE]
+> If a user is placed in the `"treatment"` group e.g. they are shown the experiment and will continue to see the treatment across all sessions from the same browser. This is because we use a hash of user's session ID cookie to deterministically set the control group. The session cookie lasts for 365 days, otherwise they might see something different on each reload.
+
+## Toggling an experiment on or off
+
+In development every session is placed into the `"treatment"` control group so that the experiment can be developed on.
+
+However, you can change which experiment to show by calling the following function in the `Console` tab in Chrome dev tools,
+
+```javascript
+window.overrideControlGroup("<experiment_key>", "treatment" | "control");
+
+// Example to see original search experience
+window.overrideControlGroup("ai_search_experiment", "control");
+```
+
+For events, you can verify that your `experiment_variation` values are being included in the event context from the `Network` tab in Chrome dev tools.
+
+## Tracking results on an experiment
+
+### Via regular events
+
+If your experiment object in [experiments.ts](./experiments.ts) included the `includeVariationInContext: true` key (and is the ONLY object to include that key) then the `experiment_variation` of your experiment will be sent in the context of an event.
+
+This means that you can send other events, like
+
+```typescript
+sendEvent({
+  type: EventType.search,
+  search_query: "How do I open pdf?",
+  search_context: "general-search",
+});
+```
+
+And the `context` on that event will include the `experiment_variation` key and value of your experiment,
+
+e.g.
+
+```javascript
+{
+  search_query: "How do I open pdf?",
+  search_context: "general-search",
+  context: {
+    ...
+    experiment_variation: "treatment" // Could also be "control" depending on the random outcome
+  }
+}
+```
+
+### Via the `experiment` event
+
+If your experiment is specific, meaning it can be tracked with a boolean event, e.g.
+
+```javascript
+{
+    experiment_name: <string>, // e.g. `new_button_experiment` for did user click new button?
+    experiment_variation: 'treatment' | 'control',
+    experiment_success: <boolean>, // e.g. true the user is using the new button!
+}
+```
+
+Then you should omit the `includeVariationInContext` key from your experiment object and use the `sendExperimentSuccess` function to track events.
+
+Example:
+
+```typescript
+import { sendExperimentSuccess } from '@/events/components/experiments/experiment-event'
+import { EXPERIMENTS } from '@/events/components/experiments/experiments'
+
+export function MyNewComponent() {
+  return (
+    <button onClick={() => {
+      console.log("The user did the thing!")
+      sendExperimentSuccess(EXPERIMENTS.new_button_experiment)
+    }}>
+  )
+}
+
+```

src/events/components/experiments/experiment-event.ts

@@ -0,0 +1,13 @@
+import { EventType } from '@/events/types'
+import { sendEvent } from '../events'
+import { getExperimentControlGroupFromSession } from './experiment'
+import { ExperimentNames } from './experiments'
+
+export function sendExperimentSuccess(experimentKey: ExperimentNames, success = true) {
+  return sendEvent({
+    type: EventType.experiment,
+    experiment_name: experimentKey,
+    experiment_variation: getExperimentControlGroupFromSession(experimentKey).toLowerCase(),
+    experiment_success: success,
+  })
+}

src/events/components/experiments/experiment.ts

@@ -0,0 +1,153 @@
+import murmur from 'imurmurhash'
+import {
+  CONTROL_VARIATION,
+  ExperimentNames,
+  TREATMENT_VARIATION,
+  getActiveExperiments,
+} from './experiments'
+import { getUserEventsId } from '../events'
+import Cookies from 'src/frame/components/lib/cookies'
+
+let experimentsInitialized = false
+
+export function shouldShowExperiment(
+  experimentKey: ExperimentNames | { key: ExperimentNames },
+  locale: string,
+) {
+  // Accept either EXPERIMENTS.<experiment_key> or EXPERIMENTS.<experiment_key>.key
+  if (typeof experimentKey === 'object') {
+    experimentKey = experimentKey.key
+  }
+
+  // Determine if user is in treatment group. If they are, show the experiment
+  const experiments = getActiveExperiments('all')
+  for (const experiment of experiments) {
+    if (experiment.key === experimentKey) {
+      // If the user has staffonly cookie, and staff override is true, show the experiment
+      if (experiment.alwaysShowForStaff) {
+        const staffCookie = Cookies.get('staffonly')
+        if (staffCookie && staffCookie.startsWith('yes')) {
+          console.log(`Staff cookie is set, showing '${experiment.key}' experiment`)
+          return true
+        }
+      }
+
+      // If there is an override for the current session, use that
+      if (controlGroupOverride[experiment.key]) {
+        const controlGroup = getExperimentControlGroupFromSession(
+          experimentKey,
+          experiment.percentOfUsersToGetExperiment,
+        )
+        return controlGroup === TREATMENT_VARIATION
+        // Otherwise use the regular logic to determine if the user is in the treatment group
+      } else if (
+        experiment.limitToLanguages?.length &&
+        experiment.limitToLanguages.includes(locale)
+      ) {
+        return (
+          getExperimentControlGroupFromSession(
+            experimentKey,
+            experiment.percentOfUsersToGetExperiment,
+          ) === TREATMENT_VARIATION
+        )
+      }
+    }
+  }
+  return false
+}
+
+// Allow developers to override their experiment group for the current session
+export const controlGroupOverride = {} as { [key in ExperimentNames]: 'treatment' | 'control' }
+if (typeof window !== 'undefined') {
+  // @ts-expect-error
+  window.overrideControlGroup = (
+    experimentKey: ExperimentNames,
+    controlGroup: 'treatment' | 'control',
+  ): string => {
+    const activeExperiments = getActiveExperiments('all')
+    // Make sure key is valid
+    if (activeExperiments.some((experiment) => experiment.key === experimentKey)) {
+      controlGroupOverride[experimentKey] = controlGroup
+      const event = new Event('controlGroupOverrideChanged')
+      window.dispatchEvent(event)
+      return `Updated ${experimentKey}. Session is now in the "${controlGroup}" group for this session.`
+    } else {
+      throw new Error(
+        `Invalid experiment key: ${experimentKey}. Must be one of: ${activeExperiments.map((experiment) => experiment.key).join(', ')}`,
+      )
+    }
+  }
+}
+
+// Determine if the user is in the treatment or control group for a given experiment
+export function getExperimentControlGroupFromSession(
+  experimentKey: ExperimentNames,
+  percentToGetExperiment = 50,
+) {
+  if (controlGroupOverride[experimentKey]) {
+    return controlGroupOverride[experimentKey]
+  } else if (process.env.NODE_ENV === 'development') {
+    return TREATMENT_VARIATION
+  } else if (process.env.NODE_ENV === 'test') {
+    return CONTROL_VARIATION
+  }
+  // We hash the user's events ID to ensure that the user is always in the same group for a given experiment
+  // This works because the hash is a deterministic and the user's ID is stored in a cookie for 365 days
+  const id = getUserEventsId()
+  const hash = murmur(experimentKey).hash(id).result()
+  const modHash = hash % 100
+  return modHash < percentToGetExperiment ? TREATMENT_VARIATION : CONTROL_VARIATION
+}
+
+export function getExperimentVariationForContext(locale: string) {
+  const experiments = getActiveExperiments(locale)
+  for (const experiment of experiments) {
+    if (experiment.includeVariationInContext) {
+      return getExperimentControlGroupFromSession(
+        experiment.key,
+        experiment.percentOfUsersToGetExperiment,
+      )
+    }
+  }
+
+  // When no experiment has `includeVariationInContext: true`
+  return null
+}
+
+export function initializeExperiments(locale: string) {
+  if (experimentsInitialized) return
+  experimentsInitialized = true
+
+  const experiments = getActiveExperiments(locale)
+
+  if (experiments.length && process.env.NODE_ENV === 'development') {
+    console.log(
+      `In development, all users are placed in the "${TREATMENT_VARIATION}" group for experiments`,
+    )
+  } else if (experiments.length && process.env.NODE_ENV === 'test') {
+    console.log(`In test, all users are placed in the "${CONTROL_VARIATION}" group for experiments`)
+  }
+
+  let numberOfExperimentsUsingContext = 0
+  for (const experiment of experiments) {
+    if (experiment.includeVariationInContext) {
+      // Validate the experiments object
+      numberOfExperimentsUsingContext++
+      if (numberOfExperimentsUsingContext > 1) {
+        throw new Error(
+          'Only one experiment can include its variation in the context at a time. Please update the experiments configuration.',
+        )
+      }
+    }
+
+    const controlGroup = getExperimentControlGroupFromSession(
+      experiment.key,
+      experiment.percentOfUsersToGetExperiment,
+    )
+
+    // Even in preview & prod it is useful to see if a given experiment is "on" or "off"
+    console.log(
+      `Experiment ${experiment.key} is in the "${controlGroup === TREATMENT_VARIATION ? TREATMENT_VARIATION : CONTROL_VARIATION}" group for this browser.\nCall function window.overrideControlGroup('${experiment.key}', 'treatment' | 'control') to change your group for this session.`,
+    )
+  }
+}