GOTH-580 A/B Test Infrastructure and first A/B test (#275)

When a user goes to the page they should get randomly assigned to either variant 0 or 1.

Sets a cookie "_experiment_no-duplicates-in-river" with a value of either 0 or 1. This value should be the same upon returning to the page (the cookie lasts for 30 days and this timer is refreshed on new visits).

The experiment name ("no-duplicates-in-river") and variant ("0" or "1") are attached to all GA events as 'experimentName' and 'experimentVariant' respectively.

If the variant is 0, the home page remains unchanged.
If the variant is 1, stories in the latest news section at the bottom of the homepage aree deduplicated from the homepage topper. In other words, anything that appears in the "Latest" section in the homepage topper shouldn't also appear in the "Latest" river.

Because the latest section in the homepage topper is also deduplicated from the curated featured articles, there's also of a special case. If either or both of the curated articles in the homepage topper are among the top 6 latest articles, they'll also be deduplicated from the river at the bottom of the page.

composables/types/Experiment.ts

@@ -0,0 +1,5 @@
+export default interface Experiment {
+    name: string;
+    maxAgeSeconds?: number;
+    variants: {weight:number}[];
+}

cypress/support/commands.ts

@@ -43,6 +43,7 @@ declare module 'Cypress' {
 }
 
 Cypress.Commands.add('loadGlobalFixtures',  () => {
+    cy.setCookie('_experiment_no-duplicates-in-river', '0', {path: '/'})
     cy.intercept('/api/v2/system_messages/*', {fixture: 'aviary/system_messages_empty.json'}).as('systemMessages')
     cy.intercept('/api/v2/sitewide_components/*', {fixture: 'aviary/sitewide_components.json'}).as('sitewideComponents')
     cy.intercept('/api/v2/navigation/*', {fixture: 'aviary/navigation.json'}).as('navigation')

experiments/index.ts

@@ -0,0 +1,5 @@
+import noDuplicatesInRiver from './no-duplicates-in-river/index'
+
+export default [
+    noDuplicatesInRiver
+]

experiments/no-duplicates-in-river/index.ts

@@ -0,0 +1,11 @@
+export default {
+    // A helper exp-{name}-{var} class will be added to the root element
+    name: 'no-duplicates-in-river',
+
+    maxAgeSeconds: 60 * 60 * 24 * 30, //30 days
+
+    variants: [
+      { weight: 1 }, // 0: Original
+      { weight: 1 }  // 1: Remove duplicate stories from the latest news river
+    ]
+  }

layouts/default.vue

@@ -9,7 +9,7 @@ const marketingBannerData = useMarketingBannerData()
 
 const config = useRuntimeConfig()
 const route = useRoute()
-const { $htlbid, $analytics } = useNuxtApp()
+const { $htlbid, $analytics, $features } = useNuxtApp()
 
 const navigationState = useNavigation()
 const navigationPromise = findNavigation().then(({ data }) => {

pages/index.vue

@@ -4,13 +4,22 @@ import { ArticlePage } from '~~/composables/types/Page'
 import { computed, ref, nextTick } from 'vue'
 import { onBeforeRouteUpdate } from 'vue-router'
 
+const { $features } = useNuxtApp()
+const possibleDuplicateCount = ref(6) // from the topper
+const actualDuplicateCount = ref(4)
 const riverStoryCount = ref(6)
 const riverAdOffset = ref(2)
 const riverAdRepeatRate = ref(6)
 const riverContainer = ref('#latest')
 
+let findArticleLimit
+if ($features.enabled['experiment-deduplicate-river']) {
+  findArticleLimit = riverStoryCount.value + possibleDuplicateCount.value
+} else {
+  findArticleLimit = riverStoryCount.value
+}
 const articlesPromise = findArticlePages({
-  limit: riverStoryCount.value,
+  limit: findArticleLimit,
   sponsored_content: false,
 }).then(({ data }) => normalizeFindArticlePagesResponse(data))
 
@@ -35,6 +44,18 @@ const latestArticles = ref([...articles])
 // the home page featured article should display only the first and second story in the home page content collection
 const featuredArticles = homePageCollections?.[0].data.map(normalizeArticlePage)
 
+actualDuplicateCount.value = 4
+const firstFour = articles.slice(0,6).map(article => article.uuid)
+if (firstFour.includes(featuredArticles[0].uuid)) { actualDuplicateCount.value += 1 }
+if (firstFour.includes(featuredArticles[1].uuid)) { actualDuplicateCount.value += 1 }
+
+const filteredLatestArticles = computed(() => {
+  return latestArticles.value.slice(actualDuplicateCount.value)
+})
+
+const riverArticles = $features.enabled['experiment-deduplicate-river'] ?
+filteredLatestArticles : latestArticles
+
 usePreloadResponsiveImage(
   useImageUrl(featuredArticles?.[0]?.listingImage, {
     width: 700,
@@ -49,19 +70,26 @@ usePreloadResponsiveImage(
 )
 
 const riverSegments = computed(() => {
-  let riverCopy = latestArticles.value.slice()
+  let riverCopy = riverArticles.value.slice()
   const segments = [] as ArticlePage[][]
-  while (riverCopy.length) {
+  while (riverCopy.length >= 6) {
     segments.push(riverCopy.splice(0, riverStoryCount.value))
   }
   return segments
 })
 
 const loadMoreArticles = async () => {
+  let loadMoreOffset
+  if ($features.enabled['experiment-deduplicate-river']) {
+    loadMoreOffset = latestArticles.value.length + actualDuplicateCount.value
+  } else {
+    loadMoreOffset = latestArticles.value.length
+  }
+
   const newArticles = await useLoadMoreArticles({
     sponsored_content: false,
     limit: riverStoryCount.value,
-    offset: latestArticles.value.length,
+    offset: loadMoreOffset,
   })
   latestArticles.value.push(...newArticles)
   await nextTick()

plugins/1.experiments.ts

@@ -0,0 +1,75 @@
+import currentExperiments from "~/experiments";
+import Experiment from "~/composables/types/Experiment";
+import { get } from "cypress/types/lodash";
+
+export default defineNuxtPlugin(() => {
+    let activeVariant:number
+    const defaultMaxAge = 60 * 60 * 24 * 30 // 30 days
+
+    const chooseWeightedRandom = (weights:number[]):number => {
+      const sumOfWeights = weights.reduce(
+        (sum, weight) => sum + weight,
+      0)
+      const choice = Math.random()
+      let threshold = 0
+      for (let i = 0; i < weights.length; i++) {
+        threshold += weights[i]/sumOfWeights
+        if (choice <= threshold) {
+          return i
+        }
+      }
+    }
+
+    const assignVariants = (experiments:Experiment[]):void => {
+      experiments.forEach(experiment => {
+        if (experiment === getCurrentExperiment() && typeof activeVariant === 'undefined') {
+          activeVariant = readVariant(experiment) ?? chooseVariant(experiment)
+        }
+      })
+    }
+
+    const chooseVariant = (experiment:Experiment):number => {
+      return chooseWeightedRandom(experiment.variants.map(variant => variant.weight))
+    }
+
+    const readVariant = (experiment:Experiment):number => {
+      const cookie = useCookie(`_experiment_${experiment.name}`, { path: '/' })
+      if (typeof cookie.value !== 'undefined') {
+        return Number(cookie.value)
+      }
+    }
+
+    const saveVariant = (experiment:Experiment, variant:number):void => {
+      const cookie = useCookie(
+        `_experiment_${experiment.name}`, 
+        { 
+          path: '/',
+          maxAge: experiment.maxAgeSeconds ?? defaultMaxAge,
+        }
+      )
+      cookie.value = String(variant)
+    }
+
+    const getCurrentExperiment = ():Experiment => {
+      if (currentExperiments.length > 0) {
+        return currentExperiments[0]
+      } else {
+        return undefined
+      }
+    }
+
+    assignVariants(currentExperiments)
+
+    if (!process.client) {
+      saveVariant(getCurrentExperiment(), activeVariant)
+    }
+
+    return {
+      provide: {
+        experiments: {
+          current: getCurrentExperiment(),
+          activeVariant
+        }
+      }
+    }
+})

plugins/2.features.ts

@@ -0,0 +1,35 @@
+export default defineNuxtPlugin(() => {
+
+  // Add your features here
+  const { $experiments } = useNuxtApp()
+
+  const features = {
+    // Remove duplicate stories from the river on the home page
+    // Part of an a/b test
+    // - Matt Walsh
+    // Experiment started: May 17 2023
+    'experiment-deduplicate-river': 
+      $experiments.current?.name === 'no-duplicates-in-river' && 
+      $experiments.activeVariant === 1
+  }
+
+  const enabled = features || {}
+  const disabled = {}
+  for (const feature of Object.entries(enabled)) {
+    disabled[feature[0]] = !feature[1]
+  }
+  const classes = Object.entries(enabled)
+    .map(entry => (entry[1] ? `${entry[0]}-enabled` : `${entry[0]}-disabled`))
+
+  useHead({
+    bodyAttrs: {
+      class: classes.join(' ')
+    },
+  })
+
+  return {
+    provide: {
+      features: { enabled, disabled, classes }
+    }
+  }
+})

plugins/analytics.client.ts → plugins/4.analytics.client.ts

@@ -3,6 +3,7 @@ import { useMembershipStatus } from "~~/composables/states"
 export default defineNuxtPlugin(() => {
   const config = useRuntimeConfig()
   const membershipStatus = useMembershipStatus()
+  const  { $experiments } = useNuxtApp()
 
   window.dataLayer = window.dataLayer || []
   // init gtag function
@@ -13,7 +14,15 @@ export default defineNuxtPlugin(() => {
 
   // event to use when sending gtag events
   const sendEvent = (name: string, params: Record<string, string>) => {
-    gtag('event', name, params)
+    if ($experiments.current) {
+      gtag('event', name, {
+        experimentName: $experiments.current.name,
+        experimentVariant: $experiments.activeVariant,
+        ...params
+      })
+    } else {
+      gtag('event', name, params)
+    }
   }
   // gtag even for reporting on page views
   const sendPageView = (params: Record<string, string>) => {

plugins/features.md

@@ -0,0 +1,76 @@
+# Feature toggles
+
+The feature toggle plugin was created to enable us to toggle a/b test experiment features
+without the rest of the code needing to know anything about the a/b testing setup. The client code
+(or someone reading it) shouldn't need to know what experiment id #123 variant #2 means, it just 
+needs to know if a feature is enabled or disabled.
+
+While this was created with a/b testing in mind it also has other applications:
+See here for more about feature toggles: https://en.wikipedia.org/wiki/Feature_toggle
+
+## Setup
+
+Add your features and conditions to the `features` object in `/plugins/features.js`
+
+```
+import { $experiments } from useNuxtApp()
+
+const features = {
+  // experiment-bigger-logo
+  // Makes the logo bigger
+  // - Matt Walsh
+  // Expires: 2023/04/30
+  'experiment-bigger-logo': 
+    $experiments.current?.name === 'test' && 
+    $experiments.activeVariant === 1
+}
+```
+
+## Usage 
+
+### Using a feature toggle in CSS
+
+Use the provided `{feature-name}-enabled` and `{featured-name}-disabled` classes to check if a feature is active
+
+```
+.experiment-bigger-logo-enabled .logo {
+  font-size: 99px
+}
+```
+
+### Using a feature toggle in a template
+
+Use `$features.enabled` or `$features.disabled` to test if a feature is active.
+
+```
+<huge-arrow v-if="!$features.enabled['experiment-bigger-logo'] />
+```
+
+### Using a feature toggle in other code
+
+Use `$features.enabled` or `$features.disabled` to test if a feature is active.
+
+```
+import { $features } from useNuxtApp()
+
+mounted () {
+  if ($features.enabled['experiment-bigger-logo']) {
+    animateLogo()
+  }
+}
+```
+
+## Best practices
+
+First, decide if it makes sense for your feature to use a feature toggle. While there
+are several cases where they're useful, too many toggles will create technical debt.
+
+When you add a new feature toggle, add a comment with a breif description of the feature 
+and your name to aid in cleaning up the toggle later. If possible add an expiration date for 
+the feature as well.
+
+Use a prefixes to designate the type of feature.
+
+| Feature Type | Prefix        |
+|--------------|---------------|
+| A/B Test     | `experiment-` |