ab_testing_calculations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1720459a",
   "metadata": {},
   "source": [
    "# PYTHON. Calculations for AB test analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dc16a7d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats as st\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c94eb2ef",
   "metadata": {},
   "source": [
    "## Basic theoretical confidence interval calculation\n",
    "Assumptions: Normal distribution (functions from **scipy.stats** library)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dd815ffc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def conf_interval(clicks, N, conf=0.95, n_tails=2, rnd=4):\n",
    "    # ppf(q, loc=0, scale=1) Percent point function (inverse of cdf — percentiles).\n",
    "    z=st.norm.ppf(1-(1-conf)/n_tails)\n",
    "\n",
    "    p_hat=clicks/N\n",
    "    # binomial distribution\n",
    "    standard_error= math.sqrt(p_hat*(1-p_hat)/N)\n",
    "\n",
    "    margin_of_error = standard_error * z\n",
    "\n",
    "    interval_min = p_hat - margin_of_error\n",
    "    interval_max = p_hat + margin_of_error\n",
    "\n",
    "    #st.norm.cdf(z)\n",
    "    print(f'P_hat:           {round(p_hat,4)}')\n",
    "    print(f'Standard error:  {round(standard_error,4)}')\n",
    "    print(f'Margin of error: {round(margin_of_error,4)}')\n",
    "    print(\"-\"*20)\n",
    "    print(f\"{round(conf*100,1)}% {n_tails}-tail confidence interval is [{round(interval_min,rnd)} ; {round(interval_max,rnd)}]\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e300432a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P_hat:           0.15\n",
      "Standard error:  0.008\n",
      "Margin of error: 0.0206\n",
      "--------------------\n",
      "99.0% 2-tail confidence interval is [0.1294 ; 0.1706]\n"
     ]
    }
   ],
   "source": [
    "N=2000\n",
    "clicks=300\n",
    "conf=0.99\n",
    "n_tails=2\n",
    "\n",
    "conf_interval(clicks, N, conf=conf, n_tails=n_tails)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0f273f2",
   "metadata": {},
   "source": [
    "## Sample size\n",
    "How many observations (at least) need to be collected to have significant results?\n",
    "\n",
    "* [Even Miller calculator](https://www.evanmiller.org/ab-testing/sample-size.html)\n",
    "* calculation based on pooled standard error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0eda0da1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://towardsdatascience.com/mathematical-intuition-behind-a-b-testing-with-python-9d024e5e7f37\n",
    "\n",
    "def calc_sample_size(alpha, beta, p, delta, method, n_tails=2):\n",
    "    \"\"\" Based on https://www.evanmiller.org/ab-testing/sample-size.html\n",
    "    Ref: https://stats.stackexchange.com/questions/357336/create-an-a-b-sample-size-calculator-using-evan-millers-post\n",
    "    Args:\n",
    "        alpha (float): How often are you willing to accept a Type I error (false positive)?\n",
    "        power (float): How often do you want to correctly detect a true positive (=1-beta)?\n",
    "        p (float): Base conversion rate\n",
    "        pct_mde (float): Minimum detectable effect, relative to base conversion rate.\n",
    "    \"\"\"\n",
    "    if method == 'evanmiller':\n",
    "        t_alpha2 = st.norm.ppf(1.0-alpha/n_tails)\n",
    "        t_beta = st.norm.ppf(1-beta)\n",
    "\n",
    "        sd1 = np.sqrt(2 * p * (1.0 - p))\n",
    "        sd2 = np.sqrt(p * (1.0 - p) + (p + delta) * (1.0 - p - delta))\n",
    "\n",
    "        n= round((t_alpha2 * sd1 + t_beta * sd2) * (t_alpha2 * sd1 + t_beta * sd2) / (delta**2))\n",
    "    elif method == 'pooled_se':\n",
    "        \"\"\"\n",
    "        References:\n",
    "            Code taken from Nguyen Ngo: https://towardsdatascience.com/the-math-behind-a-b-testing-with-example-code-part-1-of-2-7be752e1d06f\n",
    "            Stanford lecture on sample sizes     \n",
    "        \"\"\"\n",
    "        # standard normal distribution to determine z-values\n",
    "        standard_norm = st.norm(0, 1)\n",
    "\n",
    "        # find Z_beta from desired power\n",
    "        Z_beta = standard_norm.ppf(1-beta)\n",
    "\n",
    "        # find Z_alpha\n",
    "        Z_alpha = standard_norm.ppf(1-alpha/n_tails)\n",
    "\n",
    "        # average of probabilities from both groups\n",
    "        pooled_prob = (p + p+delta) / 2\n",
    "\n",
    "        n= round(2*(pooled_prob * (1 - pooled_prob) * (Z_beta + Z_alpha)**2\n",
    "                 / delta**2))\n",
    "    return n\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5b9cfc03",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample size \n",
      "1) Evan Miller: 3623  \n",
      "2) Pooled_se:  3842 \n",
      "Average: 3732\n"
     ]
    }
   ],
   "source": [
    "alpha=0.05\n",
    "beta=0.2\n",
    "power=1-beta\n",
    "d_min=0.02 # delta\n",
    "p_hat=0.1\n",
    "pct_mde=p_hat-d_min/p_hat\n",
    "delta=d_min\n",
    "\n",
    "n_evanmiller=calc_sample_size(alpha, beta, p_hat, d_min, method='evanmiller')\n",
    "n_stanford=calc_sample_size(alpha, beta, p_hat, d_min, method='pooled_se')\n",
    "\n",
    "print (f\"Sample size \\n1) Evan Miller: {n_evanmiller}  \\n2) Pooled_se:  {n_stanford} \\nAverage: {round(np.mean([n_stanford,n_evanmiller]))}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88af04e6",
   "metadata": {},
   "source": [
    "## AB test experiment analysis\n",
    "\n",
    "Pooled Standard error theoretical confidence interval calculation and comparison to practical minimal uplift. \n",
    "Verdict on the further actions based on the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "553ac564",
   "metadata": {},
   "outputs": [],
   "source": [
    "#theoretical\n",
    "def ab_pooled_theor_sign(N_cont, action_cont, N_exp, action_exp, d_min, conf=0.95, n_tails=2):\n",
    "    print(f\"Control \\n N      {N_cont}\\n action {action_cont}\")\n",
    "    print(f\"Experiment \\n N      {N_exp}\\n action {action_exp}\")\n",
    "    print(\"-\"*20)\n",
    "    \n",
    "    p_hat_cont=action_cont/N_cont\n",
    "    print(f\"P control:    {round(p_hat_cont,4)}\")\n",
    "\n",
    "    p_hat_exp=action_exp/N_exp\n",
    "    print(f\"P experiment: {round(p_hat_exp,4)}\")\n",
    "\n",
    "    p_hat_pool=(action_cont+action_exp)/(N_cont+N_exp)\n",
    "    print(f\"P poopled:    {round(p_hat_pool,4)}\")\n",
    "\n",
    "    standard_error_pool=math.sqrt(p_hat_pool*(1-p_hat_pool)*((1/N_cont) + (1/N_exp)))\n",
    "\n",
    "    z=st.norm.ppf(1-(1-conf)/n_tails)\n",
    "    margin_of_error_pool = standard_error_pool * z\n",
    "\n",
    "    dif_hat=p_hat_exp-p_hat_cont\n",
    "\n",
    "    interval_min=dif_hat-margin_of_error_pool\n",
    "    interval_max=dif_hat+margin_of_error_pool\n",
    "    print(\"-\"*20)\n",
    "    print(f'Minimal practical significance {round(d_min,4)}')\n",
    "    print(f\"Confidence interval [{round(interval_min,4)},{round(interval_max,4)}]\")\n",
    "    \n",
    "    print(\"-\"*20)\n",
    "    #stat significance - interval doesn't contain zero\n",
    "    if interval_min<0 and interval_max>0:\n",
    "        print(\"Confidence interval contains zero. We cannot reject null hypotesis that there is no difference between groups\")\n",
    "    elif interval_min>0:\n",
    "        print(\"There is a significant POSITIVE result\")\n",
    "    else:   \n",
    "        print(\"There is a significant NEGATIVE result\")\n",
    "\n",
    "\n",
    "    #practical significance   - the interval is to the right from min uplift\n",
    "    if d_min<=interval_min:\n",
    "        print(f\"With {conf*100}% confidence there is practical and significant effect greater than minimal prectical uplift ({d_min}). \\nVerdict: LAUNCH the change\")\n",
    "    elif interval_min>=-d_min and interval_max<=d_min:\n",
    "        print(\"There is no practical significance. \\nVerdict: NO LAUNCH\")\n",
    "    elif interval_min<=-d_min and interval_max>=d_min:\n",
    "        print (\"Overlap. Results do not provide clarity. \\nVerdict: Additional test may bring more solid results\")\n",
    "    elif interval_min>=-d_min and interval_max>=d_min:\n",
    "        print (\"Intersection on positive side. Results do not provide clarity. \\nVerdict: Additional test may bring more solid results\")   \n",
    "    else: \n",
    "        print(\"No Launch\")\n",
    "    print(\"-\"*20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5e203a90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Control \n",
      " N      10072\n",
      " action 974\n",
      "Experiment \n",
      " N      9886\n",
      " action 1242\n",
      "--------------------\n",
      "P control:    0.0967\n",
      "P experiment: 0.1256\n",
      "P poopled:    0.111\n",
      "--------------------\n",
      "Minimal practical significance 0.02\n",
      "Confidence interval [0.0202,0.0376]\n",
      "--------------------\n",
      "There is a significant POSITIVE result\n",
      "With 95.0% confidence there is practical and significant effect greater than minimal prectical uplift (0.02). \n",
      "Verdict: LAUNCH the change\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "n_tails=2\n",
    "conf=0.95\n",
    "d_min=0.02\n",
    "#experiment --------------------------------------------------------------\n",
    "#control\n",
    "N_cont=10072 # pageviews\n",
    "action_cont=974 # clicks, views, etc\n",
    "\n",
    "#experiment (target)\n",
    "N_exp=9886 # pageviews\n",
    "action_exp=1242 #clicks, views, etc\n",
    "# calculations --------------------------------------------------------------\n",
    "\n",
    "ab_pooled_theor_sign(N_cont, action_cont, N_exp, action_exp, d_min)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06418e08",
   "metadata": {},
   "source": [
    "## Empirical confidence interval\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bded5c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "def empir_conf_interval(data, conf=0.95, n_tails=2, rnd=4):\n",
    "    N=len(data)\n",
    "    data_mean=np.mean(data)\n",
    "    data_std=np.std(data, ddof=1)\n",
    "    data_standard_error=data_std/np.sqrt(N)\n",
    "\n",
    "    # ppf(q, loc=0, scale=1) Percent point function (inverse of cdf — percentiles).\n",
    "    z=st.norm.ppf(1-(1-conf)/n_tails)\n",
    "    data_margin_of_error=z*data_standard_error\n",
    "\n",
    "\n",
    "    interval_min = data_mean - data_margin_of_error\n",
    "    interval_max = data_mean + data_margin_of_error\n",
    "    print(f\"{round(conf*100,1)}% {n_tails}-tail confidence interval is [{round(interval_min,rnd)} ; {round(interval_max,rnd)}]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8c8e357e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95.0% 2-tail confidence interval is [79158.0 ; 104367.0]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "data=[87029, 113407, 84843, 104994, 99327, 92052, 60684]\n",
    "conf=0.95\n",
    "n_tails=2\n",
    "\n",
    "empir_conf_interval(data, conf=0.95, n_tails=2, rnd=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "961b43f6",
   "metadata": {},
   "source": [
    "## Variability analysis (bootstrap)\n",
    "\n",
    "**Bootstrap** - run one bigger experiment and then randomly split this large sample of users to smaller groups, calculate metric and compare results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c9262a46",
   "metadata": {},
   "outputs": [],
   "source": [
    "def conf_iterval_diff(data, conf=0.95, n_tails=2,rnd=4):\n",
    "    # theoretical confidence interval\n",
    "    N=len(data)\n",
    "    data_mean=np.mean(data)\n",
    "    data_std=np.std(data, ddof=1)\n",
    "    print(f\"Mean difference is {round(data_mean,rnd)} and standard deviation is {round(data_std,rnd)}\")\n",
    "\n",
    "    #Theoretical confidence interval\n",
    "    # ppf(q, loc=0, scale=1) Percent point function (inverse of cdf — percentiles).\n",
    "    z=st.norm.ppf(1-(1-conf)/n_tails)\n",
    "    data_margin_of_error=z*data_std\n",
    "\n",
    "\n",
    "    interval_min = data_mean - data_margin_of_error\n",
    "    interval_max = data_mean + data_margin_of_error\n",
    "    print(\"-\"*20)\n",
    "    print(\"Theoretical confidence interval\")\n",
    "    print(f\"{round(conf*100,1)}% {n_tails}-tail confidence interval is [{round(interval_min,rnd)} ; {round(interval_max,rnd)}]\")\n",
    "\n",
    "    # Empirical confidence interval (percentile)\n",
    "\n",
    "    ran=np.percentile(data,[2.5,97.5])\n",
    "    print(\"-\"*20)\n",
    "    print(\"Empirical confidence interval\")\n",
    "    print(f\"{round(conf*100,1)}% {n_tails}-tail confidence interval is [{round(ran[0],rnd)} ; {round(ran[1],rnd)}]\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8e74b93b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean difference is 0.0042 and standard deviation is 0.0364\n",
      "--------------------\n",
      "Theoretical confidence interval\n",
      "95.0% 2-tail confidence interval is [-0.067 ; 0.0755]\n",
      "--------------------\n",
      "Empirical confidence interval\n",
      "95.0% 2-tail confidence interval is [-0.0605 ; 0.0802]\n"
     ]
    }
   ],
   "source": [
    "conf=0.95\n",
    "n_tails=2\n",
    "\n",
    "# test results\n",
    "g1=[0.02,0.11,0.14,0.05,0.09,0.11,0.09,0.1,0.14,0.08,0.09,0.08,0.09,0.08,0.12,0.09,0.16,0.11,0.12,0.11,0.06,0.11,0.13,0.1,0.08,0.14,0.1,0.08,0.12,0.09,0.14,0.1,0.08,0.08,0.07,0.13,0.11,0.08,0.1,0.11]\n",
    "g2=[0.07,0.11,0.05,0.07,0.1,0.07,0.1,0.1,0.12,0.14,0.04,0.07,0.07,0.06,0.15,0.09,0.12,0.1,0.08,0.09,0.08,0.08,0.14,0.09,0.1,0.08,0.08,0.09,0.08,0.11,0.11,0.1,0.14,0.1,0.08,0.05,0.19,0.11,0.08,0.13]\n",
    "\n",
    "# difference between A and B per test\n",
    "g_diff=[round(x[0]-x[1],2) for x in zip(g1,g2)]\n",
    "\n",
    "# g_diff=[0]*40\n",
    "# for i in range(40):\n",
    "#     g_diff[i]=round(g1[i]-g2[i],2)\n",
    "   \n",
    "conf_iterval_diff(g_diff, conf, n_tails)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff8de04d",
   "metadata": {},
   "source": [
    "## Sanity check example\n",
    "**Given**: We know true probability - users are assigned to A and B randomly (binomial) with probability of 0.5\n",
    "\n",
    "**Metric**: #cookies (Unit of diversion = cookie)\n",
    "\n",
    "**Result**: We get data by day on number of users (cookies) in each group\n",
    "\n",
    "**Experiment duration**: 2 weeks\n",
    "\n",
    "**Question to answer**: is the data distributed as expected = we have around 50/50 randomly distributed data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cab4b4b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day</th>\n",
       "      <th>N_A</th>\n",
       "      <th>N_B</th>\n",
       "      <th>p_hat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mon</td>\n",
       "      <td>5077</td>\n",
       "      <td>4877</td>\n",
       "      <td>0.510046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tue</td>\n",
       "      <td>5495</td>\n",
       "      <td>4729</td>\n",
       "      <td>0.537461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>wed</td>\n",
       "      <td>5294</td>\n",
       "      <td>5063</td>\n",
       "      <td>0.511152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thu</td>\n",
       "      <td>5446</td>\n",
       "      <td>5035</td>\n",
       "      <td>0.519607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fri</td>\n",
       "      <td>5126</td>\n",
       "      <td>5010</td>\n",
       "      <td>0.505722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>sat</td>\n",
       "      <td>3382</td>\n",
       "      <td>3193</td>\n",
       "      <td>0.514373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>sun</td>\n",
       "      <td>2891</td>\n",
       "      <td>3226</td>\n",
       "      <td>0.472617</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   day   N_A   N_B     p_hat\n",
       "0  mon  5077  4877  0.510046\n",
       "1  tue  5495  4729  0.537461\n",
       "2  wed  5294  5063  0.511152\n",
       "3  thu  5446  5035  0.519607\n",
       "4  fri  5126  5010  0.505722\n",
       "5  sat  3382  3193  0.514373\n",
       "6  sun  2891  3226  0.472617"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day</th>\n",
       "      <th>N_A</th>\n",
       "      <th>N_B</th>\n",
       "      <th>p_hat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mon</td>\n",
       "      <td>5029</td>\n",
       "      <td>5092</td>\n",
       "      <td>0.496888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tue</td>\n",
       "      <td>5166</td>\n",
       "      <td>5048</td>\n",
       "      <td>0.505776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>wed</td>\n",
       "      <td>4902</td>\n",
       "      <td>5063</td>\n",
       "      <td>0.491922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thu</td>\n",
       "      <td>4923</td>\n",
       "      <td>4805</td>\n",
       "      <td>0.506065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fri</td>\n",
       "      <td>4816</td>\n",
       "      <td>4741</td>\n",
       "      <td>0.503924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>sat</td>\n",
       "      <td>3411</td>\n",
       "      <td>2939</td>\n",
       "      <td>0.537165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>sun</td>\n",
       "      <td>3496</td>\n",
       "      <td>3075</td>\n",
       "      <td>0.532035</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   day   N_A   N_B     p_hat\n",
       "0  mon  5029  5092  0.496888\n",
       "1  tue  5166  5048  0.505776\n",
       "2  wed  4902  5063  0.491922\n",
       "3  thu  4923  4805  0.506065\n",
       "4  fri  4816  4741  0.503924\n",
       "5  sat  3411  2939  0.537165\n",
       "6  sun  3496  3075  0.532035"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total in group A 63844\n",
      "Total in group B 62506\n"
     ]
    }
   ],
   "source": [
    "#DATA PREPARATION\n",
    "# data\n",
    "w1=[\"mon\", 5077,4877,'tue',5495,4729,'wed',5294,5063,'thu',5446,5035,'fri',5126,5010,'sat',3382,3193,'sun',2891,3226]\n",
    "w1=np.array(w1)\n",
    "df_w1=pd.DataFrame(w1.reshape(7,3),columns=['day', 'N_A', 'N_B'])\n",
    "w2=[\"mon\", 5029,5092,'tue',5166,5048,'wed',4902,5063,'thu',4923,4805,'fri',4816,4741,'sat',3411,2939,'sun',3496,3075]\n",
    "w2=np.array(w2)\n",
    "df_w2=pd.DataFrame(w2.reshape(7,3),columns=['day', 'N_A', 'N_B'])\n",
    "\n",
    "for c in ['N_A', 'N_B']:\n",
    "    df_w1[c]=pd.to_numeric(df_w1[c])    \n",
    "    df_w2[c]=pd.to_numeric(df_w2[c]) \n",
    "\n",
    "df_w1['p_hat']=df_w1['N_A']/(df_w1['N_A']+df_w1['N_B'])\n",
    "df_w2['p_hat']=df_w2['N_A']/(df_w2['N_A']+df_w2['N_B']) \n",
    "  \n",
    "\n",
    "display(df_w1)\n",
    "display(df_w2)\n",
    "\n",
    "# total per week\n",
    "N_w1_a=df_w1['N_A'].sum()\n",
    "N_w1_b=df_w1['N_B'].sum()\n",
    "N_w2_a=df_w2['N_A'].sum()\n",
    "N_w2_b=df_w2['N_B'].sum()\n",
    "\n",
    "# total for 2 weeks\n",
    "N_A = N_w1_a + N_w1_b\n",
    "N_B = N_w2_a + N_w2_b\n",
    "\n",
    "print(f'Total in group A {N_A}\\nTotal in group B {N_B}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9f9f9fde",
   "metadata": {},
   "outputs": [],
   "source": [
    "def theor_split_check (p_true, N_cont, N_exp, conf=0.95, n_tails=2, rnd=4):\n",
    "\n",
    "    # binomial\n",
    "    standard_deviation=math.sqrt(p_true*(1-p_true)/(N_cont+N_exp))\n",
    "    \n",
    "    \n",
    "    # multiply by z score to get margin of error\n",
    "    z=st.norm.ppf(1-(1-conf)/n_tails)\n",
    "    margin_of_error=z*standard_deviation\n",
    "    \n",
    "    # compute confidence interval around 0.5\n",
    "    interval_min = p_true - margin_of_error\n",
    "    interval_max = p_true + margin_of_error\n",
    "    \n",
    "    print(\"Theoretical confidence interval\")\n",
    "    print(f\"{round(conf*100,1)}% {n_tails}-tail confidence interval is [{round(interval_min,rnd)} ; {round(interval_max,rnd)}]\")\n",
    "    \n",
    "    p_hat=N_cont/(N_cont+N_exp)\n",
    "    \n",
    "    \n",
    "    print(\"-\"*20)\n",
    "    print(f'Observed ratio for A vs B is {round(p_hat,rnd)}')\n",
    "    if p_hat>interval_min and p_hat<interval_max:\n",
    "        print (f\"Results ARE statistically significant with {round(conf*100,0)} confidence. Data is distributed according to expected probability\")\n",
    "    else:\n",
    "        print (f\"Results DO NOT follow expected probability with {round(conf*100,0)} confidence.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f960bec1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theoretical confidence interval\n",
      "95.0% 2-tail confidence interval is [0.4972 ; 0.5028]\n",
      "--------------------\n",
      "Observed ratio for A vs B is 0.5053\n",
      "Results DO NOT follow expected probability with 95.0 confidence.\n"
     ]
    }
   ],
   "source": [
    "p_true=0.5\n",
    "conf=0.95\n",
    "n_tails=2\n",
    "theor_split_check (p_true, N_A, N_B, conf=0.95, n_tails=2, rnd=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1953a5f",
   "metadata": {},
   "source": [
    "## Single metric sanity check\n",
    "\n",
    "**Experiment**: change color and placement of a button\n",
    "\n",
    "**Metric**: click trough rate\n",
    "\n",
    "**Unit of diversion**: cookie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6133f11b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day</th>\n",
       "      <th>click_A</th>\n",
       "      <th>N_A</th>\n",
       "      <th>click_B</th>\n",
       "      <th>N_B</th>\n",
       "      <th>cta_A</th>\n",
       "      <th>cta_B</th>\n",
       "      <th>success</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d1</td>\n",
       "      <td>51</td>\n",
       "      <td>1292</td>\n",
       "      <td>115</td>\n",
       "      <td>1305</td>\n",
       "      <td>0.039474</td>\n",
       "      <td>0.088123</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d2</td>\n",
       "      <td>39</td>\n",
       "      <td>853</td>\n",
       "      <td>73</td>\n",
       "      <td>835</td>\n",
       "      <td>0.045721</td>\n",
       "      <td>0.087425</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d3</td>\n",
       "      <td>64</td>\n",
       "      <td>1129</td>\n",
       "      <td>91</td>\n",
       "      <td>1133</td>\n",
       "      <td>0.056687</td>\n",
       "      <td>0.080318</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4</td>\n",
       "      <td>43</td>\n",
       "      <td>873</td>\n",
       "      <td>60</td>\n",
       "      <td>871</td>\n",
       "      <td>0.049255</td>\n",
       "      <td>0.068886</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d5</td>\n",
       "      <td>55</td>\n",
       "      <td>1197</td>\n",
       "      <td>78</td>\n",
       "      <td>1134</td>\n",
       "      <td>0.045948</td>\n",
       "      <td>0.068783</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>d6</td>\n",
       "      <td>44</td>\n",
       "      <td>1023</td>\n",
       "      <td>72</td>\n",
       "      <td>1015</td>\n",
       "      <td>0.043011</td>\n",
       "      <td>0.070936</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>d7</td>\n",
       "      <td>56</td>\n",
       "      <td>1003</td>\n",
       "      <td>76</td>\n",
       "      <td>977</td>\n",
       "      <td>0.055833</td>\n",
       "      <td>0.077789</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  day  click_A   N_A  click_B   N_B     cta_A     cta_B  success\n",
       "0  d1       51  1292      115  1305  0.039474  0.088123        1\n",
       "1  d2       39   853       73   835  0.045721  0.087425        1\n",
       "2  d3       64  1129       91  1133  0.056687  0.080318        1\n",
       "3  d4       43   873       60   871  0.049255  0.068886        1\n",
       "4  d5       55  1197       78  1134  0.045948  0.068783        1\n",
       "5  d6       44  1023       72  1015  0.043011  0.070936        1\n",
       "6  d7       56  1003       76   977  0.055833  0.077789        1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "totals\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'click_A': 352, 'N_A': 7370, 'click_B': 565, 'N_B': 7270}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success rate (= cta in B> cta in A) by day is 100.0%\n",
      "Difference in metric (exp-control): 0.03\n"
     ]
    }
   ],
   "source": [
    "#DATA\n",
    "w=[\"d1\", 51,1292, 115, 1305,\n",
    "    'd2',39,853,73,835,\n",
    "    'd3',64,1129,91,1133,\n",
    "    'd4',43,873,60,871,\n",
    "    'd5',55,1197,78,1134,\n",
    "    'd6',44,1023,72,1015,\n",
    "    'd7',56,1003,76,977]\n",
    "w=np.array(w)\n",
    "df_w=pd.DataFrame(w.reshape(7,5),columns=['day', 'click_A', 'N_A', 'click_B', 'N_B'])\n",
    "\n",
    "totals={}\n",
    "for c in [ 'click_A', 'N_A', 'click_B', 'N_B']:\n",
    "    df_w[c]=pd.to_numeric(df_w[c]) \n",
    "    totals[c]=df_w[c].sum()\n",
    "\n",
    "for c in ['A','B']:\n",
    "    df_w[f'cta_{c}']=df_w[f'click_{c}']/df_w[f'N_{c}']\n",
    "\n",
    "df_w['success']=(df_w['cta_B']>df_w['cta_A'])*1\n",
    "\n",
    "display(df_w)\n",
    "print('totals')\n",
    "display(totals)\n",
    "\n",
    "print(f\"Success rate (= cta in B> cta in A) by day is {round(df_w['success'].sum()/df_w.shape[0]*100,2)}%\")\n",
    "\n",
    "for c in ['A', 'B']:\n",
    "    totals[f'cta_{c}']=totals[f'click_{c}']/totals[f'N_{c}']\n",
    "\n",
    "dif=totals['cta_B']-totals['cta_A']\n",
    "print(f\"Difference in metric (exp-control): {round(dif,4)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "be70ab2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "d_min=0.01\n",
    "alpha=0.05\n",
    "beta=0.2\n",
    "conf=0.95\n",
    "\n",
    "emperical_SE=0.0035\n",
    "emperical_SE_N=10000\n",
    "\n",
    "\n",
    "    \n",
    "def sanity_check_single_metric(dif, d_min, emperical_SE, emperical_SE_N, N_A, N_B, conf=0.95, n_tails=2, rnd=4):\n",
    "    print(f\"Observed metric: {round(dif,4)}\")\n",
    "    standard_error=(emperical_SE/math.sqrt(1/emperical_SE_N + 1/emperical_SE_N))*math.sqrt(1/N_B + 1/N_A)\n",
    "    print(f\"Standard error: {round(standard_error,4)}\")\n",
    "\n",
    "    # multiply by z score to get margin of error\n",
    "    z=st.norm.ppf(1-(1-conf)/n_tails)\n",
    "    margin_of_error=z*standard_error\n",
    "    \n",
    "    # compute confidence interval around 0.5\n",
    "    interval_min = dif - margin_of_error\n",
    "    interval_max = dif + margin_of_error\n",
    "\n",
    "    print(\"-\"*20)\n",
    "    print(f'Minimal practical significance {round(d_min,4)}')\n",
    "    print(f\"Confidence interval [{round(interval_min,4)},{round(interval_max,4)}]\")\n",
    "    \n",
    "    print(\"-\"*20)\n",
    "    #stat significance - interval doesn't contain zero\n",
    "    if interval_min<0 and interval_max>0:\n",
    "        print(\"Confidence interval contains zero. We cannot reject null hypotesis that there is no difference between groups\")\n",
    "    elif interval_min>0:\n",
    "        print(\"There is a significant POSITIVE result\")\n",
    "    else:   \n",
    "        print(\"There is a significant NEGATIVE result\")\n",
    "\n",
    "\n",
    "    #practical significance   - the interval is to the right from min uplift\n",
    "    if d_min<=interval_min:\n",
    "        print(f\"With {conf*100}% confidence there is practical and significant effect greater than minimal practical uplift ({d_min}). \\nVerdict: LAUNCH the change\")\n",
    "    elif interval_min>=-d_min and interval_max<=d_min:\n",
    "        print(\"There is no practical significance. \\nVerdict: NO LAUNCH\")\n",
    "    elif interval_min<=-d_min and interval_max>=d_min:\n",
    "        print (\"Overlap. Results do not provide clarity. \\nVerdict: Additional test may bring more solid results\")\n",
    "    elif interval_min>=-d_min and interval_max>=d_min:\n",
    "        print (\"Intersection on positive side. Results do not provide clarity. \\nVerdict: Additional test may bring more solid results\")   \n",
    "    else: \n",
    "        print(\"NO Launch\")\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d9aed971",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observed metric: 0.03\n",
      "Standard error: 0.0041\n",
      "--------------------\n",
      "Minimal practical significance 0.01\n",
      "Confidence interval [0.0219,0.038]\n",
      "--------------------\n",
      "There is a significant POSITIVE result\n",
      "With 95.0% confidence there is practical and significant effect greater than minimal practical uplift (0.01). \n",
      "Verdict: LAUNCH the change\n"
     ]
    }
   ],
   "source": [
    "sanity_check_single_metric(dif, d_min, emperical_SE, emperical_SE_N, totals['N_A'], totals['N_B'], conf=0.95, n_tails=2, rnd=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa84330d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}