06_variance_reduction_my_solutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A/B Testing Course"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lesson 6. Variance Reduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Homework"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 580,
     "status": "ok",
     "timestamp": 1674542192205,
     "user": {
      "displayName": "Nikolay Nazarov",
      "userId": "18018785782997408667"
     },
     "user_tz": -420
    },
    "id": "agQEC38nE-72",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import ttest_ind"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 6399,
     "status": "ok",
     "timestamp": 1674542198600,
     "user": {
      "displayName": "Nikolay Nazarov",
      "userId": "18018785782997408667"
     },
     "user_tz": -420
    },
    "id": "nQGxlqByE-73",
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_web_logs = pd.read_csv('2022-04-13T12_df_web_logs.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Data Description"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "df_exp_users - users split in the experiment:\n",
    "- user_id - user identifier;\n",
    "- pilot - test group flag."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nNRTGF6aBHrZ"
   },
   "source": [
    "df_web_logs - web logs information:\n",
    "- user_id - user identifier;\n",
    "- page - visited page;\n",
    "- date - date and time of a page visit;\n",
    "- load_time - page load time."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Checking Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>page</th>\n",
       "      <th>date</th>\n",
       "      <th>load_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f25239</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:45:37</td>\n",
       "      <td>80.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>06d6df</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:49:56</td>\n",
       "      <td>70.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>06d6df</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:51:16</td>\n",
       "      <td>89.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f25239</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:51:43</td>\n",
       "      <td>74.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>697870</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:53:12</td>\n",
       "      <td>66.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_id page                 date  load_time\n",
       "0  f25239    m  2022-02-03 23:45:37       80.8\n",
       "1  06d6df    m  2022-02-03 23:49:56       70.5\n",
       "2  06d6df    m  2022-02-03 23:51:16       89.7\n",
       "3  f25239    m  2022-02-03 23:51:43       74.4\n",
       "4  697870    m  2022-02-03 23:53:12       66.8"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_web_logs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2401709 entries, 0 to 2401708\n",
      "Data columns (total 4 columns):\n",
      " #   Column     Dtype  \n",
      "---  ------     -----  \n",
      " 0   user_id    object \n",
      " 1   page       object \n",
      " 2   date       object \n",
      " 3   load_time  float64\n",
      "dtypes: float64(1), object(3)\n",
      "memory usage: 73.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df_web_logs.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_web_logs.load_time.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Task 1. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare the powers of tests with different fractions of removable data.  \n",
    "The significance level is 0.05.  \n",
    "The group sizes are 1000 people (sample sizes will be larger, as there are many values for each person).  \n",
    "We test the hypothesis of equality of means using the Student's t-test.  \n",
    "The expected effect is an increase in processing time by 1%.  \n",
    "The effect in synthetic A/B tests is added by multiplying by a constant."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.2: 0.964, 0.1: 0.967, 0.02: 0.939, 0.002: 0.337, 0.0002: 0.064}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sim_power(df_log, outliers):\n",
    "    result = dict()\n",
    "    begin_date = datetime(2022, 3, 1)\n",
    "    end_date = datetime(2022, 3, 8)\n",
    "    df_log['date'] = pd.to_datetime(df_log['date'])\n",
    "    df = (\n",
    "    df_log\n",
    "    [(df_log['date'] >= begin_date) & (df_log['date'] < end_date)]\n",
    "    [['user_id', 'load_time']]\n",
    "    )\n",
    "    all_users = df.user_id.unique().tolist()\n",
    "    for outlier in outliers[0:]:\n",
    "        p_vals = []\n",
    "        for _ in range(1000):\n",
    "            exp_users = np.random.choice(all_users, 2000, replace=False)\n",
    "            users_a = exp_users[0:1000]\n",
    "            users_b = exp_users[1000:]\n",
    "            group_a = df[df['user_id'].isin(users_a)].load_time\n",
    "            group_b = df[df['user_id'].isin(users_b)].load_time\n",
    "            group_bb = group_b * 1.01\n",
    "            \n",
    "            lower_a = np.quantile(group_a, outlier/2)\n",
    "            upper_a = np.quantile(group_a, 1 - outlier/2)\n",
    "            lower_b = np.quantile(group_bb, outlier/2)\n",
    "            upper_b = np.quantile(group_bb, 1 - outlier/2)\n",
    "            group_a_filt = [x for x in group_a if x > lower_a  and x < upper_a]\n",
    "            group_b_filt = [x for x in group_bb if x > lower_b and x < upper_b]\n",
    "\n",
    "            p_val = ttest_ind(group_a_filt, group_b_filt).pvalue\n",
    "            p_vals.append(p_val)\n",
    "        \n",
    "        sig_cnt = [x for x in p_vals if x < 0.05]\n",
    "        power = len(sig_cnt) / len(p_vals)\n",
    "        \n",
    "        result[outlier] = power\n",
    "                                                                                                      \n",
    "    return result\n",
    "\n",
    "\n",
    "sim_power(df_web_logs, [0.2, 0.1, 0.02, 0.002, 0.0002])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Task 2. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Complete the same task, but change the way the effect is added. In synthetic A/B tests, the effect is added by adding a constant to 1% of the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.2: 0.324, 0.1: 0.318, 0.02: 0.521, 0.002: 0.372, 0.0002: 0.115}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sim_power(df_log, outliers):\n",
    "    result = dict()\n",
    "    begin_date = datetime(2022, 3, 1)\n",
    "    end_date = datetime(2022, 3, 8)\n",
    "    df_log['date'] = pd.to_datetime(df_log['date'])\n",
    "    df = (\n",
    "    df_log\n",
    "    [(df_log['date'] >= begin_date) & (df_log['date'] < end_date)]\n",
    "    [['user_id', 'load_time']]\n",
    "    )\n",
    "    all_users = df.user_id.unique().tolist()\n",
    "    for outlier in outliers[0:]:\n",
    "        p_vals = []\n",
    "        for _ in range(1000):\n",
    "            exp_users = np.random.choice(all_users, 2000, replace=False)\n",
    "            users_a = exp_users[0:1000]\n",
    "            users_b = exp_users[1000:]\n",
    "            group_a = df[df['user_id'].isin(users_a)].load_time\n",
    "            group_b = df[df['user_id'].isin(users_b)].load_time\n",
    "\n",
    "            len_b = len(group_b)\n",
    "            one_pct = int(len_b / 100)\n",
    "            group_b99 = group_b[0:len_b-one_pct]\n",
    "            group_b1 = group_b[len_b-one_pct:]\n",
    "            effect = 1 * len_b / len(group_b1)\n",
    "            group_b1_effect = group_b1 + effect\n",
    "            group_bb = pd.concat([group_b99, group_b1_effect])\n",
    "            \n",
    "            lower_a = np.quantile(group_a, outlier/2)\n",
    "            upper_a = np.quantile(group_a, 1 - outlier/2)\n",
    "            lower_b = np.quantile(group_bb, outlier/2)\n",
    "            upper_b = np.quantile(group_bb, 1 - outlier/2)\n",
    "            group_a_filt = [x for x in group_a if x > lower_a  and x < upper_a]\n",
    "            group_b_filt = [x for x in group_bb if x > lower_b and x < upper_b]\n",
    "\n",
    "            p_val = ttest_ind(group_a_filt, group_b_filt).pvalue\n",
    "            p_vals.append(p_val)\n",
    "        \n",
    "        sig_cnt = [x for x in p_vals if x < 0.05]\n",
    "        power = len(sig_cnt) / len(p_vals)\n",
    "        \n",
    "        result[outlier] = power\n",
    "                                                                                                      \n",
    "    return result\n",
    "\n",
    "\n",
    "sim_power(df_web_logs, [0.2, 0.1, 0.02, 0.002, 0.0002])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Task 3. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create method `process_outliers` of a class `MetricsService`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "simple test passed\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from pydantic import BaseModel\n",
    "\n",
    "\n",
    "class Design(BaseModel):\n",
    "    \"\"\"A data class with a description of experiment parameters:\n",
    "\n",
    "    statistical_test - type of statistical test. ['ttest', 'bootstrap']\n",
    "    effect - effect size in percentage\n",
    "    alpha - significance level\n",
    "    beta - allowable probability of Type II error\n",
    "    bootstrap_iter - number of bootstrap iterations\n",
    "    bootstrap_ci_type - method for constructing a confidence interval. ['normal', 'percentile', 'pivotal']\n",
    "    bootstrap_agg_func - experiment metric. ['mean', 'quantile 95']\n",
    "    metric_name - name of the target experiment metric\n",
    "    metric_outlier_lower_bound - lower acceptable boundary of the metric; anything below this is considered an outlier\n",
    "    metric_outlier_upper_bound - upper acceptable boundary of the metric; anything above this is considered an outlier\n",
    "    metric_outlier_process_type - method for handling outliers. ['drop', 'clip'].\n",
    "        'drop' - remove measurement, \n",
    "        'clip' - replace outlier with the nearest boundary value (lower_bound, upper_bound).\n",
    "    \"\"\"\n",
    "    statistical_test: str = 'ttest'\n",
    "    effect: float = 3.\n",
    "    alpha: float = 0.05\n",
    "    beta: float = 0.1\n",
    "    bootstrap_iter: int = 1000\n",
    "    bootstrap_ci_type: str = 'normal'\n",
    "    bootstrap_agg_func: str = 'mean'\n",
    "    metric_name: str\n",
    "    metric_outlier_lower_bound: float\n",
    "    metric_outlier_upper_bound: float\n",
    "    metric_outlier_process_type: str\n",
    "\n",
    "\n",
    "class MetricsService:\n",
    "\n",
    "    def process_outliers(self, metrics, design):\n",
    "        \"\"\"Returns a new dataframe with processed outliers in the metric measurements.\n",
    "\n",
    "        :param metrics (pd.DataFrame): table with metric values, columns=['user_id', 'metric'].\n",
    "        :param design (Design): object with experiment parameter data.\n",
    "        :return df: columns=['user_id', 'metric']\n",
    "        \"\"\"\n",
    "        # YOUR_CODE_HERE\n",
    "        lower = design.metric_outlier_lower_bound\n",
    "        upper = design.metric_outlier_upper_bound\n",
    "        method = design.metric_outlier_process_type\n",
    "        \n",
    "        if method == 'drop':\n",
    "            df_filt = metrics[(metrics['metric'] >= lower) & (metrics['metric'] <= upper)]\n",
    "        elif method == 'clip':\n",
    "            df_mid = metrics[(metrics['metric'] >= lower) & (metrics['metric'] <= upper)]\n",
    "            min_v = lower\n",
    "            max_v = upper\n",
    "            df_lower = metrics[metrics['metric'] < lower]\n",
    "            df_lower['metric'] = min_v\n",
    "            df_upper = metrics[metrics['metric'] > upper]\n",
    "            df_upper['metric'] = max_v\n",
    "            df_filt = pd.concat([df_lower, df_mid, df_upper]).reset_index(drop=True)\n",
    "            \n",
    "        return df_filt.copy()\n",
    "\n",
    "\n",
    "def _chech_df(df, df_ideal, sort_by, reindex=False, set_dtypes=False):\n",
    "    assert isinstance(df, pd.DataFrame), 'The function did not return a pd.DataFrame.'\n",
    "    assert len(df) == len(df_ideal), 'Wrong numner of rows.'\n",
    "    assert len(df.T) == len(df_ideal.T), 'Wrong number of columns.'\n",
    "    columns = df_ideal.columns\n",
    "    assert df.columns.isin(columns).sum() == len(df.columns), 'Wrong column name.'\n",
    "    df = df[columns].sort_values(sort_by)\n",
    "    df_ideal = df_ideal.sort_values(sort_by)\n",
    "    if reindex:\n",
    "        df_ideal.index = range(len(df_ideal))\n",
    "        df.index = range(len(df))\n",
    "    if set_dtypes:\n",
    "        for column, dtype in df_ideal.dtypes.to_dict().items():\n",
    "            df[column] = df[column].astype(dtype)\n",
    "    assert df_ideal.equals(df), 'The resulting dataframe does not match the expected result.'\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    metrics = pd.DataFrame({\n",
    "        'user_id': ['1', '2', '3'],\n",
    "        'metric': [1., 2, 3]\n",
    "    })\n",
    "    design = Design(\n",
    "        metric_name='response_time',\n",
    "        metric_outlier_lower_bound=0.1,\n",
    "        metric_outlier_upper_bound=2.2,\n",
    "        metric_outlier_process_type='drop',\n",
    "    )\n",
    "    ideal_processed_metrics = pd.DataFrame({\n",
    "        'user_id': ['1', '2'],\n",
    "        'metric': [1., 2]\n",
    "    })\n",
    "\n",
    "    metrics_service = MetricsService()\n",
    "    processed_metrics = metrics_service.process_outliers(metrics, design)\n",
    "    _chech_df(processed_metrics, ideal_processed_metrics, ['user_id', 'metric'], True, True)\n",
    "    print('simple test passed')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "KRzEGsPUE-7k"
   ],
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}