pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "\n",
    "from collect_functions import *\n",
    "from constants import *\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process raw data and organize into dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get data files\n",
    "filenames = [y for x in os.walk(RAW_DATA_PATH) for y in glob(os.path.join(x[0], '*.json'))]\n",
    "\n",
    "# write dataframes to disk\n",
    "store_all_df(filenames, agg_features=\"max\", agg_survey=False)\n",
    "store_all_df(filenames, agg_features=\"slope\", agg_survey=False)\n",
    "store_all_time_series(filenames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read main dataframes\n",
    "df_features_max = pd.read_csv(DATAFRAMES_PATH + \"df_features_agg_max.csv\")\n",
    "df_features_slope = pd.read_csv(DATAFRAMES_PATH + \"df_features_agg_slope.csv\")\n",
    "\n",
    "df_generals = pd.read_csv(DATAFRAMES_PATH + \"df_generals.csv\")\n",
    "df_answers = pd.read_csv(DATAFRAMES_PATH + \"df_answers.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "video_names = {1: \"arm\", 2: \"pepper\", 3: \"sophia\"}\n",
    "def replace_video_id(df):\n",
    "    df[\"video\"] = df.video_id.replace(video_names)\n",
    "    df.drop(\"video_id\", axis=1, inplace=True)\n",
    "    return df\n",
    "\n",
    "df_features_max = replace_video_id(df_features_max)\n",
    "df_features_slope = replace_video_id(df_features_slope)\n",
    "df_answers = replace_video_id(df_answers)\n",
    "\n",
    "# set indexes of dataframes\n",
    "df_features_max.set_index([\"user_id\", \"video\"], inplace=True)\n",
    "df_features_slope.set_index([\"user_id\", \"video\"], inplace=True)\n",
    "df_answers.set_index([\"user_id\", \"video\"], inplace=True)\n",
    "df_generals.set_index(\"user_id\", inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# minimum length of the time series for proper analysis\n",
    "threshold = 100\n",
    "# non-meaningful data\n",
    "indices_to_drop = df_features_max[df_features_max.nb_timestamps < 100].index\n",
    "\n",
    "# remove such videos, and their corresponding answers in the survey\n",
    "df_features_max.drop(indices_to_drop, inplace=True)\n",
    "df_features_slope.drop(indices_to_drop, inplace=True)\n",
    "df_answers.drop(indices_to_drop, inplace=True)\n",
    "\n",
    "df_features_max.drop(\"nb_timestamps\", axis=1, inplace=True)\n",
    "df_features_slope.drop(\"nb_timestamps\", axis=1, inplace=True)\n",
    "\n",
    "assert(len(df_answers) == len(df_features_max))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NB_PARTICIPANTS = len(df_generals)\n",
    "print(\"We had {0} participants\".format(NB_PARTICIPANTS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df_generals.age, bins=range(10, 90, 5))\n",
    "plt.title(\"Age distribution\")\n",
    "plt.ylabel(\"Frequency\")\n",
    "plt.xlabel(\"Age\")\n",
    "plt.savefig(\"plot/age.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_generals.robotRealLife = df_generals.robotRealLife.replace({ 0: \"No\", 1: \"Yes\" })\n",
    "general = df_generals[[\"gender\", \"robotRealLife\"]]\n",
    "\n",
    "general_frequency = pd.crosstab(\n",
    "    [np.repeat(\"gender\", NB_PARTICIPANTS), general.gender],\n",
    "    [np.repeat(\"robotRealLife\", NB_PARTICIPANTS), general.robotRealLife],\n",
    "    margins=True\n",
    ")\n",
    "\n",
    "general_frequency.index.names = [None, None]\n",
    "general_frequency.columns.names = [None, None]\n",
    "\n",
    "general_frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_answers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_answers.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_features_slope.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_features_slope.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pearson correlation analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Aggregate overall survey scale\n",
    "overall_score = df_answers.mean(axis=1)\n",
    "\n",
    "# Aggregate answers for each subscale\n",
    "anthropomorphism_score = df_answers[ANTHROPOMORPHISM_COLUMNS].mean(axis=1)\n",
    "animacy_score = df_answers[ANIMACY_COLUMNS].mean(axis=1)\n",
    "likeability_score = df_answers[LIKEABILITY_COLUMNS].mean(axis=1)\n",
    "intelligence_score = df_answers[INTELLIGENCE_COLUMNS].mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_features_max.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_max = pd.DataFrame({\n",
    "    \"anthropomorphism\": df_features_max.corrwith(anthropomorphism_score),\n",
    "    \"animacy\": df_features_max.corrwith(animacy_score),\n",
    "    \"likeability\": df_features_max.corrwith(likeability_score),\n",
    "    \"intelligence\": df_features_max.corrwith(intelligence_score),\n",
    "    \"overall\": df_features_max.corrwith(overall_score)\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"max correlation\", corr_max.max().max())\n",
    "print(\"min correlation\", corr_max.min().min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,6))\n",
    "plt.title(\"Correlations of max-aggregated features with the survey response\", fontsize=16, pad=25)\n",
    "sns.heatmap(corr_max.round(2).T, vmin=-1, vmax=1, cmap=\"coolwarm\", square=True, annot=True)\n",
    "plt.savefig(\"plot/corr_features_max_survey.png\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_slope = pd.DataFrame({\n",
    "    \"anthropomorphism\": df_features_slope.corrwith(anthropomorphism_score),\n",
    "    \"animacy\": df_features_slope.corrwith(animacy_score),\n",
    "    \"likeability\": df_features_slope.corrwith(likeability_score),\n",
    "    \"intelligence\": df_features_slope.corrwith(intelligence_score),\n",
    "    \"overall\": df_features_slope.corrwith(overall_score)\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"max correlation\", corr_slope.max().max())\n",
    "print(\"min correlation\", corr_slope.min().min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,6))\n",
    "plt.title(\"Correlations of slope-aggregated features with the survey response\", fontsize=16, pad=25)\n",
    "sns.heatmap(corr_slope.round(2).T, vmin=-1, vmax=1, cmap=\"coolwarm\", square=True, annot=True)\n",
    "plt.savefig(\"plot/corr_features_slope_survey.png\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_features_max = df_features_max.corr()\n",
    "corr_features_slope = df_features_slope.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"max correlation\", corr_features_max.replace(1, 0).max().max())\n",
    "print(\"min correlation\", corr_features_max.replace(1, 0).min().min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(13, 11))\n",
    "plt.title(\"Pairwise correlations of max-aggregated features\", fontsize=20, pad=25)\n",
    "sns.heatmap(corr_features_max.round(2).T, vmin=-1, vmax=1, cmap=\"coolwarm\", square=True, annot=True)\n",
    "plt.savefig(\"plot/corr_pairwise_features_max.png\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(13, 11))\n",
    "plt.title(\"Pairwise correlations of slope-aggregated features\", fontsize=20, pad=25)\n",
    "sns.heatmap(corr_features_slope.round(2).T, vmin=-1, vmax=1, cmap=\"coolwarm\", square=True, annot=True)\n",
    "plt.savefig(\"plot/corr_pairwise_features_slope.png\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_features_max.columns.name = \"feature\"\n",
    "df_features_slope.columns.name = \"feature\"\n",
    "\n",
    "df_features_max_transformed = (\n",
    "    df_features_max\n",
    "    .drop(\"eyeClosure\", axis=1) # drop eyeClosure to make plot readable\n",
    "    .stack()\n",
    "    .to_frame(\"value\")\n",
    "    .reset_index()\n",
    ")\n",
    "\n",
    "df_features_slope_transformed = (\n",
    "    df_features_slope\n",
    "    .drop(\"eyeClosure\", axis=1) # drop eyeClosure to make plot readable\n",
    "    .stack()\n",
    "    .to_frame(\"value\")\n",
    "    .reset_index()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_features_slope_transformed.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# seed for reproducibility\n",
    "seed = 42\n",
    "\n",
    "plt.figure(figsize=(11, 4))\n",
    "sns.barplot(x=\"feature\", hue=\"video\", y=\"value\", data=df_features_max_transformed, seed=seed, errwidth=0.5)\n",
    "plt.title(\"Mean with 95% CI of max-aggregated features per video clip\", fontsize=16, pad=10)\n",
    "plt.xticks(rotation=90)\n",
    "plt.xlabel(None)\n",
    "plt.legend(loc=\"upper right\")\n",
    "plt.savefig(\"plot/mean_max_features_per_video.png\", bbox_inches=\"tight\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(11, 4))\n",
    "sns.barplot(x=\"feature\", hue=\"video\", y=\"value\", data=df_features_slope_transformed, seed=seed, errwidth=0.5)\n",
    "plt.title(\"Mean with 95% CI of slope-aggregated features per video clip\", fontsize=16, pad=25)\n",
    "plt.xticks(rotation=90)\n",
    "plt.xlabel(None)\n",
    "plt.savefig(\"plot/mean_slope_features_per_video.png\", bbox_inches=\"tight\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = pd.concat(\n",
    "    objs={\n",
    "        \"anthropomorphism\": anthropomorphism_score,\n",
    "        \"animacy\": animacy_score,\n",
    "        \"likeability\": likeability_score,\n",
    "        \"intelligence\": intelligence_score,\n",
    "        \"overall\": overall_score\n",
    "    },\n",
    "    axis=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores.columns.name = \"godspeed_scale\"\n",
    "\n",
    "scores_transformed = (\n",
    "    scores\n",
    "    .stack()\n",
    "    .to_frame(\"value\")\n",
    "    .reset_index()\n",
    ")\n",
    "\n",
    "\n",
    "scores_transformed.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(8, 5))\n",
    "sns.barplot(x=\"godspeed_scale\", hue=\"video\", y=\"value\", data=scores_transformed, seed=seed, errwidth=1)\n",
    "plt.title(\"Mean with 95% CI of overall and subscale survey responses per video clip\", fontsize=16, pad=25)\n",
    "plt.xlabel(None)\n",
    "plt.savefig(\"plot/mean_survey_per_video.png\", bbox_inches=\"tight\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LassoCV\n",
    "\n",
    "def lasso_cv(X, y, test_size=0.2, seed=42):\n",
    "    \"\"\"\n",
    "    Splits the data into training and test samples.\n",
    "    Fits the training samples to a Lasso linear model and\n",
    "    selects the best model by cross-validation.\n",
    "    Returns the R-squared coefficient of determination of\n",
    "    the test prediction.\n",
    "    \"\"\"\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=seed)\n",
    "    model = LassoCV(random_state=seed).fit(X_train, y_train)\n",
    "    r2 = model.score(X_test, y_test)\n",
    "    print(len([x for x in model.coef_ if x != 0]), \" nonzero coefficients out of 14\")\n",
    "    return r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_max = {}\n",
    "results_slope = {}\n",
    "results_mean = {}\n",
    "\n",
    "for survey_scale in scores.columns:\n",
    "    target = scores[survey_scale]\n",
    "    results_max[survey_scale] = lasso_cv(df_features_max, target)\n",
    "    results_slope[survey_scale] = lasso_cv(df_features_slope, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.from_dict(results_slope, orient='index', columns=[\"R-squared\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.from_dict(results_max, orient='index', columns=[\"R-squared\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame.from_dict(results_slope, orient='index', columns=[\"R-squared\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try to predict video_id from features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_features_max[CONSIDERED_FEATURES]\n",
    "y = df_features_max.index.get_level_values(1)\n",
    "n = 100\n",
    "err = []\n",
    "# repeat with different seed\n",
    "for i in range(n):\n",
    "    # split data into training and testing set\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=i)\n",
    "\n",
    "    gnb = GaussianNB()\n",
    "    y_pred = gnb.fit(X_train, y_train).predict(X_test)\n",
    "    err.append((y_test == y_pred).sum())\n",
    "print(\"Average accuracy: {0}%\".format(np.mean(err)/len(y_test) *100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_features_slope[CONSIDERED_FEATURES]\n",
    "y = df_features_max.index.get_level_values(1)\n",
    "n = 100\n",
    "err = []\n",
    "# repeat with different seed\n",
    "for i in range(n):\n",
    "    # split data into training and testing set\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=i)\n",
    "\n",
    "    gnb = GaussianNB()\n",
    "    y_pred = gnb.fit(X_train, y_train).predict(X_test)\n",
    "    err.append((y_test == y_pred).sum())\n",
    "print(\"Average accuracy: {0}%\".format(np.mean(err)/len(y_test) *100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try to predict video_id from answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_answers\n",
    "y = df_features_max.index.get_level_values(1)\n",
    "n = 100\n",
    "err = []\n",
    "# repeat with different seed\n",
    "for i in range(n):\n",
    "    # split data into training and testing set\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=i)\n",
    "\n",
    "    gnb = GaussianNB()\n",
    "    y_pred = gnb.fit(X_train, y_train).predict(X_test)\n",
    "    err.append((y_test == y_pred).sum())\n",
    "print(\"Average accuracy: {0}%\".format(np.mean(err)/len(y_test) *100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(20,10))\n",
    "\n",
    "feature_name = \"smile\"\n",
    "t = pd.read_csv(DATAFRAMES_PATH + \"df_\" + feature_name + \"_3.csv\")\n",
    "t.drop(\"video_id\",inplace=True, axis=1)\n",
    "t.set_index(\"user_id\", inplace=True)\n",
    "t[:15].T.plot()\n",
    "plt.legend(title=\"user_id\", loc='best', ncol=3, bbox_to_anchor=(1, 1, 0, 0))\n",
    "plt.ylabel(\"facial action score\")\n",
    "plt.title(\"Smiling of 15 participants while watching Sophia\")\n",
    "plt.savefig(\"plot/smile.png\", bbox_inches=\"tight\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Features for each video and user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_features_max.values[:,1:]\n",
    "pca = PCA(n_components=2)\n",
    "y = pca.fit_transform(X)\n",
    "vid1 = plt.scatter(y[::3][:,0], y[::3][:,1])\n",
    "vid2 = plt.scatter(y[1::3][:,0], y[1::3][:,1])\n",
    "vid3 = plt.scatter(y[2::3][:,0], y[2::3][:,1])\n",
    "\n",
    "plt.legend((vid1,vid2,vid3), video_names.values(), fontsize=8)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_features_slope.values[:,1:]\n",
    "pca = PCA(n_components=2)\n",
    "y = pca.fit_transform(X)\n",
    "vid1 = plt.scatter(y[::3][:,0], y[::3][:,1])\n",
    "vid2 = plt.scatter(y[1::3][:,0], y[1::3][:,1])\n",
    "vid3 = plt.scatter(y[2::3][:,0], y[2::3][:,1])\n",
    "\n",
    "plt.legend((vid1,vid2,vid3), video_names.values(), fontsize=8)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Answers for each video and user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_answers.values\n",
    "pca = PCA(n_components=2)\n",
    "y = pca.fit_transform(X)\n",
    "vid1 = plt.scatter(y[::3][:,0], y[::3][:,1])\n",
    "vid2 = plt.scatter(y[1::3][:,0], y[1::3][:,1])\n",
    "vid3 = plt.scatter(y[2::3][:,0], y[2::3][:,1])\n",
    "\n",
    "plt.legend((vid1,vid2,vid3), video_names.values(), fontsize=8)\n",
    "plt.title(\"PCA plot of overall survey response\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PCA().fit(X).explained_variance_ratio_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}