Process_AUT_GT_Maxime.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/massivetexts/llm_aut_study/blob/main/notebooks/Process_AUT_GT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eXqu-c24fsZs"
      },
      "source": [
        "This notebook collects, normalizes and processes AUT human judged ground truth from various sources. This is the data used in [Beyond Semantic Distance: Automated Scoring of Divergent Thinking Greatly Improves with Large Language Models](https://www.researchgate.net/publication/363456838_Beyond_Semantic_Distance_Automated_Scoring_of_Divergent_Thinking_Greatly_Improves_with_Large_Language_Models/stats).\n",
        "\n",
        "This is an assortment of studies, with different demographics, goals, and test setups. It is most appropriate for supervised learning of automated scoring, where we're not necessarily trying to learn about the participants but the raters in general."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sAfU9ApyfppM",
        "outputId": "c3202312-63b7-4ccd-ecf9-7404c6f620bb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: xlrd in c:\\users\\maxime\\anaconda3\\envs\\ml\\lib\\site-packages (2.0.1)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING: Ignoring invalid distribution -atplotlib (c:\\users\\maxime\\anaconda3\\envs\\ml\\lib\\site-packages)\n",
            "WARNING: Ignoring invalid distribution -atplotlib (c:\\users\\maxime\\anaconda3\\envs\\ml\\lib\\site-packages)\n",
            "WARNING: Ignoring invalid distribution -atplotlib (c:\\users\\maxime\\anaconda3\\envs\\ml\\lib\\site-packages)\n",
            "WARNING: Ignoring invalid distribution -atplotlib (c:\\users\\maxime\\anaconda3\\envs\\ml\\lib\\site-packages)\n",
            "WARNING: Ignoring invalid distribution -atplotlib (c:\\users\\maxime\\anaconda3\\envs\\ml\\lib\\site-packages)\n",
            "WARNING: Ignoring invalid distribution -atplotlib (c:\\users\\maxime\\anaconda3\\envs\\ml\\lib\\site-packages)\n",
            "c:\\Users\\Maxime\\anaconda3\\envs\\ml\\lib\\site-packages\\outdated\\utils.py:14: OutdatedPackageWarning: The package pingouin is out of date. Your version is 0.5.3, the latest is 0.5.4.\n",
            "Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.\n",
            "  return warn(\n"
          ]
        }
      ],
      "source": [
        "import dataclasses\n",
        "#@title import and setup defs\n",
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "import hashlib\n",
        "from pathlib import Path\n",
        "from sklearn.model_selection import train_test_split\n",
        "import json\n",
        "\n",
        "!pip install --upgrade xlrd\n",
        "!pip -qq install pingouin\n",
        "import pingouin as pg\n",
        "\n",
        "def simple_stats(df, rater_cols=False):\n",
        "    print(\"# of prompts\", len(df.prompt.unique()))\n",
        "    print(\"# of participants\", len(df.participant.unique()))\n",
        "    print(\"# of data points\", len(df))\n",
        "    print(\"Prompts\", df.prompt.unique())\n",
        "    if rater_cols:\n",
        "        print(\"# of raters\", len(rater_cols))\n",
        "        print(\"Intraclass correlation coefficients (report ICC2k)\")\n",
        "        x = df[['id']+rater_cols].melt(id_vars='id', var_name='rater', value_name='rating')\n",
        "        icc = pg.intraclass_corr(data=x, targets='id', raters='rater', ratings='rating', nan_policy='omit')\n",
        "        display(icc.round(2))\n",
        "\n",
        "include_rater_std = False #@param {type:'boolean'}\n",
        "returncols = ['type', 'src', 'question', 'prompt', 'response', 'id', 'target', 'participant', 'response_num']\n",
        "if include_rater_std:\n",
        "    returncols += ['rating_std']\n",
        "\n",
        "def prep_general(data, src, rater_cols=None, return_full=False,\n",
        "                 aggregate_scores=True,\n",
        "                 drop_noresponse=True, include_stats=True, round_adjust=False,\n",
        "                 include_rater_std=False, overwrite_q=False, inputrange=None):\n",
        "    ''' General cleaning that repeats for multiple datasets.\n",
        "\n",
        "    If aggregate_scores is False, this will (eventually) return original individual rater scores, rather than multi-rater.\n",
        "        I haven't implemented this disagregation yet because I'm not sure how well disagregated\n",
        "        scored could be used in a proper training, without data leakage.\n",
        "    '''\n",
        "    if not rater_cols:\n",
        "        # assume columns that say 'rater'\n",
        "        rater_cols = [col for col in data.columns if 'rater' in col.lower()]\n",
        "\n",
        "    print(\"Rater cols:\", rater_cols)\n",
        "\n",
        "    if drop_noresponse:\n",
        "        data = data[~data.response.isna()]\n",
        "\n",
        "    data['src'] = src\n",
        "    data['avg_rating'] = data[rater_cols].mean(1)\n",
        "    if round_adjust:\n",
        "        # add a tiny bit to avg rater to round in direction of median. Only need if there are tiebreakers\n",
        "        data['median_rating'] = data[rater_cols].median(1)\n",
        "        data['avg_rating'] = data.avg_rating + (data.median_rating - data.avg_rating).div(10**3)\n",
        "    if include_rater_std:\n",
        "        data['rating_std'] = data[rater_cols].std(1)\n",
        "\n",
        "    data['target'] = normalize_values(data.avg_rating, oldrange=inputrange)\n",
        "\n",
        "    missing = data.target.isna()\n",
        "    if missing.sum():\n",
        "        print(f'Dropping {missing.sum()} unrated items')\n",
        "        data = data[~missing]\n",
        "\n",
        "    data['participant'] = src + data['participant'].astype(str)\n",
        "\n",
        "    if 'type' not in data.columns:\n",
        "        print('assuming type=\"uses\"')\n",
        "        data['type'] = 'uses'\n",
        "\n",
        "    if ('question' not in data.columns) or overwrite_q:\n",
        "        try:\n",
        "            assert data.type.unique().tolist() == ['uses']\n",
        "        except:\n",
        "            raise Exception(\"can't infer question format for anything other than uses; please explicitly supply a 'question' column\")\n",
        "        data.loc[data.type=='uses', 'question'] = data.loc[data.type=='uses', 'prompt'].apply(lambda x: f\"What is a surprising use for a {x.upper()}?\")\n",
        "\n",
        "    idhash = (data.participant+data.response).apply(lambda x: hashlib.md5(x.encode('utf-8')).hexdigest()[:6])\n",
        "    # task can be a custom string that is used instead of prompt (e.g. 'g2_red' instead of 'red')\n",
        "    data['id'] = f'{src}_' + data['task' if 'task' in data.columns else 'prompt'].astype(str) + '-' + idhash\n",
        "\n",
        "    if include_stats:\n",
        "        simple_stats(data, rater_cols)\n",
        "\n",
        "    if return_full:\n",
        "        return data\n",
        "    else:\n",
        "        cols = ['type', 'src', 'question', 'prompt', 'response', 'id', 'target', 'participant', 'response_num']\n",
        "        if include_rater_std:\n",
        "            cols += ['rating_std']\n",
        "        return data[cols]\n",
        "\n",
        "datasets = dict()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "p6OHgu0tat-Q"
      },
      "outputs": [],
      "source": [
        "#base_dir = 'drive/MyDrive/Grants/MOTES/Data/aut_ground_truth' #@param { type: 'string' }\n",
        "#base_dir = Path(base_dir)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_7bna3Sx8A39",
        "outputId": "733784fb-01c3-4899-f5d3-52ee4018cb43"
      },
      "outputs": [],
      "source": [
        "# All AUT ICC2ks, to get a sense of reliability\n",
        "#x = np.array([0.85, 0.69, 0.79, 0.48, 0.67, 0.81, 0.67, 0.72])\n",
        "#np.mean(x), np.std(x).round(3), np.median(x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "cellView": "form",
        "id": "47HJ3XN7asiY"
      },
      "outputs": [],
      "source": [
        "#@title Normalization Settings\n",
        "#@markdown Ensure that all the datasets have the same scale\n",
        "scale_min = 1 #@param{type:'number'}\n",
        "scale_max = 5 #@param{type:'number'}\n",
        "#@markdown Round to nearest 0.1 by default.\n",
        "round_func = lambda x: x.round(1) #@param [\"lambda x: x.multiply(2).round(0).div(2)\", \"None\", \"lambda x: x.round(1)\"] {type:\"raw\"}\n",
        "\n",
        "def normalize_values(series, outrange=(1,5), oldrange=None,\n",
        "                     round_func=round_func):\n",
        "    min, max = outrange\n",
        "    if oldrange:\n",
        "        oldmin, oldmax = oldrange\n",
        "    else:\n",
        "        oldmin, oldmax = series.min(), series.max()\n",
        "\n",
        "    x = (series - oldmin)/(oldmax-oldmin)\n",
        "    x = min + (max-min)*x\n",
        "    if round_func:\n",
        "        # none of our data has even numbers of raters, so there shouldn't be need for rounding tiebreakers\n",
        "        # but for future proofing, add or subtract a tiny number randomly, so that rounding can go either way\n",
        "        modifier = (2*np.random.randint(0,2,size=x.shape)-1)/10**5\n",
        "        x = round_func(x+modifier)\n",
        "    return x\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zltQcwkXeRvk"
      },
      "source": [
        "## Normalize Dumas et al 2020\n",
        "\n",
        "Download from OSF."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2NxCzyyjeF1S",
        "outputId": "8870286f-a488-4502-db92-950956c7f54f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Rater cols: ['rater1', 'rater2', 'rater3', 'rater4']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 10\n",
            "# of participants 92\n",
            "# of data points 5490\n",
            "Prompts ['book' 'bottle' 'brick' 'fork' 'pants' 'rope' 'shoe' 'shovel' 'table'\n",
            " 'tire']\n",
            "# of raters 4\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.57</td>\n",
              "      <td>6.27</td>\n",
              "      <td>5376</td>\n",
              "      <td>16131</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.56, 0.58]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.58</td>\n",
              "      <td>7.78</td>\n",
              "      <td>5376</td>\n",
              "      <td>16128</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.49, 0.65]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.63</td>\n",
              "      <td>7.78</td>\n",
              "      <td>5376</td>\n",
              "      <td>16128</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.62, 0.64]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.84</td>\n",
              "      <td>6.27</td>\n",
              "      <td>5376</td>\n",
              "      <td>16131</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.83, 0.85]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.85</td>\n",
              "      <td>7.78</td>\n",
              "      <td>5376</td>\n",
              "      <td>16128</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.79, 0.88]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.87</td>\n",
              "      <td>7.78</td>\n",
              "      <td>5376</td>\n",
              "      <td>16128</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.87, 0.88]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Type              Description   ICC     F   df1    df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.57  6.27  5376  16131   0.0  [0.56, 0.58]\n",
              "1   ICC2     Single random raters  0.58  7.78  5376  16128   0.0  [0.49, 0.65]\n",
              "2   ICC3      Single fixed raters  0.63  7.78  5376  16128   0.0  [0.62, 0.64]\n",
              "3  ICC1k  Average raters absolute  0.84  6.27  5376  16131   0.0  [0.83, 0.85]\n",
              "4  ICC2k    Average random raters  0.85  7.78  5376  16128   0.0  [0.79, 0.88]\n",
              "5  ICC3k     Average fixed raters  0.87  7.78  5376  16128   0.0  [0.87, 0.88]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4180</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for a TIRE?</td>\n",
              "      <td>tire</td>\n",
              "      <td>gravel substitute</td>\n",
              "      <td>dod20_tire-30e5e6</td>\n",
              "      <td>3.0</td>\n",
              "      <td>dod2061</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3445</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for a FORK?</td>\n",
              "      <td>fork</td>\n",
              "      <td>hair straightener</td>\n",
              "      <td>dod20_fork-925d45</td>\n",
              "      <td>3.7</td>\n",
              "      <td>dod2048</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      type    src                              question prompt  \\\n",
              "4180  uses  dod20  What is a surprising use for a TIRE?   tire   \n",
              "3445  uses  dod20  What is a surprising use for a FORK?   fork   \n",
              "\n",
              "               response                 id  target participant  response_num  \n",
              "4180  gravel substitute  dod20_tire-30e5e6     3.0     dod2061             0  \n",
              "3445  hair straightener  dod20_fork-925d45     3.7     dod2048             0  "
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#!wget -q -O dod20.csv https://osf.io/download/u3yv4/\n",
        "\n",
        "src = 'dod20' \n",
        "paca = pd.read_csv('./../data/dod20.csv', index_col=0)\n",
        "paca = paca[~paca.response.str.contains('!!!')]\n",
        "datasets[src] = prep_general(paca, src, include_rater_std=include_rater_std)\n",
        "datasets[src].sample(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gIwAL3uqaikc"
      },
      "source": [
        "## Process Silvia et 2009\n",
        "\n",
        "Silvia, P. J., Nusbaum, E. C., Berg, C., Martin, C., & O'Connor, A. (2009). Openness to experience, plasticity, and creativity: Exploring lower-order, high-order, and interactive effects. Journal of Research in Personality, 43(6), 1087–1090. https://doi.org/10.1016/j.jrp.2009.04.015"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pM8plqKLe-Iv",
        "outputId": "f04ba20b-f0a0-499d-db20-3d3989d3185a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Rater cols: ['rater_1', 'rater_2', 'rater_3', 'rater_4']\n",
            "Dropping 10 unrated items\n",
            "assuming type=\"uses\"\n",
            "# of prompts 3\n",
            "# of participants 202\n",
            "# of data points 4099\n",
            "Prompts ['brick' 'knife' 'box']\n",
            "# of raters 4\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.33</td>\n",
              "      <td>2.97</td>\n",
              "      <td>4097</td>\n",
              "      <td>12294</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.31, 0.35]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.36</td>\n",
              "      <td>4.10</td>\n",
              "      <td>4097</td>\n",
              "      <td>12291</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.25, 0.45]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.44</td>\n",
              "      <td>4.10</td>\n",
              "      <td>4097</td>\n",
              "      <td>12291</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.42, 0.45]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.66</td>\n",
              "      <td>2.97</td>\n",
              "      <td>4097</td>\n",
              "      <td>12294</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.65, 0.68]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.69</td>\n",
              "      <td>4.10</td>\n",
              "      <td>4097</td>\n",
              "      <td>12291</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.57, 0.77]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.76</td>\n",
              "      <td>4.10</td>\n",
              "      <td>4097</td>\n",
              "      <td>12291</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.74, 0.77]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Type              Description   ICC     F   df1    df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.33  2.97  4097  12294   0.0  [0.31, 0.35]\n",
              "1   ICC2     Single random raters  0.36  4.10  4097  12291   0.0  [0.25, 0.45]\n",
              "2   ICC3      Single fixed raters  0.44  4.10  4097  12291   0.0  [0.42, 0.45]\n",
              "3  ICC1k  Average raters absolute  0.66  2.97  4097  12294   0.0  [0.65, 0.68]\n",
              "4  ICC2k    Average random raters  0.69  4.10  4097  12291   0.0  [0.57, 0.77]\n",
              "5  ICC3k     Average fixed raters  0.76  4.10  4097  12291   0.0  [0.74, 0.77]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#!wget -q -O silvia09.csv https://osf.io/download/qdrv8/\n",
        "src = 'snbmo09'\n",
        "data = pd.read_csv('./../data/silvia09.csv').rename(columns={'subject':'participant','response_order':'response_num'})\n",
        "data['prompt'] = data.task.apply(lambda x: x.split('_')[-1])\n",
        "datasets[src] = prep_general(data, src, include_rater_std=include_rater_std)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cTj8vh0wEc4j",
        "outputId": "f9fc87c9-c4d7-4079-c192-da8ce35c52f8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>131</th>\n",
              "      <td>uses</td>\n",
              "      <td>snbmo09</td>\n",
              "      <td>What is a surprising use for a BRICK?</td>\n",
              "      <td>brick</td>\n",
              "      <td>building</td>\n",
              "      <td>snbmo09_1_brick-455391</td>\n",
              "      <td>1.0</td>\n",
              "      <td>snbmo099</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     type      src                               question prompt  response  \\\n",
              "131  uses  snbmo09  What is a surprising use for a BRICK?  brick  building   \n",
              "\n",
              "                         id  target participant  response_num  \n",
              "131  snbmo09_1_brick-455391     1.0    snbmo099             2  "
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "datasets[src].sample(1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jn8ziOcamzaH"
      },
      "source": [
        "## Process Hass 2017\n",
        "\n",
        "Hass, R. W. (2017). Semantic search during divergent thinking. Cognition, 166, 344–357. https://doi.org/10.1016/j.cognition.2017.05.039\n",
        "\n",
        "Data at https://osf.io/ng598.\n",
        "\n",
        "This study looked at uses for *bottle* and *brick*. There were 54 participants after data cleaning.\n",
        "\n",
        "Rating was on a 5-point scale. For verification, their reported inter-rater reliability was ICC2k was 0.80 for brick and 0.78 for bottle, which is about what we see below.\n",
        "\n",
        "The rating data was stoplisted, so I need to reconstruct the original responses here."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OJaYB4X6mtTn"
      },
      "outputs": [],
      "source": [
        "#!wget -q -O hass17ratings.xslx https://osf.io/download/mcykr/ # rater scores\n",
        "#!wget -q -O hass17responses1.xslx https://osf.io/download/27bx8/ # responses 1\n",
        "#!wget -q -O hass17responses2.xslx https://osf.io/download/rzvyd/ # responses 2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "c2uGiyokCtkG"
      },
      "outputs": [],
      "source": [
        "src = 'hass17'\n",
        "all_ratings = []\n",
        "for sheet, prompt in [('br_exp1', 'brick'),('br_exp2', 'brick'),('bot_exp1', 'bottle'),('bot_exp2', 'bottle')]:\n",
        "    data = pd.read_excel('./../data/hass17ratings.xlsx', sheet_name=sheet) #.rename(columns={'subject':'participant','response_order':'response_num'}) # typo here: xls instead of xlsx\n",
        "    data['prompt'] = prompt\n",
        "    all_ratings.append(data)\n",
        "hassratings = pd.concat(all_ratings).rename(columns={'response':'cleaned'})\n",
        "participants = pd.concat([pd.read_excel('./../data/hass17responses1.xls'), pd.read_excel('./../data/hass17responses2.xlsx')]) # typo here: xls instead of xlsx"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 451
        },
        "id": "FMGwqsjhDHgY",
        "outputId": "a7fd7b11-6057-4444-b2c7-b53e00e75e89"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to\n",
            "[nltk_data]     C:\\Users\\Maxime\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to\n",
            "[nltk_data]     C:\\Users\\Maxime\\AppData\\Roaming\\nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Rater cols: ['r1', 'r2', 'r3']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 2\n",
            "# of participants 57\n",
            "# of data points 1093\n",
            "Prompts ['bottle' 'brick']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.55</td>\n",
              "      <td>4.68</td>\n",
              "      <td>794</td>\n",
              "      <td>1590</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.51, 0.59]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.56</td>\n",
              "      <td>5.04</td>\n",
              "      <td>794</td>\n",
              "      <td>1588</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.5, 0.6]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.57</td>\n",
              "      <td>5.04</td>\n",
              "      <td>794</td>\n",
              "      <td>1588</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.54, 0.61]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.79</td>\n",
              "      <td>4.68</td>\n",
              "      <td>794</td>\n",
              "      <td>1590</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.76, 0.81]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.79</td>\n",
              "      <td>5.04</td>\n",
              "      <td>794</td>\n",
              "      <td>1588</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.75, 0.82]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.80</td>\n",
              "      <td>5.04</td>\n",
              "      <td>794</td>\n",
              "      <td>1588</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.78, 0.82]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Type              Description   ICC     F  df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.55  4.68  794  1590   0.0  [0.51, 0.59]\n",
              "1   ICC2     Single random raters  0.56  5.04  794  1588   0.0    [0.5, 0.6]\n",
              "2   ICC3      Single fixed raters  0.57  5.04  794  1588   0.0  [0.54, 0.61]\n",
              "3  ICC1k  Average raters absolute  0.79  4.68  794  1590   0.0  [0.76, 0.81]\n",
              "4  ICC2k    Average random raters  0.79  5.04  794  1588   0.0  [0.75, 0.82]\n",
              "5  ICC3k     Average fixed raters  0.80  5.04  794  1588   0.0  [0.78, 0.82]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# melt original responses to long, reconstructe the cleaned columns, then join with ratings\n",
        "long_part = participants.melt(id_vars='ID', value_name='response').rename(columns={'ID':'participant'})\n",
        "long_part = long_part[long_part.variable.str.contains('resp') & ~long_part.variable.str.contains('time')].dropna()\n",
        "long_part[['prompt', 'response_num']] = long_part.variable.str.split('_', expand=True)\n",
        "long_part.loc[long_part.prompt.str.contains('resp1'), 'prompt'] = 'bottle'\n",
        "long_part.loc[long_part.prompt.str.contains('resp2'), 'prompt'] = 'brick'\n",
        "long_part.sample(10)\n",
        "\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "stops = stopwords.words('english')\n",
        "# not sure which list the study used, so just adjust based on testing\n",
        "stops += ['could']\n",
        "stops = [w for w in stops if w not in ['can']]\n",
        "stops = set(stops)\n",
        "\n",
        "def stop_clean(x):\n",
        "    x = x.lower()\n",
        "    x = x.replace('i.e.', 'e') # quirk of the tokenization in original study\n",
        "    for c in list(\"/\\\\'-()\"):\n",
        "        x = x.replace(c, '')\n",
        "    words = [word for word in word_tokenize(x) if word not in stops]\n",
        "    return \" \".join(words)\n",
        "\n",
        "long_part['cleaned'] = long_part.response.apply(stop_clean)\n",
        "hass07 = long_part.merge(hassratings, how='left', on=['prompt', 'cleaned'])\n",
        "datasets[src] = prep_general(hass07, src, rater_cols=['r1','r2','r3'], include_rater_std=include_rater_std)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vJqyd2UHDCMZ"
      },
      "source": [
        "## Process Silvia et al Data 2008\n",
        "\n",
        "From https://osf.io/dh7ey, this was the order of creativity tasks:\n",
        "\n",
        "1. Please list all of the creative, unusual uses for a brick that you can think of.\n",
        "2. Please list all of the creative, unusual instances of things that are round that you can think of.\n",
        "3. Imagine that people no longer needed to sleep. Please list creative, unusual consequences that would follow.\n",
        "4. Please list all of the creative, unusual uses for a knife that you can think of.\n",
        "5. Please list all of the creative, unusual instances of things that will make a noise that you can think of.\n",
        "6. Imagine that everyone shrank to 12 inches tall. Please list creative, unusual consequences that would follow.\n",
        "\n",
        "Numbers 1 and 4 are AUT.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7jNMs7mcDh_A",
        "outputId": "8e5e5750-81fa-46ec-dee0-5826d80328c1"
      },
      "outputs": [],
      "source": [
        "#!pip -q install pyreadstat\n",
        "#!wget -q -O SilvaStudy2.zip https://files.osf.io/v1/resources/4ketx/providers/osfstorage/5dd70d1f83135e000ec3c242/?zip=\n",
        "#!unzip -o SilvaStudy2.zip\n",
        "import pyreadstat"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vhVizY5jg1v6",
        "outputId": "3ffe6030-2ffc-48e0-a6cf-e975594a75e3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Rater cols: ['rater1', 'rater2', 'rater3']\n",
            "Dropping 37 unrated items\n",
            "# of prompts 6\n",
            "# of participants 242\n",
            "# of data points 11490\n",
            "Prompts ['brick' 'round' 'no sleep' 'knife' 'noise' 'shrank']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.12</td>\n",
              "      <td>1.40</td>\n",
              "      <td>11488</td>\n",
              "      <td>22978</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.11, 0.13]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.20</td>\n",
              "      <td>2.16</td>\n",
              "      <td>11488</td>\n",
              "      <td>22976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.09, 0.3]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.28</td>\n",
              "      <td>2.16</td>\n",
              "      <td>11488</td>\n",
              "      <td>22976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.27, 0.29]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.29</td>\n",
              "      <td>1.40</td>\n",
              "      <td>11488</td>\n",
              "      <td>22978</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.26, 0.31]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.43</td>\n",
              "      <td>2.16</td>\n",
              "      <td>11488</td>\n",
              "      <td>22976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.22, 0.57]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.54</td>\n",
              "      <td>2.16</td>\n",
              "      <td>11488</td>\n",
              "      <td>22976</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.52, 0.55]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Type              Description   ICC     F    df1    df2  pval  \\\n",
              "0   ICC1   Single raters absolute  0.12  1.40  11488  22978   0.0   \n",
              "1   ICC2     Single random raters  0.20  2.16  11488  22976   0.0   \n",
              "2   ICC3      Single fixed raters  0.28  2.16  11488  22976   0.0   \n",
              "3  ICC1k  Average raters absolute  0.29  1.40  11488  22978   0.0   \n",
              "4  ICC2k    Average random raters  0.43  2.16  11488  22976   0.0   \n",
              "5  ICC3k     Average fixed raters  0.54  2.16  11488  22976   0.0   \n",
              "\n",
              "          CI95%  \n",
              "0  [0.11, 0.13]  \n",
              "1   [0.09, 0.3]  \n",
              "2  [0.27, 0.29]  \n",
              "3  [0.26, 0.31]  \n",
              "4  [0.22, 0.57]  \n",
              "5  [0.52, 0.55]  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Testing stats just for AUT (burczak et al reported ICC2k as 0.48)\n",
            "Rater cols: ['rater1', 'rater2', 'rater3']\n",
            "# of prompts 2\n",
            "# of participants 241\n",
            "# of data points 3425\n",
            "Prompts ['brick' 'knife']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.12</td>\n",
              "      <td>1.42</td>\n",
              "      <td>3424</td>\n",
              "      <td>6850</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.1, 0.15]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.23</td>\n",
              "      <td>2.76</td>\n",
              "      <td>3424</td>\n",
              "      <td>6848</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.06, 0.39]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.37</td>\n",
              "      <td>2.76</td>\n",
              "      <td>3424</td>\n",
              "      <td>6848</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.35, 0.39]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.30</td>\n",
              "      <td>1.42</td>\n",
              "      <td>3424</td>\n",
              "      <td>6850</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.26, 0.34]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.48</td>\n",
              "      <td>2.76</td>\n",
              "      <td>3424</td>\n",
              "      <td>6848</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.15, 0.65]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.64</td>\n",
              "      <td>2.76</td>\n",
              "      <td>3424</td>\n",
              "      <td>6848</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.62, 0.66]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Type              Description   ICC     F   df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.12  1.42  3424  6850   0.0   [0.1, 0.15]\n",
              "1   ICC2     Single random raters  0.23  2.76  3424  6848   0.0  [0.06, 0.39]\n",
              "2   ICC3      Single fixed raters  0.37  2.76  3424  6848   0.0  [0.35, 0.39]\n",
              "3  ICC1k  Average raters absolute  0.30  1.42  3424  6850   0.0  [0.26, 0.34]\n",
              "4  ICC2k    Average random raters  0.48  2.76  3424  6848   0.0  [0.15, 0.65]\n",
              "5  ICC3k     Average fixed raters  0.64  2.76  3424  6848   0.0  [0.62, 0.66]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6721</th>\n",
              "      <td>uses</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What is a surprising use for a KNIFE?</td>\n",
              "      <td>knife</td>\n",
              "      <td>pick a lock</td>\n",
              "      <td>setal08_4.0-9cc70e</td>\n",
              "      <td>2.0</td>\n",
              "      <td>setal08143</td>\n",
              "      <td>5.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>991</th>\n",
              "      <td>uses</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What is a surprising use for a KNIFE?</td>\n",
              "      <td>knife</td>\n",
              "      <td>open a can</td>\n",
              "      <td>setal08_4.0-628b62</td>\n",
              "      <td>2.0</td>\n",
              "      <td>setal0823</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      type      src                               question prompt  \\\n",
              "6721  uses  setal08  What is a surprising use for a KNIFE?  knife   \n",
              "991   uses  setal08  What is a surprising use for a KNIFE?  knife   \n",
              "\n",
              "         response                  id  target participant  response_num  \n",
              "6721  pick a lock  setal08_4.0-9cc70e     2.0  setal08143           5.0  \n",
              "991    open a can  setal08_4.0-628b62     2.0   setal0823           2.0  "
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "src = 'setal08'\n",
        "promptref = ['brick', 'round', 'no sleep', 'knife', 'noise', 'shrank']\n",
        "qref = [\"What is a surprising use for a BRICK?\",\n",
        "        \"What is a surprising thing that is ROUND?\",\n",
        "        \"What would be a surprising consequence if PEOPLE NEEDED NO SLEEP?\",\n",
        "        \"What is a surprising use for a KNIFE?\",\n",
        "        \"What is a surprising thing that makes a NOISE?\",\n",
        "        \"What would be a surprising consequence if EVERYONE SHRANK TO 12 INCHES TALL?\"]\n",
        "typeref = ['uses', 'instances', 'consequences', 'uses', 'instances', 'consequences']\n",
        "qref_dict = dict(zip(promptref, qref))\n",
        "\n",
        "df, meta = pyreadstat.read_sav('./../data/silvia_2008/DT_Responses_PACA_2008_Study_2.sav')\n",
        "df = df.rename(columns={'subject':'participant', 'order':'response_num'})\n",
        "df['prompt'] = df.task.apply(lambda x: promptref[int(x)-1])\n",
        "df['type'] = df.task.apply(lambda x: typeref[int(x)-1])\n",
        "df['participant'] = df['participant'].astype(int)\n",
        "df['question'] = df.prompt.apply(lambda x: qref_dict[x])\n",
        "df = prep_general(df, src, include_stats=True, return_full=True,\n",
        "                  include_rater_std=include_rater_std)\n",
        "# doublecheck - burczak reported ICC2k as 0.48 (for the s)\n",
        "print(\"Testing stats just for AUT (burczak et al reported ICC2k as 0.48)\")\n",
        "prep_general(df.query(\"type=='uses'\"), src, include_stats=True)\n",
        "datasets[src] = df[returncols]\n",
        "datasets[src].sample(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UH3yOwIon4Ky"
      },
      "source": [
        "## Process Hofelich Mohr, Sell, and Lindsay\n",
        "\n",
        "Doublecheck ICC2k - buczak paper had icc2k=0.67"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JHgvPMmynu6i"
      },
      "outputs": [],
      "source": [
        "#!wget -qO hofelich.zip https://conservancy.umn.edu/bitstream/handle/11299/172116/HMSL_CSV%20Data%20Files.zip?sequence=28&isAllowed=y\n",
        "#!unzip -qo hofelich.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BaeZVSf_oCSI",
        "outputId": "b07e9f80-09c3-4a30-b447-93ffda9b4a90"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Rater cols: ['J1_Rating', 'J2_Rating', 'J3_Rating', 'J4_Rating']\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\Maxime\\AppData\\Local\\Temp\\ipykernel_21688\\2386548369.py:51: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['src'] = src\n",
            "C:\\Users\\Maxime\\AppData\\Local\\Temp\\ipykernel_21688\\2386548369.py:52: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  data['avg_rating'] = data[rater_cols].mean(1)\n"
          ]
        },
        {
          "ename": "TypeError",
          "evalue": "loop of ufunc does not support argument 0 of type float which has no callable rint method",
          "output_type": "error",
          "traceback": [
            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[1;31mAttributeError\u001b[0m: 'float' object has no attribute 'rint'",
            "\nThe above exception was the direct cause of the following exception:\n",
            "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "Cell \u001b[1;32mIn [42], line 8\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[38;5;66;03m# remove three ratings that are 11\u001b[39;00m\n\u001b[0;32m      6\u001b[0m df[rater_cols] \u001b[38;5;241m=\u001b[39m df[rater_cols]\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;241m11\u001b[39m, pd\u001b[38;5;241m.\u001b[39mNA)\n\u001b[1;32m----> 8\u001b[0m datasets[src] \u001b[38;5;241m=\u001b[39m \u001b[43mprep_general\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msrc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrater_cols\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrater_cols\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minclude_rater_std\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minclude_rater_std\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      9\u001b[0m datasets[src]\u001b[38;5;241m.\u001b[39msample(\u001b[38;5;241m2\u001b[39m)\n\u001b[0;32m     10\u001b[0m df\u001b[38;5;241m.\u001b[39mhead()\n",
            "Cell \u001b[1;32mIn [1], line 60\u001b[0m, in \u001b[0;36mprep_general\u001b[1;34m(data, src, rater_cols, return_full, aggregate_scores, drop_noresponse, include_stats, round_adjust, include_rater_std, overwrite_q, inputrange)\u001b[0m\n\u001b[0;32m     57\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m include_rater_std:\n\u001b[0;32m     58\u001b[0m     data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrating_std\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m data[rater_cols]\u001b[38;5;241m.\u001b[39mstd(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m---> 60\u001b[0m data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtarget\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[43mnormalize_values\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mavg_rating\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moldrange\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputrange\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     62\u001b[0m missing \u001b[38;5;241m=\u001b[39m data\u001b[38;5;241m.\u001b[39mtarget\u001b[38;5;241m.\u001b[39misna()\n\u001b[0;32m     63\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m missing\u001b[38;5;241m.\u001b[39msum():\n",
            "Cell \u001b[1;32mIn [2], line 22\u001b[0m, in \u001b[0;36mnormalize_values\u001b[1;34m(series, outrange, oldrange, round_func)\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m round_func:\n\u001b[0;32m     19\u001b[0m     \u001b[38;5;66;03m# none of our data has even numbers of raters, so there shouldn't be need for rounding tiebreakers\u001b[39;00m\n\u001b[0;32m     20\u001b[0m     \u001b[38;5;66;03m# but for future proofing, add or subtract a tiny number randomly, so that rounding can go either way\u001b[39;00m\n\u001b[0;32m     21\u001b[0m     modifier \u001b[38;5;241m=\u001b[39m (\u001b[38;5;241m2\u001b[39m\u001b[38;5;241m*\u001b[39mnp\u001b[38;5;241m.\u001b[39mrandom\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m0\u001b[39m,\u001b[38;5;241m2\u001b[39m,size\u001b[38;5;241m=\u001b[39mx\u001b[38;5;241m.\u001b[39mshape)\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m/\u001b[39m\u001b[38;5;241m10\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m5\u001b[39m\n\u001b[1;32m---> 22\u001b[0m     x \u001b[38;5;241m=\u001b[39m \u001b[43mround_func\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43mmodifier\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     23\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m x\n",
            "Cell \u001b[1;32mIn [2], line 6\u001b[0m, in \u001b[0;36m<lambda>\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m      4\u001b[0m scale_max \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m5\u001b[39m \u001b[38;5;66;03m#@param{type:'number'}\u001b[39;00m\n\u001b[0;32m      5\u001b[0m \u001b[38;5;66;03m#@markdown Round to nearest 0.1 by default.\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m round_func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mlambda\u001b[39;00m x: \u001b[43mx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mround\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m#@param [\"lambda x: x.multiply(2).round(0).div(2)\", \"None\", \"lambda x: x.round(1)\"] {type:\"raw\"}\u001b[39;00m\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mnormalize_values\u001b[39m(series, outrange\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m1\u001b[39m,\u001b[38;5;241m5\u001b[39m), oldrange\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m      9\u001b[0m                      round_func\u001b[38;5;241m=\u001b[39mround_func):\n\u001b[0;32m     10\u001b[0m     \u001b[38;5;28mmin\u001b[39m, \u001b[38;5;28mmax\u001b[39m \u001b[38;5;241m=\u001b[39m outrange\n",
            "File \u001b[1;32mc:\\Users\\Maxime\\anaconda3\\envs\\ml\\lib\\site-packages\\pandas\\core\\series.py:2688\u001b[0m, in \u001b[0;36mSeries.round\u001b[1;34m(self, decimals, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2656\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m   2657\u001b[0m \u001b[38;5;124;03mRound each value in a Series to the given number of decimals.\u001b[39;00m\n\u001b[0;32m   2658\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   2685\u001b[0m \u001b[38;5;124;03mdtype: float64\u001b[39;00m\n\u001b[0;32m   2686\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m   2687\u001b[0m nv\u001b[38;5;241m.\u001b[39mvalidate_round(args, kwargs)\n\u001b[1;32m-> 2688\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_values\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mround\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecimals\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   2689\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_constructor(result, index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindex, copy\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\u001b[38;5;241m.\u001b[39m__finalize__(\n\u001b[0;32m   2690\u001b[0m     \u001b[38;5;28mself\u001b[39m, method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mround\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   2691\u001b[0m )\n\u001b[0;32m   2693\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
            "\u001b[1;31mTypeError\u001b[0m: loop of ufunc does not support argument 0 of type float which has no callable rint method"
          ]
        }
      ],
      "source": [
        "src = 'hmsl'\n",
        "df = pd.read_csv(f'./../data/hofelich/HMSL_Originality_scores_all.csv')\n",
        "df = df.rename(columns={'Item': 'prompt', 'QLogin_1':'participant'})\n",
        "rater_cols = ['J1_Rating','J2_Rating','J3_Rating','J4_Rating']\n",
        "# remove three ratings that are 11\n",
        "df[rater_cols] = df[rater_cols].replace(11, pd.NA)\n",
        "\n",
        "datasets[src] = prep_general(df, src, rater_cols=rater_cols, include_rater_std=include_rater_std)\n",
        "datasets[src].sample(2)\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VgNoUTqg_dQb"
      },
      "source": [
        "## Process dataset used by Beaty and Johnson\n",
        "\n",
        "From SemDis paper:\n",
        "\n",
        "- Study 1 was re-analysis of AUT responses from Beaty et al., 2018 to see if ensemble approaches work better. Two tests: `box` and `rope`\n",
        "   - according to their paper, using additive composition was slightly negative correlation, while multiplicative 'results revealed a large correlation between latent semantic distance and human ratings:$r=.91$, p<.001'. This uses a model that weighs the factors, but is (I think) tailored to the dataset without held out data.\n",
        "\n",
        "- Study 2 was re-analysis of results from Silvia et al. 2017, also on box and rope\n",
        "- Study 3 was brick - yet again - via Beaty and Silvia 2012\n",
        "- Study 4 and 5- Heinen and Johnson (2018) - were noun matching, not relevant here"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "a4b_fi1y_SdN"
      },
      "outputs": [],
      "source": [
        "#!wget -qO beaty.zip https://files.osf.io/v1/resources/gz4fc/providers/osfstorage/5e45b6c73e86a800be6e662e/?zip=\n",
        "#!unzip -qo beaty.zip\n",
        "files = sorted(list(Path('.').glob('./../data/Data/Study*/*data_long.xlsx')))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v3xBdaTnSktv",
        "outputId": "282c1875-5a7a-4fab-f975-b03ca51a421d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "-----------BETAL18------------\n",
            "Rater cols: ['rater1', 'rater2', 'rater3', 'rater4']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 2\n",
            "# of participants 171\n",
            "# of data points 2918\n",
            "Prompts ['box' 'rope']\n",
            "# of raters 4\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.52</td>\n",
              "      <td>5.26</td>\n",
              "      <td>2894</td>\n",
              "      <td>8685</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.5, 0.53]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.52</td>\n",
              "      <td>6.18</td>\n",
              "      <td>2894</td>\n",
              "      <td>8682</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.46, 0.58]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.56</td>\n",
              "      <td>6.18</td>\n",
              "      <td>2894</td>\n",
              "      <td>8682</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.55, 0.58]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.81</td>\n",
              "      <td>5.26</td>\n",
              "      <td>2894</td>\n",
              "      <td>8685</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.8, 0.82]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.82</td>\n",
              "      <td>6.18</td>\n",
              "      <td>2894</td>\n",
              "      <td>8682</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.77, 0.85]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.84</td>\n",
              "      <td>6.18</td>\n",
              "      <td>2894</td>\n",
              "      <td>8682</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.83, 0.85]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Type              Description   ICC     F   df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.52  5.26  2894  8685   0.0   [0.5, 0.53]\n",
              "1   ICC2     Single random raters  0.52  6.18  2894  8682   0.0  [0.46, 0.58]\n",
              "2   ICC3      Single fixed raters  0.56  6.18  2894  8682   0.0  [0.55, 0.58]\n",
              "3  ICC1k  Average raters absolute  0.81  5.26  2894  8685   0.0   [0.8, 0.82]\n",
              "4  ICC2k    Average random raters  0.82  6.18  2894  8682   0.0  [0.77, 0.85]\n",
              "5  ICC3k     Average fixed raters  0.84  6.18  2894  8682   0.0  [0.83, 0.85]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "------------SNB17-------------\n",
            "Rater cols: ['rater1', 'rater2', 'rater3']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 2\n",
            "# of participants 142\n",
            "# of data points 2372\n",
            "Prompts ['box' 'rope']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.38</td>\n",
              "      <td>2.82</td>\n",
              "      <td>2352</td>\n",
              "      <td>4706</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.35, 0.4]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.40</td>\n",
              "      <td>3.57</td>\n",
              "      <td>2352</td>\n",
              "      <td>4704</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.29, 0.5]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.46</td>\n",
              "      <td>3.57</td>\n",
              "      <td>2352</td>\n",
              "      <td>4704</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.44, 0.49]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.65</td>\n",
              "      <td>2.82</td>\n",
              "      <td>2352</td>\n",
              "      <td>4706</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.62, 0.67]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.67</td>\n",
              "      <td>3.57</td>\n",
              "      <td>2352</td>\n",
              "      <td>4704</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.55, 0.75]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.72</td>\n",
              "      <td>3.57</td>\n",
              "      <td>2352</td>\n",
              "      <td>4704</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.7, 0.74]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Type              Description   ICC     F   df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.38  2.82  2352  4706   0.0   [0.35, 0.4]\n",
              "1   ICC2     Single random raters  0.40  3.57  2352  4704   0.0   [0.29, 0.5]\n",
              "2   ICC3      Single fixed raters  0.46  3.57  2352  4704   0.0  [0.44, 0.49]\n",
              "3  ICC1k  Average raters absolute  0.65  2.82  2352  4706   0.0  [0.62, 0.67]\n",
              "4  ICC2k    Average random raters  0.67  3.57  2352  4704   0.0  [0.55, 0.75]\n",
              "5  ICC3k     Average fixed raters  0.72  3.57  2352  4704   0.0   [0.7, 0.74]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "-------------BS12-------------\n",
            "Rater cols: ['br_rater1', 'br_rater2', 'br_rater3']\n",
            "assuming type=\"uses\"\n",
            "# of prompts 1\n",
            "# of participants 133\n",
            "# of data points 1807\n",
            "Prompts ['brick']\n",
            "# of raters 3\n",
            "Intraclass correlation coefficients (report ICC2k)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Type</th>\n",
              "      <th>Description</th>\n",
              "      <th>ICC</th>\n",
              "      <th>F</th>\n",
              "      <th>df1</th>\n",
              "      <th>df2</th>\n",
              "      <th>pval</th>\n",
              "      <th>CI95%</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ICC1</td>\n",
              "      <td>Single raters absolute</td>\n",
              "      <td>0.43</td>\n",
              "      <td>3.24</td>\n",
              "      <td>1800</td>\n",
              "      <td>3602</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.4, 0.46]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ICC2</td>\n",
              "      <td>Single random raters</td>\n",
              "      <td>0.46</td>\n",
              "      <td>4.46</td>\n",
              "      <td>1800</td>\n",
              "      <td>3600</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.3, 0.57]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ICC3</td>\n",
              "      <td>Single fixed raters</td>\n",
              "      <td>0.54</td>\n",
              "      <td>4.46</td>\n",
              "      <td>1800</td>\n",
              "      <td>3600</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.51, 0.56]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ICC1k</td>\n",
              "      <td>Average raters absolute</td>\n",
              "      <td>0.69</td>\n",
              "      <td>3.24</td>\n",
              "      <td>1800</td>\n",
              "      <td>3602</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.67, 0.72]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ICC2k</td>\n",
              "      <td>Average random raters</td>\n",
              "      <td>0.72</td>\n",
              "      <td>4.46</td>\n",
              "      <td>1800</td>\n",
              "      <td>3600</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.56, 0.8]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ICC3k</td>\n",
              "      <td>Average fixed raters</td>\n",
              "      <td>0.78</td>\n",
              "      <td>4.46</td>\n",
              "      <td>1800</td>\n",
              "      <td>3600</td>\n",
              "      <td>0.0</td>\n",
              "      <td>[0.76, 0.79]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Type              Description   ICC     F   df1   df2  pval         CI95%\n",
              "0   ICC1   Single raters absolute  0.43  3.24  1800  3602   0.0   [0.4, 0.46]\n",
              "1   ICC2     Single random raters  0.46  4.46  1800  3600   0.0   [0.3, 0.57]\n",
              "2   ICC3      Single fixed raters  0.54  4.46  1800  3600   0.0  [0.51, 0.56]\n",
              "3  ICC1k  Average raters absolute  0.69  3.24  1800  3602   0.0  [0.67, 0.72]\n",
              "4  ICC2k    Average random raters  0.72  4.46  1800  3600   0.0   [0.56, 0.8]\n",
              "5  ICC3k     Average fixed raters  0.78  4.46  1800  3600   0.0  [0.76, 0.79]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "for fname,src in zip(files[:3], ['betal18','snb17', 'bs12']):\n",
        "    print(src.upper().center(30, '-'))\n",
        "    df = pd.read_excel(fname)\n",
        "\n",
        "    df = df.rename(columns={'id':'participant', 'item':'prompt'})\n",
        "    df['response_num'] = None\n",
        "    datasets[src] = prep_general(df, src, include_rater_std=include_rater_std)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "laa40OuiNV2X"
      },
      "source": [
        "### Doublecheck Beaty data\n",
        "Confirm that this data is what's seen in table 1 of paper - this is correct."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kl7M4fLvLzt6",
        "outputId": "fd68d1a6-be51-435c-cd5a-a99b1f525d5f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>b_rater1</th>\n",
              "      <th>b_rater2</th>\n",
              "      <th>b_rater3</th>\n",
              "      <th>b_rater4</th>\n",
              "      <th>b_cbowu</th>\n",
              "      <th>b_cbows</th>\n",
              "      <th>b_cboww</th>\n",
              "      <th>b_tasa</th>\n",
              "      <th>b_glov</th>\n",
              "      <th>r_rater1</th>\n",
              "      <th>r_rater2</th>\n",
              "      <th>r_rater3</th>\n",
              "      <th>r_rater4</th>\n",
              "      <th>r_cbowu</th>\n",
              "      <th>r_cbows</th>\n",
              "      <th>r_cboww</th>\n",
              "      <th>r_tasa</th>\n",
              "      <th>r_glov</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>b_rater1</th>\n",
              "      <td>1.00</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.73</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.30</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.27</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.25</td>\n",
              "      <td>0.37</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_rater2</th>\n",
              "      <td>0.65</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.59</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.27</td>\n",
              "      <td>0.24</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.42</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_rater3</th>\n",
              "      <td>0.54</td>\n",
              "      <td>0.59</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.73</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.35</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_rater4</th>\n",
              "      <td>0.73</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.73</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_cbowu</th>\n",
              "      <td>0.46</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.45</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.80</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.28</td>\n",
              "      <td>0.43</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_cbows</th>\n",
              "      <td>0.32</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.76</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.69</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.23</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.34</td>\n",
              "      <td>0.44</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.40</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_cboww</th>\n",
              "      <td>0.43</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.69</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.71</td>\n",
              "      <td>0.09</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_tasa</th>\n",
              "      <td>0.30</td>\n",
              "      <td>0.27</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.41</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.31</td>\n",
              "      <td>-0.05</td>\n",
              "      <td>0.06</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.15</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>b_glov</th>\n",
              "      <td>0.26</td>\n",
              "      <td>0.24</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.80</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.71</td>\n",
              "      <td>0.31</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_rater1</th>\n",
              "      <td>0.29</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.09</td>\n",
              "      <td>-0.05</td>\n",
              "      <td>0.07</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_rater2</th>\n",
              "      <td>0.27</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.06</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.65</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.75</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.34</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_rater3</th>\n",
              "      <td>0.36</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.23</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.75</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.53</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_rater4</th>\n",
              "      <td>0.37</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.76</td>\n",
              "      <td>0.87</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_cbowu</th>\n",
              "      <td>0.37</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.54</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_cbows</th>\n",
              "      <td>0.32</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.34</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.38</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.84</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.81</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_cboww</th>\n",
              "      <td>0.36</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.44</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.15</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.34</td>\n",
              "      <td>0.53</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.87</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.87</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_tasa</th>\n",
              "      <td>0.25</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.28</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.60</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>r_glov</th>\n",
              "      <td>0.37</td>\n",
              "      <td>0.42</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.12</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.88</td>\n",
              "      <td>0.81</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0.59</td>\n",
              "      <td>1.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          b_rater1  b_rater2  b_rater3  b_rater4  b_cbowu  b_cbows  b_cboww  \\\n",
              "b_rater1      1.00      0.65      0.54      0.73     0.46     0.32     0.43   \n",
              "b_rater2      0.65      1.00      0.59      0.74     0.48     0.37     0.45   \n",
              "b_rater3      0.54      0.59      1.00      0.73     0.38     0.45     0.36   \n",
              "b_rater4      0.73      0.74      0.73      1.00     0.45     0.38     0.39   \n",
              "b_cbowu       0.46      0.48      0.38      0.45     1.00     0.76     0.84   \n",
              "b_cbows       0.32      0.37      0.45      0.38     0.76     1.00     0.69   \n",
              "b_cboww       0.43      0.45      0.36      0.39     0.84     0.69     1.00   \n",
              "b_tasa        0.30      0.27      0.22      0.22     0.36     0.32     0.41   \n",
              "b_glov        0.26      0.24      0.21      0.26     0.80     0.57     0.71   \n",
              "r_rater1      0.29      0.38      0.37      0.43     0.17     0.10     0.09   \n",
              "r_rater2      0.27      0.38      0.39      0.41     0.17     0.13     0.10   \n",
              "r_rater3      0.36      0.48      0.51      0.54     0.31     0.23     0.20   \n",
              "r_rater4      0.37      0.43      0.48      0.51     0.31     0.21     0.21   \n",
              "r_cbowu       0.37      0.40      0.36      0.49     0.42     0.35     0.41   \n",
              "r_cbows       0.32      0.42      0.39      0.45     0.39     0.34     0.40   \n",
              "r_cboww       0.36      0.43      0.37      0.50     0.48     0.44     0.46   \n",
              "r_tasa        0.25      0.35      0.29      0.37     0.28     0.19     0.17   \n",
              "r_glov        0.37      0.42      0.35      0.45     0.43     0.40     0.45   \n",
              "\n",
              "          b_tasa  b_glov  r_rater1  r_rater2  r_rater3  r_rater4  r_cbowu  \\\n",
              "b_rater1    0.30    0.26      0.29      0.27      0.36      0.37     0.37   \n",
              "b_rater2    0.27    0.24      0.38      0.38      0.48      0.43     0.40   \n",
              "b_rater3    0.22    0.21      0.37      0.39      0.51      0.48     0.36   \n",
              "b_rater4    0.22    0.26      0.43      0.41      0.54      0.51     0.49   \n",
              "b_cbowu     0.36    0.80      0.17      0.17      0.31      0.31     0.42   \n",
              "b_cbows     0.32    0.57      0.10      0.13      0.23      0.21     0.35   \n",
              "b_cboww     0.41    0.71      0.09      0.10      0.20      0.21     0.41   \n",
              "b_tasa      1.00    0.31     -0.05      0.06      0.08      0.10     0.10   \n",
              "b_glov      0.31    1.00      0.07      0.08      0.19      0.22     0.37   \n",
              "r_rater1   -0.05    0.07      1.00      0.65      0.74      0.74     0.32   \n",
              "r_rater2    0.06    0.08      0.65      1.00      0.75      0.76     0.38   \n",
              "r_rater3    0.08    0.19      0.74      0.75      1.00      0.87     0.57   \n",
              "r_rater4    0.10    0.22      0.74      0.76      0.87      1.00     0.54   \n",
              "r_cbowu     0.10    0.37      0.32      0.38      0.57      0.54     1.00   \n",
              "r_cbows     0.13    0.31      0.33      0.38      0.54      0.54     0.84   \n",
              "r_cboww     0.15    0.39      0.32      0.34      0.53      0.51     0.90   \n",
              "r_tasa      0.02    0.22      0.33      0.35      0.54      0.47     0.66   \n",
              "r_glov      0.12    0.33      0.29      0.33      0.50      0.50     0.88   \n",
              "\n",
              "          r_cbows  r_cboww  r_tasa  r_glov  \n",
              "b_rater1     0.32     0.36    0.25    0.37  \n",
              "b_rater2     0.42     0.43    0.35    0.42  \n",
              "b_rater3     0.39     0.37    0.29    0.35  \n",
              "b_rater4     0.45     0.50    0.37    0.45  \n",
              "b_cbowu      0.39     0.48    0.28    0.43  \n",
              "b_cbows      0.34     0.44    0.19    0.40  \n",
              "b_cboww      0.40     0.46    0.17    0.45  \n",
              "b_tasa       0.13     0.15    0.02    0.12  \n",
              "b_glov       0.31     0.39    0.22    0.33  \n",
              "r_rater1     0.33     0.32    0.33    0.29  \n",
              "r_rater2     0.38     0.34    0.35    0.33  \n",
              "r_rater3     0.54     0.53    0.54    0.50  \n",
              "r_rater4     0.54     0.51    0.47    0.50  \n",
              "r_cbowu      0.84     0.90    0.66    0.88  \n",
              "r_cbows      1.00     0.87    0.61    0.81  \n",
              "r_cboww      0.87     1.00    0.60    0.87  \n",
              "r_tasa       0.61     0.60    1.00    0.59  \n",
              "r_glov       0.81     0.87    0.59    1.00  "
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test = pd.read_excel('./../data/Data/Study 1/s1_data_agg.xlsx')\n",
        "test[[col for col in test if (col.startswith('b_') or col.startswith('r_')) and not col.endswith('_a')]].corr().round(2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8H-WPj9MeGrS"
      },
      "source": [
        "## Normalize MOTES Pilot Data\n",
        "\n",
        "Score on multi-point scale.\n",
        "\n",
        "MOTES is related to the \"Measuring Original Thinking in Elementary Students: A Text-Mining Approach\" (IES #R305A200519). This dataset is very, very new and not yet ready to share. Hopefully it is open access by the time our first LLM paper is published, until then I think we can share with researchers by request (contact <peter.organisciak@du.edu> and/or <selcuk.acar@unt.edu>)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9d5mIJsYQ7oD",
        "outputId": "ff237725-162d-4dad-8344-e620c245113d"
      },
      "outputs": [],
      "source": [
        "#src = 'motesp'\n",
        "#motes_pilot = pd.read_csv(os.path.join(base_dir, 'motes_pilot_gt_scores.csv')).rename(columns=dict(D='rater1', K='rater2', T='rater3', ID='participant'))\n",
        "#motes_pilot['response_num'] = None\n",
        "#motes_pilot['type'] = motes_pilot.task.apply(lambda x: x.split('_')[0]).replace({'g1':'uses', 'g2':'instances', 'g3': 'consequences', 'g4':'completion'})\n",
        "#motes_pilot = prep_general(motes_pilot, src, include_stats=False, return_full=True, include_rater_std=include_rater_std)\n",
        "## Normalize question styles\n",
        "#motes_pilot.question = (motes_pilot.question.str.replace(\"What would be a surprising\", \"What is a surprising\", regex=True)\n",
        "#                            .str.replace(\"Think of an example of something that is (.*)\", \"What is a surprising example of something \\\\1?\", regex=True)\n",
        "#                            .str.replace(\"What would happen if (.*)\", \"What would be a surprising consequence if \\\\1\", regex=True)\n",
        "#)\n",
        "#comprows = (motes_pilot.type == 'completion')\n",
        "#motes_pilot.loc[comprows, 'question'] = motes_pilot.loc[comprows, 'question'].str.replace(\"^(.*)$\", 'Complete this sentence in a surprising way: \"\\\\1...\"', regex=True)\n",
        "#datasets[src] = motes_pilot[returncols]\n",
        "#\n",
        "#simple_stats(motes_pilot[motes_pilot.id.str.contains('_g1_')], ['rater1', 'rater2', 'rater3'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tbICpoGS4nvR"
      },
      "source": [
        "## Process MOTES Main Data\n",
        "\n",
        "This is the post-pilot data. As with the pilot data, this dataset is not yet released, but please reach out.\n",
        "\n",
        "There are currently 5 raters, though early analysis had 4, which meant the randomized rounding strategy sometimes went into effect - e.g. 2.25 might become 2.3 or 2.2."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "9l-yEQEQ1p2f"
      },
      "outputs": [],
      "source": [
        "#df = pd.read_csv(os.path.join(base_dir, 'motes_full_gt_scores.csv')).replace(-999, pd.NA).copy()\n",
        "#df['participant'] = df.Order.astype(str).apply(lambda x: hashlib.md5(x.encode('utf-8')).hexdigest()[:6])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dDWDbuTf94vS",
        "outputId": "e3e783d6-cfaf-4786-cb6c-00d1232d9fc1"
      },
      "outputs": [],
      "source": [
        "#src = 'motesf'\n",
        "#\n",
        "#corrected = True # use spelling corrected columns\n",
        "#items = [col.replace('_prompt', '') for col in df.columns if col.startswith('G') and col.endswith('_prompt')]\n",
        "#\n",
        "#collector = []\n",
        "## RESHAPE TO long\n",
        "#for item in items:\n",
        "#    subset = df[['participant'] + [col for col in df.columns if col.startswith(item)]].copy()\n",
        "#    subset.columns = [col.split('_')[-1] for col in subset.columns]\n",
        "#    subset['game'] = item.split('_')[0]\n",
        "#    subset['prompt_code'] = item\n",
        "#    collector.append(subset)\n",
        "#reshaped = pd.concat(collector)\n",
        "## remove non-responses\n",
        "#reshaped = reshaped[~reshaped.raw.isna()]\n",
        "## restore original wording in the test\n",
        "#reshaped.prompt =reshaped.prompt.str.replace('light bulbs', 'lightbulb').str.replace('hat cap', 'hat').str.replace('soccer ball', 'ball').str.replace('lead pencil', 'pencil').str.replace('spoons', 'spoon')\n",
        "#\n",
        "## add display order\n",
        "#displayorder = df[['participant'] + [col for col in df.columns if 'DO' in col]]\n",
        "#displayorder = displayorder.melt(id_vars='participant', value_name='prompt_code')\n",
        "#displayorder['response_num'] = displayorder.variable.apply(lambda x:x[-1])\n",
        "#reshaped = reshaped.merge(displayorder[['participant', 'prompt_code', 'response_num']])\n",
        "## use spelling corrected response, unless set otherwise\n",
        "#reshaped = reshaped.rename(columns={('corrected' if corrected else 'raw'):'response'})\n",
        "#reshaped['type'] = reshaped['game'].replace({'G1':'uses', 'G2': 'instances', 'G3':'completion'})\n",
        "#\n",
        "#completion_ref = {\n",
        "#    \"playground\": \"When the friends met on the playground...\",\n",
        "#    \"school bus\": \"When I got on the school bus...\",\n",
        "#    \"games\": \"At a sleepover we...\",\n",
        "#    'library': \"When the kids were in the library...\",\n",
        "#    'lecture': \"When the teacher was talking...\",\n",
        "#    'phone': \"My friend called me on the phone to tell me...\",\n",
        "#    'rain': \"It started raining and...\",\n",
        "#    'closet': \"When I opened my closet...\",\n",
        "#    'lunchroom': \"When I was at lunch...\"\n",
        "#}\n",
        "#reshaped.loc[reshaped.type == 'uses', 'question'] = reshaped.loc[reshaped.type == 'uses', 'prompt'].apply(lambda x: f\"What is a surprising use for a {x.upper()}?\")\n",
        "#reshaped.loc[reshaped.type == 'instances','question'] = reshaped.loc[reshaped.type == 'instances', 'prompt'].apply(lambda x: f\"What is a surprising example of something {x.upper()}?\")\n",
        "#reshaped.loc[(reshaped.type == 'completion'), 'question'] = reshaped.loc[reshaped.type == 'completion', 'prompt'].replace(completion_ref).str.replace(\"(.*)\", 'Complete this sentence in a surprising way: \"\\\\1\"...', regex=True)\n",
        "#\n",
        "#datasets[src] = prep_general(reshaped, src,\n",
        "#                             rater_cols=[col for col in reshaped if 'score' in col.lower()],\n",
        "#                             include_rater_std=include_rater_std, inputrange=(1,5))\n",
        "#\n",
        "## This is code for a separate purpose - exporting spell corrected and uncorrected data\n",
        "#if False:\n",
        "#    #corrected_res = datasets[src]\n",
        "#    #uncorrected = datasets[src]\n",
        "#    participant_info = df[['Grade', 'Age at MOTES', 'participant']]\n",
        "#    participant_info.columns = ['grade', 'age', 'participant']\n",
        "#    participant_info.grade = participant_info.grade.apply(lambda x: int(x[0]))\n",
        "#    participant_info.participant = src + participant_info.participant\n",
        "#    spelldata = uncorrected.rename(columns={'response':'raw'}).merge(corrected_res.rename(columns={'response':'corrected'}), on=[col for col in uncorrected.columns if col not in ['id', 'response']]).merge(participant_info)\n",
        "#    spelldata.to_csv(base_dir / 'motes_full_spelldata.csv')\n",
        "#\n",
        "#datasets[src].sample()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9mbZjzsjIuW8"
      },
      "source": [
        "## Combine data and basic stats"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HjzAueLF-G2j"
      },
      "source": [
        "Check correlation among ratings for responses which have been submitted more than once. Here, I sample one rating vs mean of all the ratings for a duplicate response. This contextualizes the max what a model might be able to do - if humans can't agree (sometimes with themselves!), then it would be impossible for a model to do so."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zHONheN0k0Q4",
        "outputId": "b2e4ee80-602c-4139-e50b-77a6581d7aa5"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1000/1000 [00:42<00:00, 23.27it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Average correlation among duplicates 0.8316375642575191\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "#@markdown Average rating-to-rating correlation on duplicates\n",
        "# run multiple times with different samples\n",
        "from tqdm import trange\n",
        "corrs = []\n",
        "for i in trange(1000):\n",
        "    check_dupe_corr = pd.concat(datasets.values()).sample(frac=1, random_state=i**2)\n",
        "    just_duped = check_dupe_corr[check_dupe_corr[['prompt', 'response']].duplicated(keep=False)]\n",
        "    first = just_duped.drop_duplicates(['prompt', 'response'], keep='first')\n",
        "    last = just_duped.drop_duplicates(['prompt', 'response'], keep='last')\n",
        "    merged = first.merge(last[['prompt', 'response', 'target']], how='inner', on=['prompt', 'response'])\n",
        "    corr = merged[['target_x', 'target_y']].corr().values[0,1]\n",
        "    corrs.append(corr)\n",
        "print(\"\\nAverage correlation among duplicates\", sum(corrs)/len(corrs))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of datasets combined: 7\n",
            "dod20: 5490 data points\n",
            "snbmo09: 4099 data points\n",
            "hass17: 1093 data points\n",
            "setal08: 11490 data points\n",
            "betal18: 2918 data points\n",
            "snb17: 2372 data points\n",
            "bs12: 1807 data points\n",
            "\n",
            "Number of data points in total: 29269\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target_x</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "      <th>target_y</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for a BOTTLE?</td>\n",
              "      <td>bottle</td>\n",
              "      <td>drinking</td>\n",
              "      <td>dod20_bottle-7d11bf</td>\n",
              "      <td>1.0</td>\n",
              "      <td>dod2021</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>uses</td>\n",
              "      <td>snb17</td>\n",
              "      <td>What is a surprising use for a BOX?</td>\n",
              "      <td>box</td>\n",
              "      <td>sled</td>\n",
              "      <td>snb17_box-24a76f</td>\n",
              "      <td>1.3</td>\n",
              "      <td>snb17104</td>\n",
              "      <td>None</td>\n",
              "      <td>1.3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for a TIRE?</td>\n",
              "      <td>tire</td>\n",
              "      <td>doorstop</td>\n",
              "      <td>dod20_tire-45cef6</td>\n",
              "      <td>3.0</td>\n",
              "      <td>dod2053</td>\n",
              "      <td>1</td>\n",
              "      <td>3.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for a PANTS?</td>\n",
              "      <td>pants</td>\n",
              "      <td>scarecrow</td>\n",
              "      <td>dod20_pants-7e08c6</td>\n",
              "      <td>2.5</td>\n",
              "      <td>dod2068</td>\n",
              "      <td>4</td>\n",
              "      <td>2.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>instances</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What is a surprising thing that makes a NOISE?</td>\n",
              "      <td>noise</td>\n",
              "      <td>hair dryer</td>\n",
              "      <td>setal08_5.0-2e313a</td>\n",
              "      <td>1.3</td>\n",
              "      <td>setal0819</td>\n",
              "      <td>25.0</td>\n",
              "      <td>1.3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        type      src                                        question  prompt  \\\n",
              "0       uses    dod20          What is a surprising use for a BOTTLE?  bottle   \n",
              "1       uses    snb17             What is a surprising use for a BOX?     box   \n",
              "2       uses    dod20            What is a surprising use for a TIRE?    tire   \n",
              "3       uses    dod20           What is a surprising use for a PANTS?   pants   \n",
              "4  instances  setal08  What is a surprising thing that makes a NOISE?   noise   \n",
              "\n",
              "     response                   id  target_x participant response_num  \\\n",
              "0    drinking  dod20_bottle-7d11bf       1.0     dod2021            1   \n",
              "1        sled     snb17_box-24a76f       1.3    snb17104         None   \n",
              "2    doorstop    dod20_tire-45cef6       3.0     dod2053            1   \n",
              "3   scarecrow   dod20_pants-7e08c6       2.5     dod2068            4   \n",
              "4  hair dryer   setal08_5.0-2e313a       1.3   setal0819         25.0   \n",
              "\n",
              "   target_y  \n",
              "0       1.0  \n",
              "1       1.3  \n",
              "2       3.0  \n",
              "3       2.5  \n",
              "4       1.3  "
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(f'Number of datasets combined: {len(datasets)}')\n",
        "# Number of datapoints in each dataset\n",
        "for src, df in datasets.items():\n",
        "    print(f'{src}: {len(df)} data points')\n",
        "print(f'\\nNumber of data points in total: {sum(len(df) for df in datasets.values())}')\n",
        "merged.head()\n",
        "\n",
        "#hmsl and motesp datasets not included in the merged dataset: error or not available"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "type             object\n",
              "src              object\n",
              "question         object\n",
              "prompt           object\n",
              "response         object\n",
              "id               object\n",
              "target_x        float64\n",
              "participant      object\n",
              "response_num     object\n",
              "target_y        float64\n",
              "dtype: object"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "merged.dtypes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dM-KuLyz0gDx",
        "outputId": "445a2f2c-dc11-465f-ab3f-90000dd530e9"
      },
      "outputs": [],
      "source": [
        "#@markdown Average rating-to-mean(other ratings) correlation on duplicates\n",
        "#corrs = []\n",
        "#for i in trange(1000):\n",
        "#    check_dupe_corr = pd.concat(datasets.values()).sample(frac=1)\n",
        "#    means_of_dupes = check_dupe_corr[check_dupe_corr[['prompt', 'response']].duplicated()].groupby(['prompt','response'], as_index=False).target.mean().round(2)\n",
        "#    print(means_of_dupes['prompt'].unique())\n",
        "#    print(means_of_dupes['response'].unique())\n",
        "#    corr = check_dupe_corr.merge(means_of_dupes[['prompt', 'response', 'target']], on=['prompt', 'response']).corr().loc['target_x', 'target_y']\n",
        "#    corrs.append(corr)\n",
        "#print(\"\\nAverage correlation among duplicates\", sum(corrs)/len(corrs))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fTgfgBWR_c-H"
      },
      "source": [
        "Merge ratings for items with duplicates, so that a response that has been rated multiple times has the average of all instances as it's ground truth."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "f7ps_G3makbo",
        "outputId": "12aad5ff-6673-489c-cb0b-971d9427614f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Pre-dedupe data size is 29269 items\n",
            "# of unique participants is 1039\n",
            "# of unique prompts 16\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "src\n",
              "setal08    11490\n",
              "dod20       5490\n",
              "snbmo09     4099\n",
              "betal18     2918\n",
              "snb17       2372\n",
              "bs12        1807\n",
              "hass17      1093\n",
              "Name: count, dtype: int64"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dropped 8494 duplicate items. 29.02%\n",
            "Final data size is 20775 items\n",
            "\n",
            "proportion of representation by each prompt\n",
            "prompt\n",
            "brick       19.52\n",
            "box         13.01\n",
            "knife       10.48\n",
            "rope         9.86\n",
            "noise        8.80\n",
            "shrank       7.31\n",
            "no sleep     7.31\n",
            "round        6.38\n",
            "bottle       3.35\n",
            "book         2.35\n",
            "table        2.23\n",
            "pants        2.13\n",
            "tire         1.99\n",
            "fork         1.96\n",
            "shoe         1.69\n",
            "shovel       1.63\n",
            "Name: count, dtype: float64\n",
            "\n",
            "Number of datasets with each prompt\n",
            "prompt\n",
            "brick       5\n",
            "rope        3\n",
            "box         3\n",
            "knife       2\n",
            "bottle      2\n",
            "pants       1\n",
            "shrank      1\n",
            "no sleep    1\n",
            "noise       1\n",
            "round       1\n",
            "tire        1\n",
            "book        1\n",
            "fork        1\n",
            "shoe        1\n",
            "shovel      1\n",
            "table       1\n",
            "Name: count, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "data = pd.concat(datasets.values()).sample(frac=1, random_state=1234)\n",
        "\n",
        "# Fix prompt grammar and spelling\n",
        "data.question = data.question.str.replace('a (PANTS)', r'\\1', regex=True).str.replace('LIGHTBULB', 'LIGHT BULB')\n",
        "#data.prompt = data.prompt.str.replace('lightbulb', 'light bulb')\n",
        "\n",
        "#@markdown save all data before deduplication for easy access later?\n",
        "save_all = True #param {type:'boolean'}\n",
        "save_location = './../data/all_data.csv' #@param {type:'raw'}\n",
        "if save_all:\n",
        "    data.to_csv(save_location)\n",
        "\n",
        "#@markdown For generating splits - focus just on aut?\n",
        "aut_only = False #@param {type:'boolean'}\n",
        "if aut_only:\n",
        "    data = data[data.type == 'uses']\n",
        "\n",
        "drop_missing = True #@param {type:'boolean'}\n",
        "if drop_missing:\n",
        "    data = data[~data.target.isna()]\n",
        "print(f\"Pre-dedupe data size is {len(data)} items\")\n",
        "print(f'# of unique participants is {len(data.participant.unique())}')\n",
        "print(\"# of unique prompts\", len(data.prompt.unique()))\n",
        "\n",
        "display(data['src'].value_counts())\n",
        "\n",
        "combine_gt_for_dupes = True #@param {type:'boolean'}\n",
        "if combine_gt_for_dupes:\n",
        "    dupe_means = data.groupby(['prompt', 'response'], as_index=False).target.aggregate(['count', 'mean', 'std']).reset_index().round(1)\n",
        "    dupe_means = dupe_means[dupe_means['count'] > 1]\n",
        "    data = data.merge(dupe_means, how='left')\n",
        "    data.loc[~data['mean'].isna(), 'target'] = data['mean'].dropna() # set target to be mean\n",
        "    data = data.drop(columns=['mean', 'std'])\n",
        "\n",
        "ensure_no_dupes = True #@param {type: 'boolean'}\n",
        "deduplication_strategy = \"Uncased Match\" #@param [\"Exact\", \"Uncased Match\"]\n",
        "og_size = len(data)\n",
        "if ensure_no_dupes:\n",
        "    if deduplication_strategy == \"Uncased Match\":\n",
        "        data['response_lower'] = data.response.str.lower()\n",
        "        data = data.drop_duplicates(['prompt', 'response_lower'])\n",
        "        data = data.drop('response_lower', axis=1)\n",
        "    elif deduplication_strategy == \"Exact\":\n",
        "        data = data.drop_duplicates(['prompt', 'response'])\n",
        "    print(f\"Dropped {og_size-len(data)} duplicate items. {100*(og_size-len(data))/og_size:.2f}%\")\n",
        "    print(f\"Final data size is {len(data)} items\")\n",
        "\n",
        "print('\\nproportion of representation by each prompt')\n",
        "print((data.prompt.value_counts() / len(data)).multiply(100).round(2))\n",
        "\n",
        "print(\"\\nNumber of datasets with each prompt\")\n",
        "print(data[['prompt', 'src']].drop_duplicates().prompt.value_counts())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {
        "id": "YzS9PSnGzo9f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'shovel': 5.759,\n",
              " 'shoe': 5.78,\n",
              " 'fork': 5.946,\n",
              " 'tire': 5.959,\n",
              " 'pants': 6.03,\n",
              " 'table': 6.051,\n",
              " 'book': 6.135,\n",
              " 'bottle': 6.455,\n",
              " 'round': 7.134,\n",
              " 'no sleep': 7.277,\n",
              " 'shrank': 7.283,\n",
              " 'noise': 7.476,\n",
              " 'rope': 7.554,\n",
              " 'knife': 7.627,\n",
              " 'box': 7.846,\n",
              " 'brick': 8.233}"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from scipy.stats import entropy\n",
        "data.groupby('prompt').target.apply(entropy).sort_values().round(3).to_dict()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GN4OT7sLKsK_"
      },
      "source": [
        "## Save Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H65oin4lXOI8",
        "outputId": "fc48541c-5edb-4190-bb35-3345b308b2b4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "'rm' n'est pas reconnu en tant que commande interne\n",
            "ou externe, un programme ex�cutable ou un fichier de commandes.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "####gt_main_with_dupes\n",
            "- Data size is 20775 items\n",
            "seed 987\n",
            "targetsplits {'train': 80, 'val': 5, 'test': 15}\n",
            "split_by_part: False; split_by_prompt: False\n",
            "Final split sizes: [80.0, 5.0, 15.0]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "tar: : Couldn't visit directory: No such file or directory\n",
            "tar: Error exit delayed from previous errors.\n",
            "'mv' n'est pas reconnu en tant que commande interne\n",
            "ou externe, un programme ex�cutable ou un fichier de commandes.\n"
          ]
        }
      ],
      "source": [
        "name = 'gt_main_with_dupes' #@param {type:'string'}\n",
        "random_seed = 987 #@param {type:'number'}\n",
        "\n",
        "# ensure subfolder to avoid rm'ing something dumb\n",
        "data_dir = os.path.join('data', name)\n",
        "!rm -rf $data_dir\n",
        "\n",
        "print(f'####{name}')\n",
        "print(f\"- Data size is {len(data)} items\")\n",
        "os.makedirs(data_dir, exist_ok=True)\n",
        "splits = {'train': 80, 'val': 5, 'test': 15} #@param {type: 'raw'}\n",
        "print('seed', random_seed)\n",
        "print('targetsplits', splits)\n",
        "#markdown ### Data to Include\n",
        "\n",
        "#@markdown ### Parameters\n",
        "#@markdown If splitting data by participant, individuals will be entirely in a single split.\n",
        "#@markdown If paired with ensure_no_dupes, the test is split out, then the train data is deduped\n",
        "#@markdown so no test data is seen. This will imbalance the split sizes from expectation.\n",
        "split_by_participant = False #@param {type: 'boolean'}\n",
        "#@markdown Split where test data has different prompts than train.\n",
        "split_by_prompt = False #@param {type: 'boolean'}\n",
        "try:\n",
        "    assert not (split_by_participant and split_by_prompt)\n",
        "except:\n",
        "    raise AssertionError(\"Can't split by participant *and* prompt\")\n",
        "\n",
        "print(f'split_by_part: {split_by_participant}; split_by_prompt: {split_by_prompt}')\n",
        "\n",
        "#@markdown `dry_run`: don't save outputs\n",
        "dry_run = False #@param {type: 'boolean'}\n",
        "\n",
        "examples = dict()\n",
        "\n",
        "def fingerprint(x):\n",
        "    x = x.lower()\n",
        "    x = \"\".join([c for c in x if c.isalpha()])\n",
        "    return x\n",
        "\n",
        "if split_by_participant or split_by_prompt:\n",
        "    try:\n",
        "        assert set(examples.keys()) == {'test', 'train', 'val'}\n",
        "    except:\n",
        "        raise Exception('split_by_participant or split_by_prompt requires test/train/val names')\n",
        "\n",
        "    sample = dict()\n",
        "    splitcol = 'participant' if split_by_participant else 'prompt'\n",
        "    if split_by_participant:\n",
        "        unique_vals = data[splitcol].unique().tolist()\n",
        "\n",
        "        sample['train'], sample['test'] = train_test_split(unique_vals, train_size=splits['train']/sum(splits.values()), shuffle=True, random_state=random_seed)\n",
        "        sample['val'], sample['test']  = train_test_split(sample['test'], train_size=splits['val']/(splits['val']+splits['test']), shuffle=True, random_state=random_seed)\n",
        "    elif split_by_prompt:\n",
        "        # most common prompts\n",
        "        promptn = data.prompt.value_counts() / len(data)\n",
        "        # put aside the n most plentiful prompts, and add them to train later\n",
        "        sample['train'] = promptn[:4].index.tolist()\n",
        "        # recalc, to use splits based on the rest of the data\n",
        "        #remaining = data[~data.prompt.isin(sample['train'])]\n",
        "        #promptn = remaining.prompt.value_counts() / len(remaining)\n",
        "        promptn = promptn[~promptn.index.isin(sample['train'])]\n",
        "\n",
        "        # fill val and test, then add remainder to train\n",
        "        pool = promptn.sample(frac=1, random_state=random_seed).cumsum()\n",
        "        sample['val'] = pool[(pool-promptn.mean()/2) < splits['val']/100].index.tolist()\n",
        "        promptn = promptn[~promptn.index.isin(sample['val'])]\n",
        "\n",
        "        pool = promptn.sample(frac=1, random_state=random_seed).cumsum()\n",
        "        sample['test'] = pool[(pool-promptn.mean()/2) < splits['test']/100].index.tolist()\n",
        "        promptn = promptn[~promptn.index.isin(sample['test'])]\n",
        "\n",
        "        sample['train'] += promptn.index.tolist()\n",
        "        print('train:', sample['train'])\n",
        "        print('val:', sample['val'])\n",
        "        print('test:', sample['test'])\n",
        "\n",
        "    for splittype in splits.keys():\n",
        "        examples[splittype] = data[data[splitcol].isin(sample[splittype])].to_dict(orient='records')\n",
        "\n",
        "    if ensure_no_dupes and split_by_participant: #split by prompt is already deduped\n",
        "        exclude_list = [fingerprint(x['prompt']+x['response']) for x in examples['test']+examples['val']]\n",
        "        examples['train_og'] = examples['train']\n",
        "        examples['train'] = [x for x in examples['train'] if fingerprint(x['prompt']+x['response']) not in exclude_list]\n",
        "\n",
        "else:\n",
        "    # this approach allows custom split names\n",
        "    data_shuff = data.sample(frac=1, random_state=random_seed)\n",
        "    n = 0\n",
        "    nd = len(data_shuff)\n",
        "    for k, v in splits.items():\n",
        "        m = np.rint(n + (nd * v)/100).astype(int)\n",
        "        examples[k] = data_shuff.iloc[n:m].to_dict(orient='records')\n",
        "        n = m\n",
        "\n",
        "sizes = [len(examples[splittype]) for splittype in splits.keys()]\n",
        "print(\"Final split sizes:\", [np.round(100*x/sum(sizes), 1) for x in sizes])\n",
        "\n",
        "if not dry_run:\n",
        "    for split_type, split in splits.items():\n",
        "        os.makedirs(os.path.join(data_dir, split_type), exist_ok=True)\n",
        "\n",
        "        for item in examples[split_type]:\n",
        "            with open(os.path.join(data_dir, split_type, f\"{item['id']}.json\"), mode='w') as f:\n",
        "                json.dump(item, f)\n",
        "\n",
        "    !tar -czf \"{name}.tar.gz\" data/${name}\n",
        "    !mv \"{name}.tar.gz\" \"{base_dir}\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KHocMN_pQABA"
      },
      "source": [
        "# Dataset Reference\n",
        "\n",
        "####gt_main2\n",
        "\n",
        "This is the main split in the first LLM paper.\n",
        "\n",
        "- Data size is 20202 items\n",
        "- seed 987\n",
        "- targetsplits {'train': 80, 'val': 5, 'test': 15}\n",
        "- split_by_part: False; split_by_prompt: False\n",
        "- Final split sizes: [80.0, 5.0, 15.0]\n",
        "- (gt_main_std *should* be identical, with stdev included, but I haven't\n",
        "doublechecked)\n",
        "\n",
        "#### gt_main_with_dupes\n",
        "\n",
        "- same context as above, but not deduplicated. This is a data leakage context, which should be avoided, though to compare with past work we trained one such model.\n",
        "\n",
        "####gt_byparticipant\n",
        "- Data size is 20202 items\n",
        "- seed 987\n",
        "- targetsplits {'train': 80, 'val': 5, 'test': 15}\n",
        "- split_by_part: True; split_by_prompt: False\n",
        "- Final split sizes: [80.7, 4.7, 14.6]\n",
        "\n",
        "####gt_byprompt\n",
        "\n",
        "This the the split used for having different prompts between test and train in the first LLM paper.\n",
        "\n",
        "- Data size is 20202 items\n",
        "- seed 987\n",
        "- targetsplits {'train': 79, 'val': 4, 'test': 17}\n",
        "- split_by_part: False; split_by_prompt: True\n",
        "- train: ['brick', 'box', 'knife', 'rope', 'book', 'table', 'tire', 'fork', 'ball', 'pencil', 'lightbulb', 'shoe', 'hat', 'sock', 'toothbrush', 'backpack']\n",
        "- val: []\n",
        "- test: ['paperclip', 'spoon', 'bottle', 'shovel', 'pants']\n",
        "- Final split sizes: [83.2, 0.0, 16.8]\n",
        "\n",
        "####all\n",
        "\n",
        "This is the condition for training the final model, for use in the Open Creativity Scoring system. It was *not* deduped for training to avoid data leakeage, since it's for applied use and that performance would be desired.\n",
        "\n",
        "- Data size is 27217 items\n",
        "- seed 987\n",
        "- targetsplits {'train': 94, 'val': 1, 'test': 5}\n",
        "- split_by_part: False; split_by_prompt: False\n",
        "- Final split sizes: [94.0, 1.0, 5.0]\n",
        "\n",
        "####gt_alltests2\n",
        "\n",
        "This is the condition which includes consequences, instances, and complete the sentence. It may grow outdated as I develop a format for training with this data.\n",
        "\n",
        "It doesn't have test/train, rather multiple numbered groups so that an ensemble can be trained.\n",
        "\n",
        "- Data size is 31567 items\n",
        "- seed 987\n",
        "- targetsplits {'group1': 32, 'group2': 32, 'group3': 32, 'val': 4}\n",
        "- split_by_part: False; split_by_prompt: False\n",
        "- Final split sizes: [32.0, 32.0, 32.0, 4.0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uJon5X15GaV4",
        "outputId": "d9d0e19e-ead5-4f01-e524-ca2cb4eec923"
      },
      "outputs": [],
      "source": [
        "#dprint(\"All gt options\")\n",
        "#print([x.stem.split('.')[0] for x in base_dir.glob('*tar.gz')])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Combined data can be loaded directly from all_data.csv now"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length of data: 20775\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type</th>\n",
              "      <th>src</th>\n",
              "      <th>question</th>\n",
              "      <th>prompt</th>\n",
              "      <th>response</th>\n",
              "      <th>id</th>\n",
              "      <th>target</th>\n",
              "      <th>participant</th>\n",
              "      <th>response_num</th>\n",
              "      <th>index</th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>uses</td>\n",
              "      <td>dod20</td>\n",
              "      <td>What is a surprising use for PANTS?</td>\n",
              "      <td>pants</td>\n",
              "      <td>fashion</td>\n",
              "      <td>dod20_pants-160974</td>\n",
              "      <td>1.0</td>\n",
              "      <td>dod2037</td>\n",
              "      <td>6</td>\n",
              "      <td>14274.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>consequences</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What would be a surprising consequence if EVER...</td>\n",
              "      <td>shrank</td>\n",
              "      <td>couldn't harvest food</td>\n",
              "      <td>setal08_6.0-5d7c8c</td>\n",
              "      <td>1.3</td>\n",
              "      <td>setal08150</td>\n",
              "      <td>7.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>consequences</td>\n",
              "      <td>setal08</td>\n",
              "      <td>What would be a surprising consequence if PEOP...</td>\n",
              "      <td>no sleep</td>\n",
              "      <td>people would not need to eat the regular healt...</td>\n",
              "      <td>setal08_3.0-5d36eb</td>\n",
              "      <td>1.0</td>\n",
              "      <td>setal08106</td>\n",
              "      <td>2.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>uses</td>\n",
              "      <td>betal18</td>\n",
              "      <td>What is a surprising use for a ROPE?</td>\n",
              "      <td>rope</td>\n",
              "      <td>use it to tie up a bicycle</td>\n",
              "      <td>betal18_rope-1dd6b7</td>\n",
              "      <td>1.9</td>\n",
              "      <td>betal182093</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>uses</td>\n",
              "      <td>betal18</td>\n",
              "      <td>What is a surprising use for a ROPE?</td>\n",
              "      <td>rope</td>\n",
              "      <td>steps</td>\n",
              "      <td>betal18_rope-672857</td>\n",
              "      <td>1.3</td>\n",
              "      <td>betal182076</td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           type      src                                           question  \\\n",
              "0          uses    dod20                What is a surprising use for PANTS?   \n",
              "1  consequences  setal08  What would be a surprising consequence if EVER...   \n",
              "2  consequences  setal08  What would be a surprising consequence if PEOP...   \n",
              "3          uses  betal18               What is a surprising use for a ROPE?   \n",
              "4          uses  betal18               What is a surprising use for a ROPE?   \n",
              "\n",
              "     prompt                                           response  \\\n",
              "0     pants                                            fashion   \n",
              "1    shrank                              couldn't harvest food   \n",
              "2  no sleep  people would not need to eat the regular healt...   \n",
              "3      rope                         use it to tie up a bicycle   \n",
              "4      rope                                              steps   \n",
              "\n",
              "                    id  target  participant response_num    index  count  \n",
              "0   dod20_pants-160974     1.0      dod2037            6  14274.0    2.0  \n",
              "1   setal08_6.0-5d7c8c     1.3   setal08150          7.0      NaN    NaN  \n",
              "2   setal08_3.0-5d36eb     1.0   setal08106          2.0      NaN    NaN  \n",
              "3  betal18_rope-1dd6b7     1.9  betal182093         None      NaN    NaN  \n",
              "4  betal18_rope-672857     1.3  betal182076         None      NaN    NaN  "
            ]
          },
          "execution_count": 59,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(f'Length of data: {len(data)}')\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {},
      "outputs": [],
      "source": [
        "# save data cleaned (-9000 samples)\n",
        "data.to_csv('./../data/cleaned_all_data.csv')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyOiLWVlkci39ayG4qPrCLCX",
      "collapsed_sections": [
        "zltQcwkXeRvk",
        "gIwAL3uqaikc",
        "jn8ziOcamzaH",
        "vJqyd2UHDCMZ",
        "UH3yOwIon4Ky",
        "VgNoUTqg_dQb",
        "laa40OuiNV2X",
        "8H-WPj9MeGrS",
        "tbICpoGS4nvR"
      ],
      "include_colab_link": true,
      "mount_file_id": "1bZ0PDifSqhWhAUwuLK4BCt0Mke50ww-Q",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}