Stage 0-6. Gerenate-Bert-Base-Cased-Embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sample_submission_stage_1.csv', 'test_stage_1.tsv', 'trees']\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"data\"))\n",
    "import zipfile\n",
    "import sys\n",
    "import time\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_uuid": "cfccbec6c87185a0db428e3ce8ecb93aa9c4547e",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import modeling\n",
    "import extract_features\n",
    "import tokenization\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "1a655a90d41802605da6f27c605313eac4af4cc2",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_offset_no_spaces(text, offset):\n",
    "\tcount = 0\n",
    "\tfor pos in range(offset):\n",
    "\t\tif text[pos] != \" \": count +=1\n",
    "\treturn count\n",
    "\n",
    "def count_chars_no_special(text):\n",
    "\tcount = 0\n",
    "\tspecial_char_list = [\"#\"]\n",
    "\tfor pos in range(len(text)):\n",
    "\t\tif text[pos] not in special_char_list: count +=1\n",
    "\treturn count\n",
    "\n",
    "def count_length_no_special(text):\n",
    "\tcount = 0\n",
    "\tspecial_char_list = [\"#\", \" \"]\n",
    "\tfor pos in range(len(text)):\n",
    "\t\tif text[pos] not in special_char_list: count +=1\n",
    "\treturn count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "9d8437cb4f7c1737e0c915d7d16cc3d004612416",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_bert(data, output, embedding_size=1024, layer=-1):\n",
    "\t'''\n",
    "\tRuns a forward propagation of BERT on input text, extracting contextual word embeddings\n",
    "\tInput: data, a pandas DataFrame containing the information in one of the GAP files\n",
    "\n",
    "\tOutput: emb, a pandas DataFrame containing contextual embeddings for the words A, B and Pronoun. Each embedding is a numpy array of shape (768)\n",
    "\tcolumns: \"emb_A\": the embedding for word A\n",
    "\t         \"emb_B\": the embedding for word B\n",
    "\t         \"emb_P\": the embedding for the pronoun\n",
    "\t         \"label\": the answer to the coreference problem: \"A\", \"B\" or \"NEITHER\"\n",
    "\t'''\n",
    "    # From the current file, take the text only, and write it in a file which will be passed to BERT\n",
    "\ttext = data[\"Text\"]\n",
    "\ttext.to_csv(\"input.txt\", index = False, header = False)\n",
    "\n",
    "    # The script extract_features.py runs forward propagation through BERT, and writes the output in the file output.jsonl\n",
    "    # I'm lazy, so I'm only saving the output of the last layer. Feel free to change --layers = -1 to save the output of other layers.\n",
    "\n",
    "\tres = os.system(\"python extract_features.py \\\n",
    "\t  --input_file=input.txt \\\n",
    "\t  --output_file=output.jsonl \\\n",
    "\t  --vocab_file=cased_L-12_H-768_A-12/vocab.txt \\\n",
    "\t  --bert_config_file=cased_L-12_H-768_A-12/bert_config.json \\\n",
    "\t  --init_checkpoint=cased_L-12_H-768_A-12/bert_model.ckpt \\\n",
    "\t  --layers={} \\\n",
    "\t  --max_seq_length=512 \\\n",
    "\t  --batch_size=1\".format(layer))\n",
    "    \n",
    "    \n",
    "\tbert_output = pd.read_json(\"output.jsonl\", lines = True)\n",
    "\n",
    "\tos.system(\"rm output.jsonl\")\n",
    "\tos.system(\"rm input.txt\")\n",
    "\n",
    "\tindex = data.index\n",
    "\tcolumns = [\"emb_A\", \"emb_B\", \"emb_P\", \"label\"]\n",
    "\temb = pd.DataFrame(index = index, columns = columns)\n",
    "\temb.index.name = \"ID\"\n",
    "\n",
    "\tfor i in range(len(data)): # For each line in the data file\n",
    "\t\t# get the words A, B, Pronoun. Convert them to lower case, since we're using the uncased version of BERT\n",
    "\t\tP = data.loc[i,\"Pronoun\"].lower()\n",
    "\t\tA = data.loc[i,\"A\"].lower()\n",
    "\t\tB = data.loc[i,\"B\"].lower()\n",
    "\n",
    "\t\t# For each word, find the offset not counting spaces. This is necessary for comparison with the output of BERT\n",
    "\t\tP_offset = compute_offset_no_spaces(data.loc[i,\"Text\"], data.loc[i,\"Pronoun-offset\"])\n",
    "\t\tA_offset = compute_offset_no_spaces(data.loc[i,\"Text\"], data.loc[i,\"A-offset\"])\n",
    "\t\tB_offset = compute_offset_no_spaces(data.loc[i,\"Text\"], data.loc[i,\"B-offset\"])\n",
    "\t\t# Figure out the length of A, B, not counting spaces or special characters\n",
    "\t\tA_length = count_length_no_special(A)\n",
    "\t\tB_length = count_length_no_special(B)\n",
    "\n",
    "\t\t# Initialize embeddings with zeros\n",
    "\t\temb_A = np.zeros(embedding_size)\n",
    "\t\temb_B = np.zeros(embedding_size)\n",
    "\t\temb_P = np.zeros(embedding_size)\n",
    "\n",
    "\t\t# Initialize counts\n",
    "\t\tcount_chars = 0\n",
    "\t\tcnt_A, cnt_B, cnt_P = 0, 0, 0\n",
    "\n",
    "\t\tfeatures = pd.DataFrame(bert_output.loc[i,\"features\"]) # Get the BERT embeddings for the current line in the data file\n",
    "\t\tfor j in range(2,len(features)):  # Iterate over the BERT tokens for the current line; we skip over the first 2 tokens, which don't correspond to words\n",
    "\t\t\ttoken = features.loc[j,\"token\"]\n",
    "\n",
    "\t\t\t# See if the character count until the current token matches the offset of any of the 3 target words\n",
    "\t\t\tif count_chars  == P_offset: \n",
    "\t\t\t\t# print(token)\n",
    "\t\t\t\temb_P += np.array(features.loc[j,\"layers\"][0]['values'])\n",
    "\t\t\t\tcnt_P += 1\n",
    "\t\t\tif count_chars in range(A_offset, A_offset + A_length): \n",
    "\t\t\t\t# print(token)\n",
    "\t\t\t\temb_A += np.array(features.loc[j,\"layers\"][0]['values'])\n",
    "\t\t\t\tcnt_A +=1\n",
    "\t\t\tif count_chars in range(B_offset, B_offset + B_length): \n",
    "\t\t\t\t# print(token)\n",
    "\t\t\t\temb_B += np.array(features.loc[j,\"layers\"][0]['values'])\n",
    "\t\t\t\tcnt_B +=1\t\t\t\t\t\t\t\t\n",
    "\t\t\t# Update the character count\n",
    "\t\t\tcount_chars += count_length_no_special(token)\n",
    "\t\t# Taking the average between tokens in the span of A or B, so divide the current value by the count\t\n",
    "\t\temb_A /= cnt_A\n",
    "\t\temb_B /= cnt_B\n",
    "\n",
    "\t\t# Work out the label of the current piece of text\n",
    "\t\tlabel = \"Neither\"\n",
    "\t\tif (data.loc[i,\"A-coref\"] == True):\n",
    "\t\t\tlabel = \"A\"\n",
    "\t\tif (data.loc[i,\"B-coref\"] == True):\n",
    "\t\t\tlabel = \"B\"\n",
    "\n",
    "\t\t# Put everything together in emb\n",
    "\t\temb.iloc[i] = [emb_A, emb_B, emb_P, label]\n",
    "\n",
    "\treturn emb\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "8298947fa33285e722bdaae2df41bca5dd795732"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Started at  Sat Apr 13 19:43:36 2019\n",
      "6 Epoch\n",
      "Finished at  Sat Apr 13 19:53:27 2019\n",
      "7 Epoch\n",
      "Finished at  Sat Apr 13 20:03:37 2019\n",
      "8 Epoch\n",
      "Finished at  Sat Apr 13 20:13:57 2019\n",
      "9 Epoch\n",
      "Finished at  Sat Apr 13 20:24:26 2019\n",
      "10 Epoch\n",
      "Finished at  Sat Apr 13 20:35:08 2019\n",
      "11 Epoch\n",
      "Finished at  Sat Apr 13 20:45:57 2019\n",
      "12 Epoch\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Expected object or value",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-bb9ce0113b17>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[0mtest_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"gap-test.tsv\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msep\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'\\t'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m     \u001b[0mtest_emb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrun_bert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0membedding_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0membedding_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"{}contextual_embeddings_gap_test.json\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtag\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlayer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m     \u001b[0mtest_emb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_json\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"vector/bert_base_cased/{}contextual_embeddings_gap_test.json\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtag\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morient\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'columns'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-5-1091fd5bfe1f>\u001b[0m in \u001b[0;36mrun_bert\u001b[1;34m(data, output, embedding_size, layer)\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m         \u001b[0mbert_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_json\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"output.jsonl\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlines\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m         \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"rm output.jsonl\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\io\\json\\json.py\u001b[0m in \u001b[0;36mread_json\u001b[1;34m(path_or_buf, orient, typ, dtype, convert_axes, convert_dates, keep_default_dates, numpy, precise_float, date_unit, encoding, lines)\u001b[0m\n\u001b[0;32m    352\u001b[0m         obj = FrameParser(json, orient, dtype, convert_axes, convert_dates,\n\u001b[0;32m    353\u001b[0m                           \u001b[0mkeep_default_dates\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprecise_float\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 354\u001b[1;33m                           date_unit).parse()\n\u001b[0m\u001b[0;32m    355\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    356\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mtyp\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'series'\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\io\\json\\json.py\u001b[0m in \u001b[0;36mparse\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    420\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    421\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 422\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parse_no_numpy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    423\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    424\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\io\\json\\json.py\u001b[0m in \u001b[0;36m_parse_no_numpy\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    637\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0morient\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"columns\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    638\u001b[0m             self.obj = DataFrame(\n\u001b[1;32m--> 639\u001b[1;33m                 loads(json, precise_float=self.precise_float), dtype=None)\n\u001b[0m\u001b[0;32m    640\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0morient\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"split\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    641\u001b[0m             decoded = dict((str(k), v)\n",
      "\u001b[1;31mValueError\u001b[0m: Expected object or value"
     ]
    }
   ],
   "source": [
    "print(\"Started at \", time.ctime())\n",
    "\n",
    "for i in range(6, 13):\n",
    "    print(\"{} Epoch\".format(i))\n",
    "    tag = 'bert-base-cased-seq512-'\n",
    "    embedding_size = 768\n",
    "    layer = i\n",
    "    tag = tag + str(layer)\n",
    "\n",
    "    test_data = pd.read_csv(\"gap-test.tsv\", sep = '\\t')\n",
    "    test_emb = run_bert(test_data, embedding_size=embedding_size, output=\"{}contextual_embeddings_gap_test.json\".format(tag), layer=layer)\n",
    "    test_emb.to_json(\"vector/bert_base_cased/{}contextual_embeddings_gap_test.json\".format(tag), orient = 'columns')\n",
    "\n",
    "    validation_data = pd.read_csv(\"gap-validation.tsv\", sep = '\\t')\n",
    "    validation_emb = run_bert(validation_data, embedding_size=embedding_size, output=\"{}contextual_embeddings_gap_validation.json\".format(tag), layer=layer)\n",
    "    validation_emb.to_json(\"vector/bert_base_cased/{}contextual_embeddings_gap_validation.json\".format(tag), orient = 'columns')\n",
    "\n",
    "    development_data = pd.read_csv(\"gap-development.tsv\", sep = '\\t')\n",
    "    development_emb = run_bert(development_data, embedding_size=embedding_size, output=\"{}contextual_embeddings_gap_development.json\".format(tag), layer=layer)\n",
    "    development_emb.to_json(\"vector/bert_base_cased/{}contextual_embeddings_gap_development.json\".format(tag), orient = 'columns')\n",
    "    print(\"Finished at \", time.ctime())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "29ba41d2570238ec22735909c76cd86c2742517c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras import backend, models, layers, initializers, regularizers, constraints, optimizers\n",
    "from keras import callbacks as kc\n",
    "from keras import optimizers as ko\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, KFold, train_test_split\n",
    "from sklearn.metrics import log_loss\n",
    "import time\n",
    "\n",
    "\n",
    "dense_layer_sizes = [37]\n",
    "dropout_rate = 0.6\n",
    "learning_rate = 0.001\n",
    "n_fold = 5\n",
    "batch_size = 32\n",
    "epochs = 1000\n",
    "patience = 100\n",
    "# n_test = 100\n",
    "lambd = 0.1 # L2 regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "763ec8591474c45d6b065cad0c7efc2bbe9ad514",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_mlp_model(input_shape):\n",
    "\tX_input = layers.Input(input_shape)\n",
    "\tX = layers.Dropout(dropout_rate, seed = 7)(X_input)\n",
    "\t# First dense layer\n",
    "\tX = layers.Dense(dense_layer_sizes[0], name = 'dense0')(X)\n",
    "\tX = layers.BatchNormalization(name = 'bn0')(X)\n",
    "\tX = layers.Activation('relu')(X)\n",
    "\tX = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "\t# Output layer\n",
    "\tX = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "\tX = layers.Activation('softmax')(X)\n",
    "\n",
    "\t# Create model\n",
    "\tmodel = models.Model(input = X_input, output = X, name = \"classif_model\")\n",
    "\treturn model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_uuid": "0ea0ac2603b0a4bbdaa2776c8e37ad7894a99f5a",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_json(embeddings):\n",
    "\t'''\n",
    "\tParses the embeddigns given by BERT, and suitably formats them to be passed to the MLP model\n",
    "\n",
    "\tInput: embeddings, a DataFrame containing contextual embeddings from BERT, as well as the labels for the classification problem\n",
    "\tcolumns: \"emb_A\": contextual embedding for the word A\n",
    "\t         \"emb_B\": contextual embedding for the word B\n",
    "\t         \"emb_P\": contextual embedding for the pronoun\n",
    "\t         \"label\": the answer to the coreference problem: \"A\", \"B\" or \"NEITHER\"\n",
    "\n",
    "\tOutput: X, a numpy array containing, for each line in the GAP file, the concatenation of the embeddings of the target words\n",
    "\t        Y, a numpy array containing, for each line in the GAP file, the one-hot encoded answer to the coreference problem\n",
    "\t'''\n",
    "\tembeddings.sort_index(inplace = True) # Sorting the DataFrame, because reading from the json file messed with the order\n",
    "\tX = np.zeros((len(embeddings),3*768))\n",
    "\tY = np.zeros((len(embeddings), 3))\n",
    "\n",
    "\t# Concatenate features\n",
    "\tfor i in range(len(embeddings)):\n",
    "\t\tA = np.array(embeddings.loc[i,\"emb_A\"])\n",
    "\t\tB = np.array(embeddings.loc[i,\"emb_B\"])\n",
    "\t\tP = np.array(embeddings.loc[i,\"emb_P\"])\n",
    "\t\tX[i] = np.concatenate((A,B,P))\n",
    "\n",
    "\t# One-hot encoding for labels\n",
    "\tfor i in range(len(embeddings)):\n",
    "\t\tlabel = embeddings.loc[i,\"label\"]\n",
    "\t\tif label == \"A\":\n",
    "\t\t\tY[i,0] = 1\n",
    "\t\telif label == \"B\":\n",
    "\t\t\tY[i,1] = 1\n",
    "\t\telse:\n",
    "\t\t\tY[i,2] = 1\n",
    "\n",
    "\treturn X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0 started at Sat Apr 13 20:51:41 2019\n",
      "WARNING:tensorflow:From C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\control_flow_ops.py:423: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Sat Apr 13 20:52:14 2019\n",
      "Fold 2 started at Sat Apr 13 20:53:11 2019\n",
      "Fold 3 started at Sat Apr 13 20:54:13 2019\n",
      "Fold 4 started at Sat Apr 13 20:54:59 2019\n",
      "In the 6 layer\n",
      "CV mean score: 0.6057, std: 0.0419.\n",
      "[0.543322131367516, 0.5708338441582095, 0.6405795915464846, 0.6528578352342197, 0.6211102080583093]\n",
      "Test score: 0.5884442300447054\n",
      "Fold 0 started at Sat Apr 13 20:56:12 2019\n",
      "Fold 1 started at Sat Apr 13 20:57:15 2019\n",
      "Fold 2 started at Sat Apr 13 20:58:05 2019\n",
      "Fold 3 started at Sat Apr 13 20:59:41 2019\n",
      "Fold 4 started at Sat Apr 13 21:00:15 2019\n",
      "In the 7 layer\n",
      "CV mean score: 0.5535, std: 0.0447.\n",
      "[0.5066575307367974, 0.5013281071450478, 0.5699701519048647, 0.621092640762094, 0.5683219570607602]\n",
      "Test score: 0.5276046293553005\n",
      "Fold 0 started at Sat Apr 13 21:01:52 2019\n",
      "Fold 1 started at Sat Apr 13 21:03:13 2019\n",
      "Fold 2 started at Sat Apr 13 21:04:28 2019\n",
      "Fold 3 started at Sat Apr 13 21:05:46 2019\n",
      "Fold 4 started at Sat Apr 13 21:06:53 2019\n",
      "In the 8 layer\n",
      "CV mean score: 0.5246, std: 0.0382.\n",
      "[0.49338926951048073, 0.48079338804140587, 0.5232742647217187, 0.5900693570929101, 0.5354614450919979]\n",
      "Test score: 0.49608130353524077\n",
      "Fold 0 started at Sat Apr 13 21:08:04 2019\n",
      "Fold 1 started at Sat Apr 13 21:09:40 2019\n",
      "Fold 2 started at Sat Apr 13 21:11:05 2019\n",
      "Fold 3 started at Sat Apr 13 21:12:16 2019\n",
      "Fold 4 started at Sat Apr 13 21:13:14 2019\n",
      "In the 9 layer\n",
      "CV mean score: 0.5285, std: 0.0403.\n",
      "[0.48875602479373736, 0.4853440811013511, 0.541492104864169, 0.5955207998479779, 0.5316355668988652]\n",
      "Test score: 0.48385449249346957\n",
      "Fold 0 started at Sat Apr 13 21:15:01 2019\n",
      "Fold 1 started at Sat Apr 13 21:16:45 2019\n",
      "Fold 2 started at Sat Apr 13 21:18:35 2019\n",
      "Fold 3 started at Sat Apr 13 21:20:02 2019\n",
      "Fold 4 started at Sat Apr 13 21:21:08 2019\n",
      "In the 10 layer\n",
      "CV mean score: 0.5374, std: 0.0460.\n",
      "[0.49743160269728887, 0.4860144542853255, 0.5564757796569776, 0.6144590071504916, 0.5325259506295563]\n",
      "Test score: 0.4859783595462804\n",
      "Fold 0 started at Sat Apr 13 21:22:32 2019\n",
      "Fold 1 started at Sat Apr 13 21:24:16 2019\n",
      "Fold 2 started at Sat Apr 13 21:25:20 2019\n",
      "Fold 3 started at Sat Apr 13 21:26:35 2019\n",
      "Fold 4 started at Sat Apr 13 21:27:31 2019\n",
      "In the 11 layer\n",
      "CV mean score: 0.5678, std: 0.0420.\n",
      "[0.5208007103184467, 0.5369537565535903, 0.5760698877972059, 0.6423187570764006, 0.5628822236328103]\n",
      "Test score: 0.5263297020286404\n"
     ]
    }
   ],
   "source": [
    "# Read development embeddigns from json file - this is the output of Bert\n",
    "for i in range(6, 12):\n",
    "    tag = 'bert-base-cased-seq512-'\n",
    "    embedding_size = 768\n",
    "    layer = i\n",
    "    tag = tag + str(layer)\n",
    "\n",
    "    development = pd.read_json(\"vector/bert_base_cased/{}contextual_embeddings_gap_development.json\".format(tag))\n",
    "    X_development, Y_development = parse_json(development)\n",
    "\n",
    "    validation = pd.read_json(\"vector/bert_base_cased/{}contextual_embeddings_gap_validation.json\".format(tag))\n",
    "    X_validation, Y_validation = parse_json(validation)\n",
    "\n",
    "    test = pd.read_json(\"vector/bert_base_cased/{}contextual_embeddings_gap_test.json\".format(tag))\n",
    "    X_test, Y_test = parse_json(test)\n",
    "\n",
    "    # There may be a few NaN values, where the offset of a target word is greater than the max_seq_length of BERT.\n",
    "    # They are very few, so I'm just dropping the rows.\n",
    "    remove_test = [row for row in range(len(X_test)) if np.sum(np.isnan(X_test[row]))]\n",
    "    X_test = np.delete(X_test, remove_test, 0)\n",
    "    Y_test = np.delete(Y_test, remove_test, 0)\n",
    "\n",
    "    remove_validation = [row for row in range(len(X_validation)) if np.sum(np.isnan(X_validation[row]))]\n",
    "    X_validation = np.delete(X_validation, remove_validation, 0)\n",
    "    Y_validation = np.delete(Y_validation, remove_validation, 0)\n",
    "\n",
    "    # We want predictions for all development rows. So instead of removing rows, make them 0\n",
    "    remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row]))]\n",
    "    X_development[remove_development] = np.zeros(3*768)\n",
    "\n",
    "    # Will train on data from the gap-test and gap-validation files, in total 2454 rows\n",
    "    X_train = np.concatenate((X_test, X_validation), axis = 0)\n",
    "    Y_train = np.concatenate((Y_test, Y_validation), axis = 0)\n",
    "\n",
    "    # Will predict probabilities for data from the gap-development file; initializing the predictions\n",
    "    prediction = np.zeros((len(X_development),3)) # testing predictions\n",
    "\n",
    "    # Training and cross-validation\n",
    "    folds = KFold(n_splits=n_fold, shuffle=True, random_state=3)\n",
    "    scores = []\n",
    "    oof = np.zeros_like(Y_train)\n",
    "    \n",
    "    for fold_n, (train_index, valid_index) in enumerate(folds.split(X_train)):\n",
    "        # split training and validation data\n",
    "        print('Fold', fold_n, 'started at', time.ctime())\n",
    "        X_tr, X_val = X_train[train_index], X_train[valid_index]\n",
    "        Y_tr, Y_val = Y_train[train_index], Y_train[valid_index]\n",
    "\n",
    "        # Define the model, re-initializing for each fold\n",
    "        classif_model = build_mlp_model([X_train.shape[1]])\n",
    "        classif_model.compile(optimizer = optimizers.Adam(lr = learning_rate), loss = \"categorical_crossentropy\")\n",
    "        callbacks = [kc.EarlyStopping(monitor='val_loss', patience=patience, restore_best_weights = True)]\n",
    "\n",
    "        # train the model\n",
    "        classif_model.fit(x = X_tr, y = Y_tr, epochs = epochs, batch_size = batch_size, callbacks = callbacks, validation_data = (X_val, Y_val), verbose = 0)\n",
    "\n",
    "        # make predictions on validation and test data\n",
    "        pred_valid = classif_model.predict(x = X_val, verbose = 0)\n",
    "        oof[valid_index] = pred_valid\n",
    "        pred = classif_model.predict(x = X_development, verbose = 0)\n",
    "\n",
    "        # oof[valid_index] = pred_valid.reshape(-1,)\n",
    "        scores.append(log_loss(Y_val, pred_valid))\n",
    "        prediction += pred\n",
    "    prediction /= n_fold\n",
    "\n",
    "    # Print CV scores, as well as score on the test data\n",
    "    print(\"In the {} layer\".format(i))\n",
    "    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n",
    "    print(scores)\n",
    "    print(\"Test score:\", log_loss(Y_development,prediction))\n",
    "    \n",
    "    # Write the prediction to file for submission\n",
    "    oof_df = pd.DataFrame(oof)\n",
    "    oof_df.to_csv(\"oof/oof_bert_cased_768_seqlen512-L{}.csv\".format(layer), index=False)\n",
    "    \n",
    "    submission = pd.read_csv(\"data/sample_submission_stage_1.csv\", index_col = \"ID\")\n",
    "    submission[\"A\"] = prediction[:,0]\n",
    "    submission[\"B\"] = prediction[:,1]\n",
    "    submission[\"NEITHER\"] = prediction[:,2]\n",
    "    submission.to_csv(\"outputs/submission_bert_cased_768_seqlen512-L{}.csv\".format(layer))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Records\n",
    "\n",
    "* Layer 01 CV mean score: 0.8874, std: 0.0154. Test score: 0.8495820849324255\n",
    "* Layer 02 CV mean score: 0.8700, std: 0.0127. Test score: 0.8438746537152055\n",
    "* Layer 03 CV mean score: 0.8042, std: 0.0209. Test score: 0.7804143238337299\n",
    "* Layer 04 CV mean score: 0.7707, std: 0.0307. Test score: 0.7545189016261488\n",
    "* Layer 05 CV mean score: 0.7750, std: 0.0202. Test score: 0.7548036766969642\n",
    "* Layer 06 CV mean score: 0.7625, std: 0.0147. Test score: 0.7379605187821101\n",
    "* Layer 07 CV mean score: 0.7603, std: 0.0148. Test score: 0.7383950867017735\n",
    "* Layer 08 CV mean score: 0.7489, std: 0.0168. Test score: 0.7151597505941151\n",
    "* Layer 09 CV mean score: 0.6968, std: 0.0223. Test score: 0.6628560374140844\n",
    "* Layer 10 CV mean score: 0.6388, std: 0.0300. Test score: 0.6216318814585869\n",
    "* Layer 11 CV mean score: 0.6115, std: 0.0214. Test score: 0.5907616362240837\n",
    "* Layer 12 CV mean score: 0.5796, std: 0.0232. Test score: 0.5618664425925821\n",
    "* Layer 13 CV mean score: 0.5436, std: 0.0287. Test score: 0.5226838338351586\n",
    "* Layer 14 CV mean score: 0.5089, std: 0.0334. Test score: 0.47300375639454456\n",
    "* Layer 15 CV mean score: 0.4669, std: 0.0401. Test score: 0.4352659449041463\n",
    "* Layer 16 CV mean score: 0.4499, std: 0.0456. Test score: 0.4087953151634497\n",
    "* Layer 17 CV mean score: 0.4361, std: 0.0257. Test score: 0.39111295910973526\n",
    "* Layer 18 CV mean score: 0.4297, std: 0.0270. Test score: 0.3832015453128307\n",
    "* Layer 19 CV mean score: 0.4181, std: 0.0243. Test score: 0.37968471044825763\n",
    "* Layer 20 CV mean score: 0.4457, std: 0.0240. Test score: 0.40375841880593566\n",
    "* Layer 21 CV mean score: 0.4636, std: 0.0275. Test score: 0.42320501486081\n",
    "* Layer 22 CV mean score: 0.4882, std: 0.0237. Test score: 0.44682373979905493\n",
    "* Layer 23 CV mean score: 0.5017, std: 0.0190. Test score: 0.4601968420633977\n",
    "* Layer 24 CV mean score: 0.5058, std: 0.0153. Test score: 0.46330062780985837"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}