Stage 0-5. Gerenate-Bert-Big-Cased-Embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sample_submission_stage_1.csv', 'test_stage_1.tsv', 'trees']\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"data\"))\n",
    "import zipfile\n",
    "import sys\n",
    "import time\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_uuid": "cfccbec6c87185a0db428e3ce8ecb93aa9c4547e",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import modeling\n",
    "import extract_features\n",
    "import tokenization\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "1a655a90d41802605da6f27c605313eac4af4cc2",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_offset_no_spaces(text, offset):\n",
    "\tcount = 0\n",
    "\tfor pos in range(offset):\n",
    "\t\tif text[pos] != \" \": count +=1\n",
    "\treturn count\n",
    "\n",
    "def count_chars_no_special(text):\n",
    "\tcount = 0\n",
    "\tspecial_char_list = [\"#\"]\n",
    "\tfor pos in range(len(text)):\n",
    "\t\tif text[pos] not in special_char_list: count +=1\n",
    "\treturn count\n",
    "\n",
    "def count_length_no_special(text):\n",
    "\tcount = 0\n",
    "\tspecial_char_list = [\"#\", \" \"]\n",
    "\tfor pos in range(len(text)):\n",
    "\t\tif text[pos] not in special_char_list: count +=1\n",
    "\treturn count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_uuid": "9d8437cb4f7c1737e0c915d7d16cc3d004612416",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_bert(data, output, embedding_size=1024, layer=-1):\n",
    "\t'''\n",
    "\tRuns a forward propagation of BERT on input text, extracting contextual word embeddings\n",
    "\tInput: data, a pandas DataFrame containing the information in one of the GAP files\n",
    "\n",
    "\tOutput: emb, a pandas DataFrame containing contextual embeddings for the words A, B and Pronoun. Each embedding is a numpy array of shape (768)\n",
    "\tcolumns: \"emb_A\": the embedding for word A\n",
    "\t         \"emb_B\": the embedding for word B\n",
    "\t         \"emb_P\": the embedding for the pronoun\n",
    "\t         \"label\": the answer to the coreference problem: \"A\", \"B\" or \"NEITHER\"\n",
    "\t'''\n",
    "    # From the current file, take the text only, and write it in a file which will be passed to BERT\n",
    "\ttext = data[\"Text\"]\n",
    "\ttext.to_csv(\"input.txt\", index = False, header = False)\n",
    "\n",
    "    # The script extract_features.py runs forward propagation through BERT, and writes the output in the file output.jsonl\n",
    "    # I'm lazy, so I'm only saving the output of the last layer. Feel free to change --layers = -1 to save the output of other layers.\n",
    "\n",
    "\tres = os.system(\"python extract_features.py \\\n",
    "\t  --input_file=input.txt \\\n",
    "\t  --output_file=output.jsonl \\\n",
    "\t  --vocab_file=cased_L-24_H-1024_A-16/vocab.txt \\\n",
    "\t  --bert_config_file=cased_L-24_H-1024_A-16/bert_config.json \\\n",
    "\t  --init_checkpoint=cased_L-24_H-1024_A-16/bert_model.ckpt \\\n",
    "\t  --layers={} \\\n",
    "\t  --max_seq_length=300 \\\n",
    "\t  --batch_size=1\".format(layer))\n",
    "    \n",
    "    \n",
    "\tbert_output = pd.read_json(\"output.jsonl\", lines = True)\n",
    "\n",
    "\tos.system(\"rm output.jsonl\")\n",
    "\tos.system(\"rm input.txt\")\n",
    "\n",
    "\tindex = data.index\n",
    "\tcolumns = [\"emb_A\", \"emb_B\", \"emb_P\", \"label\"]\n",
    "\temb = pd.DataFrame(index = index, columns = columns)\n",
    "\temb.index.name = \"ID\"\n",
    "\n",
    "\tfor i in range(len(data)): # For each line in the data file\n",
    "\t\t# get the words A, B, Pronoun. Convert them to lower case, since we're using the uncased version of BERT\n",
    "\t\tP = data.loc[i,\"Pronoun\"].lower()\n",
    "\t\tA = data.loc[i,\"A\"].lower()\n",
    "\t\tB = data.loc[i,\"B\"].lower()\n",
    "\n",
    "\t\t# For each word, find the offset not counting spaces. This is necessary for comparison with the output of BERT\n",
    "\t\tP_offset = compute_offset_no_spaces(data.loc[i,\"Text\"], data.loc[i,\"Pronoun-offset\"])\n",
    "\t\tA_offset = compute_offset_no_spaces(data.loc[i,\"Text\"], data.loc[i,\"A-offset\"])\n",
    "\t\tB_offset = compute_offset_no_spaces(data.loc[i,\"Text\"], data.loc[i,\"B-offset\"])\n",
    "\t\t# Figure out the length of A, B, not counting spaces or special characters\n",
    "\t\tA_length = count_length_no_special(A)\n",
    "\t\tB_length = count_length_no_special(B)\n",
    "\n",
    "\t\t# Initialize embeddings with zeros\n",
    "\t\temb_A = np.zeros(embedding_size)\n",
    "\t\temb_B = np.zeros(embedding_size)\n",
    "\t\temb_P = np.zeros(embedding_size)\n",
    "\n",
    "\t\t# Initialize counts\n",
    "\t\tcount_chars = 0\n",
    "\t\tcnt_A, cnt_B, cnt_P = 0, 0, 0\n",
    "\n",
    "\t\tfeatures = pd.DataFrame(bert_output.loc[i,\"features\"]) # Get the BERT embeddings for the current line in the data file\n",
    "\t\tfor j in range(2,len(features)):  # Iterate over the BERT tokens for the current line; we skip over the first 2 tokens, which don't correspond to words\n",
    "\t\t\ttoken = features.loc[j,\"token\"]\n",
    "\n",
    "\t\t\t# See if the character count until the current token matches the offset of any of the 3 target words\n",
    "\t\t\tif count_chars  == P_offset: \n",
    "\t\t\t\t# print(token)\n",
    "\t\t\t\temb_P += np.array(features.loc[j,\"layers\"][0]['values'])\n",
    "\t\t\t\tcnt_P += 1\n",
    "\t\t\tif count_chars in range(A_offset, A_offset + A_length): \n",
    "\t\t\t\t# print(token)\n",
    "\t\t\t\temb_A += np.array(features.loc[j,\"layers\"][0]['values'])\n",
    "\t\t\t\tcnt_A +=1\n",
    "\t\t\tif count_chars in range(B_offset, B_offset + B_length): \n",
    "\t\t\t\t# print(token)\n",
    "\t\t\t\temb_B += np.array(features.loc[j,\"layers\"][0]['values'])\n",
    "\t\t\t\tcnt_B +=1\t\t\t\t\t\t\t\t\n",
    "\t\t\t# Update the character count\n",
    "\t\t\tcount_chars += count_length_no_special(token)\n",
    "\t\t# Taking the average between tokens in the span of A or B, so divide the current value by the count\t\n",
    "\t\temb_A /= cnt_A\n",
    "\t\temb_B /= cnt_B\n",
    "\n",
    "\t\t# Work out the label of the current piece of text\n",
    "\t\tlabel = \"Neither\"\n",
    "\t\tif (data.loc[i,\"A-coref\"] == True):\n",
    "\t\t\tlabel = \"A\"\n",
    "\t\tif (data.loc[i,\"B-coref\"] == True):\n",
    "\t\t\tlabel = \"B\"\n",
    "\n",
    "\t\t# Put everything together in emb\n",
    "\t\temb.iloc[i] = [emb_A, emb_B, emb_P, label]\n",
    "\n",
    "\treturn emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "8298947fa33285e722bdaae2df41bca5dd795732"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Started at  Sat Apr 13 16:46:49 2019\n",
      "17 Epoch\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:75: RuntimeWarning: invalid value encountered in true_divide\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:76: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished at  Sat Apr 13 17:01:05 2019\n",
      "18 Epoch\n",
      "Finished at  Sat Apr 13 17:15:29 2019\n",
      "19 Epoch\n",
      "Finished at  Sat Apr 13 17:30:41 2019\n",
      "20 Epoch\n",
      "Finished at  Sat Apr 13 17:45:36 2019\n",
      "21 Epoch\n",
      "Finished at  Sat Apr 13 18:00:47 2019\n",
      "22 Epoch\n",
      "Finished at  Sat Apr 13 18:16:46 2019\n",
      "23 Epoch\n",
      "Finished at  Sat Apr 13 18:33:09 2019\n"
     ]
    }
   ],
   "source": [
    "print(\"Started at \", time.ctime())\n",
    "\n",
    "for i in range(17, 24):\n",
    "    print(\"{} Epoch\".format(i))\n",
    "    tag = 'bert-large-cased-seq300-'\n",
    "    embedding_size = 1024\n",
    "    layer = i\n",
    "    tag = tag + str(layer)\n",
    "\n",
    "    test_data = pd.read_csv(\"gap-test.tsv\", sep = '\\t')\n",
    "    test_emb = run_bert(test_data, embedding_size=embedding_size, output=\"{}contextual_embeddings_gap_test.json\".format(tag), layer=layer)\n",
    "    test_emb.to_json(\"vector/bert_big_cased/{}contextual_embeddings_gap_test.json\".format(tag), orient = 'columns')\n",
    "\n",
    "    validation_data = pd.read_csv(\"gap-validation.tsv\", sep = '\\t')\n",
    "    validation_emb = run_bert(validation_data, embedding_size=embedding_size, output=\"{}contextual_embeddings_gap_validation.json\".format(tag), layer=layer)\n",
    "    validation_emb.to_json(\"vector/bert_big_cased/{}contextual_embeddings_gap_validation.json\".format(tag), orient = 'columns')\n",
    "\n",
    "    development_data = pd.read_csv(\"gap-development.tsv\", sep = '\\t')\n",
    "    development_emb = run_bert(development_data, embedding_size=embedding_size, output=\"{}contextual_embeddings_gap_development.json\".format(tag), layer=layer)\n",
    "    development_emb.to_json(\"vector/bert_big_cased/{}contextual_embeddings_gap_development.json\".format(tag), orient = 'columns')\n",
    "    print(\"Finished at \", time.ctime())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "29ba41d2570238ec22735909c76cd86c2742517c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras import backend, models, layers, initializers, regularizers, constraints, optimizers\n",
    "from keras import callbacks as kc\n",
    "from keras import optimizers as ko\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, KFold, train_test_split\n",
    "from sklearn.metrics import log_loss\n",
    "import time\n",
    "\n",
    "\n",
    "dense_layer_sizes = [37]\n",
    "dropout_rate = 0.6\n",
    "learning_rate = 0.001\n",
    "n_fold = 5\n",
    "batch_size = 32\n",
    "epochs = 1000\n",
    "patience = 100\n",
    "# n_test = 100\n",
    "lambd = 0.1 # L2 regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_uuid": "763ec8591474c45d6b065cad0c7efc2bbe9ad514",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_mlp_model(input_shape):\n",
    "\tX_input = layers.Input(input_shape)\n",
    "\tX = layers.Dropout(dropout_rate, seed = 7)(X_input)\n",
    "\t# First dense layer\n",
    "\tX = layers.Dense(dense_layer_sizes[0], name = 'dense0')(X)\n",
    "\tX = layers.BatchNormalization(name = 'bn0')(X)\n",
    "\tX = layers.Activation('relu')(X)\n",
    "\tX = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "\t# Output layer\n",
    "\tX = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "\tX = layers.Activation('softmax')(X)\n",
    "\n",
    "\t# Create model\n",
    "\tmodel = models.Model(input = X_input, output = X, name = \"classif_model\")\n",
    "\treturn model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_uuid": "0ea0ac2603b0a4bbdaa2776c8e37ad7894a99f5a",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_json(embeddings):\n",
    "\t'''\n",
    "\tParses the embeddigns given by BERT, and suitably formats them to be passed to the MLP model\n",
    "\n",
    "\tInput: embeddings, a DataFrame containing contextual embeddings from BERT, as well as the labels for the classification problem\n",
    "\tcolumns: \"emb_A\": contextual embedding for the word A\n",
    "\t         \"emb_B\": contextual embedding for the word B\n",
    "\t         \"emb_P\": contextual embedding for the pronoun\n",
    "\t         \"label\": the answer to the coreference problem: \"A\", \"B\" or \"NEITHER\"\n",
    "\n",
    "\tOutput: X, a numpy array containing, for each line in the GAP file, the concatenation of the embeddings of the target words\n",
    "\t        Y, a numpy array containing, for each line in the GAP file, the one-hot encoded answer to the coreference problem\n",
    "\t'''\n",
    "\tembeddings.sort_index(inplace = True) # Sorting the DataFrame, because reading from the json file messed with the order\n",
    "\tX = np.zeros((len(embeddings),3*1024))\n",
    "\tY = np.zeros((len(embeddings), 3))\n",
    "\n",
    "\t# Concatenate features\n",
    "\tfor i in range(len(embeddings)):\n",
    "\t\tA = np.array(embeddings.loc[i,\"emb_A\"])\n",
    "\t\tB = np.array(embeddings.loc[i,\"emb_B\"])\n",
    "\t\tP = np.array(embeddings.loc[i,\"emb_P\"])\n",
    "\t\tX[i] = np.concatenate((A,B,P))\n",
    "\n",
    "\t# One-hot encoding for labels\n",
    "\tfor i in range(len(embeddings)):\n",
    "\t\tlabel = embeddings.loc[i,\"label\"]\n",
    "\t\tif label == \"A\":\n",
    "\t\t\tY[i,0] = 1\n",
    "\t\telif label == \"B\":\n",
    "\t\t\tY[i,1] = 1\n",
    "\t\telse:\n",
    "\t\t\tY[i,2] = 1\n",
    "\n",
    "\treturn X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0 started at Sat Apr 13 18:37:20 2019\n",
      "WARNING:tensorflow:From C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\control_flow_ops.py:423: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Sat Apr 13 18:38:06 2019\n",
      "Fold 2 started at Sat Apr 13 18:39:03 2019\n",
      "Fold 3 started at Sat Apr 13 18:39:39 2019\n",
      "Fold 4 started at Sat Apr 13 18:40:40 2019\n",
      "In the 17 layer\n",
      "CV mean score: 0.4829, std: 0.0397.\n",
      "[0.4346496581889228, 0.44157181763493675, 0.5370932914761926, 0.5123143833008907, 0.4888957987282213]\n",
      "Test score: 0.43846707336013385\n",
      "Fold 0 started at Sat Apr 13 18:42:06 2019\n",
      "Fold 1 started at Sat Apr 13 18:43:00 2019\n",
      "Fold 2 started at Sat Apr 13 18:44:10 2019\n",
      "Fold 3 started at Sat Apr 13 18:45:14 2019\n",
      "Fold 4 started at Sat Apr 13 18:45:57 2019\n",
      "In the 18 layer\n",
      "CV mean score: 0.4581, std: 0.0343.\n",
      "[0.4193595699975912, 0.4183773830971185, 0.49989506578700443, 0.4901620703687713, 0.46277756677692217]\n",
      "Test score: 0.42972178614517226\n",
      "Fold 0 started at Sat Apr 13 18:46:55 2019\n",
      "Fold 1 started at Sat Apr 13 18:48:07 2019\n",
      "Fold 2 started at Sat Apr 13 18:49:36 2019\n",
      "Fold 3 started at Sat Apr 13 18:50:43 2019\n",
      "Fold 4 started at Sat Apr 13 18:51:38 2019\n",
      "In the 19 layer\n",
      "CV mean score: 0.4616, std: 0.0406.\n",
      "[0.42694433931571424, 0.4011178471579696, 0.4940855499225573, 0.5060295508571051, 0.4799860385664571]\n",
      "Test score: 0.43622680131277647\n",
      "Fold 0 started at Sat Apr 13 18:52:41 2019\n",
      "Fold 1 started at Sat Apr 13 18:54:11 2019\n",
      "Fold 2 started at Sat Apr 13 18:55:29 2019\n",
      "Fold 3 started at Sat Apr 13 18:56:46 2019\n",
      "Fold 4 started at Sat Apr 13 18:57:38 2019\n",
      "In the 20 layer\n",
      "CV mean score: 0.4751, std: 0.0431.\n",
      "[0.43767003520066344, 0.4132725541780882, 0.5089639065927796, 0.5277389604327991, 0.48761352518884044]\n",
      "Test score: 0.45204942501800666\n",
      "Fold 0 started at Sat Apr 13 19:00:14 2019\n",
      "Fold 1 started at Sat Apr 13 19:03:30 2019\n",
      "Fold 2 started at Sat Apr 13 19:07:36 2019\n",
      "Fold 3 started at Sat Apr 13 19:10:04 2019\n",
      "Fold 4 started at Sat Apr 13 19:10:57 2019\n",
      "In the 21 layer\n",
      "CV mean score: 0.4924, std: 0.0458.\n",
      "[0.46487460450202656, 0.41821025506580956, 0.5194842722263208, 0.5482539533334689, 0.5112084306587645]\n",
      "Test score: 0.4698730536699334\n",
      "Fold 0 started at Sat Apr 13 19:12:00 2019\n",
      "Fold 1 started at Sat Apr 13 19:14:16 2019\n",
      "Fold 2 started at Sat Apr 13 19:16:00 2019\n",
      "Fold 3 started at Sat Apr 13 19:17:21 2019\n",
      "Fold 4 started at Sat Apr 13 19:18:27 2019\n",
      "In the 22 layer\n",
      "CV mean score: 0.5193, std: 0.0444.\n",
      "[0.49275417588081516, 0.4477072774663682, 0.552950433003913, 0.5718185284069826, 0.5311560006160794]\n",
      "Test score: 0.4888120474687927\n",
      "Fold 0 started at Sat Apr 13 19:20:02 2019\n",
      "Fold 1 started at Sat Apr 13 19:21:14 2019\n",
      "Fold 2 started at Sat Apr 13 19:23:25 2019\n",
      "Fold 3 started at Sat Apr 13 19:24:29 2019\n",
      "Fold 4 started at Sat Apr 13 19:25:46 2019\n",
      "In the 23 layer\n",
      "CV mean score: 0.5563, std: 0.0456.\n",
      "[0.5374659632709372, 0.47934232456882714, 0.568694280906696, 0.6138326857717236, 0.581927944508165]\n",
      "Test score: 0.539190848914898\n"
     ]
    }
   ],
   "source": [
    "# Read development embeddigns from json file - this is the output of Bert\n",
    "for i in range(17, 24):\n",
    "    tag = 'bert-large-cased-seq300-'\n",
    "    embedding_size = 1024\n",
    "    layer = i\n",
    "    tag = tag + str(layer)\n",
    "\n",
    "    development = pd.read_json(\"vector/bert_big_cased/{}contextual_embeddings_gap_development.json\".format(tag))\n",
    "    X_development, Y_development = parse_json(development)\n",
    "\n",
    "    validation = pd.read_json(\"vector/bert_big_cased/{}contextual_embeddings_gap_validation.json\".format(tag))\n",
    "    X_validation, Y_validation = parse_json(validation)\n",
    "\n",
    "    test = pd.read_json(\"vector/bert_big_cased/{}contextual_embeddings_gap_test.json\".format(tag))\n",
    "    X_test, Y_test = parse_json(test)\n",
    "\n",
    "    # There may be a few NaN values, where the offset of a target word is greater than the max_seq_length of BERT.\n",
    "    # They are very few, so I'm just dropping the rows.\n",
    "    remove_test = [row for row in range(len(X_test)) if np.sum(np.isnan(X_test[row]))]\n",
    "    X_test = np.delete(X_test, remove_test, 0)\n",
    "    Y_test = np.delete(Y_test, remove_test, 0)\n",
    "\n",
    "    remove_validation = [row for row in range(len(X_validation)) if np.sum(np.isnan(X_validation[row]))]\n",
    "    X_validation = np.delete(X_validation, remove_validation, 0)\n",
    "    Y_validation = np.delete(Y_validation, remove_validation, 0)\n",
    "\n",
    "    # We want predictions for all development rows. So instead of removing rows, make them 0\n",
    "    remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row]))]\n",
    "    X_development[remove_development] = np.zeros(3*1024)\n",
    "\n",
    "    # Will train on data from the gap-test and gap-validation files, in total 2454 rows\n",
    "    X_train = np.concatenate((X_test, X_validation), axis = 0)\n",
    "    Y_train = np.concatenate((Y_test, Y_validation), axis = 0)\n",
    "\n",
    "    # Will predict probabilities for data from the gap-development file; initializing the predictions\n",
    "    prediction = np.zeros((len(X_development),3)) # testing predictions\n",
    "\n",
    "    # Training and cross-validation\n",
    "    folds = KFold(n_splits=n_fold, shuffle=True, random_state=3)\n",
    "    scores = []\n",
    "    oof = np.zeros_like(Y_train)\n",
    "    \n",
    "    for fold_n, (train_index, valid_index) in enumerate(folds.split(X_train)):\n",
    "        # split training and validation data\n",
    "        print('Fold', fold_n, 'started at', time.ctime())\n",
    "        X_tr, X_val = X_train[train_index], X_train[valid_index]\n",
    "        Y_tr, Y_val = Y_train[train_index], Y_train[valid_index]\n",
    "\n",
    "        # Define the model, re-initializing for each fold\n",
    "        classif_model = build_mlp_model([X_train.shape[1]])\n",
    "        classif_model.compile(optimizer = optimizers.Adam(lr = learning_rate), loss = \"categorical_crossentropy\")\n",
    "        callbacks = [kc.EarlyStopping(monitor='val_loss', patience=patience, restore_best_weights = True)]\n",
    "\n",
    "        # train the model\n",
    "        classif_model.fit(x = X_tr, y = Y_tr, epochs = epochs, batch_size = batch_size, callbacks = callbacks, validation_data = (X_val, Y_val), verbose = 0)\n",
    "\n",
    "        # make predictions on validation and test data\n",
    "        pred_valid = classif_model.predict(x = X_val, verbose = 0)\n",
    "        oof[valid_index] = pred_valid\n",
    "        pred = classif_model.predict(x = X_development, verbose = 0)\n",
    "\n",
    "        # oof[valid_index] = pred_valid.reshape(-1,)\n",
    "        scores.append(log_loss(Y_val, pred_valid))\n",
    "        prediction += pred\n",
    "    prediction /= n_fold\n",
    "\n",
    "    # Print CV scores, as well as score on the test data\n",
    "    print(\"In the {} layer\".format(i))\n",
    "    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n",
    "    print(scores)\n",
    "    print(\"Test score:\", log_loss(Y_development,prediction))\n",
    "    \n",
    "    # Write the prediction to file for submission\n",
    "    oof_df = pd.DataFrame(oof)\n",
    "    oof_df.to_csv(\"oof/oof_bert_cased_large_1024_seqlen300-L{}.csv\".format(layer), index=False)\n",
    "    \n",
    "    submission = pd.read_csv(\"data/sample_submission_stage_1.csv\", index_col = \"ID\")\n",
    "    submission[\"A\"] = prediction[:,0]\n",
    "    submission[\"B\"] = prediction[:,1]\n",
    "    submission[\"NEITHER\"] = prediction[:,2]\n",
    "    submission.to_csv(\"outputs/submission_bert_cased_large_1024_seqlen300-L{}.csv\".format(layer))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Records\n",
    "\n",
    "* Layer 01 CV mean score: 0.8874, std: 0.0154. Test score: 0.8495820849324255\n",
    "* Layer 02 CV mean score: 0.8700, std: 0.0127. Test score: 0.8438746537152055\n",
    "* Layer 03 CV mean score: 0.8042, std: 0.0209. Test score: 0.7804143238337299\n",
    "* Layer 04 CV mean score: 0.7707, std: 0.0307. Test score: 0.7545189016261488\n",
    "* Layer 05 CV mean score: 0.7750, std: 0.0202. Test score: 0.7548036766969642\n",
    "* Layer 06 CV mean score: 0.7625, std: 0.0147. Test score: 0.7379605187821101\n",
    "* Layer 07 CV mean score: 0.7603, std: 0.0148. Test score: 0.7383950867017735\n",
    "* Layer 08 CV mean score: 0.7489, std: 0.0168. Test score: 0.7151597505941151\n",
    "* Layer 09 CV mean score: 0.6968, std: 0.0223. Test score: 0.6628560374140844\n",
    "* Layer 10 CV mean score: 0.6388, std: 0.0300. Test score: 0.6216318814585869\n",
    "* Layer 11 CV mean score: 0.6115, std: 0.0214. Test score: 0.5907616362240837\n",
    "* Layer 12 CV mean score: 0.5796, std: 0.0232. Test score: 0.5618664425925821\n",
    "* Layer 13 CV mean score: 0.5436, std: 0.0287. Test score: 0.5226838338351586\n",
    "* Layer 14 CV mean score: 0.5089, std: 0.0334. Test score: 0.47300375639454456\n",
    "* Layer 15 CV mean score: 0.4669, std: 0.0401. Test score: 0.4352659449041463\n",
    "* Layer 16 CV mean score: 0.4499, std: 0.0456. Test score: 0.4087953151634497\n",
    "* Layer 17 CV mean score: 0.4361, std: 0.0257. Test score: 0.39111295910973526\n",
    "* Layer 18 CV mean score: 0.4297, std: 0.0270. Test score: 0.3832015453128307\n",
    "* Layer 19 CV mean score: 0.4181, std: 0.0243. Test score: 0.37968471044825763\n",
    "* Layer 20 CV mean score: 0.4457, std: 0.0240. Test score: 0.40375841880593566\n",
    "* Layer 21 CV mean score: 0.4636, std: 0.0275. Test score: 0.42320501486081\n",
    "* Layer 22 CV mean score: 0.4882, std: 0.0237. Test score: 0.44682373979905493\n",
    "* Layer 23 CV mean score: 0.5017, std: 0.0190. Test score: 0.4601968420633977\n",
    "* Layer 24 CV mean score: 0.5058, std: 0.0153. Test score: 0.46330062780985837"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}