Stage 1.2 Naive-concat-classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sample_submission_stage_1.csv', 'test_stage_1.tsv', 'trees']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"data\"))\n",
    "import zipfile\n",
    "import sys\n",
    "import time\n",
    "\n",
    "from keras import backend, models, layers, initializers, regularizers, constraints, optimizers\n",
    "from keras import callbacks as kc\n",
    "from keras import optimizers as ko\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, KFold, train_test_split\n",
    "from sklearn.metrics import log_loss\n",
    "import time\n",
    "\n",
    "\n",
    "dense_layer_sizes = [37]\n",
    "dropout_rate = 0.6\n",
    "learning_rate = 0.001\n",
    "n_fold = 5\n",
    "batch_size = 32\n",
    "epochs = 1000\n",
    "patience = 100\n",
    "# n_test = 100\n",
    "lambd = 0.1 # L2 regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_mlp_model(input_shape):\n",
    "\tX_input = layers.Input(input_shape)\n",
    "\tX = layers.Dropout(dropout_rate, seed = 7)(X_input)\n",
    "\t# First dense layer\n",
    "\tX = layers.Dense(dense_layer_sizes[0], name = 'dense0')(X)\n",
    "\tX = layers.BatchNormalization(name = 'bn0')(X)\n",
    "\tX = layers.Activation('relu')(X)\n",
    "\tX = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "\t# Output layer\n",
    "\tX = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "\tX = layers.Activation('softmax')(X)\n",
    "\n",
    "\t# Create model\n",
    "\tmodel = models.Model(input = X_input, output = X, name = \"classif_model\")\n",
    "\treturn model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "0ea0ac2603b0a4bbdaa2776c8e37ad7894a99f5a",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_json(embeddings, embedding_size):\n",
    "    '''\n",
    "    Parses the embeddigns given by BERT, and suitably formats them to be passed to the MLP model\n",
    "\n",
    "    Input: embeddings, a DataFrame containing contextual embeddings from BERT, as well as the labels for the classification problem\n",
    "    columns: \"emb_A\": contextual embedding for the word A\n",
    "             \"emb_B\": contextual embedding for the word B\n",
    "             \"emb_P\": contextual embedding for the pronoun\n",
    "             \"label\": the answer to the coreference problem: \"A\", \"B\" or \"NEITHER\"\n",
    "\n",
    "    Output: X, a numpy array containing, for each line in the GAP file, the concatenation of the embeddings of the target words\n",
    "            Y, a numpy array containing, for each line in the GAP file, the one-hot encoded answer to the coreference problem\n",
    "    '''\n",
    "    embeddings.sort_index(inplace = True) # Sorting the DataFrame, because reading from the json file messed with the order\n",
    "    X = np.zeros((len(embeddings),3* embedding_size))\n",
    "    Y = np.zeros((len(embeddings), 3))\n",
    "\n",
    "    # Concatenate features\n",
    "    for i in range(len(embeddings)):\n",
    "        A = np.array(embeddings.loc[i,\"emb_A\"])\n",
    "        B = np.array(embeddings.loc[i,\"emb_B\"])\n",
    "        P = np.array(embeddings.loc[i,\"emb_P\"])\n",
    "        X[i] = np.concatenate((A,B,P))\n",
    "\n",
    "    # One-hot encoding for labels\n",
    "    for i in range(len(embeddings)):\n",
    "        label = embeddings.loc[i,\"label\"]\n",
    "        if label == \"A\":\n",
    "            Y[i,0] = 1\n",
    "        elif label == \"B\":\n",
    "            Y[i,1] = 1\n",
    "        else:\n",
    "            Y[i,2] = 1\n",
    "\n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_filename = \"contextual_embeddings_gap_development.json\"\n",
    "val_filename = \"contextual_embeddings_gap_validation.json\"\n",
    "test_filename = \"contextual_embeddings_gap_test.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Keep all BERTs learn on the same corpus\n",
    "remove_test = []\n",
    "remove_validation = []\n",
    "remove_development = [209, 1506, 1988]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_tag,\n",
    "                dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "                oof_folder='oof/', pred_folder='outputs/'):\n",
    "    '''\n",
    "    Arguments:\n",
    "        tag (data_tag): the tag of model and vec\n",
    "        embedding_size: the size of bert embedding\n",
    "        layer: to indicate which layer of bert is going to use\n",
    "        checkpoint_path: the path to model_checkpoint folder\n",
    "        n_fold: the number of CV folds\n",
    "        model_tag: the prefix of the prediction and oof file and model chekpoint\n",
    "    '''\n",
    "\n",
    "    tag = tag + str(layer) # follow the original naming style\n",
    "    dev_filename = tag + dev_filename\n",
    "    val_filename = tag + val_filename\n",
    "    test_filename = tag + test_filename\n",
    "    \n",
    "    development = pd.read_json(os.path.join(dev_folder_path, dev_filename))\n",
    "    X_development, Y_development = parse_json(development, embedding_size)\n",
    "\n",
    "    validation = pd.read_json(os.path.join(val_folder_path, val_filename))\n",
    "    X_validation, Y_validation = parse_json(validation, embedding_size)\n",
    "\n",
    "    test = pd.read_json(os.path.join(test_folder_path, test_filename))\n",
    "    X_test, Y_test = parse_json(test, embedding_size)\n",
    "\n",
    "    # There may be a few NaN values, where the offset of a target word is greater than the max_seq_length of BERT.\n",
    "    # They are very few, so I'm just dropping the rows.\n",
    "    # remove_test = [row for row in range(len(X_test)) if np.sum(np.isnan(X_test[row].reshape(-1)))]\n",
    "    X_test = np.delete(X_test, remove_test, 0)\n",
    "    Y_test = np.delete(Y_test, remove_test, 0)\n",
    "\n",
    "    # remove_validation = [row for row in range(len(X_validation)) if np.sum(np.isnan(X_validation[row].reshape(-1)))]\n",
    "    X_validation = np.delete(X_validation, remove_validation, 0)\n",
    "    Y_validation = np.delete(Y_validation, remove_validation, 0)\n",
    "\n",
    "    # We want predictions for all development rows. So instead of removing rows, make them 0\n",
    "    # remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row].reshape(-1)))]\n",
    "    X_development = np.delete(X_development, remove_development, 0)\n",
    "    Y_development = np.delete(Y_development, remove_development, 0)\n",
    "    \n",
    "    # Will train on data from the gap-test and gap-validation files, in total 2454 rows\n",
    "    X_train = np.concatenate((X_test, X_validation, X_development), axis = 0)\n",
    "    Y_train = np.concatenate((Y_test, Y_validation, Y_development), axis = 0)\n",
    "\n",
    "    # Will predict probabilities for data from the gap-development file; initializing the predictions\n",
    "    prediction = np.zeros((len(X_development), 3)) # testing predictions\n",
    "\n",
    "    # Training and cross-validation\n",
    "    folds = KFold(n_splits=n_fold, shuffle=True, random_state=3)\n",
    "    scores = []\n",
    "    oof = np.zeros_like(Y_train)\n",
    "    \n",
    "    for fold_n, (train_index, valid_index) in enumerate(folds.split(X_train)):\n",
    "        # split training and validation data\n",
    "        print('Fold', fold_n, 'started at', time.ctime())\n",
    "        X_tr, X_val = X_train[train_index], X_train[valid_index]\n",
    "        Y_tr, Y_val = Y_train[train_index], Y_train[valid_index]\n",
    "\n",
    "        # Define the model, re-initializing for each fold\n",
    "        classif_model = build_mlp_model([X_train.shape[-1]])\n",
    "        classif_model.compile(optimizer=optimizers.Adam(lr=learning_rate), loss=\"categorical_crossentropy\")\n",
    "\n",
    "        callbacks = [\n",
    "                     kc.EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=True),\n",
    "                     kc.ModelCheckpoint(os.path.join(checkpoint_path, model_tag + tag + str(fold_n) + '.pt'), monitor='val_loss', verbose=0, save_best_only=True, mode='min')]\n",
    "\n",
    "        # train the model\n",
    "        classif_model.fit(x=X_tr, y=Y_tr, epochs=epochs, batch_size=batch_size, \n",
    "                          callbacks=callbacks, validation_data=(X_val, Y_val), verbose=0)\n",
    "\n",
    "        # make predictions on validation and test data\n",
    "        pred_valid = classif_model.predict(x=X_val, verbose=0)\n",
    "        oof[valid_index] = pred_valid\n",
    "        pred = classif_model.predict(x=X_development, verbose=0)\n",
    "\n",
    "        # oof[valid_index] = pred_valid.reshape(-1,)\n",
    "        scores.append(log_loss(Y_val, pred_valid))\n",
    "        prediction += pred\n",
    "    \n",
    "    prediction /= n_fold    \n",
    "    \n",
    "    print(\"-\" * 30)\n",
    "    print(\"For the model\", tag)\n",
    "    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))\n",
    "    print(scores)\n",
    "    \n",
    "    # Write the prediction to file for submission\n",
    "    oof_df = pd.DataFrame(oof)\n",
    "    oof_df.to_csv(oof_folder + model_tag + tag + \".csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bert Base\n",
    "## Uncased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-uncased-seq512-\"\n",
    "pred_tag = \"naive-\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0 started at Mon Apr 15 14:30:58 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Mon Apr 15 14:32:27 2019\n",
      "Fold 2 started at Mon Apr 15 14:33:55 2019\n",
      "Fold 3 started at Mon Apr 15 14:35:15 2019\n",
      "Fold 4 started at Mon Apr 15 14:36:32 2019\n",
      "Fold 5 started at Mon Apr 15 14:38:04 2019\n",
      "Fold 6 started at Mon Apr 15 14:40:07 2019\n",
      "------------------------------\n",
      "For the model bert-base-uncased-seq512-8\n",
      "CV mean score: 0.4393, std: 0.0198.\n",
      "[0.4528021516174784, 0.4419618785703496, 0.422456534245503, 0.4218643427577846, 0.4103121842551371, 0.4633743767757231, 0.4626042989704471]\n",
      "Test score: 0.22137542657638595\n",
      "------------------------------\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Length of values does not match length of index",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-367cf3bddce0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n\u001b[0;32m      2\u001b[0m             \u001b[0mdev_filename\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdev_filename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_filename\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mval_filename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_filename\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtest_filename\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m             oof_folder='oof/', pred_folder='outputs/', model_tag=pred_tag)\n\u001b[0m",
      "\u001b[1;32m<ipython-input-6-104b924c741c>\u001b[0m in \u001b[0;36mtrain_folds\u001b[1;34m(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_tag, dev_filename, val_filename, test_filename, oof_folder, pred_folder)\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m     \u001b[0msubmission\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"data/sample_submission_stage_1.csv\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex_col\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"ID\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 96\u001b[1;33m     \u001b[0msubmission\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"A\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mprediction\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     97\u001b[0m     \u001b[0msubmission\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"B\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mprediction\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m     \u001b[0msubmission\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"NEITHER\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mprediction\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__setitem__\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m   3117\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3118\u001b[0m             \u001b[1;31m# set column\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3119\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3120\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3121\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_setitem_slice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_set_item\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m   3192\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3193\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_ensure_valid_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3194\u001b[1;33m         \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sanitize_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3195\u001b[0m         \u001b[0mNDFrame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3196\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_sanitize_column\u001b[1;34m(self, key, value, broadcast)\u001b[0m\n\u001b[0;32m   3389\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3390\u001b[0m             \u001b[1;31m# turn me into an ndarray\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3391\u001b[1;33m             \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_sanitize_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3392\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mIndex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3393\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36m_sanitize_index\u001b[1;34m(data, index, copy)\u001b[0m\n\u001b[0;32m   3998\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3999\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4000\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Length of values does not match length of '\u001b[0m \u001b[1;34m'index'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4001\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4002\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mABCIndexClass\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Length of values does not match length of index"
     ]
    }
   ],
   "source": [
    "train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base_cased\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-cased-seq512-\"\n",
    "pred_tag = \"naive-\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0 started at Mon Apr 15 14:43:29 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Mon Apr 15 14:44:23 2019\n",
      "Fold 2 started at Mon Apr 15 14:45:26 2019\n",
      "Fold 3 started at Mon Apr 15 14:46:30 2019\n",
      "Fold 4 started at Mon Apr 15 14:47:30 2019\n",
      "Fold 5 started at Mon Apr 15 14:48:46 2019\n",
      "Fold 6 started at Mon Apr 15 14:49:36 2019\n",
      "------------------------------\n",
      "For the model bert-base-cased-seq512-8\n",
      "CV mean score: 0.4905, std: 0.0260.\n",
      "[0.4972360841906117, 0.46110386217475, 0.4736347498743591, 0.49113536845971745, 0.4619680452013188, 0.5084957606328465, 0.5395958285293162]\n",
      "Test score: 0.2906273791398692\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bert Big\n",
    "## Uncased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big\"\n",
    "embedding_size = 1024\n",
    "layer = 19\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-uncased-seq300-\"\n",
    "pred_tag = \"naive-\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0 started at Mon Apr 15 14:51:11 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Mon Apr 15 14:51:58 2019\n",
      "Fold 2 started at Mon Apr 15 14:53:08 2019\n",
      "Fold 3 started at Mon Apr 15 14:54:27 2019\n",
      "Fold 4 started at Mon Apr 15 14:55:58 2019\n",
      "Fold 5 started at Mon Apr 15 14:56:57 2019\n",
      "Fold 6 started at Mon Apr 15 14:57:45 2019\n",
      "------------------------------\n",
      "For the model bert-large-uncased-seq300-19\n",
      "CV mean score: 0.3805, std: 0.0313.\n",
      "[0.37305250019210306, 0.3738718491489601, 0.37252183842618986, 0.3491585163411635, 0.34228276159344245, 0.42119303584404866, 0.4313765856804964]\n",
      "Test score: 0.17803088601250697\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big_cased\"\n",
    "embedding_size = 1024\n",
    "layer = 18\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-cased-seq300-\"\n",
    "pred_tag = \"naive-\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0 started at Mon Apr 15 14:58:37 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 1 started at Mon Apr 15 14:59:26 2019\n",
      "Fold 2 started at Mon Apr 15 15:00:32 2019\n",
      "Fold 3 started at Mon Apr 15 15:01:37 2019\n",
      "Fold 4 started at Mon Apr 15 15:02:27 2019\n",
      "Fold 5 started at Mon Apr 15 15:03:44 2019\n",
      "Fold 6 started at Mon Apr 15 15:04:40 2019\n",
      "------------------------------\n",
      "For the model bert-large-cased-seq300-18\n",
      "CV mean score: 0.4190, std: 0.0237.\n",
      "[0.4233008980611559, 0.4071883598085637, 0.40699723165057805, 0.41427951452648853, 0.37972288588196007, 0.4472983241099176, 0.454350395469127]\n",
      "Test score: 0.22459776728373343\n",
      "------------------------------\n"
     ]
    }
   ],
   "source": [
    "train_folds(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}