model_6_cnn_with_cropped_imgs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 6: CNN with cropped images ##"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Needs editing.\n",
    "\n",
    "In this notebook, I implement a basic CNN from scratch. \n",
    "\n",
    "Predictions made using this model scored 0.281 by Kaggle, ranking 1011 out of 1335 (76th percentile) (submitted on Jan 18, 2019).\n",
    "\n",
    "Hardware used: CPU: i5 2.10GHz x 6, GPU: none: RAM: 16Gb + 32Gb virtual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# load libraries\n",
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "# import matplotlib.pyplot as plt\n",
    "# from PIL import Image\n",
    "from keras.preprocessing import image\n",
    "# from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Conv2D, AveragePooling2D, MaxPooling2D, BatchNormalization, GlobalAveragePooling2D, Flatten, Dropout, Dense\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.applications.imagenet_utils import preprocess_input\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Image         Id\n",
      "0  0000e88ab.jpg  w_f48451c\n",
      "1  0001f9222.jpg  w_c3d896a\n",
      "2  00029d126.jpg  w_20df2c5\n",
      "3  00050a15a.jpg  new_whale\n",
      "4  0005c1ef8.jpg  new_whale\n",
      "25361\n"
     ]
    }
   ],
   "source": [
    "# load train files and labels into dataframe\n",
    "traindf_all = pd.read_csv('train.csv')\n",
    "print(traindf_all.head())\n",
    "print(len(traindf_all))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Image         Id\n",
      "0  0000e88ab.jpg  w_f48451c\n",
      "1  0001f9222.jpg  w_c3d896a\n",
      "2  00029d126.jpg  w_20df2c5\n",
      "3  000a6daec.jpg  w_dd88965\n",
      "4  0016b897a.jpg  w_64404ac\n",
      "15697\n"
     ]
    }
   ],
   "source": [
    "# remove unlabeled images\n",
    "traindf = traindf_all.drop(traindf_all[traindf_all.Id == 'new_whale'].index.tolist())\n",
    "traindf.reset_index(drop=True, inplace=True)\n",
    "del traindf_all\n",
    "print(traindf.head())\n",
    "print(len(traindf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          Id  Count\n",
      "0  w_f48451c     14\n",
      "1  w_c3d896a      4\n",
      "2  w_20df2c5      4\n",
      "5004\n"
     ]
    }
   ],
   "source": [
    "# create dataframe with distinct ids and count of images per id\n",
    "ids = pd.DataFrame(traindf['Id'].unique(), columns=['Id'])\n",
    "ids['Count'] = 0\n",
    "for r in ids.itertuples():\n",
    "    id = r.Id\n",
    "    cnt = len(traindf[traindf['Id'] == id])\n",
    "    ind = ids[ids['Id'] == id].index.values[0]\n",
    "    ids.loc[ind, 'Count'] = cnt\n",
    "print(ids.head(3))\n",
    "print(len(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get image dimensions and color mode for all training images\n",
    "traindf['Width'] = 0\n",
    "traindf['Height'] = 0\n",
    "traindf['Mode'] = ''\n",
    "i = 0\n",
    "for r in traindf.itertuples(): \n",
    "    img_name = r.Image \n",
    "    img_path = 'train/' + img_name\n",
    "    img = Image.open(img_path) \n",
    "    width, height = img.size\n",
    "    mode = img.mode\n",
    "    traindf.loc[i, ['Width', 'Height', 'Mode']] = width, height, mode\n",
    "    i += 1\n",
    "print(traindf.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bbox = pd.read_csv('bounding_boxes.csv')\n",
    "print(bbox.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindf['x0'] = 0\n",
    "traindf['y0'] = 0\n",
    "traindf['x1'] = 0\n",
    "traindf['y1'] = 0\n",
    "i = 0\n",
    "for img in traindf.Image:\n",
    "    bb = np.array(bbox.loc[bbox.Image==img, ['x0','y0','x1','y1']])\n",
    "    x0 = bb[0][0]\n",
    "    y0 = bb[0][1]\n",
    "    x1 = bb[0][2]\n",
    "    y1 = bb[0][3]\n",
    "    traindf.loc[i, ['x0', 'y0', 'x1', 'y1']] = x0, y0, x1, y1\n",
    "    i += 1\n",
    "print(traindf.head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindf.to_csv('traindf.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to convert images to tensors\n",
    "def imgs_to_tensors(df, path, size=(100, 100)):\n",
    "    '''\n",
    "    df: dataframe listing image file names in column \"Image\"\n",
    "    path: directory where image files are located (don't include /)\n",
    "    size: target height and width to resize images to\n",
    "    '''\n",
    "    HEIGHT, WIDTH = size\n",
    "    LEN=df.shape[0]   \n",
    "    tensors = np.zeros((LEN, HEIGHT, WIDTH, 3))\n",
    "    i = 0\n",
    "    for im_name in df.Image:\n",
    "        if (i%1000==0):\n",
    "            print('Processing image {}: {}'.format(i, im_name))\n",
    "        im_path = path + '/' + im_name\n",
    "        # load image to PIL format\n",
    "        im = image.load_img(path=im_path, \n",
    "                            grayscale=False, \n",
    "                            color_mode='rgb', \n",
    "                            target_size=(HEIGHT, WIDTH), \n",
    "                            interpolation='nearest')\n",
    "        # convert to numpy array/tensor with shape (HEIGHT, WIDTH, 3)\n",
    "        x = image.img_to_array(im)\n",
    "        x = preprocess_input(x) # important line! I am not sure why\n",
    "        tensors[i] = x\n",
    "        i += 1   \n",
    "    return tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing image 0: 0000e88ab.jpg\n",
      "Processing image 1000: 10b694367.jpg\n",
      "Processing image 2000: 21e28ae02.jpg\n",
      "Processing image 3000: 32533a7fb.jpg\n",
      "Processing image 4000: 42f134dea.jpg\n",
      "Processing image 5000: 5297b6c40.jpg\n",
      "Processing image 6000: 6311688b7.jpg\n",
      "Processing image 7000: 7390cbfab.jpg\n",
      "Processing image 8000: 83336c385.jpg\n",
      "Processing image 9000: 92f450203.jpg\n",
      "Processing image 10000: a39babc55.jpg\n",
      "Processing image 11000: b36da6f7c.jpg\n",
      "Processing image 12000: c4160ee65.jpg\n",
      "Processing image 13000: d3b15e280.jpg\n",
      "Processing image 14000: e3fe27a84.jpg\n",
      "Processing image 15000: f3f3f8b92.jpg\n",
      "(15697, 100, 100, 3)\n"
     ]
    }
   ],
   "source": [
    "# create tensors and save on disk\n",
    "# (divide by 255 to normalize pixel values)\n",
    "tensors_train = imgs_to_tensors(df=traindf, path='cropped/train')/255\n",
    "np.save('tensors/model_6/tensors_train', tensors_train)\n",
    "print(tensors_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = np.array(ids.Id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create labels and save on disk\n",
    "tensors_train_labels = np.zeros((len(traindf), len(ids)))\n",
    "i = 0\n",
    "for id in traindf.Id:\n",
    "    j = np.argwhere(labels==id)[0, 0]\n",
    "    tensors_train_labels[i, j] = 1\n",
    "    i += 1\n",
    "np.save('tensors/model_1/tensors_train_labels', tensors_train_labels)\n",
    "print(tensors_train_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load previously saved tensors and labels, if any\n",
    "tensors_train = np.load('tensors/model_6/tensors_train.npy')\n",
    "tensors_train_labels = np.load('tensors/model_1/tensors_train_labels.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 100, 100, 16)      208       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 50, 50, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 50, 50, 32)        2080      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 25, 25, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 25, 25, 64)        8256      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 12, 12, 64)        0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 9216)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 500)               4608500   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 5004)              2507004   \n",
      "=================================================================\n",
      "Total params: 7,126,048\n",
      "Trainable params: 7,126,048\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# build basic model\n",
    "# (similar to one described in Lesson 2.18 in Deep Learning section of ML Nanodegree)\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Conv2D(filters=16, kernel_size=2, padding='same', activation='relu',\n",
    "                       input_shape=(tensors_train.shape[1], tensors_train.shape[2], 3)))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "model.add(Conv2D(filters=32, kernel_size=2, padding='same', activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "model.add(Conv2D(filters=64, kernel_size=2, padding='same', activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=2))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(500, activation='relu'))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(tensors_train_labels.shape[1], activation='softmax'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer=\"adam\", metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 14127 samples, validate on 1570 samples\n",
      "Epoch 1/10\n",
      "14127/14127 [==============================] - 85s 6ms/step - loss: 8.2750 - acc: 0.0042 - val_loss: 8.1431 - val_acc: 0.0115\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 8.14315, saving model to saved_models/weights.model_6.h5\n",
      "Epoch 2/10\n",
      "14127/14127 [==============================] - 83s 6ms/step - loss: 7.7928 - acc: 0.0118 - val_loss: 8.1618 - val_acc: 0.0172\n",
      "\n",
      "Epoch 00002: val_loss did not improve from 8.14315\n",
      "Epoch 3/10\n",
      "14127/14127 [==============================] - 84s 6ms/step - loss: 7.4095 - acc: 0.0173 - val_loss: 7.9881 - val_acc: 0.0299\n",
      "\n",
      "Epoch 00003: val_loss improved from 8.14315 to 7.98809, saving model to saved_models/weights.model_6.h5\n",
      "Epoch 4/10\n",
      "14127/14127 [==============================] - 165s 12ms/step - loss: 6.9877 - acc: 0.0281 - val_loss: 8.0409 - val_acc: 0.0389\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 7.98809\n",
      "Epoch 5/10\n",
      "14127/14127 [==============================] - 87s 6ms/step - loss: 6.4007 - acc: 0.0477 - val_loss: 8.2173 - val_acc: 0.0408\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 7.98809\n",
      "Epoch 6/10\n",
      "14127/14127 [==============================] - 88s 6ms/step - loss: 5.6035 - acc: 0.0890 - val_loss: 8.6822 - val_acc: 0.0510\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 7.98809\n",
      "Epoch 7/10\n",
      "14127/14127 [==============================] - 88s 6ms/step - loss: 4.5153 - acc: 0.1797 - val_loss: 9.2346 - val_acc: 0.0541\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 7.98809\n",
      "Epoch 8/10\n",
      "14127/14127 [==============================] - 89s 6ms/step - loss: 3.3251 - acc: 0.3377 - val_loss: 9.8960 - val_acc: 0.0535\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 7.98809\n",
      "Epoch 9/10\n",
      "14127/14127 [==============================] - 89s 6ms/step - loss: 2.2169 - acc: 0.5194 - val_loss: 10.6306 - val_acc: 0.0516\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 7.98809\n",
      "Epoch 10/10\n",
      "14127/14127 [==============================] - 85s 6ms/step - loss: 1.4291 - acc: 0.6590 - val_loss: 11.0277 - val_acc: 0.0580\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 7.98809\n"
     ]
    }
   ],
   "source": [
    "# train the model\n",
    "EPOCHS = 10\n",
    "BATCH_SIZE = 16\n",
    "checkpointer = ModelCheckpoint(filepath='saved_models/weights.model_6.h5', verbose=1, save_best_only=True)\n",
    "history = model.fit(\n",
    "        x=tensors_train,\n",
    "        y=tensors_train_labels,\n",
    "        batch_size=BATCH_SIZE,\n",
    "        epochs=EPOCHS,\n",
    "        callbacks=[checkpointer],\n",
    "        validation_split=0.1,\n",
    "        verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load best weights\n",
    "model.load_weights('saved_models/weights.model_6.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Image\n",
      "0  21253f840.jpg\n",
      "1  769f8d32b.jpg\n",
      "2  a69dc856e.jpg\n",
      "7960\n"
     ]
    }
   ],
   "source": [
    "# load test files into dataframe\n",
    "filelist = os.listdir('test')\n",
    "testdf = pd.DataFrame(filelist, columns=['Image'])\n",
    "print(testdf.head(3))\n",
    "print(len(testdf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing image 0: 21253f840.jpg\n",
      "Processing image 1000: b14876130.jpg\n",
      "Processing image 2000: 8d8c7a728.jpg\n",
      "Processing image 3000: ca3921cc1.jpg\n",
      "Processing image 4000: e23615e20.jpg\n",
      "Processing image 5000: 0e5538c86.jpg\n",
      "Processing image 6000: 3234bf468.jpg\n",
      "Processing image 7000: 465b5b1ab.jpg\n",
      "(7960, 100, 100, 3)\n"
     ]
    }
   ],
   "source": [
    "# create tensors for test images and save on disk\n",
    "tensors_test = imgs_to_tensors(df=testdf, path='cropped/test')/255\n",
    "np.save('tensors/model_6/tensors_test', tensors_test)\n",
    "print(tensors_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load previously saved test tensors, if any\n",
    "tensors_test = np.load('tensors/model_6/tensors_test.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7960/7960 [==============================] - 9s 1ms/step\n"
     ]
    }
   ],
   "source": [
    "# make predictions\n",
    "predictions = model.predict(tensors_test, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Image                                                 Id\n",
      "0  21253f840.jpg  new_whale w_23a388d w_e966f36 w_d405854 w_f765256\n",
      "1  769f8d32b.jpg  new_whale w_9c506f6 w_6cda039 w_f765256 w_60ce6fc\n",
      "2  a69dc856e.jpg  new_whale w_f0fe284 w_4bf625b w_23a388d w_9c506f6\n",
      "3  79bee536e.jpg  new_whale w_9c506f6 w_6cda039 w_fd3e556 w_23a388d\n",
      "4  7eb9a6f1b.jpg  new_whale w_3de579a w_23a388d w_789c969 w_d405854\n",
      "5  8e0a9e74b.jpg  new_whale w_3de579a w_789c969 w_4690940 w_f0fe284\n",
      "6  4853537ad.jpg  new_whale w_343f088 w_3de579a w_789c969 w_700ebb4\n",
      "7  8cba4a867.jpg  new_whale w_9c506f6 w_23a388d w_60ce6fc w_6cda039\n",
      "8  8da08a11a.jpg  new_whale w_9b5109b w_cd4cb49 w_8da30ad w_0369a5c\n",
      "9  48a937823.jpg  new_whale w_3de579a w_789c969 w_4690940 w_f0fe284\n"
     ]
    }
   ],
   "source": [
    "# get 5 best predictions per image and decode to whale ids\n",
    "# insert \"new_whale\" where prediction probability drops below 10% \n",
    "testdf['Id'] = ''\n",
    "for i, pred in enumerate(predictions):\n",
    "    inx = np.argsort(pred)[-5:][::-1].tolist()\n",
    "    preds = labels[inx].tolist()\n",
    "    probs = pred[inx]\n",
    "    try:\n",
    "        # get index of first prediction with prob less than 10%\n",
    "        j = (probs < 0.1).tolist().index(True)\n",
    "        # enter \"new_whale\" in that index, and shift any remaining preds to right\n",
    "        for ii in range(4, (j-1), -1):\n",
    "            if ii==j:\n",
    "                preds[ii] = 'new_whale'\n",
    "            else:\n",
    "                preds[ii] = preds[ii-1]\n",
    "    except ValueError:\n",
    "        pass\n",
    "    testdf.loc[i,'Id'] = ' '.join(preds)\n",
    "print(testdf.head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write to file and submit to Kaggle\n",
    "testdf.to_csv('submissions/submit_0122_02.txt', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This submission scored 0.281 in Kaggle, ranking me 1011 out of 1335 (76 percentile):\n",
    "\n",
    "![title](submissions/screenshots/submit_0118_03.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}