AmExpert_keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import datetime\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.keras.models import Sequential\n",
    "from tensorflow.python.keras.layers import Dense\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "from tensorflow.python.keras.wrappers.scikit_learn import KerasRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((32820, 531), (14067, 531))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = pd.read_csv('./dataset/X.csv')\n",
    "df_test  = pd.read_csv('./dataset/X_test.csv')\n",
    "X.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32820, 1)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y=pd.read_csv('./dataset/df_labels.csv')\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test[np.isnan(df_test)]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scaler_x = MinMaxScaler()\n",
    "# scaler_y = MinMaxScaler()\n",
    "# print(scaler_x.fit(X))\n",
    "# xscale=scaler_x.transform(X)\n",
    "# print(scaler_y.fit(y))\n",
    "# yscale=scaler_y.transform(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    model = Sequential([\n",
    "    Dense(64, activation=tf.nn.relu, input_shape=[len(X_train.keys())]),\n",
    "    Dense(64, activation=tf.nn.relu),         \n",
    "    Dense(1)\n",
    "  ])\n",
    "    \n",
    "    optimizer = tf.keras.optimizers.RMSprop(0.0001)\n",
    "    \n",
    "    model.compile(loss='mse',\n",
    "                optimizer=optimizer,\n",
    "                metrics=['mse','msle'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "regressor = KerasRegressor(build_fn=build_model, batch_size=32, epochs=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "22974/22974 [==============================] - 3s 121us/sample - loss: 12.4341 - mean_squared_error: 12.4341 - mean_squared_logarithmic_error: 0.4433\n",
      "Epoch 2/50\n",
      "22974/22974 [==============================] - 3s 115us/sample - loss: 1.6279 - mean_squared_error: 1.6279 - mean_squared_logarithmic_error: 0.0239\n",
      "Epoch 3/50\n",
      "22974/22974 [==============================] - 3s 114us/sample - loss: 1.4628 - mean_squared_error: 1.4628 - mean_squared_logarithmic_error: 0.0217\n",
      "Epoch 4/50\n",
      "22974/22974 [==============================] - 3s 114us/sample - loss: 1.4319 - mean_squared_error: 1.4319 - mean_squared_logarithmic_error: 0.0213\n",
      "Epoch 5/50\n",
      "22974/22974 [==============================] - 3s 115us/sample - loss: 1.4142 - mean_squared_error: 1.4142 - mean_squared_logarithmic_error: 0.0211\n",
      "Epoch 6/50\n",
      "22974/22974 [==============================] - 3s 115us/sample - loss: 1.4036 - mean_squared_error: 1.4036 - mean_squared_logarithmic_error: 0.0209\n",
      "Epoch 7/50\n",
      "22974/22974 [==============================] - 3s 116us/sample - loss: 1.3948 - mean_squared_error: 1.3948 - mean_squared_logarithmic_error: 0.0208\n",
      "Epoch 8/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3893 - mean_squared_error: 1.3893 - mean_squared_logarithmic_error: 0.0208\n",
      "Epoch 9/50\n",
      "22974/22974 [==============================] - 3s 116us/sample - loss: 1.3847 - mean_squared_error: 1.3847 - mean_squared_logarithmic_error: 0.0207\n",
      "Epoch 10/50\n",
      "22974/22974 [==============================] - 3s 119us/sample - loss: 1.3815 - mean_squared_error: 1.3815 - mean_squared_logarithmic_error: 0.0207\n",
      "Epoch 11/50\n",
      "22974/22974 [==============================] - 3s 117us/sample - loss: 1.3766 - mean_squared_error: 1.3766 - mean_squared_logarithmic_error: 0.0206\n",
      "Epoch 12/50\n",
      "22974/22974 [==============================] - 3s 117us/sample - loss: 1.3752 - mean_squared_error: 1.3752 - mean_squared_logarithmic_error: 0.0206\n",
      "Epoch 13/50\n",
      "22974/22974 [==============================] - 3s 111us/sample - loss: 1.3717 - mean_squared_error: 1.3717 - mean_squared_logarithmic_error: 0.0205\n",
      "Epoch 14/50\n",
      "22974/22974 [==============================] - 3s 112us/sample - loss: 1.3709 - mean_squared_error: 1.3709 - mean_squared_logarithmic_error: 0.0205\n",
      "Epoch 15/50\n",
      "22974/22974 [==============================] - 3s 112us/sample - loss: 1.3677 - mean_squared_error: 1.3677 - mean_squared_logarithmic_error: 0.0205\n",
      "Epoch 16/50\n",
      "22974/22974 [==============================] - 3s 113us/sample - loss: 1.3658 - mean_squared_error: 1.3658 - mean_squared_logarithmic_error: 0.0205\n",
      "Epoch 17/50\n",
      "22974/22974 [==============================] - 3s 114us/sample - loss: 1.3637 - mean_squared_error: 1.3637 - mean_squared_logarithmic_error: 0.0204\n",
      "Epoch 18/50\n",
      "22974/22974 [==============================] - 3s 124us/sample - loss: 1.3640 - mean_squared_error: 1.3640 - mean_squared_logarithmic_error: 0.0204\n",
      "Epoch 19/50\n",
      "22974/22974 [==============================] - 3s 122us/sample - loss: 1.3638 - mean_squared_error: 1.3638 - mean_squared_logarithmic_error: 0.0204\n",
      "Epoch 20/50\n",
      "22974/22974 [==============================] - 3s 111us/sample - loss: 1.3597 - mean_squared_error: 1.3597 - mean_squared_logarithmic_error: 0.0204\n",
      "Epoch 21/50\n",
      "22974/22974 [==============================] - 3s 110us/sample - loss: 1.3586 - mean_squared_error: 1.3586 - mean_squared_logarithmic_error: 0.0204\n",
      "Epoch 22/50\n",
      "22974/22974 [==============================] - 3s 114us/sample - loss: 1.3576 - mean_squared_error: 1.3576 - mean_squared_logarithmic_error: 0.0204\n",
      "Epoch 23/50\n",
      "22974/22974 [==============================] - 3s 115us/sample - loss: 1.3567 - mean_squared_error: 1.3567 - mean_squared_logarithmic_error: 0.0203\n",
      "Epoch 24/50\n",
      "22974/22974 [==============================] - 3s 110us/sample - loss: 1.3550 - mean_squared_error: 1.3550 - mean_squared_logarithmic_error: 0.0203\n",
      "Epoch 25/50\n",
      "22974/22974 [==============================] - 3s 111us/sample - loss: 1.3546 - mean_squared_error: 1.3546 - mean_squared_logarithmic_error: 0.0203\n",
      "Epoch 26/50\n",
      "22974/22974 [==============================] - 3s 112us/sample - loss: 1.3513 - mean_squared_error: 1.3513 - mean_squared_logarithmic_error: 0.0203\n",
      "Epoch 27/50\n",
      "22974/22974 [==============================] - 3s 122us/sample - loss: 1.3524 - mean_squared_error: 1.3524 - mean_squared_logarithmic_error: 0.0203s - loss: 1.2693 - mean_squared_err\n",
      "Epoch 28/50\n",
      "22974/22974 [==============================] - 3s 120us/sample - loss: 1.3508 - mean_squared_error: 1.3508 - mean_squared_logarithmic_error: 0.0203\n",
      "Epoch 29/50\n",
      "22974/22974 [==============================] - 3s 119us/sample - loss: 1.3497 - mean_squared_error: 1.3497 - mean_squared_logarithmic_error: 0.0202\n",
      "Epoch 30/50\n",
      "22974/22974 [==============================] - 3s 121us/sample - loss: 1.3466 - mean_squared_error: 1.3466 - mean_squared_logarithmic_error: 0.0202\n",
      "Epoch 31/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3463 - mean_squared_error: 1.3463 - mean_squared_logarithmic_error: 0.0202\n",
      "Epoch 32/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3448 - mean_squared_error: 1.3448 - mean_squared_logarithmic_error: 0.0202\n",
      "Epoch 33/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3435 - mean_squared_error: 1.3435 - mean_squared_logarithmic_error: 0.0202\n",
      "Epoch 34/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3431 - mean_squared_error: 1.3431 - mean_squared_logarithmic_error: 0.0202\n",
      "Epoch 35/50\n",
      "22974/22974 [==============================] - 3s 117us/sample - loss: 1.3430 - mean_squared_error: 1.3430 - mean_squared_logarithmic_error: 0.0202\n",
      "Epoch 36/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3412 - mean_squared_error: 1.3412 - mean_squared_logarithmic_error: 0.0201\n",
      "Epoch 37/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3396 - mean_squared_error: 1.3396 - mean_squared_logarithmic_error: 0.0201\n",
      "Epoch 38/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3398 - mean_squared_error: 1.3398 - mean_squared_logarithmic_error: 0.0201\n",
      "Epoch 39/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3372 - mean_squared_error: 1.3372 - mean_squared_logarithmic_error: 0.0201\n",
      "Epoch 40/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3371 - mean_squared_error: 1.3371 - mean_squared_logarithmic_error: 0.0201\n",
      "Epoch 41/50\n",
      "22974/22974 [==============================] - 3s 122us/sample - loss: 1.3344 - mean_squared_error: 1.3344 - mean_squared_logarithmic_error: 0.0201\n",
      "Epoch 42/50\n",
      "22974/22974 [==============================] - 3s 122us/sample - loss: 1.3333 - mean_squared_error: 1.3333 - mean_squared_logarithmic_error: 0.0200\n",
      "Epoch 43/50\n",
      "22974/22974 [==============================] - 3s 129us/sample - loss: 1.3336 - mean_squared_error: 1.3336 - mean_squared_logarithmic_error: 0.0200\n",
      "Epoch 44/50\n",
      "22974/22974 [==============================] - 3s 123us/sample - loss: 1.3315 - mean_squared_error: 1.3315 - mean_squared_logarithmic_error: 0.0200\n",
      "Epoch 45/50\n",
      "22974/22974 [==============================] - 3s 122us/sample - loss: 1.3315 - mean_squared_error: 1.3315 - mean_squared_logarithmic_error: 0.0200\n",
      "Epoch 46/50\n",
      "22974/22974 [==============================] - 3s 118us/sample - loss: 1.3309 - mean_squared_error: 1.3309 - mean_squared_logarithmic_error: 0.0200\n",
      "Epoch 47/50\n",
      "22974/22974 [==============================] - 3s 122us/sample - loss: 1.3281 - mean_squared_error: 1.3281 - mean_squared_logarithmic_error: 0.0200\n",
      "Epoch 48/50\n",
      "22974/22974 [==============================] - 3s 111us/sample - loss: 1.3276 - mean_squared_error: 1.3276 - mean_squared_logarithmic_error: 0.0200\n",
      "Epoch 49/50\n",
      "22974/22974 [==============================] - 3s 114us/sample - loss: 1.3260 - mean_squared_error: 1.3260 - mean_squared_logarithmic_error: 0.0199\n",
      "Epoch 50/50\n",
      "22974/22974 [==============================] - 3s 113us/sample - loss: 1.3242 - mean_squared_error: 1.3242 - mean_squared_logarithmic_error: 0.0199\n"
     ]
    }
   ],
   "source": [
    "results = regressor.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.027307256289269\n",
      "2.108539500034366\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_log_error\n",
    "\n",
    "pred_train= regressor.predict(X_train)\n",
    "print(mean_squared_log_error(y_train,pred_train)*100)\n",
    "\n",
    "pred_test= regressor.predict(X_test)\n",
    "print(mean_squared_log_error(y_test,pred_test)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14067, 2)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read in sample_submission dataframe\n",
    "submission = pd.read_csv(\"./dataset/sample_submission_iwBpW0t.csv\")\n",
    "submission.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = regressor.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Append predictions from blended models\n",
    "submission.iloc[:,1] = np.floor(np.expm1(y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fix outleir predictions\n",
    "q1 = submission['cc_cons'].quantile(0.0001)\n",
    "q2 = submission['cc_cons'].quantile(0.98)\n",
    "submission['cc_cons'] = submission['cc_cons'].apply(lambda x: x if x > q1 else x*0.77)\n",
    "submission['cc_cons'] = submission['cc_cons'].apply(lambda x: x if x < q2 else x*1.1)\n",
    "submission.to_csv(\"./dataset/submission_keras1.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale predictions\n",
    "submission['cc_cons'] *= 1.001619\n",
    "submission.to_csv(\"./dataset/submission_keras2.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}