05_01_missing_values.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:14.804289Z",
     "start_time": "2017-06-22T08:23:14.542624Z"
    }
   },
   "outputs": [],
   "source": [
    "# numpyの準備\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:14.808621Z",
     "start_time": "2017-06-22T08:23:14.805204Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# csvファイルの表示\n",
    "\n",
    "#!cat 2D_example_dame.csv\n",
    "\n",
    "with open('2D_example_dame.csv') as f:\n",
    "    print(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:14.813616Z",
     "start_time": "2017-06-22T08:23:14.810129Z"
    }
   },
   "outputs": [],
   "source": [
    "# dataにデータをセット\n",
    "data = np.loadtxt(\"2D_example_dame.csv\", delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:14.817735Z",
     "start_time": "2017-06-22T08:23:14.814995Z"
    }
   },
   "outputs": [],
   "source": [
    "y = data[:,0].astype(int) # 1列目がラベル．整数に変換"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:14.830913Z",
     "start_time": "2017-06-22T08:23:14.819496Z"
    }
   },
   "outputs": [],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:14.834815Z",
     "start_time": "2017-06-22T08:23:14.832431Z"
    }
   },
   "outputs": [],
   "source": [
    "X = data[:,1:3] # 2,3列目がデータ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:14.840956Z",
     "start_time": "2017-06-22T08:23:14.836446Z"
    }
   },
   "outputs": [],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:14.846860Z",
     "start_time": "2017-06-22T08:23:14.842476Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X[:, 0] # Xの1列目（csvファイルの2列目）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:14.852763Z",
     "start_time": "2017-06-22T08:23:14.848267Z"
    }
   },
   "outputs": [],
   "source": [
    "X[:, 1] # Xの2列目（csvファイルの3列目）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.079546Z",
     "start_time": "2017-06-22T08:23:14.854414Z"
    }
   },
   "outputs": [],
   "source": [
    "# matplotlibの準備\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.set_cmap(plt.cm.Paired) # 色設定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.204656Z",
     "start_time": "2017-06-22T08:23:15.080720Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolors='k'); # 2次元散布図でプロット"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.335773Z",
     "start_time": "2017-06-22T08:23:15.205715Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolors='k'); # 2次元散布図でプロット\n",
    "plt.xlim(-10,10)\n",
    "plt.ylim(-10,10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "対策の例：NaNと外れ値を除外する"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.340919Z",
     "start_time": "2017-06-22T08:23:15.336794Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "~np.isnan(X[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.346099Z",
     "start_time": "2017-06-22T08:23:15.342061Z"
    }
   },
   "outputs": [],
   "source": [
    "~np.isnan(X[:, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.352647Z",
     "start_time": "2017-06-22T08:23:15.347517Z"
    }
   },
   "outputs": [],
   "source": [
    "~np.isnan(X[:, 1]) & ~np.isnan(X[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.356851Z",
     "start_time": "2017-06-22T08:23:15.353708Z"
    }
   },
   "outputs": [],
   "source": [
    "# X1にはNaNがない\n",
    "X1 = X[~np.isnan(X[:, 1]) & ~np.isnan(X[:, 0])] \n",
    "y1 = y[~np.isnan(X[:, 1]) & ~np.isnan(X[:, 0])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.362259Z",
     "start_time": "2017-06-22T08:23:15.357829Z"
    }
   },
   "outputs": [],
   "source": [
    "X1, X1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.369141Z",
     "start_time": "2017-06-22T08:23:15.363911Z"
    }
   },
   "outputs": [],
   "source": [
    "(abs(X1[:, 0]) < 10), (abs(X1[:, 1]) < 10) # 外れ値対策"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.373819Z",
     "start_time": "2017-06-22T08:23:15.370279Z"
    }
   },
   "outputs": [],
   "source": [
    "X2 = X1[(abs(X1[:, 0]) < 10) & (abs(X1[:, 1]) < 10)]\n",
    "y2 = y1[(abs(X1[:, 0]) < 10) & (abs(X1[:, 1]) < 10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.379444Z",
     "start_time": "2017-06-22T08:23:15.375232Z"
    }
   },
   "outputs": [],
   "source": [
    "X2, X2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.500916Z",
     "start_time": "2017-06-22T08:23:15.380845Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.scatter(X2[:, 0], X2[:, 1], c=y2, s=50, edgecolors='k'); # 2次元散布図でプロット"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "対策の例：NaNを埋める"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.788943Z",
     "start_time": "2017-06-22T08:23:15.502493Z"
    }
   },
   "outputs": [],
   "source": [
    "# from sklearn.preprocessing import Imputer\n",
    "from sklearn.impute import SimpleImputer as Imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.792069Z",
     "start_time": "2017-06-22T08:23:15.789795Z"
    }
   },
   "outputs": [],
   "source": [
    "missing_value_to_mean = Imputer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.798240Z",
     "start_time": "2017-06-22T08:23:15.793593Z"
    }
   },
   "outputs": [],
   "source": [
    "missing_value_to_mean.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.804509Z",
     "start_time": "2017-06-22T08:23:15.799690Z"
    }
   },
   "outputs": [],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.810763Z",
     "start_time": "2017-06-22T08:23:15.806016Z"
    }
   },
   "outputs": [],
   "source": [
    "X_new = missing_value_to_mean.transform(X)\n",
    "X_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.941330Z",
     "start_time": "2017-06-22T08:23:15.812181Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.scatter(X_new[:, 0], X_new[:, 1], c=y, s=50, edgecolors='k'); # 2次元散布図でプロット"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:15.949404Z",
     "start_time": "2017-06-22T08:23:15.942739Z"
    }
   },
   "outputs": [],
   "source": [
    "missing_value_to_median = Imputer(strategy='median')\n",
    "missing_value_to_median.fit(X)\n",
    "X_new2 = missing_value_to_median.transform(X)\n",
    "X_new2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-22T08:23:16.074703Z",
     "start_time": "2017-06-22T08:23:15.950975Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.scatter(X_new2[:, 0], X_new2[:, 1], c=y, s=50, edgecolors='k'); # 2次元散布図でプロット"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "12px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}