01.4.kNN-tf2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow2实现kNN分类和回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python version: sys.version_info(major=3, minor=7, micro=2, releaselevel='final', serial=0)\n",
      "tensorflow 2.3.0\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from collections.abc import Iterable\n",
    "from collections import Counter\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.datasets import mnist, boston_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import load_iris\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "print(\"python version:\", sys.version_info)\n",
    "print(tf.__name__, tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. TensorFlow实现kNN基类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFKNNBase:\n",
    "    \"\"\"KNN basic class with TensorFlow.\n",
    "    \n",
    "    Attributes:\n",
    "        n_neighbors: A int number, number of neighbors.\n",
    "        _metric: A method object, choose from {_manhattan_distance, _euclidean_distance, _chebyshev_distance}.\n",
    "        _X_train: feature data for training. A tf.Tensor matrix of (n_samples, n_features) shape, \n",
    "            data type must be continuous value type. \n",
    "        _y_train: label data for training. A tf.Tensor array of (n_samples, ) shape, \n",
    "            data type must be discrete value.\n",
    "    \"\"\"\n",
    "        \n",
    "    def __init__(self, n_neighbors=5, metric=\"euclidean\"):\n",
    "        \"\"\"Init method.\n",
    "    \n",
    "        Args:\n",
    "            n_neighbors: int, optional (default = 5), the integer must greater then 0.\n",
    "                Number of neighbors to use by default for :meth:`kneighbors` queries.\n",
    "            metric: {\"manhattan\", \"euclidean\", \"chebyshev\"}, optional, default 'euclidean'.\n",
    "        \n",
    "        Raises:\n",
    "            ValueError: metric value is out of options.\n",
    "            AssertionError: n_neighbors value is not a integer or n_neighbors > 0. \n",
    "        \"\"\"\n",
    "        assert isinstance(n_neighbors, int) and n_neighbors > 0\n",
    "        self.n_neighbors = n_neighbors\n",
    "        if metric == \"manhattan\":\n",
    "            self._metric = self._manhattan_distance\n",
    "        elif metric == \"euclidean\":\n",
    "            self._metric = self._euclidean_distance\n",
    "        elif metric == \"chebyshev\":\n",
    "            self._metric = self._chebyshev_distance\n",
    "        else:\n",
    "            raise ValueError(f'No such metric as {metric}, please option from: {\"manhattan\", \"euclidean\", \"chebyshev\"}')\n",
    "        self._X_train, self._y_train = [None] * 2\n",
    "    \n",
    "    def fit(self, X_train, y_train):\n",
    "        \"\"\"method for training model. \n",
    "        \n",
    "        Args:\n",
    "            X_train: A matrix of (n_samples, n_features) shape, data type must be continuous value type. \n",
    "            y_train: A array of (n_samples, ) shape.\n",
    "        \n",
    "        Raises:\n",
    "            AssertionError: X_train value or y_train value with a mismatched shape.\n",
    "        \"\"\"\n",
    "        assert isinstance(X_train, Iterable) and isinstance(y_train, Iterable)\n",
    "        assert len(X_train) == len(y_train)\n",
    "        self._X_train = tf.constant(X_train, dtype=tf.float32)\n",
    "        self._y_train = y_train if isinstance(y_train, tf.Tensor) else tf.constant(y_train)\n",
    "    \n",
    "    def predict(self, X_test):\n",
    "        \"\"\"predict test data.\n",
    "        \n",
    "        Args:\n",
    "            X_test: A np.ndarray matrix of (n_samples, n_features) shape, \n",
    "                or a np.ndarray array of (n_features, ) shape.\n",
    "            \n",
    "        Returns:\n",
    "            A list for samples predictions or a single prediction.\n",
    "        \n",
    "        Raises:\n",
    "            ValueError: X_test value with a mismatched shape.\n",
    "        \"\"\"\n",
    "        assert isinstance(X_test, Iterable)\n",
    "        X_test = tf.constant(X_test, dtype=tf.float32)\n",
    "        \n",
    "        if X_test.shape == (self._X_train.shape[1], ):\n",
    "            y_pred = self._predict_sample(X_test)\n",
    "        elif X_test.shape[1] == self._X_train.shape[1]: \n",
    "            y_pred = []\n",
    "            for sample in X_test:\n",
    "                y_pred.append(self._predict_sample(sample))\n",
    "        else:\n",
    "            raise ValueError(\"Mismatched shape for X_test\")\n",
    "        return y_pred\n",
    "    \n",
    "    def _manhattan_distance(self, x):\n",
    "        return tf.reduce_sum(tf.abs(self._X_train - x), axis=1)    \n",
    "    \n",
    "    def _euclidean_distance(self, x):\n",
    "        return tf.sqrt(tf.reduce_sum(tf.square(self._X_train - x), axis=1))\n",
    "    \n",
    "    def _chebyshev_distance(self, x):\n",
    "        return tf.reduce_max(tf.abs(self._X_train - x), axis=1)\n",
    "    \n",
    "    def _find_k_labels(self, sample):\n",
    "        distance = self._metric(sample)\n",
    "        sorted_idx = tf.argsort(distance)\n",
    "        k_labels = tf.gather(self._y_train, indices=sorted_idx[:self.n_neighbors])\n",
    "        return k_labels\n",
    "    \n",
    "    def _predict_sample(self, sample):\n",
    "        raise Exception(\"Can call predict method for NumpyKNNBase object! \")\n",
    "        \n",
    "    def _score_validation(self, X_test, y_test):\n",
    "        assert isinstance(X_test, Iterable) and isinstance(y_test, Iterable)\n",
    "        assert len(X_test) == len(y_test)\n",
    "        X_test = tf.constant(X_test, dtype=tf.float32)\n",
    "        y_test = y_test if isinstance(y_test, tf.Tensor) else tf.constant(y_test)\n",
    "        return X_test, y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. TensorFlow实现kNN Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFKNNClassifier(TFKNNBase):\n",
    "    \"\"\"kNN Classifier with TensorFlow, explicitly inherits from TFKNNBase already.\n",
    "    \n",
    "    Attributes:\n",
    "        n_neighbors: A int number, number of neighbors.\n",
    "        _metric: A method object, choose from {_manhattan_distance, _euclidean_distance, _chebyshev_distance}.\n",
    "        _X_train: feature data for training. A tf.Tensor matrix of (n_samples, n_features) shape, \n",
    "            data type must be continuous value type. \n",
    "        _y_train: label data for training. A tf.Tensor array of (n_samples, ) shape, \n",
    "            data type must be discrete value.\n",
    "    \"\"\"\n",
    "    def __new__(cls, *args, **kwargs):\n",
    "        return object.__new__(cls, *args, **kwargs)\n",
    "    \n",
    "    def score(self, X_test, y_test):\n",
    "        \"\"\"Use test dataset to evaluate the trained model.\n",
    "        \n",
    "        Args:\n",
    "            X_test: A np.ndarray matrix of (n_samples, n_features) shape.\n",
    "            y_test: A np.ndarray array of (n_samples, ) shape. data type must be\n",
    "                discrete value.\n",
    "        Returns:\n",
    "            return accuracy, a float number. accuracy = correct_count / y_test.shape[0]\n",
    "        \"\"\"\n",
    "        X_test, y_test = self._score_validation(X_test, y_test)\n",
    "        \n",
    "        y_pred = self.predict(X_test)\n",
    "        correct_count = tf.reduce_sum(tf.cast(y_pred == y_test, dtype=tf.int32))\n",
    "        accuracy = correct_count / y_test.shape[0]\n",
    "        return accuracy.numpy()\n",
    "    \n",
    "    def _predict_sample(self, sample):\n",
    "        k_labels = self._find_k_labels(sample)\n",
    "        pred = Counter(k_labels.numpy()).most_common(1)[0][0]\n",
    "        return pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 iris数据集验证算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_data, y_data = load_iris(return_X_y=True)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, random_state=1, shuffle=True)\n",
    "clf = TFKNNClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 mnist手写数字识别"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://tensorflow.google.cn/api_docs/python/tf/keras/datasets/mnist/load_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://yann.lecun.com/exdb/mnist/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 28, 28), (60000,), (10000, 28, 28), (10000,))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 784)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = X_train.reshape([60000, -1])\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 784)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = X_test.reshape([10000, -1])\n",
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = TFKNNClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "clf.score(X_test[:100], y_test[:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Numpy实现Min-Max归一化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NumpyMinMaxScaler:\n",
    "    def __init__(self):\n",
    "        self.min_ = None\n",
    "        self.max_ = None\n",
    "    \n",
    "    def fit(self, X_train):\n",
    "        self.max_ = np.max(X_train, axis=0)\n",
    "        self.min_ = np.min(X_train, axis=0)\n",
    "    \n",
    "    def transform(self, x):\n",
    "        return (x - self.min_) / (self.max_ - self.min_ + 0.00001)\n",
    "    \n",
    "    def fit_transform(self, X_train):\n",
    "        self.fit(X_train)\n",
    "        return self.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 784)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler = NumpyMinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.76"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = TFKNNClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "clf.score(X_test[:100], y_test[:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 红酒数据集验证NumpyMinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6888888888888889"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import load_wine\n",
    "\n",
    "X_data, y_data = load_wine(return_X_y=True)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, random_state=1, shuffle=True)\n",
    "clf = TFKNNClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9777777777777777"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_data, y_data = load_wine(return_X_y=True)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, random_state=1, shuffle=True)\n",
    "\n",
    "scaler = NumpyMinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)\n",
    "\n",
    "clf = TFKNNClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. TensorFlow实现kNN Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFKNNRegressor(TFKNNBase): \n",
    "    \"\"\"kNN Regressor with tensorflow, explicitly inherits from TFKNNBase already.\n",
    "    \n",
    "    Attributes:\n",
    "        n_neighbors: A int number, number of neighbors.\n",
    "        _metric: A method object, choose from {_manhattan_distance, _euclidean_distance, _chebyshev_distance}.\n",
    "        _X_train: feature data for training. A tf.Tensor matrix of (n_samples, n_features) shape, \n",
    "            data type must be continuous value type. \n",
    "        _y_train: label data for training. A tf.Tensor array of (n_samples, ) shape, \n",
    "            data type must be discrete value.\n",
    "    \"\"\"\n",
    "    def __new__(cls, *args, **kwargs):\n",
    "        return object.__new__(cls, *args, **kwargs)\n",
    "    \n",
    "    def score(self, X_test, y_test):\n",
    "        \"\"\"Use test dataset to evaluate the trained model.\n",
    "        \n",
    "        Args:\n",
    "            X_test: A np.ndarray matrix of (n_samples, n_features) shape.\n",
    "            y_test: A np.ndarray array of (n_samples, ) shape. data type must be\n",
    "                discrete value.\n",
    "        Returns:\n",
    "            return R^2, R^2 = 1 - u / v. u = sum((y_pred - y_true)^2), v = sum((y_true - y_true_mean)^2)\n",
    "        \"\"\"\n",
    "        X_test, y_test = self._score_validation(X_test, y_test)\n",
    "        \n",
    "        y_pred = self.predict(X_test)\n",
    "        y_true_mean = tf.reduce_mean(y_test, axis=0)\n",
    "        u = tf.reduce_sum(tf.square(y_pred - y_test))\n",
    "        v = tf.reduce_sum(tf.square(y_test - y_true_mean))\n",
    "        r_squared = 1 - u / v\n",
    "        return r_squared.numpy()\n",
    "    \n",
    "    def _predict_sample(self, sample):\n",
    "        k_labels = self._find_k_labels(sample)\n",
    "        pred = tf.reduce_mean(k_labels, axis=0)\n",
    "        return pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 boston房价数据集验证算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, y_train), (X_test, y_test) = boston_housing.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5558913435303654"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rgs = TFKNNRegressor()\n",
    "rgs.fit(X_train, y_train)\n",
    "rgs.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6401800189130427"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler = NumpyMinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)\n",
    "\n",
    "rgs = TFKNNRegressor()\n",
    "rgs.fit(X_train, y_train)\n",
    "rgs.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. TensorFlow API\n",
    "    > tf.constant(x ,dtype=tf.int32)  \n",
    "    > tf.abs()  \n",
    "    > tf.sqrt()  \n",
    "    > tf.square()  \n",
    "    > tf.reduce_sum()  \n",
    "    > tf.reduce_max(input_tensor, axis=1)  \n",
    "    > tf.argsort(distance)  \n",
    "    > tf.gather(self._y_train, indices)  \n",
    "2. TensorFlow的数据类型\n",
    "    > tf.Tensor\n",
    "    > tf.float32  \n",
    "    > tf.int32  \n",
    "3. tensorflow.keras的数据集\n",
    "    > from tensorflow.keras.datasets import mnist, boston_housing\n",
    "    > (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "4. Numpy实现Min-Max Normalization\n",
    "    > 分母加一个极小值， 防止分母为0  \n",
    "    > Min-Max Normalization不一定能得到更好训练效果  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 作业\n",
    "\n",
    "1. 惯例 至少敲两遍。第一遍拷贝kNN-numpy，照着改。第二遍从零开始默写。并完善注释 \n",
    "2. tensorflow实现Min-Max Normalization。\n",
    "3. 弄清上述tf API的功能及其参数，总结tensorflow API和numpy的区别。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 相关链接\n",
    "\n",
    "<a href=\"./01.1.kNN.ipynb\" style=\"\"> 1.1 kNN k近邻算法原理 </a>  \n",
    "<a href=\"./01.2.kNN-sklearn.ipynb\" style=\"\"> 1.2 sklearn中使用kNN做分类、回归任务 </a>  \n",
    "<a href=\"./01.3.kNN-numpy.ipynb\" style=\"\"> 1.3 numpy实现kNN分类和回归 </a>    \n",
    "  \n",
    "<a href=\"./01.5.kNN-torch1.ipynb\"> 1.5 Pytorch1实现kNN分类和回归 </a>  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 项目源码  \n",
    "  \n",
    "https://github.com/LossJ     \n",
    "进入后点击Statistic-Machine-Learning"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}