ghodsi_supervised_pca.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Supervised Principal Component Analysis: Visualization, Classification and Regression on Subspaces and Submanifolds\n",
    "\n",
    "https://uwaterloo.ca/data-analytics/sites/ca.data-analytics/files/uploads/files/barshan_supervised_preprint.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy import linalg\n",
    "from scipy.sparse.linalg import eigsh as ssl_eigsh\n",
    "from time import clock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import TransformerMixin, BaseEstimator\n",
    "from sklearn import utils\n",
    "from sklearn.preprocessing import KernelCenterer, scale\n",
    "from sklearn.metrics.pairwise import pairwise_kernels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Algorithm: Supervised PCA\n",
    "\n",
    "Input: training data matrix, X, testing data example, x, kernel matrix of target variable, L, and training data size, n.\n",
    "\n",
    "Onput: Dimension reduced training and testing data, Z and z.\n",
    "\n",
    "1: H ← I − (n^−1)ee^T\n",
    "\n",
    "2: Q ← XHLHX^T\n",
    "\n",
    "3: Compute basis: U ← eigenvectors of Q corresponding to the top d eigenvalues.\n",
    "\n",
    "4: Encode training data: Z ← U^T X\n",
    "\n",
    "5: Encode test example: z ← U^T x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class spca(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    def __init__(self, num_components, kernel=\"linear\", eigen_solver='auto', \n",
    "                 max_iterations=None, gamma=0, degree=3, coef0=1, alpha=1.0, \n",
    "                 tolerance=0, fit_inverse_transform=False):\n",
    "        \n",
    "        self._num_components = num_components\n",
    "        self._gamma = gamma\n",
    "        self._tolerance = tolerance\n",
    "        self._fit_inverse_transform = fit_inverse_transform\n",
    "        self._max_iterations = max_iterations\n",
    "        self._degree = degree\n",
    "        self._kernel = kernel\n",
    "        self._eigen_solver = eigen_solver\n",
    "        self._coef0 = coef0\n",
    "        self._centerer = KernelCenterer()\n",
    "        self._alpha = alpha\n",
    "        \n",
    "        \n",
    "    def _get_kernel(self, X, Y=None):\n",
    "        # Returns a kernel matrix K such that K_{i, j} is the kernel between the ith and jth vectors \n",
    "        # of the given matrix X, if Y is None. \n",
    "        \n",
    "        # If Y is not None, then K_{i, j} is the kernel between the ith array from X and the jth array from Y.\n",
    "        \n",
    "        # valid kernels are 'linear, rbf, poly, sigmoid, precomputed'\n",
    "        \n",
    "        args = {\"gamma\": self._gamma, \"degree\": self._degree, \"coef0\": self._coef0}\n",
    "        \n",
    "        return pairwise_kernels(X, Y, metric=self._kernel, n_jobs=-1, filter_params=True, **args)\n",
    "    \n",
    "    \n",
    "    \n",
    "    def _fit(self, X, Y):\n",
    "        \n",
    "        # calculate kernel matrix of the labels Y and centre it and call it K (=H.L.H)\n",
    "        K = self._centerer.fit_transform(self._get_kernel(Y))\n",
    "        \n",
    "        # deciding on the number of components to use\n",
    "        if self._num_components is not None:\n",
    "            num_components = min(K.shape[0], self._num_components)\n",
    "        else:\n",
    "            num_components = self.K.shape[0]\n",
    "        \n",
    "        # Scale X\n",
    "        # scaled_X = scale(X)\n",
    "        \n",
    "        # calculate the eigen values and eigen vectors for X^T.K.X\n",
    "        Q = (X.T).dot(K).dot(X)\n",
    "        \n",
    "        # If n_components is much less than the number of training samples, \n",
    "        # arpack may be more efficient than the dense eigensolver.\n",
    "        if (self._eigen_solver=='auto'):\n",
    "            if (Q.shape[0]/num_components) > 20:\n",
    "                eigen_solver = 'arpack'\n",
    "            else:\n",
    "                eigen_solver = 'dense'\n",
    "        else:\n",
    "            eigen_solver = self._eigen_solver\n",
    "        \n",
    "        if eigen_solver == 'dense':\n",
    "            # Return the eigenvalues (in ascending order) and eigenvectors of a Hermitian or symmetric matrix.\n",
    "            self._lambdas, self._alphas = linalg.eigh(Q, eigvals=(Q.shape[0] - num_components, Q.shape[0] - 1))\n",
    "            # argument eigvals = Indexes of the smallest and largest (in ascending order) eigenvalues\n",
    "        \n",
    "        elif eigen_solver == 'arpack':\n",
    "            # deprecated :: self._lambdas, self._alphas = utils.arpack.eigsh(A=Q, num_components, which=\"LA\", tol=self._tolerance)\n",
    "            self._lambdas, self._alphas = ssl_eigsh(A=Q, k=num_components, which=\"LA\", tol=self._tolerance)\n",
    "            \n",
    "        indices = self._lambdas.argsort()[::-1]\n",
    "        \n",
    "        self._lambdas = self._lambdas[indices]\n",
    "        self._lambdas = self._lambdas[self._lambdas > 0]  # selecting values only for non zero eigen values\n",
    "        \n",
    "        self._alphas = self.alphas[:, indices]\n",
    "        self._alphas = self.alphas[:, self._lambdas > 0]  # selecting values only for non zero eigen values\n",
    "        \n",
    "        self.X_fit = X\n",
    "\n",
    "        \n",
    "        def _transform(self):\n",
    "            return self.X_fit.dot(self._alphas)\n",
    "        \n",
    "        \n",
    "        def transform(self, X):\n",
    "            return X.dot(self._alphas)\n",
    "        \n",
    "        \n",
    "        def fit(self, X, Y):\n",
    "            self._fit(X,Y)\n",
    "            return\n",
    "        \n",
    "        \n",
    "        def fit_and_transform(self, X, Y):\n",
    "            self.fit(X, Y)\n",
    "            return self._transform()\n",
    "               \n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}