logreg_diabetes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy.linalg import norm\n",
    "import pickle\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "from scipy.stats import norm as norm_d\n",
    "from scipy.stats import expon\n",
    "from scipy.stats import weibull_min as weibull\n",
    "from scipy.stats import burr12 as burr\n",
    "from scipy.stats import randint\n",
    "from scipy.stats import uniform\n",
    "from scipy.optimize import minimize\n",
    "import copy\n",
    "import math\n",
    "import time\n",
    "from scipy.optimize import minimize\n",
    "from scipy.sparse.linalg import svds\n",
    "from scipy.linalg import svdvals\n",
    "import scipy\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "import pickle\n",
    "from pathlib import Path\n",
    "from functions import *\n",
    "from algorithms import *\n",
    "from utils import *\n",
    "from tests import *\n",
    "\n",
    "matplotlib.rcParams['pdf.fonttype'] = 42\n",
    "matplotlib.rcParams['ps.fonttype'] = 42\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## diabetes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "dataset = \"diabetes\"\n",
    "A, y, m, n, sparsity_A = prepare_data(dataset)\n",
    "print(\"Number of functions: \", m, \", dimension: \", n)\n",
    "L, average_L, worst_L = compute_L(dataset, A)\n",
    "denseA = A.toarray()\n",
    "print(\"Smoothness const: \", L)\n",
    "print(\"Average smoothness const: \", average_L)\n",
    "print(\"Worst smoothness const: \", worst_L)\n",
    "print(\"Percentage of non-zeros: \", sparsity_A)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find the solution (Run this code only once)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename=dataset+\"_x_init_all_ones\"\n",
    "x_init = np.ones(n)\n",
    "l2 = 0\n",
    "l1 = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "param = [A, y, l2, False, l1]\n",
    "res_solver = minimize(F, x_init, args = param, jac=logreg_grad_plus_lasso, \n",
    "                      method='L-BFGS-B',\n",
    "                      options={'maxiter':10000, 'disp':True}, tol=1e-18)\n",
    "\n",
    "print(res_solver.success)\n",
    "print(res_solver.fun)\n",
    "print(res_solver)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_solution(dataset, l2, l1, np.array(res_solver.x), res_solver.fun)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Noise distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_star = read_solution(dataset, l2, l1)[0]\n",
    "f_star = read_solution(dataset, l2, l1)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "x = deepcopy(x_star)\n",
    "grad_deviations = np.array([])\n",
    "full_grad = logreg_grad(x, [denseA,y,0,False])\n",
    "for i in range(m):\n",
    "    grad_deviations = np.append(grad_deviations, np.linalg.norm(logreg_grad(x, [denseA[[i]],y[[i]],0,False]) - full_grad))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mu = np.mean(grad_deviations)\n",
    "sigma = np.sqrt(np.mean(grad_deviations**2 - mu))\n",
    "print(mu)\n",
    "print(sigma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,8))\n",
    "plt.xticks(fontsize=30)\n",
    "plt.yticks(fontsize=30)\n",
    "t, bins, patches = plt.hist(np.sort(grad_deviations), bins=50, density=1, log=True)\n",
    "temp2 = np.linspace(0, 700, 1000)\n",
    "plt.semilogy(temp2, norm_d.pdf(temp2, mu, sigma), linewidth=2, color='red', marker='d',\n",
    "            markersize = 15, \n",
    "            markevery=range(0, 1000, 100))\n",
    "plt.title('diabetes, real samples', fontsize=35)\n",
    "plt.xlabel(\"Noise norm\", fontsize=40)\n",
    "plt.ylabel(\"Density\", fontsize=40)\n",
    "plt.savefig(\"plot/diabetes_real.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,8))\n",
    "plt.xticks(fontsize=30)\n",
    "plt.yticks(fontsize=30)\n",
    "sample = norm_d.rvs(loc=mu, scale=sigma, size=m)\n",
    "sample = sample[sample > 0]\n",
    "t, bins, patches = plt.hist(sample, bins=50, density=(1 - norm_d.cdf(0,loc=mu,scale=sigma)), log=True)\n",
    "temp2 = np.linspace(0, 700, 1000)\n",
    "plt.semilogy(temp2, norm_d.pdf(temp2, mu, sigma), linewidth=2, color='red', marker='d',\n",
    "            markersize = 15, \n",
    "            markevery=range(0, 1000, 100))\n",
    "plt.title('diabetes, synthetic Gaussian samples', fontsize=35)\n",
    "plt.xlabel(\"Noise norm\", fontsize=40)\n",
    "plt.ylabel(\"Density\", fontsize=40)\n",
    "plt.savefig(\"plot/diabetes_synthetic.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "shift_param = 0.1\n",
    "filename=dataset+\"_x_star_shifted_by_all_ones_shift_\"+str(shift_param)\n",
    "l2 = 0\n",
    "l1 = 0\n",
    "batch_size = 100\n",
    "M = int(2*m/batch_size)\n",
    "gamma = 1.0/(L+l2)*0.1\n",
    "x_star = read_solution(dataset, l2, l1)[0]\n",
    "f_star = read_solution(dataset, l2, l1)[1]\n",
    "x_init = x_star + shift_param*np.ones(n)\n",
    "lambd = 2*np.sqrt(2)*(L+l2)*norm(x_star-x_init)*0.01\n",
    "lambd_schedule = [lambd, 1000, 0.7]\n",
    "S = 10000\n",
    "save_info_period = 100\n",
    "beta = 0.001\n",
    "N = int(S*m/batch_size)\n",
    "a = ((np.log(4*N/beta)) ** 2)\n",
    "print(\"a = \", a)\n",
    "\n",
    "B = norm(x_star-x_init) / (np.log(4*N/beta))\n",
    "print(\"B = \", B)\n",
    "\n",
    "sparse_full = False\n",
    "sparse_stoch = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"gamma = \", gamma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Initial value: \", logreg_loss(x_init, [denseA,y,l2,False,0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(lambd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "res = sstm(filename=filename, x_init=x_init, A=A, y=y, a=5000.0,\n",
    "             L=L+l2, l2=l2, \n",
    "             sparse_full=sparse_full, sparse_stoch=sparse_stoch, \n",
    "             S=S, max_t=np.inf,\n",
    "             batch_size=batch_size, indices=None, save_info_period=save_info_period, \n",
    "             x_star=x_star, f_star=f_star)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "res = clipped_sstm(filename=filename, x_init=x_init, A=A, y=y, a=5000,\n",
    "             B=B*0.05, L=L+l2, l2=l2, \n",
    "             sparse_full=sparse_full, sparse_stoch=sparse_stoch, \n",
    "             S=S, max_t=np.inf,\n",
    "             batch_size=batch_size, indices=None, save_info_period=save_info_period, \n",
    "             x_star=x_star, f_star=f_star)\n",
    "        i += 1\n",
    "        print(\"Value: \", res['func_vals'][-1]+f_star)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SGD and clipped-SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "res = clipped_sgd_const_stepsize_decr_clip(filename=filename, x_init=x_init, A=A, y=y, gamma=gamma,\n",
    "     lambd_schedule=lambd_schedule, l2=l2, \n",
    "     sparse_full=sparse_full, sparse_stoch=sparse_stoch, \n",
    "     l1=l1, S=S, max_t=np.inf,\n",
    "     batch_size=batch_size, indices=None, save_info_period=save_info_period, \n",
    "     x_star=x_star, f_star=f_star)\n",
    "print(\"Value: \", res['func_vals'][-1]+f_star)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "res = clipped_sgd_const_stepsize(filename=filename, x_init=x_init, A=A, y=y, gamma=gamma,\n",
    "     lambd=lambd*0.01, l2=l2, \n",
    "     sparse_full=sparse_full, sparse_stoch=sparse_stoch, \n",
    "     l1=l1, S=S, max_t=np.inf,\n",
    "     batch_size=batch_size, indices=None, save_info_period=save_info_period, \n",
    "     x_star=x_star, f_star=f_star)\n",
    "print(\"Value: \", res['func_vals'][-1]+f_star)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "res = sgd_const_stepsize(filename=filename, x_init=x_init, A=A, y=y, gamma=gamma,\n",
    "     l2=l2, \n",
    "     sparse_full=sparse_full, sparse_stoch=sparse_stoch, \n",
    "     l1=l1, S=S, max_t=np.inf,\n",
    "     batch_size=batch_size, indices=None, save_info_period=save_info_period, \n",
    "     x_star=x_star, f_star=f_star)\n",
    "print(\"Value: \", res['func_vals'][-1]+f_star)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = [\n",
    "             ['SGD_const_stepsize', [gamma, l2, l1, S, batch_size], \n",
    "           'SGD', None],\n",
    "         ['clipped-SGD_const_stepsize', [gamma, lambd*0.01, l2, l1, S, batch_size], \n",
    "           'clipped-SGD', None],\n",
    "         ['clipped-SGD_const_stepsize', [gamma, lambd, l2, l1, S, batch_size], \n",
    "           'clipped-SGD', None],\n",
    "         ['clipped-SGD_const_stepsize_decr_clip', [gamma, lambd_schedule, l2, l1, S, batch_size], \n",
    "           'd-clipped-SGD', None],\n",
    "           ['clipped-SSTM', [5000.0, B*0.05, L+l2, l2, S, batch_size], \n",
    "           'clipped-SSTM', None],\n",
    "           ['SSTM', [5000.0, L+l2, l2, S, batch_size], \n",
    "           'SSTM', None],\n",
    "\n",
    "]\n",
    "mode_y = 'func_vals'\n",
    "mode_x = 'data_passes'\n",
    "figsize = (12, 8)\n",
    "title_size = 30\n",
    "linewidth = 2\n",
    "markersize = 15\n",
    "legend_size = 25\n",
    "xlabel_size = 30\n",
    "ylabel_size = 40\n",
    "xticks_size = 20\n",
    "yticks_size = 30\n",
    "sizes = [title_size, linewidth, markersize, legend_size, xlabel_size, ylabel_size, xticks_size, yticks_size]\n",
    "title = dataset\n",
    "bbox_to_anchor = (1,1)\n",
    "legend_loc=\"upper right\"\n",
    "\n",
    "\n",
    "save_fig = [True, filename+\"_test.pdf\"]\n",
    "\n",
    "args_for_plots = [filename, mode_y, mode_x, figsize, sizes, title, methods, bbox_to_anchor, legend_loc, save_fig]\n",
    "make_plots(args_for_plots)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = read_results_from_file(filename, 'SGD_const_stepsize', [gamma, l2, l1, S, batch_size])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "x = deepcopy(res['last_iter'])\n",
    "grad_deviations = np.array([])\n",
    "full_grad = logreg_grad(x, [denseA,y,0,False])\n",
    "for i in range(m):\n",
    "    grad_deviations = np.append(grad_deviations, np.linalg.norm(logreg_grad(x, [denseA[[i]],y[[i]],0,False]) - full_grad))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mu = np.mean(grad_deviations)\n",
    "sigma = np.sqrt(np.mean(grad_deviations**2 - mu))\n",
    "print(mu)\n",
    "print(sigma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,8))\n",
    "plt.xticks(fontsize=30)\n",
    "plt.yticks(fontsize=30)\n",
    "t, bins, patches = plt.hist(np.sort(grad_deviations), bins=50, density=1, log=True)\n",
    "temp2 = np.linspace(0, 700, 1000)\n",
    "plt.semilogy(temp2, norm_d.pdf(temp2, mu, sigma), linewidth=2, color='red', marker='d',\n",
    "            markersize = 15, \n",
    "            markevery=range(0, 1000, 100))\n",
    "plt.title('diabetes, real samples', fontsize=35)\n",
    "plt.xlabel(\"Noise norm\", fontsize=40)\n",
    "plt.ylabel(\"Density\", fontsize=40)\n",
    "plt.savefig(\"plot/diabetes_sgd_last_point.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,8))\n",
    "plt.xticks(fontsize=30)\n",
    "plt.yticks(fontsize=30)\n",
    "sample = norm_d.rvs(loc=mu, scale=sigma, size=m)\n",
    "sample = sample[sample > 0]\n",
    "t, bins, patches = plt.hist(sample, bins=50, density=(1 - norm_d.cdf(0,loc=mu,scale=sigma)), log=True)\n",
    "temp2 = np.linspace(0, 700, 1000)\n",
    "plt.semilogy(temp2, norm_d.pdf(temp2, mu, sigma), linewidth=2, color='red', marker='d',\n",
    "            markersize = 15, \n",
    "            markevery=range(0, 1000, 100))\n",
    "plt.title('diabetes, synthetic Gaussian samples', fontsize=35)\n",
    "plt.xlabel(\"Noise norm\", fontsize=40)\n",
    "plt.ylabel(\"Density\", fontsize=40)\n",
    "plt.savefig(\"plot/diabetes_synthetic_sgd_last_iter.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}