H2O_AutoML_IPYNB/penalized_logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import h2o package and specific estimator \n",
    "import h2o\n",
    "from h2o.estimators.glm import H2OGeneralizedLinearEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321..... not found.\n",
      "Attempting to start a local H2O server...\n",
      "  Java Version: java version \"1.8.0_112\"; Java(TM) SE Runtime Environment (build 1.8.0_112-b16); Java HotSpot(TM) 64-Bit Server VM (build 25.112-b16, mixed mode)\n",
      "  Starting server from /Users/phall/anaconda/lib/python3.5/site-packages/h2o/backend/bin/h2o.jar\n",
      "  Ice root: /var/folders/tc/0ss1l73113j3wdyjsxmy1j2r0000gn/T/tmp7et25rt0\n",
      "  JVM stdout: /var/folders/tc/0ss1l73113j3wdyjsxmy1j2r0000gn/T/tmp7et25rt0/h2o_phall_started_from_python.out\n",
      "  JVM stderr: /var/folders/tc/0ss1l73113j3wdyjsxmy1j2r0000gn/T/tmp7et25rt0/h2o_phall_started_from_python.err\n",
      "  Server is running at http://127.0.0.1:54321\n",
      "Connecting to H2O server at http://127.0.0.1:54321... successful.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>01 secs</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.12.0.1</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>29 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_phall_gqrp80</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>3.556 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>accepting new members, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://127.0.0.1:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>None</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.5.2 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ------------------------------\n",
       "H2O cluster uptime:         01 secs\n",
       "H2O cluster version:        3.12.0.1\n",
       "H2O cluster version age:    29 days\n",
       "H2O cluster name:           H2O_from_python_phall_gqrp80\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    3.556 Gb\n",
       "H2O cluster total cores:    8\n",
       "H2O cluster allowed cores:  8\n",
       "H2O cluster status:         accepting new members, healthy\n",
       "H2O connection url:         http://127.0.0.1:54321\n",
       "H2O connection proxy:\n",
       "H2O internal security:      False\n",
       "Python version:             3.5.2 final\n",
       "--------------------------  ------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h2o.init() # start h2o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# location of clean data file\n",
    "path = 'https://raw.githubusercontent.com/h2oai/app-consumer-loan/master/data/loan.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define input variable measurement levels \n",
    "# strings automatically parsed as enums (nominal)\n",
    "# numbers automatically parsed as numeric\n",
    "col_types = {'bad_loan': 'enum'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "frame = h2o.import_file(path=path, col_types=col_types) # import from url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows:163987\n",
      "Cols:15\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>       </th><th>loan_amnt         </th><th>term     </th><th>int_rate          </th><th>emp_length       </th><th>home_ownership  </th><th>annual_inc       </th><th>purpose           </th><th>addr_state  </th><th>dti               </th><th>delinq_2yrs        </th><th>revol_util        </th><th>total_acc         </th><th>bad_loan  </th><th>longest_credit_length  </th><th>verification_status  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>type   </td><td>int               </td><td>enum     </td><td>real              </td><td>int              </td><td>enum            </td><td>real             </td><td>enum              </td><td>enum        </td><td>real              </td><td>int                </td><td>real              </td><td>int               </td><td>enum      </td><td>int                    </td><td>enum                 </td></tr>\n",
       "<tr><td>mins   </td><td>500.0             </td><td>         </td><td>5.42              </td><td>0.0              </td><td>                </td><td>1896.0           </td><td>                  </td><td>            </td><td>0.0               </td><td>0.0                </td><td>0.0               </td><td>1.0               </td><td>          </td><td>0.0                    </td><td>                     </td></tr>\n",
       "<tr><td>mean   </td><td>13074.169141456332</td><td>         </td><td>13.715904065566189</td><td>5.684352932995338</td><td>                </td><td>71915.67051974905</td><td>                  </td><td>            </td><td>15.881530121290167</td><td>0.22735700606252723</td><td>54.07917280242262 </td><td>24.579733834274574</td><td>          </td><td>14.854273655448333     </td><td>                     </td></tr>\n",
       "<tr><td>maxs   </td><td>35000.0           </td><td>         </td><td>26.060000000000002</td><td>10.0             </td><td>                </td><td>7141778.0        </td><td>                  </td><td>            </td><td>39.99             </td><td>29.0               </td><td>150.70000000000002</td><td>118.0             </td><td>          </td><td>65.0                   </td><td>                     </td></tr>\n",
       "<tr><td>sigma  </td><td>7993.556188734672 </td><td>         </td><td>4.391939870545809 </td><td>3.610663731100238</td><td>                </td><td>59070.91565491818</td><td>                  </td><td>            </td><td>7.5876682241925355</td><td>0.6941679229284191 </td><td>25.285366766770498</td><td>11.685190365910666</td><td>          </td><td>6.947732922546689      </td><td>                     </td></tr>\n",
       "<tr><td>zeros  </td><td>0                 </td><td>         </td><td>0                 </td><td>14248            </td><td>                </td><td>0                </td><td>                  </td><td>            </td><td>270               </td><td>139459             </td><td>1562              </td><td>0                 </td><td>          </td><td>11                     </td><td>                     </td></tr>\n",
       "<tr><td>missing</td><td>0                 </td><td>0        </td><td>0                 </td><td>5804             </td><td>0               </td><td>4                </td><td>0                 </td><td>0           </td><td>0                 </td><td>29                 </td><td>193               </td><td>29                </td><td>0         </td><td>29                     </td><td>0                    </td></tr>\n",
       "<tr><td>0      </td><td>5000.0            </td><td>36 months</td><td>10.65             </td><td>10.0             </td><td>RENT            </td><td>24000.0          </td><td>credit_card       </td><td>AZ          </td><td>27.650000000000002</td><td>0.0                </td><td>83.7              </td><td>9.0               </td><td>0         </td><td>26.0                   </td><td>verified             </td></tr>\n",
       "<tr><td>1      </td><td>2500.0            </td><td>60 months</td><td>15.27             </td><td>0.0              </td><td>RENT            </td><td>30000.0          </td><td>car               </td><td>GA          </td><td>1.0               </td><td>0.0                </td><td>9.4               </td><td>4.0               </td><td>1         </td><td>12.0                   </td><td>verified             </td></tr>\n",
       "<tr><td>2      </td><td>2400.0            </td><td>36 months</td><td>15.96             </td><td>10.0             </td><td>RENT            </td><td>12252.0          </td><td>small_business    </td><td>IL          </td><td>8.72              </td><td>0.0                </td><td>98.5              </td><td>10.0              </td><td>0         </td><td>10.0                   </td><td>not verified         </td></tr>\n",
       "<tr><td>3      </td><td>10000.0           </td><td>36 months</td><td>13.49             </td><td>10.0             </td><td>RENT            </td><td>49200.0          </td><td>other             </td><td>CA          </td><td>20.0              </td><td>0.0                </td><td>21.0              </td><td>37.0              </td><td>0         </td><td>15.0                   </td><td>verified             </td></tr>\n",
       "<tr><td>4      </td><td>5000.0            </td><td>36 months</td><td>7.9               </td><td>3.0              </td><td>RENT            </td><td>36000.0          </td><td>wedding           </td><td>AZ          </td><td>11.200000000000001</td><td>0.0                </td><td>28.3              </td><td>12.0              </td><td>0         </td><td>7.0                    </td><td>verified             </td></tr>\n",
       "<tr><td>5      </td><td>3000.0            </td><td>36 months</td><td>18.64             </td><td>9.0              </td><td>RENT            </td><td>48000.0          </td><td>car               </td><td>CA          </td><td>5.3500000000000005</td><td>0.0                </td><td>87.5              </td><td>4.0               </td><td>0         </td><td>4.0                    </td><td>verified             </td></tr>\n",
       "<tr><td>6      </td><td>5600.0            </td><td>60 months</td><td>21.28             </td><td>4.0              </td><td>OWN             </td><td>40000.0          </td><td>small_business    </td><td>CA          </td><td>5.55              </td><td>0.0                </td><td>32.6              </td><td>13.0              </td><td>1         </td><td>7.0                    </td><td>verified             </td></tr>\n",
       "<tr><td>7      </td><td>5375.0            </td><td>60 months</td><td>12.69             </td><td>0.0              </td><td>RENT            </td><td>15000.0          </td><td>other             </td><td>TX          </td><td>18.080000000000002</td><td>0.0                </td><td>36.5              </td><td>3.0               </td><td>1         </td><td>7.0                    </td><td>verified             </td></tr>\n",
       "<tr><td>8      </td><td>6500.0            </td><td>60 months</td><td>14.65             </td><td>5.0              </td><td>OWN             </td><td>72000.0          </td><td>debt_consolidation</td><td>AZ          </td><td>16.12             </td><td>0.0                </td><td>20.6              </td><td>23.0              </td><td>0         </td><td>13.0                   </td><td>not verified         </td></tr>\n",
       "<tr><td>9      </td><td>12000.0           </td><td>36 months</td><td>12.69             </td><td>10.0             </td><td>OWN             </td><td>75000.0          </td><td>debt_consolidation</td><td>CA          </td><td>10.78             </td><td>0.0                </td><td>67.10000000000001 </td><td>34.0              </td><td>0         </td><td>22.0                   </td><td>verified             </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "frame.describe() # summarize table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# split into training and test for cross validation\n",
    "train, test = frame.split_frame([0.7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bad_loan\n",
      "['loan_amnt', 'term', 'int_rate', 'emp_length', 'home_ownership', 'annual_inc', 'purpose', 'addr_state', 'dti', 'delinq_2yrs', 'revol_util', 'total_acc', 'longest_credit_length', 'verification_status']\n"
     ]
    }
   ],
   "source": [
    "# assign target and inputs for logistic regression\n",
    "y = 'bad_loan'\n",
    "X = [name for name in frame.columns if name != y]\n",
    "print(y)\n",
    "print(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['verification_status', 'term', 'home_ownership', 'addr_state', 'purpose']\n",
      "['dti', 'annual_inc', 'delinq_2yrs', 'int_rate', 'longest_credit_length', 'revol_util', 'emp_length', 'loan_amnt', 'total_acc']\n"
     ]
    }
   ],
   "source": [
    "# determine column types\n",
    "reals, enums = [], []\n",
    "for key, val in frame.types.items():\n",
    "    if key in X:\n",
    "        if val == 'enum':\n",
    "            enums.append(key)\n",
    "        else: \n",
    "            reals.append(key)\n",
    "\n",
    "print(enums)\n",
    "print(reals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# impute missing values\n",
    "_ = frame[reals].impute(method='mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set target to factor for logisitic regression\n",
    "# just to be safe ...\n",
    "train[y] = train[y].asfactor()\n",
    "test[y] = test[y].asfactor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm Model Build progress: |███████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "# elastic net regularized regression\n",
    "#   - binomial family for logistic regression\n",
    "#   - L1 for variable selection\n",
    "#   - L2 for handling multicollinearity\n",
    "#   - IRLS for handling outliers\n",
    "#   - with lamba parameter tuning for variable selection and regularization\n",
    "\n",
    "# initialize\n",
    "loan_glm = H2OGeneralizedLinearEstimator(family='binomial',\n",
    "                                         model_id='loan_glm2',\n",
    "                                         solver='IRLSM',\n",
    "                                         standardize=True,\n",
    "                                         lambda_search=True)\n",
    "\n",
    "# train \n",
    "loan_glm.train(X, y, training_frame=train)\n",
    "\n",
    "# view detailed results at http://ip:port/flow/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "purpose.car :  -0.11190117054653674\n",
      "emp_length :  -0.0017696551450257373\n",
      "addr_state.TN :  0.012526186420074192\n",
      "term.60 months :  0.15562963741151795\n",
      "revol_util :  0.0037916526358893828\n",
      "addr_state.CO :  -0.15983916847312812\n",
      "addr_state.MI :  0.02091853569392306\n",
      "loan_amnt :  1.2785877478098858e-05\n",
      "home_ownership.RENT :  0.09227455982790071\n",
      "addr_state.NJ :  0.11907705662989007\n",
      "addr_state.NV :  0.11757735824973783\n",
      "purpose.other :  0.07133767802649965\n",
      "total_acc :  -0.010167858694788761\n",
      "addr_state.OR :  -0.005529884439359159\n",
      "verification_status.not verified :  -0.009727378241438905\n",
      "longest_credit_length :  0.004224928762668751\n",
      "purpose.major_purchase :  -0.04211397535003131\n",
      "addr_state.WV :  -0.00318184897313048\n",
      "purpose.small_business :  0.555619849621502\n",
      "purpose.credit_card :  -0.19244800923871863\n",
      "purpose.debt_consolidation :  -0.09928103992055347\n",
      "delinq_2yrs :  0.02652109180190443\n",
      "home_ownership.MORTGAGE :  -0.035726793917526437\n",
      "int_rate :  0.10380632783073096\n",
      "annual_inc :  -6.0929659692125846e-06\n",
      "verification_status.verified :  0.00957869891171434\n",
      "addr_state.FL :  0.16155591143358577\n",
      "addr_state.MO :  0.034963558598601015\n",
      "dti :  0.022635959362114613\n",
      "addr_state.NY :  0.03399174562042058\n",
      "Intercept :  -3.0355150801737745\n",
      "term.36 months :  -0.16945440197587225\n",
      "addr_state.TX :  -0.12320745319468676\n"
     ]
    }
   ],
   "source": [
    "# print non-zero model parameters\n",
    "for name, val in loan_glm.coef().items():\n",
    "    if val != 0.0:\n",
    "        print(name, ': ', val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6991436801020542\n",
      "0.6957946293349209\n"
     ]
    }
   ],
   "source": [
    "# measure train and test AUC\n",
    "print(loan_glm.auc(train=True))\n",
    "print(loan_glm.model_performance(test_data=test).auc())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Are you sure you want to shutdown the H2O instance running at http://127.0.0.1:54321 (Y/N)? y\n",
      "H2O session _sid_955e closed.\n"
     ]
    }
   ],
   "source": [
    "# shutdown h2o ... be careful this can erase your work\n",
    "h2o.cluster().shutdown(prompt=True)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}