Final_Book.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "executionInfo": {
     "elapsed": 1772,
     "status": "ok",
     "timestamp": 1616428530729,
     "user": {
      "displayName": "Ajay Jogiya",
      "photoUrl": "",
      "userId": "12844089788344129075"
     },
     "user_tz": -330
    },
    "id": "5fuSeh64vjuh"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import time\n",
    "import pickle\n",
    "import joblib\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "from joblib import dump, load\n",
    "from def_functions import remove_funtion , age_replace , get_d_m_y , \\\n",
    "    separate_genre , song_len, like_language , count_funtion , get_composer_name , \\\n",
    "    get_artist_name , extract_code , get_isrc , calculate_groupby_features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377418, 25)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('data.csv')\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_point = pd.read_csv('q_point_1.csv')\n",
    "q_point = q_point.drop('id',axis = 1) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LHvDcTnJm9G4"
   },
   "source": [
    "\n",
    "# **Function_1: Prediction Function**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "executionInfo": {
     "elapsed": 1269,
     "status": "ok",
     "timestamp": 1616430816438,
     "user": {
      "displayName": "Ajay Jogiya",
      "photoUrl": "",
      "userId": "12844089788344129075"
     },
     "user_tz": -330
    },
    "id": "8xWtkkQu24kV"
   },
   "outputs": [],
   "source": [
    "def function_1(q_point):\n",
    "    start = time.time()\n",
    "    \n",
    "\n",
    "    # merge data on test file\n",
    "\n",
    "    q_point = remove_funtion(q_point)\n",
    "\n",
    "    ''' fill missing values '''\n",
    "\n",
    "    q_point['composer'] = q_point['composer'].fillna('no_name')\n",
    "\n",
    "    q_point['lyricist'] = q_point['lyricist'].fillna('no_name')\n",
    "\n",
    "    q_point['source_screen_name'] = q_point['source_screen_name'].fillna('not_define')\n",
    "\n",
    "    q_point['gender'] = q_point['gender'].fillna('not_define')\n",
    "\n",
    "    # for numeric feature\n",
    "    q_point['isrc'] = q_point['isrc'].fillna('0')\n",
    "\n",
    "    # replace missing values with mode \n",
    "    q_point['source_system_tab'] = q_point['source_system_tab'].fillna('my library')\n",
    "\n",
    "    q_point['source_type'] = q_point['source_type'].fillna('local-library')\n",
    "\n",
    "    q_point['name'] = q_point['name'].fillna('演員')\n",
    "\n",
    "    q_point['song_length'] = q_point['song_length'].fillna(241812.0)# meadian of song_length \n",
    "\n",
    "    q_point['artist_name'] = q_point['artist_name'].fillna('Various Artists')\n",
    "\n",
    "    q_point['language'] = q_point['language'].fillna('3.0')\n",
    "\n",
    "    ''' feature extraction '''\n",
    "    \n",
    "    # seprate each genre\n",
    "    q_point = separate_genre(q_point)\n",
    "\n",
    "    mean = 245120.9693600806\n",
    "    q_point = song_len(mean,q_point)\n",
    "\n",
    "    q_point = like_language(q_point)\n",
    "\n",
    "    # count of composer artist\n",
    "    composer_count = []\n",
    "    for i in q_point['composer']:\n",
    "        s = count_funtion(i,'no_name')\n",
    "        composer_count.append(s)\n",
    "    q_point['composer_count'] = composer_count\n",
    "\n",
    "    # count of lyricist artist\n",
    "    lyricist_count = []\n",
    "    for i in q_point['lyricist']:\n",
    "        s = count_funtion(i,'no_name')\n",
    "        lyricist_count.append(s)\n",
    "    q_point['lyricist_count'] = lyricist_count\n",
    "\n",
    "    # get first composer name\n",
    "    q_point = get_composer_name(q_point)\n",
    "\n",
    "    # get first artist name\n",
    "    q_point = get_artist_name(q_point)\n",
    "\n",
    "    # extract features from isrc: country_code , regi_code , year , designation_code\n",
    "    q_point = get_isrc(q_point)\n",
    "\n",
    "    # group by features\n",
    "    q_point = calculate_groupby_features(q_point)\n",
    "\n",
    "    # drop feature which have decrease performance\n",
    "    drop_list  = ['city','composer_count','country_code ', 'eight_genre',\n",
    "                  'expire_month', 'five_genre','four_genre', 'gender',\n",
    "                  'isrc', 'one_genre','regi_day', 'seven_genre', 'six_genre', 'three_genre']\n",
    "    q_point = q_point.drop(drop_list , axis = 1)\n",
    "\n",
    "    cat_features = ['msno','song_id', 'source_system_tab', 'source_screen_name',\n",
    "                    'source_type','registered_via', 'regi_month','regi_year', \n",
    "                    'expire_day', 'expire_year',\n",
    "                    'language', 'two_genre','composer_first_name',\n",
    "                    'first_artist_name','regi_code', 'year','designation_code',\n",
    "                    'artist_name','lyricist','composer','name']\n",
    "\n",
    "    q_point[cat_features] = q_point[cat_features].astype(str) \n",
    "    \n",
    "    # label encoding\n",
    "    label_dic = ['song_id.pkl','source_system_tab.pkl','source_screen_name.pkl',\n",
    "                'source_type.pkl','registered_via.pkl','regi_month.pkl','regi_year.pkl',\n",
    "                'expire_day.pkl','expire_year.pkl','language.pkl','two_genre.pkl',\n",
    "                'composer_first_name.pkl','first_artist_name.pkl','regi_code.pkl','year.pkl',\n",
    "                'designation_code.pkl','msno.pkl','artist_name.pkl','composer.pkl',\n",
    "                'lyricist.pkl']\n",
    "    \n",
    "    label_dic = os.listdir(r\"G:\\Project 1\\Music Recommandation Systam\\Deployment\\label_dicts\\old\")\n",
    "\n",
    "    for name in label_dic:\n",
    "        # load pickle file of dict for labels\n",
    "        #print(name)\n",
    "        pkl_file = open(r'G:\\Project 1\\Music Recommandation Systam\\Deployment\\label_dicts\\old\\{}'.format(name), 'rb')\n",
    "        le = pickle.load(pkl_file) \n",
    "        \n",
    "        col_name = name[:-7]\n",
    "        # replace key(categories) with there values(labels)\n",
    "        q_point[col_name] = q_point[col_name].astype(str)\n",
    "        q_point[col_name] = q_point[col_name].map(le)\n",
    "        #b.close()\n",
    "        #pkl_file.close()\n",
    "        #print(q_point)\n",
    "\n",
    "    name = 'name.pickle'\n",
    "    # label encoding\n",
    "    with open(r\"G:\\Project 1\\Music Recommandation Systam\\Deployment\\label_dicts\\new\\name.pickle\", 'rb') as handle:\n",
    "        b = pickle.load(handle)\n",
    "    col_name = name[:-7]\n",
    "    # replace key(categories) with there values(labels)\n",
    "    q_point[col_name] = q_point[col_name].astype(str)\n",
    "    q_point[col_name] = q_point[col_name].map(b)\n",
    "    handle.close()\n",
    "    \n",
    "    # standar scaler\n",
    "    std = os.listdir(r\"G:\\Project 1\\Music Recommandation Systam\\Deployment\\std_scaler\")\n",
    "    for name in std:\n",
    "        # load standard scaler\n",
    "        scaler = load(r\"G:\\Project 1\\Music Recommandation Systam\\Deployment\\std_scaler\\{}\".format(name))\n",
    "        col_name = name[:-15]\n",
    "        q_point[col_name] = q_point[col_name].astype(float)\n",
    "        q_point[col_name] = scaler.transform(q_point[col_name].values.reshape(-1,1))\n",
    "    \n",
    "    prob = ['source_screen_name.pickle', 'source_type.pickle']\n",
    "    #print('std Done.')\n",
    "    for name in prob:\n",
    "        with open(r\"G:\\Project 1\\Music Recommandation Systam\\Deployment\\prob\\{}\".format(name), 'rb') as handle:\n",
    "            b = pickle.load(handle)\n",
    "        col_name = name[:-7]\n",
    "        q_point['prob_'+col_name] = q_point[col_name].map(b)\n",
    "    #print('prob Done')\n",
    "    lgbm = joblib.load(r\"G:\\Project 1\\Music Recommandation Systam\\Deployment\\best_model\\lgbm.pkl\")\n",
    "    #print('start prediction.')\n",
    "    pred = lgbm.predict(q_point)\n",
    "\n",
    "    #print('Run Time :',(time.time() - start) , 'sec' )\n",
    "\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sxHBiUK5Rs15"
   },
   "source": [
    "# **Prediction**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 78963,
     "status": "ok",
     "timestamp": 1616430905554,
     "user": {
      "displayName": "Ajay Jogiya",
      "photoUrl": "",
      "userId": "12844089788344129075"
     },
     "user_tz": -330
    },
    "id": "F44razTi4O_Q",
    "outputId": "91c9fa0f-ddac-46c0-e94a-4ca4f76c5480"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predicted Class is 1.\n",
      "Run Time : 4.499501705169678\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "prediction = function_1(q_point)\n",
    "print()\n",
    "print(\"Predicted Class is {}.\".format(prediction[0]))\n",
    "print('Run Time :',time.time() - start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "x126OFpuySeb"
   },
   "source": [
    "# **Function_2 : Compute Performance Matric**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "executionInfo": {
     "elapsed": 1127,
     "status": "ok",
     "timestamp": 1616432550256,
     "user": {
      "displayName": "Ajay Jogiya",
      "photoUrl": "",
      "userId": "12844089788344129075"
     },
     "user_tz": -330
    },
    "id": "KZnClM9AT6-q"
   },
   "outputs": [],
   "source": [
    "def function_2(q_point,y_label):\n",
    "    start = time.time()\n",
    "    # merge data on test file\n",
    "    member = pd.read_csv('/content/drive/MyDrive/Case_study/new_member.csv')\n",
    "    song_info = pd.read_csv('/content/drive/MyDrive/Case_study/csv_files/songs_info.csv')\n",
    "\n",
    "    test_member = q_point.merge(member , how = 'left' , on = 'msno')\n",
    "    q_point = test_member.merge(songs_info, how='left', on='song_id')\n",
    "    q_point = remove_funtion(q_point)\n",
    "\n",
    "    ''' fill missing values '''\n",
    "\n",
    "    q_point['composer'] = q_point['composer'].fillna('no_name')\n",
    "\n",
    "    q_point['lyricist'] = q_point['lyricist'].fillna('no_name')\n",
    "\n",
    "    q_point['source_screen_name'] = q_point['source_screen_name'].fillna('not_define')\n",
    "\n",
    "    q_point['gender'] = q_point['gender'].fillna('not_define')\n",
    "\n",
    "    # for numeric feature\n",
    "    q_point['isrc'] = q_point['isrc'].fillna('0')\n",
    "\n",
    "    # replace missing values with mode \n",
    "    q_point['source_system_tab'] = q_point['source_system_tab'].fillna('my library')\n",
    "\n",
    "    q_point['source_type'] = q_point['source_type'].fillna('local-library')\n",
    "\n",
    "    q_point['name'] = q_point['name'].fillna('演員')\n",
    "\n",
    "    q_point['song_length'] = q_point['song_length'].fillna(241812.0)# meadian of song_length \n",
    "\n",
    "    q_point['artist_name'] = q_point['artist_name'].fillna('Various Artists')\n",
    "\n",
    "    q_point['language'] = q_point['language'].fillna('3.0')\n",
    "\n",
    "    ''' feature extraction '''\n",
    "    \n",
    "    # seprate each genre\n",
    "    q_point = separate_genre(q_point)\n",
    "\n",
    "    mean = 245120.9693600806\n",
    "    q_point = song_len(mean,q_point)\n",
    "\n",
    "    q_point = like_language(q_point)\n",
    "\n",
    "    # count of composer artist\n",
    "    composer_count = []\n",
    "    for i in q_point['composer']:\n",
    "        s = count_funtion(i,'no_name')\n",
    "        composer_count.append(s)\n",
    "    q_point['composer_count'] = composer_count\n",
    "\n",
    "    # count of lyricist artist\n",
    "    lyricist_count = []\n",
    "    for i in q_point['lyricist']:\n",
    "        s = count_funtion(i,'no_name')\n",
    "        lyricist_count.append(s)\n",
    "    q_point['lyricist_count'] = lyricist_count\n",
    "\n",
    "    # get first composer name\n",
    "    q_point = get_composer_name(q_point)\n",
    "\n",
    "    # get first artist name\n",
    "    q_point = get_artist_name(q_point)\n",
    "\n",
    "    # extract features from isrc: country_code , regi_code , year , designation_code\n",
    "    q_point = get_isrc(q_point)\n",
    "\n",
    "    # group by features\n",
    "    q_point = calculate_groupby_features(q_point)\n",
    "\n",
    "    # drop feature which have decrease performance\n",
    "    drop_list  = ['city','composer_count','country_code ', 'eight_genre',\n",
    "                  'expire_month', 'five_genre','four_genre', 'gender',\n",
    "                  'isrc', 'one_genre','regi_day', 'seven_genre', 'six_genre', 'three_genre']\n",
    "    q_point = q_point.drop(drop_list , axis = 1)\n",
    "\n",
    "    cat_features = ['msno','song_id', 'source_system_tab', 'source_screen_name',\n",
    "                    'source_type','registered_via', 'regi_month','regi_year', \n",
    "                    'expire_day', 'expire_year',\n",
    "                    'language', 'two_genre','composer_first_name',\n",
    "                    'first_artist_name','regi_code', 'year','designation_code',\n",
    "                    'artist_name','lyricist','composer','name']\n",
    "\n",
    "    q_point[cat_features] = q_point[cat_features].astype(str) \n",
    "\n",
    "    # label encoding\n",
    "    label_dic = ['song_id.pkl','source_system_tab.pkl','source_screen_name.pkl',\n",
    "                'source_type.pkl','registered_via.pkl','regi_month.pkl','regi_year.pkl',\n",
    "                'expire_day.pkl','expire_year.pkl','language.pkl','two_genre.pkl',\n",
    "                'composer_first_name.pkl','first_artist_name.pkl','regi_code.pkl','year.pkl',\n",
    "                'designation_code.pkl','msno.pkl','artist_name.pkl','composer.pkl',\n",
    "                'lyricist.pkl']\n",
    "    for name in label_dic:\n",
    "        # load pickle file of dict for labels\n",
    "        pkl_file = open('/content/drive/MyDrive/Case_study/label_dicts/new/{}'.format(name), 'rb')\n",
    "        le = pickle.load(pkl_file) \n",
    "        \n",
    "        col_name = name[:-4]\n",
    "        #print(col_name)\n",
    "        # replace key(categories) with there values(labels)\n",
    "        q_point[col_name] = le.transform(q_point[col_name])\n",
    "        #b.close()\n",
    "        pkl_file.close()\n",
    "\n",
    "    name = 'name.pickle'\n",
    "    # label encoding\n",
    "    with open('/content/drive/MyDrive/Case_study/label_dicts/new/name.pickle', 'rb') as handle:\n",
    "        b = pickle.load(handle)\n",
    "    col_name = name[:-7]\n",
    "    # replace key(categories) with there values(labels)\n",
    "    q_point[col_name] = q_point[col_name].astype(str)\n",
    "    q_point[col_name] = q_point[col_name].map(b)\n",
    "    handle.close()\n",
    "\n",
    "    # standar scaler\n",
    "    std = os.listdir('/content/drive/MyDrive/Case_study/std_scaler')\n",
    "    for name in std:\n",
    "        # load standard scaler\n",
    "        scaler = load('/content/drive/MyDrive/Case_study/std_scaler/{}'.format(name))\n",
    "        col_name = name[:-15]\n",
    "        q_point[col_name] = q_point[col_name].astype(float)\n",
    "        q_point[col_name] = scaler.transform(q_point[col_name].values.reshape(-1,1))\n",
    "    \n",
    "    #find conditional probability of both features\n",
    "    prob = ['source_screen_name.pickle', 'source_type.pickle']\n",
    "    for name in prob:\n",
    "        with open('/content/drive/MyDrive/Case_study/prob/{}'.format(name), 'rb') as handle:\n",
    "            b = pickle.load(handle)\n",
    "        col_name = name[:-7]\n",
    "        q_point['prob_'+col_name] = q_point[col_name].map(b)\n",
    "\n",
    "    # for auc score we need probability of class\n",
    "    # for improve performance we take average of both model\n",
    "    stacking = joblib.load('/content/drive/MyDrive/Case_study/best_model/stacking.pkl')\n",
    "    lgbm = joblib.load('/content/drive/MyDrive/Case_study/best_model/lgbm.pkl')\n",
    "\n",
    "    pred1 = stacking.predict_proba(q_point)\n",
    "    pred2 = lgbm.predict_proba(q_point)\n",
    "\n",
    "    pred1 = pred1[:,1]\n",
    "    pred2 = pred2[:,1]\n",
    "\n",
    "    pred = (pred1 + pred2) / 2\n",
    "\n",
    "    print('AUC Score  : ',roc_auc_score(y_label , pred))\n",
    "    print('Run Time   :',(time.time() - start) , 'sec' )\n",
    "\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "executionInfo": {
     "elapsed": 1067,
     "status": "ok",
     "timestamp": 1616434384055,
     "user": {
      "displayName": "Ajay Jogiya",
      "photoUrl": "",
      "userId": "12844089788344129075"
     },
     "user_tz": -330
    },
    "id": "GOEeia5npStr"
   },
   "outputs": [],
   "source": [
    "# take sample from train data \n",
    "test_data = train.sample(100)\n",
    "y_label   = test_data['target']\n",
    "test_data = test_data.drop('target',axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 100922,
     "status": "ok",
     "timestamp": 1616434484571,
     "user": {
      "displayName": "Ajay Jogiya",
      "photoUrl": "",
      "userId": "12844089788344129075"
     },
     "user_tz": -330
    },
    "id": "xh7ugaMRQus9",
    "outputId": "831bcfa1-bb4b-43e2-b5ed-71890f06a7b3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score  :  0.6398559423769508\n",
      "Run Time   : 99.38778257369995 sec\n"
     ]
    }
   ],
   "source": [
    "pred = function_2(test_data , y_label)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Final_Book.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}