18BIS0103 LAB TAS-2 2nd part.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "derived-burst",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Date  Month  Year  Hours  Min  out/in  temp\n",
      "0     8     12  2018      9   30       2  29.0\n",
      "1     8     12  2018      9   30       2  29.0\n",
      "2     8     12  2018      9   29       1  41.0\n",
      "3     8     12  2018      9   29       1  41.0\n",
      "4     8     12  2018      9   29       2  31.0\n"
     ]
    }
   ],
   "source": [
    "#We are going to import from Pandas,Numpy, and sklearn and store the data frame into the variable df [18BIS0103]\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "df = pd.read_csv(\"IOT-temp dataset task 2_UPDATED.csv\")\n",
    "print (df.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "terminal-twins",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37281\n"
     ]
    }
   ],
   "source": [
    "#REMOVING DUPLICATE\n",
    "df_dropped = df\n",
    "\n",
    "#We are going to Remove duplicate values\n",
    "df_dropped.drop_duplicates(inplace=True)\n",
    "df_dropped[df_dropped.duplicated()]\n",
    "entries=df_dropped.shape\n",
    "entries = int(entries[0])\n",
    "print(entries)\n",
    "#We also stored the number of entries for later use\n",
    "#These are the number of rows after the duplicated values are removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "significant-edgar",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Date      0\n",
      "Month     0\n",
      "Year      0\n",
      "Hours     0\n",
      "Min       0\n",
      "out/in    0\n",
      "temp      0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#We are going to remove the missing data and put in the MEAN value\n",
    "\n",
    "missing_value = ['temp']\n",
    "\n",
    "for column in missing_value:\n",
    "    df_dropped[column] = df_dropped[column].replace(np.NaN, np.NaN)\n",
    "    mean = int(df_dropped[column].mean(skipna=True))\n",
    "    df_dropped[column] = df_dropped[column].replace(np.NaN, mean)\n",
    "    \n",
    "print(df_dropped.isnull().sum())\n",
    "\n",
    "#The below is a proof that the Data is free of null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "scientific-electric",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Date      0\n",
      "Month     0\n",
      "Year      0\n",
      "Hours     0\n",
      "Min       0\n",
      "out/in    0\n",
      "temp      0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#NOTE : THIS IS THE SAME STEP AS THE PREVIOUS ONE, EXECPT WE USE MEDIAN VALUE\n",
    "#We are going to remove the missing data and put in the median value\n",
    "\n",
    "missing_value = ['temp']\n",
    "\n",
    "for column in missing_value:\n",
    "    df_dropped[column] = df_dropped[column].replace(np.NaN, np.NaN)\n",
    "    median = int(df_dropped[column].median(skipna=True))\n",
    "    df_dropped[column] = df_dropped[column].replace(np.NaN, median)\n",
    "    \n",
    "print(df_dropped.isnull().sum())\n",
    "\n",
    "#The below is a proof that the Data is free of null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "fossil-award",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    Date  Month  Year  Hours  Min  out/in  temp\n",
      "2      8     12  2018      9   29       1  41.0\n",
      "10     8     12  2018      9   25       1  42.0\n",
      "16     8     12  2018      9   21       1  41.0\n",
      "20     8     12  2018      9   19       1  42.0\n",
      "24     8     12  2018      9   17       1  41.0\n",
      "    Date  Month  Year  Hours  Min  out/in  temp\n",
      "0      8     12  2018      9   30       2  29.0\n",
      "4      8     12  2018      9   29       2  31.0\n",
      "6      8     12  2018      9   28       2  29.0\n",
      "8      8     12  2018      9   26       2  29.0\n",
      "12     8     12  2018      9   24       2  29.0\n"
     ]
    }
   ],
   "source": [
    "#Split the dataset into out and in\n",
    " \n",
    "df_dropped_out = df_dropped[df_dropped['out/in'] ==1] #This Varialble has OUT in it\n",
    "df_dropped_in = df_dropped[df_dropped['out/in'] ==2] #This variable has IN in it\n",
    "\n",
    "print(df_dropped_out.head())\n",
    "print(df_dropped_in.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "muslim-pierre",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Splitting dataset for training and testing for OUT --> 1 for out, 2 --> IN\n",
    "\n",
    "X1 = df_dropped_out.iloc[:, 0:6]\n",
    "y1 = df_dropped_out.iloc[:, 6]\n",
    "X1_train, X1_test, y1_train, y1_test = train_test_split(X1,y1,random_state=0, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "complimentary-suspension",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Feature scaling for OUT\n",
    "sc_X1 = StandardScaler()\n",
    "X1_train = sc_X1.fit_transform(X1_train)\n",
    "X1_test = sc_X1.transform(X1_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "aboriginal-physiology",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Splitting dataset for training and testing for IN --> 1 for out, 2 for in\n",
    "\n",
    "X2 = df_dropped_in.iloc[:, 0:6]\n",
    "y2 = df_dropped_in.iloc[:, 6]\n",
    "X2_train, X2_test, y2_train, y2_test = train_test_split(X2,y2,random_state=0, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "dutch-barrier",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Feature scaling for IN\n",
    "sc_X2 = StandardScaler()\n",
    "X2_train = sc_X2.fit_transform(X2_train)\n",
    "X2_test = sc_X2.transform(X2_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "stunning-yugoslavia",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define the model: Init K-NN for OUT | n_neighbors=3\n",
    "classifier_OUT = KNeighborsClassifier(n_neighbors=3, p='weighted', metric='euclidean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "wicked-cambodia",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(metric='euclidean', n_neighbors=3, p='weighted')"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Fit Model for OUT\n",
    "classifier_OUT.fit(X1_train, y1_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "happy-sessions",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define the model: Init K-NN for IN | n_neighbors=3\n",
    "classifier_IN = KNeighborsClassifier(n_neighbors=3, p=2, metric='euclidean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "first-active",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(metric='euclidean', n_neighbors=3)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Fit Model for IN\n",
    "classifier_IN.fit(X1_train, y1_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "every-medication",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73.93916418245476"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import math #OUT\n",
    "math.sqrt(len(y1_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "creative-cemetery",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44.609416046390926"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#IN\n",
    "math.sqrt(len(y2_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "moral-share",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Predict the test results for OUT\n",
    "y1_pred = classifier_OUT.predict(X1_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "funky-newspaper",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Predict the test results for IN\n",
    "y2_pred = classifier_IN.predict(X2_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "above-implement",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  1   0   0   0   1   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   2   2   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  1   2   3  10   3   1   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  1   4  16  28  84   6   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  1   1  12  96  61  85   0   1   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   1  22  98  36  43   4   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   1   9  46  84  92   6   0   0   0   1   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   2   3  18 127  62  40   1   0   0   0   0   1   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   2  16  62  29  25   5   1   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   2  13  52  31  40   3   1   0   0   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   1   0   2  14  64  33   9   4   1   2   0   0   0\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   3   5  38  18  37   8   0   2   0   1\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   2   4  10  13  54  59  75   9   2   2   1\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   1   2   2   1   7  27 107 112  50   4   1   0\n",
      "    3   1   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   2   4  10  34  94  78  93   6   3\n",
      "    0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   1   3   4   9  36 111 115  82  21\n",
      "    2   0   1   1   0   1   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   4   6  13  26 136  77  69\n",
      "   10   1   1   1   2   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   2   0   0   0   0   0   1   6  20  56 103 130\n",
      "   84   8   1   0   2   1   0   0   0   0]\n",
      " [  0   0   0   1   0   0   1   0   0   0   0   2   3   6   3  20  14 117\n",
      "   83  59   5   1   2   0   0   0   0   0]\n",
      " [  0   0   0   0   1   1   0   0   0   0   0   0   3   2   4   7  16  24\n",
      "   71  69  61   9   4   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   1   0   0   1   3   1   8   6  16\n",
      "   33  69  87  60   5   1   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   5   3   0   9\n",
      "   12  17  97  55  36   2   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   4   6   2   3\n",
      "    8  15  29  79  46  33   3   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  10   3   3\n",
      "    1   1  10  13  48  53  26   4   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  10   4   1\n",
      "    5   3   5  16  16  57  59  15   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   1   1   2   2\n",
      "    0   1   1   2   1  14  45  13   2   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   1   3   3   4   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
      "    0   0   0   0   0   0   1   0   0   0]]\n",
      "0.2601960420218053\n"
     ]
    }
   ],
   "source": [
    "#Evaluate model OUT\n",
    "cm_out = confusion_matrix(y1_test, y1_pred)\n",
    "print (cm_out)\n",
    "print (f1_score(y1_test, y1_pred, average='weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "dress-dominant",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0   0   0   0   0   0   0   0   0   1   0   0   0   0   1   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   1   0   0   0   0   4   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   5   0   0   0   0  13   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   3   0   0   2   2  32   0   1   0]\n",
      " [  0   0   0   0   0   0   0   0   0  13   0   0  11   2  76   0  11   0]\n",
      " [  0   0   0   0   0   0   0   0   0   8   0   0  19   2 116   0  11   0]\n",
      " [  0   0   0   0   0   0   0   0   0   2   0   0  13   9 267   0   9   0]\n",
      " [  0   0   0   0   0   0   0   0   0   5   0   0  45   9 219   1   3   0]\n",
      " [  0   0   0   0   0   0   0   0   0   9   0   0  41  13 264   0   4   0]\n",
      " [  0   0   0   0   0   0   0   0   0  27   0   0  24   7 228   0   4   1]\n",
      " [  0   0   0   0   0   0   0   0   0   2   0   0  38  13 171   0   7   1]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0  19   6 121   0   6   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   6   4  50   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   8   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0]]\n",
      "0.022817772236006055\n"
     ]
    }
   ],
   "source": [
    "#Evaluate model IN\n",
    "cm_in = confusion_matrix(y2_test, y2_pred)\n",
    "print (cm_in)\n",
    "print (f1_score(y2_test, y2_pred, average='weighted'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}