SVM-NSL_KDD.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sys\n",
    "import sklearn\n",
    "import io\n",
    "import random\n",
    "import time\n",
    "train_url = 'https://raw.githubusercontent.com/merteroglu/NSL-KDD-Network-Instrusion-Detection/master/NSL_KDD_Train.csv'\n",
    "test_url = 'https://raw.githubusercontent.com/merteroglu/NSL-KDD-Network-Instrusion-Detection/master/NSL_KDD_Test.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimensions of the Training set: (125973, 42)\n",
      "Dimensions of the Test set: (22544, 42)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>ftp_data</td>\n",
       "      <td>SF</td>\n",
       "      <td>491</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>25</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>udp</td>\n",
       "      <td>other</td>\n",
       "      <td>SF</td>\n",
       "      <td>146</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>private</td>\n",
       "      <td>S0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>26</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>neptune</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>232</td>\n",
       "      <td>8153</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>199</td>\n",
       "      <td>420</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration protocol_type   service flag  src_bytes  dst_bytes  land  \\\n",
       "0         0           tcp  ftp_data   SF        491          0     0   \n",
       "1         0           udp     other   SF        146          0     0   \n",
       "2         0           tcp   private   S0          0          0     0   \n",
       "3         0           tcp      http   SF        232       8153     0   \n",
       "4         0           tcp      http   SF        199        420     0   \n",
       "\n",
       "   wrong_fragment  urgent  hot  ...  dst_host_srv_count  \\\n",
       "0               0       0    0  ...                  25   \n",
       "1               0       0    0  ...                   1   \n",
       "2               0       0    0  ...                  26   \n",
       "3               0       0    0  ...                 255   \n",
       "4               0       0    0  ...                 255   \n",
       "\n",
       "   dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n",
       "0                    0.17                    0.03   \n",
       "1                    0.00                    0.60   \n",
       "2                    0.10                    0.05   \n",
       "3                    1.00                    0.00   \n",
       "4                    1.00                    0.00   \n",
       "\n",
       "   dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "0                         0.17                         0.00   \n",
       "1                         0.88                         0.00   \n",
       "2                         0.00                         0.00   \n",
       "3                         0.03                         0.04   \n",
       "4                         0.00                         0.00   \n",
       "\n",
       "   dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "0                  0.00                      0.00                  0.05   \n",
       "1                  0.00                      0.00                  0.00   \n",
       "2                  1.00                      1.00                  0.00   \n",
       "3                  0.03                      0.01                  0.00   \n",
       "4                  0.00                      0.00                  0.00   \n",
       "\n",
       "   dst_host_srv_rerror_rate    label  \n",
       "0                      0.00   normal  \n",
       "1                      0.00   normal  \n",
       "2                      0.00  neptune  \n",
       "3                      0.01   normal  \n",
       "4                      0.00   normal  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_names = [\"duration\",\"protocol_type\",\"service\",\"flag\",\"src_bytes\",\n",
    "    \"dst_bytes\",\"land\",\"wrong_fragment\",\"urgent\",\"hot\",\"num_failed_logins\",\n",
    "    \"logged_in\",\"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\n",
    "    \"num_file_creations\",\"num_shells\",\"num_access_files\",\"num_outbound_cmds\",\n",
    "    \"is_host_login\",\"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\n",
    "    \"srv_serror_rate\",\"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\n",
    "    \"diff_srv_rate\",\"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\",\n",
    "    \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\"dst_host_same_src_port_rate\",\n",
    "    \"dst_host_srv_diff_host_rate\",\"dst_host_serror_rate\",\"dst_host_srv_serror_rate\",\n",
    "    \"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\",\"label\"]\n",
    "\n",
    "df = pd.read_csv(train_url,header=None, names = col_names)\n",
    "df_test = pd.read_csv(test_url, header=None, names = col_names)\n",
    "print('Dimensions of the Training set:',df.shape)\n",
    "print('Dimensions of the Test set:',df_test.shape)\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label distribution Training set:\n",
      "normal             67343\n",
      "neptune            41214\n",
      "satan               3633\n",
      "ipsweep             3599\n",
      "portsweep           2931\n",
      "smurf               2646\n",
      "nmap                1493\n",
      "back                 956\n",
      "teardrop             892\n",
      "warezclient          890\n",
      "pod                  201\n",
      "guess_passwd          53\n",
      "buffer_overflow       30\n",
      "warezmaster           20\n",
      "land                  18\n",
      "imap                  11\n",
      "rootkit               10\n",
      "loadmodule             9\n",
      "ftp_write              8\n",
      "multihop               7\n",
      "phf                    4\n",
      "perl                   3\n",
      "spy                    2\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print('Label distribution Training set:')\n",
    "print(df['label'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label distribution Test set:\n",
      "normal             9711\n",
      "neptune            4657\n",
      "guess_passwd       1231\n",
      "mscan               996\n",
      "warezmaster         944\n",
      "apache2             737\n",
      "satan               735\n",
      "processtable        685\n",
      "smurf               665\n",
      "back                359\n",
      "snmpguess           331\n",
      "saint               319\n",
      "mailbomb            293\n",
      "snmpgetattack       178\n",
      "portsweep           157\n",
      "ipsweep             141\n",
      "httptunnel          133\n",
      "nmap                 73\n",
      "pod                  41\n",
      "buffer_overflow      20\n",
      "multihop             18\n",
      "named                17\n",
      "ps                   15\n",
      "sendmail             14\n",
      "rootkit              13\n",
      "xterm                13\n",
      "teardrop             12\n",
      "xlock                 9\n",
      "land                  7\n",
      "xsnoop                4\n",
      "ftp_write             3\n",
      "worm                  2\n",
      "sqlattack             2\n",
      "perl                  2\n",
      "udpstorm              2\n",
      "phf                   2\n",
      "loadmodule            2\n",
      "imap                  1\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print('Label distribution Test set:')\n",
    "print(df_test['label'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set:\n",
      "Feature 'protocol_type' has 3 categories\n",
      "Feature 'service' has 70 categories\n",
      "Feature 'flag' has 11 categories\n",
      "Feature 'label' has 23 categories\n"
     ]
    }
   ],
   "source": [
    "# categorical to binary: protocol_type (column 2), service (column 3), flag (column 4).\n",
    "print('Training set:')\n",
    "for col_name in df.columns:\n",
    "    if df[col_name].dtypes == 'object' :\n",
    "        unique_cat = len(df[col_name].unique())\n",
    "        print(\"Feature '{col_name}' has {unique_cat} categories\".format(col_name=col_name, unique_cat=unique_cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set:\n",
      "Feature 'protocol_type' has 3 categories\n",
      "Feature 'service' has 64 categories\n",
      "Feature 'flag' has 11 categories\n",
      "Feature 'label' has 38 categories\n"
     ]
    }
   ],
   "source": [
    "print('Test set:')\n",
    "for col_name in df_test.columns:\n",
    "    if df_test[col_name].dtypes == 'object' :\n",
    "        unique_cat = len(df_test[col_name].unique())\n",
    "        print(\"Feature '{col_name}' has {unique_cat} categories\".format(col_name=col_name, unique_cat=unique_cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>ftp_data</td>\n",
       "      <td>SF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>udp</td>\n",
       "      <td>other</td>\n",
       "      <td>SF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>tcp</td>\n",
       "      <td>private</td>\n",
       "      <td>S0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  protocol_type   service flag\n",
       "0           tcp  ftp_data   SF\n",
       "1           udp     other   SF\n",
       "2           tcp   private   S0\n",
       "3           tcp      http   SF\n",
       "4           tcp      http   SF"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LabelEncoder\n",
    "from sklearn.preprocessing import LabelEncoder,OneHotEncoder\n",
    "categorical_columns=['protocol_type', 'service', 'flag']\n",
    "df_categorical_values = df[categorical_columns]\n",
    "testdf_categorical_values = df_test[categorical_columns]\n",
    "\n",
    "df_categorical_values.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Protocol_type_icmp', 'Protocol_type_tcp', 'Protocol_type_udp']\n"
     ]
    }
   ],
   "source": [
    "# protocol type\n",
    "unique_protocol=sorted(df.protocol_type.unique())\n",
    "string1 = 'Protocol_type_'\n",
    "unique_protocol2=[string1 + x for x in unique_protocol]\n",
    "print(unique_protocol2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['service_IRC', 'service_X11', 'service_Z39_50', 'service_aol', 'service_auth', 'service_bgp', 'service_courier', 'service_csnet_ns', 'service_ctf', 'service_daytime', 'service_discard', 'service_domain', 'service_domain_u', 'service_echo', 'service_eco_i', 'service_ecr_i', 'service_efs', 'service_exec', 'service_finger', 'service_ftp', 'service_ftp_data', 'service_gopher', 'service_harvest', 'service_hostnames', 'service_http', 'service_http_2784', 'service_http_443', 'service_http_8001', 'service_imap4', 'service_iso_tsap', 'service_klogin', 'service_kshell', 'service_ldap', 'service_link', 'service_login', 'service_mtp', 'service_name', 'service_netbios_dgm', 'service_netbios_ns', 'service_netbios_ssn', 'service_netstat', 'service_nnsp', 'service_nntp', 'service_ntp_u', 'service_other', 'service_pm_dump', 'service_pop_2', 'service_pop_3', 'service_printer', 'service_private', 'service_red_i', 'service_remote_job', 'service_rje', 'service_shell', 'service_smtp', 'service_sql_net', 'service_ssh', 'service_sunrpc', 'service_supdup', 'service_systat', 'service_telnet', 'service_tftp_u', 'service_tim_i', 'service_time', 'service_urh_i', 'service_urp_i', 'service_uucp', 'service_uucp_path', 'service_vmnet', 'service_whois']\n"
     ]
    }
   ],
   "source": [
    "# service\n",
    "unique_service=sorted(df.service.unique())\n",
    "string2 = 'service_'\n",
    "unique_service2=[string2 + x for x in unique_service]\n",
    "print(unique_service2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['flag_OTH', 'flag_REJ', 'flag_RSTO', 'flag_RSTOS0', 'flag_RSTR', 'flag_S0', 'flag_S1', 'flag_S2', 'flag_S3', 'flag_SF', 'flag_SH']\n"
     ]
    }
   ],
   "source": [
    "# flag\n",
    "unique_flag=sorted(df.flag.unique())\n",
    "string3 = 'flag_'\n",
    "unique_flag2=[string3 + x for x in unique_flag]\n",
    "print(unique_flag2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['service_IRC', 'service_X11', 'service_Z39_50', 'service_auth', 'service_bgp', 'service_courier', 'service_csnet_ns', 'service_ctf', 'service_daytime', 'service_discard', 'service_domain', 'service_domain_u', 'service_echo', 'service_eco_i', 'service_ecr_i', 'service_efs', 'service_exec', 'service_finger', 'service_ftp', 'service_ftp_data', 'service_gopher', 'service_hostnames', 'service_http', 'service_http_443', 'service_imap4', 'service_iso_tsap', 'service_klogin', 'service_kshell', 'service_ldap', 'service_link', 'service_login', 'service_mtp', 'service_name', 'service_netbios_dgm', 'service_netbios_ns', 'service_netbios_ssn', 'service_netstat', 'service_nnsp', 'service_nntp', 'service_ntp_u', 'service_other', 'service_pm_dump', 'service_pop_2', 'service_pop_3', 'service_printer', 'service_private', 'service_remote_job', 'service_rje', 'service_shell', 'service_smtp', 'service_sql_net', 'service_ssh', 'service_sunrpc', 'service_supdup', 'service_systat', 'service_telnet', 'service_tftp_u', 'service_tim_i', 'service_time', 'service_urp_i', 'service_uucp', 'service_uucp_path', 'service_vmnet', 'service_whois']\n"
     ]
    }
   ],
   "source": [
    "# put together\n",
    "dumcols=unique_protocol2 + unique_service2 + unique_flag2\n",
    "#do it for test set\n",
    "unique_service_test=sorted(df_test.service.unique())\n",
    "unique_service2_test=[string2 + x for x in unique_service_test]\n",
    "print(unique_service2_test)\n",
    "testdumcols=unique_protocol2 + unique_service2_test + unique_flag2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  protocol_type   service flag\n",
      "0           tcp  ftp_data   SF\n",
      "1           udp     other   SF\n",
      "2           tcp   private   S0\n",
      "3           tcp      http   SF\n",
      "4           tcp      http   SF\n",
      "--------------------\n",
      "   protocol_type  service  flag\n",
      "0              1       20     9\n",
      "1              2       44     9\n",
      "2              1       49     5\n",
      "3              1       24     9\n",
      "4              1       24     9\n"
     ]
    }
   ],
   "source": [
    "df_categorical_values_enc=df_categorical_values.apply(LabelEncoder().fit_transform)\n",
    "print(df_categorical_values.head())\n",
    "print('--------------------')\n",
    "print(df_categorical_values_enc.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test set\n",
    "testdf_categorical_values_enc=testdf_categorical_values.apply(LabelEncoder().fit_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol_type_icmp</th>\n",
       "      <th>Protocol_type_tcp</th>\n",
       "      <th>Protocol_type_udp</th>\n",
       "      <th>service_IRC</th>\n",
       "      <th>service_X11</th>\n",
       "      <th>service_Z39_50</th>\n",
       "      <th>service_aol</th>\n",
       "      <th>service_auth</th>\n",
       "      <th>service_bgp</th>\n",
       "      <th>service_courier</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_REJ</th>\n",
       "      <th>flag_RSTO</th>\n",
       "      <th>flag_RSTOS0</th>\n",
       "      <th>flag_RSTR</th>\n",
       "      <th>flag_S0</th>\n",
       "      <th>flag_S1</th>\n",
       "      <th>flag_S2</th>\n",
       "      <th>flag_S3</th>\n",
       "      <th>flag_SF</th>\n",
       "      <th>flag_SH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 84 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Protocol_type_icmp  Protocol_type_tcp  Protocol_type_udp  service_IRC  \\\n",
       "0                 0.0                1.0                0.0          0.0   \n",
       "1                 0.0                0.0                1.0          0.0   \n",
       "2                 0.0                1.0                0.0          0.0   \n",
       "3                 0.0                1.0                0.0          0.0   \n",
       "4                 0.0                1.0                0.0          0.0   \n",
       "\n",
       "   service_X11  service_Z39_50  service_aol  service_auth  service_bgp  \\\n",
       "0          0.0             0.0          0.0           0.0          0.0   \n",
       "1          0.0             0.0          0.0           0.0          0.0   \n",
       "2          0.0             0.0          0.0           0.0          0.0   \n",
       "3          0.0             0.0          0.0           0.0          0.0   \n",
       "4          0.0             0.0          0.0           0.0          0.0   \n",
       "\n",
       "   service_courier  ...  flag_REJ  flag_RSTO  flag_RSTOS0  flag_RSTR  flag_S0  \\\n",
       "0              0.0  ...       0.0        0.0          0.0        0.0      0.0   \n",
       "1              0.0  ...       0.0        0.0          0.0        0.0      0.0   \n",
       "2              0.0  ...       0.0        0.0          0.0        0.0      1.0   \n",
       "3              0.0  ...       0.0        0.0          0.0        0.0      0.0   \n",
       "4              0.0  ...       0.0        0.0          0.0        0.0      0.0   \n",
       "\n",
       "   flag_S1  flag_S2  flag_S3  flag_SF  flag_SH  \n",
       "0      0.0      0.0      0.0      1.0      0.0  \n",
       "1      0.0      0.0      0.0      1.0      0.0  \n",
       "2      0.0      0.0      0.0      0.0      0.0  \n",
       "3      0.0      0.0      0.0      1.0      0.0  \n",
       "4      0.0      0.0      0.0      1.0      0.0  \n",
       "\n",
       "[5 rows x 84 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# One-hot encoding\n",
    "enc = OneHotEncoder(categories='auto')\n",
    "df_categorical_values_encenc = enc.fit_transform(df_categorical_values_enc)\n",
    "df_cat_data = pd.DataFrame(df_categorical_values_encenc.toarray(),columns=dumcols)\n",
    "df_cat_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test set\n",
    "testdf_categorical_values_encenc = enc.fit_transform(testdf_categorical_values_enc)\n",
    "testdf_cat_data = pd.DataFrame(testdf_categorical_values_encenc.toarray(),columns=testdumcols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['service_http_2784',\n",
       " 'service_urh_i',\n",
       " 'service_harvest',\n",
       " 'service_http_8001',\n",
       " 'service_aol',\n",
       " 'service_red_i']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# service difference between train and test\n",
    "trainservice=df['service'].tolist()\n",
    "testservice= df_test['service'].tolist()\n",
    "difference=list(set(trainservice) - set(testservice))\n",
    "string = 'service_'\n",
    "difference=[string + x for x in difference]\n",
    "difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(125973, 84)\n",
      "(22544, 84)\n"
     ]
    }
   ],
   "source": [
    "for col in difference:\n",
    "    testdf_cat_data[col] = 0\n",
    "print(df_cat_data.shape)    \n",
    "print(testdf_cat_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(125973, 123)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_REJ</th>\n",
       "      <th>flag_RSTO</th>\n",
       "      <th>flag_RSTOS0</th>\n",
       "      <th>flag_RSTR</th>\n",
       "      <th>flag_S0</th>\n",
       "      <th>flag_S1</th>\n",
       "      <th>flag_S2</th>\n",
       "      <th>flag_S3</th>\n",
       "      <th>flag_SF</th>\n",
       "      <th>flag_SH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>491</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>146</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>232</td>\n",
       "      <td>8153</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>199</td>\n",
       "      <td>420</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 123 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration  src_bytes  dst_bytes  land  wrong_fragment  urgent  hot  \\\n",
       "0         0        491          0     0               0       0    0   \n",
       "1         0        146          0     0               0       0    0   \n",
       "2         0          0          0     0               0       0    0   \n",
       "3         0        232       8153     0               0       0    0   \n",
       "4         0        199        420     0               0       0    0   \n",
       "\n",
       "   num_failed_logins  logged_in  num_compromised  ...  flag_REJ  flag_RSTO  \\\n",
       "0                  0          0                0  ...       0.0        0.0   \n",
       "1                  0          0                0  ...       0.0        0.0   \n",
       "2                  0          0                0  ...       0.0        0.0   \n",
       "3                  0          1                0  ...       0.0        0.0   \n",
       "4                  0          1                0  ...       0.0        0.0   \n",
       "\n",
       "   flag_RSTOS0  flag_RSTR  flag_S0  flag_S1  flag_S2  flag_S3  flag_SF  \\\n",
       "0          0.0        0.0      0.0      0.0      0.0      0.0      1.0   \n",
       "1          0.0        0.0      0.0      0.0      0.0      0.0      1.0   \n",
       "2          0.0        0.0      1.0      0.0      0.0      0.0      0.0   \n",
       "3          0.0        0.0      0.0      0.0      0.0      0.0      1.0   \n",
       "4          0.0        0.0      0.0      0.0      0.0      0.0      1.0   \n",
       "\n",
       "   flag_SH  \n",
       "0      0.0  \n",
       "1      0.0  \n",
       "2      0.0  \n",
       "3      0.0  \n",
       "4      0.0  \n",
       "\n",
       "[5 rows x 123 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newdf=df.join(df_cat_data)\n",
    "newdf.drop('flag', axis=1, inplace=True)\n",
    "newdf.drop('protocol_type', axis=1, inplace=True)\n",
    "newdf.drop('service', axis=1, inplace=True)\n",
    "print(newdf.shape)\n",
    "newdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22544, 123)\n"
     ]
    }
   ],
   "source": [
    "# test data\n",
    "newdf_test=df_test.join(testdf_cat_data)\n",
    "newdf_test.drop('flag', axis=1, inplace=True)\n",
    "newdf_test.drop('protocol_type', axis=1, inplace=True)\n",
    "newdf_test.drop('service', axis=1, inplace=True)\n",
    "print(newdf_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "labeldf=newdf['label']\n",
    "labeldf_test=newdf_test['label']\n",
    "# change the label column\n",
    "newlabeldf=labeldf.replace({ 'normal' : 0, 'neptune' : 1 ,'back': 1, 'land': 1, 'pod': 1, 'smurf': 1, 'teardrop': 1,'mailbomb': 1, 'apache2': 1, 'processtable': 1, 'udpstorm': 1, 'worm': 1,\n",
    "                           'ipsweep' : 1,'nmap' : 1,'portsweep' : 1,'satan' : 1,'mscan' : 1,'saint' : 1,\n",
    "                            'ftp_write': 1,'guess_passwd': 1,'imap': 1,'multihop': 1,'phf': 1,'spy': 1,'warezclient': 1,'warezmaster': 1,'sendmail': 1,'named': 1,'snmpgetattack': 1,'snmpguess': 1,'xlock': 1,'xsnoop': 1,'httptunnel': 1,\n",
    "                           'buffer_overflow': 1,'loadmodule': 1,'perl': 1,'rootkit': 1,'ps': 1,'sqlattack': 1,'xterm': 1 })\n",
    "newlabeldf_test=labeldf_test.replace({ 'normal' : 0, 'neptune' : 1 ,'back': 1, 'land': 1, 'pod': 1, 'smurf': 1, 'teardrop': 1,'mailbomb': 1, 'apache2': 1, 'processtable': 1, 'udpstorm': 1, 'worm': 1,\n",
    "                           'ipsweep' : 1,'nmap' : 1,'portsweep' : 1,'satan' : 1,'mscan' : 1,'saint' : 1\n",
    "                           ,'ftp_write': 1,'guess_passwd': 1,'imap': 1,'multihop': 1,'phf': 1,'spy': 1,'warezclient': 1,'warezmaster': 1,'sendmail': 1,'named': 1,'snmpgetattack': 1,'snmpguess': 1,'xlock': 1,'xsnoop': 1,'httptunnel': 1,\n",
    "                           'buffer_overflow': 1,'loadmodule': 1,'perl': 1,'rootkit': 1,'ps': 1,'sqlattack': 1,'xterm': 1})\n",
    "# put the new label column back\n",
    "newdf['label'] = newlabeldf\n",
    "newdf_test['label'] = newlabeldf_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = newdf.drop(['duration','land','wrong_fragment','urgent','num_failed_logins','logged_in','num_compromised','num_file_creations','num_root','root_shell','su_attempted','num_shells','num_access_files','num_outbound_cmds','is_host_login','is_guest_login','flag_S2','flag_S3','flag_SH','srv_rerror_rate','service_csnet_ns','service_ctf','service_daytime','service_discard','service_domain','service_domain_u','service_echo','service_eco_i','service_ecr_i','service_efs','service_exec','service_finger','service_ftp','service_ftp_data','service_gopher','service_netbios_ns','service_ldap','service_kshell','service_klogin','service_iso_tsap','service_imap4','service_http_443','service_hostnames','service_netbios_dgm','service_name','service_mtp','service_login','service_link','service_pop_3','service_pop_2','service_pm_dump','service_other','service_ntp_u','service_nntp','service_nnsp','service_netstat','service_netbios_ssn','service_ssh','service_sql_net','service_sunrpc','service_smtp','service_shell','service_rje','service_remote_job','service_private','service_printer','service_uucp_path','service_uucp','service_urp_i','service_time','service_tim_i','service_tftp_u','service_telnet','service_systat','service_supdup','dst_host_count','srv_diff_host_rate','diff_srv_rate','flag_S0','flag_S1','rerror_rate','flag_RSTR','flag_RSTOS0','flag_RSTO','flag_REJ','flag_OTH','service_whois','service_vmnet','srv_serror_rate','serror_rate','service_urh_i','service_red_i','service_harvest','service_http_2784','dst_host_srv_rerror_rate','dst_host_rerror_rate','dst_host_srv_serror_rate','dst_host_serror_rate','dst_host_srv_diff_host_rate','Protocol_type_tcp','Protocol_type_udp','service_IRC','service_X11','service_Z39_50','service_auth','service_bgp','service_courier','service_http_8001','service_aol'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test =newdf_test.drop(['duration','land','wrong_fragment','urgent','num_failed_logins','logged_in','num_compromised','num_file_creations','num_root','root_shell','su_attempted','num_shells','num_access_files','num_outbound_cmds','is_host_login','is_guest_login','flag_S2','flag_S3','flag_SH','srv_rerror_rate','service_csnet_ns','service_ctf','service_daytime','service_discard','service_domain','service_domain_u','service_echo','service_eco_i','service_ecr_i','service_efs','service_exec','service_finger','service_ftp','service_ftp_data','service_gopher','service_netbios_ns','service_ldap','service_kshell','service_klogin','service_iso_tsap','service_imap4','service_http_443','service_hostnames','service_netbios_dgm','service_name','service_mtp','service_login','service_link','service_pop_3','service_pop_2','service_pm_dump','service_other','service_ntp_u','service_nntp','service_nnsp','service_netstat','service_netbios_ssn','service_ssh','service_sql_net','service_sunrpc','service_smtp','service_shell','service_rje','service_remote_job','service_private','service_printer','service_uucp_path','service_uucp','service_urp_i','service_time','service_tim_i','service_tftp_u','service_telnet','service_systat','service_supdup','dst_host_count','srv_diff_host_rate','diff_srv_rate','flag_S0','flag_S1','rerror_rate','flag_RSTR','flag_RSTOS0','flag_RSTO','flag_REJ','flag_OTH','service_whois','service_vmnet','srv_serror_rate','serror_rate','service_urh_i','service_red_i','service_harvest','service_http_2784','dst_host_srv_rerror_rate','dst_host_rerror_rate','dst_host_srv_serror_rate','dst_host_serror_rate','dst_host_srv_diff_host_rate','Protocol_type_tcp','Protocol_type_udp','service_IRC','service_X11','service_Z39_50','service_auth','service_bgp','service_courier','service_http_8001','service_aol'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(125973, 13)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# input and label\n",
    "X_Df = x.drop('label',1)\n",
    "Y_Df = newdf.label\n",
    "X_Df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test set\n",
    "X_Df_test = x_test.drop('label',1)\n",
    "Y_Df_test = newdf_test.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "colNames=list(X_Df)\n",
    "colNames_test=list(X_Df_test)\n",
    "from sklearn import preprocessing\n",
    "scaler1 = preprocessing.StandardScaler().fit(X_Df)\n",
    "X_Df=scaler1.transform(X_Df) \n",
    "# test data\n",
    "scaler5 = preprocessing.StandardScaler().fit(X_Df_test)\n",
    "X_Df_test=scaler5.transform(X_Df_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from secml.data import CDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.0076786 , -0.00491864, -0.09507567, ..., -0.26542904,\n",
       "        -0.6863277 ,  0.82515007],\n",
       "       [-0.00773737, -0.00491864, -0.09507567, ..., -0.26542904,\n",
       "        -0.6863277 ,  0.82515007],\n",
       "       [-0.00776224, -0.00491864, -0.09507567, ..., -0.26542904,\n",
       "        -0.6863277 , -1.21190076],\n",
       "       ...,\n",
       "       [-0.00738219, -0.00482315, -0.09507567, ..., -0.26542904,\n",
       "        -0.6863277 ,  0.82515007],\n",
       "       [-0.00776224, -0.00491864, -0.09507567, ..., -0.26542904,\n",
       "        -0.6863277 , -1.21190076],\n",
       "       [-0.00773652, -0.00491864, -0.09507567, ..., -0.26542904,\n",
       "        -0.6863277 ,  0.82515007]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_Df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_Df.tolist()\n",
    "Y_train = Y_Df.tolist()\n",
    "X_test = X_Df_test.tolist()\n",
    "Y_test = Y_Df_test.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr = CDataset(X_train, Y_train)\n",
    "ts = CDataset(X_test, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CArray(125973, 13)(dense: [[3.558064e-07 0.000000e+00 0.000000e+00 ... 0.000000e+00 0.000000e+00  1.000000e+00] [1.057999e-07 0.000000e+00 0.000000e+00 ... 0.000000e+00 0.000000e+00  1.000000e+00] [0.000000e+00 0.000000e+00 0.000000e+00 ... 0.000000e+00 0.000000e+00  0.000000e+00] ... [1.616709e-06 2.931438e-07 0.000000e+00 ... 0.000000e+00 0.000000e+00  1.000000e+00] [0.000000e+00 0.000000e+00 0.000000e+00 ... 0.000000e+00 0.000000e+00  0.000000e+00] [1.094232e-07 0.000000e+00 0.000000e+00 ... 0.000000e+00 0.000000e+00  1.000000e+00]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Normalize the data\n",
    "from secml.ml.features import CNormalizerMinMax\n",
    "nmz = CNormalizerMinMax()\n",
    "tr.X = nmz.fit_transform(tr.X)\n",
    "ts.X = nmz.transform(ts.X)\n",
    "\n",
    "# Metric to use for training and performance evaluation\n",
    "from secml.ml.peval.metrics import CMetricAccuracy\n",
    "metric = CMetricAccuracy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimating the best training parameters...\n"
     ]
    }
   ],
   "source": [
    "random_state = 999\n",
    "\n",
    "n_features = 2  # Number of features\n",
    "n_samples = 1100  # Number of samples\n",
    "centers = [[-2, 0], [2, -2], [2, 2]]  # Centers of the clusters\n",
    "cluster_std = 0.8  # Standard deviation of the clusters\n",
    "\n",
    "# Creation of the multiclass classifier\n",
    "from secml.ml.classifiers import CClassifierSVM\n",
    "from secml.ml.classifiers.multiclass import CClassifierMulticlassOVA\n",
    "from secml.ml.kernels import CKernelRBF\n",
    "clf = CClassifierMulticlassOVA(CClassifierSVM, kernel=CKernelRBF())\n",
    "\n",
    "# Parameters for the Cross-Validation procedure\n",
    "xval_params = {'C': [1e-2, 0.1, 1], 'kernel.gamma': [10, 100, 1e3]}\n",
    "\n",
    "# Let's create a 3-Fold data splitter\n",
    "from secml.data.splitter import CDataSplitterKFold\n",
    "xval_splitter = CDataSplitterKFold(num_folds=3, random_state=random_state)\n",
    "\n",
    "# Select and set the best training parameters for the classifier\n",
    "print(\"Estimating the best training parameters...\")\n",
    "best_params = clf.estimate_parameters(\n",
    "    dataset=tr,\n",
    "    parameters=xval_params,\n",
    "    splitter=xval_splitter,\n",
    "    metric='accuracy',\n",
    "    perf_evaluator='xval'\n",
    ")\n",
    "\n",
    "print(\"The best training parameters are: \",\n",
    "      [(k, best_params[k]) for k in sorted(best_params)])\n",
    "\n",
    "# We can now fit the classifier\n",
    "clf.fit(tr.X, tr.Y)\n",
    "\n",
    "# Compute predictions on a test set\n",
    "y_pred = clf.predict(ts.X)\n",
    "\n",
    "# Evaluate the accuracy of the classifier\n",
    "acc = metric.performance_score(y_true=ts.Y, y_pred=y_pred)\n",
    "\n",
    "print(\"Accuracy on test set: {:.2%}\".format(acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Linear kernel\n",
    "from sklearn.svm import SVC\n",
    "clf_SVM_Df=SVC(kernel='linear', C=1.0, random_state=0)\n",
    "clf_SVM_Df.fit(X_Df, Y_Df.astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_Df_pred=clf_SVM_Df.predict(X_Df_test)\n",
    "# Create confusion matrix\n",
    "pd.crosstab(Y_Df_test, Y_Df_pred, rownames=['Actual attacks'], colnames=['Predicted attacks'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn import metrics\n",
    "accuracy = cross_val_score(clf_SVM_Df, X_Df_test, Y_Df_test, cv=10, scoring='accuracy')\n",
    "print(\"Accuracy: %0.5f (+/- %0.5f)\" % (accuracy.mean(), accuracy.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "clf_SVM_Df1=SVC(kernel='rbf', C=1.0, random_state=0)\n",
    "clf_SVM_Df1.fit(X_Df, Y_Df.astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_Df_pred=clf_SVM_Df1.predict(X_Df_test)\n",
    "# Create confusion matrix\n",
    "pd.crosstab(Y_Df_test, Y_Df_pred, rownames=['Actual attacks'], colnames=['Predicted attacks'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn import metrics\n",
    "accuracy = cross_val_score(clf_SVM_Df1, X_Df_test, Y_Df_test, cv=10, scoring='accuracy')\n",
    "print(\"Accuracy: %0.5f (+/- %0.5f)\" % (accuracy.mean(), accuracy.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}