VAE_nb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finding Anomalies in Bitcoin Price Data\n",
    "\n",
    "### VAE EXPLANATION:\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Imports\n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Date    High     Low    Last     Bid     Ask        Volume    VWAP\n",
      "0 2014-04-15  515.00  453.16  499.01  500.01  505.04  28535.844106  491.41\n",
      "1 2014-04-16  548.00  494.02  534.00  535.01  536.00  31159.941300  520.21\n",
      "2 2014-04-17  537.24  481.63  506.52  504.70  505.38  21126.375080  504.83\n",
      "3 2014-04-18  508.43  470.00  487.00  484.14  487.00  11879.484756  485.72\n",
      "4 2014-04-19  507.43  472.81  504.74  504.74  505.00  10262.195861  492.22\n"
     ]
    }
   ],
   "source": [
    "#Importing btc stock data\n",
    "data = pd.read_csv('bit_data.csv', parse_dates=['Date']) \n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Generating Additional features\n",
    "data['Day']=data['Date'].dt.day\n",
    "data['Day_of_week']=data['Date'].dt.weekday\n",
    "data['Raw_Volume']=data['Volume']\n",
    "data['Raw_Last']=data['Last']\n",
    "data['Raw_VWAP']=data['VWAP']\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Adding moving averages\n",
    "close_avg_3 = data['Last'].rolling(3).mean()\n",
    "close_avg_5 =  data['Last'].rolling(5).mean()\n",
    "close_avg_10 = data['Last'].rolling(10).mean()\n",
    "\n",
    "data['3rd Closing Avg']=close_avg_3\n",
    "data['5th Closing Avg']=close_avg_5\n",
    "data['10th Closing Avg']=close_avg_10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Removing NAN valued rows\n",
    "data = data.dropna(how='any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features to be trained on\n",
    "features = [f for f in list(data) if f not in ['Date','Raw_Volume','Raw_Last','Raw_VWAP']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['High', 'Low', 'Last', 'Bid', 'Ask', 'Volume', 'VWAP', 'Day', 'Day_of_week', '3rd Closing Avg', '5th Closing Avg', '10th Closing Avg']\n"
     ]
    }
   ],
   "source": [
    "df = data.copy()\n",
    "\n",
    "# Normalize column data between 0 and 1\n",
    "df[features] = df[features].apply(lambda x: (x - x.min()) / (x.max() - x.min()))\n",
    "\n",
    "print(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         Date      High       Low      Last       Bid       Ask    Volume  \\\n",
      "9  2014-04-24  0.014710  0.017889  0.017248  0.017172  0.017156  0.050171   \n",
      "10 2014-04-25  0.015019  0.015599  0.015421  0.015399  0.015327  0.257063   \n",
      "11 2014-04-26  0.013269  0.016310  0.015183  0.015215  0.015143  0.082784   \n",
      "12 2014-04-27  0.012807  0.015644  0.014547  0.014578  0.014457  0.054306   \n",
      "13 2014-04-28  0.012302  0.014831  0.014554  0.014585  0.014486  0.147186   \n",
      "\n",
      "        VWAP       Day  Day_of_week    Raw_Volume  Raw_Last  Raw_VWAP  \\\n",
      "9   0.015677  0.766667     0.500000   6875.473428    498.32    488.46   \n",
      "10  0.014382  0.800000     0.666667  32311.719807    463.54    463.95   \n",
      "11  0.014200  0.833333     0.833333  10885.104866    459.00    460.51   \n",
      "12  0.013577  0.866667     1.000000   7383.826598    446.90    448.73   \n",
      "13  0.013001  0.900000     0.000000  18802.958300    447.03    437.82   \n",
      "\n",
      "    3rd Closing Avg  5th Closing Avg  10th Closing Avg  \n",
      "9          0.015814         0.016185          0.016831  \n",
      "10         0.015355         0.015827          0.016625  \n",
      "11         0.014783         0.015410          0.016190  \n",
      "12         0.013872         0.014942          0.015843  \n",
      "13         0.013580         0.014457          0.015611  \n"
     ]
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train-Test split\n",
    "from sklearn.model_selection import train_test_split\n",
    "features = [f for f in list(df)]\n",
    "X_train, X_test = train_test_split(df[features], test_size=0.20, random_state=26)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X shape:  (1294, 12) (324, 12)\n"
     ]
    }
   ],
   "source": [
    "features = [f for f in list(data) if f not in ['Date','Raw_Volume','Raw_Last','Raw_VWAP']]\n",
    "X_test = X_test[features]\n",
    "X_train = X_train[features]\n",
    "\n",
    "print(\"X train shape: \", X_train.shape)\n",
    "print(\"X test shape: \", X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25\n",
      "(1250, 12)\n",
      "(300, 12)\n"
     ]
    }
   ],
   "source": [
    "# Batch processing\n",
    "m = 50\n",
    "\n",
    "num_batches_train = X_train.shape[0]//m\n",
    "print(num_batches_train)\n",
    "X_train_trun = X_train.head(num_batches_train*m)\n",
    "print(X_train_trun.shape)\n",
    "\n",
    "num_batches_test = X_test.shape[0]//m\n",
    "X_test_trun = X_test.head(num_batches_test*m)\n",
    "print(X_test_trun.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.autograd import Variable\n",
    "\n",
    "# Variables\n",
    "batch_size = 50\n",
    "representation_size = 2\n",
    "encoder_dim = 6\n",
    "decoder_dim = 6\n",
    "decoder_out_dim = X_train_trun.shape[1] \n",
    "n_x = X_train_trun.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VAE(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(VAE, self).__init__()\n",
    "        self.en1 = nn.Linear(n_x, encoder_dim)\n",
    "        self.en_mu = nn.Linear(encoder_dim, representation_size)\n",
    "        self.en_std = nn.Linear(encoder_dim, representation_size)\n",
    "        self.de1 = nn.Linear(representation_size, decoder_dim)\n",
    "        self.de2 = nn.Linear(decoder_dim, decoder_out_dim)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "        \n",
    "    def encode(self, x):\n",
    "        #Encode a batch of samples, and return posterior parameters for each point\n",
    "        h1 = self.relu(self.en1(x))\n",
    "        return self.en_mu(h1), self.en_std(h1)\n",
    "    \n",
    "    def decode(self, z):\n",
    "        #Decode a batch of latent variables\n",
    "        \n",
    "        h2 = self.relu(self.de1(z))\n",
    "        return self.sigmoid(self.de2(h2))\n",
    "    \n",
    "    def reparam(self, mu, logvar):\n",
    "        #Reparameterisation trick to sample z values. \n",
    "        #This is stochastic during training, and returns the mode during evaluation.\n",
    "        \n",
    "        if self.training:\n",
    "            std = logvar.mul(0.5).exp_()\n",
    "            eps = Variable(std.data.new(std.size()).normal_())\n",
    "            return eps.mul(std).add_(mu)\n",
    "        else:\n",
    "            return mu\n",
    "            \n",
    "    \n",
    "    def forward(self, x):\n",
    "        #Takes a batch of samples, encodes them, and then decodes them again to compare.\n",
    "        mu, logvar = self.encode(x.contiguous().view(-1, n_x))\n",
    "        z = self.reparam(mu, logvar)\n",
    "        return self.decode(z), mu, logvar\n",
    "    \n",
    "    def loss(self, reconstruction, x, mu, logvar):\n",
    "        #ELBO assuming entries of x are binary variables, with closed form KLD\n",
    "        \n",
    "        bce = torch.nn.functional.binary_cross_entropy(reconstruction, x.contiguous().view(-1, n_x))\n",
    "        KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())\n",
    "        # Normalise by same number of elements as in reconstruction\n",
    "        KLD /= x.contiguous().view(-1, n_x).data.shape[0] * n_x\n",
    "\n",
    "        return bce + KLD\n",
    "    \n",
    "    def rget_z(self, x):\n",
    "        #Encode a batch of data points, x, into their z representations.\n",
    "        \n",
    "        mu, logvar = self.encode(x.contiguous().view(-1,n_x))\n",
    "        return self.reparam(mu, logvar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VAE(\n",
      "  (en1): Linear(in_features=12, out_features=6, bias=True)\n",
      "  (en_mu): Linear(in_features=6, out_features=2, bias=True)\n",
      "  (en_std): Linear(in_features=6, out_features=2, bias=True)\n",
      "  (de1): Linear(in_features=2, out_features=6, bias=True)\n",
      "  (de2): Linear(in_features=6, out_features=12, bias=True)\n",
      "  (relu): ReLU()\n",
      "  (sigmoid): Sigmoid()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model =VAE()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Training model\n",
    "def train(epoch, batches_per_epoch = 501, log_interval=500):\n",
    "    model.train()\n",
    "    ind = np.arange(X_train_trun.shape[0]) # Get array of nos.\n",
    "    \n",
    "    for i in range(batches_per_epoch):\n",
    "        d = X_train_trun.iloc[[x for x in ind],:]\n",
    "        data = torch.Tensor(d.values)\n",
    "        data = Variable(data, requires_grad=False)\n",
    "        \n",
    "        optimizer.zero_grad() # Make gradients zero before starting to accumulate them\n",
    "        \n",
    "        recon_batch, mu, logvar = model(data)\n",
    "        \n",
    "        loss = model.loss(recon_batch, data, mu, logvar)\n",
    "        loss.backward() # Calc grads\n",
    "        optimizer.step() # Weight update\n",
    "        if (i % log_interval == 0) and (epoch % 5 ==0):\n",
    "            #Print progress\n",
    "            print('Train Epoch: {} [{}/{}]\\tLoss: {:.6f}'.format(\n",
    "                epoch, i * batch_size, batch_size*batches_per_epoch,\n",
    "                loss.data[0] / len(data)))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 5 [0/25050]\tLoss: 0.000313\n",
      "Train Epoch: 5 [25000/25050]\tLoss: 0.000309\n",
      "Train Epoch: 10 [0/25050]\tLoss: 0.000310\n",
      "Train Epoch: 10 [25000/25050]\tLoss: 0.000313\n",
      "Train Epoch: 15 [0/25050]\tLoss: 0.000311\n",
      "Train Epoch: 15 [25000/25050]\tLoss: 0.000310\n",
      "Train Epoch: 20 [0/25050]\tLoss: 0.000309\n",
      "Train Epoch: 20 [25000/25050]\tLoss: 0.000309\n",
      "Train Epoch: 25 [0/25050]\tLoss: 0.000310\n",
      "Train Epoch: 25 [25000/25050]\tLoss: 0.000310\n",
      "Train Epoch: 30 [0/25050]\tLoss: 0.000312\n",
      "Train Epoch: 30 [25000/25050]\tLoss: 0.000310\n",
      "Train Epoch: 35 [0/25050]\tLoss: 0.000310\n",
      "Train Epoch: 35 [25000/25050]\tLoss: 0.000310\n",
      "Train Epoch: 40 [0/25050]\tLoss: 0.000310\n",
      "Train Epoch: 40 [25000/25050]\tLoss: 0.000313\n",
      "Train Epoch: 45 [0/25050]\tLoss: 0.000310\n",
      "Train Epoch: 45 [25000/25050]\tLoss: 0.000311\n",
      "Train Epoch: 50 [0/25050]\tLoss: 0.000309\n",
      "Train Epoch: 50 [25000/25050]\tLoss: 0.000312\n",
      "Train Epoch: 55 [0/25050]\tLoss: 0.000310\n",
      "Train Epoch: 55 [25000/25050]\tLoss: 0.000308\n",
      "Train Epoch: 60 [0/25050]\tLoss: 0.000311\n",
      "Train Epoch: 60 [25000/25050]\tLoss: 0.000309\n",
      "Train Epoch: 65 [0/25050]\tLoss: 0.000311\n",
      "Train Epoch: 65 [25000/25050]\tLoss: 0.000310\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, 70):\n",
    "    train(epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def construct_numvec(z = None):\n",
    "    out = np.zeros((1, representation_size))\n",
    "    out[:representation_size] = 1.\n",
    "    if z is None:\n",
    "        return(out)\n",
    "    else:\n",
    "        for i in range(len(z)):\n",
    "            out[:,i] = z[i]\n",
    "        return(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting for random selection \n",
    "_, X_rem = train_test_split(X_train_trun,test_size=0.20, random_state=19)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = torch.Tensor(X_rem.values)\n",
    "data = Variable(d, requires_grad=False)\n",
    "model.eval()\n",
    "z_dataset = model.rget_z(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Swapnilbp\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "C:\\Users\\Swapnilbp\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "z=[]\n",
    "# Get the z repr\n",
    "z=z_dataset[:,0].data.numpy()\n",
    "z1 = z_dataset[:,1].data.numpy\n",
    "\n",
    "X_rem['z1'] = z\n",
    "X_rem['z2'] = z1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calc recon err\n",
    "def recon_error_cal(actual,predictions):\n",
    "    recon_error = np.mean(np.power(actual - predictions, 2))\n",
    "    recon_array = np.power(actual - predictions, 2)\n",
    "    max_col = np.argmax(recon_array)\n",
    "    return recon_error, max_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Swapnilbp\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\numpy\\core\\fromnumeric.py:51: FutureWarning: 'argmax' is deprecated, use 'idxmax' instead. The behavior of 'argmax'\n",
      "will be corrected to return the positional maximum in the future.\n",
      "Use 'series.values.argmax' to get the position of the maximum now.\n",
      "  return getattr(obj, method)(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "recon_on_train = [] \n",
    "max_contributor =[]\n",
    "for i in range(X_rem.shape[0]): # For each row in tr\n",
    "    #z1 = X_rem.iloc[i]['z1']\n",
    "    z1 = z_dataset[:,0][i]\n",
    "    #z2 = X_rem.iloc[i]['z2']\n",
    "    z2 = z_dataset[:,0][i]\n",
    "    z_ = torch.cat([z1, z2])   # Define z vector\n",
    "    \n",
    "    test_pred = model.decode(z_)\n",
    "    test_pred = test_pred.data.numpy()\n",
    "    transpose = test_pred.T\n",
    "    transpose = np.squeeze(transpose) # Decoder predictions\n",
    "    actuals = X_rem.iloc[i][features]\n",
    "\n",
    "    recon_error, max_col = recon_error_cal(actuals, transpose)\n",
    "    recon_on_train.append(recon_error)\n",
    "    max_contributor.append(max_col[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.02269214, 0.02540397, 0.02457797, 0.02462611, 0.0246597 ,\n",
       "       0.08123855, 0.02361177, 0.49336454, 0.48685247, 0.02334212,\n",
       "       0.02396036, 0.0243868 ], dtype=float32)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90th percentile recon error 0.03356949664021628\n",
      "95th percentile recon error 0.038235091257865526\n",
      "Selected anomaly threshold:  0.038235091257865526\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "recon_train_se = pd.Series(recon_on_train)\n",
    "print(\"95th percentile recon error\", np.percentile(recon_train_se, 95))\n",
    "anomaly_threshold =  np.percentile(recon_train_se, 95)\n",
    "print('Selected anomaly threshold: ', anomaly_threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "z_test = model.rget_z(Variable(torch.Tensor(X_test.values))) \n",
    "z_test = z_test.data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Swapnilbp\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\numpy\\core\\fromnumeric.py:51: FutureWarning: 'argmax' is deprecated, use 'idxmax' instead. The behavior of 'argmax'\n",
      "will be corrected to return the positional maximum in the future.\n",
      "Use 'series.values.argmax' to get the position of the maximum now.\n",
      "  return getattr(obj, method)(*args, **kwds)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(324,)"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recon_on_test = []\n",
    "max_contributor_test =[]\n",
    "\n",
    "for i in range(z_test.shape[0]): # For each row\n",
    "    z_ = Variable(torch.Tensor(z_test[i]))  # Define z vector\n",
    "    test_pred = model.decode(z_)\n",
    "    test_pred=test_pred.data.numpy()\n",
    "    transpose = test_pred.T\n",
    "    transpose = np.squeeze(transpose) #Decode predictions\n",
    "    actuals = X_test.iloc[i]\n",
    "\n",
    "    recon_error , max_col = recon_error_cal(actuals, transpose) # Error calculations \n",
    "    # Add this error to recon list\n",
    "    recon_on_test.append(recon_error)\n",
    "    max_contributor_test.append(max_col)\n",
    "    \n",
    "\n",
    "recon_test_se = pd.Series(recon_on_test)\n",
    "recon_test_se.shape\n",
    "\n",
    "max_col_test = pd.Series(max_contributor_test)\n",
    "max_col_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.05864197530864197\n"
     ]
    }
   ],
   "source": [
    "anomaly_count = 0\n",
    "for i in range(len(recon_test_se)):\n",
    "    if recon_test_se[i] > anomaly_threshold:\n",
    "        anomaly_count +=1\n",
    "print(anomaly_count/X_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test['recon_error'] = recon_on_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Last</th>\n",
       "      <th>Bid</th>\n",
       "      <th>Ask</th>\n",
       "      <th>Volume</th>\n",
       "      <th>VWAP</th>\n",
       "      <th>Day</th>\n",
       "      <th>Day_of_week</th>\n",
       "      <th>3rd Closing Avg</th>\n",
       "      <th>5th Closing Avg</th>\n",
       "      <th>10th Closing Avg</th>\n",
       "      <th>recon_error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1324</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.984502</td>\n",
       "      <td>0.986347</td>\n",
       "      <td>0.984551</td>\n",
       "      <td>0.073541</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.566667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.973398</td>\n",
       "      <td>0.956703</td>\n",
       "      <td>0.954588</td>\n",
       "      <td>0.071988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1330</th>\n",
       "      <td>0.799016</td>\n",
       "      <td>0.728683</td>\n",
       "      <td>0.759133</td>\n",
       "      <td>0.760018</td>\n",
       "      <td>0.759149</td>\n",
       "      <td>0.171495</td>\n",
       "      <td>0.771382</td>\n",
       "      <td>0.766667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.772154</td>\n",
       "      <td>0.847341</td>\n",
       "      <td>0.969844</td>\n",
       "      <td>0.057393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>0.006897</td>\n",
       "      <td>0.007836</td>\n",
       "      <td>0.008559</td>\n",
       "      <td>0.008541</td>\n",
       "      <td>0.008466</td>\n",
       "      <td>0.609295</td>\n",
       "      <td>0.006538</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.006731</td>\n",
       "      <td>0.007691</td>\n",
       "      <td>0.009030</td>\n",
       "      <td>0.052084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1366</th>\n",
       "      <td>0.605367</td>\n",
       "      <td>0.612044</td>\n",
       "      <td>0.604973</td>\n",
       "      <td>0.606066</td>\n",
       "      <td>0.604967</td>\n",
       "      <td>0.075228</td>\n",
       "      <td>0.608543</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.595620</td>\n",
       "      <td>0.611736</td>\n",
       "      <td>0.651572</td>\n",
       "      <td>0.049912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1365</th>\n",
       "      <td>0.586929</td>\n",
       "      <td>0.582301</td>\n",
       "      <td>0.592415</td>\n",
       "      <td>0.593307</td>\n",
       "      <td>0.592405</td>\n",
       "      <td>0.078105</td>\n",
       "      <td>0.586524</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.585961</td>\n",
       "      <td>0.602590</td>\n",
       "      <td>0.649281</td>\n",
       "      <td>0.048451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401</th>\n",
       "      <td>0.580788</td>\n",
       "      <td>0.595356</td>\n",
       "      <td>0.594122</td>\n",
       "      <td>0.595055</td>\n",
       "      <td>0.594382</td>\n",
       "      <td>0.049800</td>\n",
       "      <td>0.586099</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.590761</td>\n",
       "      <td>0.594335</td>\n",
       "      <td>0.600371</td>\n",
       "      <td>0.043106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1545</th>\n",
       "      <td>0.415724</td>\n",
       "      <td>0.434528</td>\n",
       "      <td>0.420875</td>\n",
       "      <td>0.421882</td>\n",
       "      <td>0.421141</td>\n",
       "      <td>0.024857</td>\n",
       "      <td>0.423584</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.425306</td>\n",
       "      <td>0.436698</td>\n",
       "      <td>0.446607</td>\n",
       "      <td>0.043097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1247</th>\n",
       "      <td>0.213393</td>\n",
       "      <td>0.221902</td>\n",
       "      <td>0.219893</td>\n",
       "      <td>0.219870</td>\n",
       "      <td>0.219797</td>\n",
       "      <td>0.050632</td>\n",
       "      <td>0.215954</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.217049</td>\n",
       "      <td>0.221802</td>\n",
       "      <td>0.220322</td>\n",
       "      <td>0.042878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>975</th>\n",
       "      <td>0.038736</td>\n",
       "      <td>0.043233</td>\n",
       "      <td>0.041834</td>\n",
       "      <td>0.041927</td>\n",
       "      <td>0.041749</td>\n",
       "      <td>0.028107</td>\n",
       "      <td>0.040377</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.040898</td>\n",
       "      <td>0.041752</td>\n",
       "      <td>0.041523</td>\n",
       "      <td>0.042570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1372</th>\n",
       "      <td>0.476959</td>\n",
       "      <td>0.437858</td>\n",
       "      <td>0.475477</td>\n",
       "      <td>0.475455</td>\n",
       "      <td>0.475440</td>\n",
       "      <td>0.126082</td>\n",
       "      <td>0.461796</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.468982</td>\n",
       "      <td>0.506775</td>\n",
       "      <td>0.589958</td>\n",
       "      <td>0.041825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1607</th>\n",
       "      <td>0.330296</td>\n",
       "      <td>0.345400</td>\n",
       "      <td>0.335570</td>\n",
       "      <td>0.336320</td>\n",
       "      <td>0.335626</td>\n",
       "      <td>0.033589</td>\n",
       "      <td>0.336622</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.339136</td>\n",
       "      <td>0.351350</td>\n",
       "      <td>0.370293</td>\n",
       "      <td>0.041173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>0.008695</td>\n",
       "      <td>0.012064</td>\n",
       "      <td>0.010681</td>\n",
       "      <td>0.010704</td>\n",
       "      <td>0.010676</td>\n",
       "      <td>0.019501</td>\n",
       "      <td>0.009735</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.009539</td>\n",
       "      <td>0.009443</td>\n",
       "      <td>0.009223</td>\n",
       "      <td>0.041073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>0.003957</td>\n",
       "      <td>0.005603</td>\n",
       "      <td>0.004729</td>\n",
       "      <td>0.004722</td>\n",
       "      <td>0.004635</td>\n",
       "      <td>0.345580</td>\n",
       "      <td>0.004275</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.004924</td>\n",
       "      <td>0.005511</td>\n",
       "      <td>0.005717</td>\n",
       "      <td>0.040925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1490</th>\n",
       "      <td>0.389543</td>\n",
       "      <td>0.405657</td>\n",
       "      <td>0.396110</td>\n",
       "      <td>0.396756</td>\n",
       "      <td>0.396073</td>\n",
       "      <td>0.032199</td>\n",
       "      <td>0.396409</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.394205</td>\n",
       "      <td>0.402028</td>\n",
       "      <td>0.418996</td>\n",
       "      <td>0.040183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1318</th>\n",
       "      <td>0.876832</td>\n",
       "      <td>0.789745</td>\n",
       "      <td>0.854822</td>\n",
       "      <td>0.856707</td>\n",
       "      <td>0.855113</td>\n",
       "      <td>0.129095</td>\n",
       "      <td>0.851077</td>\n",
       "      <td>0.366667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.799938</td>\n",
       "      <td>0.847216</td>\n",
       "      <td>0.784620</td>\n",
       "      <td>0.040058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>0.005971</td>\n",
       "      <td>0.009056</td>\n",
       "      <td>0.008168</td>\n",
       "      <td>0.008139</td>\n",
       "      <td>0.008076</td>\n",
       "      <td>0.058733</td>\n",
       "      <td>0.006960</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.007143</td>\n",
       "      <td>0.007417</td>\n",
       "      <td>0.007785</td>\n",
       "      <td>0.039817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1341</th>\n",
       "      <td>0.785845</td>\n",
       "      <td>0.786010</td>\n",
       "      <td>0.787884</td>\n",
       "      <td>0.789621</td>\n",
       "      <td>0.787995</td>\n",
       "      <td>0.094912</td>\n",
       "      <td>0.780494</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.756485</td>\n",
       "      <td>0.755351</td>\n",
       "      <td>0.822040</td>\n",
       "      <td>0.039451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>579</th>\n",
       "      <td>0.008747</td>\n",
       "      <td>0.011725</td>\n",
       "      <td>0.010957</td>\n",
       "      <td>0.010968</td>\n",
       "      <td>0.010864</td>\n",
       "      <td>0.141317</td>\n",
       "      <td>0.009734</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009208</td>\n",
       "      <td>0.008991</td>\n",
       "      <td>0.007704</td>\n",
       "      <td>0.038873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128</th>\n",
       "      <td>0.121171</td>\n",
       "      <td>0.126012</td>\n",
       "      <td>0.123848</td>\n",
       "      <td>0.124043</td>\n",
       "      <td>0.123771</td>\n",
       "      <td>0.068085</td>\n",
       "      <td>0.123213</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.123569</td>\n",
       "      <td>0.123462</td>\n",
       "      <td>0.122424</td>\n",
       "      <td>0.038368</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          High       Low      Last       Bid       Ask    Volume      VWAP  \\\n",
       "1324  1.000000  1.000000  0.984502  0.986347  0.984551  0.073541  1.000000   \n",
       "1330  0.799016  0.728683  0.759133  0.760018  0.759149  0.171495  0.771382   \n",
       "168   0.006897  0.007836  0.008559  0.008541  0.008466  0.609295  0.006538   \n",
       "1366  0.605367  0.612044  0.604973  0.606066  0.604967  0.075228  0.608543   \n",
       "1365  0.586929  0.582301  0.592415  0.593307  0.592405  0.078105  0.586524   \n",
       "1401  0.580788  0.595356  0.594122  0.595055  0.594382  0.049800  0.586099   \n",
       "1545  0.415724  0.434528  0.420875  0.421882  0.421141  0.024857  0.423584   \n",
       "1247  0.213393  0.221902  0.219893  0.219870  0.219797  0.050632  0.215954   \n",
       "975   0.038736  0.043233  0.041834  0.041927  0.041749  0.028107  0.040377   \n",
       "1372  0.476959  0.437858  0.475477  0.475455  0.475440  0.126082  0.461796   \n",
       "1607  0.330296  0.345400  0.335570  0.336320  0.335626  0.033589  0.336622   \n",
       "221   0.008695  0.012064  0.010681  0.010704  0.010676  0.019501  0.009735   \n",
       "256   0.003957  0.005603  0.004729  0.004722  0.004635  0.345580  0.004275   \n",
       "1490  0.389543  0.405657  0.396110  0.396756  0.396073  0.032199  0.396409   \n",
       "1318  0.876832  0.789745  0.854822  0.856707  0.855113  0.129095  0.851077   \n",
       "194   0.005971  0.009056  0.008168  0.008139  0.008076  0.058733  0.006960   \n",
       "1341  0.785845  0.786010  0.787884  0.789621  0.787995  0.094912  0.780494   \n",
       "579   0.008747  0.011725  0.010957  0.010968  0.010864  0.141317  0.009734   \n",
       "1128  0.121171  0.126012  0.123848  0.124043  0.123771  0.068085  0.123213   \n",
       "\n",
       "           Day  Day_of_week  3rd Closing Avg  5th Closing Avg  \\\n",
       "1324  0.566667     0.000000         0.973398         0.956703   \n",
       "1330  0.766667     1.000000         0.772154         0.847341   \n",
       "168   0.166667     0.000000         0.006731         0.007691   \n",
       "1366  0.933333     0.000000         0.595620         0.611736   \n",
       "1365  0.900000     1.000000         0.585961         0.602590   \n",
       "1401  0.133333     0.000000         0.590761         0.594335   \n",
       "1545  0.933333     1.000000         0.425306         0.436698   \n",
       "1247  0.000000     1.000000         0.217049         0.221802   \n",
       "975   0.000000     1.000000         0.040898         0.041752   \n",
       "1372  0.100000     1.000000         0.468982         0.506775   \n",
       "1607  0.000000     0.000000         0.339136         0.351350   \n",
       "221   0.966667     1.000000         0.009539         0.009443   \n",
       "256   0.100000     1.000000         0.004924         0.005511   \n",
       "1490  0.066667     1.000000         0.394205         0.402028   \n",
       "1318  0.366667     0.166667         0.799938         0.847216   \n",
       "194   0.033333     1.000000         0.007143         0.007417   \n",
       "1341  0.100000     0.500000         0.756485         0.755351   \n",
       "579   0.966667     0.000000         0.009208         0.008991   \n",
       "1128  0.100000     1.000000         0.123569         0.123462   \n",
       "\n",
       "      10th Closing Avg  recon_error  \n",
       "1324          0.954588     0.071988  \n",
       "1330          0.969844     0.057393  \n",
       "168           0.009030     0.052084  \n",
       "1366          0.651572     0.049912  \n",
       "1365          0.649281     0.048451  \n",
       "1401          0.600371     0.043106  \n",
       "1545          0.446607     0.043097  \n",
       "1247          0.220322     0.042878  \n",
       "975           0.041523     0.042570  \n",
       "1372          0.589958     0.041825  \n",
       "1607          0.370293     0.041173  \n",
       "221           0.009223     0.041073  \n",
       "256           0.005717     0.040925  \n",
       "1490          0.418996     0.040183  \n",
       "1318          0.784620     0.040058  \n",
       "194           0.007785     0.039817  \n",
       "1341          0.822040     0.039451  \n",
       "579           0.007704     0.038873  \n",
       "1128          0.122424     0.038368  "
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errs = X_test[X_test['recon_error'] > anomaly_threshold]\n",
    "errs.sort_values('recon_error', ascending=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TensorFlow-GPU",
   "language": "python",
   "name": "tf-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}