!Z-Score.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Required packages for processing\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pytz\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_dataframe(df):\n",
    "    df = df.replace(-1, np.NaN)\n",
    "    df[\"dt\"] = pd.to_datetime(df.index, utc=True, unit=\"s\")\n",
    "    df[\"dt\"] = df[\"dt\"].dt.tz_convert(pytz.timezone(\"Europe/Amsterdam\")).dt.tz_localize(None)\n",
    "    df = df.set_index(\"dt\")\n",
    "    df[\"Total\"]= df.sum(axis=1)\n",
    "    return(df)\n",
    "\n",
    "def resample_dataframe(df, window):\n",
    "    df = df.resample(window).mean()\n",
    "    return(df)\n",
    "    \n",
    "def zscore(x, window):\n",
    "    r = x.rolling(window=window)\n",
    "    m = r.mean().shift(1)\n",
    "    s = r.std(ddof=0).shift(1)\n",
    "    z = (x-m)/s\n",
    "    z = z.to_frame()\n",
    "    z = z.dropna()\n",
    "    return(z)\n",
    "\n",
    "def moving_average(x, window):\n",
    "    y = x.rolling(window=window).mean()\n",
    "    return(y)\n",
    "\n",
    "def top_anomalies(df, threshold):\n",
    "    column = df.iloc[:, 0]\n",
    "    df['temp'] = column.apply(lambda x: x if x > 0 else (x*-1))\n",
    "    df['pct'] = df['temp'].rank(pct=True)\n",
    "    df['over_threshold'] = df['pct'].apply(lambda x: 1 if x >= threshold else 0)\n",
    "    df = df.drop(columns=['pct'])\n",
    "    df = df.drop(columns=['temp'])\n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in sample data - load1 metric\n",
    "node_load1 = pd.read_parquet(\"node_load1\")\n",
    "# Preprocess data - resample to 5 minute time bins\n",
    "node_load1 = preprocess_dataframe(node_load1)\n",
    "node_load1_5min = resample_dataframe(node_load1,\"5T\")\n",
    "print(node_load1_5min.plot(y=\"Total\", figsize=(15,5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize subset\n",
    "print(node_load1_5min['2020-01-15 00:00:00':'2020-01-28 00:00:00'].plot(y=\"Total\", figsize=(15,5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate z-scores based on a 50 minute time window\n",
    "zscore = zscore((node_load1_5min['Total']),10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize results\n",
    "print(zscore['2020-01-15 00:00:00':'2020-01-28 00:00:00'].plot(figsize=(15,5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate top z-score 'anomalies' (top 3%)\n",
    "top_anomalies = top_anomalies(zscore, 0.97)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print/visualize results\n",
    "print(top_anomalies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(top_anomalies.plot(y='Total', figsize=(15,5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter out only those z-scores and their corresponding timestamps which are over the defined threshold\n",
    "selection = top_anomalies[top_anomalies['over_threshold']==1]\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n",
    "    print(selection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}