AutoARIMA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c53995ed-813e-414f-af9d-abdd3a7c565a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt\n",
    "import boto3\n",
    "from datetime import datetime\n",
    "from statsmodels.tsa.stattools import adfuller\n",
    "import pmdarima as pm\n",
    "from pmdarima import model_selection\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "649ab477-dd59-4710-92e9-0759a6ca085c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- Read in data\n",
    "\n",
    "df = pd.read_csv('s3://xxxxxx/xxxxx.csv')\n",
    "\n",
    "df = df.rename(columns = {'COUNT(X.REFNUM)': 'AVOL'})\n",
    "\n",
    "# -- Select the matter name for time series analysis\n",
    "\n",
    "df['M_NAME'].unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da18aca4-eeb3-4d5f-b506-744631c8ace8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df['M_NAME'] == 'MNS8']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b921a14e-42aa-4347-b4fa-e80bacc8e257",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- Inspect data\n",
    "\n",
    "print(df.head())\n",
    "print('\\n Data Types:')\n",
    "print(df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1607eda-c20c-47d7-9979-d7e06fa066ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- convert month received column to dtype dates\n",
    "\n",
    "df['MONTH'] = pd.to_datetime(df['MONTH'])\n",
    "df.set_index('MONTH', inplace=True)\n",
    "\n",
    " # -- check dtype of index\n",
    "df.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e186fd95-b7a5-496d-b436-cb39a8fe05b3",
   "metadata": {},
   "outputs": [],
   "source": [
    " # -- convert apps volume to time series\n",
    "time_series = pd.DataFrame(df['AVOL'])\n",
    "time_series.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82fa4f67-61b1-485b-ac07-69bbace33405",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- Sort on index\n",
    "\n",
    "time_series = time_series.sort_index(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21f74055-69e3-4624-a39b-907adc75b9ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- Find total range\n",
    "\n",
    "# print(min(df['MONTH']) + \" \" +\n",
    "# max(df['MONTH']))\n",
    "\n",
    "# -- required (minus March as data incomplete). total range = '2016-03':'2022-02'\n",
    "\n",
    "time_series = time_series['2016-04':'2022-02']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2db70d7-90f5-4926-b67c-a5607f4e3b79",
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_plot = plt.plot(time_series)\n",
    "ts_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef738667-2cf7-4947-a470-c01c1eaaa1a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "adf_test = ADFTest(alpha = 0.05)\n",
    "adf_test.should_diff(time_series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc9e295c-c8f0-4796-b08d-475dd9ee0f16",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- Test Stationarity of Time Series\n",
    "\n",
    "def test_stationarity(tseries):\n",
    "    # Determining rolling statistics:\n",
    "    rolmean = tseries.rolling(window=12).mean()\n",
    "    rolstd = tseries.rolling(window=12).std()\n",
    "    \n",
    "    # Plot rolling statistics:\n",
    "    plt.plot(tseries, color='blue', label='Original')\n",
    "    plt.plot(rolmean, color='red', label='Rolling Mean')\n",
    "    plt.plot(rolstd, color='black', label = 'Rolling StD')\n",
    "    plt.legend(loc='best')\n",
    "    plt.title('Rolling Mean & Standard Deviation')\n",
    "    plt.show()\n",
    "    # Perform Dickey-Fuller test:\n",
    "    print('Results of Dickey-Fuller Test:')\n",
    "    dftest = adfuller(tseries, autolag='AIC')\n",
    "    dfoutput = pd.Series(dftest[0:3], index=['Test Statistic', 'p-value', '#Lag Used' 'Number of Observations Used'])\n",
    "    for key, value in dftest[4].items():\n",
    "        dfoutput['Critical Value (%s)'%key] = value\n",
    "    print(dfoutput)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c56b6feb-6eb1-42d0-87aa-296c158d533e",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_stationarity(time_series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "743a42ae-806b-4ed9-8f03-7e9e78f4f671",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- Plot test and train data\n",
    "train = time_series[:85].dropna()\n",
    "test = time_series[-20:].dropna()\n",
    "plt.plot(train)\n",
    "plt.plot(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2353939f-b01c-45d6-b7eb-23e8fff7aa72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# arima_model = pm.auto_arima(train, error_action='ignore', trace=True,\n",
    "#                       suppress_warnings=True, maxiter=150,\n",
    "#                       seasonal=True, m=12, return_conf_int=True)\n",
    "\n",
    "\"\"\"-- MNS8 Best model: ARIMA(p=0,d=1,q=1)(P=2/4?,D=0,Q=0)[12]\n",
    "    -- MNDA Best model: ARIMA(2,1,0)(2,0,0)[12]\n",
    "    -- MNPLF Best model:  ARIMA(0,1,1)(2,0,0)[12]\n",
    "    -- MNPL Best model:  ARIMA(4,1,1)(2,0,0)[12]  \n",
    "\"\"\"\n",
    "\n",
    "arima_model = pm.arima.ARIMA(order=(0,1,1),seasonal_order=(2,0,0,12),error_action='ignore', trace=True,\n",
    "                             suppress_warnings=True, maxiter=10, seasonal=True, m=12, return_conf_int=True)\n",
    "arima_model.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fb98b3e-ae58-43d2-95e5-42ebff9e007a",
   "metadata": {},
   "outputs": [],
   "source": [
    "arima_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94155d57-b989-4833-ba03-d32b20af54ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- collate and define results / figures\n",
    "\n",
    "predictions = arima_model.predict(n_periods = 20, return_conf_int=True)\n",
    "forecast = predictions[0]\n",
    "conf_int = predictions[1]\n",
    "low_conf, high_conf = zip(*conf_int)\n",
    "\n",
    "forecast = pd.Series(forecast, index=test.index)\n",
    "high_conf = pd.Series(high_conf, index=test.index)\n",
    "low_conf = pd.Series(low_conf,index=test.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2f7bae9-7e81-4e36-900a-f61095ecab22",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- plot results\n",
    "\n",
    "plt.figure(figsize=(8,5))\n",
    "plt.fill_between(test.index,high_conf,low_conf,color=\"springgreen\", label=\"Confidence Int (95%)\")\n",
    "plt.plot(train, label=\"Training\", color=\"black\")\n",
    "plt.plot(forecast,label=\"Predicted\", color=\"seagreen\")\n",
    "plt.legend(loc = \"upper left\")\n",
    "plt.grid(which='minor')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4129806d-1960-4976-be3b-70e7c1f46f1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- show R squared\n",
    "\n",
    "print(r2_score(test, forecast))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d59cf37-366e-43fd-bdb5-5bb39e4ad474",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- Apply forecast to new time periods\n",
    "\n",
    "new_dates =pd.Index(['2022-03-01', '2022-04-01','2022-05-01','2022-06-01',\n",
    "            '2022-07-01','2022-08-01','2022-09-01','2022-10-01',\n",
    "            '2022-11-01','2022-12-01','2023-01-01','2023-02-01'])\n",
    "time_series.index.append(new_dates)\n",
    "time_series.index = pd.to_datetime(time_series.index)\n",
    "#time_series.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdb38312-11ae-4c11-9907-4b516f019e8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- plot final forecast results\n",
    "\n",
    "plt.plot(time_series)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "forecastvenv",
   "language": "python",
   "name": "forecastvenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}