earning_per_share_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aabaa077-2488-404c-a774-c3dcaf3e7913",
   "metadata": {},
   "source": [
    "# Multivariate LSTM for Predicting EPS (Earnings per Share) over Company Fundamentals\n",
    "\n",
    "- In this problem, we will focus on predicting Earnings Per Share (EPS) by jointly modeling historical fundamentals where fundamentals for multiple companies in are stored the \"fundamentals.csv\" file for each year.\n",
    "- We will try different number of latent dimensions of LSTM: [5, 10, 30] as well as learning rate and number of epochs.\n",
    "- We will perform hyperparemeter tuning based on the regression evaluation metric, Mean Absolute Percentage Error (MAPE)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1bbf9b67-a63d-4a5f-9926-df5bfbf8f496",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import mean_absolute_percentage_error\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Input, LSTM, Dense, Conv1D, MaxPooling1D, Flatten, Dropout\n",
    "from keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from sklearn.model_selection import train_test_split\n",
    "import itertools\n",
    "import os\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "03cc2d86-cede-43a9-8444-b5665a785e37",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_directory = r\"C:\\Users\\sb013698\\Desktop\\github\\Machine Learning in Finance\\Datasets\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aa81b2f7-90e6-4a30-a71d-420d0fff1db6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1781, 79)\n"
     ]
    }
   ],
   "source": [
    "# Load dataset\n",
    "data = pd.read_csv(os.path.join(data_directory, 'fundamentals.csv'))\n",
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eb53fbea-b6ff-4a9b-9248-a0211f54ec2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Ticker Symbol</th>\n",
       "      <th>Period Ending</th>\n",
       "      <th>Accounts Payable</th>\n",
       "      <th>Accounts Receivable</th>\n",
       "      <th>Add'l income/expense items</th>\n",
       "      <th>After Tax ROE</th>\n",
       "      <th>Capital Expenditures</th>\n",
       "      <th>Capital Surplus</th>\n",
       "      <th>Cash Ratio</th>\n",
       "      <th>...</th>\n",
       "      <th>Total Current Assets</th>\n",
       "      <th>Total Current Liabilities</th>\n",
       "      <th>Total Equity</th>\n",
       "      <th>Total Liabilities</th>\n",
       "      <th>Total Liabilities &amp; Equity</th>\n",
       "      <th>Total Revenue</th>\n",
       "      <th>Treasury Stock</th>\n",
       "      <th>For Year</th>\n",
       "      <th>Earnings Per Share</th>\n",
       "      <th>Estimated Shares Outstanding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>AAL</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>3.068000e+09</td>\n",
       "      <td>-222000000.0</td>\n",
       "      <td>-1.961000e+09</td>\n",
       "      <td>23.0</td>\n",
       "      <td>-1.888000e+09</td>\n",
       "      <td>4.695000e+09</td>\n",
       "      <td>53.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7.072000e+09</td>\n",
       "      <td>9.011000e+09</td>\n",
       "      <td>-7.987000e+09</td>\n",
       "      <td>2.489100e+10</td>\n",
       "      <td>1.690400e+10</td>\n",
       "      <td>2.485500e+10</td>\n",
       "      <td>-367000000.0</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>-5.60</td>\n",
       "      <td>3.350000e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AAL</td>\n",
       "      <td>2013-12-31</td>\n",
       "      <td>4.975000e+09</td>\n",
       "      <td>-93000000.0</td>\n",
       "      <td>-2.723000e+09</td>\n",
       "      <td>67.0</td>\n",
       "      <td>-3.114000e+09</td>\n",
       "      <td>1.059200e+10</td>\n",
       "      <td>75.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.432300e+10</td>\n",
       "      <td>1.380600e+10</td>\n",
       "      <td>-2.731000e+09</td>\n",
       "      <td>4.500900e+10</td>\n",
       "      <td>4.227800e+10</td>\n",
       "      <td>2.674300e+10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>-11.25</td>\n",
       "      <td>1.630222e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AAL</td>\n",
       "      <td>2014-12-31</td>\n",
       "      <td>4.668000e+09</td>\n",
       "      <td>-160000000.0</td>\n",
       "      <td>-1.500000e+08</td>\n",
       "      <td>143.0</td>\n",
       "      <td>-5.311000e+09</td>\n",
       "      <td>1.513500e+10</td>\n",
       "      <td>60.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.175000e+10</td>\n",
       "      <td>1.340400e+10</td>\n",
       "      <td>2.021000e+09</td>\n",
       "      <td>4.120400e+10</td>\n",
       "      <td>4.322500e+10</td>\n",
       "      <td>4.265000e+10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>4.02</td>\n",
       "      <td>7.169154e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>AAL</td>\n",
       "      <td>2015-12-31</td>\n",
       "      <td>5.102000e+09</td>\n",
       "      <td>352000000.0</td>\n",
       "      <td>-7.080000e+08</td>\n",
       "      <td>135.0</td>\n",
       "      <td>-6.151000e+09</td>\n",
       "      <td>1.159100e+10</td>\n",
       "      <td>51.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9.985000e+09</td>\n",
       "      <td>1.360500e+10</td>\n",
       "      <td>5.635000e+09</td>\n",
       "      <td>4.278000e+10</td>\n",
       "      <td>4.841500e+10</td>\n",
       "      <td>4.099000e+10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>11.39</td>\n",
       "      <td>6.681299e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>AAP</td>\n",
       "      <td>2012-12-29</td>\n",
       "      <td>2.409453e+09</td>\n",
       "      <td>-89482000.0</td>\n",
       "      <td>6.000000e+05</td>\n",
       "      <td>32.0</td>\n",
       "      <td>-2.711820e+08</td>\n",
       "      <td>5.202150e+08</td>\n",
       "      <td>23.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.184200e+09</td>\n",
       "      <td>2.559638e+09</td>\n",
       "      <td>1.210694e+09</td>\n",
       "      <td>3.403120e+09</td>\n",
       "      <td>4.613814e+09</td>\n",
       "      <td>6.205003e+09</td>\n",
       "      <td>-27095000.0</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>5.29</td>\n",
       "      <td>7.328355e+07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0 Ticker Symbol Period Ending  Accounts Payable  \\\n",
       "0           0           AAL    2012-12-31      3.068000e+09   \n",
       "1           1           AAL    2013-12-31      4.975000e+09   \n",
       "2           2           AAL    2014-12-31      4.668000e+09   \n",
       "3           3           AAL    2015-12-31      5.102000e+09   \n",
       "4           4           AAP    2012-12-29      2.409453e+09   \n",
       "\n",
       "   Accounts Receivable  Add'l income/expense items  After Tax ROE  \\\n",
       "0         -222000000.0               -1.961000e+09           23.0   \n",
       "1          -93000000.0               -2.723000e+09           67.0   \n",
       "2         -160000000.0               -1.500000e+08          143.0   \n",
       "3          352000000.0               -7.080000e+08          135.0   \n",
       "4          -89482000.0                6.000000e+05           32.0   \n",
       "\n",
       "   Capital Expenditures  Capital Surplus  Cash Ratio  ...  \\\n",
       "0         -1.888000e+09     4.695000e+09        53.0  ...   \n",
       "1         -3.114000e+09     1.059200e+10        75.0  ...   \n",
       "2         -5.311000e+09     1.513500e+10        60.0  ...   \n",
       "3         -6.151000e+09     1.159100e+10        51.0  ...   \n",
       "4         -2.711820e+08     5.202150e+08        23.0  ...   \n",
       "\n",
       "   Total Current Assets  Total Current Liabilities  Total Equity  \\\n",
       "0          7.072000e+09               9.011000e+09 -7.987000e+09   \n",
       "1          1.432300e+10               1.380600e+10 -2.731000e+09   \n",
       "2          1.175000e+10               1.340400e+10  2.021000e+09   \n",
       "3          9.985000e+09               1.360500e+10  5.635000e+09   \n",
       "4          3.184200e+09               2.559638e+09  1.210694e+09   \n",
       "\n",
       "   Total Liabilities  Total Liabilities & Equity  Total Revenue  \\\n",
       "0       2.489100e+10                1.690400e+10   2.485500e+10   \n",
       "1       4.500900e+10                4.227800e+10   2.674300e+10   \n",
       "2       4.120400e+10                4.322500e+10   4.265000e+10   \n",
       "3       4.278000e+10                4.841500e+10   4.099000e+10   \n",
       "4       3.403120e+09                4.613814e+09   6.205003e+09   \n",
       "\n",
       "   Treasury Stock  For Year  Earnings Per Share  Estimated Shares Outstanding  \n",
       "0    -367000000.0    2012.0               -5.60                  3.350000e+08  \n",
       "1             0.0    2013.0              -11.25                  1.630222e+08  \n",
       "2             0.0    2014.0                4.02                  7.169154e+08  \n",
       "3             0.0    2015.0               11.39                  6.681299e+08  \n",
       "4     -27095000.0    2012.0                5.29                  7.328355e+07  \n",
       "\n",
       "[5 rows x 79 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2d9cdfa6-a8b3-4fab-9e55-e9dd461111f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1299, 31)\n"
     ]
    }
   ],
   "source": [
    "# Remove missing rows\n",
    "clean_df = data.dropna()\n",
    "\n",
    "# Compute the correlation matrix\n",
    "correlation_with_eps = clean_df.select_dtypes(include=[np.number]).corr()[\"Earnings Per Share\"]\n",
    "\n",
    "# Filter the columns that have an absolute correlation greater than 10%\n",
    "filtered_columns = correlation_with_eps[correlation_with_eps.abs() > 0.1]\n",
    "clean_df = clean_df[filtered_columns.index.to_list()]\n",
    "print(clean_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3448f8d5-a0d9-4fa2-94fa-b281d96eb5a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 1299 entries, 0 to 1779\n",
      "Data columns (total 31 columns):\n",
      " #   Column                                        Non-Null Count  Dtype  \n",
      "---  ------                                        --------------  -----  \n",
      " 0   Accounts Payable                              1299 non-null   float64\n",
      " 1   Cost of Revenue                               1299 non-null   float64\n",
      " 2   Depreciation                                  1299 non-null   float64\n",
      " 3   Earnings Before Interest and Tax              1299 non-null   float64\n",
      " 4   Earnings Before Tax                           1299 non-null   float64\n",
      " 5   Gross Profit                                  1299 non-null   float64\n",
      " 6   Income Tax                                    1299 non-null   float64\n",
      " 7   Investments                                   1299 non-null   float64\n",
      " 8   Liabilities                                   1299 non-null   float64\n",
      " 9   Long-Term Investments                         1299 non-null   float64\n",
      " 10  Net Cash Flow-Operating                       1299 non-null   float64\n",
      " 11  Net Cash Flows-Financing                      1299 non-null   float64\n",
      " 12  Net Income                                    1299 non-null   float64\n",
      " 13  Net Income Adjustments                        1299 non-null   float64\n",
      " 14  Net Income Applicable to Common Shareholders  1299 non-null   float64\n",
      " 15  Net Income-Cont. Operations                   1299 non-null   float64\n",
      " 16  Net Receivables                               1299 non-null   float64\n",
      " 17  Non-Recurring Items                           1299 non-null   float64\n",
      " 18  Operating Income                              1299 non-null   float64\n",
      " 19  Operating Margin                              1299 non-null   float64\n",
      " 20  Other Equity                                  1299 non-null   float64\n",
      " 21  Other Operating Items                         1299 non-null   float64\n",
      " 22  Pre-Tax Margin                                1299 non-null   float64\n",
      " 23  Profit Margin                                 1299 non-null   float64\n",
      " 24  Retained Earnings                             1299 non-null   float64\n",
      " 25  Sale and Purchase of Stock                    1299 non-null   float64\n",
      " 26  Total Current Assets                          1299 non-null   float64\n",
      " 27  Total Current Liabilities                     1299 non-null   float64\n",
      " 28  Total Revenue                                 1299 non-null   float64\n",
      " 29  Treasury Stock                                1299 non-null   float64\n",
      " 30  Earnings Per Share                            1299 non-null   float64\n",
      "dtypes: float64(31)\n",
      "memory usage: 324.8 KB\n"
     ]
    }
   ],
   "source": [
    "clean_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "31c77c0b-158c-4c1e-bc62-88bce2af9f72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1299, 31)\n"
     ]
    }
   ],
   "source": [
    "# Normalize data\n",
    "new_df = clean_df.copy()\n",
    "# Specify the column to be excluded from scaling\n",
    "column_not_scaled = 'Earnings Per Share'\n",
    "# Select only the numeric columns (exclude any non-numeric ones)\n",
    "numeric_columns = new_df.select_dtypes(include=[np.number]).columns.tolist()\n",
    "# Exclude the target column ('Earnings Per Share') from scaling\n",
    "columns_to_scale = [col for col in numeric_columns if col != column_not_scaled]\n",
    "# Extract the subset of the numeric columns to scale\n",
    "subset_to_scale = new_df[columns_to_scale]\n",
    "# Initialize and fit the scaler on the subset\n",
    "scaler = StandardScaler()\n",
    "scaled_subset = scaler.fit_transform(subset_to_scale)\n",
    "# Replace the original values with scaled values in the DataFrame\n",
    "new_df[columns_to_scale] = scaled_subset\n",
    "\n",
    "# Define features and target & re-order columns\n",
    "# Define the target column\n",
    "target = [\"Earnings Per Share\"]\n",
    "# Get the column names as a list\n",
    "features = [col for col in new_df.columns if col not in target]\n",
    "# Combine target and features into new column order\n",
    "# Make sure the target is the first column\n",
    "new_column_order = target + features\n",
    "new_df = new_df[new_column_order]\n",
    "print(new_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e7423de7-edce-4e69-a19f-c7592368297a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Earnings Per Share</th>\n",
       "      <th>Accounts Payable</th>\n",
       "      <th>Cost of Revenue</th>\n",
       "      <th>Depreciation</th>\n",
       "      <th>Earnings Before Interest and Tax</th>\n",
       "      <th>Earnings Before Tax</th>\n",
       "      <th>Gross Profit</th>\n",
       "      <th>Income Tax</th>\n",
       "      <th>Investments</th>\n",
       "      <th>Liabilities</th>\n",
       "      <th>...</th>\n",
       "      <th>Other Equity</th>\n",
       "      <th>Other Operating Items</th>\n",
       "      <th>Pre-Tax Margin</th>\n",
       "      <th>Profit Margin</th>\n",
       "      <th>Retained Earnings</th>\n",
       "      <th>Sale and Purchase of Stock</th>\n",
       "      <th>Total Current Assets</th>\n",
       "      <th>Total Current Liabilities</th>\n",
       "      <th>Total Revenue</th>\n",
       "      <th>Treasury Stock</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-5.60</td>\n",
       "      <td>-0.086035</td>\n",
       "      <td>-0.099624</td>\n",
       "      <td>-0.065782</td>\n",
       "      <td>-0.714485</td>\n",
       "      <td>-0.793056</td>\n",
       "      <td>0.489305</td>\n",
       "      <td>-0.637361</td>\n",
       "      <td>0.182437</td>\n",
       "      <td>0.312047</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.843813</td>\n",
       "      <td>0.091423</td>\n",
       "      <td>-0.315517</td>\n",
       "      <td>-0.274099</td>\n",
       "      <td>-0.678030</td>\n",
       "      <td>0.290847</td>\n",
       "      <td>-0.070032</td>\n",
       "      <td>0.336829</td>\n",
       "      <td>0.078146</td>\n",
       "      <td>0.251187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-11.25</td>\n",
       "      <td>0.169587</td>\n",
       "      <td>-0.084166</td>\n",
       "      <td>-0.057914</td>\n",
       "      <td>-0.634982</td>\n",
       "      <td>-0.749141</td>\n",
       "      <td>0.586905</td>\n",
       "      <td>-0.522659</td>\n",
       "      <td>-0.342454</td>\n",
       "      <td>-0.323817</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.483568</td>\n",
       "      <td>0.095052</td>\n",
       "      <td>-0.406789</td>\n",
       "      <td>-0.333069</td>\n",
       "      <td>-0.745522</td>\n",
       "      <td>0.290847</td>\n",
       "      <td>0.430957</td>\n",
       "      <td>0.815652</td>\n",
       "      <td>0.120217</td>\n",
       "      <td>0.274049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.02</td>\n",
       "      <td>0.128436</td>\n",
       "      <td>0.052609</td>\n",
       "      <td>0.075425</td>\n",
       "      <td>0.246706</td>\n",
       "      <td>0.144395</td>\n",
       "      <td>1.393538</td>\n",
       "      <td>-0.174953</td>\n",
       "      <td>0.709446</td>\n",
       "      <td>-1.034223</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.443842</td>\n",
       "      <td>0.295535</td>\n",
       "      <td>-0.406789</td>\n",
       "      <td>-0.333069</td>\n",
       "      <td>-0.644910</td>\n",
       "      <td>-0.051547</td>\n",
       "      <td>0.253182</td>\n",
       "      <td>0.775508</td>\n",
       "      <td>0.474674</td>\n",
       "      <td>0.274049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.39</td>\n",
       "      <td>0.186611</td>\n",
       "      <td>-0.081877</td>\n",
       "      <td>0.135469</td>\n",
       "      <td>0.473835</td>\n",
       "      <td>0.377059</td>\n",
       "      <td>1.597871</td>\n",
       "      <td>-1.884679</td>\n",
       "      <td>0.230796</td>\n",
       "      <td>-0.681265</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.509583</td>\n",
       "      <td>0.326832</td>\n",
       "      <td>-0.269881</td>\n",
       "      <td>0.374560</td>\n",
       "      <td>-0.375093</td>\n",
       "      <td>-0.960909</td>\n",
       "      <td>0.131234</td>\n",
       "      <td>0.795580</td>\n",
       "      <td>0.437684</td>\n",
       "      <td>0.274049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.29</td>\n",
       "      <td>-0.174309</td>\n",
       "      <td>-0.319369</td>\n",
       "      <td>-0.401802</td>\n",
       "      <td>-0.312756</td>\n",
       "      <td>-0.284464</td>\n",
       "      <td>-0.313901</td>\n",
       "      <td>-0.223095</td>\n",
       "      <td>0.074423</td>\n",
       "      <td>0.270125</td>\n",
       "      <td>...</td>\n",
       "      <td>0.289618</td>\n",
       "      <td>-0.291852</td>\n",
       "      <td>-0.315517</td>\n",
       "      <td>-0.392038</td>\n",
       "      <td>-0.303520</td>\n",
       "      <td>0.284794</td>\n",
       "      <td>-0.338649</td>\n",
       "      <td>-0.307397</td>\n",
       "      <td>-0.337434</td>\n",
       "      <td>0.272361</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Earnings Per Share  Accounts Payable  Cost of Revenue  Depreciation  \\\n",
       "0               -5.60         -0.086035        -0.099624     -0.065782   \n",
       "1              -11.25          0.169587        -0.084166     -0.057914   \n",
       "2                4.02          0.128436         0.052609      0.075425   \n",
       "3               11.39          0.186611        -0.081877      0.135469   \n",
       "4                5.29         -0.174309        -0.319369     -0.401802   \n",
       "\n",
       "   Earnings Before Interest and Tax  Earnings Before Tax  Gross Profit  \\\n",
       "0                         -0.714485            -0.793056      0.489305   \n",
       "1                         -0.634982            -0.749141      0.586905   \n",
       "2                          0.246706             0.144395      1.393538   \n",
       "3                          0.473835             0.377059      1.597871   \n",
       "4                         -0.312756            -0.284464     -0.313901   \n",
       "\n",
       "   Income Tax  Investments  Liabilities  ...  Other Equity  \\\n",
       "0   -0.637361     0.182437     0.312047  ...     -0.843813   \n",
       "1   -0.522659    -0.342454    -0.323817  ...     -0.483568   \n",
       "2   -0.174953     0.709446    -1.034223  ...     -1.443842   \n",
       "3   -1.884679     0.230796    -0.681265  ...     -1.509583   \n",
       "4   -0.223095     0.074423     0.270125  ...      0.289618   \n",
       "\n",
       "   Other Operating Items  Pre-Tax Margin  Profit Margin  Retained Earnings  \\\n",
       "0               0.091423       -0.315517      -0.274099          -0.678030   \n",
       "1               0.095052       -0.406789      -0.333069          -0.745522   \n",
       "2               0.295535       -0.406789      -0.333069          -0.644910   \n",
       "3               0.326832       -0.269881       0.374560          -0.375093   \n",
       "4              -0.291852       -0.315517      -0.392038          -0.303520   \n",
       "\n",
       "   Sale and Purchase of Stock  Total Current Assets  \\\n",
       "0                    0.290847             -0.070032   \n",
       "1                    0.290847              0.430957   \n",
       "2                   -0.051547              0.253182   \n",
       "3                   -0.960909              0.131234   \n",
       "4                    0.284794             -0.338649   \n",
       "\n",
       "   Total Current Liabilities  Total Revenue  Treasury Stock  \n",
       "0                   0.336829       0.078146        0.251187  \n",
       "1                   0.815652       0.120217        0.274049  \n",
       "2                   0.775508       0.474674        0.274049  \n",
       "3                   0.795580       0.437684        0.274049  \n",
       "4                  -0.307397      -0.337434        0.272361  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4f43db2f-476b-453b-9231-3489135734f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequence shape: (1269, 30, 31)\n",
      "Target shape: (1269,)\n"
     ]
    }
   ],
   "source": [
    "# Create sequences\n",
    "sequences = list()\n",
    "targets = list()\n",
    "\n",
    "# Define a sequence length\n",
    "sequence_length = 30\n",
    "\n",
    "for i in range(len(new_df) - sequence_length):\n",
    "    \n",
    "    seq_features = new_df.iloc[i:(i+sequence_length)].values\n",
    "    seq_target = new_df.iloc[(i+sequence_length), 0]\n",
    "    \n",
    "    sequences.append(seq_features)\n",
    "    targets.append(seq_target)\n",
    "\n",
    "# Convert sequences and targets to numpy arrays\n",
    "X = np.array(sequences, dtype=np.float32)\n",
    "y = np.array(targets, dtype=np.float32)\n",
    "\n",
    "print(f\"Sequence shape: {X.shape}\")\n",
    "print(f\"Target shape: {y.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "832ec885-29ae-4554-b85e-26e7ce98618f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create function to build LSTM models with specific latent dimensions\n",
    "def build_model(latent_dim, input_shape, lr):\n",
    "    model = Sequential()\n",
    "    model.add(\n",
    "        LSTM(\n",
    "            units=latent_dim, \n",
    "            input_shape=input_shape, \n",
    "            activation='tanh', \n",
    "            return_sequences=False,\n",
    "        )\n",
    "    )\n",
    "    model.add(Dense(32, activation='tanh'))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(1, activation='linear'))  # Predict EPS for the next time step\n",
    "    model.compile(optimizer=Adam(learning_rate=lr), loss='mean_squared_error')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8774f211-f908-4981-8e7a-c2b0ef8bf3d2",
   "metadata": {},
   "source": [
    "### Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f28c9d55-5660-405c-8f19-ac98ce34eac7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape: (1015, 30, 31)\n",
      "Validation shape: (127, 30, 31)\n",
      "Test shape: (127, 30, 31)\n"
     ]
    }
   ],
   "source": [
    "# Split the data into training, validation, and test sets\n",
    "train_size = 0.8\n",
    "val_size = 0.1\n",
    "test_size = 0.1\n",
    "\n",
    "X_train, X_temp, y_train, y_temp = train_test_split(\n",
    "    X, y, test_size=(val_size+test_size), random_state=42,\n",
    ")\n",
    "X_val, X_test, y_val, y_test = train_test_split(\n",
    "    X_temp, y_temp, test_size=(test_size/(val_size+test_size)), random_state=42,\n",
    ")\n",
    "\n",
    "print(f\"Train shape: {X_train.shape}\")\n",
    "print(f\"Validation shape: {X_val.shape}\")\n",
    "print(f\"Test shape: {X_test.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e6fbc8f9-2e3c-46fa-a8b5-95bc661cfdcd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of combinations: 36\n",
      "[(5, 0.0001, 50), (5, 0.0001, 100), (5, 0.0001, 150), (5, 0.0001, 200), (5, 0.0005, 50)]\n"
     ]
    }
   ],
   "source": [
    "# Create scenarios for hyperparameter tuning\n",
    "latent_dimensions = [5, 10, 30]\n",
    "lr_list = [1e-4, 5e-4, 1e-3]\n",
    "epoch_list = [50, 100, 150, 200]\n",
    "\n",
    "combinations = list(itertools.product(\n",
    "    latent_dimensions, lr_list, epoch_list,\n",
    "))\n",
    "\n",
    "print(f\"Number of combinations: {len(combinations)}\")\n",
    "print(combinations[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "910f0d20-68f0-4e4b-b9c7-9d39d803e965",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 36/36 [31:14<00:00, 52.06s/it]\n"
     ]
    }
   ],
   "source": [
    "# Check if the results file already exists\n",
    "results_file = os.path.join(data_directory, \"hp_results_eps.csv\")\n",
    "\n",
    "# Initialize an empty dataframe if the file doesn't exist\n",
    "if os.path.exists(results_file):\n",
    "    # Load existing results if file exists\n",
    "    results_df = pd.read_csv(results_file)\n",
    "else:\n",
    "    # Create an empty df to store results\n",
    "    results_df = pd.DataFrame(columns=[\"Scenario\", \"Latent Dimension\", \"Learning Rate\", \"Number of Epochs\", \"MAPE\"])\n",
    "\n",
    "scenario_id = 1\n",
    "\n",
    "for combo in tqdm(combinations):\n",
    "\n",
    "    # Define parameter values from combinations\n",
    "    latent_dim, lr, epoch = combo\n",
    "\n",
    "    # Define and train the LSTM model using given parameter values\n",
    "    model = build_model(latent_dim, (X_train.shape[1], X_train.shape[2]), lr)  # input_shape=(timesteps, features)\n",
    "    model.fit(X_train, y_train, epochs=epoch, batch_size=64, verbose=0)\n",
    "\n",
    "    # Make predictions over the validation set and evaluate model's performance using MAPE\n",
    "    preds = model.predict(X_val, verbose=0)\n",
    "    mape = mean_absolute_percentage_error(preds, y_val)\n",
    "\n",
    "    # Store sceanario results in a dictionary\n",
    "    result = {\n",
    "        \"Scenario\": f\"S{scenario_id}\",\n",
    "        \"Latent Dimension\": latent_dim,\n",
    "        \"Learning Rate\": lr,\n",
    "        \"Number of Epochs\": epoch,\n",
    "        \"MAPE\": mape,\n",
    "    }\n",
    "\n",
    "    scenario_id += 1\n",
    "\n",
    "    # Convert the result to a DataFrame\n",
    "    scenario_result = pd.DataFrame([result])\n",
    "    \n",
    "    # Drop columns that are completely empty or contain all NaN values\n",
    "    scenario_result = scenario_result.dropna(axis=1, how='all')\n",
    "    \n",
    "    # Concatenate the new result with the existing DataFrame\n",
    "    results_df = pd.concat([results_df, scenario_result], ignore_index=True)\n",
    "    \n",
    "    # Save results to CSV after each scenario\n",
    "    results_df.to_csv(results_file, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53a5be55-7eb3-499c-b9e9-2fb30ee7d16e",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "47e5a712-e895-4687-881a-b85138366576",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Scenario</th>\n",
       "      <th>Latent Dimension</th>\n",
       "      <th>Learning Rate</th>\n",
       "      <th>Number of Epochs</th>\n",
       "      <th>MAPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>S13</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>50</td>\n",
       "      <td>0.699085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S3</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>150</td>\n",
       "      <td>0.749119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>S14</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>100</td>\n",
       "      <td>0.809940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>S2</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>100</td>\n",
       "      <td>0.816036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>S25</td>\n",
       "      <td>30</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>50</td>\n",
       "      <td>0.866264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>S15</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>150</td>\n",
       "      <td>0.909409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>S7</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>150</td>\n",
       "      <td>1.050857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>S16</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>200</td>\n",
       "      <td>1.082592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>S1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>50</td>\n",
       "      <td>1.104181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S5</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0005</td>\n",
       "      <td>50</td>\n",
       "      <td>1.152777</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Scenario  Latent Dimension  Learning Rate  Number of Epochs      MAPE\n",
       "12      S13                10         0.0001                50  0.699085\n",
       "2        S3                 5         0.0001               150  0.749119\n",
       "13      S14                10         0.0001               100  0.809940\n",
       "1        S2                 5         0.0001               100  0.816036\n",
       "24      S25                30         0.0001                50  0.866264\n",
       "14      S15                10         0.0001               150  0.909409\n",
       "6        S7                 5         0.0005               150  1.050857\n",
       "15      S16                10         0.0001               200  1.082592\n",
       "0        S1                 5         0.0001                50  1.104181\n",
       "4        S5                 5         0.0005                50  1.152777"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Obtain best hp values\n",
    "hp_results = pd.read_csv(results_file)\n",
    "hp_results.sort_values(by=\"MAPE\", ascending=True)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f7471459-c5a8-47c0-ba15-5507c12fec00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4/4 [==============================] - 1s 9ms/step\n",
      "Test MAPE: 0.890\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model's performance on the test set using optimal hp values\n",
    "# Split the data into training and test sets\n",
    "# This time, use 90% and 10% of data for training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.1, random_state=42,\n",
    ")\n",
    "\n",
    "# Use optimal HP settings to evaluate the model's overall performance\n",
    "best_model = build_model(\n",
    "    latent_dim=10, \n",
    "    input_shape=(X_train.shape[1], X_train.shape[2]), \n",
    "    lr=1e-4,\n",
    ")\n",
    "best_model.fit(X_train, y_train, epochs=50, batch_size=64, verbose=0)\n",
    "\n",
    "# Make predictions over the test set\n",
    "preds = best_model.predict(X_test, verbose=1)\n",
    "test_mape = mean_absolute_percentage_error(preds, y_test)\n",
    "\n",
    "print(f\"Test MAPE: {test_mape:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c667be84-fade-41f6-ac1d-b2ae394ec153",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09263b2f-0900-49d7-818c-1ac2330dc3bd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "1e2d24e6-3023-45d2-8bf6-fc16497f0fe2",
   "metadata": {},
   "source": [
    "# END"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}