common.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyN//3EPv1tsGq9I8Yl+JeZA",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/j03m/Algorithmic-trading/blob/master/common.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "id": "notpCbCROSOU",
        "outputId": "f6e9886e-084a-4eb1-d532-a5afd6476875"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-a1ecc0dddd6c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtalib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas_ta\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mta\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mplotly\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpress\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcopy\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'talib'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ],
      "source": [
        "import talib\n",
        "import pandas as pd\n",
        "import pandas_ta as ta\n",
        "import plotly.express as px\n",
        "from copy import copy\n",
        "from scipy import stats\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import plotly.figure_factory as ff\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from sklearn.svm import SVR\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV\n",
        "from sklearn.metrics import r2_score, confusion_matrix, classification_report, accuracy_score, f1_score\n",
        "from tensorflow import keras\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "import requests\n",
        "from requests.exceptions import HTTPError\n",
        "import json as js\n",
        "from datetime import datetime, timedelta\n",
        "import time\n",
        "from os.path import exists\n",
        "from decimal import *\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "from keras.wrappers.scikit_learn import KerasClassifier\n",
        "from keras import backend as K\n",
        "from sklearn.model_selection import KFold, ParameterGrid\n",
        "from keras.layers import Input, LSTM, Attention, Dense\n",
        "from keras.models import Model\n",
        "from keras.optimizers import Adam\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras.regularizers import l2\n",
        "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
        "from sklearn.svm import SVR\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.pipeline import Pipeline\n",
        "import joblib\n",
        "import os\n",
        "\n",
        "pd.options.display.float_format = '{:f}'.format\n",
        "np.set_printoptions(formatter={'float': '{:f}'.format})\n",
        "\n",
        "# Function to plot interactive plots using Plotly Express\n",
        "sc = MinMaxScaler()\n",
        "num_features = 2 #3\n",
        "candle_features = 5 #6\n",
        "\n",
        "\n",
        "# commands\n",
        "coin_base = False\n",
        "ku_coin = True\n",
        "load_models = True\n",
        "train_models = False\n",
        "save_models = False\n",
        "\n",
        "\n",
        "COINBASE_REST_API = 'https://api.pro.coinbase.com'\n",
        "COINBASE_PRODUCTS = COINBASE_REST_API+'/products'\n",
        "KUCOIN_REST_API = \"https://api.kucoin.com\"\n",
        "KUCOIN_PRODUCTS = KUCOIN_REST_API+ \"/api/v1/market/allTickers\"\n",
        "KUCOIN_CANDLES = KUCOIN_REST_API+ \"/api/v1/market/candles\"\n",
        "\n",
        "data_path = '/content/drive/My Drive/ml-trde-notebooks/data'\n",
        "model_path = \"/content/drive/My Drive/ml-trde-notebooks/models\"\n",
        "\n",
        "#pull training data\n",
        "tickers = [\"SPY\", \"TSLA\", \"AAPL\", \"IBM\", \"F\", \"CAT\", \"BAC\", \"B\", \"META\", \"AMZN\", \"XOM\", \"BP\"]\n",
        "coins = [\"FCON-USDT\", \"GMT3L-USDT\", \"NEAR3L-USDT\", \"H2O-USDT\", \"DOGE3L-USDT\", \"DOGE3S-USDT\"]\n",
        "\n",
        "all_stock_dfs = []\n",
        "all_coin_dfs = []\n",
        "gbl_all_features = [\n",
        "                    \"Open\", \"High\",\n",
        "                    \"Low\",\n",
        "                    \"Close\",\n",
        "                    \"Volume\",\n",
        "                    \"CDL_ENGULFING\",\n",
        "                    \"CDL_HARAMI\",\n",
        "                    \"CDL_HARAMICROSS\",\n",
        "                    \"CDL_PIERCING\",\n",
        "                    \"CDL_DARKCLOUDCOVER\",\n",
        "                    \"CDL_HAMMER\",\n",
        "                    \"CDL_INVERTEDHAMMER\",\n",
        "                    \"MACD_12_26_9\",\n",
        "                    \"MACDh_12_26_9\",\n",
        "                    \"MACDs_12_26_9\",\n",
        "                    \"RSI_14\",\n",
        "                    \"STOCHk_14_3_3\",\n",
        "                    \"STOCHd_14_3_3\",\n",
        "                    \"BBL_5_2.0\",\n",
        "                    \"BBM_5_2.0\",\n",
        "                    \"BBU_5_2.0\",\n",
        "                    \"BBB_5_2.0\",\n",
        "                    \"BBP_5_2.0\",\n",
        "                    \"OBV\",\n",
        "                    \"AD\",\n",
        "                    \"MFI_14\",\n",
        "                    \"WILLR_14\",\n",
        "                    \"RVI\", \"VWAP\", \"VWAPD\"]\n",
        "\n",
        "gbl_target_column = [\"Target\"]\n",
        "gbl_all_columns = gbl_all_features + gbl_target_column\n",
        "\n",
        "all_model_names = [\"lstm_cv\", \n",
        "        \"lstm_coins_cv\", \n",
        "        \"lstm_att_cv\", \n",
        "        \"lstm_att_ohlcv\", \n",
        "        \"lstm_cv_rvi\",\n",
        "        \"lstm_cv_vwap\", \n",
        "        \"lstm_ohlc\",\n",
        "        \"svm_cv\",\n",
        "        \"svm_cv_vwap\"\n",
        "        \"lstm_xgb_cols\"\n",
        "        ]\n",
        "\n",
        "model_config = {\n",
        "    \"day_bar_models\": [\n",
        "        \"lstm_cv\", \n",
        "        \"lstm_att_cv\", \n",
        "        \"lstm_cv_rvi\",\n",
        "        \"svm_cv\",\n",
        "        \"lstm_xgb_cols\"],\n",
        "\n",
        "    \"training_filter\": [],\n",
        "    \"backtest_filter\": [],\n",
        "    \"15m_bars\": [\"lstm_15m\"],\n",
        "    \"training_types\":{\n",
        "        \"lstm_coins_cv\":\"all\"\n",
        "    },\n",
        "    \"column_sets\": {\n",
        "        \"lstm_cv\": [\"Close\", \"Volume\"],\n",
        "        \"lstm_15m\": [\"Close\", \"Volume\"],\n",
        "        \"lstm_coins_cv\": [\"Close\", \"Volume\"],\n",
        "        \"lstm_ohlc\": [\"Open\", \"High\", \"Low\", \"Close\", \"Volume\"],\n",
        "        \"lstm_att_cv\": [\"Close\", \"Volume\"],\n",
        "        \"lstm_att_ohlcv\": [\"Open\", \"High\", \"Low\", \"Close\", \"Volume\"],\n",
        "        \"lstm_cv_rvi\": [\"Close\", \"Volume\", \"RVI\"],\n",
        "        \"lstm_cv_vwap\": [\"Close\", \"Volume\", \"VWAP\", \"VWAPD\"],\n",
        "        \"svm_cv\": [\"Close\", \"Volume\"],\n",
        "        \"svm_cv_vwap\": [\"Close\", \"Volume\", \"VWAP\", \"VWAPD\"],\n",
        "        \"lstm_xgb_cols\": [\n",
        "                          'High',\n",
        "                          'Low',\n",
        "                          'Close',\n",
        "                          'MACDh_12_26_9',\n",
        "                          'Open',\n",
        "                          'BBL_5_2.0',\n",
        "                          'AD',\n",
        "                          'MACDs_12_26_9',\n",
        "                          'MACD_12_26_9']\n",
        "    },\n",
        "    \"build_type\":{\n",
        "        \"lstm_att_cv\":\"att\",\n",
        "        \"lstm_att_ohlcv\":\"att\",\n",
        "        \"lstm_xgb_cols\": \"att\"\n",
        "    },\n",
        "    \"load_type\": {\n",
        "      \"svm_cv\": \"joblib\",\n",
        "      \"svm_cv_vwap\": \"joblib\"\n",
        "    }\n",
        "}\n",
        "\n",
        "def pluck(nparray, all_columns, desired_columns):\n",
        "  df = pd.DataFrame(nparray, columns=all_columns)\n",
        "  return df[desired_columns].values\n",
        "\n",
        "def interactive_plot(df, title):\n",
        "  fig = px.line(title = title)\n",
        "  for i in df.columns[1:]:\n",
        "    fig.add_scatter(x = df['Date'], y = df[i], name = i)\n",
        "  fig.show()\n",
        "\n",
        "\n",
        "def scale_data(data):\n",
        "  # Scale the data\n",
        "  scaled_data = sc.fit_transform(data)\n",
        "  return scaled_data\n",
        "\n",
        "def sort_date(pric_df):\n",
        "  pric_df.reset_index()\n",
        "  pric_df = pric_df.sort_values(by = ['Date'])\n",
        "  return pric_df\n",
        "\n",
        "def append_price_dif(df):\n",
        "  df['Target'] = df['Close'].shift(-1)\n",
        "  df = df[:-1]\n",
        "  return df\n",
        "\n",
        "def show_plot(data, title):\n",
        "  plt.figure(figsize = (13, 5))\n",
        "  plt.plot(data, linewidth = 3)\n",
        "  plt.title(title)\n",
        "  plt.grid()\n",
        "\n",
        "def build_model_original(features, outcomes):\n",
        "  # Create the model\n",
        "  inputs = keras.layers.Input(shape=(features,outcomes))\n",
        "  x = keras.layers.LSTM(150, return_sequences= True)(inputs)\n",
        "  x = keras.layers.Dropout(0.3)(x)\n",
        "  x = keras.layers.LSTM(150, return_sequences=True)(x)\n",
        "  x = keras.layers.Dropout(0.3)(x)\n",
        "  x = keras.layers.LSTM(150)(x)\n",
        "  outputs = keras.layers.Dense(1, activation='linear')(x)\n",
        "\n",
        "  model = keras.Model(inputs=inputs, outputs=outputs)\n",
        "  model.compile(optimizer='adam', loss=\"mse\")\n",
        "  return model\n",
        "\n",
        "def build_model(features, outcomes):\n",
        "  # Set the dropout rate and weight decay coefficient\n",
        "  dropout_rate = 0.3\n",
        "  weight_decay = 1e-6\n",
        "\n",
        "  # Define the model\n",
        "  inputs = keras.layers.Input(shape=(features, outcomes))\n",
        "  x = keras.layers.LSTM(150, return_sequences=True)(inputs)\n",
        "  x = keras.layers.Dropout(dropout_rate)(x)\n",
        "  x = keras.layers.BatchNormalization()(x)\n",
        "  x = keras.layers.LSTM(150, return_sequences=True)(x)\n",
        "  x = keras.layers.Dropout(dropout_rate)(x)\n",
        "  x = keras.layers.BatchNormalization()(x)\n",
        "  x = keras.layers.LSTM(150)(x)\n",
        "  outputs = keras.layers.Dense(1, activation='linear', kernel_regularizer=l2(weight_decay))(x)\n",
        "\n",
        "  model = keras.Model(inputs=inputs, outputs=outputs)\n",
        "  model.compile(optimizer='adam', loss=\"mse\")\n",
        "  return model\n",
        "\n",
        "def build_attention_model(features, outcomes):\n",
        "  # Create the model\n",
        "  inputs = keras.layers.Input(shape=(features,outcomes))\n",
        "  x = keras.layers.LSTM(150, return_sequences= True)(inputs)\n",
        "  x = keras.layers.Dropout(0.3)(x)\n",
        "  x = keras.layers.BatchNormalization()(x)\n",
        "  x = keras.layers.LSTM(150, return_sequences=True)(x)\n",
        "  x = keras.layers.Dropout(0.3)(x)\n",
        "  x = keras.layers.BatchNormalization()(x)\n",
        "  x = keras.layers.LSTM(150)(x)\n",
        "  attention_layer = Attention()([x, x])\n",
        "  outputs = keras.layers.Dense(1, activation='linear')(x)\n",
        "  model = Model(inputs=inputs, outputs=outputs)\n",
        "  model.compile(optimizer='adam', loss=\"mse\")\n",
        "  return model\n",
        "\n",
        "def connect(url, params):\n",
        "  response = requests.get(url,params)\n",
        "  response.raise_for_status()\n",
        "  return response\n",
        "\n",
        "def coinbase_json_to_df(delta, product, granularity='86400'):\n",
        "  start_date = (datetime.today() - timedelta(seconds=delta*int(granularity))).isoformat()\n",
        "  end_date = datetime.now().isoformat()\n",
        "  # Please refer to the coinbase documentation on the expected parameters\n",
        "  params = {'start':start_date, 'end':end_date, 'granularity':granularity}\n",
        "  response = connect(COINBASE_PRODUCTS+'/' + product + '/candles', params)\n",
        "  response_text = response.text\n",
        "  df_history = pd.read_json(response_text)\n",
        "  # Add column names in line with the Coinbase Pro documentation\n",
        "  df_history.columns = ['time','low','high','open','close','volume']\n",
        "  df_history['time'] = [datetime.fromtimestamp(x) for x in df_history['time']]\n",
        "  return df_history\n",
        "\n",
        "def ku_coin_json_to_df(delta, product, granularity='86400'):\n",
        "  granularity = int(granularity)\n",
        "  start_date = (datetime.today() - timedelta(seconds=delta*granularity))\n",
        "  end_date = datetime.now()\n",
        "\n",
        "  # Please refer to the kucoin documentation on the expected parameters\n",
        "  params = {'startAt':int(start_date.timestamp()), 'endAt':int(end_date.timestamp()), 'type':gran_to_string(granularity), 'symbol':product}\n",
        "  response = connect(KUCOIN_CANDLES, params)\n",
        "  response_text = response.text\n",
        "  response_data = js.loads(response_text);\n",
        "  if (response_data[\"code\"] != \"200000\"):\n",
        "    raise Exception(\"Illegal response: \" + response_text)\n",
        "\n",
        "  df_history = pd.DataFrame(response_data[\"data\"])\n",
        "\n",
        "  # kucoin is weird in that they don't have candles for everything. IF we don't have the requested\n",
        "  # number of bars here, it throws off the whole algo. I don't want to try and project so we\n",
        "  # just won't trade those instruments\n",
        "  got_bars = len(df_history)\n",
        "  if ( got_bars < delta-1):\n",
        "    raise Exception(\"Requested:\" + str(delta) + \" bars \" + \" but only got:\" + str(got_bars))\n",
        "\n",
        "  df_history.columns = ['time','open','close','high','low','volume', 'amount']\n",
        "  df_history['time'] = [datetime.fromtimestamp(int(x)) for x in df_history['time']]\n",
        "  df_history['open'] = [float(x) for x in df_history['open']]\n",
        "  df_history['close'] = [float(x) for x in df_history['close']]\n",
        "  df_history['high'] = [float(x) for x in df_history['high']]\n",
        "  df_history['low'] = [float(x) for x in df_history['low']]\n",
        "  df_history['volume'] = [float(x) for x in df_history['volume']]\n",
        "  df_history['amount'] = [float(x) for x in df_history['amount']]\n",
        "  return df_history\n",
        "\n",
        "def gran_to_string(granularity):\n",
        "  #todo implement this actually\n",
        "  if granularity == 86400:\n",
        "    return \"1day\"\n",
        "  if granularity == 900:\n",
        "    return \"15min\"\n",
        "  raise Exception(\"Joe didn't implement a proper granularity to string. Lazy, lazy.\")\n",
        "\n",
        "def get_coin_data_frames(time, product, granularity='86400', feature_set = [\"Close\", \"Volume\"]):\n",
        "  if coin_base:\n",
        "    df_raw = coinbase_json_to_df(time, product, granularity)\n",
        "  else:\n",
        "    df_raw = ku_coin_json_to_df(time, product, granularity)\n",
        "\n",
        "  if len(df_raw.index) == 0:\n",
        "    raise Exception(\"No data for \" + product)\n",
        "    \n",
        "\n",
        "  df_raw = df_raw.rename(columns={\"time\":\"Date\", \"open\":\"Open\", \"high\":\"High\", \"low\":\"Low\", \"close\":\"Close\", \"volume\":\"Volume\"})\n",
        "  return df_raw\n",
        "\n",
        "def build_profit_estimate(predicted, df_btc_history, budget = 5000):\n",
        "  df_predicted_chart = pd.DataFrame();\n",
        "  df_predicted_chart[\"Date\"] = df_btc_history[\"Date\"]\n",
        "  df_predicted_chart[\"Predicted\"] = predicted\n",
        "  df_predicted_chart[\"Predicted-Target\"] = df_predicted_chart[\"Predicted\"].shift(-1)\n",
        "  df_predicted_chart[\"Predicted-Diff\"] = df_predicted_chart[\"Predicted-Target\"] - df_predicted_chart[\"Predicted\"]\n",
        "  df_predicted_chart[\"Should-Trade\"] = np.where(df_predicted_chart[\"Predicted-Diff\"] > 0, True, False)\n",
        "  df_predicted_chart[\"RealDiff\"] = df_btc_history['Target'] - df_btc_history['Close']\n",
        "  df_predicted_chart[\"Percent\"] = df_predicted_chart[\"RealDiff\"] / df_btc_history[\"Close\"]\n",
        "  df_predicted_chart[\"Profit\"] = np.where(df_predicted_chart[\"Should-Trade\"] > 0, df_predicted_chart[\"Percent\"] * budget, 0)\n",
        "  profit = df_predicted_chart[\"Profit\"].sum()\n",
        "  return [df_predicted_chart, profit]\n",
        "\n",
        "def debug_prediction_frame(predicted, df_history, df_history_scaled):\n",
        "  df_predicted_chart = pd.DataFrame();\n",
        "  df_predicted_chart[\"Date\"] = df_history[\"Date\"]\n",
        "  df_predicted_chart[\"Predicted\"] = predicted\n",
        "  df_predicted_chart[\"Original\"] = df_history_scaled[:,0]\n",
        "  #Trend\n",
        "  #df_predicted_chart[\"Original-Target\"] = df_history_scaled[:,2]\n",
        "  df_predicted_chart[\"Original-Target\"] = df_history_scaled[:,1]\n",
        "  df_predicted_chart[\"Target-Date\"] = df_predicted_chart[\"Date\"].shift(-1)\n",
        "  df_predicted_chart[\"Predicted-Diff\"] = df_predicted_chart[\"Predicted\"] - df_predicted_chart[\"Original\"]\n",
        "  df_predicted_chart[\"Actual-Diff\"] = df_predicted_chart[\"Original-Target\"] - df_predicted_chart[\"Original\"]\n",
        "  df_predicted_chart[\"Should-Trade\"] = np.where(df_predicted_chart[\"Predicted-Diff\"] > 0, True, False)\n",
        "  df_predicted_chart[\"Close\"] = df_history[\"Close\"]\n",
        "  df_predicted_chart[\"Target\"] = df_history[\"Target\"]\n",
        "  df_predicted_chart[\"RealDiff\"] = df_history[\"Diff\"]\n",
        "  df_predicted_chart[\"Percent\"] = df_predicted_chart[\"RealDiff\"] / df_predicted_chart[\"Close\"]\n",
        "  df_predicted_chart[\"Profit\"] = np.where(df_predicted_chart[\"Should-Trade\"] > 0, df_predicted_chart[\"Percent\"] * budget, 0)\n",
        "  return df_predicted_chart\n",
        "\n",
        "def get_all_products():\n",
        "  if coin_base:\n",
        "    return get_all_coinbase_products()\n",
        "\n",
        "  if ku_coin:\n",
        "    return get_all_kucoin_products()\n",
        "\n",
        "def get_all_kucoin_products():\n",
        "  response = connect(KUCOIN_PRODUCTS, {})\n",
        "  products = js.loads(response.text)\n",
        "  df_products = pd.DataFrame(products[\"data\"][\"ticker\"])\n",
        "  df_products = df_products.rename(columns={\"symbol\":\"id\"})\n",
        "  return df_products\n",
        "\n",
        "def get_all_coinbase_products():\n",
        "  response = connect(COINBASE_PRODUCTS, {})\n",
        "  response_text = response.text\n",
        "  df_products = pd.read_json(response_text)\n",
        "  return df_products\n",
        "\n",
        "def predict_trade(model, product, X, columns):\n",
        "  predicted = model.predict(X).flatten()\n",
        "  df_pred = pd.DataFrame(predicted, columns = [\"Close\"])\n",
        "  for column in gbl_all_columns:\n",
        "    if column != \"Close\":\n",
        "      df_pred[column] = 0\n",
        "  return [predicted, sc.inverse_transform(df_pred)[:,[0]].flatten()]\n",
        "\n",
        "def build_trade_model(predicted, predicted_scaled, df, product, name, mse):\n",
        "  # add predicted\n",
        "  df_trade = df\n",
        "  df_trade[\"Predicted_Scaled\"] = predicted_scaled\n",
        "  df_trade = df_trade.tail(1)\n",
        "\n",
        "  # add the product, derive a move and percent\n",
        "  df_trade[\"Predicted\"] = [predicted]\n",
        "  df_trade[\"Product\"] = product\n",
        "  df_trade[\"Model Name\"] = name\n",
        "  df_trade[\"Move\"] = df_trade[\"Predicted\"] - df_trade[\"Close\"]\n",
        "  df_trade[\"MSE\"] = mse\n",
        "  df_trade[\"Percent\"] = (df_trade[\"Move\"] / df_trade[\"Close\"]) * 100\n",
        "  df_trade[\"RawPercent\"] = df_trade[\"Move\"] / df_trade[\"Close\"]\n",
        "  df_trade[\"250Fees\"] = (250 * 0.004) * 2\n",
        "  df_trade[\"5kFees\"] = (5000 * 0.004) * 2\n",
        "  df_trade[\"10kFees\"] = (10000 * 0.0025) * 2\n",
        "  df_trade[\"250Profit\"] = (250 * df_trade[\"RawPercent\"]) - df_trade[\"250Fees\"]\n",
        "  df_trade[\"5kProfit\"] = (5000 * df_trade[\"RawPercent\"]) - df_trade[\"5kFees\"]\n",
        "  df_trade[\"10k0Profit\"] = (10000 * df_trade[\"RawPercent\"]) - df_trade[\"10kFees\"]\n",
        "  \n",
        "  return df_trade\n",
        "\n",
        "\n",
        "def convert_to_training_dataset(df, columns):\n",
        "  df = sort_date(df)\n",
        "  target_df = attach_technicals(df)\n",
        "  target_df = append_price_dif(target_df)\n",
        "  features = target_df[gbl_all_columns]\n",
        "  scaled_features = scale_data(features)\n",
        "\n",
        "  X = pluck(scaled_features, gbl_all_columns, columns)\n",
        "  y = pluck(scaled_features, gbl_all_columns, gbl_target_column)\n",
        "\n",
        "  return [scaled_features, X, y, features]\n",
        "\n",
        "def get_group_bars(df):\n",
        "  df = pd.DataFrame(sc.fit_transform(df[[\"Close\", \"Volume\"]]), columns=[\"Close\",\"Volume\"])\n",
        "  # Split into input sequences and target values\n",
        "  n_steps = 4*4  # 4 hours of data at 15 minute intervals\n",
        "  X = []\n",
        "  Y = []\n",
        "  for i in range(0, len(df), n_steps):\n",
        "    df_group = df.iloc[i:i+n_steps]\n",
        "    if len(df_group) != n_steps:\n",
        "      continue\n",
        "    X.append(np.array(df_group.values))\n",
        "    Y.append(df_group.values[-1,0])\n",
        "\n",
        "  # Convert the lists to NumPy arrays\n",
        "  X = np.array(X)\n",
        "  Y = np.array(Y)\n",
        "  return [X, Y]\n",
        "\n",
        "def getTrainingVanilla15mSet(ticker):\n",
        "  file_path = data_path + \"/\" + ticker + \"-15.csv\"\n",
        "  df = sort_date(pd.read_csv(file_path).rename(columns={\"Datetime\":\"Date\"}))\n",
        "  df['Date'] = pd.to_datetime(df['Date'])\n",
        "  return get_group_bars(df)\n",
        "\n",
        "\n",
        "def build_15m_model(getTrainingSet=getTrainingVanilla15mSet):\n",
        "  # the 15 min bar model\n",
        "  # Build the model\n",
        "  group_size = 4*4\n",
        "  features = 2\n",
        "  model15 = build_model(group_size, features)\n",
        "\n",
        "  # Compile the model\n",
        "  model15.compile(loss='mean_squared_error', optimizer='adam')\n",
        "\n",
        "  early_stopping = EarlyStopping(monitor='val_loss', patience=5)\n",
        "  for ticker in tickers:\n",
        "    [X,Y] = getTrainingSet(ticker)\n",
        "    model15.fit(X, Y,\n",
        "      epochs = 200,\n",
        "      validation_split = 0.2,\n",
        "      callbacks=[early_stopping])\n",
        "  return model15\n",
        "\n",
        "def attachVWAPS(df, length=30):\n",
        "  vwaps = df\n",
        "  vwaps.set_index(pd.DatetimeIndex(vwaps[\"Date\"]), inplace=True)\n",
        "  vwaps[\"VWAP\"] = df.ta.vwap(length=length)\n",
        "  vwaps = vwaps.dropna(subset=[\"VWAP\"])\n",
        "  vwaps['VWAPD'] = vwaps['Close'] - vwaps['VWAP']\n",
        "  return vwaps\n",
        "\n",
        "def attachRVI(df):\n",
        "  vol_df = df\n",
        "  vol_df[\"RVI\"] = df.ta.rvi()\n",
        "  return vol_df.fillna(0)\n",
        "\n",
        "def get_raw_data_for_coin_list(days, coins = [\"FCON-USDT\", \"GMT3L-USDT\", \"NEAR3L-USDT\", \"H2O-USDT\", \"DOGE3L-USDT\", \"DOGE3S-USDT\"]):\n",
        "  coin_dfs = {}\n",
        "  for coin in coins:\n",
        "    print(\"coin: \", coin)\n",
        "    coin_dfs[coin] = get_coin_data_frames(days, coin)\n",
        "    time.sleep(1)\n",
        "  return coin_dfs\n",
        "\n",
        "def quick_train(df, model):\n",
        "  [scaled_features, X, y, normal_features] = convert_to_training_dataset(df, columns=[\"Close\", \"Volume\", \"Target\"])  \n",
        "  history = model.fit(X, y)\n",
        "  return model\n",
        "\n",
        "def predict_config_model_for_product(df_raw, name, product):\n",
        "  columns = model_config[\"column_sets\"][name]+gbl_target_column\n",
        "\n",
        "  [scaled_features, X, y, normal_features] = convert_to_training_dataset(df_raw, columns)  \n",
        "  [predictions_scaled, predictions] = predict_trade(models[name], product, X, columns)\n",
        "  \n",
        "  scaled_close = pluck(X, columns, [\"Close\"])\n",
        "  mse = mean_squared_error(scaled_close, predictions_scaled)\n",
        "  \n",
        "  prediction = predictions[-1]\n",
        "  \n",
        "  return build_trade_model(prediction, predictions_scaled, normal_features, product, name, mse)\n",
        "\n",
        "def fetch_and_predict_short_term(model, product):\n",
        "  if coin_base:\n",
        "    df_raw = coinbase_json_to_df(16, product, 900)\n",
        "  else:\n",
        "    df_raw = ku_coin_json_to_df(16, product, 900)\n",
        "  df_raw = df_raw.rename(columns={\"close\":\"Close\", \"volume\": \"Volume\"})\n",
        "  [X, Y] = get_group_bars(df_raw[[\"Close\", \"Volume\"]])\n",
        "  predicted = model.predict(X)\n",
        "  df_pred = pd.DataFrame(predicted, columns = [\"Close\"])\n",
        "  df_pred[\"Volume\"] = 0\n",
        "  return [predicted.flatten()[0], sc.inverse_transform(df_pred).flatten()[0]]\n",
        "\n",
        "def consensus_percent(df):\n",
        "  return mse_weighted_average(df, \"Percent\")\n",
        "\n",
        "\n",
        "def consensus_prediction(df):\n",
        "  return mse_weighted_average(df, \"Predicted\")\n",
        "\n",
        "def mse_weighted_average(df, column):\n",
        "    mse = df[\"MSE\"]\n",
        "    predictions = df[column]\n",
        "    weights = np.array([1/(np.sqrt(mse)) for mse in mse])\n",
        "    weights = weights / np.sum(weights)\n",
        "    weighted_average = np.dot(predictions, weights)\n",
        "    return weighted_average\n",
        "\n",
        "def consensus_overall(df):\n",
        "  return (1 - (df['Predicted'].std()/df['Predicted'].mean())) * 100\n",
        "\n",
        "\n",
        "# attach candle patterns\n",
        "def attach_technicals(df_raw):\n",
        "  df_raw = attachVWAPS(df_raw)\n",
        "  df_raw = attachRVI(df_raw)\n",
        "  df_candles = df_raw.ta.cdl_pattern(name=[\"engulfing\", \"harami\", \"haramicross\", \"piercing\", \"darkcloudcover\", \"hammer\", \"invertedhammer\"])\n",
        "  macds = df_raw.ta.macd()\n",
        "  rsi = df_raw.ta.rsi()\n",
        "  stoch = df_raw.ta.stoch()\n",
        "  bbands = df_raw.ta.bbands()\n",
        "  obv = df_raw.ta.obv()\n",
        "  ad = df_raw.ta.ad()\n",
        "  mfi = df_raw.ta.mfi()\n",
        "  willr = df_raw.ta.willr()\n",
        "  df_final = pd.concat([df_raw, df_candles, macds, rsi, stoch, bbands, obv, ad, mfi, willr], axis=1)\n",
        "  df_final = df_final.fillna(0)\n",
        "  return df_final\n",
        "\n",
        "def build_model_from_config(models, name, config):    \n",
        "  features = config[\"column_sets\"][name]\n",
        "  num_features = len(features)\n",
        "  print(\"build model:\", name, \" features:\", num_features)\n",
        "  if name in config[\"build_type\"] and config[\"build_type\"][name] == \"att\":\n",
        "    models[name] = build_attention_model(num_features, 1)\n",
        "  if name in config[\"build_type\"] and config[\"build_type\"][name] == \"svm\":\n",
        "    models[name] = SVR(kernel='rbf')\n",
        "  else:\n",
        "    models[name] = build_model(num_features, 1)\n",
        "\n",
        "def train_config_model_against_df(name, config, df):\n",
        "  features = config[\"column_sets\"][name]\n",
        "  num_features = len(features)\n",
        "  print(\"Training model:\", name, \" features:\", num_features)\n",
        "  [scaled_features, X, y, normal_features] = convert_to_training_dataset(df, columns=features)  \n",
        "  model = models[name]\n",
        "\n",
        "  # Create an early stopping callback\n",
        "  early_stopping = EarlyStopping(monitor='val_loss', patience=15)\n",
        "\n",
        "  history = model.fit(\n",
        "      X, y,\n",
        "      epochs = 200,\n",
        "      validation_split = 0.2,\n",
        "      callbacks=[early_stopping]\n",
        "  )\n",
        "  return [model, history]    \n",
        "\n",
        "def train_model(model, X, y):\n",
        "\n",
        "  # Reshape the 1D arrays to 3D arrays to feed in the model\n",
        "  X_train = np.reshape(X, (X.shape[0], X.shape[1], 1))\n",
        "\n",
        "  # Create an early stopping callback\n",
        "  early_stopping = EarlyStopping(monitor='val_loss', patience=15)\n",
        "\n",
        "  history = model.fit(\n",
        "      X_train, y,\n",
        "      epochs = 200,\n",
        "      validation_split = 0.2,\n",
        "      callbacks=[early_stopping]\n",
        "  )\n",
        "  return [model, history]\n",
        "\n",
        "def build_and_stash_all_config_models():\n",
        "  training_filter = model_config[\"training_filter\"]\n",
        "  for name in model_config[\"day_bar_models\"]:\n",
        "    if (len(training_filter)!=0 and not name in training_filter):\n",
        "      print(\"skipping: \", name)\n",
        "      continue\n",
        "    build_model_from_config(models, name, model_config)\n",
        "\n",
        "def get_training_datasets_for_model(name):\n",
        "  if not name in model_config[\"training_types\"]:\n",
        "    print(name, \" should train on stocks\")\n",
        "    return all_stock_dfs\n",
        "  training_type = model_config[\"training_types\"][name]\n",
        "  if (training_type == \"coin\"):\n",
        "    print(name, \" should train on coins\")\n",
        "    return all_coin_dfs\n",
        "  if (training_type == \"stocks\"):\n",
        "    print(name, \" should train on stocks\")\n",
        "    return all_stock_dfs\n",
        "  if (training_type == \"all\"):\n",
        "    print(name, \" should train on everything\")\n",
        "    return all_stock_dfs + all_coin_dfs\n",
        "  raise Exception(\"Bad name or config error:\" + name + \" type:\" + training_type)\n",
        "\n",
        "me = '/content/drive/My Drive/ml-trde-notebooks/common.ipynb'\n",
        "print(\"Modified\")\n",
        "print(os.stat(me)[-2])\n",
        "print(os.stat(me).st_mtime)\n",
        "print(os.path.getmtime(me))"
      ]
    }
  ]
}