actor_critic_duel_agent.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "actor-critic-duel agent.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "vzcHHaSQuwQ1",
        "colab_type": "code",
        "outputId": "bbfad86e-6f6e-4f44-9d5b-dce33d97948e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 63
        }
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "sns.set()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T2G6_QOz6m6H",
        "colab_type": "code",
        "outputId": "cd878ea4-ff6e-41c5-f7b9-a3efcd484e70",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "df = pd.read_csv('EURUSD_H1_2010-2019.csv',encoding='utf-16',sep='\\t')\n",
        "df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Symbol</th>\n",
              "      <th>Timeframe</th>\n",
              "      <th>Date</th>\n",
              "      <th>Seconds</th>\n",
              "      <th>Open</th>\n",
              "      <th>High</th>\n",
              "      <th>Low</th>\n",
              "      <th>Close</th>\n",
              "      <th>Volume</th>\n",
              "      <th>Ticks</th>\n",
              "      <th>Spread</th>\n",
              "      <th>FX_PLSTM_Indicator_v1.09 B#0</th>\n",
              "      <th>FX_PLSTM_Indicator_v1.09 B#1</th>\n",
              "      <th>FX_PLSTM_Indicator_v1.09 B#2</th>\n",
              "      <th>FX_PLSTM_Indicator_v1.09 B#3</th>\n",
              "      <th>FX_PLSTM_Indicator_v1.09 B#4</th>\n",
              "      <th>FX_PLSTM_Indicator_v1.09 B#5</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>EURUSD</td>\n",
              "      <td>H1</td>\n",
              "      <td>01-10-10</td>\n",
              "      <td>19:00:00</td>\n",
              "      <td>1.37531</td>\n",
              "      <td>1.37571</td>\n",
              "      <td>1.37389</td>\n",
              "      <td>1.37422</td>\n",
              "      <td>0</td>\n",
              "      <td>1812</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>EURUSD</td>\n",
              "      <td>H1</td>\n",
              "      <td>01-10-10</td>\n",
              "      <td>20:00:00</td>\n",
              "      <td>1.37416</td>\n",
              "      <td>1.37656</td>\n",
              "      <td>1.37413</td>\n",
              "      <td>1.37638</td>\n",
              "      <td>0</td>\n",
              "      <td>1636</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>EURUSD</td>\n",
              "      <td>H1</td>\n",
              "      <td>01-10-10</td>\n",
              "      <td>21:00:00</td>\n",
              "      <td>1.37636</td>\n",
              "      <td>1.37786</td>\n",
              "      <td>1.37613</td>\n",
              "      <td>1.37782</td>\n",
              "      <td>0</td>\n",
              "      <td>1476</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>EURUSD</td>\n",
              "      <td>H1</td>\n",
              "      <td>01-10-10</td>\n",
              "      <td>22:00:00</td>\n",
              "      <td>1.37780</td>\n",
              "      <td>1.37828</td>\n",
              "      <td>1.37740</td>\n",
              "      <td>1.37785</td>\n",
              "      <td>0</td>\n",
              "      <td>1068</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>EURUSD</td>\n",
              "      <td>H1</td>\n",
              "      <td>01-10-10</td>\n",
              "      <td>23:00:00</td>\n",
              "      <td>1.37791</td>\n",
              "      <td>1.37915</td>\n",
              "      <td>1.37771</td>\n",
              "      <td>1.37907</td>\n",
              "      <td>0</td>\n",
              "      <td>855</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Symbol Timeframe  ... FX_PLSTM_Indicator_v1.09 B#4 FX_PLSTM_Indicator_v1.09 B#5\n",
              "0  EURUSD        H1  ...                            0                            0\n",
              "1  EURUSD        H1  ...                            0                            0\n",
              "2  EURUSD        H1  ...                            0                            0\n",
              "3  EURUSD        H1  ...                            0                            0\n",
              "4  EURUSD        H1  ...                            0                            0\n",
              "\n",
              "[5 rows x 17 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LCfQ-BfL6ty0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import deque\n",
        "import random\n",
        "\n",
        "class Actor:\n",
        "    def __init__(self, name, input_size, output_size, size_layer):\n",
        "        with tf.variable_scope(name):\n",
        "            self.X = tf.placeholder(tf.float32, (None, input_size))\n",
        "            feed_actor = tf.layers.dense(self.X, size_layer, activation = tf.nn.relu)\n",
        "            tensor_action, tensor_validation = tf.split(feed_actor,2,1)\n",
        "            feed_action = tf.layers.dense(tensor_action, output_size)\n",
        "            feed_validation = tf.layers.dense(tensor_validation, 1)\n",
        "            self.logits = feed_validation + tf.subtract(feed_action,\n",
        "                                                        tf.reduce_mean(feed_action,axis=1,keep_dims=True))\n",
        "\n",
        "class Critic:\n",
        "    def __init__(self, name, input_size, output_size, size_layer, learning_rate):\n",
        "        with tf.variable_scope(name):\n",
        "            self.X = tf.placeholder(tf.float32, (None, input_size))\n",
        "            self.Y = tf.placeholder(tf.float32, (None, output_size))\n",
        "            self.REWARD = tf.placeholder(tf.float32, (None, 1))\n",
        "            feed_critic = tf.layers.dense(self.X, size_layer, activation = tf.nn.relu)\n",
        "            tensor_action, tensor_validation = tf.split(feed_critic,2,1)\n",
        "            feed_action = tf.layers.dense(tensor_action, output_size)\n",
        "            feed_validation = tf.layers.dense(tensor_validation, 1)\n",
        "            feed_critic = feed_validation + tf.subtract(feed_action,tf.reduce_mean(feed_action,axis=1,keep_dims=True))\n",
        "            feed_critic = tf.nn.relu(feed_critic) + self.Y\n",
        "            feed_critic = tf.layers.dense(feed_critic, size_layer//2, activation = tf.nn.relu)\n",
        "            self.logits = tf.layers.dense(feed_critic, 1)\n",
        "            self.cost = tf.reduce_mean(tf.square(self.REWARD - self.logits))\n",
        "            self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
        "            \n",
        "class Agent:\n",
        "\n",
        "    LEARNING_RATE = 0.001\n",
        "    BATCH_SIZE = 32\n",
        "    LAYER_SIZE = 256\n",
        "    OUTPUT_SIZE = 3\n",
        "    EPSILON = 0.5\n",
        "    DECAY_RATE = 0.005\n",
        "    MIN_EPSILON = 0.1\n",
        "    GAMMA = 0.99\n",
        "    MEMORIES = deque()\n",
        "    MEMORY_SIZE = 300\n",
        "    COPY = 1000\n",
        "    T_COPY = 0\n",
        "\n",
        "    def __init__(self, state_size, window_size, trend, skip):\n",
        "        self.state_size = state_size\n",
        "        self.window_size = window_size\n",
        "        self.half_window = window_size // 2\n",
        "        self.trend = trend\n",
        "        self.skip = skip\n",
        "        tf.reset_default_graph()\n",
        "        self.actor = Actor('actor-original', self.state_size, self.OUTPUT_SIZE, self.LAYER_SIZE)\n",
        "        self.actor_target = Actor('actor-target', self.state_size, self.OUTPUT_SIZE, self.LAYER_SIZE)\n",
        "        self.critic = Critic('critic-original', self.state_size, self.OUTPUT_SIZE, self.LAYER_SIZE, self.LEARNING_RATE)\n",
        "        self.critic_target = Critic('critic-target', self.state_size, self.OUTPUT_SIZE, \n",
        "                                    self.LAYER_SIZE, self.LEARNING_RATE)\n",
        "        self.grad_critic = tf.gradients(self.critic.logits, self.critic.Y)\n",
        "        self.actor_critic_grad = tf.placeholder(tf.float32, [None, self.OUTPUT_SIZE])\n",
        "        weights_actor = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='actor')\n",
        "        self.grad_actor = tf.gradients(self.actor.logits, weights_actor, -self.actor_critic_grad)\n",
        "        grads = zip(self.grad_actor, weights_actor)\n",
        "        self.optimizer = tf.train.AdamOptimizer(self.LEARNING_RATE).apply_gradients(grads)\n",
        "        self.sess = tf.InteractiveSession()\n",
        "        self.sess.run(tf.global_variables_initializer())\n",
        "    \n",
        "    def _assign(self, from_name, to_name):\n",
        "        from_w = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=from_name)\n",
        "        to_w = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=to_name)\n",
        "        for i in range(len(from_w)):\n",
        "            assign_op = to_w[i].assign(from_w[i])\n",
        "            self.sess.run(assign_op)\n",
        "            \n",
        "    def _memorize(self, state, action, reward, new_state, dead):\n",
        "        self.MEMORIES.append((state, action, reward, new_state, dead))\n",
        "        if len(self.MEMORIES) > self.MEMORY_SIZE:\n",
        "            self.MEMORIES.popleft()\n",
        "            \n",
        "    def _select_action(self, state):\n",
        "        if np.random.rand() < self.EPSILON:\n",
        "            action = np.random.randint(self.OUTPUT_SIZE)\n",
        "        else:\n",
        "            prediction = self.sess.run(self.actor.logits, feed_dict={self.actor.X:[state]})[0]\n",
        "            action = np.argmax(prediction)\n",
        "        return action\n",
        "    \n",
        "    def _construct_memories_and_train(self, replay):\n",
        "        states = np.array([a[0] for a in replay])\n",
        "        new_states = np.array([a[3] for a in replay])\n",
        "        Q = self.sess.run(self.actor.logits, feed_dict={self.actor.X: states})\n",
        "        Q_target = self.sess.run(self.actor_target.logits, feed_dict={self.actor_target.X: states})\n",
        "        grads = self.sess.run(self.grad_critic, feed_dict={self.critic.X:states, self.critic.Y:Q})[0]\n",
        "        self.sess.run(self.optimizer, feed_dict={self.actor.X:states, self.actor_critic_grad:grads})\n",
        "        \n",
        "        rewards = np.array([a[2] for a in replay]).reshape((-1, 1))\n",
        "        rewards_target = self.sess.run(self.critic_target.logits, \n",
        "                                       feed_dict={self.critic_target.X:new_states,self.critic_target.Y:Q_target})\n",
        "        for i in range(len(replay)):\n",
        "            if not replay[0][-1]:\n",
        "                rewards[i] += self.GAMMA * rewards_target[i]\n",
        "        cost, _ = self.sess.run([self.critic.cost, self.critic.optimizer], \n",
        "                                feed_dict={self.critic.X:states, self.critic.Y:Q, self.critic.REWARD:rewards})\n",
        "        return cost\n",
        "    \n",
        "    def get_state(self, t):\n",
        "        window_size = self.window_size + 1\n",
        "        d = t - window_size + 1\n",
        "        block = self.trend[d : t + 1] if d >= 0 else -d * [self.trend[0]] + self.trend[0 : t + 1]\n",
        "        res = []\n",
        "        for i in range(window_size - 1):\n",
        "            res.append(block[i + 1] - block[i])\n",
        "        return np.array(res)\n",
        "    \n",
        "    def buy(self, initial_money):\n",
        "        starting_money = initial_money\n",
        "        states_sell = []\n",
        "        states_buy = []\n",
        "        inventory = []\n",
        "        state = self.get_state(0)\n",
        "        for t in range(0, len(self.trend) - 1, self.skip):\n",
        "            action = self._select_action(state)\n",
        "            next_state = self.get_state(t + 1)\n",
        "            \n",
        "            if action == 1 and initial_money >= self.trend[t]:\n",
        "                inventory.append(self.trend[t])\n",
        "                initial_money -= self.trend[t]\n",
        "                states_buy.append(t)\n",
        "                print('day %d: buy 1 unit at price %f, total balance %f'% (t, self.trend[t], initial_money))\n",
        "            \n",
        "            elif action == 2 and len(inventory):\n",
        "                bought_price = inventory.pop(0)\n",
        "                initial_money += self.trend[t]\n",
        "                states_sell.append(t)\n",
        "                try:\n",
        "                    invest = ((close[t] - bought_price) / bought_price) * 100\n",
        "                except:\n",
        "                    invest = 0\n",
        "                print(\n",
        "                    'day %d, sell 1 unit at price %f, investment %f %%, total balance %f,'\n",
        "                    % (t, close[t], invest, initial_money)\n",
        "                )\n",
        "            \n",
        "            state = next_state\n",
        "        invest = ((initial_money - starting_money) / starting_money) * 100\n",
        "        total_gains = initial_money - starting_money\n",
        "        return states_buy, states_sell, total_gains, invest\n",
        "    \n",
        "    def train(self, iterations, checkpoint, initial_money):\n",
        "        for i in range(iterations):\n",
        "            print(i);\n",
        "            total_profit = 0\n",
        "            inventory = []\n",
        "            state = self.get_state(0)\n",
        "            starting_money = initial_money\n",
        "            for t in range(0, len(self.trend) - 1, self.skip):\n",
        "                if (self.T_COPY + 1) % self.COPY == 0:\n",
        "                    self._assign('actor-original', 'actor-target')\n",
        "                    self._assign('critic-original', 'critic-target')\n",
        "                \n",
        "                action = self._select_action(state)\n",
        "                next_state = self.get_state(t + 1)\n",
        "                \n",
        "                if action == 1 and starting_money >= self.trend[t]:\n",
        "                    inventory.append(self.trend[t])\n",
        "                    starting_money -= self.trend[t]\n",
        "                \n",
        "                elif action == 2 and len(inventory) > 0:\n",
        "                    bought_price = inventory.pop(0)\n",
        "                    total_profit += self.trend[t] - bought_price\n",
        "                    starting_money += self.trend[t]\n",
        "                    \n",
        "                invest = ((starting_money - initial_money) / initial_money)\n",
        "                # print(\"r\")\n",
        "                self._memorize(state, action, invest, next_state, starting_money < initial_money)\n",
        "                batch_size = min(len(self.MEMORIES), self.BATCH_SIZE)\n",
        "                state = next_state\n",
        "                replay = random.sample(self.MEMORIES, batch_size)\n",
        "                cost = self._construct_memories_and_train(replay)\n",
        "                self.T_COPY += 1\n",
        "                self.EPSILON = self.MIN_EPSILON + (1.0 - self.MIN_EPSILON) * np.exp(-self.DECAY_RATE * i)\n",
        "            if (i+1) % checkpoint == 0:\n",
        "                print('epoch: %d, total rewards: %f.3, cost: %f, total money: %f'%(i + 1, total_profit, cost,\n",
        "                                                                                  starting_money))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UW9utUZ761gr",
        "colab_type": "code",
        "outputId": "22fe8305-6ae4-40c1-8d66-7f06dd26976f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "close = df.Close.values.tolist()[:5000]\n",
        "initial_money = 1000\n",
        "window_size = 30\n",
        "skip = 1\n",
        "batch_size = 32\n",
        "agent = Agent(state_size = window_size, \n",
        "              window_size = window_size, \n",
        "              trend = close, \n",
        "              skip = skip)\n",
        "agent.train(iterations = 200, checkpoint = 1, initial_money = initial_money)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-2-cf9619bde11f>:8: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.Dense instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `layer.__call__` method instead.\n",
            "0\n",
            "epoch: 1, total rewards: 2.990380.3, cost: 0.000030, total money: 940.604180\n",
            "1\n",
            "epoch: 2, total rewards: -1.431450.3, cost: 0.000393, total money: 968.718330\n",
            "2\n",
            "epoch: 3, total rewards: 6.238010.3, cost: 0.000203, total money: 965.041390\n",
            "3\n",
            "epoch: 4, total rewards: 2.205320.3, cost: 0.000046, total money: 888.794020\n",
            "4\n",
            "epoch: 5, total rewards: 1.162570.3, cost: 0.000067, total money: 978.417390\n",
            "5\n",
            "epoch: 6, total rewards: -0.052700.3, cost: 0.000030, total money: 925.058880\n",
            "6\n",
            "epoch: 7, total rewards: -2.423030.3, cost: 0.000062, total money: 979.063550\n",
            "7\n",
            "epoch: 8, total rewards: -0.001320.3, cost: 0.000084, total money: 951.916590\n",
            "8\n",
            "epoch: 9, total rewards: 1.498600.3, cost: 0.000046, total money: 953.240800\n",
            "9\n",
            "epoch: 10, total rewards: -0.916860.3, cost: 0.000023, total money: 952.291870\n",
            "10\n",
            "epoch: 11, total rewards: 4.741500.3, cost: 0.000250, total money: 940.957420\n",
            "11\n",
            "epoch: 12, total rewards: 1.229320.3, cost: 0.000268, total money: 849.681660\n",
            "12\n",
            "epoch: 13, total rewards: -0.124710.3, cost: 0.000041, total money: 957.407270\n",
            "13\n",
            "epoch: 14, total rewards: 7.148470.3, cost: 0.000023, total money: 838.861970\n",
            "14\n",
            "epoch: 15, total rewards: 1.456190.3, cost: 0.000081, total money: 935.051590\n",
            "15\n",
            "epoch: 16, total rewards: 2.481870.3, cost: 0.000018, total money: 951.489370\n",
            "16\n",
            "epoch: 17, total rewards: 3.040490.3, cost: 0.000054, total money: 833.035000\n",
            "17\n",
            "epoch: 18, total rewards: 1.253250.3, cost: 0.000286, total money: 944.684210\n",
            "18\n",
            "epoch: 19, total rewards: -1.318740.3, cost: 0.000075, total money: 939.130410\n",
            "19\n",
            "epoch: 20, total rewards: 1.618820.3, cost: 0.000106, total money: 994.463200\n",
            "20\n",
            "epoch: 21, total rewards: -1.977540.3, cost: 0.000012, total money: 938.471360\n",
            "21\n",
            "epoch: 22, total rewards: 4.833720.3, cost: 0.000024, total money: 987.716360\n",
            "22\n",
            "epoch: 23, total rewards: 0.709430.3, cost: 0.000100, total money: 986.465970\n",
            "23\n",
            "epoch: 24, total rewards: 2.103410.3, cost: 0.000031, total money: 989.274850\n",
            "24\n",
            "epoch: 25, total rewards: -0.303940.3, cost: 0.000018, total money: 934.560410\n",
            "25\n",
            "epoch: 26, total rewards: 6.437720.3, cost: 0.000015, total money: 959.718330\n",
            "26\n",
            "epoch: 27, total rewards: 0.194140.3, cost: 0.000026, total money: 966.167080\n",
            "27\n",
            "epoch: 28, total rewards: 1.260770.3, cost: 0.000039, total money: 961.562500\n",
            "28\n",
            "epoch: 29, total rewards: 1.591970.3, cost: 0.000009, total money: 994.444330\n",
            "29\n",
            "epoch: 30, total rewards: 0.940510.3, cost: 0.000038, total money: 959.875350\n",
            "30\n",
            "epoch: 31, total rewards: 0.104650.3, cost: 0.000060, total money: 928.043370\n",
            "31\n",
            "epoch: 32, total rewards: 0.059450.3, cost: 0.000109, total money: 970.191120\n",
            "32\n",
            "epoch: 33, total rewards: -0.903710.3, cost: 0.000144, total money: 979.197650\n",
            "33\n",
            "epoch: 34, total rewards: 3.527430.3, cost: 0.000063, total money: 895.820620\n",
            "34\n",
            "epoch: 35, total rewards: 3.899300.3, cost: 0.000027, total money: 978.353910\n",
            "35\n",
            "epoch: 36, total rewards: -0.065710.3, cost: 0.000196, total money: 953.159690\n",
            "36\n",
            "epoch: 37, total rewards: -0.481680.3, cost: 0.000048, total money: 920.250670\n",
            "37\n",
            "epoch: 38, total rewards: 1.289720.3, cost: 0.000024, total money: 957.392900\n",
            "38\n",
            "epoch: 39, total rewards: 4.256200.3, cost: 0.000052, total money: 876.348560\n",
            "39\n",
            "epoch: 40, total rewards: 1.421910.3, cost: 0.000188, total money: 961.774170\n",
            "40\n",
            "epoch: 41, total rewards: 0.697600.3, cost: 0.000047, total money: 990.691820\n",
            "41\n",
            "epoch: 42, total rewards: 1.659500.3, cost: 0.000043, total money: 966.252070\n",
            "42\n",
            "epoch: 43, total rewards: -0.281490.3, cost: 0.000015, total money: 955.855790\n",
            "43\n",
            "epoch: 44, total rewards: -2.899430.3, cost: 0.000037, total money: 895.024970\n",
            "44\n",
            "epoch: 45, total rewards: -2.165430.3, cost: 0.000098, total money: 938.333420\n",
            "45\n",
            "epoch: 46, total rewards: 0.759350.3, cost: 0.000022, total money: 959.653580\n",
            "46\n",
            "epoch: 47, total rewards: 2.319890.3, cost: 0.000136, total money: 916.072810\n",
            "47\n",
            "epoch: 48, total rewards: -2.012820.3, cost: 0.000031, total money: 895.904300\n",
            "48\n",
            "epoch: 49, total rewards: 6.755580.3, cost: 0.000012, total money: 906.184370\n",
            "49\n",
            "epoch: 50, total rewards: 3.848440.3, cost: 0.000022, total money: 917.562250\n",
            "50\n",
            "epoch: 51, total rewards: 1.325380.3, cost: 0.000086, total money: 932.090610\n",
            "51\n",
            "epoch: 52, total rewards: 0.287280.3, cost: 0.000025, total money: 974.736580\n",
            "52\n",
            "epoch: 53, total rewards: 0.341010.3, cost: 0.000070, total money: 946.566320\n",
            "53\n",
            "epoch: 54, total rewards: -0.293430.3, cost: 0.000037, total money: 982.678940\n",
            "54\n",
            "epoch: 55, total rewards: 1.352310.3, cost: 0.000303, total money: 997.056390\n",
            "55\n",
            "epoch: 56, total rewards: 1.705090.3, cost: 0.000025, total money: 974.649940\n",
            "56\n",
            "epoch: 57, total rewards: 0.425780.3, cost: 0.000032, total money: 984.744590\n",
            "57\n",
            "epoch: 58, total rewards: 1.071210.3, cost: 0.000129, total money: 962.839610\n",
            "58\n",
            "epoch: 59, total rewards: -0.509850.3, cost: 0.000059, total money: 938.720500\n",
            "59\n",
            "epoch: 60, total rewards: 0.963480.3, cost: 0.000061, total money: 921.827330\n",
            "60\n",
            "epoch: 61, total rewards: 0.253740.3, cost: 0.000057, total money: 866.248270\n",
            "61\n",
            "epoch: 62, total rewards: 9.633680.3, cost: 0.000024, total money: 796.999560\n",
            "62\n",
            "epoch: 63, total rewards: 3.491980.3, cost: 0.000006, total money: 889.921510\n",
            "63\n",
            "epoch: 64, total rewards: 8.704990.3, cost: 0.000156, total money: 887.637350\n",
            "64\n",
            "epoch: 65, total rewards: 1.182100.3, cost: 0.000041, total money: 1001.182100\n",
            "65\n",
            "epoch: 66, total rewards: 3.356320.3, cost: 0.000013, total money: 896.909420\n",
            "66\n",
            "epoch: 67, total rewards: 2.906810.3, cost: 0.000099, total money: 973.061670\n",
            "67\n",
            "epoch: 68, total rewards: 3.207590.3, cost: 0.000016, total money: 919.740130\n",
            "68\n",
            "epoch: 69, total rewards: 2.252390.3, cost: 0.000037, total money: 958.284610\n",
            "69\n",
            "epoch: 70, total rewards: 1.438920.3, cost: 0.000015, total money: 907.959150\n",
            "70\n",
            "epoch: 71, total rewards: 0.842080.3, cost: 0.000099, total money: 982.375680\n",
            "71\n",
            "epoch: 72, total rewards: -0.348280.3, cost: 0.000025, total money: 892.943140\n",
            "72\n",
            "epoch: 73, total rewards: -1.051410.3, cost: 0.000100, total money: 973.398910\n",
            "73\n",
            "epoch: 74, total rewards: 4.055500.3, cost: 0.000061, total money: 945.910030\n",
            "74\n",
            "epoch: 75, total rewards: 1.165320.3, cost: 0.000007, total money: 972.829100\n",
            "75\n",
            "epoch: 76, total rewards: 2.824820.3, cost: 0.000053, total money: 998.529790\n",
            "76\n",
            "epoch: 77, total rewards: -1.772790.3, cost: 0.000009, total money: 993.927770\n",
            "77\n",
            "epoch: 78, total rewards: 2.940010.3, cost: 0.000020, total money: 954.778820\n",
            "78\n",
            "epoch: 79, total rewards: -0.762810.3, cost: 0.000010, total money: 985.003920\n",
            "79\n",
            "epoch: 80, total rewards: 2.091780.3, cost: 0.000021, total money: 928.444900\n",
            "80\n",
            "epoch: 81, total rewards: 1.448330.3, cost: 0.000040, total money: 922.024110\n",
            "81\n",
            "epoch: 82, total rewards: 8.559620.3, cost: 0.000013, total money: 764.241510\n",
            "82\n",
            "epoch: 83, total rewards: 0.197010.3, cost: 0.000036, total money: 914.980140\n",
            "83\n",
            "epoch: 84, total rewards: 0.458980.3, cost: 0.000105, total money: 953.673020\n",
            "84\n",
            "epoch: 85, total rewards: 0.531750.3, cost: 0.000068, total money: 943.939320\n",
            "85\n",
            "epoch: 86, total rewards: -0.102370.3, cost: 0.000013, total money: 994.187460\n",
            "86\n",
            "epoch: 87, total rewards: 0.470520.3, cost: 0.000223, total money: 939.571990\n",
            "87\n",
            "epoch: 88, total rewards: -0.448180.3, cost: 0.000018, total money: 945.757030\n",
            "88\n",
            "epoch: 89, total rewards: 2.139170.3, cost: 0.000058, total money: 968.111790\n",
            "89\n",
            "epoch: 90, total rewards: 0.187160.3, cost: 0.000037, total money: 927.967940\n",
            "90\n",
            "epoch: 91, total rewards: -0.004480.3, cost: 0.000011, total money: 995.725390\n",
            "91\n",
            "epoch: 92, total rewards: 3.413880.3, cost: 0.000090, total money: 987.838660\n",
            "92\n",
            "epoch: 93, total rewards: 1.606810.3, cost: 0.000024, total money: 936.547910\n",
            "93\n",
            "epoch: 94, total rewards: 2.447770.3, cost: 0.000609, total money: 995.293740\n",
            "94\n",
            "epoch: 95, total rewards: 0.823810.3, cost: 0.000048, total money: 932.937720\n",
            "95\n",
            "epoch: 96, total rewards: 3.312850.3, cost: 0.000198, total money: 997.574200\n",
            "96\n",
            "epoch: 97, total rewards: -0.709580.3, cost: 0.000034, total money: 983.664150\n",
            "97\n",
            "epoch: 98, total rewards: 1.422140.3, cost: 0.000040, total money: 1001.422140\n",
            "98\n",
            "epoch: 99, total rewards: 0.741460.3, cost: 0.000102, total money: 956.761600\n",
            "99\n",
            "epoch: 100, total rewards: 1.734410.3, cost: 0.000070, total money: 947.934630\n",
            "100\n",
            "epoch: 101, total rewards: -0.617300.3, cost: 0.000044, total money: 945.467710\n",
            "101\n",
            "epoch: 102, total rewards: 1.353760.3, cost: 0.000204, total money: 980.085830\n",
            "102\n",
            "epoch: 103, total rewards: 1.159290.3, cost: 0.000042, total money: 924.815860\n",
            "103\n",
            "epoch: 104, total rewards: 4.470720.3, cost: 0.000134, total money: 900.826390\n",
            "104\n",
            "epoch: 105, total rewards: 0.569550.3, cost: 0.000101, total money: 966.545170\n",
            "105\n",
            "epoch: 106, total rewards: 2.486270.3, cost: 0.000020, total money: 904.134140\n",
            "106\n",
            "epoch: 107, total rewards: 0.556800.3, cost: 0.000102, total money: 949.669570\n",
            "107\n",
            "epoch: 108, total rewards: 2.818410.3, cost: 0.000019, total money: 874.341210\n",
            "108\n",
            "epoch: 109, total rewards: -0.399890.3, cost: 0.000097, total money: 967.070520\n",
            "109\n",
            "epoch: 110, total rewards: 3.502550.3, cost: 0.000018, total money: 916.966430\n",
            "110\n",
            "epoch: 111, total rewards: -2.233680.3, cost: 0.000032, total money: 997.766320\n",
            "111\n",
            "epoch: 112, total rewards: 1.827820.3, cost: 0.000118, total money: 969.241830\n",
            "112\n",
            "epoch: 113, total rewards: 5.127900.3, cost: 0.000017, total money: 928.673960\n",
            "113\n",
            "epoch: 114, total rewards: -0.237800.3, cost: 0.000056, total money: 884.824410\n",
            "114\n",
            "epoch: 115, total rewards: 0.921600.3, cost: 0.000050, total money: 897.034420\n",
            "115\n",
            "epoch: 116, total rewards: 1.237960.3, cost: 0.000029, total money: 994.079090\n",
            "116\n",
            "epoch: 117, total rewards: 3.855110.3, cost: 0.000091, total money: 1000.980550\n",
            "117\n",
            "epoch: 118, total rewards: 4.734540.3, cost: 0.000052, total money: 997.608760\n",
            "118\n",
            "epoch: 119, total rewards: 1.174830.3, cost: 0.000013, total money: 979.872840\n",
            "119\n",
            "epoch: 120, total rewards: 2.440540.3, cost: 0.000085, total money: 995.307650\n",
            "120\n",
            "epoch: 121, total rewards: -0.273790.3, cost: 0.000020, total money: 991.141680\n",
            "121\n",
            "epoch: 122, total rewards: 1.521740.3, cost: 0.000019, total money: 919.301390\n",
            "122\n",
            "epoch: 123, total rewards: -0.658810.3, cost: 0.000028, total money: 972.435170\n",
            "123\n",
            "epoch: 124, total rewards: 2.893650.3, cost: 0.000148, total money: 950.483430\n",
            "124\n",
            "epoch: 125, total rewards: 0.591670.3, cost: 0.000015, total money: 984.954210\n",
            "125\n",
            "epoch: 126, total rewards: 3.211150.3, cost: 0.000008, total money: 930.822230\n",
            "126\n",
            "epoch: 127, total rewards: -0.735150.3, cost: 0.000009, total money: 928.375810\n",
            "127\n",
            "epoch: 128, total rewards: 1.930750.3, cost: 0.000004, total money: 932.603870\n",
            "128\n",
            "epoch: 129, total rewards: -0.778600.3, cost: 0.000012, total money: 996.380300\n",
            "129\n",
            "epoch: 130, total rewards: -0.259590.3, cost: 0.000012, total money: 945.977700\n",
            "130\n",
            "epoch: 131, total rewards: -0.685050.3, cost: 0.000014, total money: 958.235970\n",
            "131\n",
            "epoch: 132, total rewards: 0.237610.3, cost: 0.000013, total money: 973.323690\n",
            "132\n",
            "epoch: 133, total rewards: -0.393560.3, cost: 0.000021, total money: 984.023280\n",
            "133\n",
            "epoch: 134, total rewards: 1.050560.3, cost: 0.000013, total money: 995.348230\n",
            "134\n",
            "epoch: 135, total rewards: 1.318610.3, cost: 0.000027, total money: 937.660580\n",
            "135\n",
            "epoch: 136, total rewards: 1.641550.3, cost: 0.000023, total money: 892.970630\n",
            "136\n",
            "epoch: 137, total rewards: -0.574250.3, cost: 0.000025, total money: 949.970960\n",
            "137\n",
            "epoch: 138, total rewards: 3.203210.3, cost: 0.000054, total money: 912.398730\n",
            "138\n",
            "epoch: 139, total rewards: 2.929690.3, cost: 0.000018, total money: 967.545630\n",
            "139\n",
            "epoch: 140, total rewards: 0.691090.3, cost: 0.000045, total money: 956.829140\n",
            "140\n",
            "epoch: 141, total rewards: -0.221420.3, cost: 0.000017, total money: 985.589230\n",
            "141\n",
            "epoch: 142, total rewards: 0.278500.3, cost: 0.000069, total money: 980.440180\n",
            "142\n",
            "epoch: 143, total rewards: 0.761610.3, cost: 0.000019, total money: 987.969970\n",
            "143\n",
            "epoch: 144, total rewards: 1.020790.3, cost: 0.000080, total money: 917.417370\n",
            "144\n",
            "epoch: 145, total rewards: -0.478410.3, cost: 0.000023, total money: 838.226340\n",
            "145\n",
            "epoch: 146, total rewards: 0.256750.3, cost: 0.000074, total money: 950.755030\n",
            "146\n",
            "epoch: 147, total rewards: 2.498180.3, cost: 0.000013, total money: 951.640470\n",
            "147\n",
            "epoch: 148, total rewards: 1.737380.3, cost: 0.000012, total money: 988.956730\n",
            "148\n",
            "epoch: 149, total rewards: 0.345340.3, cost: 0.000006, total money: 963.564990\n",
            "149\n",
            "epoch: 150, total rewards: -0.630400.3, cost: 0.000013, total money: 985.165180\n",
            "150\n",
            "epoch: 151, total rewards: 2.064330.3, cost: 0.000050, total money: 980.785070\n",
            "151\n",
            "epoch: 152, total rewards: 1.690780.3, cost: 0.000023, total money: 884.676620\n",
            "152\n",
            "epoch: 153, total rewards: -0.465480.3, cost: 0.000085, total money: 948.602700\n",
            "153\n",
            "epoch: 154, total rewards: 0.196280.3, cost: 0.000073, total money: 963.513730\n",
            "154\n",
            "epoch: 155, total rewards: -1.169430.3, cost: 0.000031, total money: 930.743430\n",
            "155\n",
            "epoch: 156, total rewards: -0.391100.3, cost: 0.000073, total money: 999.608900\n",
            "156\n",
            "epoch: 157, total rewards: 1.385740.3, cost: 0.000009, total money: 984.385900\n",
            "157\n",
            "epoch: 158, total rewards: 0.926470.3, cost: 0.000469, total money: 995.221070\n",
            "158\n",
            "epoch: 159, total rewards: -1.273400.3, cost: 0.000066, total money: 943.656260\n",
            "159\n",
            "epoch: 160, total rewards: 3.071550.3, cost: 0.000018, total money: 929.311860\n",
            "160\n",
            "epoch: 161, total rewards: 4.478170.3, cost: 0.000006, total money: 929.267990\n",
            "161\n",
            "epoch: 162, total rewards: -0.074490.3, cost: 0.000012, total money: 971.561170\n",
            "162\n",
            "epoch: 163, total rewards: 3.761330.3, cost: 0.000088, total money: 954.091870\n",
            "163\n",
            "epoch: 164, total rewards: 3.497130.3, cost: 0.000127, total money: 1003.497130\n",
            "164\n",
            "epoch: 165, total rewards: 1.570770.3, cost: 0.000006, total money: 924.727850\n",
            "165\n",
            "epoch: 166, total rewards: -0.804680.3, cost: 0.000017, total money: 969.410840\n",
            "166\n",
            "epoch: 167, total rewards: -0.486550.3, cost: 0.000024, total money: 923.000090\n",
            "167\n",
            "epoch: 168, total rewards: 0.258110.3, cost: 0.000015, total money: 920.330140\n",
            "168\n",
            "epoch: 169, total rewards: 0.697670.3, cost: 0.000018, total money: 941.329610\n",
            "169\n",
            "epoch: 170, total rewards: 0.619040.3, cost: 0.000124, total money: 944.093860\n",
            "170\n",
            "epoch: 171, total rewards: 0.170980.3, cost: 0.000008, total money: 949.353250\n",
            "171\n",
            "epoch: 172, total rewards: 2.012350.3, cost: 0.000011, total money: 968.015630\n",
            "172\n",
            "epoch: 173, total rewards: 0.352160.3, cost: 0.000010, total money: 929.095750\n",
            "173\n",
            "epoch: 174, total rewards: 1.217840.3, cost: 0.000075, total money: 963.032730\n",
            "174\n",
            "epoch: 175, total rewards: 0.992130.3, cost: 0.000015, total money: 889.474850\n",
            "175\n",
            "epoch: 176, total rewards: -0.615410.3, cost: 0.000016, total money: 934.182480\n",
            "176\n",
            "epoch: 177, total rewards: 3.441710.3, cost: 0.000017, total money: 925.234750\n",
            "177\n",
            "epoch: 178, total rewards: 1.092420.3, cost: 0.000026, total money: 985.468030\n",
            "178\n",
            "epoch: 179, total rewards: 0.282800.3, cost: 0.000014, total money: 987.482210\n",
            "179\n",
            "epoch: 180, total rewards: -1.018040.3, cost: 0.000046, total money: 980.551480\n",
            "180\n",
            "epoch: 181, total rewards: 1.306270.3, cost: 0.000094, total money: 865.782150\n",
            "181\n",
            "epoch: 182, total rewards: 3.043140.3, cost: 0.000009, total money: 930.430970\n",
            "182\n",
            "epoch: 183, total rewards: 1.201490.3, cost: 0.000015, total money: 918.356100\n",
            "183\n",
            "epoch: 184, total rewards: 0.955820.3, cost: 0.000006, total money: 978.261650\n",
            "184\n",
            "epoch: 185, total rewards: -0.028930.3, cost: 0.000037, total money: 988.652540\n",
            "185\n",
            "epoch: 186, total rewards: 1.381550.3, cost: 0.000013, total money: 987.188090\n",
            "186\n",
            "epoch: 187, total rewards: 3.725450.3, cost: 0.000005, total money: 937.096880\n",
            "187\n",
            "epoch: 188, total rewards: 1.377130.3, cost: 0.000112, total money: 968.773280\n",
            "188\n",
            "epoch: 189, total rewards: 0.262530.3, cost: 0.000041, total money: 841.909220\n",
            "189\n",
            "epoch: 190, total rewards: 1.780590.3, cost: 0.000061, total money: 950.743400\n",
            "190\n",
            "epoch: 191, total rewards: -0.104110.3, cost: 0.000024, total money: 934.692720\n",
            "191\n",
            "epoch: 192, total rewards: 0.227750.3, cost: 0.000004, total money: 987.412920\n",
            "192\n",
            "epoch: 193, total rewards: 3.019220.3, cost: 0.000036, total money: 926.160360\n",
            "193\n",
            "epoch: 194, total rewards: 1.771470.3, cost: 0.000033, total money: 914.734540\n",
            "194\n",
            "epoch: 195, total rewards: 2.330750.3, cost: 0.000009, total money: 935.359150\n",
            "195\n",
            "epoch: 196, total rewards: 2.826270.3, cost: 0.000014, total money: 947.690320\n",
            "196\n",
            "epoch: 197, total rewards: 0.392200.3, cost: 0.000043, total money: 956.466910\n",
            "197\n",
            "epoch: 198, total rewards: 0.054630.3, cost: 0.000023, total money: 998.616830\n",
            "198\n",
            "epoch: 199, total rewards: 0.982010.3, cost: 0.000115, total money: 954.030920\n",
            "199\n",
            "epoch: 200, total rewards: 0.899170.3, cost: 0.000018, total money: 972.583870\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MgB_oFAo65Sl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "states_buy, states_sell, total_gains, invest = agent.buy(initial_money = initial_money)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s-GoEjY77AHV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fig = plt.figure(figsize = (15,5))\n",
        "plt.plot(close, color='r', lw=2.)\n",
        "plt.plot(close, '^', markersize=10, color='m', label = 'buying signal', markevery = states_buy)\n",
        "plt.plot(close, 'v', markersize=10, color='k', label = 'selling signal', markevery = states_sell)\n",
        "plt.title('total gains %f, total investment %f%%'%(total_gains, invest))\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}