implementing the xor example: losses, optimizers, datasets training etc.

.gitignore

@@ -1,2 +1,3 @@
 .vscode/
+.vs/
 build/

CMakeLists.txt

@@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.0.0)
 
 project(Tipousi VERSION 0.0.1)
 
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
 # Set the default build type to Release if not specified
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build" FORCE)
@@ -20,6 +23,8 @@ include_directories(
     ${PROJECT_SOURCE_DIR}/include/layer
     ${PROJECT_SOURCE_DIR}/include/graph
     ${PROJECT_SOURCE_DIR}/include/base
+    ${PROJECT_SOURCE_DIR}/include/optimizer
+    ${PROJECT_SOURCE_DIR}/include/data
 )
 
 # Source files

CMakeSettings.json

@@ -0,0 +1,15 @@
+{
+  "configurations": [
+    {
+      "name": "x64-Debug",
+      "generator": "Ninja",
+      "configurationType": "Debug",
+      "inheritEnvironments": [ "msvc_x64_x64" ],
+      "buildRoot": "${projectDir}\\out\\build\\${name}",
+      "installRoot": "${projectDir}\\out\\install\\${name}",
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "",
+      "ctestCommandArgs": ""
+    }
+  ]
+}

include/data/dataset.hpp

@@ -0,0 +1,40 @@
+#pragma once
+#include <Eigen/Dense>
+#include <vector>
+
+namespace Tipousi
+{
+    namespace Data
+    {
+        class Dataset
+        {
+          public:
+            Dataset(const Eigen::MatrixXd &X, const Eigen::MatrixXd &Y);
+            ~Dataset() = default;
+
+            using DataPair = std::pair<Eigen::MatrixXf, Eigen::MatrixXf>;
+
+            class Iterator
+            {
+              public:
+                Iterator(const Eigen::MatrixXd &X, const Eigen::MatrixXd &Y,
+                         size_t index);
+                Iterator &operator++();
+                bool      operator!=(const Iterator &other) const;
+                DataPair  operator*() const;
+
+              private:
+                const Eigen::MatrixXd &m_X;
+                const Eigen::MatrixXd &m_y;
+                size_t                 m_index;
+            };
+
+            Iterator begin() const;
+            Iterator end() const;
+
+          private:
+            Eigen::MatrixXd m_X;
+            Eigen::MatrixXd m_y;
+        };
+    };  // namespace Data
+};      // namespace Tipousi

include/graph/sequential.hpp

@@ -1,13 +1,14 @@
 #pragma once
 
 #include "graph/node.hpp"
+#include "graph/trainable.hpp"
 #include <vector>
 
 namespace Tipousi
 {
     namespace Graph
     {
-        class Sequential
+        class Sequential : public Trainable
         {
 
           public:
@@ -27,6 +28,11 @@ namespace Tipousi
             void forward(const Eigen::MatrixXf &in, Eigen::MatrixXf &out);
             void backward();
 
+            void train(const Data::Dataset            &dataset,
+                       const Optimizer::OptimizerBase &optimizer,
+                       const Loss::LossBase           &loss,
+                       const uint32_t                  n_epochs) override;
+
           private:
             Node               *m_input_node  = nullptr;
             Node               *m_output_node = nullptr;

include/graph/trainable.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "data/dataset.hpp"
+#include "loss/base.hpp"
+#include "optimizer/base.hpp"
+
+namespace Tipousi
+{
+    namespace Graph
+    {
+        class Trainable
+        {
+          public:
+            virtual void train(const Data::Dataset            &dataset,
+                               const Optimizer::OptimizerBase &optimizer,
+                               const Loss::LossBase           &loss,
+                               const uint32_t                  n_epochs) = 0;
+
+          protected:
+            Trainable()          = default;
+            virtual ~Trainable() = default;
+        };
+
+    }  // namespace Graph
+}  // namespace Tipousi

include/loss/base.hpp

@@ -0,0 +1,22 @@
+#pragma once
+#include <Eigen/Dense>
+
+namespace Tipousi
+{
+    namespace Loss
+    {
+        class LossBase
+        {
+          public:
+            LossBase()          = default;
+            virtual ~LossBase() = default;
+
+            virtual float compute(const Eigen::MatrixXf &y,
+                                  const Eigen::MatrixXf &y_pred) const = 0;
+
+            virtual void grad(Eigen::MatrixXf       &out_grad,
+                              const Eigen::MatrixXf &y,
+                              const Eigen::MatrixXf &y_pred) const = 0;
+        };
+    };  // namespace Loss
+};      // namespace Tipousi

include/loss/mse.hpp

@@ -1,11 +1,20 @@
 #pragma once
+#include "loss/base.hpp"
 
 namespace Tipousi
 {
     namespace Loss
     {
-        class MSE
+        class MSE : public LossBase
         {
+            MSE()  = default;
+            ~MSE() = default;
+
+            float compute(const Eigen::MatrixXf &y,
+                          const Eigen::MatrixXf &y_pred) const  override;
+
+            void grad(Eigen::MatrixXf &out_grad, const Eigen::MatrixXf &y,
+                      const Eigen::MatrixXf &y_pred) const override;
         };
     };  // namespace Loss
 };      // namespace Tipousi

include/loss/template.hpp → include/optimizer/base.hpp

@@ -2,10 +2,10 @@
 
 namespace Tipousi
 {
-    namespace Loss
+    namespace Optimizer
     {
-        class MyTemplate
+        class OptimizerBase
         {
         };
-    };  // namespace Loss
+    };  // namespace Optimizer
 };      // namespace Tipousi

include/optimizer/sgd.hpp

@@ -0,0 +1,15 @@
+#pragma once
+#include "optimizer/base.hpp"
+
+namespace Tipousi
+{
+    namespace Optimizer
+    {
+        class SGD : public OptimizerBase
+        {
+          public:
+            SGD();
+            ~SGD() = default;
+        };
+    };  // namespace Optimizer
+};      // namespace Tipousi

src/data/dataset.cpp

@@ -0,0 +1,46 @@
+#include "data/dataset.hpp"
+
+namespace Tipousi
+{
+    namespace Data
+    {
+        Dataset::Dataset(const Eigen::MatrixXd &X, const Eigen::MatrixXd &Y)
+            : m_X(X), m_y(Y)
+        {
+        }
+
+        Dataset::Iterator::Iterator(const Eigen::MatrixXd &X,
+                                    const Eigen::MatrixXd &Y, size_t index)
+            : m_X(X), m_y(Y), m_index(index)
+        {
+        }
+
+        Dataset::Iterator &Dataset::Iterator::operator++()
+        {
+            ++m_index;
+            return *this;
+        }
+
+        bool Dataset::Iterator::operator!=(const Dataset::Iterator &other) const
+        {
+            return m_index != other.m_index;
+        }
+
+        Dataset::DataPair Dataset::Iterator::operator*() const
+        {
+            Eigen::MatrixXf x = m_X.row(m_index).cast<float>();
+            Eigen::MatrixXf y = m_y.row(m_index).cast<float>();
+            return {x, y};
+        }
+
+        Dataset::Iterator Dataset::begin() const
+        {
+            return Iterator(m_X, m_y, 0);
+        }
+
+        Dataset::Iterator Dataset::end() const
+        {
+            return Iterator(m_X, m_y, m_X.rows());
+        }
+    }  // namespace Data
+}  // namespace Tipousi

src/graph/sequential.cpp

@@ -1,4 +1,5 @@
 #include "graph/sequential.hpp"
+#include <iostream>
 
 namespace Tipousi
 {
@@ -58,5 +59,29 @@ namespace Tipousi
             //
         }
 
+        void Sequential::train(const Data::Dataset            &dataset,
+                               const Optimizer::OptimizerBase &optimizer,
+                               const Loss::LossBase           &loss_func,
+                               const uint32_t                  n_epochs)
+        {
+            for (uint32_t i{0}; i < n_epochs; i++)
+            {
+                float total_loss = 0.0f;
+                uint32_t counter{0};
+                for (const auto &[x, y] : dataset)
+                {
+                    Eigen::MatrixXf output;
+                    Eigen::MatrixXf out_grad;
+                    forward(x, output);
+                    total_loss += loss_func.compute(y, output);
+                    loss_func.grad(out_grad, y, output);
+                    backward();
+                    counter++;
+                }
+                std::cout << "Epoch: " << i
+                          << ", Loss: " << total_loss / counter << std::endl;
+            }
+        }
+
     }  // namespace Graph
 }  // namespace Tipousi

src/loss/mse.cpp

@@ -0,0 +1,19 @@
+#include "loss/mse.hpp"
+
+namespace Tipousi
+{
+    namespace Loss
+    {
+        float MSE::compute(const Eigen::MatrixXf &y,
+                           const Eigen::MatrixXf &y_pred) const 
+        {
+            return (y - y_pred).array().square().mean();
+        }
+
+        void MSE::grad(Eigen::MatrixXf &out_grad, const Eigen::MatrixXf &y,
+                       const Eigen::MatrixXf &y_pred) const 
+        {
+            out_grad = 2.0f * (y_pred - y) / y.rows();
+        }
+    }  // namespace Loss
+}  // namespace Tipousi

src/optimizer/sgd.cpp

@@ -0,0 +1,9 @@
+#include "optimizer/sgd.hpp"
+
+namespace Tipousi
+{
+    namespace Optimizer
+    {
+        SGD::SGD() {}
+    }  // namespace Optimizer
+}  // namespace Tipousi