Skip to content

Commit

Permalink
switch test over to NN
Browse files Browse the repository at this point in the history
  • Loading branch information
@bgranzow
bgranzow committed Mar 1, 2024
1 parent 4dda73b commit 19bdbc3
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 56 deletions.
35 changes: 35 additions & 0 deletions src/NN.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,41 @@ typename FFNN<T>::Vector FFNN<T>::evaluate(Vector const& x_in)
return x[end];
}

template <class T>
typename FFNN<T>::Vector const& FFNN<T>::get_params()
{
int idx = 0;
for (size_t i = 0; i < W.size(); ++i) {
for (int j = 0; j < W[i].rows(); ++j) {
for (int k = 0; k < W[i].cols(); ++k) {
params[idx++] = W[i](j,k);
}
}
for (int j = 0; j < b[i].size(); ++j) {
params[idx++] = b[i][j];
}
}
return params;
}

template <class T>
void FFNN<T>::set_params(Vector const& p)
{
params = p;
int idx = 0;
for (size_t i = 0; i < W.size(); ++i) {
for (int j = 0; j < W[i].rows(); ++j) {
for (int k = 0; k < W[i].cols(); ++k) {
W[i](j,k) = params[idx++];
}
}
for (int j = 0; j < b[i].size(); ++j) {
b[i][j] = params[idx++];
}
}
ffnn_seed(num_params, W, b);
}

template class FFNN<double>;
template class FFNN<SFADT>;
template class FFNN<DFADT>;
Expand Down
104 changes: 48 additions & 56 deletions test/unit/nn.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,114 +4,106 @@

using namespace ML;

#if 0
static void check_components_wrt_params(
const char* activation,
std::vector<int> const& topology,
Vector const& x)
FFNN<DFADT>::Vector const& x)
{
using Vector = FFNN<DFADT>::Vector;
assert(x.size() == 1);
assert(topology[0] == 1);
assert(topology.back() == 1);
NN network(activation, topology);
FFNN<DFADT> network(activation, topology);
Vector const p = network.get_params();
Vector const y = network.feed_forward(x);
auto const D = network.differentiate(y);
double const h = 1.e-8;
Vector const y = network.evaluate(x);
DFADT const h = 1.e-8;
for (int i = 0; i < p.size(); ++i) {
Vector p_perturb = p;
p_perturb[i] += h;
network.set_params(p_perturb);
Vector const y_perturb = network.feed_forward(x);
Vector const y_perturb = network.evaluate(x);
Vector const dy_dp_fd = (y_perturb - y)/h;
printf("dy[0]/dp[%d], FD: %.15e, backprop: %.15e\n", i, dy_dp_fd[0], D[0](0,i));
printf("dy[0]/dp[%d], FD: %.15e, AD: %.15e\n",
i,
dy_dp_fd[0].val(),
y[0].fastAccessDx(i));
}
printf("\n");
}

static void check_components_wrt_inputs(
const char* activation,
std::vector<int> const& topology,
Vector const& x)
FFNN<SFADT>::Vector const& x)
{
using Vector = FFNN<SFADT>::Vector;
assert(x.size() == 1);
assert(topology[0] == 1);
assert(topology.back() == 1);
NN network(activation, topology);
FFNN<SFADT> network(activation, topology);
Vector const p = network.get_params();
Vector const y = network.feed_forward(x);
auto const D = network.differentiate(y);
double const h = 1.e-6;
for (int i = 0; i < x.size(); ++i) {
Vector x_perturb = x;
x_perturb[i] += h;
Vector const y_perturb = network.feed_forward(x_perturb);
Vector const dy_dx_fd = (y_perturb - y)/h;
printf("dy[0]/dx[%d], FD: %.15e, backprop: %.15e\n", i, dy_dx_fd[0], D[1](0,i));
}
printf("\n");
Vector const y = network.evaluate(x);
SFADT const h = 1.e-6;
Vector x_perturb = x;
x_perturb[0] += h;
Vector const y_perturb = network.evaluate(x_perturb);
Vector const dy_dx_fd = (y_perturb - y)/h;
printf("dy[0]/dx, FD: %.15e, AD: %.15e\n",
dy_dx_fd[0].val(),
y[0].fastAccessDx(0));
}

static void check_direction(
const char* activation,
std::vector<int> const& topology,
Vector const& x)
FFNN<DFADT>::Vector const& x)
{
using Vector = FFNN<DFADT>::Vector;
assert(topology[0] == 1);
assert(topology.back() == 1);
assert(x.size() == 1);
NN network(activation, topology);
FFNN<DFADT> network(activation, topology);
Vector const p = network.get_params();
Vector const y = network.feed_forward(x);
auto const D = network.differentiate(y);
Vector const y = network.evaluate(x);
Vector dir = Vector(p.size());
dir.setOnes();
Vector const dy_dir = D[0] * dir;
double dy_dir = 0.;
for (int i = 0; i < p.size(); ++i) {
dy_dir += y[0].fastAccessDx(i) * dir[i].val();
}
for (int i = 0; i >= -14; --i) {
double const h = std::pow(10., double(i));
DFADT const h = std::pow(10., double(i));
Vector const p_perturb = p + dir * h;
network.set_params(p_perturb);
Vector const y_perturb = network.feed_forward(x);
Vector const y_perturb = network.evaluate(x);
Vector const dy_dir_fd = (y_perturb - y) / h;
double const err = std::abs(dy_dir[0] - dy_dir_fd[0]);
printf("[%.2e] %.15e\n", h, err);
double const err = std::abs(dy_dir - dy_dir_fd[0].val());
printf("[%.2e] %.15e\n", h.val(), err);
}
printf("\n");
}

TEST(NN, components_wrt_params)
TEST(FFNN, components_wrt_params)
{
Vector x(1);
FFNN<DFADT>::Vector x(1);
x[0] = 1.;
check_components_wrt_params("relu", {1, 4, 4, 1}, x);
check_components_wrt_params("sigmoid", {1, 2, 3, 1}, x);
check_components_wrt_params("tanh", {1, 2, 1}, x);
check_components_wrt_params("relu", {1,4,4,1}, x);
check_components_wrt_params("sigmoid", {1,2,3,1}, x);
check_components_wrt_params("tanh", {1,2,1}, x);
}

TEST(NN, components_wrt_inputs)
TEST(FFNN, components_wrt_inputs)
{
Vector x(1);
FFNN<SFADT>::Vector x(1);
x[0] = 1.23;
check_components_wrt_inputs("relu", {1, 5, 1}, x);
check_components_wrt_inputs("sigmoid", {1, 3, 2, 1}, x);
check_components_wrt_inputs("tanh", {1, 3, 1}, x);
}

TEST(NN, check_direction)
{
Vector x(1);
x[0] = 1.;
check_direction("relu", {1, 4, 4, 1}, x);
check_direction("sigmoid", {1, 2, 3, 1}, x);
check_direction("tanh", {1, 10, 1}, x);
x[0].diff(0, 1);
check_components_wrt_inputs("relu", {1,5,1}, x);
check_components_wrt_inputs("sigmoid", {1,3,2,1}, x);
check_components_wrt_inputs("tanh", {1,3,1}, x);
}
#endif

TEST(NN, ADrewrite)
TEST(FFNN, check_direction)
{
FFNN<DFADT>::Vector x(1);
x[0] = 1;
FFNN<DFADT> nn("relu", {1, 4, 4, 1});
auto y = nn.evaluate(x);
std::cout << y << "\n";
x[0] = 1.0;
check_direction("relu", {1,4,4,1}, x);
}

0 comments on commit 19bdbc3

Please sign in to comment.