Skip to content

Commit

Permalink
Showing 2 changed files with 13 additions and 5 deletions.
15 changes: 11 additions & 4 deletions src/nn_primitives/activation_primitives.nim
Original file line number Diff line number Diff line change
@@ -43,14 +43,16 @@ proc sigmoid*[T: SomeReal](t: Tensor[T]): Tensor[T] {.inline.}=

# stable: proc sigmoid_closure(x: T): T = 0.5.T * (tanh(0.5.T * x) + 1.T)

return t.mapT(1.T / (1.T + exp(-x)))
result = mapT(t):
1.T / (1.T + exp(-x))

proc msigmoid*[T: SomeReal](t: var Tensor[T]) {.inline.}=
## Logistic sigmoid activation function, :math:`f(x) = 1 / (1 + \exp(-x))`
## Note: Canonical sigmoid is not stable for large negative value

# stable: proc sigmoid_closure(x: T): T = 0.5.T * (tanh(0.5.T * x) + 1.T)
t.applyT(1.T / (1.T + exp(-x)))
applyT(t):
1.T / (1.T + exp(-x))

proc relu*[T](t: Tensor[T]): Tensor[T] {.inline.}=
t.mapT max(0.T,x)
@@ -60,9 +62,14 @@ proc mrelu*[T](t: var Tensor[T]) {.inline.}=


proc relu_backward*[T](gradient: Tensor[T], cached_tensor: Tensor[T]): Tensor[T]{.inline.}=
result = cached_tensor.mapT(if x <= 0.T: 0.T else: 1.T)
result = mapT(cached_tensor):
if x <= 0.T:
0.T
else:
1.T
result .*= gradient

proc sigmoid_backward*[T](gradient: Tensor[T], cached_tensor: Tensor[T]): Tensor[T]{.inline.}=
result = cached_tensor.mapT(x * (1 - x))
result = mapT(cached_tensor):
x * (1 - x)
result .*= gradient
3 changes: 2 additions & 1 deletion src/nn_primitives/sigmoid_cross_entropy_primitives.nim
Original file line number Diff line number Diff line change
@@ -67,7 +67,8 @@ proc sigmoid_cross_entropy_backward*[T](
elif gradient is Tensor:
let grad = gradient.data[gradient.offset]

return map2T(cached_tensor, target, grad * (1 / (1 + exp(-x)) - y) / T(batch_size))
result = map2T(cached_tensor, target):
grad * (1 / (1 + exp(-x)) - y) / T(batch_size)

# ################################################
# Explanation of sigmoid cross-entropy algorithms:

0 comments on commit 598eb41

Please sign in to comment.