From 6a69e678ebab78a93f1191e659687f4b2c5e369f Mon Sep 17 00:00:00 2001
From: Alan <41682961+alan-cooney@users.noreply.github.com>
Date: Mon, 13 Nov 2023 08:56:19 +0800
Subject: [PATCH] Switch loss to mean (#67)

Closes Should we use mean rather than sum for batchwise loss reduction? #63
---
 sparse_autoencoder/train/train_autoencoder.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sparse_autoencoder/train/train_autoencoder.py b/sparse_autoencoder/train/train_autoencoder.py
index 474f0837..192ab181 100644
--- a/sparse_autoencoder/train/train_autoencoder.py
+++ b/sparse_autoencoder/train/train_autoencoder.py
@@ -88,7 +88,7 @@ def train_autoencoder(
             learned_activations_fired_count.add_(fired.sum(dim=0))
 
             # Backwards pass
-            total_loss.sum().backward()
+            total_loss.mean().backward()
 
             optimizer.step()