correct the formula for k (#23)

ayaka14732 · Jan 8, 2024 · 6bb1905 · 6bb1905
1 parent b0c337c
commit 6bb1905
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/scripts/determine_params.py b/scripts/determine_params.py
@@ -14,7 +14,7 @@
 n = len(model.model.layers)
 r = q_size // k_size
 h = model.config.num_attention_heads // r
-k = q_size // m // h
+k = k_size // m // h
 
 n_params = sum(x.numel() for x in model.parameters())