From 6bb19056f2e10b052321c9c510df8d9474aca333 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 8 Jan 2024 16:36:09 +0300 Subject: [PATCH] correct the formula for k (#23) --- scripts/determine_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/determine_params.py b/scripts/determine_params.py index d97746e..799706f 100644 --- a/scripts/determine_params.py +++ b/scripts/determine_params.py @@ -14,7 +14,7 @@ n = len(model.model.layers) r = q_size // k_size h = model.config.num_attention_heads // r -k = q_size // m // h +k = k_size // m // h n_params = sum(x.numel() for x in model.parameters())