diff --git a/model.py b/model.py
index 7829944d0b..ca086f1ab8 100644
--- a/model.py
+++ b/model.py
@@ -890,6 +890,10 @@ def from_pretrained(cls, config, model_type):
                     if key == "lm_head.weight":
                         continue
 
+                if not config.use_abs_pos_embeddings:
+                    if key == "transformer.wpe.weight":
+                        continue
+
                 assert sd_hf[key].shape == sd[key].shape
                 with torch.no_grad():
                     print(key)