From 363dbd5c05a5df364ddd338dbabe7a23b0983a0a Mon Sep 17 00:00:00 2001
From: "ferdinand.mom" <ferdinand.mom@epita.fr>
Date: Wed, 30 Oct 2024 15:12:06 +0000
Subject: [PATCH] need to update max position embeding  when seq_len is greater
 (for rope)

---
 train.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/train.py b/train.py
index c9e99b4..ac8d588 100644
--- a/train.py
+++ b/train.py
@@ -121,6 +121,7 @@ if __name__ == "__main__":
     model_config.num_hidden_layers = config["model"]["num_hidden_layers"]
     model_config.num_attention_heads = config["model"]["num_attention_heads"]
     model_config.num_key_value_heads = config["model"]["num_key_value_heads"]
+    model_config.max_position_embeddings = SEQ_LEN
 
     start_time = time.time()
     model = Llama(config=model_config)