squall/flash-attention
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Gen] Add back num_last_tokens in gpt.py

Browse Source
This commit is contained in:
Tri Dao 2023-09-03 20:44:40 -07:00
parent 5953c4f58c
commit 7b33743a72
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
flash_attn/models/gpt.py
View File

@ -634,6 +634,8 @@ class GPTLMHeadModel(GPTPreTrainedModel, GenerationMixin):
input_ids, position_ids=position_ids, inference_params=inference_params input_ids, position_ids=position_ids, inference_params=inference_params
) )
assert hidden_states.ndim == 3, "sequence_parallel is not supported in generation mode" assert hidden_states.ndim == 3, "sequence_parallel is not supported in generation mode"
if num_last_tokens > 0:
hidden_states = hidden_states[:, -num_last_tokens:]
if self.project_out is not None: if self.project_out is not None:
hidden_states = self.project_out(hidden_states) hidden_states = self.project_out(hidden_states)
lm_logits = self.lm_head(hidden_states) lm_logits = self.lm_head(hidden_states)