2022-11-29 09:31:19 +08:00
|
|
|
# @package _global_
|
|
|
|
|
defaults:
|
|
|
|
|
- /experiment/owt/gpt2s-flash.yaml
|
|
|
|
|
- override /model/gpt2model: gpt2-medium
|
|
|
|
|
|
|
|
|
|
# Can enable mlp_checkpoint_lvl to fit batch_size 32 to A100 40GB
|
2022-11-29 20:13:51 +08:00
|
|
|
# model:
|
|
|
|
|
# config:
|
|
|
|
|
# mlp_checkpoint_lvl: 1
|
2022-11-29 09:31:19 +08:00
|
|
|
|
|
|
|
|
datamodule:
|
|
|
|
|
# batch_size: 32
|
2022-11-29 20:13:51 +08:00
|
|
|
batch_size: ${eval:"8 if ${train.gpu_mem} < 24 else (16 if ${train.gpu_mem} < 40 else (32 if ${train.gpu_mem} < 80 else 64))"}
|
2022-11-29 09:31:19 +08:00
|
|
|
|
|
|
|
|
train:
|
|
|
|
|
optimizer:
|
|
|
|
|
lr: 1.5e-4
|
