From 9ce7ac69259f142e7bdc51fc72381d998cf7f3b7 Mon Sep 17 00:00:00 2001 From: lewtun Date: Sun, 24 Mar 2024 14:29:22 +0100 Subject: [PATCH] Fix hyperparameters in KTO example (#1474) * Fix hparams in KTO example * Clean * Fix --- examples/scripts/kto.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/scripts/kto.py b/examples/scripts/kto.py index b47a17984a..49a3edffff 100644 --- a/examples/scripts/kto.py +++ b/examples/scripts/kto.py @@ -19,9 +19,9 @@ # regular: python examples/scripts/kto.py \ --model_name_or_path=gpt2 \ - --per_device_train_batch_size 4 \ + --per_device_train_batch_size 16 \ --max_steps 1000 \ - --learning_rate 1e-3 \ + --learning_rate 2e-5 \ --gradient_accumulation_steps 1 \ --logging_steps 10 \ --eval_steps 500 \ @@ -35,14 +35,13 @@ # peft: python examples/scripts/kto.py \ --model_name_or_path=gpt2 \ - --per_device_train_batch_size 4 \ + --per_device_train_batch_size 16 \ --max_steps 1000 \ - --learning_rate 1e-3 \ + --learning_rate 2e-4 \ --gradient_accumulation_steps 1 \ --logging_steps 10 \ --eval_steps 500 \ --output_dir="kto_anthropic_hh" \ - --optim rmsprop \ --warmup_steps 150 \ --report_to wandb \ --bf16 \