update

examples/tutorials/rlhf/gpt2_sst2_ppo/step_2_train_model_ddp.py

@@ -39,11 +39,11 @@ kl_coef：0.03
 response_length： 256
 
 nohup torchrun --nproc_per_node=2 step_2_train_model_ddp.py \
---learning_rate 1e-6 \
+--learning_rate 5e-6 \
 --response_length 128 \
---kl_coef 0.3 \
---cliprange 0.05 \
---cliprange_value 0.05 \
+--kl_coef 0.05 \
+--cliprange 0.2 \
+--cliprange_value 0.2 \
 &
 
 

toolbox/trl/trainer/ppo_trainer.py

@@ -364,8 +364,8 @@ class PPOTrainer(Trainer):
             top_k=0.0,
             top_p=1.0,
             do_sample=True,
-            pad_token_id=processing_class.pad_token_id, eos_token_id=processing_class.eos_token_id,
-            repetition_penalty=3.0,
+            # pad_token_id=processing_class.pad_token_id, eos_token_id=processing_class.eos_token_id,
+            # repetition_penalty=3.0,
         )
 
         accelerator.print("===training policy===")
@@ -692,8 +692,8 @@ class PPOTrainer(Trainer):
             top_k=0.0,
             top_p=1.0,
             do_sample=True,
-            pad_token_id=processing_class.pad_token_id, eos_token_id=processing_class.eos_token_id,
-            repetition_penalty=3.0,
+            # pad_token_id=processing_class.pad_token_id, eos_token_id=processing_class.eos_token_id,
+            # repetition_penalty=3.0,
         )
 
         table = defaultdict(list)