tangger/Search-R1
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix kl loss issue

Browse Source
This commit is contained in:
PeterGriffinJin
2025-03-18 20:07:47 +00:00
parent e85506f143
commit 4b3c09451a
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
verl/workers/actor/dp_actor.py
View File

@@ -268,7 +268,7 @@ class DataParallelPPOActor(BasePPOActor):
kl_penalty=self.config.kl_loss_type) kl_penalty=self.config.kl_loss_type)
kl_loss = masked_mean(kld, response_mask) kl_loss = masked_mean(kld, response_mask)
policy_loss = policy_loss - kl_loss * self.config.kl_loss_coef policy_loss = policy_loss + kl_loss * self.config.kl_loss_coef
metrics['actor/kl_loss'] = kl_loss.detach().item() metrics['actor/kl_loss'] = kl_loss.detach().item()
metrics['actor/kl_coef'] = self.config.kl_loss_coef metrics['actor/kl_coef'] = self.config.kl_loss_coef