[BugFix] Fix KL penalty

ghstack-source-id: 475dccb
Pull Request resolved: #2908

Author: Vincent Moens

torchrl/envs/transforms/llm.py

@@ -765,7 +765,7 @@ def _step(
         kl = curr_log_prob - log_prob
         if reward is None:
             reward = 0
-        next_tensordict.set(self.out_keys[0], reward + self.coef * kl)
+        next_tensordict.set(self.out_keys[0], reward - self.coef * kl)
         return next_tensordict
 
     def forward(self, tensordict: TensorDictBase) -> TensorDictBase: