[Feature] Replace D4RLExperienceReplay with MinariExperienceReplay in CQL offline

Author: jorge.ibinarriaga.robles.becas

sota-implementations/cql/=0.9.0,

@@ -190,6 +190,11 @@ def update(data, policy_eval_start, iteration):
         with timeit("log"):
             metrics_to_log.update(timeit.todict(prefix="time"))
             metrics_to_log["time/speed"] = pbar.format_dict["rate"]
+
+            if i % evaluation_interval == 0:
+                print(
+                    f"Step {i}: loss={loss.item():.4f}, eval_reward={eval_reward:.4f}"
+                )
             log_metrics(logger, metrics_to_log, i)
 
     pbar.close()

sota-implementations/cql/cql_offline.py

@@ -22,7 +22,7 @@ collector:
 
 # Logger
 logger:
-  backend: wandb
+  backend: csv
   project_name: torchrl_example_cql
   group_name: null
   exp_name: cql_cartpole_gym

sota-implementations/cql/discrete_cql_config.yaml

@@ -1,6 +1,6 @@
 # env and task
 env:
-  name: Hopper-v4
+  name: Hopper-v5
   task: ""
   library: gym
   n_samples_stats: 1000
@@ -9,7 +9,7 @@ env:
 
 # logger
 logger:
-  backend: wandb
+  backend: csv
   project_name: torchrl_example_cql
   group_name: null
   exp_name: cql_${replay_buffer.dataset}
@@ -18,11 +18,11 @@ logger:
   eval_steps: 1000
   mode: online
   eval_envs: 5
-  video: False
+  video: True
 
 # replay buffer
 replay_buffer:
-  dataset: hopper-medium-v2
+  dataset: mujoco/hopper/expert-v0
   batch_size: 256
 
 # optimization

sota-implementations/cql/offline_config.yaml

@@ -19,6 +19,7 @@
     TensorDictReplayBuffer,
 )
 from torchrl.data.datasets.d4rl import D4RLExperienceReplay
+from torchrl.data.datasets.minari_data  import MinariExperienceReplay 
 from torchrl.data.replay_buffers import SamplerWithoutReplacement
 from torchrl.envs import (
     CatTensors,
@@ -181,13 +182,13 @@ def make_replay_buffer(
 
 
 def make_offline_replay_buffer(rb_cfg):
-    data = D4RLExperienceReplay(
+    data = MinariExperienceReplay(
         dataset_id=rb_cfg.dataset,
         split_trajs=False,
         batch_size=rb_cfg.batch_size,
         sampler=SamplerWithoutReplacement(drop_last=True),
         prefetch=4,
-        direct_download=True,
+        download=True,
     )
 
     data.append_transform(DoubleToFloat())