Update (base update)

[ghstack-poisoned]

Author: Vincent Moens

.github/scripts/pre-build-script-win.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+pip install --upgrade setuptools
+
+export TORCHRL_BUILD_VERSION=0.7.0
+
+${CONDA_RUN} pip install git+https://github.com/pytorch/tensordict.git -U

.github/scripts/pre-build-script.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+pip install --upgrade setuptools
+
+${CONDA_RUN} pip install git+https://github.com/pytorch/tensordict.git -U

.github/scripts/td_script.sh

@@ -1,5 +1,25 @@
 #!/bin/bash
 
 export TORCHRL_BUILD_VERSION=0.7.0
+pip install --upgrade setuptools
 
-${CONDA_RUN} pip install git+https://github.com/pytorch/tensordict.git -U
+# Check if ARCH is set to aarch64
+ARCH=${ARCH:-}  # This sets ARCH to an empty string if it's not defined
+
+if pip list | grep -q torch; then
+    echo "Torch is installed."
+    if [[ "$ARCH" == "aarch64" ]]; then
+        ${CONDA_RUN} pip install git+https://github.com/pytorch/tensordict.git -U --no-deps
+    else
+        ${CONDA_RUN} pip install tensordict-nightly -U
+    fi
+elif [[ -n "${SMOKE_TEST_SCRIPT:-}" ]]; then
+    ${CONDA_RUN} ${PIP_INSTALL_TORCH}
+    if [[ "$ARCH" == "aarch64" ]]; then
+        ${CONDA_RUN} pip install git+https://github.com/pytorch/tensordict.git -U --no-deps
+    else
+        ${CONDA_RUN} pip install tensordict-nightly -U
+    fi
+else
+    echo "Torch is not installed - tensordict will be installed later."
+fi

.github/scripts/version_script.bat

@@ -36,6 +36,9 @@ if "%CU_VERSION%" == "xpu" call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat
 
 set DISTUTILS_USE_SDK=1
 
+:: Upgrade setuptools before installing PyTorch
+pip install --upgrade setuptools==72.1.0 || exit /b 1
+
 set args=%1
 shift
 :start

.github/unittest/linux/scripts/environment.yml

@@ -17,6 +17,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-timeout
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_distributed/scripts/environment.yml

@@ -16,6 +16,7 @@ dependencies:
     - pytest-mock
     - pytest-instafail
     - pytest-rerunfailures
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_ataridqn/environment.yml

@@ -15,6 +15,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_brax/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_chess/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_d4rl/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_envpool/environment.yml

@@ -15,6 +15,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_gen-dgrl/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_gym/environment.yml

@@ -19,6 +19,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_habitat/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-error-for-skips
     - pytest-rerunfailures
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy==1.9.1

.github/unittest/linux_libs/scripts_jumanji/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_llm/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_meltingpot/environment.yml

@@ -12,4 +12,5 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest

.github/unittest/linux_libs/scripts_minari/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_openx/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_robohive/environment.yml

@@ -19,6 +19,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_roboset/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_sklearn/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_smacv2/environment.yml

@@ -14,6 +14,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - numpy==1.23.0

.github/unittest/linux_libs/scripts_vd4rl/environment.yml

@@ -13,6 +13,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/unittest/linux_libs/scripts_vmas/environment.yml

@@ -19,6 +19,7 @@ dependencies:
     - pytest-instafail
     - pytest-rerunfailures
     - pytest-error-for-skips
+    - pytest-asyncio
     - expecttest
     - pyyaml
     - scipy

.github/workflows/build-wheels-aarch64-linux.yml

@@ -35,6 +35,7 @@ jobs:
           - repository: pytorch/rl
             smoke-test-script: test/smoke_test.py
             package-name: torchrl
+            pre-script: .github/scripts/pre-build-script.sh
     name: pytorch/rl
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
     with:

.github/workflows/build-wheels-linux.yml

@@ -34,6 +34,7 @@ jobs:
           - repository: pytorch/rl
             smoke-test-script: test/smoke_test.py
             package-name: torchrl
+            pre-script: .github/scripts/pre-build-script.sh
     name: pytorch/rl
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
     with:

.github/workflows/build-wheels-m1.yml

@@ -34,6 +34,7 @@ jobs:
           - repository: pytorch/rl
             smoke-test-script: test/smoke_test.py
             package-name: torchrl
+            pre-script: .github/scripts/pre-build-script.sh
     name: ${{ matrix.repository }}
     uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main
     with:

.github/workflows/build-wheels-windows.yml

@@ -32,7 +32,7 @@ jobs:
       matrix:
         include:
           - repository: pytorch/rl
-            pre-script: .github/scripts/td_script.sh
+            pre-script: .github/scripts/pre-build-script-win.sh
             env-script: .github/scripts/version_script.bat
             post-script: "python packaging/wheel/relocate.py"
             smoke-test-script: test/smoke_test.py

docs/source/_static/js/torchrl_theme.js

@@ -944,7 +944,7 @@ var downloadNote = $(".sphx-glr-download-link-note.admonition.note");
 if (downloadNote.length >= 1) {
     var tutorialUrlArray = $("#tutorial-type").text().split('/');
 
-    var githubLink = "https://github.com/pytorch/rl/tree/tutorial_py_dup/sphinx-tutorials/" + tutorialUrlArray[tutorialUrlArray.length - 1] + ".py",
+    var githubLink = "https://github.com/pytorch/rl/tree/tutorial_py_dup/sphinx-" + tutorialUrlArray[tutorialUrlArray.length - 1] + ".py",
         notebookLink = $(".sphx-glr-download-jupyter").find(".download.reference")[0].href,
         notebookDownloadPath = notebookLink.split('_downloads')[1],
         colabLink = "https://colab.research.google.com/github/pytorch/rl/blob/gh-pages/_downloads" + notebookDownloadPath;

docs/source/_templates/layout.html

@@ -60,7 +60,7 @@
 	  var downloadNote = $(".sphx-glr-download-link-note.admonition.note");
 	  if (downloadNote.length >= 1) {
 	      var tutorialUrl = $("#tutorial-type").text();
-	      var githubLink = "https://github.com/pytorch/rl/blob/main/tutorials/sphinx-tutorials/"  + tutorialUrl + ".py",
+	      var githubLink = "https://github.com/pytorch/rl/blob/main/tutorials/sphinx-"  + tutorialUrl + ".py",
 		  notebookLink = $(".sphx-glr-download-jupyter").find(".download.reference")[0].href,
 		  notebookDownloadPath = notebookLink.split('_downloads')[1],
 		  colabLink = "https://colab.research.google.com/github/pytorch/rl/blob/gh-pages/main/_downloads" + notebookDownloadPath;

docs/source/reference/collectors.rst

@@ -126,16 +126,16 @@ mechanism for updating policy weights across different devices and processes, ac
 Local and Remote Weight Updaters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The weight synchronization process is facilitated by two main components: :class:`~torchrl.collectors.LocalWeightUpdaterBase`
-and :class:`~torchrl.collectors.RemoteWeightUpdaterBase`. These base classes provide a structured interface for
+The weight synchronization process is facilitated by two main components: :class:`~torchrl.collectors.WeightUpdateReceiverBase`
+and :class:`~torchrl.collectors.WeightUpdateSenderBase`. These base classes provide a structured interface for
 implementing custom weight update logic, allowing users to tailor the synchronization process to their specific needs.
 
-- :class:`~torchrl.collectors.LocalWeightUpdaterBase`: This component is responsible for updating the policy weights on
+- :class:`~torchrl.collectors.WeightUpdateReceiverBase`: This component is responsible for updating the policy weights on
   the local inference worker. It is particularly useful when the training and inference occur on the same machine but on
   different devices. Users can extend this class to define how weights are fetched from a server and applied locally.
   It is also the extension point for collectors where the workers need to ask for weight updates (in contrast with
   situations where the server decides when to update the worker policies).
-- :class:`~torchrl.collectors.RemoteWeightUpdaterBase`: This component handles the distribution of policy weights to
+- :class:`~torchrl.collectors.WeightUpdateSenderBase`: This component handles the distribution of policy weights to
   remote inference workers. It is essential in distributed systems where multiple workers need to be kept in sync with
   the central policy. Users can extend this class to implement custom logic for synchronizing weights across a network of
   devices or processes.
@@ -153,8 +153,8 @@ Default Implementations
 
 For common scenarios, the API provides default implementations of these updaters, such as
 :class:`~torchrl.collectors.VanillaLocalWeightUpdater`, :class:`~torchrl.collectors.MultiProcessedRemoteWeightUpdate`,
-:class:`~torchrl.collectors.RayRemoteWeightUpdater`, :class:`~torchrl.collectors.RPCRemoteWeightUpdater`, and
-:class:`~torchrl.collectors.DistributedRemoteWeightUpdater`.
+:class:`~torchrl.collectors.RayWeightUpdateSender`, :class:`~torchrl.collectors.RPCWeightUpdateSender`, and
+:class:`~torchrl.collectors.DistributedWeightUpdateSender`.
 These implementations cover a range of typical deployment configurations, from single-device setups to large-scale
 distributed systems.
 
@@ -180,13 +180,13 @@ scenarios, ensuring that their policies remain up-to-date and performant.
     :toctree: generated/
     :template: rl_template.rst
 
-    LocalWeightUpdaterBase
-    RemoteWeightUpdaterBase
+    WeightUpdateReceiverBase
+    WeightUpdateSenderBase
     VanillaLocalWeightUpdater
     MultiProcessedRemoteWeightUpdate
-    RayRemoteWeightUpdater
-    DistributedRemoteWeightUpdater
-    RPCRemoteWeightUpdater
+    RayWeightUpdateSender
+    DistributedWeightUpdateSender
+    RPCWeightUpdateSender
 
 Collectors and replay buffers interoperability
 ----------------------------------------------
@@ -319,6 +319,21 @@ node or across multiple nodes.
     submitit_delayed_launcher
     RayCollector
 
+LLM Collectors
+---------------------------
+TorchRL also provides a data collectors for large language models. These collectors
+are meant to include a subset of the functionality of other data collectors, targeted
+at supporting researchers in fine-tuning large language models.  These classes 
+currently derive from the :class:`~torchrl.collectors.SyncDataCollector` class.
+These classes are experimental and subject to change.
+
+.. currentmodule:: torchrl.collectors.llm_collectors
+
+.. autosummary::
+    :toctree: generated/
+    :template: rl_template.rst
+
+    LLMCollector
 
 Helper functions
 ----------------

docs/source/reference/data.rst

@@ -1107,10 +1107,10 @@ and the tree can be expanded for each of these. The following figure shows how t
     Tree
 
 
-Reinforcement Learning From Human Feedback (RLHF)
--------------------------------------------------
+Large language models and Reinforcement Learning From Human Feedback (RLHF)
+---------------------------------------------------------------------------
 
-Data is of utmost importance in Reinforcement Learning from Human Feedback (RLHF).
+Data is of utmost importance in LLM post-training (e.g., GRPO or Reinforcement Learning from Human Feedback (RLHF)).
 Given that these techniques are commonly employed in the realm of language,
 which is scarcely addressed in other subdomains of RL within the library,
 we offer specific utilities to facilitate interaction with external libraries
@@ -1124,6 +1124,7 @@ efficient sampling.
     :toctree: generated/
     :template: rl_template.rst
 
+    History
     PairwiseDataset
     PromptData
     PromptTensorDictTokenizer