[Doc] Create your pendulum tutorial (#911)

Author: vmoens

.circleci/docs/setup_env.sh

@@ -0,0 +1,66 @@
+#apt-get update -y
+#apt-get install software-properties-common -y
+#add-apt-repository ppa:git-core/candidate -y
+#apt-get update -y
+#apt-get upgrade -y
+#apt-get -y install libglfw3 libglew2.0 gcc curl g++ unzip \
+#  wget sudo git cmake libz-dev \
+#  zlib1g-dev python3.8 python3-pip ninja
+
+#yum install -y mesa-libGL freeglut egl-utils glew glfw
+#yum install -y glew glfw
+apt-get update && apt-get install -y git wget gcc g++
+
+root_dir="$(pwd)"
+conda_dir="${root_dir}/conda"
+env_dir="${root_dir}/env"
+
+os=Linux
+
+# 1. Install conda at ./conda
+printf "* Installing conda\n"
+wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh"
+bash ./miniconda.sh -b -f -p "${conda_dir}"
+
+eval "$(${conda_dir}/bin/conda shell.bash hook)"
+
+printf "* Creating a test environment\n"
+conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION"
+
+printf "* Activating\n"
+conda activate "${env_dir}"
+
+conda install -c conda-forge zlib -y
+
+pip3 install --upgrade pip --quiet --root-user-action=ignore
+
+printf "python version\n"
+python --version
+
+pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu118 --quiet --root-user-action=ignore
+#pip3 install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu --quiet --root-user-action=ignore
+
+printf "Installing tensordict\n"
+pip3 install git+https://github.com/pytorch-labs/tensordict.git --quiet --root-user-action=ignore
+
+printf "Installing torchrl\n"
+pip3 install -e . --quiet --root-user-action=ignore
+
+printf "Installing requirements\n"
+pip3 install -r docs/requirements.txt --quiet --root-user-action=ignore
+printf "Installed all dependencies\n"
+
+printf "smoke test\n"
+PYOPENGL_PLATFORM=egl MUJOCO_GL=egl python3 -c """from torchrl.envs.libs.dm_control import DMControlEnv
+print(DMControlEnv('cheetah', 'run').reset())
+"""
+
+printf "building docs...\n"
+cd ./docs
+#timeout 7m bash -ic "MUJOCO_GL=egl sphinx-build SPHINXOPTS=-v ./source _local_build" || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+PYOPENGL_PLATFORM=egl MUJOCO_GL=egl sphinx-build ./source _local_build
+cd ..
+printf "done!\n"
+
+git clone --branch gh-pages https://github.com/pytorch-labs/tensordict.git docs/_local_build/tensordict
+rm -rf docs/_local_build/tensordict/.git

.circleci/unittest/linux_libs/scripts_habitat/install.sh

@@ -5,6 +5,7 @@ unset PYTORCH_VERSION
 # so no need to set PYTORCH_VERSION.
 # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
 apt-get update && apt-get install -y git wget gcc g++
+#apt-get update && apt-get install -y git wget freeglut3 freeglut3-dev
 
 set -e
 

.github/workflows/docs.yml

@@ -10,91 +10,53 @@ on:
   workflow_dispatch:
 jobs:
   build_docs_job:
+    strategy:
+      matrix:
+        python_version: ["3.8"] # "3.8", "3.9", "3.10", "3.11"
+        cuda_arch_version: ["11.8"] # "11.6", "11.7"
+      fail-fast: false
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    with:
+      runner: linux.g5.4xlarge.nvidia.gpu
+      repository: pytorch/rl
+      gpu-arch-type: cuda
+      gpu-arch-version: ${{ matrix.cuda_arch_version }}
+      docker-image: nvidia/cudagl:11.4.0-base
+      timeout: 45
+      script: |
+        # Set env vars from matrix
+        export PYTHON_VERSION=${{ matrix.python_version }}
+        # Commenting these out for now because the GPU test are not working inside docker
+        export CUDA_ARCH_VERSION=${{ matrix.cuda_arch_version }}
+        export CU_VERSION="cu${CUDA_ARCH_VERSION:0:2}${CUDA_ARCH_VERSION:3:1}"
+        # Remove the following line when the GPU tests are working inside docker, and uncomment the above lines
+        #export CU_VERSION="cpu"
+
+        echo "PYTHON_VERSION: $PYTHON_VERSION"
+        echo "CU_VERSION: $CU_VERSION"
+        
+        cd /work
+
+        ## setup_env.sh
+        ./.circleci/docs/setup_env.sh
+
+  deploy:
+    needs: build_docs_job
     runs-on: ubuntu-20.04
     strategy:
       matrix:
         include:
-         - os: linux.4xlarge.nvidia.gpu
+         - os: linux.12xlarge
            python-version: 3.8
     defaults:
       run:
         shell: bash -l {0}
     container: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu18.04
     steps:
-    - name: Install deps
-      run: |
-        apt-get update -y
-        apt-get install software-properties-common -y
-        add-apt-repository ppa:git-core/candidate -y
-        apt-get update -y
-        apt-get upgrade -y
-        apt-get -y install libglu1-mesa libgl1-mesa-glx libosmesa6 gcc curl g++ unzip wget libglfw3-dev libgles2-mesa-dev libglew-dev sudo git cmake libz-dev
-    - name: Install rsync 📚
-      run: |
-        apt-get update && apt-get install -y rsync
-    - name: Check ldd --version
-      run: ldd --version
     - name: Checkout
       uses: actions/checkout@v3
-    # Update references
-    - name: Update pip
-      run: |
-        apt-get install python3.8 python3-pip -y
-        pip3 install --upgrade pip
-    - name: Setup conda
-      run: |
-        rm -rf $HOME/miniconda
-        wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
-        bash ~/miniconda.sh -b -p $HOME/miniconda
-    - name: setup Path
-      run: |
-        echo "$HOME/miniconda/bin" >> $GITHUB_PATH
-        echo "CONDA=$HOME/miniconda" >> $GITHUB_PATH
-    - name: create and activate conda env
-      run: |
-        $HOME/miniconda/bin/conda create --name build_binary python=${{ matrix.python-version }}
-        $HOME/miniconda/bin/conda info
-        $HOME/miniconda/bin/activate build_binary
-    - name: check python version
-      run: |
-        python --version
-    - name: Check git version
-      run: git version
-    - name: setup Path
-      run: |
-        echo /usr/local/bin >> $GITHUB_PATH
-    - name: Install PyTorch
-      shell: bash
-      run: |
-        python -m pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
-    - name: Install tensordict
-      run: |
-        python3 -m pip install git+https://github.com/pytorch-labs/tensordict.git
-    - name: Install TorchRL
-      run: |
-        python -m pip install -e .
-    - name: Test torchrl installation
-      shell: bash
-      run: |
-        mkdir _tmp
-        cd _tmp
-        python -c "import torchrl;from torchrl.envs.libs.dm_control import DMControlEnv"
-        cd ..
-    - name: Build the docset
-      id: build_doc
-      run: |
-        python -m pip install -r docs/requirements.txt
-        cd ./docs
-        timeout 7m bash -ic "MUJOCO_GL=egl sphinx-build ./source _local_build" || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
-        cd ..
-    - name: Pull TensorDict docs
-      run: |
-        git clone --branch gh-pages https://github.com/pytorch-labs/tensordict.git docs/_local_build/tensordict
-        rm -rf docs/_local_build/tensordict/.git
-    - name: Get output time
-      run: echo "The time was ${{ steps.build.outputs.time }}"
     - name: Deploy
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
       uses: JamesIves/github-pages-deploy-action@releases/v4
       with:
           token: ${{ secrets.GITHUB_TOKEN }}

docs/Makefile

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS    ?=
+SPHINXOPTS    = -v
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source
 BUILDDIR      = build

docs/requirements.txt

@@ -12,16 +12,14 @@ sphinxcontrib-htmlhelp
 myst-parser
 docutils
 
-functorch
-gym[classic_control]
 torchvision
 dm_control
 atari-py
 ale-py
-gym[accept-rom-license]
+gym[classic_control,accept-rom-license]
 pygame
 tqdm
 ipython
-imageio
-imageio[ffmpeg]
-imageio[pyav]
+imageio[ffmpeg,pyav]
+memory_profiler
+pyrender

docs/source/conf.py

@@ -88,6 +88,9 @@
     "filename_pattern": "reference/generated/tutorials/",  # files to parse
     "notebook_images": "reference/generated/tutorials/media/",  # images to parse
     "download_all_examples": True,
+    "abort_on_example_error": False,
+    "only_warn_on_example_error": True,
+    "show_memory": True,
 }
 
 napoleon_use_ivar = True

docs/source/index.rst

@@ -34,18 +34,19 @@ Basics
 ------
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
-   tutorials/torchrl_demo
    tutorials/coding_ppo
+   tutorials/pendulum
    tutorials/tensordict_tutorial
    tutorials/tensordict_module
+   tutorials/torchrl_demo
 
 Intermediate
 ------------
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
    tutorials/torch_envs
    tutorials/pretrained_models
@@ -54,7 +55,7 @@ Advanced
 --------
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
    tutorials/multi_task
    tutorials/coding_ddpg

docs/source/reference/trainers.rst

@@ -14,6 +14,40 @@ The :obj:`trainer.train()` method can be sketched as follows:
 .. code-block::
    :caption: Trainer loops
 
+           >>> for batch in collector:
+           ...     batch = self._process_batch_hook(batch)  # "batch_process"
+           ...     self._pre_steps_log_hook(batch)  # "pre_steps_log"
+           ...     self._pre_optim_hook()  # "pre_optim_steps"
+           ...     for j in range(self.optim_steps_per_batch):
+           ...         sub_batch = self._process_optim_batch_hook(batch)  # "process_optim_batch"
+           ...         losses = self.loss_module(sub_batch)
+           ...         self._post_loss_hook(sub_batch)  # "post_loss"
+           ...         self.optimizer.step()
+           ...         self.optimizer.zero_grad()
+           ...         self._post_optim_hook()  # "post_optim"
+           ...         self._post_optim_log(sub_batch)  # "post_optim_log"
+           ...     self._post_steps_hook()  # "post_steps"
+           ...     self._post_steps_log_hook(batch)  #  "post_steps_log"
+
+   There are 10 hooks that can be used in a trainer loop:
+
+           >>> for batch in collector:
+           ...     batch = self._process_batch_hook(batch)  # "batch_process"
+           ...     self._pre_steps_log_hook(batch)  # "pre_steps_log"
+           ...     self._pre_optim_hook()  # "pre_optim_steps"
+           ...     for j in range(self.optim_steps_per_batch):
+           ...         sub_batch = self._process_optim_batch_hook(batch)  # "process_optim_batch"
+           ...         losses = self.loss_module(sub_batch)
+           ...         self._post_loss_hook(sub_batch)  # "post_loss"
+           ...         self.optimizer.step()
+           ...         self.optimizer.zero_grad()
+           ...         self._post_optim_hook()  # "post_optim"
+           ...         self._post_optim_log(sub_batch)  # "post_optim_log"
+           ...     self._post_steps_hook()  # "post_steps"
+           ...     self._post_steps_log_hook(batch)  #  "post_steps_log"
+
+   There are 10 hooks that can be used in a trainer loop:
+
         >>> for batch in collector:
         ...     batch = self._process_batch_hook(batch)  # "batch_process"
         ...     self._pre_steps_log_hook(batch)  # "pre_steps_log"

knowledge_base/MUJOCO_INSTALLATION.md

@@ -129,21 +129,23 @@ issues when running `import mujoco_py` and some troubleshooting for each of them
      #include <GL/glew.h>
               ^~~~~~~~~~~
     ```
+
     _Solution_: make sure glew is installed (see above: `conda install -c conda-forge glew` or the `apt-get` version of it).
 2. 
     ```
     include/GL/glu.h:38:10: fatal error: GL/gl.h: No such file or directory
       #include <GL/gl.h>
                ^~~~~~~~~
     ```
+
     _Solution_: This should disappear once `mesalib` is installed: `conda install -y -c conda-forge mesalib`
-3. 
+4. 
    ```
    FileNotFoundError: [Errno 2] No such file or directory: 'patchelf'
    ```
 
-   _Solution_: `pip install patchelf`
-4. 
+    _Solution_: `pip install patchelf`
+5. 
     ```
     ImportError: /usr/lib/x86_64-linux-gnu/libOpenGL.so.0: undefined symbol: _glapi_tls_Current
     ```
@@ -155,7 +157,7 @@ issues when running `import mujoco_py` and some troubleshooting for each of them
     conda env config vars set LD_PRELOAD=/path/to/conda/envs/mujoco_env/x86_64-conda-linux-gnu/sysroot/usr/lib64/libGLdispatch.so.0
     ```
 
-5.
+6. 
     ```
     mujoco.FatalError: gladLoadGL error
     
@@ -166,6 +168,7 @@ issues when running `import mujoco_py` and some troubleshooting for each of them
 
 
 **Sanity check**
+
 To check that your mujoco-py has been built against the GPU, run
 ```python
 >>> import mujoco_py
@@ -191,17 +194,18 @@ RuntimeError: Failed to initialize OpenGL
 
 2. Rendered images are completely black.
 
-> Make sure to call `env.render()` before reading the pixels.
+   _Solution_: Make sure to call `env.render()` before reading the pixels.
 
 3. `patchelf` dependency is missing.
 
-> Install using `conda install patchelf` or `pip install patchelf`
+   _Solution_: Install using `conda install patchelf` or `pip install patchelf`
 
 4. Errors like "Onscreen rendering needs 101 device"
 
-> Make sure to set `DISPLAY` environment variable correctly.
+   _Solution_: Make sure to set `DISPLAY` environment variable correctly.
 
 5. `ImportError: Cannot initialize a headless EGL display.`
-  Make sure you have installed mujoco and all its dependencies (see instructions above).
-  Make sure you have set the `MUJOCO_GL=egl`.
-  Make sure you have a GPU accessible on your machine.
+
+   _Solution_: Make sure you have installed mujoco and all its dependencies (see instructions above).
+   Make sure you have set the `MUJOCO_GL=egl`.
+   Make sure you have a GPU accessible on your machine.