Better requirements.txt docs (#369)

* Better requirements.txt docs

* add c51 docs

* update c51 docs

* update dqn docs

* update optuna docs

* update docs for ddpg, sac, and td3

* add dependencies

* remove poetry groups

* remove poetry groups for tests

* Update PPO, RND, and RPO docs

* update PPG docs

* update docs

* trigger CI

* fix CI

* Fix depdenencies

* Move up pip installation

.github/workflows/tests.yaml

@@ -26,22 +26,22 @@ jobs:
 
       # classic control tests
       - name: Install core dependencies
-        run: poetry install --with pytest
+        run: poetry install -E pytest
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: Run core tests
         run: poetry run pytest tests/test_classic_control.py
       - name: Install jax
         if: runner.os == 'Linux' || runner.os == 'macOS'
-        run: poetry install --with jax
+        run: poetry install -E "pytest jax"
       - name: Run core tests with jax
         if: runner.os == 'Linux' || runner.os == 'macOS'
         run: poetry run pytest tests/test_classic_control_jax.py
       - name: Run gae tests with jax
         if: runner.os == 'Linux' || runner.os == 'macOS'
         run: poetry run pytest tests/test_jax_compute_gae.py
       - name: Install tuner dependencies
-        run: poetry install --with optuna
+        run: poetry install -E "pytest optuna"
       - name: Run tuner tests
         run: poetry run pytest tests/test_tuner.py
 
@@ -65,14 +65,14 @@ jobs:
 
       # atari tests
       - name: Install atari dependencies
-        run: poetry install --with pytest,atari
+        run: poetry install -E "pytest atari"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: Run atari tests
         run: poetry run pytest tests/test_atari.py
       - name: Install jax
         if: runner.os == 'Linux' || runner.os == 'macOS'
-        run: poetry install --with jax
+        run: poetry install -E "pytest atari jax"
       - name: Run core tests with jax
         if: runner.os == 'Linux' || runner.os == 'macOS'
         run: poetry run pytest tests/test_atari_jax.py
@@ -97,9 +97,9 @@ jobs:
 
       # pybullet tests
       - name: Install core dependencies
-        run: poetry install --with pytest
+        run: poetry install -E pytest
       - name: Install pybullet dependencies
-        run: poetry install --with pybullet
+        run: poetry install -E "pytest pybullet"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: Run pybullet tests
@@ -125,7 +125,7 @@ jobs:
 
       # procgen tests
       - name: Install core dependencies
-        run: poetry install --with pytest,procgen
+        run: poetry install -E "pytest procgen"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: Run pybullet tests
@@ -151,7 +151,7 @@ jobs:
 
       # mujoco tests
       - name: Install dependencies
-        run: poetry install --with pytest,mujoco,dm_control
+        run: poetry install -E "pytest mujoco dm_control"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: install mujoco dependencies
@@ -181,7 +181,7 @@ jobs:
 
       # mujoco tests
       - name: Install dependencies
-        run: poetry install --with pytest,mujoco,dm_control
+        run: poetry install -E "pytest mujoco dm_control"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: Run mujoco tests
@@ -208,7 +208,7 @@ jobs:
 
       # mujoco_py tests
       - name: Install dependencies
-        run: poetry install --with pytest,pybullet,mujoco_py,mujoco,jax
+        run: poetry install -E "pytest pybullet mujoco_py mujoco jax"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: install mujoco_py dependencies
@@ -241,7 +241,7 @@ jobs:
 
       # envpool tests
       - name: Install envpool dependencies
-        run: poetry install --with pytest,envpool,jax
+        run: poetry install -E "pytest envpool jax"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: Run envpool tests
@@ -267,7 +267,7 @@ jobs:
 
       # atari multigpu tests
       - name: Install atari dependencies
-        run: poetry install --with pytest,atari
+        run: poetry install -E "pytest atari"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: Run atari tests
@@ -293,7 +293,7 @@ jobs:
 
       # pettingzoo tests
       - name: Install pettingzoo dependencies
-        run: poetry install --with pytest,pettingzoo,atari
+        run: poetry install -E "pytest pettingzoo atari"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: Install ROMs

.github/workflows/utils_test.yaml

@@ -25,9 +25,9 @@ jobs:
           poetry-version: ${{ matrix.poetry-version }}
 
       - name: Install test dependencies
-        run: poetry install --with pytest
+        run: poetry install -E pytest
       - name: Install cloud dependencies
-        run: poetry install --with cloud
+        run: poetry install -E "pytest cloud"
       - name: Downgrade setuptools
         run: poetry run pip install setuptools==59.5.0
       - name: Run utils tests

.pre-commit-config.yaml

@@ -46,37 +46,49 @@ repos:
         stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-atari.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-atari.txt", "--with", "atari"]
+        args: ["--without-hashes", "-o", "requirements/requirements-atari.txt", "-E", "atari"]
         stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-pybullet.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-pybullet.txt", "--with", "pybullet"]
+        args: ["--without-hashes", "-o", "requirements/requirements-pybullet.txt", "-E", "pybullet"]
+        stages: [manual]
+      - id: poetry-export
+        name: poetry-export requirements-mujoco.txt
+        args: ["--without-hashes", "-o", "requirements/requirements-mujoco.txt", "-E", "mujoco"]
+        stages: [manual]
+      - id: poetry-export
+        name: poetry-export requirements-dm_control.txt
+        args: ["--without-hashes", "-o", "requirements/requirements-dm_control.txt", "-E", "dm_control"]
         stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-mujoco_py.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-mujoco_py.txt", "--with", "mujoco_py"]
+        args: ["--without-hashes", "-o", "requirements/requirements-mujoco_py.txt", "-E", "mujoco_py"]
         stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-procgen.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-procgen.txt", "--with", "procgen"]
+        args: ["--without-hashes", "-o", "requirements/requirements-procgen.txt", "-E", "procgen"]
         stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-envpool.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-envpool.txt", "--with", "envpool"]
+        args: ["--without-hashes", "-o", "requirements/requirements-envpool.txt", "-E", "envpool"]
         stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-pettingzoo.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-pettingzoo.txt", "--with", "pettingzoo"]
+        args: ["--without-hashes", "-o", "requirements/requirements-pettingzoo.txt", "-E", "pettingzoo"]
         stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-jax.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-jax.txt", "--with", "jax"]
+        args: ["--without-hashes", "-o", "requirements/requirements-jax.txt", "-E", "jax"]
+        stages: [manual]
+      - id: poetry-export
+        name: poetry-export requirements-optuna.txt
+        args: ["--without-hashes", "-o", "requirements/requirements-optuna.txt", "-E", "optuna"]
         stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-docs.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-docs.txt", "--with", "docs"]
+        args: ["--without-hashes", "-o", "requirements/requirements-docs.txt", "-E", "docs"]
         stages: [manual]
       - id: poetry-export
         name: poetry-export requirements-cloud.txt
-        args: ["--without-hashes", "-o", "requirements/requirements-cloud.txt", "--with", "cloud"]
+        args: ["--without-hashes", "-o", "requirements/requirements-cloud.txt", "-E", "cloud"]
         stages: [manual]

Dockerfile

@@ -12,16 +12,14 @@ RUN pip install poetry --upgrade
 COPY pyproject.toml pyproject.toml
 COPY poetry.lock poetry.lock
 RUN poetry install
-RUN poetry install --with atari
-RUN poetry install --with pybullet
 
 # install mujoco_py
 RUN apt-get -y install wget unzip software-properties-common \
     libgl1-mesa-dev \
     libgl1-mesa-glx \
     libglew-dev \
     libosmesa6-dev patchelf
-RUN poetry install --with mujoco_py
+RUN poetry install -E "atari mujoco_py pybullet"
 RUN poetry run python -c "import mujoco_py"
 
 COPY entrypoint.sh /usr/local/bin/

README.md

@@ -70,6 +70,26 @@ poetry run python cleanrl/ppo.py \
     --wandb-project-name cleanrltest
 ```
 
+If you are not using `poetry`, you can install CleanRL with `requirements.txt`:
+
+```bash
+# core dependencies
+pip install -r requirements/requirements.txt
+
+# optional dependencies
+pip install -r requirements/requirements-atari.txt
+pip install -r requirements/requirements-pybullet.txt
+pip install -r requirements/requirements-mujoco.txt
+pip install -r requirements/requirements-mujoco_py.txt
+pip install -r requirements/requirements/requirements-mujoco.txt
+pip install -r requirements/requirements-procgen.txt
+pip install -r requirements/requirements-envpool.txt
+pip install -r requirements/requirements-pettingzoo.txt
+pip install -r requirements/requirements-jax.txt
+pip install -r requirements/requirements-docs.txt
+pip install -r requirements/requirements-cloud.txt
+```
+
 To run training scripts in other games:
 ```
 poetry shell
@@ -80,31 +100,32 @@ python cleanrl/ppo.py --env-id CartPole-v1
 python cleanrl/c51.py --env-id CartPole-v1
 
 # atari
-poetry install --with atari
+poetry install -E atari
 python cleanrl/dqn_atari.py --env-id BreakoutNoFrameskip-v4
 python cleanrl/c51_atari.py --env-id BreakoutNoFrameskip-v4
 python cleanrl/ppo_atari.py --env-id BreakoutNoFrameskip-v4
 python cleanrl/sac_atari.py --env-id BreakoutNoFrameskip-v4
 
 # NEW: 3-4x side-effects free speed up with envpool's atari (only available to linux)
-poetry install --with envpool
+poetry install -E envpool
 python cleanrl/ppo_atari_envpool.py --env-id BreakoutNoFrameskip-v4
 # Learn Pong-v5 in ~5-10 mins
 # Side effects such as lower sample efficiency might occur
 poetry run python ppo_atari_envpool.py --clip-coef=0.2 --num-envs=16 --num-minibatches=8 --num-steps=128 --update-epochs=3
 
 # pybullet
-poetry install --with pybullet
+poetry install -E pybullet
 python cleanrl/td3_continuous_action.py --env-id MinitaurBulletDuckEnv-v0
 python cleanrl/ddpg_continuous_action.py --env-id MinitaurBulletDuckEnv-v0
 python cleanrl/sac_continuous_action.py --env-id MinitaurBulletDuckEnv-v0
 
 # procgen
-poetry install --with procgen
+poetry install -E procgen
 python cleanrl/ppo_procgen.py --env-id starpilot
 python cleanrl/ppg_procgen.py --env-id starpilot
 
 # ppo + lstm
+poetry install -E atari
 python cleanrl/ppo_atari_lstm.py --env-id BreakoutNoFrameskip-v4
 ```
 

benchmark/c51.sh

@@ -5,7 +5,7 @@ OMP_NUM_THREADS=1 xvfb-run -a poetry run python -m cleanrl_utils.benchmark \
     --num-seeds 3 \
     --workers 9
 
-poetry install --with atari
+poetry install -E atari
 OMP_NUM_THREADS=1 xvfb-run -a poetry run python -m cleanrl_utils.benchmark \
     --env-ids PongNoFrameskip-v4 BeamRiderNoFrameskip-v4 BreakoutNoFrameskip-v4 \
     --command "poetry run python cleanrl/c51_atari.py --track --capture-video" \

benchmark/ddpg.sh

@@ -1,12 +1,12 @@
-poetry install --with mujoco_py,pybullet
+poetry install -E "mujoco_py pybullet"
 python -c "import mujoco_py"
 xvfb-run -a python -m cleanrl_utils.benchmark \
     --env-ids HalfCheetah-v2 Walker2d-v2 Hopper-v2 InvertedPendulum-v2 Humanoid-v2 Pusher-v2 \
     --command "poetry run python cleanrl/ddpg_continuous_action.py --track --capture-video" \
     --num-seeds 3 \
     --workers 1
 
-poetry install --with mujoco_py,jax
+poetry install -E "mujoco_py pybullet jax"
 poetry run pip install --upgrade "jax[cuda]==0.3.17" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
 poetry run python -c "import mujoco_py"
 xvfb-run -a poetry run python -m cleanrl_utils.benchmark \

benchmark/dqn.sh

@@ -5,22 +5,22 @@ OMP_NUM_THREADS=1 xvfb-run -a poetry run python -m cleanrl_utils.benchmark \
     --num-seeds 3 \
     --workers 9
 
-poetry install --with atari
+poetry install -E atari
 OMP_NUM_THREADS=1 xvfb-run -a poetry run python -m cleanrl_utils.benchmark \
     --env-ids PongNoFrameskip-v4 BeamRiderNoFrameskip-v4 BreakoutNoFrameskip-v4 \
     --command "poetry run python cleanrl/dqn_atari.py --track --capture-video" \
     --num-seeds 3 \
     --workers 1
 
-poetry install --with jax
+poetry install -E jax
 poetry run pip install --upgrade "jax[cuda]==0.3.17" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
 xvfb-run -a python -m cleanrl_utils.benchmark \
     --env-ids CartPole-v1 Acrobot-v1 MountainCar-v0 \
     --command "poetry run python cleanrl/dqn_jax.py --track --capture-video" \
     --num-seeds 3 \
     --workers 1
 
-poetry install --with atari,jax
+poetry install -E "atari jax"
 poetry run pip install --upgrade "jax[cuda]==0.3.17" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
 xvfb-run -a python -m cleanrl_utils.benchmark \
     --env-ids PongNoFrameskip-v4 BeamRiderNoFrameskip-v4 BreakoutNoFrameskip-v4 \

benchmark/ppg.sh

@@ -1,6 +1,6 @@
 # export WANDB_ENTITY=openrlbenchmark
 
-poetry install --with procgen
+poetry install -E procgen
 xvfb-run -a poetry run python -m cleanrl_utils.benchmark \
     --env-ids starpilot bossfight bigfish \
     --command "poetry run python cleanrl/ppg_procgen.py --track --capture-video" \