diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2f898dd2987..bbb9a767727 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -206,7 +206,30 @@ jobs:
     steps:
       - checkout_merge
       - designate_upload_channel
-      - run: packaging/build_wheel.sh
+      - run: packaging/build_wheels.sh
+      - store_artifacts:
+          path: dist
+      - persist_to_workspace:
+          root: dist
+          paths:
+            - "*"
+
+  binary_macos_wheel:
+    <<: *binary_common
+    macos:
+      xcode: "14.0"
+    steps:
+      - checkout_merge
+      - designate_upload_channel
+      - run:
+          # Cannot easily deduplicate this as source'ing activate
+          # will set environment variables which we need to propagate
+          # to build_wheel.sh
+          command: |
+            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            sh conda.sh -b
+            source $HOME/miniconda3/bin/activate
+            packaging/build_wheels.sh
       - store_artifacts:
           path: dist
       - persist_to_workspace:
@@ -229,25 +252,24 @@ jobs:
     steps:
       - checkout
       - designate_upload_channel
-#      - run:
-#          name: Generate cache key
-#          # This will refresh cache on Sundays, nightly build should generate new cache.
-#          command: echo "$(date +"%Y-%U")" > .circleci-weekly
-#      - restore_cache:
-#          keys:
-#            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
-
+      - run:
+          name: Generate cache key
+          # This will refresh cache on Sundays, nightly build should generate new cache.
+          command: echo "$(date +"%Y-%U")" > .circleci-weekly
+      - restore_cache:
+          keys:
+            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
 
-#      - save_cache:
-#
-#          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
-#
-#          paths:
-#            - conda
-#            - env
+      - save_cache:
+
+          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+
+          paths:
+            - conda
+            - env
       - run:
           name: Install torchrl
           command: .circleci/unittest/linux/scripts/install.sh
@@ -274,25 +296,25 @@ jobs:
     steps:
       - checkout
       - designate_upload_channel
-#      - run:
-#          name: Generate cache key
-#          # This will refresh cache on Sundays, nightly build should generate new cache.
-#          command: echo "$(date +"%Y-%U")" > .circleci-weekly
-#      - restore_cache:
-#
-#          keys:
-#            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+      - run:
+          name: Generate cache key
+          # This will refresh cache on Sundays, nightly build should generate new cache.
+          command: echo "$(date +"%Y-%U")" > .circleci-weekly
+      - restore_cache:
+
+          keys:
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
 
       - run:
           name: Setup
           command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
-#      - save_cache:
-#
-#          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
-#
-#          paths:
-#            - conda
-#            - env
+      - save_cache:
+
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+
+          paths:
+            - conda
+            - env
       - run:
           # Here we create an envlist file that contains some env variables that we want the docker container to be aware of.
           # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables.
@@ -327,25 +349,25 @@ jobs:
     steps:
       - checkout
       - designate_upload_channel
-#      - run:
-#          name: Generate cache key
-#          # This will refresh cache on Sundays, nightly build should generate new cache.
-#          command: echo "$(date +"%Y-%U")" > .circleci-weekly
-#      - restore_cache:
-#
-#          keys:
-#            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_optdeps/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+      - run:
+          name: Generate cache key
+          # This will refresh cache on Sundays, nightly build should generate new cache.
+          command: echo "$(date +"%Y-%U")" > .circleci-weekly
+      - restore_cache:
+
+          keys:
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_optdeps/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
 
       - run:
           name: Setup
           command: .circleci/unittest/linux_optdeps/scripts/setup_env.sh
-#      - save_cache:
-#
-#          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_optdeps/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
-#
-#          paths:
-#            - conda
-#            - env
+      - save_cache:
+
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_optdeps/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+
+          paths:
+            - conda
+            - env
       - run:
           # Here we create an envlist file that contains some env variables that we want the docker container to be aware of.
           # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables.
@@ -381,26 +403,26 @@ jobs:
     steps:
       - checkout
       - designate_upload_channel
-#      - run:
-#          name: Generate cache key
-#          # This will refresh cache on Sundays, nightly build should generate new cache.
-#          command: echo "$(date +"%Y-%U")" > .circleci-weekly
-#      - restore_cache:
-#
-#          keys:
-#            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+      - run:
+          name: Generate cache key
+          # This will refresh cache on Sundays, nightly build should generate new cache.
+          command: echo "$(date +"%Y-%U")" > .circleci-weekly
+      - restore_cache:
+
+          keys:
+            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
 
       - run:
           name: Setup
           command: .circleci/unittest/linux_stable/scripts/setup_env.sh
 
-#      - save_cache:
-#
-#          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
-#
-#          paths:
-#            - conda
-#            - env
+      - save_cache:
+
+          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+
+          paths:
+            - conda
+            - env
       - run:
           name: Install torchrl
           command: .circleci/unittest/linux_stable/scripts/install.sh
@@ -427,25 +449,25 @@ jobs:
     steps:
       - checkout
       - designate_upload_channel
-#      - run:
-#          name: Generate cache key
-#          # This will refresh cache on Sundays, nightly build should generate new cache.
-#          command: echo "$(date +"%Y-%U")" > .circleci-weekly
-#      - restore_cache:
-#
-#          keys:
-#            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+      - run:
+          name: Generate cache key
+          # This will refresh cache on Sundays, nightly build should generate new cache.
+          command: echo "$(date +"%Y-%U")" > .circleci-weekly
+      - restore_cache:
+
+          keys:
+            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
 
       - run:
           name: Setup
           command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux_stable/scripts/setup_env.sh
-#      - save_cache:
-#
-#          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
-#
-#          paths:
-#            - conda
-#            - env
+      - save_cache:
+
+          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux_stable/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+
+          paths:
+            - conda
+            - env
       - run:
           # Here we create an envlist file that contains some env variables that we want the docker container to be aware of.
           # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables.
@@ -479,25 +501,25 @@ jobs:
           name: Install wget
           command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
           # Disable brew auto update which is very slow
-#      - run:
-#          name: Generate cache key
-#          # This will refresh cache on Sundays, nightly build should generate new cache.
-#          command: echo "$(date +"%Y-%U")" > .circleci-weekly
-#      - restore_cache:
-#
-#          keys:
-#            - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+      - run:
+          name: Generate cache key
+          # This will refresh cache on Sundays, nightly build should generate new cache.
+          command: echo "$(date +"%Y-%U")" > .circleci-weekly
+      - restore_cache:
+
+          keys:
+            - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
 
       - run:
           name: Setup
           command: .circleci/unittest/linux/scripts/setup_env.sh
-#      - save_cache:
-#
-#          key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
-#
-#          paths:
-#            - conda
-#            - env
+      - save_cache:
+
+          key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
+
+          paths:
+            - conda
+            - env
       - run:
           name: Install torchrl
           command: .circleci/unittest/linux/scripts/install.sh
@@ -518,6 +540,120 @@ workflows:
       - lint_c
 #      - type_check_python
 
+  build:
+    jobs:
+      - binary_linux_wheel:
+          conda_docker_image: pytorch/conda-builder:cpu
+          cu_version: cpu
+          name: binary_linux_wheel_py3.7_cpu
+          python_version: '3.7'
+          wheel_docker_image: pytorch/manylinux-cuda102
+
+      - binary_linux_wheel:
+          conda_docker_image: pytorch/conda-builder:cpu
+          cu_version: cpu
+          name: binary_linux_wheel_py3.8_cpu
+          python_version: '3.8'
+          wheel_docker_image: pytorch/manylinux-cuda102
+
+      - binary_linux_wheel:
+          conda_docker_image: pytorch/conda-builder:cpu
+          cu_version: cpu
+          name: binary_linux_wheel_py3.9_cpu
+          python_version: '3.9'
+          wheel_docker_image: pytorch/manylinux-cuda102
+
+      - binary_linux_wheel:
+          conda_docker_image: pytorch/conda-builder:cpu
+          cu_version: cpu
+          name: binary_linux_wheel_py3.10_cpu
+          python_version: '3.10'
+          wheel_docker_image: pytorch/manylinux-cuda102
+
+#      - binary_linux_wheel:
+#          conda_docker_image: pytorch/conda-builder:cuda102
+#          cu_version: cu102
+#          name: binary_linux_wheel_py3.7_cu102
+#          python_version: '3.7'
+#          wheel_docker_image: pytorch/manylinux-cuda102
+#
+#      - binary_linux_wheel:
+#          conda_docker_image: pytorch/conda-builder:cuda102
+#          cu_version: cu102
+#          name: binary_linux_wheel_py3.8_cu102
+#          python_version: '3.8'
+#          wheel_docker_image: pytorch/manylinux-cuda102
+#
+#      - binary_linux_wheel:
+#          conda_docker_image: pytorch/conda-builder:cuda102
+#          cu_version: cu102
+#          name: binary_linux_wheel_py3.9_cu102
+#          python_version: '3.9'
+#          wheel_docker_image: pytorch/manylinux-cuda102
+#
+#      - binary_linux_wheel:
+#          conda_docker_image: pytorch/conda-builder:cuda102
+#          cu_version: cu102
+#          name: binary_linux_wheel_py3.10_cu102
+#          python_version: '3.10'
+#          wheel_docker_image: pytorch/manylinux-cuda102
+
+#      - binary_linux_wheel:
+#          conda_docker_image: pytorch/conda-builder:cuda113
+#          cu_version: cu113
+#          name: binary_linux_wheel_py3.7_cu113
+#          python_version: '3.7'
+#          wheel_docker_image: pytorch/manylinux-cuda113
+#
+#      - binary_linux_wheel:
+#          conda_docker_image: pytorch/conda-builder:cuda113
+#          cu_version: cu113
+#          name: binary_linux_wheel_py3.8_cu113
+#          python_version: '3.8'
+#          wheel_docker_image: pytorch/manylinux-cuda113
+#
+#      - binary_linux_wheel:
+#          conda_docker_image: pytorch/conda-builder:cuda113
+#          cu_version: cu113
+#          name: binary_linux_wheel_py3.9_cu113
+#          python_version: '3.9'
+#          wheel_docker_image: pytorch/manylinux-cuda113
+#
+#      - binary_linux_wheel:
+#          conda_docker_image: pytorch/conda-builder:cuda113
+#          cu_version: cu113
+#          name: binary_linux_wheel_py3.10_cu113
+#          python_version: '3.10'
+#          wheel_docker_image: pytorch/manylinux-cuda113
+
+      - binary_macos_wheel:
+          conda_docker_image: pytorch/conda-builder:cpu
+          cu_version: cpu
+          name: binary_macos_wheel_py3.7_cpu
+          python_version: '3.7'
+          wheel_docker_image: pytorch/manylinux-cuda102
+
+      - binary_macos_wheel:
+          conda_docker_image: pytorch/conda-builder:cpu
+          cu_version: cpu
+          name: binary_macos_wheel_py3.8_cpu
+          python_version: '3.8'
+          wheel_docker_image: pytorch/manylinux-cuda102
+
+      - binary_macos_wheel:
+          conda_docker_image: pytorch/conda-builder:cpu
+          cu_version: cpu
+          name: binary_macos_wheel_py3.9_cpu
+          python_version: '3.9'
+          wheel_docker_image: pytorch/manylinux-cuda102
+
+      - binary_macos_wheel:
+          conda_docker_image: pytorch/conda-builder:cpu
+          cu_version: cpu
+          name: binary_macos_wheel_py3.10_cpu
+          python_version: '3.10'
+          wheel_docker_image: pytorch/manylinux-cuda102
+
   unittest:
     jobs:
       - unittest_macos_cpu:
diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml
index c0e9f45e519..01676cfe837 100644
--- a/.circleci/unittest/linux/scripts/environment.yml
+++ b/.circleci/unittest/linux/scripts/environment.yml
@@ -25,3 +25,4 @@ dependencies:
     - mujoco_py
     - hydra-core
     - pyrender
+    - tensorboard
diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh
index f16caf6ed36..23267bdc8d1 100755
--- a/.circleci/unittest/linux/scripts/install.sh
+++ b/.circleci/unittest/linux/scripts/install.sh
@@ -53,7 +53,7 @@ pip install "git+https://github.com/pytorch/functorch.git"
 python -c "import functorch"
 
 printf "* Installing torchrl\n"
-pip install -e .
+python setup.py develop
 
 if [[ $OSTYPE == 'darwin'* ]]; then
   PRIVATE_MUJOCO_GL=glfw
diff --git a/.circleci/unittest/linux_optdeps/scripts/install.sh b/.circleci/unittest/linux_optdeps/scripts/install.sh
index 57b3db9caea..9cba799bd3a 100755
--- a/.circleci/unittest/linux_optdeps/scripts/install.sh
+++ b/.circleci/unittest/linux_optdeps/scripts/install.sh
@@ -53,7 +53,7 @@ pip install "git+https://github.com/pytorch/functorch.git"
 python -c "import functorch"
 
 printf "* Installing torchrl\n"
-pip install -e .
+python setup.py develop
 
 # smoke test
 python -c "import torchrl"
diff --git a/.circleci/unittest/linux_stable/scripts/environment.yml b/.circleci/unittest/linux_stable/scripts/environment.yml
index c73b477f608..99b4ae39649 100644
--- a/.circleci/unittest/linux_stable/scripts/environment.yml
+++ b/.circleci/unittest/linux_stable/scripts/environment.yml
@@ -26,3 +26,4 @@ dependencies:
     - mujoco_py
     - hydra-core
     - pyrender
+    - tensorboard
diff --git a/.circleci/unittest/linux_stable/scripts/install.sh b/.circleci/unittest/linux_stable/scripts/install.sh
index b214b7b66d8..47321c32001 100755
--- a/.circleci/unittest/linux_stable/scripts/install.sh
+++ b/.circleci/unittest/linux_stable/scripts/install.sh
@@ -52,7 +52,7 @@ printf "* Installing torchrl\n"
 printf "g++ version: "
 gcc --version
 
-pip install -e .
+python setup.py develop
 
 if [[ $OSTYPE == 'darwin'* ]]; then
   PRIVATE_MUJOCO_GL=glfw
diff --git a/packaging/build_wheels.sh b/packaging/build_wheels.sh
new file mode 100755
index 00000000000..d16471b21d9
--- /dev/null
+++ b/packaging/build_wheels.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+set -ex
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+. "$script_dir/pkg_helpers.bash"
+
+export BUILD_TYPE=wheel
+setup_env
+setup_wheel_python
+pip_install numpy pyyaml future ninja
+pip_install --upgrade setuptools
+setup_pip_pytorch_version
+python setup.py clean
+
+# Copy binaries to be included in the wheel distribution
+if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
+    python_exec="$(which python)"
+    bin_path=$(dirname $python_exec)
+    env_path=$(dirname $bin_path)
+    if [[ "$(uname)" == Darwin ]]; then
+        # Install delocate to relocate the required binaries
+        pip_install "delocate>=0.9"
+    else
+        cp "$bin_path/Library/bin/libpng16.dll" torchvision
+        cp "$bin_path/Library/bin/libjpeg.dll" torchvision
+    fi
+else
+    # Install auditwheel to get some inspection utilities
+    pip_install auditwheel
+
+    # Point to custom libraries
+    export LD_LIBRARY_PATH=$(pwd)/ext_libraries/lib:$LD_LIBRARY_PATH
+fi
+
+if [[ "$OSTYPE" == "msys" ]]; then
+  echo "ERROR: Windows installation is not supported yet." && exit 100
+else
+    python setup.py bdist_wheel
+    if [[ "$(uname)" != Darwin ]]; then
+      rename "linux_x86_64" "manylinux1_x86_64" dist/*.whl
+    fi
+fi
+
+#if [[ "$(uname)" == Darwin ]]; then
+#    pushd dist/
+#    python_exec="$(which python)"
+#    bin_path=$(dirname $python_exec)
+#    env_path=$(dirname $bin_path)
+#    for whl in *.whl; do
+#        DYLD_FALLBACK_LIBRARY_PATH="$env_path/lib/:$DYLD_FALLBACK_LIBRARY_PATH" delocate-wheel -v --ignore-missing-dependencies $whl
+#    done
+#else
+#    if [[ "$OSTYPE" == "msys" ]]; then
+#        "$script_dir/windows/internal/vc_env_helper.bat" python $script_dir/wheel/relocate.py
+#    else
+#        LD_LIBRARY_PATH="/usr/local/lib:$CUDA_HOME/lib64:$LD_LIBRARY_PATH" python $script_dir/wheel/relocate.py
+#    fi
+#fi
diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash
new file mode 100644
index 00000000000..734a6ae4403
--- /dev/null
+++ b/packaging/pkg_helpers.bash
@@ -0,0 +1,399 @@
+# A set of useful bash functions for common functionality we need to do in
+# many build scripts
+
+
+# Setup CUDA environment variables, based on CU_VERSION
+#
+# Inputs:
+#   CU_VERSION (cpu, cu92, cu100)
+#   NO_CUDA_PACKAGE (bool)
+#   BUILD_TYPE (conda, wheel)
+#
+# Outputs:
+#   VERSION_SUFFIX (e.g., "")
+#   PYTORCH_VERSION_SUFFIX (e.g., +cpu)
+#   WHEEL_DIR (e.g., cu100/)
+#   CUDA_HOME (e.g., /usr/local/cuda-9.2, respected by torch.utils.cpp_extension)
+#   FORCE_CUDA (respected by torchvision setup.py)
+#   NVCC_FLAGS (respected by torchvision setup.py)
+#
+# Precondition: CUDA versions are installed in their conventional locations in
+# /usr/local/cuda-*
+#
+# NOTE: Why VERSION_SUFFIX versus PYTORCH_VERSION_SUFFIX?  If you're building
+# a package with CUDA on a platform we support CUDA on, VERSION_SUFFIX ==
+# PYTORCH_VERSION_SUFFIX and everyone is happy.  However, if you are building a
+# package with only CPU bits (e.g., torchaudio), then VERSION_SUFFIX is always
+# empty, but PYTORCH_VERSION_SUFFIX is +cpu (because that's how you get a CPU
+# version of a Python package.  But that doesn't apply if you're on OS X,
+# since the default CU_VERSION on OS X is cpu.
+setup_cuda() {
+
+  # First, compute version suffixes.  By default, assume no version suffixes
+  export VERSION_SUFFIX=""
+  export PYTORCH_VERSION_SUFFIX=""
+  export WHEEL_DIR=""
+  # Wheel builds need suffixes (but not if they're on OS X, which never has suffix)
+  if [[ "$BUILD_TYPE" == "wheel" ]] && [[ "$(uname)" != Darwin ]]; then
+    export PYTORCH_VERSION_SUFFIX="+$CU_VERSION"
+    # Match the suffix scheme of pytorch, unless this package does not have
+    # CUDA builds (in which case, use default)
+    if [[ -z "$NO_CUDA_PACKAGE" ]]; then
+      export VERSION_SUFFIX="$PYTORCH_VERSION_SUFFIX"
+      export WHEEL_DIR="$CU_VERSION/"
+    fi
+  fi
+
+  # Now work out the CUDA settings
+  case "$CU_VERSION" in
+    cu115)
+      if [[ "$OSTYPE" == "msys" ]]; then
+        export CUDA_HOME="C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.5"
+      else
+        export CUDA_HOME=/usr/local/cuda-11.5/
+      fi
+      export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
+      ;;
+    cu113)
+      if [[ "$OSTYPE" == "msys" ]]; then
+        export CUDA_HOME="C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.3"
+      else
+        export CUDA_HOME=/usr/local/cuda-11.3/
+      fi
+      export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
+      ;;
+    cu112)
+      if [[ "$OSTYPE" == "msys" ]]; then
+        export CUDA_HOME="C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.2"
+      else
+        export CUDA_HOME=/usr/local/cuda-11.2/
+      fi
+      export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
+      ;;
+    cu111)
+      if [[ "$OSTYPE" == "msys" ]]; then
+        export CUDA_HOME="C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.1"
+      else
+        export CUDA_HOME=/usr/local/cuda-11.1/
+      fi
+      export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
+      ;;
+    cu110)
+      if [[ "$OSTYPE" == "msys" ]]; then
+        export CUDA_HOME="C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.0"
+      else
+        export CUDA_HOME=/usr/local/cuda-11.0/
+      fi
+      export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0"
+      ;;
+    cu102)
+      if [[ "$OSTYPE" == "msys" ]]; then
+        export CUDA_HOME="C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2"
+      else
+        export CUDA_HOME=/usr/local/cuda-10.2/
+      fi
+      export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5"
+      ;;
+    cu101)
+      if [[ "$OSTYPE" == "msys" ]]; then
+        export CUDA_HOME="C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.1"
+      else
+        export CUDA_HOME=/usr/local/cuda-10.1/
+      fi
+      export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5"
+      ;;
+    cu100)
+      if [[ "$OSTYPE" == "msys" ]]; then
+        export CUDA_HOME="C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.0"
+      else
+        export CUDA_HOME=/usr/local/cuda-10.0/
+      fi
+      export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5"
+      ;;
+    cu92)
+      if [[ "$OSTYPE" == "msys" ]]; then
+        export CUDA_HOME="C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.2"
+      else
+        export CUDA_HOME=/usr/local/cuda-9.2/
+      fi
+      export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0"
+      ;;
+    cpu)
+      ;;
+    rocm*)
+      export FORCE_CUDA=1
+      ;;
+    *)
+      echo "Unrecognized CU_VERSION=$CU_VERSION"
+      exit 1
+      ;;
+  esac
+  if [[ -n "$CUDA_HOME" ]]; then
+    # Adds nvcc binary to the search path so that CMake's `find_package(CUDA)` will pick the right one
+    export PATH="$CUDA_HOME/bin:$PATH"
+    export FORCE_CUDA=1
+  fi
+}
+
+# Populate build version if necessary, and add version suffix
+#
+# Inputs:
+#   BUILD_VERSION (e.g., 0.1.0 or empty)
+#   VERSION_SUFFIX (e.g., +cpu)
+#
+# Outputs:
+#   BUILD_VERSION (e.g., 0.1.0.dev20190807+cpu)
+#
+# Fill BUILD_VERSION if it doesn't exist already with a nightly string
+# Usage: setup_build_version 0.1.0
+setup_build_version() {
+  if [[ -z "$BUILD_VERSION" ]]; then
+    if [[ -z "$1" ]]; then
+      setup_base_build_version
+    else
+      BUILD_VERSION="$1"
+    fi
+    BUILD_VERSION="$BUILD_VERSION.dev$(date "+%Y%m%d")$VERSION_SUFFIX"
+  else
+    BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX"
+  fi
+
+  # Set build version based on tag if on tag
+  if [[ -n "${CIRCLE_TAG}" ]]; then
+    # Strip tag
+    export BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')${VERSION_SUFFIX}"
+  fi
+}
+
+setup_base_build_version() {
+  SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+  # version.txt for some reason has `a` character after major.minor.rev
+  # command below yields 0.10.0 from version.txt containing 0.10.0a0
+  BUILD_VERSION=$( cut -f 1 -d a "$SCRIPT_DIR/../version.txt" )
+  export BUILD_VERSION
+}
+
+# Set some useful variables for OS X, if applicable
+setup_macos() {
+  if [[ "$(uname)" == Darwin ]]; then
+    export MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++
+  fi
+}
+
+
+# Top-level entry point for things every package will need to do
+#
+# Usage: setup_env 0.1.0
+setup_env() {
+  setup_cuda
+  setup_build_version "$1"
+  setup_macos
+}
+
+# Function to retry functions that sometimes timeout or have flaky failures
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+# Inputs:
+#   PYTHON_VERSION (3.7, 3.8, 3.9)
+#   UNICODE_ABI (bool)
+#
+# Outputs:
+#   PATH modified to put correct Python version in PATH
+#
+# Precondition: If Linux, you are in a soumith/manylinux-cuda* Docker image
+setup_wheel_python() {
+  if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
+    eval "$(conda shell.bash hook)"
+    conda env remove -n "env$PYTHON_VERSION" || true
+    conda create ${CONDA_CHANNEL_FLAGS} -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
+    conda activate "env$PYTHON_VERSION"
+  else
+    case "$PYTHON_VERSION" in
+      3.7) python_abi=cp37-cp37m ;;
+      3.8) python_abi=cp38-cp38 ;;
+      3.9) python_abi=cp39-cp39 ;;
+      3.10) python_abi=cp310-cp310 ;;
+      *)
+        echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION"
+        exit 1
+        ;;
+    esac
+    # Download all the dependencies required to compile image and video_reader
+    # extensions
+
+    mkdir -p ext_libraries
+    pushd ext_libraries
+    popd
+    export PATH="/opt/python/$python_abi/bin:$(pwd)/ext_libraries/bin:$PATH"
+    export LD_LIBRARY_PATH=/opt/python/$python_abi/lib/python$PYTHON_VERSION/site-packages/torch/lib/:LD_LIBRARY_PATH
+  fi
+}
+
+# Install with pip a bit more robustly than the default
+pip_install() {
+  retry pip install --progress-bar off "$@"
+}
+
+# Install torch with pip, respecting PYTORCH_VERSION, and record the installed
+# version into PYTORCH_VERSION, if applicable
+setup_pip_pytorch_version() {
+  if [[ -z "$PYTORCH_VERSION" ]]; then
+    # Install latest prerelease version of torch, per our nightlies, consistent
+    # with the requested cuda version
+    pip_install --pre torch -f "https://download.pytorch.org/whl/nightly/${WHEEL_DIR}torch_nightly.html"
+    if [[ "$CUDA_VERSION" == "cpu" ]]; then
+      # CUDA and CPU are ABI compatible on the CPU-only parts, so strip
+      # in this case
+      export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version:  *//' | sed 's/+.\+//')"
+    else
+      export PYTORCH_VERSION="$(pip show torch | grep ^Version: | sed 's/Version:  *//')"
+    fi
+  else
+    pip_install "torch==$PYTORCH_VERSION$PYTORCH_VERSION_SUFFIX" \
+      -f "https://download.pytorch.org/whl/${CU_VERSION}/torch_stable.html" \
+      -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${CU_VERSION}/torch_${UPLOAD_CHANNEL}.html"
+  fi
+}
+
+# Fill PYTORCH_VERSION with the latest conda nightly version, and
+# CONDA_CHANNEL_FLAGS with appropriate flags to retrieve these versions
+#
+# You MUST have populated PYTORCH_VERSION_SUFFIX before hand.
+setup_conda_pytorch_constraint() {
+  if [[ -z "$PYTORCH_VERSION" ]]; then
+    export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch-nightly -c pytorch"
+    export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | \
+                              python -c "import os, sys, json, re; cuver = os.environ.get('CU_VERSION'); \
+                               cuver_1 = cuver.replace('cu', 'cuda') if cuver != 'cpu' else cuver; \
+                               cuver_2 = (cuver[:-1] + '.' + cuver[-1]).replace('cu', 'cuda') if cuver != 'cpu' else cuver; \
+                               print(re.sub(r'\\+.*$', '', \
+                                [x['version'] for x in json.load(sys.stdin)['pytorch'] \
+                                  if (x['platform'] == 'darwin' or cuver_1 in x['fn'] or cuver_2 in x['fn']) \
+                                    and 'py' + os.environ['PYTHON_VERSION'] in x['fn']][-1]))")"
+    if [[ -z "$PYTORCH_VERSION" ]]; then
+      echo "PyTorch version auto detection failed"
+      echo "No package found for CU_VERSION=$CU_VERSION and PYTHON_VERSION=$PYTHON_VERSION"
+      exit 1
+    fi
+  else
+    export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch -c pytorch-${UPLOAD_CHANNEL}"
+  fi
+  if [[ "$CU_VERSION" == cpu ]]; then
+    export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}"
+    export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION"
+  else
+    export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}"
+    export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}${PYTORCH_VERSION_SUFFIX}"
+  fi
+  if [[ "$OSTYPE" == msys && "$CU_VERSION" == cu92 ]]; then
+    export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c defaults -c numba/label/dev"
+  fi
+}
+
+# Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT
+setup_conda_cudatoolkit_constraint() {
+  export CONDA_BUILD_VARIANT="cuda"
+  if [[ "$(uname)" == Darwin ]]; then
+    export CONDA_BUILD_VARIANT="cpu"
+  else
+    case "$CU_VERSION" in
+      cu115)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=11.5,<11.6 # [not osx]"
+        ;;
+      cu113)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=11.3,<11.4 # [not osx]"
+        ;;
+      cu112)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=11.2,<11.3 # [not osx]"
+        ;;
+      cu111)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=11.1,<11.2 # [not osx]"
+        ;;
+      cu110)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=11.0,<11.1 # [not osx]"
+        ;;
+      cu102)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.2,<10.3 # [not osx]"
+        ;;
+      cu101)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.1,<10.2 # [not osx]"
+        ;;
+      cu100)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]"
+        ;;
+      cu92)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]"
+        ;;
+      cpu)
+        export CONDA_CUDATOOLKIT_CONSTRAINT=""
+        export CONDA_BUILD_VARIANT="cpu"
+        ;;
+      *)
+        echo "Unrecognized CU_VERSION=$CU_VERSION"
+        exit 1
+        ;;
+    esac
+  fi
+}
+
+setup_conda_cudatoolkit_plain_constraint() {
+  export CONDA_BUILD_VARIANT="cuda"
+  export CMAKE_USE_CUDA=1
+  if [[ "$(uname)" == Darwin ]]; then
+    export CONDA_BUILD_VARIANT="cpu"
+    export CMAKE_USE_CUDA=0
+  else
+    case "$CU_VERSION" in
+      cu115)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="cudatoolkit=11.5"
+        ;;
+      cu113)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="cudatoolkit=11.3"
+        ;;
+      cu112)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="cudatoolkit=11.2"
+        ;;
+      cu111)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="cudatoolkit=11.1"
+        ;;
+      cu102)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="cudatoolkit=10.2"
+        ;;
+      cu101)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="cudatoolkit=10.1"
+        ;;
+      cu100)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="cudatoolkit=10.0"
+        ;;
+      cu92)
+        export CONDA_CUDATOOLKIT_CONSTRAINT="cudatoolkit=9.2"
+        ;;
+      cpu)
+        export CONDA_CUDATOOLKIT_CONSTRAINT=""
+        export CONDA_BUILD_VARIANT="cpu"
+        export CMAKE_USE_CUDA=0
+        ;;
+      *)
+        echo "Unrecognized CU_VERSION=$CU_VERSION"
+        exit 1
+        ;;
+    esac
+  fi
+}
+
+# Build the proper compiler package before building the final package
+setup_visual_studio_constraint() {
+  if [[ "$OSTYPE" == "msys" ]]; then
+      export VSTOOLCHAIN_PACKAGE=vs$VC_YEAR
+      conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload packaging/$VSTOOLCHAIN_PACKAGE
+      cp packaging/$VSTOOLCHAIN_PACKAGE/conda_build_config.yaml packaging/torchvision/conda_build_config.yaml
+  fi
+}
+
+setup_junit_results_folder() {
+  if [[ "$CI" == "true" ]]; then
+    export CONDA_PYTORCH_BUILD_RESULTS_DIRECTORY="${SOURCE_ROOT_DIR}/build_results/results.xml"
+  fi
+}
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000000..72aa19d2266
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,10 @@
+#[tool.usort]
+
+first_party_detection = false
+
+target-version = ["py38"]
+
+excludes = [
+    "gallery",
+    "tutorials",
+]
diff --git a/setup.cfg b/setup.cfg
index 15de32f1c70..6ff0ba25874 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,3 @@
-[bdist_wheel]
-universal=1
-
 [metadata]
 license_file = LICENSE
 
diff --git a/setup.py b/setup.py
index 2d1ca5f4722..84f94f79b05 100644
--- a/setup.py
+++ b/setup.py
@@ -17,10 +17,41 @@
     BuildExtension,
 )
 
+cwd = os.path.dirname(os.path.abspath(__file__))
+version_txt = os.path.join(cwd, "version.txt")
+with open(version_txt, "r") as f:
+    version = f.readline().strip()
+
+
+ROOT_DIR = Path(__file__).parent.resolve()
+
+
+try:
+    sha = (
+        subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd)
+        .decode("ascii")
+        .strip()
+    )
+except Exception:
+    sha = "Unknown"
+package_name = "torchrl"
+
+if os.getenv("BUILD_VERSION"):
+    version = os.getenv("BUILD_VERSION")
+elif sha != "Unknown":
+    version += "+" + sha[:7]
+
+
+def write_version_file():
+    version_path = os.path.join(cwd, "torchrl", "version.py")
+    with open(version_path, "w") as f:
+        f.write("__version__ = '{}'\n".format(version))
+        f.write("git_version = {}\n".format(repr(sha)))
+
 
 def _get_pytorch_version():
-    if "PYTORCH_VERSION" in os.environ:
-        return f"torch=={os.environ['PYTORCH_VERSION']}"
+    # if "PYTORCH_VERSION" in os.environ:
+    #     return f"torch=={os.environ['PYTORCH_VERSION']}"
     return "torch"
 
 
@@ -57,11 +88,11 @@ def run(self):
                 shutil.rmtree(str(path), ignore_errors=True)
 
 
-def _run_cmd(cmd):
-    try:
-        return subprocess.check_output(cmd, cwd=ROOT_DIR).decode("ascii").strip()
-    except Exception:
-        return None
+# def _run_cmd(cmd):
+#     try:
+#         return subprocess.check_output(cmd, cwd=ROOT_DIR).decode("ascii").strip()
+#     except Exception:
+#         return None
 
 
 def get_extensions():
@@ -119,7 +150,7 @@ def _main():
 
     setup(
         name="torchrl",
-        version="0.1",
+        version=version,
         author="torchrl contributors",
         author_email="vmoens@fb.com",
         packages=find_packages(),
@@ -128,7 +159,7 @@ def _main():
             "build_ext": BuildExtension.with_options(no_python_abi_suffix=True),
             "clean": clean,
         },
-        install_requires=[pytorch_package_dep, "numpy", "tensorboard", "packaging"],
+        install_requires=[pytorch_package_dep, "numpy", "packaging"],
         extras_require={
             "atari": ["gym", "atari-py", "ale-py", "gym[accept-rom-license]", "pygame"],
             "dm_control": ["dm_control"],
@@ -136,14 +167,23 @@ def _main():
             "rendering": ["moviepy"],
             "tests": ["pytest", "pyyaml"],
             "utils": [
+                "tensorboard",
                 "tqdm",
-                "configargparse",
                 "hydra-core>=1.1",
                 "hydra-submitit-launcher",
             ],
         },
+        url="https://github.com/facebookresearch/rl",
+        # classifiers = [
+        #    "Programming Language :: Python :: 3",
+        #    "License :: OSI Approved :: MIT License",
+        #    "Operating System :: OS Independent",
+        # ]
     )
 
 
 if __name__ == "__main__":
+
+    write_version_file()
+    print("Building wheel {}-{}".format(package_name, version))
     _main()
diff --git a/torchrl/__init__.py b/torchrl/__init__.py
index 9e5b3f23442..6a5cea6ba25 100644
--- a/torchrl/__init__.py
+++ b/torchrl/__init__.py
@@ -13,7 +13,10 @@
 
 from ._extension import _init_extension
 
-__version__ = "0.1"
+try:
+    from .version import __version__
+except ImportError:
+    __version__ = None
 
 _init_extension()
 
diff --git a/version.txt b/version.txt
new file mode 100644
index 00000000000..8acdd82b765
--- /dev/null
+++ b/version.txt
@@ -0,0 +1 @@
+0.0.1