[Feature] Consistent dropout

[vmoens]

Requested via https://discuss.pytorch.org/t/feature-request-consistent-dropout-implementation/183778

Implements https://arxiv.org/abs/2202.11818

[github-actions]

$\color{#D29922}\textsf{\Large⚠\kern{0.2cm}\normalsize Warning}$ Result of CPU Benchmark Tests

Total Benchmarks: 89. Improved: $\large\color{#35bf28}4$. Worsened: $\large\color{#d91a1a}5$.

Expand to view detailed results

Name	Max	Mean	Ops	Ops on Repo HEAD	Change
test_single	65.7558ms	64.4588ms	15.5138 Ops/s	14.8073 Ops/s	$\color{#35bf28}+4.77\%$
test_sync	43.4873ms	39.4983ms	25.3175 Ops/s	27.3641 Ops/s	$\textbf{\color{#d91a1a}-7.48\%}$
test_async	55.8261ms	32.8799ms	30.4137 Ops/s	30.5909 Ops/s	$\color{#d91a1a}-0.58\%$
test_simple	0.5096s	0.4468s	2.2381 Ops/s	2.2747 Ops/s	$\color{#d91a1a}-1.61\%$
test_transformed	0.6834s	0.6216s	1.6088 Ops/s	1.6080 Ops/s	$\color{#35bf28}+0.05\%$
test_serial	1.4323s	1.3712s	0.7293 Ops/s	0.7408 Ops/s	$\color{#d91a1a}-1.56\%$
test_parallel	1.4076s	1.3582s	0.7363 Ops/s	0.7414 Ops/s	$\color{#d91a1a}-0.69\%$
test_step_mdp_speed[True-True-True-True-True]	0.1530ms	23.0447μs	43.3939 KOps/s	43.7747 KOps/s	$\color{#d91a1a}-0.87\%$
test_step_mdp_speed[True-True-True-True-False]	36.9980μs	14.0508μs	71.1701 KOps/s	73.3651 KOps/s	$\color{#d91a1a}-2.99\%$
test_step_mdp_speed[True-True-True-False-True]	47.8990μs	14.1061μs	70.8913 KOps/s	71.7074 KOps/s	$\color{#d91a1a}-1.14\%$
test_step_mdp_speed[True-True-True-False-False]	35.9670μs	8.3592μs	119.6280 KOps/s	121.3002 KOps/s	$\color{#d91a1a}-1.38\%$
test_step_mdp_speed[True-True-False-True-True]	48.7100μs	24.7367μs	40.4258 KOps/s	41.4708 KOps/s	$\color{#d91a1a}-2.52\%$
test_step_mdp_speed[True-True-False-True-False]	56.2550μs	15.4596μs	64.6846 KOps/s	66.5844 KOps/s	$\color{#d91a1a}-2.85\%$
test_step_mdp_speed[True-True-False-False-True]	50.1140μs	15.2951μs	65.3803 KOps/s	66.2548 KOps/s	$\color{#d91a1a}-1.32\%$
test_step_mdp_speed[True-True-False-False-False]	49.1110μs	9.7574μs	102.4863 KOps/s	105.0698 KOps/s	$\color{#d91a1a}-2.46\%$
test_step_mdp_speed[True-False-True-True-True]	55.3330μs	26.0589μs	38.3746 KOps/s	39.1162 KOps/s	$\color{#d91a1a}-1.90\%$
test_step_mdp_speed[True-False-True-True-False]	67.2450μs	16.8442μs	59.3674 KOps/s	61.3338 KOps/s	$\color{#d91a1a}-3.21\%$
test_step_mdp_speed[True-False-True-False-True]	72.3350μs	15.4411μs	64.7623 KOps/s	65.9830 KOps/s	$\color{#d91a1a}-1.85\%$
test_step_mdp_speed[True-False-True-False-False]	33.6630μs	9.8570μs	101.4507 KOps/s	103.8782 KOps/s	$\color{#d91a1a}-2.34\%$
test_step_mdp_speed[True-False-False-True-True]	65.8620μs	27.1774μs	36.7953 KOps/s	37.2329 KOps/s	$\color{#d91a1a}-1.18\%$
test_step_mdp_speed[True-False-False-True-False]	46.5470μs	18.1000μs	55.2485 KOps/s	57.6798 KOps/s	$\color{#d91a1a}-4.22\%$
test_step_mdp_speed[True-False-False-False-True]	47.9190μs	16.6069μs	60.2158 KOps/s	61.0246 KOps/s	$\color{#d91a1a}-1.33\%$
test_step_mdp_speed[True-False-False-False-False]	42.3690μs	11.0414μs	90.5681 KOps/s	92.7722 KOps/s	$\color{#d91a1a}-2.38\%$
test_step_mdp_speed[False-True-True-True-True]	59.1100μs	26.0239μs	38.4262 KOps/s	39.1469 KOps/s	$\color{#d91a1a}-1.84\%$
test_step_mdp_speed[False-True-True-True-False]	42.6990μs	16.9126μs	59.1274 KOps/s	61.2996 KOps/s	$\color{#d91a1a}-3.54\%$
test_step_mdp_speed[False-True-True-False-True]	61.6340μs	17.8126μs	56.1402 KOps/s	57.5454 KOps/s	$\color{#d91a1a}-2.44\%$
test_step_mdp_speed[False-True-True-False-False]	46.3770μs	11.1587μs	89.6165 KOps/s	91.1136 KOps/s	$\color{#d91a1a}-1.64\%$
test_step_mdp_speed[False-True-False-True-True]	64.8800μs	27.0060μs	37.0288 KOps/s	37.2060 KOps/s	$\color{#d91a1a}-0.48\%$
test_step_mdp_speed[False-True-False-True-False]	67.4550μs	17.9679μs	55.6549 KOps/s	56.8197 KOps/s	$\color{#d91a1a}-2.05\%$
test_step_mdp_speed[False-True-False-False-True]	45.6750μs	18.9265μs	52.8361 KOps/s	53.5455 KOps/s	$\color{#d91a1a}-1.32\%$
test_step_mdp_speed[False-True-False-False-False]	39.2830μs	12.3644μs	80.8771 KOps/s	83.2325 KOps/s	$\color{#d91a1a}-2.83\%$
test_step_mdp_speed[False-False-True-True-True]	67.7970μs	28.4722μs	35.1220 KOps/s	35.2427 KOps/s	$\color{#d91a1a}-0.34\%$
test_step_mdp_speed[False-False-True-True-False]	61.0230μs	19.3878μs	51.5789 KOps/s	52.7586 KOps/s	$\color{#d91a1a}-2.24\%$
test_step_mdp_speed[False-False-True-False-True]	50.5550μs	19.1205μs	52.3000 KOps/s	53.0955 KOps/s	$\color{#d91a1a}-1.50\%$
test_step_mdp_speed[False-False-True-False-False]	55.3930μs	12.5705μs	79.5512 KOps/s	83.2521 KOps/s	$\color{#d91a1a}-4.45\%$
test_step_mdp_speed[False-False-False-True-True]	69.9600μs	29.4497μs	33.9562 KOps/s	34.3815 KOps/s	$\color{#d91a1a}-1.24\%$
test_step_mdp_speed[False-False-False-True-False]	79.6490μs	20.6497μs	48.4269 KOps/s	50.2886 KOps/s	$\color{#d91a1a}-3.70\%$
test_step_mdp_speed[False-False-False-False-True]	47.3280μs	19.8802μs	50.3014 KOps/s	51.5016 KOps/s	$\color{#d91a1a}-2.33\%$
test_step_mdp_speed[False-False-False-False-False]	40.9660μs	13.4634μs	74.2755 KOps/s	76.3415 KOps/s	$\color{#d91a1a}-2.71\%$
test_values[generalized_advantage_estimate-True-True]	17.1997ms	12.3933ms	80.6886 Ops/s	81.5070 Ops/s	$\color{#d91a1a}-1.00\%$
test_values[vec_generalized_advantage_estimate-True-True]	33.6469ms	26.4621ms	37.7899 Ops/s	37.7800 Ops/s	$\color{#35bf28}+0.03\%$
test_values[td0_return_estimate-False-False]	0.2480ms	0.1934ms	5.1700 KOps/s	5.4040 KOps/s	$\color{#d91a1a}-4.33\%$
test_values[td1_return_estimate-False-False]	26.6334ms	26.2068ms	38.1580 Ops/s	38.1107 Ops/s	$\color{#35bf28}+0.12\%$
test_values[vec_td1_return_estimate-False-False]	0.1094s	28.1754ms	35.4920 Ops/s	37.7370 Ops/s	$\textbf{\color{#d91a1a}-5.95\%}$
test_values[td_lambda_return_estimate-True-False]	37.2673ms	36.2709ms	27.5703 Ops/s	27.3335 Ops/s	$\color{#35bf28}+0.87\%$
test_values[vec_td_lambda_return_estimate-True-False]	33.4426ms	26.5953ms	37.6006 Ops/s	37.8989 Ops/s	$\color{#d91a1a}-0.79\%$
test_gae_speed[generalized_advantage_estimate-False-1-512]	8.4059ms	8.1909ms	122.0869 Ops/s	122.7016 Ops/s	$\color{#d91a1a}-0.50\%$
test_gae_speed[vec_generalized_advantage_estimate-True-1-512]	9.9070ms	1.9285ms	518.5362 Ops/s	537.8400 Ops/s	$\color{#d91a1a}-3.59\%$
test_gae_speed[vec_generalized_advantage_estimate-False-1-512]	0.5777ms	0.4332ms	2.3084 KOps/s	2.3277 KOps/s	$\color{#d91a1a}-0.83\%$
test_gae_speed[vec_generalized_advantage_estimate-True-32-512]	48.9629ms	40.0820ms	24.9488 Ops/s	25.0599 Ops/s	$\color{#d91a1a}-0.44\%$
test_gae_speed[vec_generalized_advantage_estimate-False-32-512]	10.1133ms	2.5393ms	393.8043 Ops/s	390.6440 Ops/s	$\color{#35bf28}+0.81\%$
test_dqn_speed	8.9575ms	1.6980ms	588.9453 Ops/s	558.3805 Ops/s	$\textbf{\color{#35bf28}+5.47\%}$
test_ddpg_speed	10.8626ms	3.8727ms	258.2197 Ops/s	266.8785 Ops/s	$\color{#d91a1a}-3.24\%$
test_sac_speed	18.1297ms	11.0384ms	90.5929 Ops/s	90.9825 Ops/s	$\color{#d91a1a}-0.43\%$
test_redq_speed	28.1998ms	20.2622ms	49.3531 Ops/s	49.5943 Ops/s	$\color{#d91a1a}-0.49\%$
test_redq_deprec_speed	0.1007s	17.9123ms	55.8275 Ops/s	61.5902 Ops/s	$\textbf{\color{#d91a1a}-9.36\%}$
test_td3_speed	20.3837ms	11.5663ms	86.4582 Ops/s	91.7307 Ops/s	$\textbf{\color{#d91a1a}-5.75\%}$
test_cql_speed	50.0496ms	40.4104ms	24.7461 Ops/s	22.8654 Ops/s	$\textbf{\color{#35bf28}+8.22\%}$
test_a2c_speed	17.3493ms	9.2878ms	107.6679 Ops/s	107.4849 Ops/s	$\color{#35bf28}+0.17\%$
test_ppo_speed	20.9833ms	9.8749ms	101.2673 Ops/s	106.8170 Ops/s	$\textbf{\color{#d91a1a}-5.20\%}$
test_reinforce_speed	16.4438ms	8.3434ms	119.8552 Ops/s	106.4364 Ops/s	$\textbf{\color{#35bf28}+12.61\%}$
test_iql_speed	45.2255ms	36.5803ms	27.3372 Ops/s	26.6801 Ops/s	$\color{#35bf28}+2.46\%$
test_sample_rb[TensorDictReplayBuffer-ListStorage-RandomSampler-4000]	2.9900ms	2.0190ms	495.2834 Ops/s	495.8020 Ops/s	$\color{#d91a1a}-0.10\%$
test_sample_rb[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000]	3.8161ms	2.1740ms	459.9746 Ops/s	467.6987 Ops/s	$\color{#d91a1a}-1.65\%$
test_sample_rb[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000]	4.0965ms	2.1812ms	458.4699 Ops/s	467.7703 Ops/s	$\color{#d91a1a}-1.99\%$
test_sample_rb[TensorDictReplayBuffer-ListStorage-SamplerWithoutReplacement-4000]	3.1349ms	2.0432ms	489.4389 Ops/s	498.3004 Ops/s	$\color{#d91a1a}-1.78\%$
test_sample_rb[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000]	3.4732ms	2.1679ms	461.2735 Ops/s	475.6391 Ops/s	$\color{#d91a1a}-3.02\%$
test_sample_rb[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000]	5.5061ms	2.1356ms	468.2456 Ops/s	476.0815 Ops/s	$\color{#d91a1a}-1.65\%$
test_sample_rb[TensorDictPrioritizedReplayBuffer-ListStorage-None-4000]	3.3603ms	2.0049ms	498.7885 Ops/s	495.5294 Ops/s	$\color{#35bf28}+0.66\%$
test_sample_rb[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000]	3.6916ms	2.1250ms	470.5836 Ops/s	472.9755 Ops/s	$\color{#d91a1a}-0.51\%$
test_sample_rb[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000]	4.1935ms	2.1423ms	466.7794 Ops/s	469.5794 Ops/s	$\color{#d91a1a}-0.60\%$
test_iterate_rb[TensorDictReplayBuffer-ListStorage-RandomSampler-4000]	2.9547ms	1.9832ms	504.2249 Ops/s	505.0755 Ops/s	$\color{#d91a1a}-0.17\%$
test_iterate_rb[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000]	4.3742ms	2.1610ms	462.7460 Ops/s	473.1661 Ops/s	$\color{#d91a1a}-2.20\%$
test_iterate_rb[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000]	3.0440ms	2.1584ms	463.3037 Ops/s	461.2771 Ops/s	$\color{#35bf28}+0.44\%$
test_iterate_rb[TensorDictReplayBuffer-ListStorage-SamplerWithoutReplacement-4000]	3.0117ms	2.0157ms	496.1068 Ops/s	487.6447 Ops/s	$\color{#35bf28}+1.74\%$
test_iterate_rb[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000]	3.8433ms	2.1596ms	463.0413 Ops/s	468.5596 Ops/s	$\color{#d91a1a}-1.18\%$
test_iterate_rb[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000]	3.5270ms	2.1520ms	464.6805 Ops/s	472.3757 Ops/s	$\color{#d91a1a}-1.63\%$
test_iterate_rb[TensorDictPrioritizedReplayBuffer-ListStorage-None-4000]	2.2780ms	2.0241ms	494.0524 Ops/s	498.3835 Ops/s	$\color{#d91a1a}-0.87\%$
test_iterate_rb[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000]	4.0059ms	2.1682ms	461.2212 Ops/s	473.7695 Ops/s	$\color{#d91a1a}-2.65\%$
test_iterate_rb[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000]	3.2334ms	2.1635ms	462.2083 Ops/s	472.0826 Ops/s	$\color{#d91a1a}-2.09\%$
test_populate_rb[TensorDictReplayBuffer-ListStorage-RandomSampler-400]	0.2051s	18.8636ms	53.0121 Ops/s	54.1080 Ops/s	$\color{#d91a1a}-2.03\%$
test_populate_rb[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-400]	0.1164s	17.0063ms	58.8019 Ops/s	57.8444 Ops/s	$\color{#35bf28}+1.66\%$
test_populate_rb[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-400]	0.1176s	17.1378ms	58.3506 Ops/s	59.0854 Ops/s	$\color{#d91a1a}-1.24\%$
test_populate_rb[TensorDictReplayBuffer-ListStorage-SamplerWithoutReplacement-400]	0.1232s	17.1177ms	58.4191 Ops/s	56.5102 Ops/s	$\color{#35bf28}+3.38\%$
test_populate_rb[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-400]	0.1228s	17.5075ms	57.1184 Ops/s	50.8615 Ops/s	$\textbf{\color{#35bf28}+12.30\%}$
test_populate_rb[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-400]	0.1229s	17.3998ms	57.4720 Ops/s	57.8024 Ops/s	$\color{#d91a1a}-0.57\%$
test_populate_rb[TensorDictPrioritizedReplayBuffer-ListStorage-None-400]	0.1210s	15.0724ms	66.3463 Ops/s	65.3910 Ops/s	$\color{#35bf28}+1.46\%$
test_populate_rb[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-400]	0.1176s	17.1180ms	58.4181 Ops/s	57.6726 Ops/s	$\color{#35bf28}+1.29\%$
test_populate_rb[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-400]	0.1205s	17.4023ms	57.4636 Ops/s	57.4471 Ops/s	$\color{#35bf28}+0.03\%$

[pytorch-bot]

🔗 Helpful Links

🧪 See artifacts and rendered test results at hud.pytorch.org/pr/pytorch/rl/1587

• 📄 Preview Python docs built from this PR

Note: Links to docs will display an error until the docs builds have been completed.

✅ You can merge normally! (8 Unrelated Failures)

As of commit 70ab0f8 with merge base bc7595f ():

FLAKY - The following jobs failed but were likely due to flakiness present on trunk:

• Examples Tests on Linux / tests (3.9, 12.1) / linux-job (gh)
• Generate documentation / build-docs (3.9, 12.1) / linux-job (gh)
• Lint / python-source-and-configs / linux-job (gh)
• Unit-tests on Linux CPU / tests (3.11) / linux-job (gh)
• Unit-tests on MacOS CPU / tests (3.11) / macos-job (gh)

BROKEN TRUNK - The following jobs failed but were present on the merge base:

👉 Rebase onto the `viable/strict` branch to avoid these failures

• Habitat Tests on Linux / tests (3.9, 11.6) / linux-job (gh)
• RLHF Tests on Linux / unittests (3.9, 12.1) / linux-job (gh)
• Unit-tests on Windows GPU / unittests / windows-job (gh)

This comment was automatically generated by Dr. CI and updates every 15 minutes.

[github-actions]

$\color{#D29922}\textsf{\Large⚠\kern{0.2cm}\normalsize Warning}$ Result of GPU Benchmark Tests

Total Benchmarks: 92. Improved: $\large\color{#35bf28}5$. Worsened: $\large\color{#d91a1a}3$.

Expand to view detailed results

Name	Max	Mean	Ops	Ops on Repo HEAD	Change
test_single	0.1230s	0.1200s	8.3301 Ops/s	8.2933 Ops/s	$\color{#35bf28}+0.44\%$
test_sync	0.1019s	0.1017s	9.8286 Ops/s	9.7555 Ops/s	$\color{#35bf28}+0.75\%$
test_async	0.2697s	98.7397ms	10.1276 Ops/s	10.0007 Ops/s	$\color{#35bf28}+1.27\%$
test_single_pixels	0.1431s	0.1427s	7.0083 Ops/s	7.0002 Ops/s	$\color{#35bf28}+0.12\%$
test_sync_pixels	99.6984ms	94.9585ms	10.5309 Ops/s	10.5436 Ops/s	$\color{#d91a1a}-0.12\%$
test_async_pixels	0.1841s	90.6840ms	11.0273 Ops/s	10.9570 Ops/s	$\color{#35bf28}+0.64\%$
test_simple	0.9308s	0.8631s	1.1586 Ops/s	1.1460 Ops/s	$\color{#35bf28}+1.10\%$
test_transformed	1.1670s	1.1058s	0.9043 Ops/s	0.8941 Ops/s	$\color{#35bf28}+1.15\%$
test_serial	2.4695s	2.4070s	0.4155 Ops/s	0.4110 Ops/s	$\color{#35bf28}+1.08\%$
test_parallel	2.5846s	2.5243s	0.3962 Ops/s	0.4024 Ops/s	$\color{#d91a1a}-1.56\%$
test_step_mdp_speed[True-True-True-True-True]	71.9610μs	35.1156μs	28.4774 KOps/s	27.8686 KOps/s	$\color{#35bf28}+2.18\%$
test_step_mdp_speed[True-True-True-True-False]	45.8410μs	20.9929μs	47.6351 KOps/s	47.1576 KOps/s	$\color{#35bf28}+1.01\%$
test_step_mdp_speed[True-True-True-False-True]	48.3110μs	21.2030μs	47.1632 KOps/s	47.4371 KOps/s	$\color{#d91a1a}-0.58\%$
test_step_mdp_speed[True-True-True-False-False]	36.7800μs	12.1936μs	82.0104 KOps/s	80.1133 KOps/s	$\color{#35bf28}+2.37\%$
test_step_mdp_speed[True-True-False-True-True]	65.0310μs	36.1393μs	27.6707 KOps/s	26.8441 KOps/s	$\color{#35bf28}+3.08\%$
test_step_mdp_speed[True-True-False-True-False]	47.0010μs	22.3476μs	44.7475 KOps/s	44.2484 KOps/s	$\color{#35bf28}+1.13\%$
test_step_mdp_speed[True-True-False-False-True]	47.4810μs	21.7309μs	46.0175 KOps/s	44.2794 KOps/s	$\color{#35bf28}+3.93\%$
test_step_mdp_speed[True-True-False-False-False]	38.3810μs	13.9012μs	71.9364 KOps/s	69.5820 KOps/s	$\color{#35bf28}+3.38\%$
test_step_mdp_speed[True-False-True-True-True]	77.5320μs	38.8675μs	25.7285 KOps/s	25.2231 KOps/s	$\color{#35bf28}+2.00\%$
test_step_mdp_speed[True-False-True-True-False]	58.3110μs	25.2335μs	39.6298 KOps/s	40.7271 KOps/s	$\color{#d91a1a}-2.69\%$
test_step_mdp_speed[True-False-True-False-True]	57.0610μs	23.1922μs	43.1180 KOps/s	43.7937 KOps/s	$\color{#d91a1a}-1.54\%$
test_step_mdp_speed[True-False-True-False-False]	37.4210μs	14.2340μs	70.2541 KOps/s	70.9671 KOps/s	$\color{#d91a1a}-1.00\%$
test_step_mdp_speed[True-False-False-True-True]	65.8110μs	40.4899μs	24.6975 KOps/s	23.8441 KOps/s	$\color{#35bf28}+3.58\%$
test_step_mdp_speed[True-False-False-True-False]	49.5610μs	25.8592μs	38.6710 KOps/s	38.1628 KOps/s	$\color{#35bf28}+1.33\%$
test_step_mdp_speed[True-False-False-False-True]	50.0210μs	24.3085μs	41.1379 KOps/s	40.6969 KOps/s	$\color{#35bf28}+1.08\%$
test_step_mdp_speed[True-False-False-False-False]	38.5200μs	15.6146μs	64.0428 KOps/s	61.8681 KOps/s	$\color{#35bf28}+3.52\%$
test_step_mdp_speed[False-True-True-True-True]	90.2410μs	38.3699μs	26.0621 KOps/s	25.3182 KOps/s	$\color{#35bf28}+2.94\%$
test_step_mdp_speed[False-True-True-True-False]	50.1600μs	24.6754μs	40.5262 KOps/s	40.4371 KOps/s	$\color{#35bf28}+0.22\%$
test_step_mdp_speed[False-True-True-False-True]	42.7710μs	26.4194μs	37.8510 KOps/s	37.2505 KOps/s	$\color{#35bf28}+1.61\%$
test_step_mdp_speed[False-True-True-False-False]	36.8010μs	15.9288μs	62.7794 KOps/s	60.8671 KOps/s	$\color{#35bf28}+3.14\%$
test_step_mdp_speed[False-True-False-True-True]	69.0110μs	41.6889μs	23.9872 KOps/s	23.9642 KOps/s	$\color{#35bf28}+0.10\%$
test_step_mdp_speed[False-True-False-True-False]	50.3900μs	26.1431μs	38.2511 KOps/s	37.4685 KOps/s	$\color{#35bf28}+2.09\%$
test_step_mdp_speed[False-True-False-False-True]	51.4300μs	28.5880μs	34.9797 KOps/s	34.5428 KOps/s	$\color{#35bf28}+1.26\%$
test_step_mdp_speed[False-True-False-False-False]	35.8610μs	17.9592μs	55.6817 KOps/s	54.9325 KOps/s	$\color{#35bf28}+1.36\%$
test_step_mdp_speed[False-False-True-True-True]	0.1067ms	42.4081μs	23.5804 KOps/s	22.8306 KOps/s	$\color{#35bf28}+3.28\%$
test_step_mdp_speed[False-False-True-True-False]	56.8910μs	27.9906μs	35.7262 KOps/s	34.6880 KOps/s	$\color{#35bf28}+2.99\%$
test_step_mdp_speed[False-False-True-False-True]	55.1010μs	28.3231μs	35.3068 KOps/s	34.4308 KOps/s	$\color{#35bf28}+2.54\%$
test_step_mdp_speed[False-False-True-False-False]	56.0810μs	18.0295μs	55.4645 KOps/s	55.2939 KOps/s	$\color{#35bf28}+0.31\%$
test_step_mdp_speed[False-False-False-True-True]	0.1054ms	43.6132μs	22.9288 KOps/s	22.0254 KOps/s	$\color{#35bf28}+4.10\%$
test_step_mdp_speed[False-False-False-True-False]	64.8420μs	29.8421μs	33.5097 KOps/s	32.3080 KOps/s	$\color{#35bf28}+3.72\%$
test_step_mdp_speed[False-False-False-False-True]	58.3310μs	28.8449μs	34.6682 KOps/s	32.9065 KOps/s	$\textbf{\color{#35bf28}+5.35\%}$
test_step_mdp_speed[False-False-False-False-False]	41.2010μs	19.4394μs	51.4420 KOps/s	48.6864 KOps/s	$\textbf{\color{#35bf28}+5.66\%}$
test_values[generalized_advantage_estimate-True-True]	25.7269ms	24.7857ms	40.3458 Ops/s	40.4340 Ops/s	$\color{#d91a1a}-0.22\%$
test_values[vec_generalized_advantage_estimate-True-True]	86.9101ms	3.2791ms	304.9621 Ops/s	312.6387 Ops/s	$\color{#d91a1a}-2.46\%$
test_values[td0_return_estimate-False-False]	94.9610μs	61.3559μs	16.2984 KOps/s	15.9680 KOps/s	$\color{#35bf28}+2.07\%$
test_values[td1_return_estimate-False-False]	55.7182ms	54.3577ms	18.3967 Ops/s	18.7622 Ops/s	$\color{#d91a1a}-1.95\%$
test_values[vec_td1_return_estimate-False-False]	1.9596ms	1.7179ms	582.1105 Ops/s	586.7441 Ops/s	$\color{#d91a1a}-0.79\%$
test_values[td_lambda_return_estimate-True-False]	89.9322ms	88.3096ms	11.3238 Ops/s	11.6331 Ops/s	$\color{#d91a1a}-2.66\%$
test_values[vec_td_lambda_return_estimate-True-False]	1.9843ms	1.7162ms	582.6682 Ops/s	585.4342 Ops/s	$\color{#d91a1a}-0.47\%$
test_gae_speed[generalized_advantage_estimate-False-1-512]	24.7016ms	23.9508ms	41.7523 Ops/s	42.0637 Ops/s	$\color{#d91a1a}-0.74\%$
test_gae_speed[vec_generalized_advantage_estimate-True-1-512]	0.8431ms	0.6960ms	1.4368 KOps/s	1.4214 KOps/s	$\color{#35bf28}+1.08\%$
test_gae_speed[vec_generalized_advantage_estimate-False-1-512]	0.7600ms	0.6789ms	1.4729 KOps/s	1.5094 KOps/s	$\color{#d91a1a}-2.42\%$
test_gae_speed[vec_generalized_advantage_estimate-True-32-512]	1.5168ms	1.4626ms	683.7223 Ops/s	683.0126 Ops/s	$\color{#35bf28}+0.10\%$
test_gae_speed[vec_generalized_advantage_estimate-False-32-512]	0.9520ms	0.6818ms	1.4668 KOps/s	1.4312 KOps/s	$\color{#35bf28}+2.49\%$
test_dqn_speed	7.7903ms	1.4546ms	687.4747 Ops/s	668.8062 Ops/s	$\color{#35bf28}+2.79\%$
test_ddpg_speed	4.7032ms	3.3237ms	300.8732 Ops/s	300.8100 Ops/s	$\color{#35bf28}+0.02\%$
test_sac_speed	94.2086ms	10.0866ms	99.1411 Ops/s	108.7263 Ops/s	$\textbf{\color{#d91a1a}-8.82\%}$
test_redq_speed	16.9641ms	16.4793ms	60.6821 Ops/s	60.1442 Ops/s	$\color{#35bf28}+0.89\%$
test_redq_deprec_speed	14.2135ms	12.9746ms	77.0739 Ops/s	77.4876 Ops/s	$\color{#d91a1a}-0.53\%$
test_td3_speed	19.2103ms	9.4817ms	105.4669 Ops/s	105.9975 Ops/s	$\color{#d91a1a}-0.50\%$
test_cql_speed	32.6804ms	31.3788ms	31.8686 Ops/s	30.4686 Ops/s	$\color{#35bf28}+4.59\%$
test_a2c_speed	8.6134ms	7.1283ms	140.2868 Ops/s	141.7486 Ops/s	$\color{#d91a1a}-1.03\%$
test_ppo_speed	8.8716ms	7.4452ms	134.3145 Ops/s	136.6788 Ops/s	$\color{#d91a1a}-1.73\%$
test_reinforce_speed	7.5503ms	6.1457ms	162.7160 Ops/s	165.2620 Ops/s	$\color{#d91a1a}-1.54\%$
test_iql_speed	28.6921ms	27.1953ms	36.7711 Ops/s	37.0608 Ops/s	$\color{#d91a1a}-0.78\%$
test_sample_rb[TensorDictReplayBuffer-ListStorage-RandomSampler-4000]	3.1846ms	2.5026ms	399.5868 Ops/s	396.2916 Ops/s	$\color{#35bf28}+0.83\%$
test_sample_rb[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000]	3.9185ms	2.6861ms	372.2848 Ops/s	326.1991 Ops/s	$\textbf{\color{#35bf28}+14.13\%}$
test_sample_rb[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000]	3.9450ms	2.6902ms	371.7231 Ops/s	372.9594 Ops/s	$\color{#d91a1a}-0.33\%$
test_sample_rb[TensorDictReplayBuffer-ListStorage-SamplerWithoutReplacement-4000]	3.1463ms	2.4867ms	402.1442 Ops/s	398.9222 Ops/s	$\color{#35bf28}+0.81\%$
test_sample_rb[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000]	3.7607ms	2.6830ms	372.7223 Ops/s	370.4507 Ops/s	$\color{#35bf28}+0.61\%$
test_sample_rb[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000]	3.9494ms	2.6937ms	371.2352 Ops/s	373.2656 Ops/s	$\color{#d91a1a}-0.54\%$
test_sample_rb[TensorDictPrioritizedReplayBuffer-ListStorage-None-4000]	2.7234ms	2.5048ms	399.2321 Ops/s	397.2757 Ops/s	$\color{#35bf28}+0.49\%$
test_sample_rb[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000]	4.0177ms	2.6917ms	371.5145 Ops/s	370.1825 Ops/s	$\color{#35bf28}+0.36\%$
test_sample_rb[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000]	3.8349ms	2.6903ms	371.7022 Ops/s	372.9640 Ops/s	$\color{#d91a1a}-0.34\%$
test_iterate_rb[TensorDictReplayBuffer-ListStorage-RandomSampler-4000]	2.7218ms	2.5148ms	397.6387 Ops/s	397.1890 Ops/s	$\color{#35bf28}+0.11\%$
test_iterate_rb[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000]	4.2305ms	2.6971ms	370.7629 Ops/s	371.6203 Ops/s	$\color{#d91a1a}-0.23\%$
test_iterate_rb[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000]	4.0843ms	2.6988ms	370.5404 Ops/s	371.2687 Ops/s	$\color{#d91a1a}-0.20\%$
test_iterate_rb[TensorDictReplayBuffer-ListStorage-SamplerWithoutReplacement-4000]	3.0685ms	2.4898ms	401.6404 Ops/s	396.7588 Ops/s	$\color{#35bf28}+1.23\%$
test_iterate_rb[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000]	4.0895ms	2.7137ms	368.5059 Ops/s	370.9746 Ops/s	$\color{#d91a1a}-0.67\%$
test_iterate_rb[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000]	4.1527ms	2.6942ms	371.1740 Ops/s	370.1471 Ops/s	$\color{#35bf28}+0.28\%$
test_iterate_rb[TensorDictPrioritizedReplayBuffer-ListStorage-None-4000]	3.0908ms	2.5128ms	397.9676 Ops/s	396.4535 Ops/s	$\color{#35bf28}+0.38\%$
test_iterate_rb[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000]	4.3490ms	2.9596ms	337.8813 Ops/s	368.9570 Ops/s	$\textbf{\color{#d91a1a}-8.42\%}$
test_iterate_rb[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000]	3.5711ms	2.6931ms	371.3154 Ops/s	370.5835 Ops/s	$\color{#35bf28}+0.20\%$
test_populate_rb[TensorDictReplayBuffer-ListStorage-RandomSampler-400]	0.2133s	19.4106ms	51.5182 Ops/s	51.8002 Ops/s	$\color{#d91a1a}-0.54\%$
test_populate_rb[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-400]	0.1250s	15.3120ms	65.3081 Ops/s	56.6158 Ops/s	$\textbf{\color{#35bf28}+15.35\%}$
test_populate_rb[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-400]	0.1268s	17.5941ms	56.8372 Ops/s	56.3637 Ops/s	$\color{#35bf28}+0.84\%$
test_populate_rb[TensorDictReplayBuffer-ListStorage-SamplerWithoutReplacement-400]	0.1272s	17.4248ms	57.3896 Ops/s	64.7084 Ops/s	$\textbf{\color{#d91a1a}-11.31\%}$
test_populate_rb[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-400]	0.1252s	17.6077ms	56.7933 Ops/s	56.4270 Ops/s	$\color{#35bf28}+0.65\%$
test_populate_rb[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-400]	0.1258s	17.6151ms	56.7696 Ops/s	56.6489 Ops/s	$\color{#35bf28}+0.21\%$
test_populate_rb[TensorDictPrioritizedReplayBuffer-ListStorage-None-400]	0.1264s	17.4903ms	57.1745 Ops/s	56.1512 Ops/s	$\color{#35bf28}+1.82\%$
test_populate_rb[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-400]	0.1256s	17.5036ms	57.1311 Ops/s	56.0507 Ops/s	$\color{#35bf28}+1.93\%$
test_populate_rb[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-400]	0.1258s	15.3529ms	65.1344 Ops/s	56.2626 Ops/s	$\textbf{\color{#35bf28}+15.77\%}$