From 2e41dcf0606e847ed4459d096ca7c664abab052b Mon Sep 17 00:00:00 2001 From: Josh Karlin Date: Thu, 23 Apr 2020 11:03:03 -0400 Subject: [PATCH 1/2] fix --- pytorch_lightning/trainer/training_loop.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/training_loop.py b/pytorch_lightning/trainer/training_loop.py index 5b3d13c72b5f1..c000b1bfa9046 100644 --- a/pytorch_lightning/trainer/training_loop.py +++ b/pytorch_lightning/trainer/training_loop.py @@ -754,7 +754,11 @@ def training_forward(self, batch, batch_idx, opt_idx, hiddens): gpu_id = 0 if isinstance(self.data_parallel_device_ids, list): gpu_id = self.data_parallel_device_ids[0] - batch = self.transfer_batch_to_gpu(copy.copy(batch), gpu_id) + + # Don't copy the batch since there is a single gpu that the batch could + # be referenced from and if there are multiple optimizers the batch will + # wind up copying it to the same device repeatedly. + batch = self.transfer_batch_to_gpu(batch, gpu_id) args[0] = batch output = self.model.training_step(*args) From e2cedee616ae00ee2dc0000aee466af3e6eef0a6 Mon Sep 17 00:00:00 2001 From: Josh Karlin Date: Thu, 23 Apr 2020 11:19:13 -0400 Subject: [PATCH 2/2] whitespace --- pytorch_lightning/trainer/training_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/training_loop.py b/pytorch_lightning/trainer/training_loop.py index c000b1bfa9046..c5fb1b9717869 100644 --- a/pytorch_lightning/trainer/training_loop.py +++ b/pytorch_lightning/trainer/training_loop.py @@ -754,7 +754,7 @@ def training_forward(self, batch, batch_idx, opt_idx, hiddens): gpu_id = 0 if isinstance(self.data_parallel_device_ids, list): gpu_id = self.data_parallel_device_ids[0] - + # Don't copy the batch since there is a single gpu that the batch could # be referenced from and if there are multiple optimizers the batch will # wind up copying it to the same device repeatedly.