From d8a09739fe34fe86bb87e9899ca50ad3ddb2b253 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Wed, 17 Mar 2021 17:01:31 +0900 Subject: [PATCH 01/21] Add test for scheduler.step in manual optimization --- .../optimization/test_manual_optimization.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/trainer/optimization/test_manual_optimization.py b/tests/trainer/optimization/test_manual_optimization.py index 8ad603a7677ea..1f221168c8327 100644 --- a/tests/trainer/optimization/test_manual_optimization.py +++ b/tests/trainer/optimization/test_manual_optimization.py @@ -1147,3 +1147,45 @@ def dis_closure(): @RunIf(min_gpus=2, special=True) def test_step_with_optimizer_closure_with_different_frequencies_ddp_with_toggle_model(tmpdir): train_manual_optimization(tmpdir, "ddp", model_cls=TestManualOptimizationDDPModelToggleModel) + + +def test_lr_scheduler_step_count(tmpdir): + """ + Test `scheduler.step` is called only in LightningModule manually. + """ + class TestModel(BoringModel): + def __init__(self): + super().__init__() + self.automatic_optimization = False + + def training_step(self, batch, batch_idx): + optimizer = self.optimizers() + lr_scheduler = self.lr_schedulers() + + output = self(batch) + loss = self.loss(batch, output) + + optimizer.zero_grad() + self.manual_backward(loss) + optimizer.step() + lr_scheduler.step() + + model = TestModel() + model.training_step_end = None + model.training_epoch_end = None + + limit_train_batches = 4 + + trainer = Trainer( + max_epochs=1, + default_root_dir=tmpdir, + limit_train_batches=limit_train_batches, + limit_test_batches=1, + limit_val_batches=1, + ) + with patch("torch.optim.SGD.step") as opt_step, \ + patch("torch.optim.lr_scheduler.StepLR.step") as lr_step: + trainer.fit(model) + + assert opt_step.call_count == limit_train_batches + assert lr_step.call_count == limit_train_batches From 761ca277b7c45a0610d20bf68f88a9da5fbaff28 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Mon, 5 Apr 2021 06:00:14 +0900 Subject: [PATCH 02/21] Update the test --- .../optimization/test_manual_optimization.py | 27 +++++-------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/tests/trainer/optimization/test_manual_optimization.py b/tests/trainer/optimization/test_manual_optimization.py index 1f221168c8327..1274e8514563c 100644 --- a/tests/trainer/optimization/test_manual_optimization.py +++ b/tests/trainer/optimization/test_manual_optimization.py @@ -1149,43 +1149,28 @@ def test_step_with_optimizer_closure_with_different_frequencies_ddp_with_toggle_ train_manual_optimization(tmpdir, "ddp", model_cls=TestManualOptimizationDDPModelToggleModel) -def test_lr_scheduler_step_count(tmpdir): +def test_lr_scheduler_step_not_called(tmpdir): """ - Test `scheduler.step` is called only in LightningModule manually. + Test `lr_scheduler.step()` is not called in manual optimization. """ class TestModel(BoringModel): def __init__(self): super().__init__() self.automatic_optimization = False - def training_step(self, batch, batch_idx): - optimizer = self.optimizers() - lr_scheduler = self.lr_schedulers() - - output = self(batch) - loss = self.loss(batch, output) - - optimizer.zero_grad() - self.manual_backward(loss) - optimizer.step() - lr_scheduler.step() - model = TestModel() model.training_step_end = None model.training_epoch_end = None - limit_train_batches = 4 - trainer = Trainer( max_epochs=1, default_root_dir=tmpdir, - limit_train_batches=limit_train_batches, + limit_train_batches=1, limit_test_batches=1, limit_val_batches=1, ) - with patch("torch.optim.SGD.step") as opt_step, \ - patch("torch.optim.lr_scheduler.StepLR.step") as lr_step: + + with patch("torch.optim.lr_scheduler.StepLR.step") as lr_step: trainer.fit(model) - assert opt_step.call_count == limit_train_batches - assert lr_step.call_count == limit_train_batches + assert lr_step.call_count == 0 From df44b16798e804c9657a3216aa8edcf81ff1e139 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Mon, 5 Apr 2021 06:01:05 +0900 Subject: [PATCH 03/21] Disable lr_scheduler.step() in manual optimization --- pytorch_lightning/trainer/connectors/optimizer_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/connectors/optimizer_connector.py b/pytorch_lightning/trainer/connectors/optimizer_connector.py index a50603bb58dbf..22189502757aa 100644 --- a/pytorch_lightning/trainer/connectors/optimizer_connector.py +++ b/pytorch_lightning/trainer/connectors/optimizer_connector.py @@ -32,7 +32,7 @@ def update_learning_rates(self, interval: str, monitor_metrics=None): interval: either 'epoch' or 'step'. monitor_metrics: dict of possible values to monitor """ - if not self.trainer.lr_schedulers: + if not self.trainer.lr_schedulers or not self.trainer.train_loop.automatic_optimization: return for scheduler_idx, lr_scheduler in enumerate(self.trainer.lr_schedulers): From d8de6b01ead5e48f74dc2079e8049390f3ef93c8 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Mon, 5 Apr 2021 06:32:18 +0900 Subject: [PATCH 04/21] Add comment to test --- tests/trainer/optimization/test_manual_optimization.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/trainer/optimization/test_manual_optimization.py b/tests/trainer/optimization/test_manual_optimization.py index 1274e8514563c..1835e0f400f70 100644 --- a/tests/trainer/optimization/test_manual_optimization.py +++ b/tests/trainer/optimization/test_manual_optimization.py @@ -1173,4 +1173,7 @@ def __init__(self): with patch("torch.optim.lr_scheduler.StepLR.step") as lr_step: trainer.fit(model) - assert lr_step.call_count == 0 + # If a lr scheduler inherits `torch.optim.lr_scheduler._LRScheduler`, + # `.step()` is called once when its instantiation. + # Thus, the call count should be 1. + assert lr_step.call_count == 1 From fe267a89e8dcb1bf89202257b3f02298e612bcdd Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Mon, 5 Apr 2021 16:23:51 +0900 Subject: [PATCH 05/21] Update CHANGLOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81846809fbf85..4d77b87dff346 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Changed `PyTorchProfiler` to use `torch.autograd.profiler.record_function` to record functions ([#6349](https://github.com/PyTorchLightning/pytorch-lightning/pull/6349)) +- Disabled `lr_schdeulder.step()` in manual optimization ([#6825](https://github.com/PyTorchLightning/pytorch-lightning/pull/6825)) + + ### Deprecated - `period` has been deprecated in favor of `every_n_val_epochs` in the `ModelCheckpoint` callback ([#6146](https://github.com/PyTorchLightning/pytorch-lightning/pull/6146)) From bf58e9c2004893f08e48439f96a26a504082f0c0 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Mon, 5 Apr 2021 19:16:05 +0900 Subject: [PATCH 06/21] Add training_step to TestModel --- tests/trainer/optimization/test_manual_optimization.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/trainer/optimization/test_manual_optimization.py b/tests/trainer/optimization/test_manual_optimization.py index 1835e0f400f70..3b85fc453e563 100644 --- a/tests/trainer/optimization/test_manual_optimization.py +++ b/tests/trainer/optimization/test_manual_optimization.py @@ -1158,6 +1158,16 @@ def __init__(self): super().__init__() self.automatic_optimization = False + def training_step(self, batch, batch_idx): + opt = self.optimizers() + + output = self(batch) + loss = self.loss(batch, output) + + opt.zero_grad() + self.manual_backward(loss) + opt.step() + model = TestModel() model.training_step_end = None model.training_epoch_end = None From 4ef09b55c409e25b28e2a133ad87ae94b7c6dc7c Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Mon, 5 Apr 2021 19:20:14 +0900 Subject: [PATCH 07/21] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli --- tests/trainer/optimization/test_manual_optimization.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/trainer/optimization/test_manual_optimization.py b/tests/trainer/optimization/test_manual_optimization.py index 3b85fc453e563..4d98dd16a9a4b 100644 --- a/tests/trainer/optimization/test_manual_optimization.py +++ b/tests/trainer/optimization/test_manual_optimization.py @@ -1175,15 +1175,13 @@ def training_step(self, batch, batch_idx): trainer = Trainer( max_epochs=1, default_root_dir=tmpdir, - limit_train_batches=1, - limit_test_batches=1, - limit_val_batches=1, + fast_dev_run=2, ) with patch("torch.optim.lr_scheduler.StepLR.step") as lr_step: trainer.fit(model) # If a lr scheduler inherits `torch.optim.lr_scheduler._LRScheduler`, - # `.step()` is called once when its instantiation. + # `.step()` is called once during its instantiation. # Thus, the call count should be 1. assert lr_step.call_count == 1 From 80eea2215c71478ddc1d76c1cecf7455ab919ee3 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Wed, 7 Apr 2021 04:10:07 +0900 Subject: [PATCH 08/21] Fix typo Co-authored-by: thomas chaton --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d77b87dff346..dfdafe7074ad4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,7 +102,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Changed `PyTorchProfiler` to use `torch.autograd.profiler.record_function` to record functions ([#6349](https://github.com/PyTorchLightning/pytorch-lightning/pull/6349)) -- Disabled `lr_schdeulder.step()` in manual optimization ([#6825](https://github.com/PyTorchLightning/pytorch-lightning/pull/6825)) +- Disabled `lr_scheduler.step()` in manual optimization ([#6825](https://github.com/PyTorchLightning/pytorch-lightning/pull/6825)) ### Deprecated From 6d263f056c946dc162297eacf4416473b43bffa9 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Wed, 7 Apr 2021 04:20:17 +0900 Subject: [PATCH 09/21] Update docs --- docs/source/common/optimizers.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index 422302ea8987e..cd1a5ddea2910 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -44,6 +44,8 @@ to manually manage the optimization process. To do so, do the following: .. warning:: Before 1.2, ``optimzer.step`` was calling ``optimizer.zero_grad()`` internally. From 1.2, it is left to the users expertise. +.. warning:: Before 1.3, ``lr_scheduler.step`` was called automatically. From 1.3, it is left to the users expertise. + .. tip:: To perform ``accumulate_grad_batches`` with one optimizer, you can do as such. .. tip:: ``self.optimizers()`` will return ``LightningOptimizer`` objects. You can access your own optimizer with ``optimizer.optimizer``. However, if you use your own optimizer to perform a step, Lightning won't be able to support accelerators and precision for you. From 523f2e860eda188e4454ff506aeea64f4cb0b387 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Thu, 8 Apr 2021 00:30:41 +0900 Subject: [PATCH 10/21] Add warning when invalid keys are specified --- pytorch_lightning/trainer/optimizers.py | 22 ++++++++++++++++--- tests/trainer/optimization/test_optimizers.py | 22 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/trainer/optimizers.py b/pytorch_lightning/trainer/optimizers.py index a247fb92cd22f..d56eeba4356e1 100644 --- a/pytorch_lightning/trainer/optimizers.py +++ b/pytorch_lightning/trainer/optimizers.py @@ -80,7 +80,8 @@ def init_optimizers(self, model: LightningModule) -> Tuple[List, List, List]: ' * A list of the previously described dict format, with an optional "frequency" key (int)' ) - lr_schedulers = self.configure_schedulers(lr_schedulers, monitor=monitor) + is_manual_optimization = not model.automatic_optimization + lr_schedulers = self.configure_schedulers(lr_schedulers, monitor, is_manual_optimization) _validate_scheduler_optimizer(optimizers, lr_schedulers) return optimizers, lr_schedulers, optimizer_frequencies @@ -98,8 +99,13 @@ def _convert_to_lightning_optimizer(trainer, optimizer): for opt_idx, opt in enumerate(self.optimizers) } - def configure_schedulers(self, schedulers: list, monitor: Optional[str] = None): - # Convert each scheduler into dict structure with relevant information + def configure_schedulers( + self, + schedulers: list, + monitor: Optional[str], + is_manual_optimization: bool, + ): + """Convert each scheduler into dict structure with relevant information""" lr_schedulers = [] default_config = _get_default_scheduler_config() for scheduler in schedulers: @@ -117,6 +123,16 @@ def configure_schedulers(self, schedulers: list, monitor: Optional[str] = None): f'The "interval" key in lr scheduler dict must be "step" or "epoch"' f' but is "{scheduler["interval"]}"' ) + if is_manual_optimization: + invalid_keys = {'interval', 'frequency', 'reduce_on_plateau', 'monitor', 'strict'} + keys_to_warn = [k for k in scheduler.keys() if k in invalid_keys] + + if keys_to_warn: + rank_zero_warn( + f'The lr scheduler dict contains the key(s) {keys_to_warn}, but the keys will be ignored.' + ' You need to call `lr_scheduler.step()` manually in manual optimization.', + RuntimeWarning, + ) scheduler['reduce_on_plateau'] = isinstance( scheduler['scheduler'], optim.lr_scheduler.ReduceLROnPlateau diff --git a/tests/trainer/optimization/test_optimizers.py b/tests/trainer/optimization/test_optimizers.py index f13448187364c..3ffaf36632626 100644 --- a/tests/trainer/optimization/test_optimizers.py +++ b/tests/trainer/optimization/test_optimizers.py @@ -476,3 +476,25 @@ def configure_optimizers(self): trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True) with pytest.raises(MisconfigurationException, match="attatched with an optimizer that wasn't returned"): trainer.fit(model) + + +def test_warn_invalid_scheduler_key_in_manual_optimization(tmpdir): + """ + Test warning when invalid scheduler keys are provided in manual optimization. + """ + + class TestModel(BoringModel): + + def __init__(self): + super().__init__() + self.automatic_optimization = False + + def configure_optimizers(self): + opt = torch.optim.SGD(self.layer.parameters(), lr=0.1) + sch = torch.optim.lr_scheduler.StepLR(opt, step_size=1) + return [opt], [{"scheduler": sch, "interval": "epoch"}] + + model = TestModel() + trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True) + with pytest.warns(RuntimeWarning, match='the keys will be ignored'): + trainer.fit(model) From ebf6a3a86776b7ab26526915fed2ac2d7a29dde4 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Thu, 8 Apr 2021 00:50:47 +0900 Subject: [PATCH 11/21] Update docs --- docs/source/common/optimizers.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index cd1a5ddea2910..be59734ae1ae5 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -44,12 +44,14 @@ to manually manage the optimization process. To do so, do the following: .. warning:: Before 1.2, ``optimzer.step`` was calling ``optimizer.zero_grad()`` internally. From 1.2, it is left to the users expertise. -.. warning:: Before 1.3, ``lr_scheduler.step`` was called automatically. From 1.3, it is left to the users expertise. +.. warning:: Before 1.3, ``lr_scheduler.step`` was called automatically in both manual and automatic optimization. From 1.3, ``lr_scheduler.step`` is disabled in manual optimization so that you can call it at arbitrary intervals. .. tip:: To perform ``accumulate_grad_batches`` with one optimizer, you can do as such. .. tip:: ``self.optimizers()`` will return ``LightningOptimizer`` objects. You can access your own optimizer with ``optimizer.optimizer``. However, if you use your own optimizer to perform a step, Lightning won't be able to support accelerators and precision for you. +.. tip:: ``self.lr_schedulers()`` will return your learning rate schedulers defined in ``LightningModule.configure_optimizers()``. + .. code-block:: python def __init__(self): @@ -77,6 +79,7 @@ Here is the same example as above using a ``closure``. def training_step(self, batch, batch_idx): opt = self.optimizers() + sch = self.lr_schedulers() def closure(): # Only zero_grad on the first batch to accumulate gradients @@ -89,6 +92,7 @@ Here is the same example as above using a ``closure``. return loss opt.step(closure=closure) + sch.step() .. tip:: Be careful where you call ``zero_grad`` or your model won't converge. It is good pratice to call ``zero_grad`` before ``manual_backward``. From 1e587f0274fc0fbf120885ce16754ad2c9ebb9ce Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Thu, 8 Apr 2021 00:55:05 +0900 Subject: [PATCH 12/21] Update docs --- docs/source/common/optimizers.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index be59734ae1ae5..b94976b33cc12 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -44,14 +44,12 @@ to manually manage the optimization process. To do so, do the following: .. warning:: Before 1.2, ``optimzer.step`` was calling ``optimizer.zero_grad()`` internally. From 1.2, it is left to the users expertise. -.. warning:: Before 1.3, ``lr_scheduler.step`` was called automatically in both manual and automatic optimization. From 1.3, ``lr_scheduler.step`` is disabled in manual optimization so that you can call it at arbitrary intervals. +.. warning:: Before 1.3, ``lr_scheduler.step`` was called automatically in both manual and automatic optimization. From 1.3, ``lr_scheduler.step`` is disabled in manual optimization so that you can call it at arbitrary intervals. Use ``self.lr_schedulers()`` in LightningModule to access your learning rate schedulers defined in ``LightningModule.configure_optimizers()``. .. tip:: To perform ``accumulate_grad_batches`` with one optimizer, you can do as such. .. tip:: ``self.optimizers()`` will return ``LightningOptimizer`` objects. You can access your own optimizer with ``optimizer.optimizer``. However, if you use your own optimizer to perform a step, Lightning won't be able to support accelerators and precision for you. -.. tip:: ``self.lr_schedulers()`` will return your learning rate schedulers defined in ``LightningModule.configure_optimizers()``. - .. code-block:: python def __init__(self): From 9f4124bb6e278b240327c8d40569cb9b0ccabeba Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Thu, 8 Apr 2021 01:02:57 +0900 Subject: [PATCH 13/21] Update the test comment --- tests/trainer/optimization/test_manual_optimization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/trainer/optimization/test_manual_optimization.py b/tests/trainer/optimization/test_manual_optimization.py index 4d98dd16a9a4b..a8f6fa0aaa243 100644 --- a/tests/trainer/optimization/test_manual_optimization.py +++ b/tests/trainer/optimization/test_manual_optimization.py @@ -1183,5 +1183,5 @@ def training_step(self, batch, batch_idx): # If a lr scheduler inherits `torch.optim.lr_scheduler._LRScheduler`, # `.step()` is called once during its instantiation. - # Thus, the call count should be 1. + # Thus, the call count should be 1, not 0. assert lr_step.call_count == 1 From 76d304b9d04ab7ab9b863476cda4d36002422306 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Thu, 8 Apr 2021 05:44:02 +0900 Subject: [PATCH 14/21] Add return type --- pytorch_lightning/trainer/optimizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/optimizers.py b/pytorch_lightning/trainer/optimizers.py index d56eeba4356e1..5a7873232b394 100644 --- a/pytorch_lightning/trainer/optimizers.py +++ b/pytorch_lightning/trainer/optimizers.py @@ -104,7 +104,7 @@ def configure_schedulers( schedulers: list, monitor: Optional[str], is_manual_optimization: bool, - ): + ) -> List[Dict[str, Any]]: """Convert each scheduler into dict structure with relevant information""" lr_schedulers = [] default_config = _get_default_scheduler_config() From 1c0c8a3551f9b4d596f484d13eb6712c3a7d3bb5 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Thu, 8 Apr 2021 20:22:23 +0900 Subject: [PATCH 15/21] Add is_last_batch to trainer --- docs/source/common/optimizers.rst | 19 +++++++++++++++++++ pytorch_lightning/trainer/training_loop.py | 1 + 2 files changed, 20 insertions(+) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index b94976b33cc12..d7496d75022f5 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -46,6 +46,25 @@ to manually manage the optimization process. To do so, do the following: .. warning:: Before 1.3, ``lr_scheduler.step`` was called automatically in both manual and automatic optimization. From 1.3, ``lr_scheduler.step`` is disabled in manual optimization so that you can call it at arbitrary intervals. Use ``self.lr_schedulers()`` in LightningModule to access your learning rate schedulers defined in ``LightningModule.configure_optimizers()``. +.. testcode:: python + + def __init__(self): + self.automatic_optimization = False + + def training_step(self, batch, batch_idx): + # do foward, backward, and optimization + ... + + # step every `n` batches + if batch_idx+1 % n == 0: + sch = self.lr_schedulers() + sch.step() + + # step every `n` epochs + if self.trainer.is_last_batch and self.trainer.current_epoch+1 % n == 0: + sch = self.lr_schedulers() + sch.step() + .. tip:: To perform ``accumulate_grad_batches`` with one optimizer, you can do as such. .. tip:: ``self.optimizers()`` will return ``LightningOptimizer`` objects. You can access your own optimizer with ``optimizer.optimizer``. However, if you use your own optimizer to perform a step, Lightning won't be able to support accelerators and precision for you. diff --git a/pytorch_lightning/trainer/training_loop.py b/pytorch_lightning/trainer/training_loop.py index 71d9407062001..d9d00a8fda037 100644 --- a/pytorch_lightning/trainer/training_loop.py +++ b/pytorch_lightning/trainer/training_loop.py @@ -413,6 +413,7 @@ def run_training_epoch(self): for batch_idx, (batch, is_last_batch) in train_dataloader: self.trainer.batch_idx = batch_idx + self.trainer.is_last_batch = is_last_batch # ------------------------------------ # TRAINING_STEP + TRAINING_STEP_END From 2073eb810d0d87502de3e600f11b049610339c7e Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Thu, 8 Apr 2021 20:25:11 +0900 Subject: [PATCH 16/21] Fix docs --- docs/source/common/optimizers.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index d7496d75022f5..7fece1a7e4b46 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -56,12 +56,12 @@ to manually manage the optimization process. To do so, do the following: ... # step every `n` batches - if batch_idx+1 % n == 0: + if (batch_idx+1) % n == 0: sch = self.lr_schedulers() sch.step() # step every `n` epochs - if self.trainer.is_last_batch and self.trainer.current_epoch+1 % n == 0: + if self.trainer.is_last_batch and (self.trainer.current_epoch+1) % n == 0: sch = self.lr_schedulers() sch.step() From 6b8f7335085dcab26eabe61461770d656d66cc1d Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Fri, 9 Apr 2021 04:15:37 +0900 Subject: [PATCH 17/21] Fix format in docs --- docs/source/common/optimizers.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index 7fece1a7e4b46..0f08337dd8dba 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -56,14 +56,14 @@ to manually manage the optimization process. To do so, do the following: ... # step every `n` batches - if (batch_idx+1) % n == 0: + if (batch_idx + 1) % n == 0: sch = self.lr_schedulers() sch.step() # step every `n` epochs - if self.trainer.is_last_batch and (self.trainer.current_epoch+1) % n == 0: - sch = self.lr_schedulers() - sch.step() + if self.trainer.is_last_batch and (self.trainer.current_epoch + 1) % n == 0: + sch = self.lr_schedulers() + sch.step() .. tip:: To perform ``accumulate_grad_batches`` with one optimizer, you can do as such. From 97cdb6dc228fe807221fe43e6037b8a1b717780d Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Fri, 9 Apr 2021 18:42:31 +0900 Subject: [PATCH 18/21] Update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfdafe7074ad4..fdb60104f34aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added `model` parameter to precision plugins' `clip_gradients` signature ([#6764](https://github.com/PyTorchLightning/pytorch-lightning/pull/6764)) +- Added `is_last_batch` attribute to `Trainer` ([#6825](https://github.com/PyTorchLightning/pytorch-lightning/pull/6825)) + + ### Changed - Renamed `pytorch_lightning.callbacks.swa` to `pytorch_lightning.callbacks.stochastic_weight_avg` ([#6259](https://github.com/PyTorchLightning/pytorch-lightning/pull/6259)) From 32445d7c8b6885705d9db0443ea34c8e07e02d0b Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Fri, 9 Apr 2021 18:47:49 +0900 Subject: [PATCH 19/21] Use train_loop.automatic_optimization --- pytorch_lightning/trainer/optimizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/optimizers.py b/pytorch_lightning/trainer/optimizers.py index 5a7873232b394..780fc5840d13b 100644 --- a/pytorch_lightning/trainer/optimizers.py +++ b/pytorch_lightning/trainer/optimizers.py @@ -80,7 +80,7 @@ def init_optimizers(self, model: LightningModule) -> Tuple[List, List, List]: ' * A list of the previously described dict format, with an optional "frequency" key (int)' ) - is_manual_optimization = not model.automatic_optimization + is_manual_optimization = not self.train_loop.automatic_optimization lr_schedulers = self.configure_schedulers(lr_schedulers, monitor, is_manual_optimization) _validate_scheduler_optimizer(optimizers, lr_schedulers) From c4579086c011cd8eb75387cf7c7c55476e95ea0d Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 19 Apr 2021 15:11:22 +0100 Subject: [PATCH 20/21] resolve merge --- docs/source/common/optimizers.rst | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index 8f3fceadf797b..f916dc2f3806c 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -35,7 +35,7 @@ To manually optimize, do the following: * ``optimizer.step()`` to update your model parameters Here is a minimal example of manual optimization. - + .. testcode:: python from pytorch_lightning import LightningModule @@ -62,6 +62,29 @@ Here is a minimal example of manual optimization. Be careful where you call ``optimizer.zero_grad()``, or your model won't converge. It is good practice to call ``optimizer.zero_grad()`` before ``self.manual_backward(loss)``. +.. warning:: Before 1.3, ``lr_scheduler.step`` was called automatically in both manual and automatic optimization. From 1.3, ``lr_scheduler.step`` is disabled in manual optimization so that you can call it at arbitrary intervals. Use ``self.lr_schedulers()`` in LightningModule to access your learning rate schedulers defined in ``LightningModule.configure_optimizers()``. + +.. testcode:: python + + def __init__(self): + self.automatic_optimization = False + + def training_step(self, batch, batch_idx): + # do forward, backward, and optimization + ... + + # step every `n` batches + if (batch_idx + 1) % n == 0: + sch = self.lr_schedulers() + sch.step() + + # step every `n` epochs + if self.trainer.is_last_batch and (self.trainer.current_epoch + 1) % n == 0: + sch = self.lr_schedulers() + sch.step() + +.. tip:: To perform ``accumulate_grad_batches`` with one optimizer, you can do as such. + Gradient accumulation --------------------- From 0ff4fea7ad94bc7912697db543e70d8aa40cfb45 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Tue, 20 Apr 2021 07:13:19 +0900 Subject: [PATCH 21/21] Revert doc change --- docs/source/common/optimizers.rst | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index f916dc2f3806c..389b1e1548026 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -62,29 +62,7 @@ Here is a minimal example of manual optimization. Be careful where you call ``optimizer.zero_grad()``, or your model won't converge. It is good practice to call ``optimizer.zero_grad()`` before ``self.manual_backward(loss)``. -.. warning:: Before 1.3, ``lr_scheduler.step`` was called automatically in both manual and automatic optimization. From 1.3, ``lr_scheduler.step`` is disabled in manual optimization so that you can call it at arbitrary intervals. Use ``self.lr_schedulers()`` in LightningModule to access your learning rate schedulers defined in ``LightningModule.configure_optimizers()``. - -.. testcode:: python - - def __init__(self): - self.automatic_optimization = False - - def training_step(self, batch, batch_idx): - # do forward, backward, and optimization - ... - - # step every `n` batches - if (batch_idx + 1) % n == 0: - sch = self.lr_schedulers() - sch.step() - - # step every `n` epochs - if self.trainer.is_last_batch and (self.trainer.current_epoch + 1) % n == 0: - sch = self.lr_schedulers() - sch.step() - -.. tip:: To perform ``accumulate_grad_batches`` with one optimizer, you can do as such. - +----- Gradient accumulation ---------------------