Fix metric setup for finetuning without a test set (#4585)

* Fix metric setup for finetuning without a test set

Signed-off-by: MaximumEntropy <sandeep.subramanian.1@umontreal.ca>

* Fix log key

Signed-off-by: MaximumEntropy <sandeep.subramanian.1@umontreal.ca>

* Remove pdb

Signed-off-by: MaximumEntropy <sandeep.subramanian.1@umontreal.ca>

* Minor

Signed-off-by: MaximumEntropy <sandeep.subramanian.1@umontreal.ca>

* Fix skip train ds building while finetuning

Signed-off-by: MaximumEntropy <sandeep.subramanian.1@umontreal.ca>

Co-authored-by: Oleksii Kuchaiev <okuchaiev@users.noreply.github.com>

examples/nlp/language_modeling/megatron_t5_seq2seq_eval.py

@@ -65,9 +65,14 @@ def main(cfg) -> None:
             trainer.callbacks[idx] = StatelessTimer(cfg.trainer.max_time,)
 
     # Get the T5 Base configuration.
-    t5_cfg = MegatronT5GLUEModel.restore_from(
-        restore_path=cfg.model.restore_from_path, trainer=trainer, return_config=True
-    )
+    if hasattr(t5_cfg.data.validation_ds, 'task_name'):
+        t5_cfg = MegatronT5GLUEModel.restore_from(
+            restore_path=cfg.model.restore_from_path, trainer=trainer, return_config=True
+        )
+    else:
+        t5_cfg = MegatronT5FinetuneModel.restore_from(
+            restore_path=cfg.model.restore_from_path, trainer=trainer, return_config=True
+        )
 
     # Override the T5 configuration with the one from the config file.
     # NOTE: Only data can be overriden here since this the file being restored here should already correspond to a GLUE/XNLI finetuned model.

nemo/collections/nlp/models/language_modeling/megatron_finetune_model.py

@@ -99,7 +99,7 @@ def setup_metric(self, data_cfg):
                         metric(average=data_cfg.metric.average, num_classes=data_cfg.metric.num_classes)
                         if metric_name != 'exact_string_match'
                         else metric()
-                        for _ in range(len(self.cfg.data.test_ds.src_file_name))
+                        for _ in range(len(data_cfg.src_file_name))
                     ]
                 else:
                     metric = [metric(average=data_cfg.metric.average, num_classes=data_cfg.metric.num_classes)]
@@ -387,7 +387,7 @@ def inference_epoch_end(self, outputs, mode, data_cfg):
                     metric = metric['acc']
             else:
                 self.log(metric_log_key, metric)
-                logging.info(f"{mode} {metric_name}: {metric}")
+                logging.info(f"{metric_log_key}: {metric}")
             metric_object.reset()
 
             averaged_loss.append(loss)
@@ -663,11 +663,11 @@ def build_train_valid_test_datasets(self, stage):
         if stage != 'test':
             self._validation_ds = self._build_eval_dataset(self.cfg.data.validation_ds)
 
-        if stage != 'validation':
+        if stage != 'validate':
             if hasattr(self.cfg.data, 'test_ds'):
                 self._test_ds = self._build_eval_dataset(self.cfg.data.test_ds)
 
-        if stage == 'validation' or stage == 'test':
+        if stage == 'validate' or stage == 'test':
             return
         self._train_ds = self._build_train_dataset(self.cfg.data.train_ds)
         logging.info(f'Finished building datasets ...')