Remove unnecessary attention mask (#8733)

* pass a config to GPTDataset

Signed-off-by: Xiaowei Ren <xren@nvidia.com>

* set attention mask to None if dataloader does not have it

Signed-off-by: Xiaowei Ren <xren@nvidia.com>

* fix function name

Signed-off-by: Xiaowei Ren <xren@nvidia.com>

* fix nsys profile

Signed-off-by: Xiaowei Ren <xren@nvidia.com>

* dataset config variable name change

Signed-off-by: Xiaowei Ren <xren@nvidia.com>

* Apply isort and black reformatting

Signed-off-by: xrennvidia <xrennvidia@users.noreply.github.com>

---------

Signed-off-by: Xiaowei Ren <xren@nvidia.com>
Signed-off-by: xrennvidia <xrennvidia@users.noreply.github.com>
Co-authored-by: xrennvidia <xrennvidia@users.noreply.github.com>

nemo/collections/nlp/models/language_modeling/megatron_gpt_model.py

@@ -1126,6 +1126,7 @@ def get_batch(self, data_iterator, tuning):
             'tokens': data["tokens"],
             'labels': data["labels"],
             'loss_mask': data["loss_mask"],
+            'attention_mask': None if "attention_mask" not in data else data["attention_mask"],
             'position_ids': data["position_ids"],
         }
         if "attention_mask" in data:
@@ -1497,6 +1498,7 @@ def build_train_valid_test_datasets(self):
                 "reset_position_ids": self.reset_position_ids,
                 "reset_attention_mask": self.reset_attention_mask,
                 "eod_mask_loss": self.eod_mask_loss,
+                "create_attention_mask": not self.get_attention_mask_from_fusion,
                 "mmap_bin_files": self.cfg.data.get("mmap_bin_files", True),
             }
 

nemo/core/optim/distributed_adam.py

@@ -122,7 +122,7 @@ def __init__(
     ):
 
         # Initialize process groups
-        if 'process_group' not in kwargs and not parallel_state.is_unitialized():
+        if 'process_group' not in kwargs and parallel_state.is_initialized():
             kwargs['process_group'] = parallel_state.get_data_parallel_group(with_context_parallel=True)
         if disable_distributed_parameters:
             world_size = torch.distributed.get_world_size()