Tutorial on ITN with Thutmose tagger and small fixes (#4117)

* 1. Add tutorial. 2. Move a function to fix import in tutorial. 3. Merge multiple spaces into one space in the final output

Signed-off-by: Alexandra Antonova <aleksandraa@nvidia.com>

* fixes for code review

Signed-off-by: Alexandra Antonova <aleksandraa@nvidia.com>

* Add tutorial to tutorials.rst

Signed-off-by: Alexandra Antonova <aleksandraa@nvidia.com>

Co-authored-by: Alexandra Antonova <aleksandraa@nvidia.com>

docs/source/starthere/tutorials.rst

@@ -151,6 +151,9 @@ To run a tutorial:
    * - Text Processing
      - Text Normalization and Inverse Normalization for ASR and TTS
      - `Text Normalization <https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/text_processing/Text_(Inverse)_Normalization.ipynb>`_
+   * - Text Processing
+     - Inverse Text Normalization for ASR - Thutmose Tagger
+     - `Inverse Text Normalization with Thutmose Tagger <https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/text_processing/ITN_with_Thutmose_Tagger.ipynb>`_
    * - Text Processing
      - Constructing Normalization Grammars with WFSTs
      - `WFST Tutorial <https://colab.research.google.com/github/NVIDIA/NeMo/blob/stable/tutorials/text_processing/WFST_Tutorial.ipynb>`_

examples/nlp/text_normalization_as_tagging/dataset_preparation/prepare_corpora_after_alignment.py

@@ -24,8 +24,7 @@
 from collections import Counter
 from typing import Dict, Optional, TextIO, Tuple
 
-from examples.nlp.text_normalization_as_tagging.dataset_preparation.utils import get_src_and_dst_for_alignment
-
+from nemo.collections.nlp.data.text_normalization_as_tagging.utils import get_src_and_dst_for_alignment
 from nemo.utils import logging
 
 parser = ArgumentParser(description="Produce data for the ThutmoseTaggerModel")

examples/nlp/text_normalization_as_tagging/dataset_preparation/prepare_corpora_for_alignment.py

@@ -47,7 +47,7 @@
 from os.path import isdir, join
 from shutil import rmtree
 
-from examples.nlp.text_normalization_as_tagging.dataset_preparation.utils import get_src_and_dst_for_alignment
+from nemo.collections.nlp.data.text_normalization_as_tagging.utils import get_src_and_dst_for_alignment
 
 parser = ArgumentParser(description='Split corpus to subcorpora for giza alignment')
 parser.add_argument('--data_dir', type=str, required=True, help='Path to folder with data')

examples/nlp/text_normalization_as_tagging/install_requirements.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+git clone https://github.com/moses-smt/giza-pp.git giza-pp
+cd giza-pp
+make
+cd ..

nemo/collections/nlp/data/text_normalization_as_tagging/tagging.py

@@ -207,9 +207,10 @@ def realize_output(self, tags: List[Tag], semiotic_labels: List[str]) -> Tuple[s
                 output_tokens.append(frag.replace(" ", "").replace("_", ""))
             else:
                 output_tokens.append(frag.strip().replace("_", ""))
-
+        output_str = " ".join(output_tokens)
+        output_str = re.sub(r" +", " ", output_str)
         return (
-            " ".join(output_tokens),
+            output_str,
             " ".join(self.source_tokens),
             " ".join(out_tags_without_swap),
             output_tags_with_swap_str,