Merge pull request #304 from yzhang123/group_unit_tests

group unittests into collections

Jenkinsfile

@@ -24,11 +24,26 @@ pipeline {
         sh 'python setup.py style'
       }
     }
-    stage('Unittests') {
+    stage('Unittests general') {
       steps {
         sh './reinstall.sh && python -m unittest tests/*.py'
       }
     }
+    stage('Unittests ASR') {
+      steps {
+        sh 'python -m unittest tests/asr/*.py'
+      }
+    }
+    stage('Unittests NLP') {
+      steps {
+        sh 'python -m unittest tests/nlp/*.py'
+      }
+    }
+    stage('Unittests TTS') {
+      steps {
+        sh 'python -m unittest tests/tts/*.py'
+      }
+    }
 
     stage('Parallel Stage1') {
       failFast true

tests/test_asr.py → tests/asr/test_asr.py

@@ -2,15 +2,14 @@
 import os
 import shutil
 import tarfile
-import unittest
 
 from ruamel.yaml import YAML
 
 import nemo
 import nemo.collections.asr as nemo_asr
-from .common_setup import NeMoUnitTest
 from nemo.collections.asr.parts import AudioDataset, WaveformFeaturizer, collections, parsers
 from nemo.core import DeviceType
+from tests.common_setup import NeMoUnitTest
 
 # ! /usr/bin/python
 # -*- coding: utf-8 -*-
@@ -65,7 +64,7 @@ class TestASRPytorch(NeMoUnitTest):
         "z",
         "'",
     ]
-    manifest_filepath = "tests/data/asr/an4_train.json"
+    manifest_filepath = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/asr/an4_train.json"))
     featurizer_config = {
         'window': 'hann',
         'dither': 1e-05,
@@ -83,23 +82,23 @@ class TestASRPytorch(NeMoUnitTest):
     @classmethod
     def setUpClass(cls) -> None:
         super().setUpClass()
-        data_folder = "tests/data/"
+        data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/"))
         print("Looking up for test ASR data")
-        if not os.path.exists(data_folder + "asr"):
-            print("Extracting ASR data to: {0}".format(data_folder + "asr"))
-            tar = tarfile.open("tests/data/asr.tar.gz", "r:gz")
+        if not os.path.exists(os.path.join(data_folder, "asr")):
+            print("Extracting ASR data to: {0}".format(os.path.join(data_folder, "asr")))
+            tar = tarfile.open(os.path.join(data_folder, "asr.tar.gz"), "r:gz")
             tar.extractall(path=data_folder)
             tar.close()
         else:
-            print("ASR data found in: {0}".format(data_folder + "asr"))
+            print("ASR data found in: {0}".format(os.path.join(data_folder, "asr")))
 
     @classmethod
     def tearDownClass(cls) -> None:
         super().tearDownClass()
-        data_folder = "tests/data/"
+        data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/"))
         print("Looking up for test ASR data")
-        if os.path.exists(data_folder + "asr"):
-            shutil.rmtree(data_folder + "asr")
+        if os.path.exists(os.path.join(data_folder, "asr")):
+            shutil.rmtree(os.path.join(data_folder, "asr"))
 
     def test_transcript_normalizers(self):
         # Create test json
@@ -154,7 +153,7 @@ def test_transcript_normalizers(self):
             "mister expand me",
             "mr don't expand me",
         ]
-        manifest_paths = "tests/data/asr/manifest_test.json"
+        manifest_paths = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/asr/manifest_test.json"))
 
         def remove_test_json():
             os.remove(manifest_paths)
@@ -231,18 +230,26 @@ def create_good_preprocessor_2():
     def test_kaldi_dataloader(self):
         batch_size = 4
         dl = nemo_asr.KaldiFeatureDataLayer(
-            kaldi_dir='tests/data/asr/kaldi_an4/', labels=self.labels, batch_size=batch_size,
+            kaldi_dir=os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/asr/kaldi_an4/')),
+            labels=self.labels,
+            batch_size=batch_size,
         )
         for data in dl.data_iterator:
             self.assertTrue(data[0].size(0) == batch_size)
 
         dl_test_min = nemo_asr.KaldiFeatureDataLayer(
-            kaldi_dir='tests/data/asr/kaldi_an4/', labels=self.labels, batch_size=batch_size, min_duration=1.0,
+            kaldi_dir=os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/asr/kaldi_an4/')),
+            labels=self.labels,
+            batch_size=batch_size,
+            min_duration=1.0,
         )
         self.assertTrue(len(dl_test_min) == 18)
 
         dl_test_max = nemo_asr.KaldiFeatureDataLayer(
-            kaldi_dir='tests/data/asr/kaldi_an4/', labels=self.labels, batch_size=batch_size, max_duration=5.0,
+            kaldi_dir=os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/asr/kaldi_an4/')),
+            labels=self.labels,
+            batch_size=batch_size,
+            max_duration=5.0,
         )
         self.assertTrue(len(dl_test_max) == 19)
 
@@ -318,7 +325,7 @@ def test_audio_preprocessors(self):
                 self.assertTrue(mfcc[0].shape[1] == 15)
 
     def test_jasper_training(self):
-        with open("tests/data/jasper_smaller.yaml") as file:
+        with open(os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/jasper_smaller.yaml"))) as file:
             jasper_model_definition = self.yaml.load(file)
         dl = nemo_asr.AudioToTextDataLayer(
             featurizer_config=self.featurizer_config,
@@ -370,7 +377,7 @@ def test_jasper_training(self):
         )
 
     def test_double_jasper_training(self):
-        with open("tests/data/jasper_smaller.yaml") as file:
+        with open(os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/jasper_smaller.yaml"))) as file:
             jasper_model_definition = self.yaml.load(file)
         dl = nemo_asr.AudioToTextDataLayer(
             featurizer_config=self.featurizer_config,
@@ -431,7 +438,7 @@ def test_double_jasper_training(self):
         )
 
     def test_quartznet_training(self):
-        with open("tests/data/quartznet_test.yaml") as f:
+        with open(os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/quartznet_test.yaml"))) as f:
             quartz_model_definition = self.yaml.load(f)
         dl = nemo_asr.AudioToTextDataLayer(
             featurizer_config=self.featurizer_config,
@@ -483,7 +490,7 @@ def test_quartznet_training(self):
         )
 
     def test_stft_conv(self):
-        with open("tests/data/jasper_smaller.yaml") as file:
+        with open(os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/jasper_smaller.yaml"))) as file:
             jasper_model_definition = self.yaml.load(file)
         dl = nemo_asr.AudioToTextDataLayer(
             featurizer_config=self.featurizer_config,
@@ -590,7 +597,7 @@ def test_clas(self):
         )
 
     def test_jasper_eval(self):
-        with open("tests/data/jasper_smaller.yaml") as file:
+        with open(os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/jasper_smaller.yaml"))) as file:
             jasper_model_definition = self.yaml.load(file)
         dl = nemo_asr.AudioToTextDataLayer(
             featurizer_config=self.featurizer_config,

tests/test_weight_share.py → tests/asr/test_weight_share.py

@@ -16,6 +16,9 @@
 # limitations under the License.
 # =============================================================================
 
+import os
+import shutil
+import tarfile
 from typing import Dict
 
 import numpy as np
@@ -24,9 +27,9 @@
 
 import nemo
 import nemo.collections.asr as nemo_asr
-from .common_setup import NeMoUnitTest
 from nemo.core import WeightShareTransform
 from nemo.core.neural_types import *
+from tests.common_setup import NeMoUnitTest
 
 
 class TestWeightSharing(NeMoUnitTest):
@@ -60,7 +63,7 @@ class TestWeightSharing(NeMoUnitTest):
         "z",
         " ",
     ]
-    manifest_filepath = "tests/data/asr/an4_train.json"
+    manifest_filepath = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/asr/an4_train.json"))
     featurizer_config = {
         'window': 'hann',
         'dither': 1e-05,
@@ -75,6 +78,27 @@ class TestWeightSharing(NeMoUnitTest):
     }
     yaml = YAML(typ="safe")
 
+    @classmethod
+    def setUpClass(cls) -> None:
+        super().setUpClass()
+        data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/"))
+        print("Looking up for test ASR data")
+        if not os.path.exists(os.path.join(data_folder, "asr")):
+            print("Extracting ASR data to: {0}".format(os.path.join(data_folder, "asr")))
+            tar = tarfile.open(os.path.join(data_folder, "asr.tar.gz"), "r:gz")
+            tar.extractall(path=data_folder)
+            tar.close()
+        else:
+            print("ASR data found in: {0}".format(os.path.join(data_folder, "asr")))
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        super().tearDownClass()
+        data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/"))
+        print("Looking up for test ASR data")
+        if os.path.exists(os.path.join(data_folder, "asr")):
+            shutil.rmtree(os.path.join(data_folder, "asr"))
+
     def __check_if_weights_are_equal(self, w1: Dict, w2: Dict):
         all_same = set(w1.keys()) == set(w2.keys())
         if not all_same:
@@ -142,7 +166,8 @@ def test_set_weights(self):
         self.assertFalse(np.array_equal(embd.embedding.weight.detach().numpy(), weights.detach().numpy(),))
 
     def test_freeze_unfreeze_TrainableNM(self):
-        with open("tests/data/jasper_smaller.yaml") as file:
+        path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/jasper_smaller.yaml"))
+        with open(path) as file:
             jasper_model_definition = self.yaml.load(file)
         dl = nemo_asr.AudioToTextDataLayer(
             featurizer_config=self.featurizer_config,

tests/test_zeroDS.py → tests/asr/test_zeroDS.py

@@ -17,15 +17,16 @@
 # =============================================================================
 
 import os
+import shutil
 import tarfile
 
 import torch
 from ruamel.yaml import YAML
 
 import nemo
 import nemo.collections.asr as nemo_asr
-from .common_setup import NeMoUnitTest
 from nemo.core.neural_types import *
+from tests.common_setup import NeMoUnitTest
 
 
 class TestZeroDL(NeMoUnitTest):
@@ -59,20 +60,29 @@ class TestZeroDL(NeMoUnitTest):
         "z",
         " ",
     ]
-    manifest_filepath = "tests/data/asr/an4_train.json"
+    manifest_filepath = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/asr/an4_train.json"))
     yaml = YAML(typ="safe")
 
-    def setUp(self) -> None:
-        super().setUp()
-        data_folder = "tests/data/"
+    @classmethod
+    def setUpClass(cls) -> None:
+        super().setUpClass()
+        data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/"))
         print("Looking up for test ASR data")
-        if not os.path.exists(data_folder + "nemo_asr"):
-            print(f"Extracting ASR data to: {data_folder + 'nemo_asr'}")
-            tar = tarfile.open("tests/data/asr.tar.gz", "r:gz")
+        if not os.path.exists(os.path.join(data_folder, "asr")):
+            print("Extracting ASR data to: {0}".format(os.path.join(data_folder, "asr")))
+            tar = tarfile.open(os.path.join(data_folder, "asr.tar.gz"), "r:gz")
             tar.extractall(path=data_folder)
             tar.close()
         else:
-            print("ASR data found in: {0}".format(data_folder + "asr"))
+            print("ASR data found in: {0}".format(os.path.join(data_folder, "asr")))
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        super().tearDownClass()
+        data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/"))
+        print("Looking up for test ASR data")
+        if os.path.exists(os.path.join(data_folder, "asr")):
+            shutil.rmtree(os.path.join(data_folder, "asr"))
 
     def test_simple_train(self):
         print("Simplest train test with ZeroDL")
@@ -103,7 +113,8 @@ def test_simple_train(self):
 
     def test_asr_with_zero_ds(self):
         print("Testing ASR NMs with ZeroDS and without pre-processing")
-        with open("tests/data/jasper_smaller.yaml") as file:
+        path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../data/jasper_smaller.yaml"))
+        with open(path) as file:
             jasper_model_definition = self.yaml.load(file)
 
         dl = nemo.backends.pytorch.common.ZerosDataLayer(

tests/common_setup.py

@@ -24,3 +24,6 @@
 class NeMoUnitTest(unittest.TestCase):
     def setUp(self) -> None:
         nemo.core.neural_factory.NeuralModuleFactory.reset_default_factory()
+        print("---------------------------------------------------------")
+        print(self._testMethodName)
+        print("---------------------------------------------------------")

tests/test_bert.py → tests/nlp/test_bert.py

@@ -17,7 +17,7 @@
 # =============================================================================
 
 import nemo.collections.nlp as nemo_nlp
-from .common_setup import NeMoUnitTest
+from tests.common_setup import NeMoUnitTest
 
 
 class TestBert(NeMoUnitTest):

tests/test_spc_tokenizer.py → tests/nlp/test_spc_tokenizer.py

@@ -16,8 +16,8 @@
 # limitations under the License.
 # =============================================================================
 
-from .common_setup import NeMoUnitTest
 from nemo.collections.nlp import SentencePieceTokenizer
+from tests.common_setup import NeMoUnitTest
 
 
 class TestSPCTokenizer(NeMoUnitTest):

tests/test_squad.py → tests/nlp/test_squad.py

@@ -22,17 +22,17 @@
 
 import nemo
 import nemo.collections.nlp as nemo_nlp
-from .common_setup import NeMoUnitTest
 from nemo.collections.nlp.utils.callbacks.squad import eval_epochs_done_callback, eval_iter_callback
 from nemo.collections.nlp.utils.download_squad import SquadDownloader
 from nemo.utils.lr_policies import get_lr_policy
+from tests.common_setup import NeMoUnitTest
 
 
 class TestSquad(NeMoUnitTest):
     @classmethod
     def setUpClass(cls) -> None:
         super().setUpClass()
-        data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data/nlp'))
+        data_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/nlp'))
         if not os.path.exists(data_folder):
             print(f"mkdir {data_folder}")
             os.mkdir(data_folder)
@@ -66,15 +66,15 @@ def setUpClass(cls) -> None:
     @classmethod
     def tearDownClass(cls) -> None:
         super().tearDownClass()
-        squad_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data/nlp/squad'))
+        squad_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/nlp/squad'))
         if os.path.exists(squad_folder):
             shutil.rmtree(squad_folder)
 
     def test_squad_v1(self):
         version_2_with_negative = False
         pretrained_bert_model = 'bert-base-uncased'
         batch_size = 3
-        data_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data/nlp/squad/v1.1'))
+        data_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/nlp/squad/v1.1'))
         max_query_length = 64
         max_seq_length = 384
         doc_stride = 128
@@ -181,7 +181,7 @@ def test_squad_v2(self):
         version_2_with_negative = True
         pretrained_bert_model = 'bert-base-uncased'
         batch_size = 3
-        data_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data/nlp/squad/v2.0'))
+        data_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/nlp/squad/v2.0'))
         max_query_length = 64
         max_seq_length = 384
         doc_stride = 128

tests/test_actions_api.py

@@ -19,7 +19,7 @@
 import os
 
 import nemo
-from .common_setup import NeMoUnitTest
+from tests.common_setup import NeMoUnitTest
 
 
 class TestTrainers(NeMoUnitTest):

tests/test_deploy_export.py

@@ -25,7 +25,7 @@
 import nemo
 import nemo.collections.asr as nemo_asr
 import nemo.collections.nlp as nemo_nlp
-from .common_setup import NeMoUnitTest
+from tests.common_setup import NeMoUnitTest
 
 
 class TestDeployExport(NeMoUnitTest):