Merge pull request #388 from blisc/logging_update

Change Final nemo.logging to logging

examples/start_here/module_configuration.py

@@ -20,6 +20,8 @@
 import nemo
 from nemo.core import DeviceType, NeuralModule, NeuralModuleFactory
 
+logging = nemo.logging
+
 # Run on CPU.
 nf = NeuralModuleFactory(placement=DeviceType.CPU)
 
@@ -46,7 +48,7 @@
 
 # SimpleLossLoggerCallback will print loss values to console.
 callback = nemo.core.SimpleLossLoggerCallback(
-    tensors=[loss], print_func=lambda x: nemo.logging.info(f'Train Loss: {str(x[0].item())}')
+    tensors=[loss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}')
 )
 
 # Invoke the "train" action.

examples/start_here/module_custom_configuration.py

@@ -26,6 +26,8 @@
 from nemo.core import DeviceType, NeuralModuleFactory, SimpleLossLoggerCallback
 from nemo.core.neural_types import ChannelType, NeuralType
 
+logging = nemo.logging
+
 
 # A custom enum.
 class Status(Enum):
@@ -38,7 +40,7 @@ class CustomTaylorNet(nemo.tutorials.TaylorNet):
 
     def __init__(self, dim, status: Status):
         super().__init__(dim)
-        nemo.logging.info("Status: {}".format(status))
+        logging.info("Status: {}".format(status))
 
     def export_to_config(self, config_file):
         """
@@ -70,7 +72,7 @@ def export_to_config(self, config_file):
         with open(abs_path_file, 'w') as outfile:
             yaml.dump(to_export, outfile)
 
-        nemo.logging.info(
+        logging.info(
             "Configuration of module {} ({}) exported to {}".format(self._uuid, type(self).__name__, abs_path_file)
         )
 
@@ -109,7 +111,7 @@ def import_from_config(cls, config_file, section_name=None, overwrite_params={})
 
         # Create and return the object.
         obj = CustomTaylorNet(**init_params)
-        nemo.logging.info(
+        logging.info(
             "Instantiated a new Neural Module of type `{}` using configuration loaded from the `{}` file".format(
                 "CustomTaylorNet", config_file
             )
@@ -144,7 +146,7 @@ def import_from_config(cls, config_file, section_name=None, overwrite_params={})
 
 # SimpleLossLoggerCallback will print loss values to console.
 callback = SimpleLossLoggerCallback(
-    tensors=[loss], print_func=lambda x: nemo.logging.info(f'Train Loss: {str(x[0].item())}')
+    tensors=[loss], print_func=lambda x: logging.info(f'Train Loss: {str(x[0].item())}')
 )
 
 # Invoke the "train" action.

nemo/__init__.py

@@ -33,9 +33,7 @@
 )
 
 if "NEMO_PACKAGE_BUILDING" not in os.environ:
-    from nemo.utils.nemo_logging import Logger as _Logger
-
-    logging = _Logger()
+    from nemo.utils import logging
 
     from nemo import backends
     from nemo import core

nemo/collections/asr/helpers.py

@@ -7,8 +7,6 @@
 
 logging = nemo.logging
 
-logging = nemo.logging
-
 
 def __ctc_decoder_predictions_tensor(tensor, labels):
     """

nemo/core/__init__.py

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .callbacks import *
-from .neural_factory import *
-from .neural_modules import *
-from .neural_types import *
+from nemo.core.callbacks import *
+from nemo.core.neural_factory import *
+from nemo.core.neural_modules import *
+from nemo.core.neural_types import *

nemo/core/callbacks.py

@@ -24,7 +24,7 @@
 from collections import namedtuple
 
 import nemo
-from ..utils import get_checkpoint_from_dir
+from nemo.utils import get_checkpoint_from_dir
 
 logging = nemo.logging
 

nemo/utils/__init__.py

@@ -16,6 +16,10 @@
 # limitations under the License.
 # =============================================================================
 
-from .argparse import NemoArgParser
-from .exp_logging import ExpManager, get_logger
-from .helpers import *
+from nemo.utils.nemo_logging import Logger as _Logger
+
+logging = _Logger()
+
+from nemo.utils.argparse import NemoArgParser
+from nemo.utils.exp_logging import ExpManager, get_logger
+from nemo.utils.helpers import *

nemo/utils/decorators/deprecated.py

@@ -20,7 +20,7 @@
 
 import wrapt
 
-import nemo
+from nemo.utils import logging
 
 # logging = nemo.logging
 
@@ -65,8 +65,7 @@ def wrapper(wrapped, instance, args, kwargs):
                 msg = msg + " " + explanation
 
             # Display the deprecated warning.
-            # logging.warning(msg)
-            nemo.logging.warning(msg)
+            logging.warning(msg)
 
         # Call the function.
         return wrapped(*args, **kwargs)

nemo/utils/exp_logging.py

@@ -5,21 +5,20 @@
 import time
 from shutil import copyfile
 
-import nemo
+from nemo.utils import logging
 from nemo.utils.decorators import deprecated
 
-# logging = nemo.logging
-
 
+# logging = nemo.logging
 @deprecated(
     version=0.11,
     explanation=(
         "Please use nemo.logging instead by using logging = nemo.logging and logging.info(), "
         "logging.warning() , etc."
     ),
 )
-def get_logger(*unused):
-    return nemo.logging
+def get_logger(unused):
+    return logging
 
 
 # class ContextFilter(logging.Filter):
@@ -99,7 +98,6 @@ def __init__(
     ):
         self.local_rank = local_rank if local_rank is not None else 0
         self.global_rank = global_rank if global_rank is not None else 0
-        self.logger = None
         self.log_file = None
         self.tb_writer = None
         self.work_dir = None
@@ -151,7 +149,8 @@ def __init__(
                         f.write(get_git_diff())
 
         # Create loggers
-        self.create_logger(log_file=bool(work_dir))
+        if bool(work_dir):
+            self.add_file_handler_to_logger()
         if use_tb and not work_dir:
             raise ValueError("ExpManager received use_tb as True but did not receive a work_dir")
 
@@ -160,25 +159,9 @@ def __init__(
         if self.ckpt_dir:
             self.make_dir(self.ckpt_dir, exist_ok)
 
-    def create_logger(self, log_file=True):
-        logger = nemo.logging
-        # tmp = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-
-        # if self.global_rank == 0:
-        #     logger.setLevel(level)
-        #     ch = logging.StreamHandler()
-        #     ch.setLevel(level)
-        #     ch.setFormatter(tmp)
-        #     logger.addHandler(ch)
-
-        if log_file:
-            self.log_file = f'{self.work_dir}/log_globalrank-{self.global_rank}_' f'localrank-{self.local_rank}.txt'
-            logger.add_file_handler(self.log_file)
-            # fh = logging.FileHandler(self.log_file)
-            # fh.setLevel(level)
-            # fh.setFormatter(tmp)
-        self.logger = logger
-        return logger
+    def add_file_handler_to_logger(self):
+        self.log_file = f'{self.work_dir}/log_globalrank-{self.global_rank}_' f'localrank-{self.local_rank}.txt'
+        logging.add_file_handler(self.log_file)
 
     def make_dir(self, dir_, exist_ok):
         # We might want to limit folder creation to only global_rank 0
@@ -199,21 +182,16 @@ def get_tb_writer(self, tb_dir=None, exist_ok=True):
                 self.tb_writer = SummaryWriter(self.tb_dir)
             except ImportError:
                 self.tb_writer = None
-                # logging.info('Not using TensorBoard.')
-                # logging.info('Install tensorboardX to use TensorBoard')
-                nemo.logging.info('Not using TensorBoard.')
-                nemo.logging.info('Install tensorboardX to use TensorBoard')
+                logging.info('Not using TensorBoard.')
+                logging.info('Install tensorboardX to use TensorBoard')
         return self.tb_writer
 
     def log_exp_info(self, params, print_everywhere=False):
         if print_everywhere or self.global_rank == 0:
-            # logging.info("NEMO MODEL'S PARAMETERS")
-            nemo.logging.info("NEMO MODEL'S PARAMETERS")
+            logging.info("NEMO MODEL'S PARAMETERS")
             for key in params:
-                # logging.info(f'{key}\t{params[key]}')
-                nemo.logging.info(f'{key}\t{params[key]}')
-            # logging.info(f'Experiment output is stored in {self.work_dir}')
-            nemo.logging.info(f'Experiment output is stored in {self.work_dir}')
+                logging.info(f'{key}\t{params[key]}')
+            logging.info(f'Experiment output is stored in {self.work_dir}')
 
 
 def get_git_hash():

nemo/utils/helpers.py

@@ -11,6 +11,7 @@
 import wget
 
 import nemo
+from nemo.utils import logging
 
 # logging = nemo.logging
 
@@ -160,6 +161,5 @@ def maybe_download_from_cloud(url, filename) -> str:
         else:
             return ""
     except (FileNotFoundError, ConnectionError, OSError):
-        # logging.info(f"Could not obtain {filename} from the cloud")
-        nemo.logging.info(f"Could not obtain {filename} from the cloud")
+        logging.info(f"Could not obtain {filename} from the cloud")
         return ""

setup.cfg

@@ -5,4 +5,4 @@ test = pytest
 known_localfolder = nemo,tests
 sections = FUTURE,STDLIB,THIRDPARTY,LOCALFOLDER
 default_section = THIRDPARTY
-skip = setup.py
+skip = setup.py, nemo/utils/__init__.py