[FEATURE] Add length normalized metrics for machine translation tasks

src/gluonnlp/metric/__init__.py

@@ -19,7 +19,9 @@
 """NLP Metrics."""
 
 from . import masked_accuracy
+from . import length_normalized_loss
 
 from .masked_accuracy import *
+from .length_normalized_loss import *
 
-__all__ = masked_accuracy.__all__
+__all__ = masked_accuracy.__all__ + length_normalized_loss.__all__

src/gluonnlp/metric/length_normalized_loss.py

@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+""" Length Normalized Loss """
+
+from mxnet import ndarray
+from mxnet.metric import EvalMetric
+
+__all__ = ['LengthNormalizedLoss']
+
+class LengthNormalizedLoss(EvalMetric):
+    """Compute length normalized loss metrics
+
+    Parameters
+    ----------
+    axis : int, default=1
+        The axis that represents classes
+    name : str
+        Name of this metric instance for display.
+    output_names : list of str, or None
+        Name of predictions that should be used when updating with update_dict.
+        By default include all predictions.
+    label_names : list of str, or None
+        Name of labels that should be used when updating with update_dict.
+        By default include all labels.
+    """
+    def __init__(self, axis=0, name='length-normalized-loss',
+                 output_names=None, label_names=None):
+        super(LengthNormalizedLoss, self).__init__(
+            name, axis=axis,
+            output_names=output_names, label_names=label_names,
+            has_global_stats=True)
+
+    def update(self, labels, preds):
+        """Update the length normalized metrics with target label and loss
+
+        Update the sum_metrics and sum_insts of metrics with provided arguments.
+
+        Parameters:
+        ----------
+        labels: list or tuple
+            It contains two elements. The first element is the target sentence and
+            the second element is the valid length of target sentence
+        preds: list or ndarray.ndarray.NDArray
+            a list of ndarray.ndarray.NDArray or scalar or a single
+            ndarray.ndarray.NDArray. It is usually the loss predicted by the model
+        """
+        typecheck = not isinstance(labels, list) and not isinstance(labels, tuple)
+        if typecheck or len(labels) != 2:
+            raise ValueError('labels must be a list. Its first element should be'
+                             ' target sequence and the second element should be'
+                             'the valid length of sequence.')
+
+        _, seq_valid_length = labels
+
+        if not isinstance(seq_valid_length, list):
+            seq_valid_length = [seq_valid_length]
+
+        if not isinstance(preds, list):
+            preds = [preds]
+
+        for length in seq_valid_length:
+            if isinstance(length, ndarray.ndarray.NDArray):
+                total_length = ndarray.sum(length).asscalar()
+            else:
+                total_length = length
+            self.num_inst += total_length
+            self.global_num_inst += total_length
+
+        for pred in preds:
+            if isinstance(pred, ndarray.ndarray.NDArray):
+                loss = ndarray.sum(pred).asscalar()
+            else:
+                loss = pred
+            self.sum_metric += loss
+            self.global_sum_metric += loss

src/gluonnlp/metric/masked_accuracy.py

@@ -18,168 +18,9 @@
 
 from mxnet import ndarray
 from mxnet.metric import check_label_shapes
+from mxnet.metric import EvalMetric
 
-__all__ = ['EvalMetric', 'MaskedAccuracy']
-
-class EvalMetric:
-    """Base class for all evaluation metrics.
-
-    .. note::
-
-        This is a base class that provides common metric interfaces.
-        One should not use this class directly, but instead create new metric
-        classes that extend it.
-
-    Parameters
-    ----------
-    name : str
-        Name of this metric instance for display.
-    output_names : list of str, or None
-        Name of predictions that should be used when updating with update_dict.
-        By default include all predictions.
-    label_names : list of str, or None
-        Name of labels that should be used when updating with update_dict.
-        By default include all labels.
-    """
-    def __init__(self, name, output_names=None,
-                 label_names=None, **kwargs):
-        self.name = str(name)
-        self.output_names = output_names
-        self.label_names = label_names
-        self._has_global_stats = kwargs.pop('has_global_stats', False)
-        self._kwargs = kwargs
-        self.reset()
-
-    def __str__(self):
-        return 'EvalMetric: {}'.format(dict(self.get_name_value()))
-
-    def get_config(self):
-        """Save configurations of metric. Can be recreated
-        from configs with metric.create(``**config``)
-        """
-        config = self._kwargs.copy()
-        config.update({
-            'metric': self.__class__.__name__,
-            'name': self.name,
-            'output_names': self.output_names,
-            'label_names': self.label_names})
-        return config
-
-    def update_dict(self, label, pred):
-        """Update the internal evaluation with named label and pred
-
-        Parameters
-        ----------
-        labels : OrderedDict of str -> NDArray
-            name to array mapping for labels.
-
-        preds : OrderedDict of str -> NDArray
-            name to array mapping of predicted outputs.
-        """
-        if self.output_names is not None:
-            pred = [pred[name] for name in self.output_names]
-        else:
-            pred = list(pred.values())
-
-        if self.label_names is not None:
-            label = [label[name] for name in self.label_names]
-        else:
-            label = list(label.values())
-
-        self.update(label, pred)
-
-    def update(self, labels, preds):
-        """Updates the internal evaluation result.
-
-        Parameters
-        ----------
-        labels : list of `NDArray`
-            The labels of the data.
-
-        preds : list of `NDArray`
-            Predicted values.
-        """
-        raise NotImplementedError()
-
-    def reset(self):
-        """Resets the internal evaluation result to initial state."""
-        self.num_inst = 0
-        self.sum_metric = 0.0
-        self.global_num_inst = 0
-        self.global_sum_metric = 0.0
-
-    def reset_local(self):
-        """Resets the local portion of the internal evaluation results
-        to initial state."""
-        self.num_inst = 0
-        self.sum_metric = 0.0
-
-    def get(self):
-        """Gets the current evaluation result.
-
-        Returns
-        -------
-        names : list of str
-           Name of the metrics.
-        values : list of float
-           Value of the evaluations.
-        """
-        if self.num_inst == 0:
-            return (self.name, float('nan'))
-        else:
-            return (self.name, self.sum_metric / self.num_inst)
-
-    def get_global(self):
-        """Gets the current global evaluation result.
-
-        Returns
-        -------
-        names : list of str
-           Name of the metrics.
-        values : list of float
-           Value of the evaluations.
-        """
-        if self._has_global_stats:
-            if self.global_num_inst == 0:
-                return (self.name, float('nan'))
-            else:
-                return (self.name, self.global_sum_metric / self.global_num_inst)
-        else:
-            return self.get()
-
-    def get_name_value(self):
-        """Returns zipped name and value pairs.
-
-        Returns
-        -------
-        list of tuples
-            A (name, value) tuple list.
-        """
-        name, value = self.get()
-        if not isinstance(name, list):
-            name = [name]
-        if not isinstance(value, list):
-            value = [value]
-        return list(zip(name, value))
-
-    def get_global_name_value(self):
-        """Returns zipped name and value pairs for global results.
-
-        Returns
-        -------
-        list of tuples
-            A (name, value) tuple list.
-        """
-        if self._has_global_stats:
-            name, value = self.get_global()
-            if not isinstance(name, list):
-                name = [name]
-            if not isinstance(value, list):
-                value = [value]
-            return list(zip(name, value))
-        else:
-            return self.get_name_value()
-
+__all__ = ['MaskedAccuracy']
 
 class MaskedAccuracy(EvalMetric):
     """Computes accuracy classification score.

tests/unittest/test_metrics.py

@@ -17,7 +17,8 @@
 
 import mxnet as mx
 import numpy as np
-from gluonnlp.metric import MaskedAccuracy
+from gluonnlp.metric import MaskedAccuracy, LengthNormalizedLoss
+from mxnet.test_utils import assert_almost_equal
 
 def test_acc():
     pred = mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])
@@ -38,3 +39,21 @@ def test_acc():
     valid_count = len(label)
     expected_acc = 1.0 * matched.sum() / valid_count
     assert acc == expected_acc
+
+def test_normalized_loss(rtol=1e-5, atol=1e-5):
+    tgt_valid_length = mx.nd.array([1, 3, 2, 7])
+    loss = mx.nd.array([1.1, 2.5, 3.8, 5.3])
+    metric = LengthNormalizedLoss()
+    metric.update([0, tgt_valid_length], loss)
+    _, metric_loss = metric.get()
+    expected_loss = loss.asnumpy().sum() / tgt_valid_length.asnumpy().sum()
+    assert_almost_equal(metric_loss, expected_loss, rtol=rtol, atol=atol)
+
+    tgt_valid_length = mx.nd.array([8, 4, 2, 7])
+    loss = mx.nd.array([8.7, 2.3, 1.8, 9.3])
+    metric = LengthNormalizedLoss()
+    metric.update([0, tgt_valid_length], loss)
+    _, metric_loss = metric.get()
+    expected_loss = loss.asnumpy().sum() / tgt_valid_length.asnumpy().sum()
+    assert_almost_equal(metric_loss, expected_loss, rtol=rtol, atol=atol)
+