add epe and auc (open-mmlab#86)

configs/top_down/resnet/README.md

@@ -63,3 +63,9 @@ Following the common setting, the models are trained on COCO train dataset, and
 | [pose_resnet_50](/configs/top_down/resnet/mpii_trb/res50_mpii_trb_256x256.py)  | 256x256 | 0.887 | 0.858 | 0.868 | [ckpt](https://openmmlab.oss-accelerate.aliyuncs.com/mmpose/top_down/resnet/res50_mpii_trb_256x256-896036b8_20200812.pth) | [log](https://openmmlab.oss-accelerate.aliyuncs.com/mmpose/top_down/resnet/res50_mpii_trb_256x256_20200812.log.json) |
 | [pose_resnet_101](/configs/top_down/resnet/mpii_trb/res101_mpii_trb_256x256.py)  | 256x256 | 0.890 | 0.863 | 0.873 | [ckpt](https://openmmlab.oss-accelerate.aliyuncs.com/mmpose/top_down/resnet/res101_mpii_trb_256x256-cfad2f05_20200812.pth) | [log](https://openmmlab.oss-accelerate.aliyuncs.com/mmpose/top_down/resnet/res101_mpii_trb_256x256_20200812.log.json) |
 | [pose_resnet_152](/configs/top_down/resnet/mpii_trb/res152_mpii_trb_256x256.py)  | 256x256 | 0.897 | 0.868 | 0.879 | [ckpt](https://openmmlab.oss-accelerate.aliyuncs.com/mmpose/top_down/resnet/res152_mpii_trb_256x256-dd369ce6_20200812.pth) | [log](https://openmmlab.oss-accelerate.aliyuncs.com/mmpose/top_down/resnet/res152_mpii_trb_256x256_20200812.log.json) |
+
+### Results on OneHand10K val set.
+
+| Arch  | Input Size | PCK@0.2 |  AUC  |  EPE  | ckpt    | log     |
+| :--- | :--------: | :------: | :------: | :------: |:------: |:------: |
+| [pose_resnet_50](/configs/top_down/resnet/onehand10k/res50_onehand10k_256x256.py) | 256x256 | 0.985 | 0.536 | 27.3 | [ckpt](https://openmmlab.oss-accelerate.aliyuncs.com/mmpose/top_down/resnet/res50_onehand10k_256x256-e67998f6_20200813.pth) | [log](https://openmmlab.oss-accelerate.aliyuncs.com/mmpose/top_down/resnet/res50_onehand10k_256x256_20200813.log.json) |

configs/top_down/resnet/onehand10k/res50_onehand10k_256x256.py

@@ -4,7 +4,7 @@
 dist_params = dict(backend='nccl')
 workflow = [('train', 1)]
 checkpoint_config = dict(interval=10)
-evaluation = dict(interval=1, metric='PCK')
+evaluation = dict(interval=1, metric=['PCK', 'AUC', 'EPE'])
 
 optimizer = dict(
     type='Adam',

mmpose/core/evaluation/__init__.py

@@ -1,9 +1,11 @@
 from .bottom_up_eval import (aggregate_results, get_group_preds,
                              get_multi_stage_outputs)
 from .eval_hooks import DistEvalHook, EvalHook
-from .top_down_eval import keypoints_from_heatmaps, pose_pck_accuracy
+from .top_down_eval import (keypoint_auc, keypoint_epe, keypoint_pck_accuracy,
+                            keypoints_from_heatmaps, pose_pck_accuracy)
 
 __all__ = [
     'EvalHook', 'DistEvalHook', 'pose_pck_accuracy', 'keypoints_from_heatmaps',
-    'get_group_preds', 'get_multi_stage_outputs', 'aggregate_results'
+    'keypoint_pck_accuracy', 'keypoint_auc', 'keypoint_epe', 'get_group_preds',
+    'get_multi_stage_outputs', 'aggregate_results'
 ]

mmpose/core/evaluation/top_down_eval.py

@@ -23,7 +23,7 @@ def _calc_distances(preds, targets, normalize):
     N, K, _ = preds.shape
     distances = np.full((K, N), -1, dtype=np.float32)
     eps = np.finfo(np.float32).eps
-    mask = (targets[..., 0] > eps) & (targets[..., 1] > eps)
+    mask = (targets[..., 0] > eps) | (targets[..., 1] > eps)
     distances[mask.T] = np.linalg.norm(
         ((preds - targets) / normalize[:, None, :])[mask], axis=-1)
     return distances
@@ -86,7 +86,7 @@ def _get_max_preds(heatmaps):
 
 def pose_pck_accuracy(output, target, thr=0.5, normalize=None):
     """Calculate the pose accuracy of PCK for each individual keypoint and the
-    averaged accuracy across all keypoints.
+    averaged accuracy across all keypoints from heatmaps.
 
     Note:
         The PCK performance metric is the percentage of joints with
@@ -117,6 +117,28 @@ def pose_pck_accuracy(output, target, thr=0.5, normalize=None):
 
     pred, _ = _get_max_preds(output)
     gt, _ = _get_max_preds(target)
+    return keypoint_pck_accuracy(pred, gt, thr, normalize)
+
+
+def keypoint_pck_accuracy(pred, gt, thr, normalize):
+    """Calculate the pose accuracy of PCK for each individual keypoint and the
+    averaged accuracy across all keypoints for coordinates.
+
+    Note:
+        batch_size: N
+        num_keypoints: K
+
+    Args:
+        pred (np.ndarray[N, K, 2]): Predicted keypoint location.
+        gt (np.ndarray[N, K, 2]): Groundtruth keypoint location.
+        thr (float): Threshold of PCK calculation.
+        normalize (np.ndarray[N, 2]): Normalization factor.
+
+    Returns:
+        np.ndarray[K]: Accuracy of each keypoint.
+        float: Averaged accuracy across all keypoints.
+        int: Number of valid keypoints.
+    """
     distances = _calc_distances(pred, gt, normalize)
 
     acc = np.array([_distance_acc(d, thr) for d in distances])
@@ -126,6 +148,56 @@ def pose_pck_accuracy(output, target, thr=0.5, normalize=None):
     return acc, avg_acc, cnt
 
 
+def keypoint_auc(pred, gt, normalize, num_step=20):
+    """Calculate the pose accuracy of PCK for each individual keypoint and the
+    averaged accuracy across all keypoints for coordinates.
+
+    Note:
+        batch_size: N
+        num_keypoints: K
+
+    Args:
+        pred (np.ndarray[N, K, 2]): Predicted keypoint location.
+        gt (np.ndarray[N, K, 2]): Groundtruth keypoint location.
+        normalize (float): Normalization factor.
+
+    Returns:
+        float: Area under curve.
+    """
+    nor = np.tile(np.array([[normalize, normalize]]), (pred.shape[0], 1))
+    x = [1.0 * i / num_step for i in range(num_step)]
+    y = []
+    for thr in x:
+        _, avg_acc, _ = keypoint_pck_accuracy(pred, gt, thr, nor)
+        y.append(avg_acc)
+
+    auc = 0
+    for i in range(num_step):
+        auc += 1.0 / num_step * y[i]
+    return auc
+
+
+def keypoint_epe(pred, gt):
+    """Calculate the end-point error.
+
+    Note:
+        batch_size: N
+        num_keypoints: K
+
+    Args:
+        pred (np.ndarray[N, K, 2]): Predicted keypoint location.
+        gt (np.ndarray[N, K, 2]): Groundtruth keypoint location.
+
+    Returns:
+        float: Average end-point error.
+    """
+    distances = _calc_distances(
+        pred, gt, np.tile(np.array([[1, 1]]), (pred.shape[0], 1)))
+    distance_valid = distances[distances != -1]
+    valid_num = len(distance_valid)
+    return distance_valid.sum() / valid_num
+
+
 def _taylor(heatmap, coord):
     """Distribution aware coordinate decoding method.
 

mmpose/datasets/datasets/top_down/topdown_onehand10k_dataset.py

@@ -6,6 +6,8 @@
 import json_tricks as json
 import numpy as np
 
+from mmpose.core.evaluation.top_down_eval import (keypoint_auc, keypoint_epe,
+                                                  keypoint_pck_accuracy)
 from mmpose.datasets.builder import DATASETS
 from .topdown_base_dataset import TopDownBaseDataset
 
@@ -157,32 +159,20 @@ def _xywh2cs(self, x, y, w, h):
 
         return center, scale
 
-    def _evaluate_kernel(self, pred, joints_3d, joints_3d_visible, bbox):
-        """Evaluate one example.
+    def evaluate(self, outputs, res_folder, metric='PCK', **kwargs):
+        """Evaluate OneHand10K keypoint results. metric (str | list[str]):
+        Metrics to be evaluated. Options are 'PCK', 'AUC', 'EPE'.
 
-        ||pre[i] - joints_3d[i]|| < 0.2 * max(w, h)
+        'PCK': ||pre[i] - joints_3d[i]|| < 0.2 * max(w, h)
+        'AUC': area under curve
+        'EPE': end-point error
         """
-        num_joints = self.ann_info['num_joints']
-        bbox = np.array(bbox)
-        threshold = np.max(bbox[2:]) * 0.2
-        hit = np.zeros(num_joints, dtype=np.float32)
-        exist = np.zeros(num_joints, dtype=np.float32)
-
-        for i in range(num_joints):
-            pred_pt = pred[i]
-            gt_pt = joints_3d[i]
-            vis = joints_3d_visible[i][0]
-            if vis:
-                exist[i] = 1
-            else:
-                continue
-            distance = np.linalg.norm(pred_pt[:2] - gt_pt[:2])
-            if distance < threshold:
-                hit[i] = 1
-        return hit, exist
+        metrics = metric if isinstance(metric, list) else [metric]
+        allowed_metrics = ['PCK', 'AUC', 'EPE']
+        for metric in metrics:
+            if metric not in allowed_metrics:
+                raise KeyError(f'metric {metric} is not supported')
 
-    def evaluate(self, outputs, res_folder, metric='PCK', **kwargs):
-        """Evaluate OneHand10K keypoint results."""
         res_file = os.path.join(res_folder, 'result_keypoints.json')
 
         kpts = []
@@ -201,7 +191,7 @@ def evaluate(self, outputs, res_folder, metric='PCK', **kwargs):
             })
 
         self._write_keypoint_results(kpts, res_file)
-        info_str = self._report_metric(res_file)
+        info_str = self._report_metric(res_file, metrics)
         name_value = OrderedDict(info_str)
 
         return name_value
@@ -212,27 +202,46 @@ def _write_keypoint_results(self, keypoints, res_file):
         with open(res_file, 'w') as f:
             json.dump(keypoints, f, sort_keys=True, indent=4)
 
-    def _report_metric(self, res_file):
+    def _report_metric(self, res_file, metrics):
         """Keypoint evaluation.
 
-        Report Mean Acc of skeleton, contour and all joints.
+        Report PCK, AUC or EPE.
         """
-        num_joints = self.ann_info['num_joints']
-        hit = np.zeros(num_joints, dtype=np.float32)
-        exist = np.zeros(num_joints, dtype=np.float32)
+        info_str = []
 
         with open(res_file, 'r') as fin:
             preds = json.load(fin)
-
         assert len(preds) == len(self.db)
-        for pred, item in zip(preds, self.db):
-            h, e = self._evaluate_kernel(pred['keypoints'], item['joints_3d'],
-                                         item['joints_3d_visible'],
-                                         item['bbox'])
-            hit += h
-            exist += e
-        pck = np.sum(hit) / np.sum(exist)
 
-        info_str = []
-        info_str.append(('PCK', pck.item()))
+        outputs = []
+        gts = []
+        for pred, item in zip(preds, self.db):
+            outputs.append(pred['keypoints'])
+            gts.append(item['joints_3d'])
+        outputs = np.array(outputs)[:, :, :-1]
+        gts = np.array(gts)[:, :, :-1]
+
+        if 'PCK' in metrics:
+            hit = 0
+            exist = 0
+
+            for pred, item in zip(preds, self.db):
+                bbox = np.array(item['bbox'])
+                threshold = np.max(bbox[2:]) * 0.2
+                h, _, e = keypoint_pck_accuracy(
+                    np.array(pred['keypoints'])[None, :, :-1],
+                    np.array(item['joints_3d'])[None, :, :-1], 1,
+                    np.array([[threshold, threshold]]))
+                hit += len(h[h > 0])
+                exist += e
+            pck = hit / exist
+
+            info_str.append(('PCK', pck))
+
+        if 'AUC' in metrics:
+            info_str.append(('AUC', keypoint_auc(outputs, gts, 30)))
+
+        if 'EPE' in metrics:
+
+            info_str.append(('EPE', keypoint_epe(outputs, gts)))
         return info_str

tests/test_evaluation/test_top_down_eval.py

@@ -2,7 +2,8 @@
 import pytest
 from numpy.testing import assert_array_almost_equal
 
-from mmpose.core import keypoints_from_heatmaps, pose_pck_accuracy
+from mmpose.core import (keypoint_auc, keypoint_epe, keypoint_pck_accuracy,
+                         keypoints_from_heatmaps, pose_pck_accuracy)
 
 
 def test_pose_pck_accuracy():
@@ -47,3 +48,76 @@ def test_keypoints_from_heatmaps():
     assert_array_almost_equal(maxvals, np.array([[[2]]]), decimal=4)
     assert isinstance(preds, np.ndarray)
     assert isinstance(maxvals, np.ndarray)
+
+
+def test_keypoint_pck_accuracy():
+    output = np.zeros((1, 5, 2))
+    target = np.zeros((1, 5, 2))
+    thr = np.full((1, 2), 10, dtype=np.float32)
+    # first channnel
+    output[0, 0] = [10, 0]
+    target[0, 0] = [10, 0]
+    # second channel
+    output[0, 1] = [20, 20]
+    target[0, 1] = [10, 10]
+    # third channel
+    output[0, 2] = [0, 0]
+    target[0, 2] = [0, 0]
+    # fourth channel
+    output[0, 3] = [30, 30]
+    target[0, 3] = [30, 30]
+    # fifth channnel
+    output[0, 4] = [0, 10]
+    target[0, 4] = [0, 10]
+
+    acc, avg_acc, cnt = keypoint_pck_accuracy(output, target, 0.5, thr)
+
+    assert_array_almost_equal(acc, np.array([1, 0, -1, 1, 1]), decimal=4)
+    assert abs(avg_acc - 0.75) < 1e-4
+    assert abs(cnt - 4) < 1e-4
+
+
+def test_keypoint_auc():
+    output = np.zeros((1, 5, 2))
+    target = np.zeros((1, 5, 2))
+    # first channnel
+    output[0, 0] = [10, 4]
+    target[0, 0] = [10, 0]
+    # second channel
+    output[0, 1] = [10, 18]
+    target[0, 1] = [10, 10]
+    # third channel
+    output[0, 2] = [0, 0]
+    target[0, 2] = [0, 0]
+    # fourth channel
+    output[0, 3] = [40, 40]
+    target[0, 3] = [30, 30]
+    # fifth channnel
+    output[0, 4] = [20, 10]
+    target[0, 4] = [0, 10]
+
+    auc = keypoint_auc(output, target, 20, 4)
+    assert abs(auc - 0.375) < 1e-4
+
+
+def test_keypoint_epe():
+    output = np.zeros((1, 5, 2))
+    target = np.zeros((1, 5, 2))
+    # first channnel
+    output[0, 0] = [10, 4]
+    target[0, 0] = [10, 0]
+    # second channel
+    output[0, 1] = [10, 18]
+    target[0, 1] = [10, 10]
+    # third channel
+    output[0, 2] = [0, 0]
+    target[0, 2] = [0, 0]
+    # fourth channel
+    output[0, 3] = [40, 40]
+    target[0, 3] = [30, 30]
+    # fifth channnel
+    output[0, 4] = [20, 10]
+    target[0, 4] = [0, 10]
+
+    epe = keypoint_epe(output, target)
+    assert abs(epe - 11.5355339) < 1e-4