-
Notifications
You must be signed in to change notification settings - Fork 0
/
evaluate.py
110 lines (84 loc) · 3.35 KB
/
evaluate.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
"""
Copyright (c) Facebook, Inc. and its affiliates.
This source code is licensed under the MIT license found in the
LICENSE file in the root directory of this source tree.
"""
import argparse
import pathlib
from argparse import ArgumentParser
import h5py
import numpy as np
from runstats import Statistics
from skimage.measure import compare_psnr, compare_ssim
def mse(gt, pred):
""" Compute Mean Squared Error (MSE) """
return np.mean((gt - pred) ** 2)
def nmse(gt, pred):
""" Compute Normalized Mean Squared Error (NMSE) """
return np.linalg.norm(gt - pred) ** 2 / np.linalg.norm(gt) ** 2
def psnr(gt, pred):
""" Compute Peak Signal to Noise Ratio metric (PSNR) """
return compare_psnr(gt, pred, data_range=gt.max())
def ssim(gt, pred):
""" Compute Structural Similarity Index Metric (SSIM). """
return compare_ssim(
gt.transpose(1, 2, 0), pred.transpose(1, 2, 0), multichannel=True, data_range=gt.max()
)
METRIC_FUNCS = dict(
MSE=mse,
NMSE=nmse,
PSNR=psnr,
SSIM=ssim,
)
class Metrics:
"""
Maintains running statistics for a given collection of metrics.
"""
def __init__(self, metric_funcs):
self.metrics = {
metric: Statistics() for metric in metric_funcs
}
def push(self, target, recons):
for metric, func in METRIC_FUNCS.items():
self.metrics[metric].push(func(target, recons))
def means(self):
return {
metric: stat.mean() for metric, stat in self.metrics.items()
}
def stddevs(self):
return {
metric: stat.stddev() for metric, stat in self.metrics.items()
}
def __repr__(self):
means = self.means()
stddevs = self.stddevs()
metric_names = sorted(list(means))
return ' '.join(
f'{name} = {means[name]:.4g} +/- {2 * stddevs[name]:.4g}' for name in metric_names
)
def evaluate(args, recons_key):
metrics = Metrics(METRIC_FUNCS)
for tgt_file in args.target_path.iterdir():
with h5py.File(tgt_file) as target, h5py.File(
args.predictions_path / tgt_file.name) as recons:
if args.acquisition and args.acquisition != target.attrs['acquisition']:
continue
target = target[recons_key].value
recons = recons['reconstruction'].value
metrics.push(target, recons)
return metrics
if __name__ == '__main__':
parser = ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--target-path', type=pathlib.Path, required=True,
help='Path to the ground truth data')
parser.add_argument('--predictions-path', type=pathlib.Path, required=True,
help='Path to reconstructions')
parser.add_argument('--challenge', choices=['singlecoil', 'multicoil'], required=True,
help='Which challenge')
parser.add_argument('--acquisition', choices=['CORPD_FBK', 'CORPDFS_FBK'], default=None,
help='If set, only volumes of the specified acquisition type are used '
'for evaluation. By default, all volumes are included.')
args = parser.parse_args()
recons_key = 'reconstruction_rss' if args.challenge == 'multicoil' else 'reconstruction_esc'
metrics = evaluate(args, recons_key)
print(metrics)