separate attack time

exp/attack.py

@@ -1,4 +1,5 @@
 import sys
+import time
 
 import numpy as np
 # noinspection PyPackageRequirements
@@ -47,6 +48,7 @@ def __init__(self, kind: str, constr: bool, conf):
         self.adv_y = None
         self.score = None
         self.conf = conf or {}
+        self.start = self.end = 0
 
     def reset(self, cls):
         self.cls = cls
@@ -55,6 +57,7 @@ def reset(self, cls):
         self.ori_x = cls.test_x.copy()
         self.adv_x = None
         self.adv_y = None
+        self.start = self.end = 0
         return self
 
     @property
@@ -63,6 +66,7 @@ def can_validate(self):
 
     def run(self, v_model: Validation):
         """Generate adversarial examples and score."""
+        self.start = time.time_ns()
         if issubclass(self.attack, CPGD):
             self.adv_x, self.adv_y = cpgd_apply_and_predict(
                 self.cls.model, self.ori_x, self.ori_y, **self.conf)
@@ -73,13 +77,18 @@ def run(self, v_model: Validation):
             self.adv_x = aml_attack.generate(x=self.ori_x)
             self.adv_y = np.array(self.cls.predict(
                 self.adv_x, self.ori_y).flatten())
+        self.end = time.time_ns()
         sys.stdout.write('\x1b[1A')
         sys.stdout.write('\x1b[2K')
 
         self.score.calculate(
-            self, v_model.constraints, v_model.scalars)
+            self, v_model.constraints, v_model.scalars,
+            dur=self.end - self.start)
         return self
 
     def to_dict(self):
-        return {'name': self.name, 'config': self.conf,
-                'can_validate': self.can_validate}
+        return {
+            'name': self.name,
+            'config': self.conf,
+            'can_validate': self.can_validate
+        }

exp/scoring.py

@@ -2,7 +2,7 @@
 import sklearn.metrics as skm
 
 from exp import CONSTR_DICT as CD, Validation
-from exp.utility import sdiv, log, logr, logd, attr_of
+from exp.utility import sdiv, log, logr, logd, attr_of, dur_sec
 
 
 def score_valid(ori: np.ndarray, adv: np.ndarray, cd: CD, scalars):
@@ -41,7 +41,8 @@ def calculate(self, true_labels, predictions):
 
     def log(self):
         for a in attr_of(self, (int, float)):
-            log(a.capitalize(), f'{getattr(self, a) * 100:.2f} %')
+            log(a.capitalize().replace('_', '-'),
+                f'{getattr(self, a) * 100:.2f} %')
 
 
 class AttackScore:
@@ -53,8 +54,9 @@ def __init__(self):
         self.n_records = 0
         self.n_valid = 0
         self.n_valid_evades = 0
+        self.dur = 0
 
-    def calculate(self, attack, constraints, attr_range):
+    def calculate(self, attack, constraints, attr_range, dur):
         ori_x, ori_y = attack.ori_x, attack.ori_y
         adv_x, adv_y = attack.adv_x, attack.adv_y
         original = attack.cls.predict(ori_x, ori_y)
@@ -67,11 +69,13 @@ def calculate(self, attack, constraints, attr_range):
         self.n_evasions = len(self.evasions)
         self.n_valid_evades = len(self.valid_evades)
         self.n_records = ori_x.shape[0]
+        self.dur = dur
 
     def log(self):
         logr('Evasions', self.n_evasions, self.n_records)
         logr('Valid', self.n_valid, self.n_records)
         logr('Valid+Evades', self.n_valid_evades, self.n_records)
+        log('Attack Duration', f'{dur_sec(self.dur) :.2f} s')
 
 
 class Result:
@@ -93,6 +97,7 @@ def __init__(self):
         self.n_evasions = Result.AvgList()
         self.n_valid = Result.AvgList()
         self.n_valid_evades = Result.AvgList()
+        self.dur = Result.AvgList()
 
     def append(self, obj):
         for a in attr_of(obj, (int, float)):
@@ -113,3 +118,4 @@ def log(self):
         logd('Valid', self.n_valid.avg, self.n_records.avg)
         logd('Valid Evasions',
              self.n_valid_evades.avg, self.n_records.avg)
+        log('Attack Duration', f'{dur_sec(self.dur.avg) :.2f} s')

exp/utility.py

@@ -78,7 +78,7 @@ def log(label: str, value):
 
 def logr(label: str, n: float, d: float):
     a, b, r = round(n, 0), round(d, 0), sdiv(n, d)
-    r_ftm = "{r:.2f} %" if str(r).isnumeric() else r
+    r_ftm = f"{r:.2f} %" if str(r).isnumeric() else r
     return log(label, f'{a} of {b} - {r_ftm}')
 
 
@@ -89,3 +89,8 @@ def logd(label: str, n: float, d: float):
 def time_sec(start: time, end: time) -> int:
     """Time difference in seconds."""
     return round((end - start) / 1e9, 1)
+
+
+def dur_sec(dur: time) -> int:
+    """Time difference in seconds."""
+    return round(dur / 1e9, 1)