Update visualization

contrib/experiments/interpretation/dutchf3_voxel/configs/texture_net.yaml

@@ -29,7 +29,7 @@ TRAIN:
   LR: 0.02
   MOMENTUM: 0.9
   WEIGHT_DECAY: 0.0001    
-  DEPTH: "voxel" # Options are No, Patch, Section and Voxel  
+  DEPTH: "voxel" # Options are none, patch, section and voxel  
   MODEL_DIR: "models"
 
 VALIDATION:

contrib/experiments/interpretation/dutchf3_voxel/default.py

@@ -24,6 +24,8 @@
 _C.PRINT_FREQ = 20
 _C.LOG_CONFIG = "logging.conf"
 _C.SEED = 42
+_C.OPENCV_BORDER_CONSTANT = 0
+
 # size of voxel cube: WINDOW_SIZE x WINDOW_SIZE x WINDOW_SIZE; used for 3D models only
 _C.WINDOW_SIZE = 65
 
@@ -50,7 +52,7 @@
 _C.TRAIN.LR = 0.01
 _C.TRAIN.MOMENTUM = 0.9
 _C.TRAIN.WEIGHT_DECAY = 0.0001
-_C.TRAIN.DEPTH = "voxel"  # Options are None, Patch and Section
+_C.TRAIN.DEPTH = "voxel"  # Options are none, patch and section
 _C.TRAIN.MODEL_DIR = "models"  # This will be a subdirectory inside OUTPUT_DIR
 
 # validation

contrib/experiments/interpretation/penobscot/local/configs/hrnet.yaml

@@ -9,6 +9,7 @@ WORKERS: 4
 PRINT_FREQ: 10
 LOG_CONFIG: logging.conf
 SEED: 2019
+OPENCV_BORDER_CONSTANT: 0
 
 
 DATASET:
@@ -75,7 +76,7 @@ TRAIN:
   WEIGHT_DECAY: 0.0001
   SNAPSHOTS: 5
   AUGMENTATION: True
-  DEPTH: "patch" # Options are none, patch and section
+  DEPTH: "patch" # Options are none, patch, and section
   STRIDE: 64
   PATCH_SIZE: 128
   AUGMENTATIONS:

contrib/experiments/interpretation/penobscot/local/configs/seresnet_unet.yaml

@@ -32,7 +32,7 @@ TRAIN:
   WEIGHT_DECAY: 0.0001
   SNAPSHOTS: 5
   AUGMENTATION: True
-  DEPTH: "patch" # Options are none, patch and section
+  DEPTH: "patch" # Options are none, patch, and section
   STRIDE: 64
   PATCH_SIZE: 128
   AUGMENTATIONS:

contrib/experiments/interpretation/penobscot/local/default.py

@@ -21,6 +21,7 @@
 _C.PIN_MEMORY = True
 _C.LOG_CONFIG = "logging.conf"
 _C.SEED = 42
+_C.OPENCV_BORDER_CONSTANT = 0
 
 # size of voxel cube: WINDOW_SIZE x WINDOW_SIZE x WINDOW_SIZE; used for 3D models only
 _C.WINDOW_SIZE = 65
@@ -72,7 +73,7 @@
 _C.TRAIN.MEAN = [-0.0001777, 0.49, -0.0000688]  # 0.0009996710808862074
 _C.TRAIN.STD = [0.14076, 0.2717, 0.06286]  # 0.20976548783479299
 _C.TRAIN.MAX = 1
-_C.TRAIN.DEPTH = "patch"  # Options are none, patch and section
+_C.TRAIN.DEPTH = "patch"  # Options are none, patch, and section
 # None adds no depth information and the num of channels remains at 1
 # Patch adds depth per patch so is simply the height of that patch from 0 to 1, channels=3
 # Section adds depth per section so contains depth information for the whole section, channels=3

contrib/experiments/interpretation/penobscot/local/test.py

@@ -18,45 +18,30 @@
 from itertools import chain
 from os import path
 
-import cv2
 import fire
 import numpy as np
 import torch
 import torchvision
 from albumentations import Compose, Normalize, PadIfNeeded, Resize
-from cv_lib.utils import load_log_configuration
+from ignite.engine import Events
+from ignite.metrics import Loss
+from ignite.utils import convert_tensor
+from toolz import compose, tail, take
+from toolz.sandbox.core import unzip
+from torch.utils import data
+
 from cv_lib.event_handlers import logging_handlers, tensorboard_handlers
-from cv_lib.event_handlers.tensorboard_handlers import (
-    create_image_writer,
-    create_summary_writer,
-)
+from cv_lib.event_handlers.tensorboard_handlers import create_image_writer, create_summary_writer
 from cv_lib.segmentation import models
-from cv_lib.segmentation.metrics import (
-    pixelwise_accuracy,
-    class_accuracy,
-    mean_class_accuracy,
-    class_iou,
-    mean_iou,
-)
-from cv_lib.segmentation.dutchf3.utils import (
-    current_datetime,
-    generate_path,
-    git_branch,
-    git_hash,
-    np_to_tb,
-)
+from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash, np_to_tb
+from cv_lib.segmentation.metrics import class_accuracy, class_iou, mean_class_accuracy, mean_iou, pixelwise_accuracy
 from cv_lib.segmentation.penobscot.engine import create_supervised_evaluator
+from cv_lib.utils import load_log_configuration
 from deepseismic_interpretation.dutchf3.data import decode_segmap
 from deepseismic_interpretation.penobscot.data import get_patch_dataset
 from deepseismic_interpretation.penobscot.metrics import InlineMeanIoU
 from default import _C as config
 from default import update_config
-from ignite.engine import Events
-from ignite.metrics import Loss
-from ignite.utils import convert_tensor
-from toolz import compose, tail, take
-from toolz.sandbox.core import unzip
-from torch.utils import data
 
 
 def _prepare_batch(batch, device=None, non_blocking=False):
@@ -139,7 +124,7 @@ def run(*options, cfg=None, debug=False):
             PadIfNeeded(
                 min_height=config.TRAIN.PATCH_SIZE,
                 min_width=config.TRAIN.PATCH_SIZE,
-                border_mode=cv2.BORDER_CONSTANT,
+                border_mode=config.OPENCV_BORDER_CONSTANT,
                 always_apply=True,
                 mask_value=mask_value,
                 value=0,
@@ -150,7 +135,7 @@ def run(*options, cfg=None, debug=False):
             PadIfNeeded(
                 min_height=config.TRAIN.AUGMENTATIONS.PAD.HEIGHT,
                 min_width=config.TRAIN.AUGMENTATIONS.PAD.WIDTH,
-                border_mode=cv2.BORDER_CONSTANT,
+                border_mode=config.OPENCV_BORDER_CONSTANT,
                 always_apply=True,
                 mask_value=mask_value,
                 value=0,

contrib/experiments/interpretation/penobscot/local/train.py

@@ -17,7 +17,6 @@
 import logging.config
 from os import path
 
-import cv2
 import fire
 import numpy as np
 import torch
@@ -26,46 +25,21 @@
 from ignite.engine import Events
 from ignite.metrics import Loss
 from ignite.utils import convert_tensor
-from toolz import compose
+from toolz import compose, take
 from torch.utils import data
 
+from cv_lib.event_handlers import SnapshotHandler, logging_handlers, tensorboard_handlers
+from cv_lib.event_handlers.logging_handlers import Evaluator
+from cv_lib.event_handlers.tensorboard_handlers import create_image_writer, create_summary_writer
+from cv_lib.segmentation import extract_metric_from, models
+from cv_lib.segmentation.dutchf3.utils import current_datetime, generate_path, git_branch, git_hash, np_to_tb
+from cv_lib.segmentation.metrics import class_accuracy, class_iou, mean_class_accuracy, mean_iou, pixelwise_accuracy
+from cv_lib.segmentation.penobscot.engine import create_supervised_evaluator, create_supervised_trainer
+from cv_lib.utils import load_log_configuration
 from deepseismic_interpretation.dutchf3.data import decode_segmap
 from deepseismic_interpretation.penobscot.data import get_patch_dataset
-from cv_lib.utils import load_log_configuration
-from cv_lib.event_handlers import (
-    SnapshotHandler,
-    logging_handlers,
-    tensorboard_handlers,
-)
-from cv_lib.event_handlers.logging_handlers import Evaluator
-from cv_lib.event_handlers.tensorboard_handlers import (
-    create_image_writer,
-    create_summary_writer,
-)
-from cv_lib.segmentation import models, extract_metric_from
-from cv_lib.segmentation.penobscot.engine import (
-    create_supervised_evaluator,
-    create_supervised_trainer,
-)
-from cv_lib.segmentation.metrics import (
-    pixelwise_accuracy,
-    class_accuracy,
-    mean_class_accuracy,
-    class_iou,
-    mean_iou,
-)
-from cv_lib.segmentation.dutchf3.utils import (
-    current_datetime,
-    generate_path,
-    git_branch,
-    git_hash,
-    np_to_tb,
-)
-
 from default import _C as config
 from default import update_config
-from toolz import take
-
 
 mask_value = 255
 _SEG_COLOURS = np.asarray(
@@ -108,7 +82,7 @@ def run(*options, cfg=None, debug=False):
     load_log_configuration(config.LOG_CONFIG)
     logger = logging.getLogger(__name__)
     logger.debug(config.WORKERS)
-    scheduler_step = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS
+    epochs_per_cycle = config.TRAIN.END_EPOCH // config.TRAIN.SNAPSHOTS
     torch.backends.cudnn.benchmark = config.CUDNN.BENCHMARK
 
     torch.manual_seed(config.SEED)
@@ -127,7 +101,7 @@ def run(*options, cfg=None, debug=False):
             PadIfNeeded(
                 min_height=config.TRAIN.PATCH_SIZE,
                 min_width=config.TRAIN.PATCH_SIZE,
-                border_mode=cv2.BORDER_CONSTANT,
+                border_mode=config.OPENCV_BORDER_CONSTANT,
                 always_apply=True,
                 mask_value=mask_value,
                 value=0,
@@ -138,7 +112,7 @@ def run(*options, cfg=None, debug=False):
             PadIfNeeded(
                 min_height=config.TRAIN.AUGMENTATIONS.PAD.HEIGHT,
                 min_width=config.TRAIN.AUGMENTATIONS.PAD.WIDTH,
-                border_mode=cv2.BORDER_CONSTANT,
+                border_mode=config.OPENCV_BORDER_CONSTANT,
                 always_apply=True,
                 mask_value=mask_value,
                 value=0,
@@ -201,7 +175,7 @@ def run(*options, cfg=None, debug=False):
         output_dir = generate_path(config.OUTPUT_DIR, config_file_name, config.TRAIN.MODEL_DIR, current_datetime(),)
 
     summary_writer = create_summary_writer(log_dir=path.join(output_dir, config.LOG_DIR))
-    snapshot_duration = scheduler_step * len(train_loader)
+    snapshot_duration = min(epochs_per_cycle * len(train_loader), 2)
     scheduler = CosineAnnealingScheduler(optimizer, "lr", config.TRAIN.MAX_LR, config.TRAIN.MIN_LR, snapshot_duration)
 
     # weights are inversely proportional to the frequency of the classes in

cv_lib/cv_lib/event_handlers/logging_handlers.py

@@ -25,11 +25,8 @@ def log_lr(optimizer, engine):
     logger.info(f"lr - {lr}")
 
 
-_DEFAULT_METRICS = {"pixacc": "Avg accuracy :", "nll": "Avg loss :"}
-
-
 @curry
-def log_metrics(log_msg, engine, metrics_dict=_DEFAULT_METRICS):
+def log_metrics(log_msg, engine, metrics_dict={"pixacc": "Avg accuracy :", "nll": "Avg loss :"}):
     logger = logging.getLogger(__name__)
     metrics = engine.state.metrics
     metrics_msg = " ".join([f"{metrics_dict[k]} {metrics[k]:.2f}" for k in metrics_dict])
@@ -44,47 +41,11 @@ def log_class_metrics(log_msg, engine, metrics_dict):
     logger.info(f"{log_msg} - Epoch {engine.state.epoch} [{engine.state.max_epochs}]\n" + metrics_msg)
 
 
+# TODO: remove Evaluator once other train.py scripts are updated
 class Evaluator:
     def __init__(self, evaluation_engine, data_loader):
         self._evaluation_engine = evaluation_engine
         self._data_loader = data_loader
 
     def __call__(self, engine):
         self._evaluation_engine.run(self._data_loader)
-
-
-class HorovodLRScheduler:
-    """
-    Horovod: using `lr = base_lr * hvd.size()` from the very beginning leads to worse final
-    accuracy. Scale the learning rate `lr = base_lr` ---> `lr = base_lr * hvd.size()` during
-    the first five epochs. See https://arxiv.org/abs/1706.02677 for details.
-    After the warmup reduce learning rate by 10 on the 30th, 60th and 80th epochs.
-    """
-
-    def __init__(
-        self, base_lr, warmup_epochs, cluster_size, data_loader, optimizer, batches_per_allreduce,
-    ):
-        self._warmup_epochs = warmup_epochs
-        self._cluster_size = cluster_size
-        self._data_loader = data_loader
-        self._optimizer = optimizer
-        self._base_lr = base_lr
-        self._batches_per_allreduce = batches_per_allreduce
-        self._logger = logging.getLogger(__name__)
-
-    def __call__(self, engine):
-        epoch = engine.state.epoch
-        if epoch < self._warmup_epochs:
-            epoch += float(engine.state.iteration + 1) / len(self._data_loader)
-            lr_adj = 1.0 / self._cluster_size * (epoch * (self._cluster_size - 1) / self._warmup_epochs + 1)
-        elif epoch < 30:
-            lr_adj = 1.0
-        elif epoch < 60:
-            lr_adj = 1e-1
-        elif epoch < 80:
-            lr_adj = 1e-2
-        else:
-            lr_adj = 1e-3
-        for param_group in self._optimizer.param_groups:
-            param_group["lr"] = self._base_lr * self._cluster_size * self._batches_per_allreduce * lr_adj
-            self._logger.debug(f"Adjust learning rate {param_group['lr']}")