added missing license headers and fixed formatting (#391)

* added missing license headers and fixed formatting

* some more license headers

conftest.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

cv_lib/cv_lib/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

cv_lib/cv_lib/event_handlers/__init__.py

@@ -31,7 +31,7 @@ def _create_checkpoint_handler(self):
     def __call__(self, engine, to_save):
         self._checkpoint_handler(engine, to_save)
         if self._snapshot_function():
-            files = glob.glob(os.path.join(self._model_save_location, self._running_model_prefix + "*"))            
+            files = glob.glob(os.path.join(self._model_save_location, self._running_model_prefix + "*"))
             name_postfix = os.path.basename(files[0]).lstrip(self._running_model_prefix)
             copyfile(
                 files[0],

cv_lib/cv_lib/event_handlers/azureml_handlers.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

cv_lib/cv_lib/event_handlers/tensorboard_handlers.py

@@ -10,6 +10,7 @@
 from cv_lib.segmentation.dutchf3.utils import np_to_tb
 from cv_lib.utils import decode_segmap
 
+
 def create_summary_writer(log_dir):
     writer = SummaryWriter(logdir=log_dir)
     return writer

cv_lib/cv_lib/segmentation/dutchf3/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

cv_lib/cv_lib/segmentation/dutchf3/utils.py

@@ -37,4 +37,3 @@ def git_branch():
 def git_hash():
     repo = Repo(search_parent_directories=True)
     return repo.active_branch.commit.hexsha
-

cv_lib/cv_lib/segmentation/models/patch_deconvnet.py

@@ -298,6 +298,7 @@ def init_vgg16_params(self, vgg16, copy_fc8=True):
                         l2.bias.data = l1.bias.data
                         i_layer = i_layer + 1
 
+
 def get_seg_model(cfg, **kwargs):
     assert (
         cfg.MODEL.IN_CHANNELS == 1

cv_lib/cv_lib/segmentation/models/seg_hrnet.py

@@ -430,15 +430,14 @@ def init_weights(
 
         if pretrained and not os.path.isfile(pretrained):
             raise FileNotFoundError(f"The file {pretrained} was not found. Please supply correct path or leave empty")
-        
+
         if os.path.isfile(pretrained):
             pretrained_dict = torch.load(pretrained)
             logger.info("=> loading pretrained model {}".format(pretrained))
             model_dict = self.state_dict()
             pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict.keys()}
             for k, _ in pretrained_dict.items():
-               logger.info(
-                   '=> loading {} pretrained model {}'.format(k, pretrained))
+                logger.info("=> loading {} pretrained model {}".format(k, pretrained))
             model_dict.update(pretrained_dict)
             self.load_state_dict(model_dict)
 

cv_lib/cv_lib/segmentation/utils.py

@@ -3,7 +3,6 @@
 
 import numpy as np
 
+
 def _chw_to_hwc(image_array_numpy):
     return np.moveaxis(image_array_numpy, 0, -1)
-
-

cv_lib/cv_lib/utils.py

@@ -13,11 +13,11 @@ def normalize(array, MIN, MAX):
     Normalizes a segmentation image array by the global range of the data, 
     MIN and MAX, for use with PIL.Image
     """
-   
+
     den = MAX - MIN
-    if den==0:
+    if den == 0:
         den += np.finfo(float).eps
- 
+
     return (array - MIN) / den
 
 

cv_lib/tests/test_metrics.py

@@ -1,3 +1,6 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
 import torch
 import numpy as np
 from pytest import approx

examples/interpretation/notebooks/utilities.py

@@ -23,9 +23,9 @@ def __init__(self, n_classes):
 
     def _fast_hist(self, label_true, label_pred, n_class):
         mask = (label_true >= 0) & (label_true < n_class)
-        hist = np.bincount(
-            n_class * label_true[mask].astype(int) + label_pred[mask], minlength=n_class ** 2,
-        ).reshape(n_class, n_class)
+        hist = np.bincount(n_class * label_true[mask].astype(int) + label_pred[mask], minlength=n_class ** 2,).reshape(
+            n_class, n_class
+        )
         return hist
 
     def update(self, label_trues, label_preds):
@@ -152,9 +152,7 @@ def compose_processing_pipeline(depth, aug=None):
 
 
 def _generate_batches(h, w, ps, patch_size, stride, batch_size=64):
-    hdc_wdx_generator = itertools.product(
-        range(0, h - patch_size + ps, stride), range(0, w - patch_size + ps, stride)
-    )
+    hdc_wdx_generator = itertools.product(range(0, h - patch_size + ps, stride), range(0, w - patch_size + ps, stride))
 
     for batch_indexes in itertoolz.partition_all(batch_size, hdc_wdx_generator):
         yield batch_indexes
@@ -166,9 +164,7 @@ def output_processing_pipeline(config, output):
     _, _, h, w = output.shape
     if config.TEST.POST_PROCESSING.SIZE != h or config.TEST.POST_PROCESSING.SIZE != w:
         output = F.interpolate(
-            output,
-            size=(config.TEST.POST_PROCESSING.SIZE, config.TEST.POST_PROCESSING.SIZE),
-            mode="bilinear",
+            output, size=(config.TEST.POST_PROCESSING.SIZE, config.TEST.POST_PROCESSING.SIZE), mode="bilinear",
         )
 
     if config.TEST.POST_PROCESSING.CROP_PIXELS > 0:
@@ -183,15 +179,7 @@ def output_processing_pipeline(config, output):
 
 
 def patch_label_2d(
-    model,
-    img,
-    pre_processing,
-    output_processing,
-    patch_size,
-    stride,
-    batch_size,
-    device,
-    num_classes,
+    model, img, pre_processing, output_processing, patch_size, stride, batch_size, device, num_classes,
 ):
     """Processes a whole section"""
     img = torch.squeeze(img)
@@ -205,19 +193,14 @@ def patch_label_2d(
     # generate output:
     for batch_indexes in _generate_batches(h, w, ps, patch_size, stride, batch_size=batch_size):
         batch = torch.stack(
-            [
-                pipe(img_p, _extract_patch(hdx, wdx, ps, patch_size), pre_processing)
-                for hdx, wdx in batch_indexes
-            ],
+            [pipe(img_p, _extract_patch(hdx, wdx, ps, patch_size), pre_processing) for hdx, wdx in batch_indexes],
             dim=0,
         )
 
         model_output = model(batch.to(device))
         for (hdx, wdx), output in zip(batch_indexes, model_output.detach().cpu()):
             output = output_processing(output)
-            output_p[
-                :, :, hdx + ps : hdx + ps + patch_size, wdx + ps : wdx + ps + patch_size
-            ] += output
+            output_p[:, :, hdx + ps : hdx + ps + patch_size, wdx + ps : wdx + ps + patch_size] += output
 
     # crop the output_p in the middle
     output = output_p[:, :, ps:-ps, ps:-ps]
@@ -325,20 +308,16 @@ def download_pretrained_model(config):
     elif "penobscot" in config.DATASET.ROOT:
         dataset = "penobscot"
     else:
-        raise NameError(
-            "Unknown dataset name. Only dutch f3 and penobscot are currently supported."
-        )
-
+        raise NameError("Unknown dataset name. Only dutch f3 and penobscot are currently supported.")
+
     if "hrnet" in config.MODEL.NAME:
         model = "hrnet"
     elif "deconvnet" in config.MODEL.NAME:
         model = "deconvnet"
     elif "resnet" in config.MODEL.NAME:
-        model = "seresnetunet"    
+        model = "seresnetunet"
     else:
-        raise NameError(
-            "Unknown model name. Only hrnet, deconvnet, and seresnet_unet are currently supported."
-        )
+        raise NameError("Unknown model name. Only hrnet, deconvnet, and seresnet_unet are currently supported.")
 
     # check if the user already supplied a URL, otherwise figure out the URL
     if "PRETRAINED" in config.MODEL.keys() and validators.url(config.MODEL.PRETRAINED):
@@ -365,39 +344,26 @@ def download_pretrained_model(config):
                 url = "https://deepseismicsharedstore.blob.core.windows.net/master-public-models/dutchf3_hrnet_patch_section_depth.pth"
             elif model == "hrnet" and config.TRAIN.DEPTH == "patch":
                 url = "https://deepseismicsharedstore.blob.core.windows.net/master-public-models/dutchf3_hrnet_patch_patch_depth.pth"
-            elif (
-                model == "deconvnet"
-                and "skip" in config.MODEL.NAME
-                and config.TRAIN.DEPTH == "none"
-            ):
+            elif model == "deconvnet" and "skip" in config.MODEL.NAME and config.TRAIN.DEPTH == "none":
                 url = "http://deepseismicsharedstore.blob.core.windows.net/master-public-models/dutchf3_deconvnetskip_patch_no_depth.pth"
-            elif (
-                model == "deconvnet"
-                and "skip" not in config.MODEL.NAME
-                and config.TRAIN.DEPTH == "none"
-            ):
-                url = "http://deepseismicsharedstore.blob.core.windows.net/master-public-models/dutchf3_deconvnet_patch_no_depth.pth"            
+            elif model == "deconvnet" and "skip" not in config.MODEL.NAME and config.TRAIN.DEPTH == "none":
+                url = "http://deepseismicsharedstore.blob.core.windows.net/master-public-models/dutchf3_deconvnet_patch_no_depth.pth"
             elif model == "seresnetunet" and config.TRAIN.DEPTH == "section":
                 url = "https://deepseismicsharedstore.blob.core.windows.net/master-public-models/dutchf3_seresnetunet_patch_section_depth.pth"
             else:
                 raise NotImplementedError(
                     "We don't store a pretrained model for Dutch F3 for this model combination yet."
                 )
-
 
         else:
-            raise NotImplementedError(
-                "We don't store a pretrained model for this dataset/model combination yet."
-            )
+            raise NotImplementedError("We don't store a pretrained model for this dataset/model combination yet.")
 
         print(f"Could not find a user-supplied URL, downloading from '{url}'")
 
     # make sure the model_dir directory is writeable
     model_dir = config.TRAIN.MODEL_DIR
 
-    if not os.path.isdir(os.path.dirname(model_dir)) or not os.access(
-        os.path.dirname(model_dir), os.W_OK
-    ):
+    if not os.path.isdir(os.path.dirname(model_dir)) or not os.access(os.path.dirname(model_dir), os.W_OK):
         print(f"Cannot write to TRAIN.MODEL_DIR={config.TRAIN.MODEL_DIR}")
         home = str(pathlib.Path.home())
         model_dir = os.path.join(home, "models")
@@ -408,14 +374,10 @@ def download_pretrained_model(config):
 
     if url:
         # Download the pretrained model:
-        pretrained_model_path = os.path.join(
-            model_dir, "pretrained_" + dataset + "_" + model + ".pth"
-        )
+        pretrained_model_path = os.path.join(model_dir, "pretrained_" + dataset + "_" + model + ".pth")
 
         # always redownload the model
-        print(
-            f"Downloading the pretrained model to '{pretrained_model_path}'. This will take a few mintues.. \n"
-        )
+        print(f"Downloading the pretrained model to '{pretrained_model_path}'. This will take a few mintues.. \n")
         urllib.request.urlretrieve(url, pretrained_model_path)
         print("Model successfully downloaded.. \n")
     else:

experiments/interpretation/dutchf3_patch/test.py

@@ -201,12 +201,23 @@ def _output_processing_pipeline(config, output):
 
 
 def _patch_label_2d(
-    model, img, pre_processing, output_processing, patch_size, stride, batch_size, device, num_classes, split, debug, MIN, MAX
+    model,
+    img,
+    pre_processing,
+    output_processing,
+    patch_size,
+    stride,
+    batch_size,
+    device,
+    num_classes,
+    split,
+    debug,
+    MIN,
+    MAX,
 ):
     """Processes a whole section
     """
-
-
+
     img = torch.squeeze(img)
     h, w = img.shape[-2], img.shape[-1]  # height and width
 
@@ -242,50 +253,51 @@ def _patch_label_2d(
                 # dump model confidence values
                 for nclass in range(num_classes):
                     image_to_disk(
-                        model_output[i, nclass, :, :].numpy(), path_prefix + f"_class_{nclass}_conf.png",
-                        MIN, MAX)
+                        model_output[i, nclass, :, :].numpy(), path_prefix + f"_class_{nclass}_conf.png", MIN, MAX
+                    )
 
     # crop the output_p in the middle
     output = output_p[:, :, ps:-ps, ps:-ps]
     return output
 
 
 def _evaluate_split(
-    split, section_aug, model, pre_processing, output_processing, device, running_metrics_overall, config, data_flow, debug=False,
+    split,
+    section_aug,
+    model,
+    pre_processing,
+    output_processing,
+    device,
+    running_metrics_overall,
+    config,
+    data_flow,
+    debug=False,
 ):
     logger = logging.getLogger(__name__)
 
     TestSectionLoader = get_test_loader(config)
 
-    test_set = TestSectionLoader(
-        config,
-        split=split,
-        is_transform=True,
-        augmentations=section_aug,
-        debug=debug,
-    )
+    test_set = TestSectionLoader(config, split=split, is_transform=True, augmentations=section_aug, debug=debug,)
 
     n_classes = test_set.n_classes
 
     if debug:
         data_flow[split] = dict()
-        data_flow[split]['test_section_loader_length'] = len(test_set)
-        data_flow[split]['test_input_shape'] = test_set.seismic.shape
-        data_flow[split]['test_label_shape'] = test_set.labels.shape
-        data_flow[split]['n_classes'] = n_classes
-
+        data_flow[split]["test_section_loader_length"] = len(test_set)
+        data_flow[split]["test_input_shape"] = test_set.seismic.shape
+        data_flow[split]["test_label_shape"] = test_set.labels.shape
+        data_flow[split]["n_classes"] = n_classes
 
     test_loader = data.DataLoader(test_set, batch_size=1, num_workers=config.WORKERS, shuffle=False)
 
     if debug:
-        data_flow[split]['test_loader_length'] = len(test_loader)
+        data_flow[split]["test_loader_length"] = len(test_loader)
         logger.info("Running in Debug/Test mode")
         take_n = 2
         test_loader = take(take_n, test_loader)
-        data_flow[split]['take_n_sections'] = take_n
+        data_flow[split]["take_n_sections"] = take_n
         pred_list, gt_list, img_list = [], [], []
 
-
     try:
         output_dir = generate_path(
             f"{config.OUTPUT_DIR}/test/{split}", git_branch(), git_hash(), config.MODEL.NAME, current_datetime(),
@@ -294,7 +306,6 @@ def _evaluate_split(
         output_dir = generate_path(f"{config.OUTPUT_DIR}/test/{split}", config.MODEL.NAME, current_datetime(),)
 
     running_metrics_split = runningScore(n_classes)
-
 
     # evaluation mode:
     with torch.no_grad():  # operations inside don't track history
@@ -314,7 +325,8 @@ def _evaluate_split(
                 split,
                 debug,
                 config.DATASET.MIN,
-                config.DATASET.MAX)
+                config.DATASET.MAX,
+            )
 
             pred = outputs.detach().max(1)[1].numpy()
             gt = labels.numpy()
@@ -331,10 +343,10 @@ def _evaluate_split(
             mask_to_disk(gt.squeeze(), os.path.join(output_dir, f"{i}_gt.png"), n_classes)
 
     if debug:
-        data_flow[split]['pred_shape'] =  pred_list
-        data_flow[split]['gt_shape'] =  gt_list
-        data_flow[split]['img_shape'] =  img_list
-    
+        data_flow[split]["pred_shape"] = pred_list
+        data_flow[split]["gt_shape"] = gt_list
+        data_flow[split]["img_shape"] = img_list
+
     # get scores
     score, class_iou = running_metrics_split.get_scores()
 
@@ -440,7 +452,7 @@ def test(*options, cfg=None, debug=False):
         config_file_name = "default_config" if not cfg else cfg.split("/")[-1].split(".")[0]
 
         fname = f"data_flow_test_{config_file_name}_{config.TRAIN.MODEL_DIR}.json"
-        with open(fname, 'w') as f:
+        with open(fname, "w") as f:
             json.dump(data_flow, f, indent=1)
 
     # FINAL TEST RESULTS:

experiments/interpretation/dutchf3_patch/test.sh

@@ -1,2 +1,4 @@
 #!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
 python test.py --cfg "configs/seresnet_unet.yaml"