Skip to content
This repository has been archived by the owner on Nov 16, 2023. It is now read-only.

Commit

Permalink
added cela copyright headers to all non-empty .py files (#3)
Browse files Browse the repository at this point in the history
  • Loading branch information
vapaunic authored Nov 7, 2019
1 parent 1c9a63c commit 93ba611
Show file tree
Hide file tree
Showing 56 changed files with 871 additions and 723 deletions.
6 changes: 5 additions & 1 deletion cv_lib/cv_lib/__version__.py
Original file line number Diff line number Diff line change
@@ -1 +1,5 @@
__version__ = '0.0.1'
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

__version__ = "0.0.1"

23 changes: 14 additions & 9 deletions cv_lib/cv_lib/event_handlers/__init__.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from ignite.handlers import ModelCheckpoint
import glob
import os
from shutil import copyfile


class SnapshotHandler:
def __init__(
self, dir_name, filename_prefix, score_function, snapshot_function
):
def __init__(self, dir_name, filename_prefix, score_function, snapshot_function):
self._model_save_location = dir_name
self._running_model_prefix = filename_prefix+"_running"
self._snapshot_prefix = filename_prefix+"_snapshot"
self._running_model_prefix = filename_prefix + "_running"
self._snapshot_prefix = filename_prefix + "_snapshot"
self._snapshot_function = snapshot_function
self._snapshot_num = 1
self._score_function = score_function
Expand All @@ -20,18 +21,20 @@ def _create_checkpoint_handler(self):
return ModelCheckpoint(
self._model_save_location,
self._running_model_prefix,
score_function=self._score_function,
score_function=self._score_function,
n_saved=1,
create_dir=True,
save_as_state_dict=True,
require_empty=False
require_empty=False,
)

def __call__(self, engine, to_save):
self._checkpoint_handler(engine, to_save)
if self._snapshot_function():
files = glob.glob(
os.path.join(self._model_save_location, self._running_model_prefix + "*")
os.path.join(
self._model_save_location, self._running_model_prefix + "*"
)
)
print(files)
name_postfix = os.path.basename(files[0]).lstrip(self._running_model_prefix)
Expand All @@ -42,5 +45,7 @@ def __call__(self, engine, to_save):
f"{self._snapshot_prefix}{self._snapshot_num}{name_postfix}",
),
)
self._checkpoint_handler = self._create_checkpoint_handler() # Reset the checkpoint handler
self._checkpoint_handler = (
self._create_checkpoint_handler()
) # Reset the checkpoint handler
self._snapshot_num += 1
11 changes: 9 additions & 2 deletions cv_lib/cv_lib/event_handlers/logging_handlers.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import logging
import logging.config
from toolz import curry
Expand All @@ -8,7 +11,9 @@ def log_training_output(engine, log_interval=100):
logger = logging.getLogger(__name__)

if engine.state.iteration % log_interval == 0:
logger.info(f"Epoch: {engine.state.epoch} Iter: {engine.state.iteration} loss {engine.state.output['loss']}")
logger.info(
f"Epoch: {engine.state.epoch} Iter: {engine.state.iteration} loss {engine.state.output['loss']}"
)


@curry
Expand All @@ -25,7 +30,9 @@ def log_lr(optimizer, engine):
def log_metrics(log_msg, engine, metrics_dict=_DEFAULT_METRICS):
logger = logging.getLogger(__name__)
metrics = engine.state.metrics
metrics_msg = " ".join([f"{metrics_dict[k]} {metrics[k]:.2f}" for k in metrics_dict])
metrics_msg = " ".join(
[f"{metrics_dict[k]} {metrics[k]:.2f}" for k in metrics_dict]
)
logger.info(
f"{log_msg} - Epoch {engine.state.epoch} [{engine.state.max_epochs}] "
+ metrics_msg
Expand Down
30 changes: 23 additions & 7 deletions cv_lib/cv_lib/event_handlers/tensorboard_handlers.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from toolz import curry
import torchvision
import logging
Expand All @@ -17,7 +20,9 @@ def create_summary_writer(log_dir):


def _log_model_output(log_label, summary_writer, engine):
summary_writer.add_scalar(log_label, engine.state.output["loss"], engine.state.iteration)
summary_writer.add_scalar(
log_label, engine.state.output["loss"], engine.state.iteration
)


@curry
Expand All @@ -43,17 +48,26 @@ def log_lr(summary_writer, optimizer, log_interval, engine):
lr = [param_group["lr"] for param_group in optimizer.param_groups]
summary_writer.add_scalar("lr", lr[0], getattr(engine.state, log_interval))


_DEFAULT_METRICS = {"accuracy": "Avg accuracy :", "nll": "Avg loss :"}


@curry
def log_metrics(summary_writer, train_engine, log_interval, engine, metrics_dict=_DEFAULT_METRICS):
def log_metrics(
summary_writer, train_engine, log_interval, engine, metrics_dict=_DEFAULT_METRICS
):
metrics = engine.state.metrics
for m in metrics_dict:
summary_writer.add_scalar(metrics_dict[m], metrics[m], getattr(train_engine.state, log_interval))
summary_writer.add_scalar(
metrics_dict[m], metrics[m], getattr(train_engine.state, log_interval)
)


def create_image_writer(summary_writer, label, output_variable, normalize=False, transform_func=lambda x: x):
def create_image_writer(
summary_writer, label, output_variable, normalize=False, transform_func=lambda x: x
):
logger = logging.getLogger(__name__)

def write_to(engine):
try:
data_tensor = transform_func(engine.state.output[output_variable])
Expand All @@ -62,6 +76,8 @@ def write_to(engine):
)
summary_writer.add_image(label, image_grid, engine.state.epoch)
except KeyError:
logger.warning("Predictions and or ground truth labels not available to report")

return write_to
logger.warning(
"Predictions and or ground truth labels not available to report"
)

return write_to
6 changes: 5 additions & 1 deletion cv_lib/cv_lib/segmentation/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from toolz import curry
import torch.nn.functional as F


@curry
def extract_metric_from(metric, engine):
metrics = engine.state.metrics
Expand All @@ -12,4 +16,4 @@ def padded_val_transform(pad_left, fine_size, x, y, y_pred):
y_pred = y_pred[
:, :, pad_left : pad_left + fine_size, pad_left : pad_left + fine_size
].contiguous()
return {"image":x, "y_pred": F.sigmoid(y_pred).detach(), "mask":y.detach()}
return {"image": x, "y_pred": F.sigmoid(y_pred).detach(), "mask": y.detach()}
86 changes: 62 additions & 24 deletions cv_lib/cv_lib/segmentation/dutchf3/augmentations.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,34 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import math
import numbers
import random
import numpy as np

from PIL import Image, ImageOps, ImageChops


class Compose(object):
def __init__(self, augmentations):
self.augmentations = augmentations

def __call__(self, img, mask):

img, mask = Image.fromarray(img, mode=None), Image.fromarray(mask, mode='L')
img, mask = Image.fromarray(img, mode=None), Image.fromarray(mask, mode="L")
assert img.size == mask.size

for a in self.augmentations:
img, mask = a(img, mask)
return np.array(img), np.array(mask, dtype=np.uint8)


class AddNoise(object):
def __call__(self, img, mask):
noise = np.random.normal(loc=0,scale=0.02,size=(img.size[1], img.size[0]))
noise = np.random.normal(loc=0, scale=0.02, size=(img.size[1], img.size[0]))
return img + noise, mask


class RandomCrop(object):
def __init__(self, size, padding=0):
if isinstance(size, numbers.Number):
Expand All @@ -42,11 +48,17 @@ def __call__(self, img, mask):
if w == tw and h == th:
return img, mask
if w < tw or h < th:
return img.resize((tw, th), Image.BILINEAR), mask.resize((tw, th), Image.NEAREST)
return (
img.resize((tw, th), Image.BILINEAR),
mask.resize((tw, th), Image.NEAREST),
)

x1 = random.randint(0, w - tw)
y1 = random.randint(0, h - th)
return img.crop((x1, y1, x1 + tw, y1 + th)), mask.crop((x1, y1, x1 + tw, y1 + th))
return (
img.crop((x1, y1, x1 + tw, y1 + th)),
mask.crop((x1, y1, x1 + tw, y1 + th)),
)


class CenterCrop(object):
Expand All @@ -60,32 +72,46 @@ def __call__(self, img, mask):
assert img.size == mask.size
w, h = img.size
th, tw = self.size
x1 = int(round((w - tw) / 2.))
y1 = int(round((h - th) / 2.))
return img.crop((x1, y1, x1 + tw, y1 + th)), mask.crop((x1, y1, x1 + tw, y1 + th))
x1 = int(round((w - tw) / 2.0))
y1 = int(round((h - th) / 2.0))
return (
img.crop((x1, y1, x1 + tw, y1 + th)),
mask.crop((x1, y1, x1 + tw, y1 + th)),
)


class RandomHorizontallyFlip(object):
def __call__(self, img, mask):
if random.random() < 0.5:
#Note: we use FLIP_TOP_BOTTOM here intentionaly. Due to the dimensions of the image,
# Note: we use FLIP_TOP_BOTTOM here intentionaly. Due to the dimensions of the image,
# it ends up being a horizontal flip.
return img.transpose(Image.FLIP_TOP_BOTTOM), mask.transpose(Image.FLIP_TOP_BOTTOM)
return (
img.transpose(Image.FLIP_TOP_BOTTOM),
mask.transpose(Image.FLIP_TOP_BOTTOM),
)
return img, mask



class RandomVerticallyFlip(object):
def __call__(self, img, mask):
if random.random() < 0.5:
return img.transpose(Image.FLIP_LEFT_RIGHT), mask.transpose(Image.FLIP_LEFT_RIGHT)
return (
img.transpose(Image.FLIP_LEFT_RIGHT),
mask.transpose(Image.FLIP_LEFT_RIGHT),
)
return img, mask



class FreeScale(object):
def __init__(self, size):
self.size = tuple(reversed(size)) # size: (h, w)

def __call__(self, img, mask):
assert img.size == mask.size
return img.resize(self.size, Image.BILINEAR), mask.resize(self.size, Image.NEAREST)
return (
img.resize(self.size, Image.BILINEAR),
mask.resize(self.size, Image.NEAREST),
)


class Scale(object):
Expand All @@ -100,11 +126,17 @@ def __call__(self, img, mask):
if w > h:
ow = self.size
oh = int(self.size * h / w)
return img.resize((ow, oh), Image.BILINEAR), mask.resize((ow, oh), Image.NEAREST)
return (
img.resize((ow, oh), Image.BILINEAR),
mask.resize((ow, oh), Image.NEAREST),
)
else:
oh = self.size
ow = int(self.size * w / h)
return img.resize((ow, oh), Image.BILINEAR), mask.resize((ow, oh), Image.NEAREST)
return (
img.resize((ow, oh), Image.BILINEAR),
mask.resize((ow, oh), Image.NEAREST),
)


class RandomSizedCrop(object):
Expand All @@ -130,10 +162,12 @@ def __call__(self, img, mask):

img = img.crop((x1, y1, x1 + w, y1 + h))
mask = mask.crop((x1, y1, x1 + w, y1 + h))
assert (img.size == (w, h))
assert img.size == (w, h)

return img.resize((self.size, self.size), Image.BILINEAR), mask.resize((self.size, self.size),
Image.NEAREST)
return (
img.resize((self.size, self.size), Image.BILINEAR),
mask.resize((self.size, self.size), Image.NEAREST),
)

# Fallback
scale = Scale(self.size)
Expand All @@ -146,26 +180,27 @@ def __init__(self, degree):
self.degree = degree

def __call__(self, img, mask):
'''
"""
PIL automatically adds zeros to the borders of images that rotated. To fix this
issue, the code in the botton sets anywhere in the labels (mask) that is zero to
255 (the value used for ignore_index).
'''
"""
rotate_degree = random.random() * 2 * self.degree - self.degree

img = img.rotate(rotate_degree, Image.BILINEAR)
mask = mask.rotate(rotate_degree, Image.NEAREST)
mask = mask.rotate(rotate_degree, Image.NEAREST)

binary_mask = Image.fromarray(np.ones([mask.size[1], mask.size[0]]))
binary_mask = binary_mask.rotate(rotate_degree, Image.NEAREST)
binary_mask = np.array(binary_mask)

mask_arr = np.array(mask)
mask_arr[binary_mask==0] = 255
mask_arr[binary_mask == 0] = 255
mask = Image.fromarray(mask_arr)

return img, mask


class RandomSized(object):
def __init__(self, size):
self.size = size
Expand All @@ -178,6 +213,9 @@ def __call__(self, img, mask):
w = int(random.uniform(0.5, 2) * img.size[0])
h = int(random.uniform(0.5, 2) * img.size[1])

img, mask = img.resize((w, h), Image.BILINEAR), mask.resize((w, h), Image.NEAREST)
img, mask = (
img.resize((w, h), Image.BILINEAR),
mask.resize((w, h), Image.NEAREST),
)

return self.crop(*self.scale(img, mask))
return self.crop(*self.scale(img, mask))
11 changes: 8 additions & 3 deletions cv_lib/cv_lib/segmentation/dutchf3/engine.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

import torch

from ignite.engine.engine import Engine, State, Events
Expand All @@ -7,13 +10,15 @@
from torch.nn import functional as F
import numpy as np


def _upscale_model_output(y_pred, y):
ph, pw = y_pred.size(2), y_pred.size(3)
h, w = y.size(2), y.size(3)
if ph != h or pw != w:
y_pred = F.upsample(input=y_pred, size=(h, w), mode='bilinear')
y_pred = F.upsample(input=y_pred, size=(h, w), mode="bilinear")
return y_pred



def create_supervised_trainer(
model,
optimizer,
Expand Down Expand Up @@ -42,7 +47,7 @@ def _update(engine, batch):

@curry
def val_transform(x, y, y_pred):
return {"image":x, "y_pred": y_pred.detach(), "mask":y.detach()}
return {"image": x, "y_pred": y_pred.detach(), "mask": y.detach()}


def create_supervised_evaluator(
Expand Down
Loading

0 comments on commit 93ba611

Please sign in to comment.