From 4ea7af5809fb0b247e02138509e7340f573a5acf Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Tue, 17 Mar 2020 16:06:44 -0400 Subject: [PATCH 01/11] Added tqdm as a dependency. --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 84ace1f..0a8bf99 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,6 +14,7 @@ install_requires = numpy opencv-python-headless scipy + tqdm [options.entry_points] console_scripts = From 865a5ed30db46fb87ee6f3dfec20e011e9a7addd Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Tue, 17 Mar 2020 18:41:45 -0400 Subject: [PATCH 02/11] Started adding CIFAR-100 download. --- .gitignore | 1 + setup.cfg | 1 + src/mosaic/imagelibrary.py | 78 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 src/mosaic/imagelibrary.py diff --git a/.gitignore b/.gitignore index ec0378a..fef43b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Project-specific .vscode/ +libraries/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/setup.cfg b/setup.cfg index 0a8bf99..29a4a1f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,6 +13,7 @@ install_requires = matplotlib numpy opencv-python-headless + requests scipy tqdm diff --git a/src/mosaic/imagelibrary.py b/src/mosaic/imagelibrary.py new file mode 100644 index 0000000..7f119bb --- /dev/null +++ b/src/mosaic/imagelibrary.py @@ -0,0 +1,78 @@ +import abc +import pathlib + +import click +import requests +import tqdm + + +class ImageLibrary(abc.ABC): + '''A library of images that are used to generate the photomosaics. + + The :class:`ImageLibrary` provides an API to access images within a + collection. These collections are, mostly, machine learning data sets. + ''' + def __init__(self, ident: str, + folder: pathlib.Path = './libraries'): + ''' + Parameters + ---------- + ident : str + a string identifier for the library + folder : pathlib.Path + top-level folder where the image libraries are stored + ''' + self._libpath = (folder / ident).resolve() # type: pathlib.Path + + # Check if the directory exists, if not, create it. + if not self._libpath.exists(): + self._libpath.mkdir(parents=True, exist_ok=True) + + def download(self, url: str, filename: str): + '''Download the contents of the URL to image library folder. + + The implementation is based on the one in + https://sumit-ghosh.com/articles/python-download-progress-bar/ + + Parameters + ---------- + url : str + download URL + filename : str + name of the downloaded file + ''' + path = self._libpath / filename + click.secho('Downloading: ', bold=True, nl=False) + click.echo(url) + + if path.exists(): + click.echo('File exists...nothing to do.') + return + + with path.open(mode='wb') as f: + response = requests.get(url, stream=True) + total_size = response.headers.get('content-length') + + if total_size is None: + f.write(response.content) + else: + total_size = int(total_size) + chunk_size = max(int(total_size/1000), 1024*1024) + t = tqdm.tqdm(total=total_size, unit='B', unit_scale=True) + for data in response.iter_content(chunk_size=chunk_size): + f.write(data) + t.update(len(data)) + t.close() + + click.echo('Done...' + click.style('\u2713', fg='green')) + click.secho('Saved to: ', bold=True, nl=False) + click.echo(filename) + + +class CIFAR100Library(ImageLibrary): + '''An image library composed of images from the CIFAR-100 dataset.''' + def __init__(self, folder=pathlib.Path('./libraries')): + super().__init__('cifar100', folder=folder) + self.download( + 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz', + 'cifar-100-python.tar.gz') From 1242e95e3ae451d642cefa7bcf803f2b35a84cba Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Tue, 17 Mar 2020 19:03:34 -0400 Subject: [PATCH 03/11] Can now unpack archive. --- src/mosaic/imagelibrary.py | 44 +++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/src/mosaic/imagelibrary.py b/src/mosaic/imagelibrary.py index 7f119bb..906317f 100644 --- a/src/mosaic/imagelibrary.py +++ b/src/mosaic/imagelibrary.py @@ -1,5 +1,6 @@ import abc import pathlib +import shutil import click import requests @@ -28,7 +29,7 @@ def __init__(self, ident: str, if not self._libpath.exists(): self._libpath.mkdir(parents=True, exist_ok=True) - def download(self, url: str, filename: str): + def _download(self, url: str, filename: str) -> pathlib.Path: '''Download the contents of the URL to image library folder. The implementation is based on the one in @@ -40,6 +41,11 @@ def download(self, url: str, filename: str): download URL filename : str name of the downloaded file + + Returns + ------- + pathlib.Path + path to the downloaded file ''' path = self._libpath / filename click.secho('Downloading: ', bold=True, nl=False) @@ -47,7 +53,7 @@ def download(self, url: str, filename: str): if path.exists(): click.echo('File exists...nothing to do.') - return + return path with path.open(mode='wb') as f: response = requests.get(url, stream=True) @@ -67,12 +73,44 @@ def download(self, url: str, filename: str): click.echo('Done...' + click.style('\u2713', fg='green')) click.secho('Saved to: ', bold=True, nl=False) click.echo(filename) + return path class CIFAR100Library(ImageLibrary): '''An image library composed of images from the CIFAR-100 dataset.''' def __init__(self, folder=pathlib.Path('./libraries')): super().__init__('cifar100', folder=folder) - self.download( + tarball = self._download( 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz', 'cifar-100-python.tar.gz') + self._unpack(tarball) + + def _unpack(self, archive: pathlib.Path) -> pathlib.Path: + '''Unpack the archive file at the given path. + + It will be extracted to the library's working directory. + + Parameters + ---------- + archive : pathlib.Path + path to the archive file + + Returns + ------- + pathlib.Path + path to the extracted archive + ''' + click.secho('Extracting: ', bold=True, nl=False) + click.echo(archive.name) + + path = archive.parent / 'cifar-100-python' + if path.exists(): + click.echo('Folder exists...nothing to do.') + return path + + shutil.unpack_archive(archive, self._libpath) + + if not path.exists(): + raise RuntimeError(f'Failed to unpack {archive}.') + + click.echo('Done...' + click.style('\u2713', fg='green', bold=True)) From 3ca1fcb3aa3006183a7a766fc1537fe2d2cd5ce0 Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Tue, 17 Mar 2020 19:39:39 -0400 Subject: [PATCH 04/11] Can now unpickle image data. --- src/mosaic/imagelibrary.py | 69 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/src/mosaic/imagelibrary.py b/src/mosaic/imagelibrary.py index 906317f..5e01cea 100644 --- a/src/mosaic/imagelibrary.py +++ b/src/mosaic/imagelibrary.py @@ -1,12 +1,29 @@ import abc +import enum +import hashlib import pathlib +import pickle +import secrets import shutil +from typing import Optional, Dict, Tuple import click import requests import tqdm +@enum.unique +class HashType(enum.Enum): + MD5 = enum.auto() + SHA256 = enum.auto() + + +_HASHES = { + HashType.MD5: hashlib.md5, + HashType.SHA256: hashlib.sha256 +} + + class ImageLibrary(abc.ABC): '''A library of images that are used to generate the photomosaics. @@ -29,7 +46,8 @@ def __init__(self, ident: str, if not self._libpath.exists(): self._libpath.mkdir(parents=True, exist_ok=True) - def _download(self, url: str, filename: str) -> pathlib.Path: + def _download(self, url: str, filename: str, + hash: Optional[Tuple[HashType, str]] = None) -> pathlib.Path: '''Download the contents of the URL to image library folder. The implementation is based on the one in @@ -41,12 +59,18 @@ def _download(self, url: str, filename: str) -> pathlib.Path: download URL filename : str name of the downloaded file + hash : (:class:`HashType`, hash) + a tuple containing the hash type and the string used for the + comparison Returns ------- pathlib.Path path to the downloaded file ''' + green_checkmark = click.style('\u2713', fg='green', bold=True) + red_cross = click.style('\u2717', fg='red', bold=True) + path = self._libpath / filename click.secho('Downloading: ', bold=True, nl=False) click.echo(url) @@ -70,6 +94,21 @@ def _download(self, url: str, filename: str) -> pathlib.Path: t.update(len(data)) t.close() + if hash is not None: + block_size = 65536 + hasher = _HASHES[hash[0]]() + with path.open('rb') as f: + data = f.read(block_size) + while len(data) > 0: + hasher.update(data) + data = f.read(block_size) + + if hasher.hexdigest() == hash[1]: + click.echo(f'Hashes match...{green_checkmark}') + else: + click.echo(f'Hashes don\'t match...{red_cross}') + raise RuntimeError(f'Expected hash {hash[0]}, got {hasher.hexdigest()}') # noqa: E501 + click.echo('Done...' + click.style('\u2713', fg='green')) click.secho('Saved to: ', bold=True, nl=False) click.echo(filename) @@ -82,8 +121,10 @@ def __init__(self, folder=pathlib.Path('./libraries')): super().__init__('cifar100', folder=folder) tarball = self._download( 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz', - 'cifar-100-python.tar.gz') - self._unpack(tarball) + 'cifar-100-python.tar.gz', + (HashType.MD5, 'eb9058c3a382ffc7106e4002c42a8d85')) + unpacked = self._unpack(tarball) + self._load_images(unpacked) def _unpack(self, archive: pathlib.Path) -> pathlib.Path: '''Unpack the archive file at the given path. @@ -114,3 +155,25 @@ def _unpack(self, archive: pathlib.Path) -> pathlib.Path: raise RuntimeError(f'Failed to unpack {archive}.') click.echo('Done...' + click.style('\u2713', fg='green', bold=True)) + return path + + def _load_images(self, unpacked: pathlib.Path): + '''Load the images from the CIFAR-100 files. + + The files are standard Python pickle files. Because they're relatively + small (~150 MB), this just loads everything into memory. There isn't + really a point to store them on disk. + + Parameters + ---------- + unpacked : pathlib.Path + path to the unpacked archive + ''' + meta = unpacked / 'meta' + train = unpacked / 'train' + + with meta.open('rb') as f: + self._labels = pickle.load(f, encoding='latin1') + + with train.open('rb') as f: + self._images = pickle.load(f, encoding='latin1') From b04ad2e980324ffeb5f765341b6e114328e9ff4e Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Tue, 17 Mar 2020 22:48:41 -0400 Subject: [PATCH 05/11] Can now load dataset images. --- setup.cfg | 1 + src/mosaic/cli.py | 18 ++++- src/mosaic/imagelibrary.py | 139 ++++++++++++++++++++++++++++++++----- 3 files changed, 138 insertions(+), 20 deletions(-) diff --git a/setup.cfg b/setup.cfg index 29a4a1f..aca0a25 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,3 +25,4 @@ console_scripts = dev = flake8 mypy + numpy-stubs @ https://github.com/numpy/numpy-stubs/archive/f3c6315738489983f5f37e1477ac68373d71b470.zip diff --git a/src/mosaic/cli.py b/src/mosaic/cli.py index 2ad7ce4..5a73fb9 100644 --- a/src/mosaic/cli.py +++ b/src/mosaic/cli.py @@ -1,9 +1,21 @@ import click +from mosaic.imagelibrary import ImageLibrary, CIFAR100Library -@click.command() -def main(): - pass + +@click.group() +@click.pass_context +def main(ctx: click.Context): + ctx.ensure_object(CIFAR100Library) + + +@main.command() +@click.pass_obj +def labels(library: ImageLibrary): + '''List the labels within the loaded library.''' + click.secho('Available Labels:', bold=True) + for label in sorted(list(library.labels)): + click.echo(f' {label}') if __name__ == '__main__': diff --git a/src/mosaic/imagelibrary.py b/src/mosaic/imagelibrary.py index 5e01cea..89baf65 100644 --- a/src/mosaic/imagelibrary.py +++ b/src/mosaic/imagelibrary.py @@ -1,15 +1,16 @@ -import abc +from abc import ABC, abstractmethod, abstractproperty # noqa: F401 +from collections.abc import Mapping import enum import hashlib import pathlib import pickle -import secrets import shutil -from typing import Optional, Dict, Tuple +from typing import FrozenSet, Iterator, List, Optional, Tuple, Union import click +import numpy as np # type: ignore import requests -import tqdm +import tqdm # type: ignore @enum.unique @@ -24,14 +25,28 @@ class HashType(enum.Enum): } -class ImageLibrary(abc.ABC): +PathLike = Union[str, pathlib.Path] + + +class ImageLibrary(ABC, Mapping): '''A library of images that are used to generate the photomosaics. The :class:`ImageLibrary` provides an API to access images within a collection. These collections are, mostly, machine learning data sets. + Subclasses will determine how to access any particular library. + + The library is implemented as a sequence on the internal library labels. + This means that, for example, the ``[]`` operator access images in batches + and not one at a time. Singular access can be obtained using the various + ``get_*()`` methods. + + Attributes + ---------- + labels : FrozenSet[str], read-only + all of the labels within the library ''' def __init__(self, ident: str, - folder: pathlib.Path = './libraries'): + folder: PathLike = './libraries'): ''' Parameters ---------- @@ -40,12 +55,70 @@ def __init__(self, ident: str, folder : pathlib.Path top-level folder where the image libraries are stored ''' + folder = pathlib.Path(folder) self._libpath = (folder / ident).resolve() # type: pathlib.Path + self._first_init = False # Check if the directory exists, if not, create it. if not self._libpath.exists(): + self._first_init = True self._libpath.mkdir(parents=True, exist_ok=True) + def __contains__(self, label: object) -> bool: + return label in self.labels + + def __getitem__(self, label: str) -> List[np.ndarray]: + return [ + self.get_image(index) for index in self.get_indices_for_label(label) # noqa: E501 + ] + + def __iter__(self) -> Iterator[List[np.ndarray]]: + for label in self.labels: + yield self[label] + + def __len__(self) -> int: + return len(self.labels) + + @abstractmethod + def get_image(self, index: int) -> np.ndarray: + '''Get an image from the library. + + This must be implemented by a subclass. + + Parameters + ---------- + index : int + the image's numerical index + + Returns + ------- + np.ndarray + the returned image + ''' + + @abstractmethod + def get_indices_for_label(self, label: str) -> List[int]: + '''Obtains the image indices for the given label. + + Parameters + ---------- + label : str + one of the labels registered with the library + + Returns + ------- + List[int] + list of image indices associated with that label + ''' + + @abstractmethod + def number_of_images(self) -> int: + '''Returns the number of images in the library.''' + + @abstractproperty + def labels(self) -> FrozenSet[str]: + '''The set of labels available within the image library.''' + def _download(self, url: str, filename: str, hash: Optional[Tuple[HashType, str]] = None) -> pathlib.Path: '''Download the contents of the URL to image library folder. @@ -72,21 +145,21 @@ def _download(self, url: str, filename: str, red_cross = click.style('\u2717', fg='red', bold=True) path = self._libpath / filename - click.secho('Downloading: ', bold=True, nl=False) - click.echo(url) if path.exists(): - click.echo('File exists...nothing to do.') return path + click.secho('Downloading: ', bold=True, nl=False) + click.echo(url) + with path.open(mode='wb') as f: response = requests.get(url, stream=True) - total_size = response.headers.get('content-length') + content_length = response.headers.get('content-length') - if total_size is None: + if content_length is None: f.write(response.content) else: - total_size = int(total_size) + total_size = int(content_length) chunk_size = max(int(total_size/1000), 1024*1024) t = tqdm.tqdm(total=total_size, unit='B', unit_scale=True) for data in response.iter_content(chunk_size=chunk_size): @@ -117,7 +190,13 @@ def _download(self, url: str, filename: str, class CIFAR100Library(ImageLibrary): '''An image library composed of images from the CIFAR-100 dataset.''' - def __init__(self, folder=pathlib.Path('./libraries')): + def __init__(self, folder: PathLike = './libraries'): + ''' + Parameters + ---------- + folder : pathlib.Path, optional + path to root library storage folder, by default './libraries' + ''' super().__init__('cifar100', folder=folder) tarball = self._download( 'http://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz', @@ -125,6 +204,33 @@ def __init__(self, folder=pathlib.Path('./libraries')): (HashType.MD5, 'eb9058c3a382ffc7106e4002c42a8d85')) unpacked = self._unpack(tarball) self._load_images(unpacked) + self._names = self._labels['coarse_label_names'] # type: List[str] + self._label_set = frozenset(self._names) + self._ids = {i: name for i, name in enumerate(self._labels['coarse_label_names'])} # noqa: E501 + + def number_of_images(self): + return len(self._images['data']) + + def get_image(self, ind: int) -> np.array: # type: ignore + row = self._images['data'][ind, :] + red = np.reshape(row[0:1024], (32, 32)) + green = np.reshape(row[1024:2048], (32, 32)) + blue = np.reshape(row[2048:3072], (32, 32)) + return np.dstack((red, green, blue)) + + def get_indices_for_label(self, label: str) -> List[int]: + if label not in self: + raise KeyError(f'Unknown label {label}.') + + target = self._names.index(label, 0, -1) + return [ + image for image, label in enumerate(self._images['coarse_labels']) + if label == target + ] + + @property + def labels(self) -> FrozenSet[str]: + return self._label_set def _unpack(self, archive: pathlib.Path) -> pathlib.Path: '''Unpack the archive file at the given path. @@ -141,14 +247,13 @@ def _unpack(self, archive: pathlib.Path) -> pathlib.Path: pathlib.Path path to the extracted archive ''' - click.secho('Extracting: ', bold=True, nl=False) - click.echo(archive.name) - path = archive.parent / 'cifar-100-python' if path.exists(): - click.echo('Folder exists...nothing to do.') return path + click.secho('Extracting: ', bold=True, nl=False) + click.echo(archive.name) + shutil.unpack_archive(archive, self._libpath) if not path.exists(): From fb509971fa14b519ae8db88598b44bcb6c7d3d63 Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Wed, 18 Mar 2020 16:50:19 -0400 Subject: [PATCH 06/11] Bug fixes. --- src/mosaic/imagelibrary.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/mosaic/imagelibrary.py b/src/mosaic/imagelibrary.py index 89baf65..dfa3dbe 100644 --- a/src/mosaic/imagelibrary.py +++ b/src/mosaic/imagelibrary.py @@ -69,7 +69,8 @@ def __contains__(self, label: object) -> bool: def __getitem__(self, label: str) -> List[np.ndarray]: return [ - self.get_image(index) for index in self.get_indices_for_label(label) # noqa: E501 + self.get_image(index) + for index in self.get_indices_for_label(label) ] def __iter__(self) -> Iterator[List[np.ndarray]]: @@ -206,7 +207,10 @@ def __init__(self, folder: PathLike = './libraries'): self._load_images(unpacked) self._names = self._labels['coarse_label_names'] # type: List[str] self._label_set = frozenset(self._names) - self._ids = {i: name for i, name in enumerate(self._labels['coarse_label_names'])} # noqa: E501 + self._ids = { + i: name + for i, name in enumerate(self._labels['coarse_label_names']) + } def number_of_images(self): return len(self._images['data']) @@ -222,7 +226,7 @@ def get_indices_for_label(self, label: str) -> List[int]: if label not in self: raise KeyError(f'Unknown label {label}.') - target = self._names.index(label, 0, -1) + target = self._names.index(label, 0, len(self._names)) return [ image for image, label in enumerate(self._images['coarse_labels']) if label == target From f2071564d50231b92717a284b6892256dbf39476 Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Wed, 18 Mar 2020 21:20:16 -0400 Subject: [PATCH 07/11] Got indexing set up. --- src/mosaic/index.py | 80 ++++++++++++++++++++++++ src/mosaic/processing.py | 131 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 src/mosaic/index.py create mode 100644 src/mosaic/processing.py diff --git a/src/mosaic/index.py b/src/mosaic/index.py new file mode 100644 index 0000000..773aa75 --- /dev/null +++ b/src/mosaic/index.py @@ -0,0 +1,80 @@ +from typing import Dict, List + +import click +import hnswlib +import numpy as np + +from mosaic.imagelibrary import ImageLibrary +from mosaic.processing import FeatureGenerator + +_GREEN_CHECKMARK = click.style('\u2713', fg='green', bold=True) +_RED_CROSS = click.style('\u2717', fg='red', bold=True) + + +class Category(object): + '''A single category within the main index. + + A "category" is just the collection of images that is used to construct a + single library index. It contains the kNN indexing structure and the + mechanism to generate feature descriptors. + + Attributes + ---------- + indexer : hnswlib.Index + the kNN indexing data structure + descriptors : FeatureGenerator + object used for generating feature descriptors for the image collection + ''' + def __init__(self, images: List[np.ndarray], dimensionality: int = 128): + click.echo(f' - Generating image features', nl=False) + self.descriptors = FeatureGenerator(images, dimensionality) + click.echo(f'...{_GREEN_CHECKMARK}') + + click.echo(f' - Building search index', nl=False) + self.indexer = hnswlib.Index('l2', dimensionality) + self.indexer.init_index(max_elements=len(images)) + self.indexer.add_items(self.descriptors.descriptors) + click.echo(f'...{_GREEN_CHECKMARK}') + + +class Index(object): + '''Indexes images in an image library for easy retrieval. + + The :class:`Index` is a wrapper around hnswlib to make working with it + easier. It also performs some of the necessary pre-processing to use the + fast nearest-neighbour library. + + Attributes + ---------- + initialized : bool + if ``False`` then the index has not been built yet + ''' + def __init__(self, ndim: int = 128): + ''' + Parameters + ---------- + ndim : int, optional + dimensionality of the search space; defaults to '128' + ''' + self._ndim = ndim + self._indices: Dict[str, Category] = {} + + @property + def initialized(self) -> bool: + return len(self._indices) != 0 + + def build(self, library: ImageLibrary): + '''Build the index from the given library. + + One index is built for each category/class within the library. + + Parameters + ---------- + library : ImageLibrary + the input image library + ''' + click.secho('Building Library Index', bold=True) + click.echo('----') + for label in library.labels: + click.secho(f'{label}:', bold=True) + self._indices[label] = Category(library[label]) diff --git a/src/mosaic/processing.py b/src/mosaic/processing.py new file mode 100644 index 0000000..c39d112 --- /dev/null +++ b/src/mosaic/processing.py @@ -0,0 +1,131 @@ +from typing import List, Tuple + +import numpy as np + + +def _svd(X: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + '''Performs an SVD on the provided array. + + Per the NumPy documentation (and any linear algebra text book), this + factorizes ``X`` into a matrix such that ``X = U @ np.diag(S) @ Vt``. This + just wraps NumPy's :func:`svd` function and then applies scikit-learn's + "svd flip" algorithm to ensure some consistency on the SVD results. + + Note + ---- + The 'K' below is the minimum of 'M' and 'N'. + + Parameters + ---------- + X : numpy.ndarray + input MxN array + + Returns + ------- + U : numpy.ndarray + SVD's left unitary matrix; will be MxK + S : numpy.ndarray + SVD's singular values; will be of length K + Vt : numpy.ndarray + SVD's right unitary matrix; will be KxN + ''' + U, S, Vt = np.linalg.svd(X, full_matrices=False) + + # Check the signs to ensure SVD consistency (see + # https://stackoverflow.com/a/44847053), which is what scikit-learn's + # svd_flip() function does; see + # https://github.com/scikit-learn/scikit-learn/blob/0.22.2/sklearn/utils/extmath.py#L526 + max_inds = np.argmax(np.abs(U), axis=0) + signs = np.sign(U[max_inds, range(U.shape[1])]) + U *= signs + Vt *= signs[:, np.newaxis] + + return U, S, Vt + + +class FeatureGenerator(object): + '''PCA-based feature descriptors for an image collection. + + Attributes + ---------- + dimensionality : int + number of dimensions in the descriptors + input_features : int + number of input features + mean : numpy.ndarray + the mean used for centering the data used to generate the descriptors + stddev : numpy.ndarray + the standard deviation used for standardizing the data used to generate + the descriptors + ''' + def __init__(self, images: List[np.ndarray], dimensionality: int = 128): + ''' + Parameters + ---------- + images : List[np.ndarray] + list of images + dimensionality : int + number of dimensions in the descriptor space + ''' + num_images = len(images) + num_features = np.prod(images[0].shape) + + self._dimensionality = dimensionality + self._input_features = num_features + + # Flatten the images into a single array of feature vectors. + data = np.zeros((num_images, num_features)) + for i, image in enumerate(images): + data[i, :] = image.flatten() + + # Normalize and standardize the image feature vectors. + self.mean = np.mean(data, axis=0) + self.stddev = np.std(data, axis=0) + data = (data - self.mean) / self.stddev + + # Compute eigenvectors using an SVD. + U, S, V = _svd(data) + + # Compute the principal components and descriptors. + self.principal_components = V[:self._dimensionality] + self.descriptors = U[:, :self._dimensionality]*S[:self._dimensionality] + + def compute(self, image: np.ndarray) -> np.ndarray: + '''Compute a feature descriptor for the provided image. + + The descriptor is computed by using the principal components to project + the higher-dimension image vector into the lower-dimension descriptor + space. + + Parameters + ---------- + image : np.ndarray + input image + + Returns + ------- + np.ndarray + feature descriptor + + Raises + ------ + ValueError + if the unwrapped image isn't the same size as the images used to + generate the descriptor space + ''' + x = image.flatten() + if x.size != self.input_features: + raise ValueError(f'The image must have {self.input_features} elements.') # noqa: E501 + + # Scale to the same range as the "training" data. + x = (x - self.mean) / self.stddev + + return x @ self.principal_components + + @property + def dimensionality(self) -> int: + return self._dimensionality + + @property + def input_features(self) -> int: + return self._input_features From 76a923539948a3ba12e8654bb983369eb6955dbc Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Wed, 18 Mar 2020 21:46:59 -0400 Subject: [PATCH 08/11] Added support to build and save indices. --- src/mosaic/cli.py | 17 +++++++++++++++++ src/mosaic/imagelibrary.py | 4 ++++ src/mosaic/index.py | 31 ++++++++++++++++++++++++++++--- src/mosaic/processing.py | 25 +++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/src/mosaic/cli.py b/src/mosaic/cli.py index 5a73fb9..6ff6adc 100644 --- a/src/mosaic/cli.py +++ b/src/mosaic/cli.py @@ -1,6 +1,7 @@ import click from mosaic.imagelibrary import ImageLibrary, CIFAR100Library +from mosaic.index import Index @click.group() @@ -9,6 +10,22 @@ def main(ctx: click.Context): ctx.ensure_object(CIFAR100Library) +@main.command() +@click.option('-l', '--label', 'labels', nargs=1, multiple=True, + help='Specific labels to build the indices for.') +@click.pass_obj +def build_index(library: ImageLibrary, labels): + '''Build the database indices needed for the photomosaic. + + The photomosaic needs to perform multiple look ups to find the best + matching image for any image patch. This will generate the indices for the + given library. + ''' + index = Index() + index.build(library, labels) + index.save(library.library_path) + + @main.command() @click.pass_obj def labels(library: ImageLibrary): diff --git a/src/mosaic/imagelibrary.py b/src/mosaic/imagelibrary.py index dfa3dbe..eb4df5d 100644 --- a/src/mosaic/imagelibrary.py +++ b/src/mosaic/imagelibrary.py @@ -80,6 +80,10 @@ def __iter__(self) -> Iterator[List[np.ndarray]]: def __len__(self) -> int: return len(self.labels) + @property + def library_path(self) -> pathlib.Path: + return self._libpath + @abstractmethod def get_image(self, index: int) -> np.ndarray: '''Get an image from the library. diff --git a/src/mosaic/index.py b/src/mosaic/index.py index 773aa75..45d7def 100644 --- a/src/mosaic/index.py +++ b/src/mosaic/index.py @@ -1,4 +1,5 @@ -from typing import Dict, List +import pathlib +from typing import Dict, List, Optional, Union import click import hnswlib @@ -9,6 +10,7 @@ _GREEN_CHECKMARK = click.style('\u2713', fg='green', bold=True) _RED_CROSS = click.style('\u2717', fg='red', bold=True) +PathLike = Union[str, pathlib.Path] class Category(object): @@ -63,7 +65,7 @@ def __init__(self, ndim: int = 128): def initialized(self) -> bool: return len(self._indices) != 0 - def build(self, library: ImageLibrary): + def build(self, library: ImageLibrary, labels: Optional[List[str]] = None): '''Build the index from the given library. One index is built for each category/class within the library. @@ -72,9 +74,32 @@ def build(self, library: ImageLibrary): ---------- library : ImageLibrary the input image library + labels : List[str], optional + if provided, only generate indices for these labels ''' click.secho('Building Library Index', bold=True) click.echo('----') - for label in library.labels: + + if labels is None: + categories = library.labels + else: + categories = frozenset(labels) + + for label in categories: click.secho(f'{label}:', bold=True) self._indices[label] = Category(library[label]) + + def save(self, folder: PathLike): + '''Save the index to disk. + + Parameters + ---------- + folder : PathLike + folder to where the indices should be stored + ''' + folder = pathlib.Path(folder) + for label, category in self._indices.items(): + index_file = folder / f'{label}.index' + descr_file = folder / f'{label}.descriptors' + category.indexer.save_index(index_file.as_posix()) + category.descriptors.save(descr_file) diff --git a/src/mosaic/processing.py b/src/mosaic/processing.py index c39d112..a7681ce 100644 --- a/src/mosaic/processing.py +++ b/src/mosaic/processing.py @@ -1,3 +1,5 @@ +import pathlib +import pickle from typing import List, Tuple import numpy as np @@ -129,3 +131,26 @@ def dimensionality(self) -> int: @property def input_features(self) -> int: return self._input_features + + def save(self, path: pathlib.Path): + '''Save the descriptor structure to disk. + + Parameters + ---------- + path : pathlib.Path + path where the descriptor should be stored + ''' + with path.open('wb') as f: + pickle.dump(self, f, pickle.HIGHEST_PROTOCOL) + + @staticmethod + def load(path: pathlib.Path): + '''Load the descriptor structure from disk. + + Parameters + ---------- + path : pathlib.Path + path to the where the discriptors are stored + ''' + with path.open('rb') as f: + return pickle.load(f) From 3b10ce60b91cb7ef3688b9c06b80fdb3e820bcdc Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Thu, 19 Mar 2020 00:09:15 -0400 Subject: [PATCH 09/11] Got the basic prototype to work! --- setup.cfg | 2 +- src/mosaic/cli.py | 30 ++++++++ src/mosaic/index.py | 21 +++++- src/mosaic/processing.py | 149 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 196 insertions(+), 6 deletions(-) diff --git a/setup.cfg b/setup.cfg index aca0a25..c1619aa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,7 @@ install_requires = hnswlib matplotlib numpy - opencv-python-headless + pillow requests scipy tqdm diff --git a/src/mosaic/cli.py b/src/mosaic/cli.py index 6ff6adc..9900eb7 100644 --- a/src/mosaic/cli.py +++ b/src/mosaic/cli.py @@ -1,7 +1,10 @@ import click +import matplotlib.image as mpimg # type: ignore +import matplotlib.pyplot as plt # type: ignore from mosaic.imagelibrary import ImageLibrary, CIFAR100Library from mosaic.index import Index +from mosaic.processing import MosaicGenerator, assemble_image @click.group() @@ -35,5 +38,32 @@ def labels(library: ImageLibrary): click.echo(f' {label}') +@main.command() +@click.argument('label', nargs=1) +@click.argument('image', nargs=1, + type=click.Path(exists=True, file_okay=True, dir_okay=False)) +@click.pass_obj +def generate(library: ImageLibrary, label, image): + '''Generate a photomosaic for the given image.''' + original = mpimg.imread(image) + + # Find tiles + indexer, descriptors = Index.load(library, label) + mosaic = MosaicGenerator(descriptors, indexer, (32, 32)) + tiles = mosaic.generate(original) + + # Generate output + output = assemble_image(tiles, library[label]) + + plt.imshow(tiles) + + plt.figure() + plt.imshow(output) + + plt.show() + + mpimg.imsave(f'{image}-mosaic.jpg', output) + + if __name__ == '__main__': main() diff --git a/src/mosaic/index.py b/src/mosaic/index.py index 45d7def..c389a6c 100644 --- a/src/mosaic/index.py +++ b/src/mosaic/index.py @@ -2,7 +2,7 @@ from typing import Dict, List, Optional, Union import click -import hnswlib +import hnswlib # type: ignore import numpy as np from mosaic.imagelibrary import ImageLibrary @@ -26,8 +26,10 @@ class Category(object): the kNN indexing data structure descriptors : FeatureGenerator object used for generating feature descriptors for the image collection + tile_size : (height, width) + image/tile size of images in this category ''' - def __init__(self, images: List[np.ndarray], dimensionality: int = 128): + def __init__(self, images: List[np.ndarray], dimensionality: int = 256): click.echo(f' - Generating image features', nl=False) self.descriptors = FeatureGenerator(images, dimensionality) click.echo(f'...{_GREEN_CHECKMARK}') @@ -38,6 +40,8 @@ def __init__(self, images: List[np.ndarray], dimensionality: int = 128): self.indexer.add_items(self.descriptors.descriptors) click.echo(f'...{_GREEN_CHECKMARK}') + self.tile_size = images[0].shape[0:2] + class Index(object): '''Indexes images in an image library for easy retrieval. @@ -51,7 +55,7 @@ class Index(object): initialized : bool if ``False`` then the index has not been built yet ''' - def __init__(self, ndim: int = 128): + def __init__(self, ndim: int = 256): ''' Parameters ---------- @@ -103,3 +107,14 @@ def save(self, folder: PathLike): descr_file = folder / f'{label}.descriptors' category.indexer.save_index(index_file.as_posix()) category.descriptors.save(descr_file) + + @staticmethod + def load(library: ImageLibrary, label: str): + index_file = library._libpath / f'{label}.index' + descr_file = library._libpath / f'{label}.descriptors' + + indexer = hnswlib.Index('l2', 256) + indexer.load_index(index_file.as_posix()) + descriptors = FeatureGenerator.load(descr_file) + + return indexer, descriptors diff --git a/src/mosaic/processing.py b/src/mosaic/processing.py index a7681ce..da92234 100644 --- a/src/mosaic/processing.py +++ b/src/mosaic/processing.py @@ -1,8 +1,11 @@ import pathlib import pickle -from typing import List, Tuple +from typing import List, Tuple, Iterator +import click +import hnswlib # type: ignore import numpy as np +import tqdm # type: ignore def _svd(X: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: @@ -45,6 +48,45 @@ def _svd(X: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: return U, S, Vt +def assemble_image(grid: np.ndarray, images: List[np.ndarray]) -> np.ndarray: + '''Assemble a mosaic image from a grid and the set of images. + + Parameters + ---------- + grid : np.ndarray + a MxN array containing image IDs for each tile location + images : List[np.ndarray] + list of image tiles + + Returns + ------- + np.ndarray + a HxW image generated from the original input list + ''' + tile_height, tile_width, _ = images[0].shape + out_height = grid.shape[0]*tile_height + out_width = grid.shape[1]*tile_width + + output = np.zeros((out_height, out_width, 3), dtype=np.uint8) + + progress = tqdm.tqdm(total=tile_width*tile_height) + for r in range(grid.shape[0]): + for c in range(grid.shape[1]): + r_start = r*tile_height + c_start = c*tile_width + + r_end = r_start + tile_height + c_end = c_start + tile_width + + output[r_start:r_end, c_start:c_end, :] = images[grid[r,c]] + + progress.update(1) + + progress.close() + + return output + + class FeatureGenerator(object): '''PCA-based feature descriptors for an image collection. @@ -122,7 +164,7 @@ def compute(self, image: np.ndarray) -> np.ndarray: # Scale to the same range as the "training" data. x = (x - self.mean) / self.stddev - return x @ self.principal_components + return x @ self.principal_components.T @property def dimensionality(self) -> int: @@ -154,3 +196,106 @@ def load(path: pathlib.Path): ''' with path.open('rb') as f: return pickle.load(f) + + +class MosaicGenerator(object): + '''Generate photomosaics from image databases.''' + + class _Tiles(object): + '''Used to generate the tiles for a particular image.''' + def __init__(self, image: np.ndarray, tile_size: Tuple[int, int]): + height, width, _ = image.shape + + self._image = image + self._tilesize = tile_size + + self._tiles_x = width // tile_size[0] + self._tiles_y = height // tile_size[1] + + self._output_width = self._tiles_x*tile_size[0] + self._output_height = self._tiles_y*tile_size[1] + + delta_width = width - self._output_width + delta_height = height - self._output_height + + self._x_start = delta_width // 2 + self._y_start = delta_height // 2 + + self._x_end = (width - 1) - (delta_width // 2) + self._y_end = (height - 1) - (delta_height // 2) + + def __len__(self) -> int: + return self._tiles_x*self._tiles_y + + def __iter__(self) -> Iterator[Tuple[int, int, np.ndarray]]: + height, width = self._tilesize + for r in range(self._tiles_y): + for c in range(self._tiles_x): + r_start = r*height + c_start = c*width + + r_end = r_start + height + c_end = c_start + width + + yield r, c, self._image[r_start:r_end, c_start:c_end, :] + + @property + def grid_size(self) -> Tuple[int, int]: + '''Size of the tile grid. + + Returns + ------- + grid_rows : int + number of grid rows + grid_cols : int + number of grid columns + ''' + return self._tiles_y, self._tiles_x + + @property + def output_size(self) -> Tuple[int, int, int]: + '''Size of the final output image. + + Returns + ------- + height, width, channels : int + the image output size. + ''' + return self._output_height, self._output_width, self._image.shape[2] # noqa: E501 + + def __init__(self, features: FeatureGenerator, indexer: hnswlib.Index, + tile_size: Tuple[int, int]): + self._features = features + self._indexer = indexer + self._tile_size = tile_size + + def generate(self, image: np.ndarray) -> np.ndarray: + '''Generate a mosaic for the given image. + + Parameters + ---------- + image : np.ndarray + input RGB image + + Returns + ------- + np.ndarray + a 2D array where each element is an ID into the image library; this + can then be used to assemble the final mosiac + ''' + tiles = MosaicGenerator._Tiles(image, self._tile_size) + + click.echo(click.style('Tile Size: ', bold=True) + + f'{self._tile_size[0]}x{self._tile_size[1]}') + click.echo(click.style('Grid Size: ', bold=True) + + f'{tiles.grid_size[0]}x{tiles.grid_size[1]}') + click.echo(click.style('Expected Output Size: ', bold=True) + + f'{tiles.output_size[0]}x{tiles.output_size[1]}') + + output = np.zeros(tiles.grid_size, dtype=int) + for r, c, tile in tqdm.tqdm(tiles, unit='tile'): + desc = self._features.compute(tile) + index, _ = self._indexer.knn_query(desc) + output[r, c] = np.squeeze(index) + + return output From 0362cf9ae24344bbdd6ad814083022e33170732d Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Thu, 26 Mar 2020 19:24:51 -0400 Subject: [PATCH 10/11] Minor bug fixes. --- src/mosaic/cli.py | 10 ++++++++-- src/mosaic/index.py | 1 + src/mosaic/processing.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/mosaic/cli.py b/src/mosaic/cli.py index 9900eb7..e668f15 100644 --- a/src/mosaic/cli.py +++ b/src/mosaic/cli.py @@ -14,17 +14,23 @@ def main(ctx: click.Context): @main.command() +@click.option('-d', '--dimensionality', nargs=1, default=128, + help='Number of dimensions in the database feature space.') @click.option('-l', '--label', 'labels', nargs=1, multiple=True, help='Specific labels to build the indices for.') @click.pass_obj -def build_index(library: ImageLibrary, labels): +def build_index(library: ImageLibrary, dimensionality: int, labels): '''Build the database indices needed for the photomosaic. The photomosaic needs to perform multiple look ups to find the best matching image for any image patch. This will generate the indices for the given library. ''' - index = Index() + index = Index(ndim=dimensionality) + if len(labels) == 0: + click.secho('Warning: ', fg='yellow', bold=True, nl=False) + click.echo('Building a complete index; this may take a while.') + labels = None index.build(library, labels) index.save(library.library_path) diff --git a/src/mosaic/index.py b/src/mosaic/index.py index c389a6c..b9fbb03 100644 --- a/src/mosaic/index.py +++ b/src/mosaic/index.py @@ -83,6 +83,7 @@ def build(self, library: ImageLibrary, labels: Optional[List[str]] = None): ''' click.secho('Building Library Index', bold=True) click.echo('----') + click.echo(click.style('Feature Size: ', bold=True) + f'{self._ndim}') if labels is None: categories = library.labels diff --git a/src/mosaic/processing.py b/src/mosaic/processing.py index da92234..97ea7bd 100644 --- a/src/mosaic/processing.py +++ b/src/mosaic/processing.py @@ -78,7 +78,7 @@ def assemble_image(grid: np.ndarray, images: List[np.ndarray]) -> np.ndarray: r_end = r_start + tile_height c_end = c_start + tile_width - output[r_start:r_end, c_start:c_end, :] = images[grid[r,c]] + output[r_start:r_end, c_start:c_end, :] = images[grid[r, c]] progress.update(1) From 757ae741567494b6e236c8827af5a3f1303afa01 Mon Sep 17 00:00:00 2001 From: Richard Rzeszutek Date: Wed, 1 Apr 2020 20:52:21 -0400 Subject: [PATCH 11/11] Outputting as PNG to maximize quality. Will add JPEG output at a later time. --- src/mosaic/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mosaic/cli.py b/src/mosaic/cli.py index e668f15..189a752 100644 --- a/src/mosaic/cli.py +++ b/src/mosaic/cli.py @@ -68,7 +68,7 @@ def generate(library: ImageLibrary, label, image): plt.show() - mpimg.imsave(f'{image}-mosaic.jpg', output) + mpimg.imsave(f'{image}-mosaic.png', output) if __name__ == '__main__':