From b322dbf1c83d0a4f68fa234960e2d19c42432c00 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 24 Jun 2022 15:43:53 +0100 Subject: [PATCH 01/50] first commit keops --- alibi_detect/cd/base.py | 2 + alibi_detect/cd/keops/__init__.py | 0 alibi_detect/cd/keops/learned_kernel.py | 0 alibi_detect/cd/keops/mmd.py | 192 ++++++++++++++++++++++++ alibi_detect/cd/mmd.py | 19 ++- alibi_detect/utils/frameworks.py | 6 + alibi_detect/utils/keops/__init__.py | 0 alibi_detect/utils/keops/kernels.py | 143 ++++++++++++++++++ 8 files changed, 357 insertions(+), 5 deletions(-) create mode 100644 alibi_detect/cd/keops/__init__.py create mode 100644 alibi_detect/cd/keops/learned_kernel.py create mode 100644 alibi_detect/cd/keops/mmd.py create mode 100644 alibi_detect/utils/keops/__init__.py create mode 100644 alibi_detect/utils/keops/kernels.py diff --git a/alibi_detect/cd/base.py b/alibi_detect/cd/base.py index 690bc39f9..b5ff7ed88 100644 --- a/alibi_detect/cd/base.py +++ b/alibi_detect/cd/base.py @@ -505,6 +505,7 @@ def __init__( self.infer_sigma = configure_kernel_from_x_ref if configure_kernel_from_x_ref and isinstance(sigma, np.ndarray): self.infer_sigma = False + # TODO: this might print a message for keops despite not existing configure_kernel_from_x_ref logger.warning('`sigma` is specified for the kernel and `configure_kernel_from_x_ref` ' 'is set to True. `sigma` argument takes priority over ' '`configure_kernel_from_x_ref` (set to False).') @@ -547,6 +548,7 @@ def preprocess(self, x: Union[np.ndarray, list]) -> Tuple[np.ndarray, np.ndarray else: return self.x_ref, x # type: ignore[return-value] + # TODO: not absolutely required for keops...?! @abstractmethod def kernel_matrix(self, x: Union['torch.Tensor', 'tf.Tensor'], y: Union['torch.Tensor', 'tf.Tensor']) \ -> Union['torch.Tensor', 'tf.Tensor']: diff --git a/alibi_detect/cd/keops/__init__.py b/alibi_detect/cd/keops/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/alibi_detect/cd/keops/learned_kernel.py b/alibi_detect/cd/keops/learned_kernel.py new file mode 100644 index 000000000..e69de29bb diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py new file mode 100644 index 000000000..3bceb97a9 --- /dev/null +++ b/alibi_detect/cd/keops/mmd.py @@ -0,0 +1,192 @@ +import logging +import numpy as np +import torch +from typing import Callable, Dict, Optional, Tuple, Union +from alibi_detect.cd.base import BaseMMDDrift +from alibi_detect.utils.keops.kernels import GaussianRBF +from alibi_detect.utils.pytorch import get_device + +logger = logging.getLogger(__name__) + + +class MMDDriftKeops(BaseMMDDrift): + def __init__( + self, + x_ref: Union[np.ndarray, list], + p_val: float = .05, + preprocess_x_ref: bool = True, + update_x_ref: Optional[Dict[str, int]] = None, + preprocess_fn: Optional[Callable] = None, + kernel: Callable = GaussianRBF, + sigma: Optional[np.ndarray] = None, + n_permutations: int = 100, + batch_size_permutations: int = 1000000, + device: Optional[str] = None, + input_shape: Optional[tuple] = None, + data_type: Optional[str] = None + ) -> None: + """ + Maximum Mean Discrepancy (MMD) data drift detector using a permutation test. + + Parameters + ---------- + x_ref + Data used as reference distribution. + p_val + p-value used for the significance of the permutation test. + preprocess_x_ref + Whether to already preprocess and store the reference data. + update_x_ref + Reference data can optionally be updated to the last n instances seen by the detector + or via reservoir sampling with size n. For the former, the parameter equals {'last': n} while + for reservoir sampling {'reservoir_sampling': n} is passed. + preprocess_fn + Function to preprocess the data before computing the data drift metrics. + kernel + Kernel used for the MMD computation, defaults to Gaussian RBF kernel. + sigma + Optionally set the GaussianRBF kernel bandwidth. Can also pass multiple bandwidth values as an array. + The kernel evaluation is then averaged over those bandwidths. + n_permutations + Number of permutations used in the permutation test. + batch_size_permutations + KeOps computes the n_permutations of the MMD^2 statistics in chunks of batch_size_permutations. + device + Device type used. The default None tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either 'cuda', 'gpu' or 'cpu'. + input_shape + Shape of input data. + data_type + Optionally specify the data type (tabular, image or time-series). Added to metadata. + """ + super().__init__( + x_ref=x_ref, + p_val=p_val, + preprocess_x_ref=preprocess_x_ref, + update_x_ref=update_x_ref, + preprocess_fn=preprocess_fn, + sigma=sigma, + n_permutations=n_permutations, + input_shape=input_shape, + data_type=data_type + ) + self.meta.update({'backend': 'keops'}) + + # set device + self.device = get_device(device) + + # initialize kernel + sigma = torch.from_numpy(sigma).to(self.device) if isinstance(sigma, # type: ignore[assignment] + np.ndarray) else None + self.kernel = kernel(sigma) if kernel == GaussianRBF else kernel + + # set the correct MMD^2 function based on the batch size for the permutations + self.batch_size = batch_size_permutations + self.n_batches = 1 + (n_permutations - 1) // batch_size_permutations + self.mmd2 = self._mmd2 if self.n_batches == 1 else self._batched_mmd2 + + def _mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) \ + -> Tuple[torch.Tensor, torch.Tensor]: + """ + Compute MMD^2 for the original test statistic and all permutations at once. + + Parameters + ---------- + x_all + Concatenated reference and test instances. + perms + List with permutation vectors. + m + Number of reference instances. + n + Number of test instances. + + Returns + ------- + MMD^2 statistic for the original and permuted reference and test sets. + """ + x_all = x_all.to(self.device) + + # construct stacked tensors with all permutations for the reference set x and test set y + x = torch.cat([x_all[None, :m, :], torch.cat([x_all[perm[:m]][None, :, :] for perm in perms], 0)], 0) + y = torch.cat([x_all[None, m:, :], torch.cat([x_all[perm[m:]][None, :, :] for perm in perms], 0)], 0) + + # compute summed kernel matrices + c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) + # TODO: check where permutations=True and reduce_sum=True belong + k_xx = self.kernel(x, x, permutations=True, reduce_sum=True) + k_yy = self.kernel(y, y, permutations=True, reduce_sum=True) + k_xy = self.kernel(x, y, permutations=True, reduce_sum=True) + stats = c_xx * (k_xx - m) + c_yy * (k_yy - n) - c_xy * k_xy # TODO: check diagonal adjustment + return stats[0], stats[1:] + + # TODO: just use _batched_mmd2?! Need to check time diff with original approach which should be minimal + def _batched_mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) \ + -> Tuple[torch.Tensor, torch.Tensor]: + """ + Batched (across the permutations) MMD^2 computation for the original test statistic and the permutations. + + Parameters + ---------- + x_all + Concatenated reference and test instances. + perms + List with permutation vectors. + m + Number of reference instances. + n + Number of test instances. + + Returns + ------- + MMD^2 statistic for the original and permuted reference and test sets. + """ + k_xx, k_yy, k_xy = [], [], [] + for batch in range(self.n_batches): + i, j = batch * self.batch_size, (batch + 1) * self.batch_size + # construct stacked tensors with a batch of permutations for the reference set x and test set y + x = torch.cat([x_all[perm[:m]][None, :, :] for perm in perms[i:j]], 0) + y = torch.cat([x_all[perm[m:]][None, :, :] for perm in perms[i:j]], 0) + if batch == 0: + x = torch.cat([x_all[None, :m, :], x], 0) + y = torch.cat([x_all[None, m:, :], y], 0) + x, y = x.to(self.device), y.to(self.device) + + # batch-wise kernel matrix computation over the permutations + k_xx.append(self.kernel(x, x, permutations=True, reduce_sum=True)) + k_yy.append(self.kernel(y, y, permutations=True, reduce_sum=True)) + k_xy.append(self.kernel(x, y, permutations=True, reduce_sum=True)) + c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) + stats = c_xx * (torch.cat(k_xx) - m) + c_yy * (torch.cat(k_yy) - n) - c_xy * torch.cat(k_xy) + return stats[0], stats[1:] + + def score(self, x: Union[np.ndarray, list]) -> Tuple[float, float, float]: + """ + Compute the p-value resulting from a permutation test using the maximum mean discrepancy + as a distance measure between the reference data and the data to be tested. + + Parameters + ---------- + x + Batch of instances. + + Returns + ------- + p-value obtained from the permutation test, the MMD^2 between the reference and test set, + and the MMD^2 threshold above which drift is flagged. + """ + x_ref, x = self.preprocess(x) + x_ref = torch.from_numpy(x_ref).float() # type: ignore[assignment] + x = torch.from_numpy(x).float() # type: ignore[assignment] + # compute kernel matrix, MMD^2 and apply permutation test + m, n = x_ref.shape[0], x.shape[0] + perms = [torch.randperm(m + n) for _ in range(self.n_permutations)] + x_all = torch.cat([x_ref, x], 0) + mmd2, mmd2_permuted = self.mmd2(x_all, perms, m, n) + if self.device.type == 'cuda': + mmd2, mmd2_permuted = mmd2.cpu(), mmd2_permuted.cpu() + p_val = (mmd2 <= mmd2_permuted).float().mean() + # compute distance threshold + idx_threshold = int(self.p_val * len(mmd2_permuted)) + distance_threshold = torch.sort(mmd2_permuted, descending=True).values[idx_threshold] + return p_val.numpy().item(), mmd2.numpy().item(), distance_threshold.numpy() diff --git a/alibi_detect/cd/mmd.py b/alibi_detect/cd/mmd.py index 0da0dec5b..0e00ad32c 100644 --- a/alibi_detect/cd/mmd.py +++ b/alibi_detect/cd/mmd.py @@ -1,7 +1,10 @@ import logging import numpy as np from typing import Callable, Dict, Optional, Union, Tuple -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow +from alibi_detect.utils.frameworks import has_keops, has_pytorch, has_tensorflow + +if has_keops: + from alibi_detect.cd.keops import MMDDriftKeops if has_pytorch: from alibi_detect.cd.pytorch.mmd import MMDDriftTorch @@ -68,10 +71,11 @@ def __init__( super().__init__() backend = backend.lower() - if backend == 'tensorflow' and not has_tensorflow or backend == 'pytorch' and not has_pytorch: + if backend == 'tensorflow' and not has_tensorflow or backend == 'pytorch' and not has_pytorch \ + or backend == 'keops' and not has_keops: raise ImportError(f'{backend} not installed. Cannot initialize and run the ' f'MMDDrift detector with {backend} backend.') - elif backend not in ['tensorflow', 'pytorch']: + elif backend not in ['tensorflow', 'pytorch', 'keops']: raise NotImplementedError(f'{backend} not implemented. Use tensorflow or pytorch instead.') kwargs = locals() @@ -82,15 +86,20 @@ def __init__( if kernel is None: if backend == 'tensorflow': from alibi_detect.utils.tensorflow.kernels import GaussianRBF - else: + elif backend == 'pytorch': from alibi_detect.utils.pytorch.kernels import GaussianRBF # type: ignore + else: + from alibi_detect.utils.keops.kernels import GaussianRBF # type: ignore kwargs.update({'kernel': GaussianRBF}) if backend == 'tensorflow' and has_tensorflow: kwargs.pop('device', None) self._detector = MMDDriftTF(*args, **kwargs) # type: ignore - else: + elif backend == 'pytorch' and has_pytorch: self._detector = MMDDriftTorch(*args, **kwargs) # type: ignore + else: + kwargs.pop('configure_kernel_from_x_ref', None) + self._detector = MMDDriftKeops(*args, **kwargs) # type: ignore self.meta = self._detector.meta def predict(self, x: Union[np.ndarray, list], return_p_val: bool = True, return_distance: bool = True) \ diff --git a/alibi_detect/utils/frameworks.py b/alibi_detect/utils/frameworks.py index 5613e21cf..f742618b1 100644 --- a/alibi_detect/utils/frameworks.py +++ b/alibi_detect/utils/frameworks.py @@ -15,3 +15,9 @@ has_sklearn = True except ImportError: has_sklearn = False + +try: + import keops # noqa + has_keops = True +except ImportError: + has_keops = False diff --git a/alibi_detect/utils/keops/__init__.py b/alibi_detect/utils/keops/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py new file mode 100644 index 000000000..0bba30c8c --- /dev/null +++ b/alibi_detect/utils/keops/kernels.py @@ -0,0 +1,143 @@ +from alibi_detect.utils.pytorch.kernels import sigma_median # TODO: keops sigma_median? +import numpy as np +from pykeops.torch import LazyTensor +import torch +import torch.nn as nn +from typing import Callable, List, Tuple, Union + + +class GaussianRBF(nn.Module): + def __init__( + self, + sigma: torch.Tensor = None, + init_sigma_fn: Callable = sigma_median, + trainable: bool = False + ) -> None: + """ + Gaussian RBF kernel: k(x,y) = exp(-(1/(2*sigma^2)||x-y||^2). A forward pass takes + a batch of instances x [Nx, features] and y [Ny, features] and returns the kernel + matrix [Nx, Ny]. + + Parameters + ---------- + sigma + Bandwidth used for the kernel. Needn't be specified if being inferred or trained. + Can pass multiple values to eval kernel with and then average. + init_sigma_fn + Function used to compute the bandwidth `sigma`. Used when `sigma` is to be inferred. + The function's signature should match :py:func:`~alibi_detect.utils.pytorch.kernels.sigma_median`, + meaning that it should take in the tensors `x`, `y` and `dist` and return `sigma`. + trainable + Whether or not to track gradients w.r.t. `sigma` to allow it to be trained. + """ + super().__init__() + if sigma is None: + self.log_sigma = nn.Parameter(torch.empty(1), requires_grad=trainable) + self.init_required = True + else: + #sigma = torch.Tensor([sigma]) # TODO: ensure it's done somewhere else + sigma = sigma.reshape(-1) # [1] + self.log_sigma = nn.Parameter(sigma.log(), requires_grad=trainable) + self.init_required = False + self.init_sigma_fn = init_sigma_fn + self.trainable = trainable + + super().__init__() + if sigma is None: + self.log_sigma = nn.Parameter(torch.empty(1), requires_grad=trainable) + self.init_required = True + else: + sigma = sigma.reshape(-1) # [Ns,] TODO: ensure this works with keops + self.log_sigma = nn.Parameter(sigma.log(), requires_grad=trainable) + self.init_required = False + self.init_sigma_fn = init_sigma_fn + self.trainable = trainable + + @property + def sigma(self) -> torch.Tensor: + return self.log_sigma.exp() + + # TODO: could use original kernel with some tweaks? + # - LazyTensor input + # - permutations input + # - reduce_sum done in main detector + def forward(self, x: Union[np.ndarray, torch.Tensor], y: Union[np.ndarray, torch.Tensor], + infer_sigma: bool = False, permutations: bool = False, reduce_sum: bool = False) -> LazyTensor: + + x, y = torch.as_tensor(x), torch.as_tensor(y) + + #if isinstance(x, np.ndarray): + # x = torch.as_tensor(x) + #if isinstance(y, np.ndarray): + # y = torch.as_tensor(y) + + if not permutations: + x_i = LazyTensor(x[:, None, :]) # [n, 1, d] + y_j = LazyTensor(y[None, :, :]) # [1, m, d] + else: + x_i = LazyTensor(x[:, :, None, :]) # [perms+1, n, 1, d] + y_j = LazyTensor(y[:, None, :, :]) # [perms+1, 1, m, d] + d_ij = ((x_i - y_j) ** 2).sum(-1) # [n, m] + + if infer_sigma or self.init_required: + if self.trainable and infer_sigma: + raise ValueError("Gradients cannot be computed w.r.t. an inferred sigma value") + sigma = self.init_sigma_fn(x, y, d_ij) # TODO: would not work with default init fn + with torch.no_grad(): + self.log_sigma.copy_(sigma.log().clone()) + self.init_required = False + + gamma = 1. / (2. * self.sigma ** 2) # [1] TODO: [Ns,]? + if not permutations: + gamma = LazyTensor(gamma[None, None, :]) # [1, 1, 1] + else: + gamma = LazyTensor(gamma[None, None, None, :]) # [1, 1, 1, 1] + k_ij = (- gamma * d_ij).exp() # [n, m] or [perms+1, n, m] + if reduce_sum: + k_ij = k_ij.sum(1).sum(1).squeeze(-1) # [1] or [perms+1] + return k_ij + + +class DeepKernelKeops(nn.Module): + def __init__( + self, + proj: nn.Module, + kernel_a: nn.Module = GaussianRBFKeops(trainable=True), + kernel_b: nn.Module = GaussianRBFKeops(trainable=True), + eps: Union[float, str] = 'trainable' + ) -> None: + super().__init__() + + self.kernel_a = kernel_a + self.kernel_b = kernel_b + self.proj = proj + if kernel_b is not None: + self._init_eps(eps) + + def _init_eps(self, eps: Union[float, str]) -> None: + if isinstance(eps, float): + if not 0 < eps < 1: + raise ValueError("eps should be in (0,1)") + self.logit_eps = nn.Parameter(torch.tensor(eps).logit(), requires_grad=False) + elif eps == 'trainable': + self.logit_eps = nn.Parameter(torch.tensor(0.)) + else: + raise NotImplementedError("eps should be 'trainable' or a float in (0,1)") + + @property + def eps(self) -> torch.Tensor: + return self.logit_eps.sigmoid() if self.kernel_b is not None else torch.tensor(0.) + + def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + similarity = self.kernel_a(self.proj(x), self.proj(y)) + if self.kernel_b is not None: + similarity = (1-self.eps)*similarity + self.eps*self.kernel_b(x, y) + return similarity + + # TODO: where does this belong? + def forward_perms(self, x_proj: torch.Tensor, y_proj: torch.Tensor, + x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + similarity = self.kernel_a(x_proj, y_proj, permutations=True) # [perms+1, n, m] + if self.kernel_b is not None: + similarity = (1-self.eps)*similarity + self.eps*self.kernel_b(x, y, permutations=True) + return similarity.sum(1).sum(1).squeeze(-1) # [perms+1] From 29ad4d9a2900d921b433922e937161e78123750f Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 24 Jun 2022 17:00:47 +0100 Subject: [PATCH 02/50] update kernel and mmd keops --- alibi_detect/cd/keops/mmd.py | 22 ++++++++++++----- alibi_detect/utils/keops/kernels.py | 38 ++++++++++++++--------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index 3bceb97a9..67b07e684 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -1,5 +1,6 @@ import logging import numpy as np +from pykeops.torch import LazyTensor import torch from typing import Callable, Dict, Optional, Tuple, Union from alibi_detect.cd.base import BaseMMDDrift @@ -114,9 +115,12 @@ def _mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) # compute summed kernel matrices c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) # TODO: check where permutations=True and reduce_sum=True belong - k_xx = self.kernel(x, x, permutations=True, reduce_sum=True) - k_yy = self.kernel(y, y, permutations=True, reduce_sum=True) - k_xy = self.kernel(x, y, permutations=True, reduce_sum=True) + #k_xx = self.kernel(x, x, permutations=True, reduce_sum=True) + #k_yy = self.kernel(y, y, permutations=True, reduce_sum=True) + #k_xy = self.kernel(x, y, permutations=True, reduce_sum=True) + k_xx = self.kernel(LazyTensor(x[:, :, None, :]), LazyTensor(x[:, None, :, :])).sum(1).sum(1).squeeze(-1) + k_yy = self.kernel(LazyTensor(y[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1) + k_xy = self.kernel(LazyTensor(x[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1) stats = c_xx * (k_xx - m) + c_yy * (k_yy - n) - c_xy * k_xy # TODO: check diagonal adjustment return stats[0], stats[1:] @@ -153,9 +157,15 @@ def _batched_mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, x, y = x.to(self.device), y.to(self.device) # batch-wise kernel matrix computation over the permutations - k_xx.append(self.kernel(x, x, permutations=True, reduce_sum=True)) - k_yy.append(self.kernel(y, y, permutations=True, reduce_sum=True)) - k_xy.append(self.kernel(x, y, permutations=True, reduce_sum=True)) + #k_xx.append(self.kernel(x, x, permutations=True, reduce_sum=True)) + #k_yy.append(self.kernel(y, y, permutations=True, reduce_sum=True)) + #k_xy.append(self.kernel(x, y, permutations=True, reduce_sum=True)) + k_xx.append(self.kernel( + LazyTensor(x[:, :, None, :]), LazyTensor(x[:, None, :, :])).sum(1).sum(1).squeeze(-1)) + k_yy.append(self.kernel( + LazyTensor(y[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1)) + k_xy.append(self.kernel( + LazyTensor(x[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1)) c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) stats = c_xx * (torch.cat(k_xx) - m) + c_yy * (torch.cat(k_yy) - n) - c_xy * torch.cat(k_xy) return stats[0], stats[1:] diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index 0bba30c8c..08667bb8f 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -30,18 +30,6 @@ def __init__( trainable Whether or not to track gradients w.r.t. `sigma` to allow it to be trained. """ - super().__init__() - if sigma is None: - self.log_sigma = nn.Parameter(torch.empty(1), requires_grad=trainable) - self.init_required = True - else: - #sigma = torch.Tensor([sigma]) # TODO: ensure it's done somewhere else - sigma = sigma.reshape(-1) # [1] - self.log_sigma = nn.Parameter(sigma.log(), requires_grad=trainable) - self.init_required = False - self.init_sigma_fn = init_sigma_fn - self.trainable = trainable - super().__init__() if sigma is None: self.log_sigma = nn.Parameter(torch.empty(1), requires_grad=trainable) @@ -58,19 +46,14 @@ def sigma(self) -> torch.Tensor: return self.log_sigma.exp() # TODO: could use original kernel with some tweaks? - # - LazyTensor input + # - LazyTensor input -> # - permutations input # - reduce_sum done in main detector - def forward(self, x: Union[np.ndarray, torch.Tensor], y: Union[np.ndarray, torch.Tensor], + def _forward(self, x: Union[np.ndarray, torch.Tensor], y: Union[np.ndarray, torch.Tensor], infer_sigma: bool = False, permutations: bool = False, reduce_sum: bool = False) -> LazyTensor: x, y = torch.as_tensor(x), torch.as_tensor(y) - #if isinstance(x, np.ndarray): - # x = torch.as_tensor(x) - #if isinstance(y, np.ndarray): - # y = torch.as_tensor(y) - if not permutations: x_i = LazyTensor(x[:, None, :]) # [n, 1, d] y_j = LazyTensor(y[None, :, :]) # [1, m, d] @@ -97,6 +80,23 @@ def forward(self, x: Union[np.ndarray, torch.Tensor], y: Union[np.ndarray, torch k_ij = k_ij.sum(1).sum(1).squeeze(-1) # [1] or [perms+1] return k_ij + def forward(self, x: LazyTensor, y: LazyTensor, infer_sigma: bool = False) -> LazyTensor: + + dist = ((x - y) ** 2).sum(-1) + + if infer_sigma or self.init_required: + if self.trainable and infer_sigma: + raise ValueError("Gradients cannot be computed w.r.t. an inferred sigma value") + sigma = self.init_sigma_fn(x, y, d_ij) # TODO: would not work with default init fn + with torch.no_grad(): + self.log_sigma.copy_(sigma.log().clone()) + self.init_required = False + + gamma = 1. / (2. * self.sigma ** 2) # [1] TODO: [Ns,]? + gamma = LazyTensor(gamma[None, None, :]) if len(dist.shape) == 2 else LazyTensor(gamma[None, None, None, :]) + kernel_mat = (- gamma * d_ij).exp() + return kernel_mat + class DeepKernelKeops(nn.Module): def __init__( From dd2d13ed2c010445f3ade5ea4a3c5b02f22d51dc Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 24 Jun 2022 18:25:34 +0100 Subject: [PATCH 03/50] allow multiple kernel bandwidths for keops --- alibi_detect/cd/keops/mmd.py | 7 ----- alibi_detect/utils/keops/kernels.py | 47 +++++------------------------ 2 files changed, 7 insertions(+), 47 deletions(-) diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index 67b07e684..62d51aeb5 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -114,10 +114,6 @@ def _mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) # compute summed kernel matrices c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) - # TODO: check where permutations=True and reduce_sum=True belong - #k_xx = self.kernel(x, x, permutations=True, reduce_sum=True) - #k_yy = self.kernel(y, y, permutations=True, reduce_sum=True) - #k_xy = self.kernel(x, y, permutations=True, reduce_sum=True) k_xx = self.kernel(LazyTensor(x[:, :, None, :]), LazyTensor(x[:, None, :, :])).sum(1).sum(1).squeeze(-1) k_yy = self.kernel(LazyTensor(y[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1) k_xy = self.kernel(LazyTensor(x[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1) @@ -157,9 +153,6 @@ def _batched_mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, x, y = x.to(self.device), y.to(self.device) # batch-wise kernel matrix computation over the permutations - #k_xx.append(self.kernel(x, x, permutations=True, reduce_sum=True)) - #k_yy.append(self.kernel(y, y, permutations=True, reduce_sum=True)) - #k_xy.append(self.kernel(x, y, permutations=True, reduce_sum=True)) k_xx.append(self.kernel( LazyTensor(x[:, :, None, :]), LazyTensor(x[:, None, :, :])).sum(1).sum(1).squeeze(-1)) k_yy.append(self.kernel( diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index 08667bb8f..d3b64973d 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -10,7 +10,7 @@ class GaussianRBF(nn.Module): def __init__( self, sigma: torch.Tensor = None, - init_sigma_fn: Callable = sigma_median, + init_sigma_fn: Callable = sigma_median, # TODO: would not work with default init fn trainable: bool = False ) -> None: """ @@ -35,7 +35,7 @@ def __init__( self.log_sigma = nn.Parameter(torch.empty(1), requires_grad=trainable) self.init_required = True else: - sigma = sigma.reshape(-1) # [Ns,] TODO: ensure this works with keops + sigma = sigma.reshape(-1) # [Ns,] self.log_sigma = nn.Parameter(sigma.log(), requires_grad=trainable) self.init_required = False self.init_sigma_fn = init_sigma_fn @@ -45,41 +45,6 @@ def __init__( def sigma(self) -> torch.Tensor: return self.log_sigma.exp() - # TODO: could use original kernel with some tweaks? - # - LazyTensor input -> - # - permutations input - # - reduce_sum done in main detector - def _forward(self, x: Union[np.ndarray, torch.Tensor], y: Union[np.ndarray, torch.Tensor], - infer_sigma: bool = False, permutations: bool = False, reduce_sum: bool = False) -> LazyTensor: - - x, y = torch.as_tensor(x), torch.as_tensor(y) - - if not permutations: - x_i = LazyTensor(x[:, None, :]) # [n, 1, d] - y_j = LazyTensor(y[None, :, :]) # [1, m, d] - else: - x_i = LazyTensor(x[:, :, None, :]) # [perms+1, n, 1, d] - y_j = LazyTensor(y[:, None, :, :]) # [perms+1, 1, m, d] - d_ij = ((x_i - y_j) ** 2).sum(-1) # [n, m] - - if infer_sigma or self.init_required: - if self.trainable and infer_sigma: - raise ValueError("Gradients cannot be computed w.r.t. an inferred sigma value") - sigma = self.init_sigma_fn(x, y, d_ij) # TODO: would not work with default init fn - with torch.no_grad(): - self.log_sigma.copy_(sigma.log().clone()) - self.init_required = False - - gamma = 1. / (2. * self.sigma ** 2) # [1] TODO: [Ns,]? - if not permutations: - gamma = LazyTensor(gamma[None, None, :]) # [1, 1, 1] - else: - gamma = LazyTensor(gamma[None, None, None, :]) # [1, 1, 1, 1] - k_ij = (- gamma * d_ij).exp() # [n, m] or [perms+1, n, m] - if reduce_sum: - k_ij = k_ij.sum(1).sum(1).squeeze(-1) # [1] or [perms+1] - return k_ij - def forward(self, x: LazyTensor, y: LazyTensor, infer_sigma: bool = False) -> LazyTensor: dist = ((x - y) ** 2).sum(-1) @@ -87,14 +52,16 @@ def forward(self, x: LazyTensor, y: LazyTensor, infer_sigma: bool = False) -> La if infer_sigma or self.init_required: if self.trainable and infer_sigma: raise ValueError("Gradients cannot be computed w.r.t. an inferred sigma value") - sigma = self.init_sigma_fn(x, y, d_ij) # TODO: would not work with default init fn + sigma = self.init_sigma_fn(x, y, dist) with torch.no_grad(): self.log_sigma.copy_(sigma.log().clone()) self.init_required = False - gamma = 1. / (2. * self.sigma ** 2) # [1] TODO: [Ns,]? + gamma = 1. / (2. * self.sigma ** 2) gamma = LazyTensor(gamma[None, None, :]) if len(dist.shape) == 2 else LazyTensor(gamma[None, None, None, :]) - kernel_mat = (- gamma * d_ij).exp() + kernel_mat = (- gamma * dist).exp() + if len(dist.shape) < len(gamma.shape): + kernel_mat = kernel_mat.sum(-1) / len(self.sigma) return kernel_mat From 0a7944c308b148bbdfa4c6c14af17b1dae6864c8 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 24 Jun 2022 19:09:58 +0100 Subject: [PATCH 04/50] fix bug --- alibi_detect/utils/keops/kernels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index d3b64973d..dc2777d4f 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -25,7 +25,7 @@ def __init__( Can pass multiple values to eval kernel with and then average. init_sigma_fn Function used to compute the bandwidth `sigma`. Used when `sigma` is to be inferred. - The function's signature should match :py:func:`~alibi_detect.utils.pytorch.kernels.sigma_median`, + The function's signature should match :py:func:`~alibi_detect.utils.keops.kernels.sigma_median`, meaning that it should take in the tensors `x`, `y` and `dist` and return `sigma`. trainable Whether or not to track gradients w.r.t. `sigma` to allow it to be trained. From 17d1662e5d2f08e53ceabb794353a020e55ef0fd Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Thu, 30 Jun 2022 16:18:43 +0100 Subject: [PATCH 05/50] update mmd --- alibi_detect/cd/keops/learned_kernel.py | 0 alibi_detect/cd/mmd.py | 22 ++++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) delete mode 100644 alibi_detect/cd/keops/learned_kernel.py diff --git a/alibi_detect/cd/keops/learned_kernel.py b/alibi_detect/cd/keops/learned_kernel.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/alibi_detect/cd/mmd.py b/alibi_detect/cd/mmd.py index 0e00ad32c..336cf9be0 100644 --- a/alibi_detect/cd/mmd.py +++ b/alibi_detect/cd/mmd.py @@ -28,6 +28,7 @@ def __init__( sigma: Optional[np.ndarray] = None, configure_kernel_from_x_ref: bool = True, n_permutations: int = 100, + batch_size_permutations: int = 1000000, device: Optional[str] = None, input_shape: Optional[tuple] = None, data_type: Optional[str] = None @@ -60,6 +61,9 @@ def __init__( Whether to already configure the kernel bandwidth from the reference data. n_permutations Number of permutations used in the permutation test. + batch_size_permutations + KeOps computes the n_permutations of the MMD^2 statistics in chunks of batch_size_permutations. + Only relevant for 'keops' backend. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. Can be specified by passing either 'cuda', 'gpu' or 'cpu'. Only relevant for 'pytorch' backend. @@ -81,6 +85,15 @@ def __init__( kwargs = locals() args = [kwargs['x_ref']] pop_kwargs = ['self', 'x_ref', 'backend', '__class__'] + if backend == 'tensorflow' and has_tensorflow: + pop_kwargs += ['device', 'batch_size_permutations'] + detector = MMDDriftTF + elif backend == 'pytorch' and has_pytorch: + pop_kwargs += ['batch_size_permutations'] + detector = MMDDriftTorch + else: + pop_kwargs += ['configure_kernel_from_x_ref'] + detector = MMDDriftKeops [kwargs.pop(k, None) for k in pop_kwargs] if kernel is None: @@ -92,14 +105,7 @@ def __init__( from alibi_detect.utils.keops.kernels import GaussianRBF # type: ignore kwargs.update({'kernel': GaussianRBF}) - if backend == 'tensorflow' and has_tensorflow: - kwargs.pop('device', None) - self._detector = MMDDriftTF(*args, **kwargs) # type: ignore - elif backend == 'pytorch' and has_pytorch: - self._detector = MMDDriftTorch(*args, **kwargs) # type: ignore - else: - kwargs.pop('configure_kernel_from_x_ref', None) - self._detector = MMDDriftKeops(*args, **kwargs) # type: ignore + self._detector = detector(*args, **kwargs) # type: ignore self.meta = self._detector.meta def predict(self, x: Union[np.ndarray, list], return_p_val: bool = True, return_distance: bool = True) \ From cf528f701ec89302ecadd85f5ff58a595d1b35c9 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Mon, 4 Jul 2022 14:30:21 +0100 Subject: [PATCH 06/50] remove learned kernel and base kernel_matrix MMD function --- alibi_detect/cd/base.py | 6 ---- alibi_detect/utils/keops/kernels.py | 45 ----------------------------- 2 files changed, 51 deletions(-) diff --git a/alibi_detect/cd/base.py b/alibi_detect/cd/base.py index b5ff7ed88..9dfad33a5 100644 --- a/alibi_detect/cd/base.py +++ b/alibi_detect/cd/base.py @@ -548,12 +548,6 @@ def preprocess(self, x: Union[np.ndarray, list]) -> Tuple[np.ndarray, np.ndarray else: return self.x_ref, x # type: ignore[return-value] - # TODO: not absolutely required for keops...?! - @abstractmethod - def kernel_matrix(self, x: Union['torch.Tensor', 'tf.Tensor'], y: Union['torch.Tensor', 'tf.Tensor']) \ - -> Union['torch.Tensor', 'tf.Tensor']: - pass - @abstractmethod def score(self, x: Union[np.ndarray, list]) -> Tuple[float, float, float]: pass diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index dc2777d4f..f1f563d8b 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -63,48 +63,3 @@ def forward(self, x: LazyTensor, y: LazyTensor, infer_sigma: bool = False) -> La if len(dist.shape) < len(gamma.shape): kernel_mat = kernel_mat.sum(-1) / len(self.sigma) return kernel_mat - - -class DeepKernelKeops(nn.Module): - def __init__( - self, - proj: nn.Module, - kernel_a: nn.Module = GaussianRBFKeops(trainable=True), - kernel_b: nn.Module = GaussianRBFKeops(trainable=True), - eps: Union[float, str] = 'trainable' - ) -> None: - super().__init__() - - self.kernel_a = kernel_a - self.kernel_b = kernel_b - self.proj = proj - if kernel_b is not None: - self._init_eps(eps) - - def _init_eps(self, eps: Union[float, str]) -> None: - if isinstance(eps, float): - if not 0 < eps < 1: - raise ValueError("eps should be in (0,1)") - self.logit_eps = nn.Parameter(torch.tensor(eps).logit(), requires_grad=False) - elif eps == 'trainable': - self.logit_eps = nn.Parameter(torch.tensor(0.)) - else: - raise NotImplementedError("eps should be 'trainable' or a float in (0,1)") - - @property - def eps(self) -> torch.Tensor: - return self.logit_eps.sigmoid() if self.kernel_b is not None else torch.tensor(0.) - - def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - similarity = self.kernel_a(self.proj(x), self.proj(y)) - if self.kernel_b is not None: - similarity = (1-self.eps)*similarity + self.eps*self.kernel_b(x, y) - return similarity - - # TODO: where does this belong? - def forward_perms(self, x_proj: torch.Tensor, y_proj: torch.Tensor, - x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - similarity = self.kernel_a(x_proj, y_proj, permutations=True) # [perms+1, n, m] - if self.kernel_b is not None: - similarity = (1-self.eps)*similarity + self.eps*self.kernel_b(x, y, permutations=True) - return similarity.sum(1).sum(1).squeeze(-1) # [perms+1] From 38ec19b7cd3f8feb8511247557bc298bab0d738d Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 5 Jul 2022 13:48:50 +0100 Subject: [PATCH 07/50] unify batched mmd2 --- alibi_detect/cd/base.py | 1 + alibi_detect/cd/keops/mmd.py | 38 +----------------------------------- 2 files changed, 2 insertions(+), 37 deletions(-) diff --git a/alibi_detect/cd/base.py b/alibi_detect/cd/base.py index 9dfad33a5..94461e789 100644 --- a/alibi_detect/cd/base.py +++ b/alibi_detect/cd/base.py @@ -502,6 +502,7 @@ def __init__( if p_val is None: logger.warning('No p-value set for the drift threshold. Need to set it to detect data drift.') + # TODO: now not supported by KeOps detector -> either support or move to framework-specific implementations self.infer_sigma = configure_kernel_from_x_ref if configure_kernel_from_x_ref and isinstance(sigma, np.ndarray): self.infer_sigma = False diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index 62d51aeb5..76c600756 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -84,46 +84,10 @@ def __init__( # set the correct MMD^2 function based on the batch size for the permutations self.batch_size = batch_size_permutations self.n_batches = 1 + (n_permutations - 1) // batch_size_permutations - self.mmd2 = self._mmd2 if self.n_batches == 1 else self._batched_mmd2 def _mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) \ -> Tuple[torch.Tensor, torch.Tensor]: """ - Compute MMD^2 for the original test statistic and all permutations at once. - - Parameters - ---------- - x_all - Concatenated reference and test instances. - perms - List with permutation vectors. - m - Number of reference instances. - n - Number of test instances. - - Returns - ------- - MMD^2 statistic for the original and permuted reference and test sets. - """ - x_all = x_all.to(self.device) - - # construct stacked tensors with all permutations for the reference set x and test set y - x = torch.cat([x_all[None, :m, :], torch.cat([x_all[perm[:m]][None, :, :] for perm in perms], 0)], 0) - y = torch.cat([x_all[None, m:, :], torch.cat([x_all[perm[m:]][None, :, :] for perm in perms], 0)], 0) - - # compute summed kernel matrices - c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) - k_xx = self.kernel(LazyTensor(x[:, :, None, :]), LazyTensor(x[:, None, :, :])).sum(1).sum(1).squeeze(-1) - k_yy = self.kernel(LazyTensor(y[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1) - k_xy = self.kernel(LazyTensor(x[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1) - stats = c_xx * (k_xx - m) + c_yy * (k_yy - n) - c_xy * k_xy # TODO: check diagonal adjustment - return stats[0], stats[1:] - - # TODO: just use _batched_mmd2?! Need to check time diff with original approach which should be minimal - def _batched_mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) \ - -> Tuple[torch.Tensor, torch.Tensor]: - """ Batched (across the permutations) MMD^2 computation for the original test statistic and the permutations. Parameters @@ -185,7 +149,7 @@ def score(self, x: Union[np.ndarray, list]) -> Tuple[float, float, float]: m, n = x_ref.shape[0], x.shape[0] perms = [torch.randperm(m + n) for _ in range(self.n_permutations)] x_all = torch.cat([x_ref, x], 0) - mmd2, mmd2_permuted = self.mmd2(x_all, perms, m, n) + mmd2, mmd2_permuted = self._mmd2(x_all, perms, m, n) if self.device.type == 'cuda': mmd2, mmd2_permuted = mmd2.cpu(), mmd2_permuted.cpu() p_val = (mmd2 <= mmd2_permuted).float().mean() From e943c6c00ee877765ae4c0ad5492877f7416025c Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Wed, 6 Jul 2022 12:01:42 +0100 Subject: [PATCH 08/50] update keops mmd --- alibi_detect/cd/base.py | 2 -- alibi_detect/cd/keops/mmd.py | 12 +++++++++++ alibi_detect/utils/keops/kernels.py | 32 ++++++++++++++++++++++++++--- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/alibi_detect/cd/base.py b/alibi_detect/cd/base.py index 94461e789..dc9900f2f 100644 --- a/alibi_detect/cd/base.py +++ b/alibi_detect/cd/base.py @@ -502,11 +502,9 @@ def __init__( if p_val is None: logger.warning('No p-value set for the drift threshold. Need to set it to detect data drift.') - # TODO: now not supported by KeOps detector -> either support or move to framework-specific implementations self.infer_sigma = configure_kernel_from_x_ref if configure_kernel_from_x_ref and isinstance(sigma, np.ndarray): self.infer_sigma = False - # TODO: this might print a message for keops despite not existing configure_kernel_from_x_ref logger.warning('`sigma` is specified for the kernel and `configure_kernel_from_x_ref` ' 'is set to True. `sigma` argument takes priority over ' '`configure_kernel_from_x_ref` (set to False).') diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index 76c600756..1ee73657a 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -20,6 +20,7 @@ def __init__( preprocess_fn: Optional[Callable] = None, kernel: Callable = GaussianRBF, sigma: Optional[np.ndarray] = None, + configure_kernel_from_x_ref: bool = True, n_permutations: int = 100, batch_size_permutations: int = 1000000, device: Optional[str] = None, @@ -48,6 +49,8 @@ def __init__( sigma Optionally set the GaussianRBF kernel bandwidth. Can also pass multiple bandwidth values as an array. The kernel evaluation is then averaged over those bandwidths. + configure_kernel_from_x_ref + Whether to already configure the kernel bandwidth from the reference data. n_permutations Number of permutations used in the permutation test. batch_size_permutations @@ -67,6 +70,7 @@ def __init__( update_x_ref=update_x_ref, preprocess_fn=preprocess_fn, sigma=sigma, + configure_kernel_from_x_ref=configure_kernel_from_x_ref, n_permutations=n_permutations, input_shape=input_shape, data_type=data_type @@ -85,6 +89,14 @@ def __init__( self.batch_size = batch_size_permutations self.n_batches = 1 + (n_permutations - 1) // batch_size_permutations + # infer the kernel bandwidth from the reference data + if self.infer_sigma or isinstance(sigma, torch.Tensor): + x = torch.from_numpy(self.x_ref).to(self.device) + _ = self.kernel(LazyTensor(x[:, None, :]), LazyTensor(x[None, :, :]), infer_sigma=self.infer_sigma) + self.infer_sigma = False + else: + self.infer_sigma = True + def _mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) \ -> Tuple[torch.Tensor, torch.Tensor]: """ diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index f1f563d8b..ab0d136f6 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -1,4 +1,3 @@ -from alibi_detect.utils.pytorch.kernels import sigma_median # TODO: keops sigma_median? import numpy as np from pykeops.torch import LazyTensor import torch @@ -6,11 +5,38 @@ from typing import Callable, List, Tuple, Union +def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor) -> torch.Tensor: + """ + Bandwidth estimation using the mean heuristic. + + Parameters + ---------- + x + LazyTensor of instances with dimension [Nx, 1, features]. + y + LazyTensor of instances with dimension [1, Ny, features]. + dist + LazyTensor with dimensions [Nx, Ny], containing the pairwise distances between `x` and `y`. + + Returns + ------- + The computed bandwidth, `sigma`. + """ + n = x.shape[0] + if (dist.min(axis=1) == 0.).all() and (torch.arange(n) == dist.argmin(axis=1).view(-1)).all() \ + and x.shape == y.shape: + n_mean = n * (n - 1) + else: + n_mean = np.prod(dist.shape) + sigma = (.5 * dist.sum(1).sum().unsqueeze(-1) / n_mean) ** .5 + return sigma + + class GaussianRBF(nn.Module): def __init__( self, sigma: torch.Tensor = None, - init_sigma_fn: Callable = sigma_median, # TODO: would not work with default init fn + init_sigma_fn: Callable = sigma_mean, trainable: bool = False ) -> None: """ @@ -25,7 +51,7 @@ def __init__( Can pass multiple values to eval kernel with and then average. init_sigma_fn Function used to compute the bandwidth `sigma`. Used when `sigma` is to be inferred. - The function's signature should match :py:func:`~alibi_detect.utils.keops.kernels.sigma_median`, + The function's signature should match :py:func:`~alibi_detect.utils.keops.kernels.sigma_mean`, meaning that it should take in the tensors `x`, `y` and `dist` and return `sigma`. trainable Whether or not to track gradients w.r.t. `sigma` to allow it to be trained. From 0487cb2ce966aea68ac838c7434cd297c2b22f7e Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Wed, 6 Jul 2022 14:20:19 +0100 Subject: [PATCH 09/50] update docs and kernel import --- alibi_detect/utils/keops/__init__.py | 5 +++++ doc/source/cd/methods/mmddrift.ipynb | 33 +++++++++++++++------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/alibi_detect/utils/keops/__init__.py b/alibi_detect/utils/keops/__init__.py index e69de29bb..235176e6b 100644 --- a/alibi_detect/utils/keops/__init__.py +++ b/alibi_detect/utils/keops/__init__.py @@ -0,0 +1,5 @@ +from .kernels import GaussianRBF + +__all__ = [ + "GaussianRBF" +] diff --git a/doc/source/cd/methods/mmddrift.ipynb b/doc/source/cd/methods/mmddrift.ipynb index 3f9c7e64e..91b59efbe 100644 --- a/doc/source/cd/methods/mmddrift.ipynb +++ b/doc/source/cd/methods/mmddrift.ipynb @@ -44,7 +44,7 @@ "\n", "Keyword arguments:\n", "\n", - "* `backend`: Both **TensorFlow** and **PyTorch** implementations of the MMD detector as well as various preprocessing steps are available. Specify the backend (*tensorflow* or *pytorch*). Defaults to *tensorflow*.\n", + "* `backend`: **TensorFlow**, **PyTorch** and [**KeOps**](https://github.com/getkeops/keops) implementations of the MMD detector as well as various preprocessing steps are available. Specify the backend (*tensorflow*, *pytorch* or *keops*). Defaults to *tensorflow*.\n", "\n", "* `p_val`: p-value used for significance of the permutation test.\n", "\n", @@ -54,11 +54,11 @@ "\n", "* `preprocess_fn`: Function to preprocess the data before computing the data drift metrics. Typically a dimensionality reduction technique.\n", "\n", - "* `kernel`: Kernel used when computing the MMD. Defaults to a Gaussian RBF kernel (`from alibi_detect.utils.pytorch import GaussianRBF` or `from alibi_detect.utils.tensorflow import GaussianRBF` dependent on the backend used).\n", + "* `kernel`: Kernel used when computing the MMD. Defaults to a Gaussian RBF kernel (`from alibi_detect.utils.pytorch import GaussianRBF`, `from alibi_detect.utils.tensorflow import GaussianRBF` or `from alibi_detect.utils.keops import GaussianRBF` dependent on the backend used).\n", "\n", "* `sigma`: Optional bandwidth for the kernel as a `np.ndarray`. We can also average over a number of different bandwidths, e.g. `np.array([.5, 1., 1.5])`.\n", "\n", - "* `configure_kernel_from_x_ref`: If `sigma` is not specified, the detector can infer it via a heuristic and set `sigma` to the median pairwise distance between 2 samples. If `configure_kernel_from_x_ref` is *True*, we can already set `sigma` at initialization of the detector by inferring it from `x_ref`, speeding up the prediction step. If set to *False*, `sigma` is computed separately for each test batch at prediction time.\n", + "* `configure_kernel_from_x_ref`: If `sigma` is not specified, the detector can infer it via a heuristic and set `sigma` to the median (*TensorFlow* and *PyTorch*) or the mean pairwise distance between 2 samples (*KeOps*) by default. If `configure_kernel_from_x_ref` is *True*, we can already set `sigma` at initialization of the detector by inferring it from `x_ref`, speeding up the prediction step. If set to *False*, `sigma` is computed separately for each test batch at prediction time.\n", "\n", "* `n_permutations`: Number of permutations used in the permutation test.\n", "\n", @@ -71,23 +71,22 @@ "\n", "* `device`: *cuda* or *gpu* to use the GPU and *cpu* for the CPU. If the device is not specified, the detector will try to leverage the GPU if possible and otherwise fall back on CPU.\n", "\n", + "Additional KeOps keyword arguments:\n", "\n", - "Initialized drift detector example:\n", + "* `batch_size_permutations`: KeOps computes the `n_permutations` of the MMD^2 statistics in chunks of `batch_size_permutations`. Defaults to 1,000,000.\n", + "\n", + "Initialized drift detector examples for each of the available backends:\n", "\n", "\n", "```python\n", "from alibi_detect.cd import MMDDrift\n", "\n", - "cd = MMDDrift(x_ref, backend='tensorflow', p_val=.05)\n", + "cd_tf = MMDDrift(x_ref, backend='tensorflow', p_val=.05)\n", + "cd_torch = MMDDrift(x_ref, backend='pytorch', p_val=.05)\n", + "cd_keops = MMDDrift(x_ref, backend='keops', p_val=.05)\n", "```\n", "\n", - "The same detector in PyTorch:\n", - "\n", - "```python\n", - "cd = MMDDrift(x_ref, backend='pytorch', p_val=.05)\n", - "```\n", - "\n", - "We can also easily add preprocessing functions for both frameworks. The following example uses a randomly initialized image encoder in PyTorch:\n", + "We can also easily add preprocessing functions for the *TensorFlow * and *PyTorch* frameworks. The following example uses a randomly initialized image encoder in PyTorch:\n", "\n", "```python\n", "from functools import partial\n", @@ -196,7 +195,7 @@ "cd = load_detector(filepath)\n", "```\n", "\n", - "Currently on the **TensorFlow** backend is supported for `save_detector` and `load_detector`. Adding **PyTorch** support is a near term priority." + "Currently on the **TensorFlow** backend is supported for `save_detector` and `load_detector`. Adding **PyTorch** and **KeOps** support is a near term priority." ] }, { @@ -213,6 +212,10 @@ "\n", "[Drift detection on CIFAR10](../../examples/cd_mmd_cifar10.ipynb)\n", "\n", + "### Tabular\n", + "\n", + "[Scaling up drift detection with KeOps](../../examples/cd_mmd_keops.ipynb)\n", + "\n", "### Text\n", "\n", "[Text drift detection on IMDB movie reviews](../../examples/cd_text_imdb.ipynb)" @@ -224,7 +227,7 @@ "hash": "ffba93b5284319fb7a107c8eacae647f441487dcc7e0323a4c0d3feb66ea8c5e" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -238,7 +241,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.11" + "version": "3.7.6" } }, "nbformat": 4, From a6a4641b3a44ece473216377004f1b6f119ca138 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Thu, 7 Jul 2022 15:50:19 +0100 Subject: [PATCH 10/50] bugfixes --- alibi_detect/cd/keops/mmd.py | 4 ++-- alibi_detect/cd/mmd.py | 4 ++-- alibi_detect/cd/pytorch/mmd.py | 2 +- alibi_detect/utils/frameworks.py | 2 +- alibi_detect/utils/keops/kernels.py | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index 1ee73657a..3c32572d7 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -2,7 +2,7 @@ import numpy as np from pykeops.torch import LazyTensor import torch -from typing import Callable, Dict, Optional, Tuple, Union +from typing import Callable, Dict, List, Optional, Tuple, Union from alibi_detect.cd.base import BaseMMDDrift from alibi_detect.utils.keops.kernels import GaussianRBF from alibi_detect.utils.pytorch import get_device @@ -83,7 +83,7 @@ def __init__( # initialize kernel sigma = torch.from_numpy(sigma).to(self.device) if isinstance(sigma, # type: ignore[assignment] np.ndarray) else None - self.kernel = kernel(sigma) if kernel == GaussianRBF else kernel + self.kernel = kernel(sigma).to(self.device) if kernel == GaussianRBF else kernel # set the correct MMD^2 function based on the batch size for the permutations self.batch_size = batch_size_permutations diff --git a/alibi_detect/cd/mmd.py b/alibi_detect/cd/mmd.py index 336cf9be0..565edf8b1 100644 --- a/alibi_detect/cd/mmd.py +++ b/alibi_detect/cd/mmd.py @@ -4,7 +4,7 @@ from alibi_detect.utils.frameworks import has_keops, has_pytorch, has_tensorflow if has_keops: - from alibi_detect.cd.keops import MMDDriftKeops + from alibi_detect.cd.keops.mmd import MMDDriftKeops if has_pytorch: from alibi_detect.cd.pytorch.mmd import MMDDriftTorch @@ -80,7 +80,7 @@ def __init__( raise ImportError(f'{backend} not installed. Cannot initialize and run the ' f'MMDDrift detector with {backend} backend.') elif backend not in ['tensorflow', 'pytorch', 'keops']: - raise NotImplementedError(f'{backend} not implemented. Use tensorflow or pytorch instead.') + raise NotImplementedError(f'{backend} not implemented. Use tensorflow, pytorch or keops instead.') kwargs = locals() args = [kwargs['x_ref']] diff --git a/alibi_detect/cd/pytorch/mmd.py b/alibi_detect/cd/pytorch/mmd.py index 1bb6ffe0b..c0ad58dcd 100644 --- a/alibi_detect/cd/pytorch/mmd.py +++ b/alibi_detect/cd/pytorch/mmd.py @@ -80,7 +80,7 @@ def __init__( # initialize kernel sigma = torch.from_numpy(sigma).to(self.device) if isinstance(sigma, # type: ignore[assignment] np.ndarray) else None - self.kernel = kernel(sigma) if kernel == GaussianRBF else kernel + self.kernel = kernel(sigma).to(self.device) if kernel == GaussianRBF else kernel # compute kernel matrix for the reference data if self.infer_sigma or isinstance(sigma, torch.Tensor): diff --git a/alibi_detect/utils/frameworks.py b/alibi_detect/utils/frameworks.py index f742618b1..7899ef748 100644 --- a/alibi_detect/utils/frameworks.py +++ b/alibi_detect/utils/frameworks.py @@ -17,7 +17,7 @@ has_sklearn = False try: - import keops # noqa + import pykeops # noqa has_keops = True except ImportError: has_keops = False diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index ab0d136f6..cedafd709 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -2,7 +2,7 @@ from pykeops.torch import LazyTensor import torch import torch.nn as nn -from typing import Callable, List, Tuple, Union +from typing import Callable, List, Optional, Tuple, Union def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor) -> torch.Tensor: @@ -23,7 +23,7 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor) -> torch.Tensor: The computed bandwidth, `sigma`. """ n = x.shape[0] - if (dist.min(axis=1) == 0.).all() and (torch.arange(n) == dist.argmin(axis=1).view(-1)).all() \ + if (dist.min(axis=1) == 0.).all() and (torch.arange(n) == dist.argmin(axis=1).cpu().view(-1)).all() \ and x.shape == y.shape: n_mean = n * (n - 1) else: @@ -35,7 +35,7 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor) -> torch.Tensor: class GaussianRBF(nn.Module): def __init__( self, - sigma: torch.Tensor = None, + sigma: Optional[torch.Tensor] = None, init_sigma_fn: Callable = sigma_mean, trainable: bool = False ) -> None: @@ -78,7 +78,7 @@ def forward(self, x: LazyTensor, y: LazyTensor, infer_sigma: bool = False) -> La if infer_sigma or self.init_required: if self.trainable and infer_sigma: raise ValueError("Gradients cannot be computed w.r.t. an inferred sigma value") - sigma = self.init_sigma_fn(x, y, dist) + sigma = self.init_sigma_fn(x, y, dist) # .to(x.device) with torch.no_grad(): self.log_sigma.copy_(sigma.log().clone()) self.init_required = False From e2b27f54a4e24ec0a0f30038dfb5d7bf0ff347bd Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 8 Jul 2022 14:38:50 +0100 Subject: [PATCH 11/50] remove unused imports --- alibi_detect/utils/keops/kernels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index cedafd709..2f471841a 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -2,7 +2,7 @@ from pykeops.torch import LazyTensor import torch import torch.nn as nn -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, Optional def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor) -> torch.Tensor: From d442be8af22481c8ae6ed2ed5dc305d7f7e5d99b Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 8 Jul 2022 14:50:24 +0100 Subject: [PATCH 12/50] add benchmarking example --- doc/source/examples/cd_mmd_keops.ipynb | 513 +++++++++++++++++++++++++ 1 file changed, 513 insertions(+) create mode 100644 doc/source/examples/cd_mmd_keops.ipynb diff --git a/doc/source/examples/cd_mmd_keops.ipynb b/doc/source/examples/cd_mmd_keops.ipynb new file mode 100644 index 000000000..280170883 --- /dev/null +++ b/doc/source/examples/cd_mmd_keops.ipynb @@ -0,0 +1,513 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "27a4394b", + "metadata": {}, + "source": [ + "# Scaling up drift detection with KeOps\n", + "\n", + "## Introduction\n", + "\n", + "A number of convenient and powerful kernel-based drift detectors such as the MMD detector ([Gretton et al., 2012](https://jmlr.csail.mit.edu/papers/v13/gretton12a.html)) do not scale favourably with increasing dataset size $n$, leading to quadratic complexity $\\mathcal{O}(n^2)$ for naive implementations. As a result, we can quickly run into memory issues by having to store the $[N_\\text{ref} + N_\\text{test}, N_\\text{ref} + N_\\text{test}]$ kernel matrix (on the GPU if applicable) used for an efficient implementation of the permutation test. Note that $N_\\text{ref}$ is the reference data size and $N_\\text{test}$ the test data size.\n", + "\n", + "We can however drastically speed up and scale up kernel-based drift detectors to large dataset sizes by working with symbolic kernel matrices instead and leverage the [KeOps](https://www.kernel-operations.io/keops/index.html) library to do so. For the user of $\\texttt{Alibi Detect}$ the only thing that changes is the specification of the detector's backend:\n", + "\n", + "\n", + "```python\n", + "from alibi_detect.cd import MMDDrift\n", + "\n", + "detector_torch = MMDDrift(x_ref, backend='pytorch')\n", + "detector_keops = MMDDrift(x_ref, backend='keops')\n", + "```\n", + "\n", + "In this notebook we will run a few simple benchmarks to illustrate the speed and memory improvements from using KeOps over vanilla PyTorch on the GPU (1x RTX 2080 Ti).\n", + "\n", + "## Data\n", + "\n", + "We randomly sample points from the standard normal distribution and run the MMD detectors with PyTorch and KeOps backends for the following settings:\n", + "\n", + "- $N_\\text{ref}, N_\\text{test} = [2, 5, 10, 20, 50, 100]$ (batch sizes in '000s)\n", + "- $D = [2, 10, 50]$\n", + "\n", + "Where $D$ denotes the number of features.\n", + "\n", + "## Requirements\n", + "\n", + "The notebook requires [PyTorch](https://pytorch.org/) and KeOps to be installed. Once PyTorch is installed, KeOps can be installed via pip:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0bf1719", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install pykeops" + ] + }, + { + "cell_type": "markdown", + "id": "7ff93d59", + "metadata": {}, + "source": [ + "Before we start let’s fix the random seeds for reproducibility:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "2ba95f29", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "\n", + "def set_seed(seed: int) -> None:\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed(seed)\n", + " np.random.seed(seed)\n", + "\n", + "set_seed(2022)" + ] + }, + { + "cell_type": "markdown", + "id": "1910895a", + "metadata": {}, + "source": [ + "\n", + "## Vanilla PyTorch vs. KeOps comparison\n", + "\n", + "### Experiments\n", + "\n", + "First we define some utility functions to run the experiments:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a1c65254", + "metadata": {}, + "outputs": [], + "source": [ + "from alibi_detect.cd import MMDDrift\n", + "import matplotlib.pyplot as plt\n", + "from scipy.stats import kstest\n", + "from timeit import default_timer as timer\n", + "\n", + "\n", + "def eval_detector(p_vals: np.ndarray, threshold: float, is_drift: bool, t_mean: float, t_std: float) -> dict:\n", + " \"\"\" In case of drifted data (ground truth) it returns the detector's power.\n", + " In case of no drift, it computes the false positive rate (FPR) and whether the p-values\n", + " are uniformly distributed U[0,1] which is checked via a KS test. \"\"\"\n", + " results = {'power': None, 'fpr': None, 'ks': None}\n", + " below_p_val_threshold = (p_vals <= threshold).mean()\n", + " if is_drift:\n", + " results['power'] = below_p_val_threshold\n", + " else:\n", + " results['fpr'] = below_p_val_threshold\n", + " stat_ks, p_val_ks = kstest(p_vals, 'uniform')\n", + " results['ks'] = {'p_val': p_val_ks, 'stat': stat_ks}\n", + " results['p_vals'] = p_vals\n", + " results['time'] = {'mean': t_mean, 'stdev': t_std}\n", + " return results\n", + "\n", + "\n", + "def experiment(backend: str, n_runs: int, n_ref: int, n_test: int, n_features: int, mu: float = 0.) -> dict:\n", + " \"\"\" Runs the experiment n_runs times, each time with newly sampled reference and test data.\n", + " Returns the p-values for each test as well as the mean and standard deviations of the runtimes. \"\"\"\n", + " p_vals, t_detect = [], []\n", + " for _ in range(n_runs):\n", + " # Sample reference and test data\n", + " x_ref = np.random.randn(*(n_ref, n_features)).astype(np.float32)\n", + " x_test = np.random.randn(*(n_test, n_features)).astype(np.float32) + mu\n", + " \n", + " # Initialise detector, make and log predictions\n", + " p_val = .05\n", + " dd = MMDDrift(x_ref, backend=backend, p_val=p_val, n_permutations=100)\n", + " start = timer()\n", + " pred = dd.predict(x_test)\n", + " end = timer()\n", + " \n", + " if _ > 0: # first run reserved for KeOps compilation\n", + " t_detect.append(end - start)\n", + " p_vals.append(pred['data']['p_val'])\n", + " \n", + " del dd, x_ref, x_test\n", + " torch.cuda.empty_cache()\n", + " \n", + " p_vals = np.array(p_vals)\n", + " t_mean, t_std = np.array(t_detect).mean(), np.array(t_detect).std()\n", + " results = eval_detector(p_vals, p_val, mu == 0., t_mean, t_std)\n", + " return results\n", + "\n", + "\n", + "def format_results(n_features: list, backends: list, max_batch_size: int = 1e10) -> dict:\n", + " T = {'batch_size': None, 'keops': None, 'pytorch': None}\n", + " T['batch_size'] = np.unique([experiments['keops'][_]['n_ref'] for _ in experiments['keops'].keys()])\n", + " T['batch_size'] = list(T['batch_size'][T['batch_size'] <= max_batch_size])\n", + " T['keops'] = {f: [] for f in n_features}\n", + " T['pytorch'] = {f: [] for f in n_features}\n", + "\n", + " for backend in backends:\n", + " for f in T[backend].keys():\n", + " for bs in T['batch_size']:\n", + " for k, v in experiments[backend].items():\n", + " if f == v['n_features'] and bs == v['n_ref']:\n", + " T[backend][f].append(results[backend][k]['time']['mean'])\n", + "\n", + " for k, v in T['keops'].items(): # apply padding\n", + " n_pad = len(v) - len(T['pytorch'][k])\n", + " T['pytorch'][k] += [np.nan for _ in range(n_pad)]\n", + " return T\n", + "\n", + "\n", + "def plot_absolute_time(results: dict, n_features: list, y_scale: str = 'linear', \n", + " detector: str = 'MMD', max_batch_size: int = 1e10):\n", + " T = format_results(n_features, ['keops', 'pytorch'], max_batch_size)\n", + " colors = ['b', 'g', 'r', 'c', 'm', 'y', 'b']\n", + " legend, n_c = [], 0\n", + " for f in n_features:\n", + " plt.plot(T['batch_size'], T['keops'][f], linestyle='solid', color=colors[n_c]);\n", + " legend.append(f'keops - {f}')\n", + " plt.plot(T['batch_size'], T['pytorch'][f], linestyle='dashed', color=colors[n_c]);\n", + " legend.append(f'pytorch - {f}')\n", + " n_c += 1\n", + " plt.title(f'{detector} drift detection time for 100 permutations')\n", + " plt.legend(legend, loc=(1.1,.1));\n", + " plt.xlabel('Batch size');\n", + " plt.ylabel('Time (s)');\n", + " plt.yscale(y_scale);\n", + " plt.show();\n", + "\n", + "\n", + "def plot_relative_time(results: dict, n_features: list, y_scale: str = 'linear',\n", + " detector: str = 'MMD', max_batch_size: int = 1e10):\n", + " T = format_results(n_features, ['keops', 'pytorch'], max_batch_size)\n", + " colors = ['b', 'g', 'r', 'c', 'm', 'y', 'b']\n", + " legend, n_c = [], 0\n", + " for f in n_features:\n", + " t_keops, t_torch = T['keops'][f], T['pytorch'][f]\n", + " ratio = [tt / tk for tt, tk in zip(t_torch, t_keops)]\n", + " plt.plot(T['batch_size'], ratio, linestyle='solid', color=colors[n_c]);\n", + " legend.append(f'pytorch/keops - {f}')\n", + " n_c += 1\n", + " plt.title(f'{detector} drift detection pytorch/keops time ratio for 100 permutations')\n", + " plt.legend(legend, loc=(1.1,.1));\n", + " plt.xlabel('Batch size');\n", + " plt.ylabel('time pytorch / keops');\n", + " plt.yscale(y_scale);\n", + " plt.show();" + ] + }, + { + "cell_type": "markdown", + "id": "43a4ee7e", + "metadata": {}, + "source": [ + "As detailed earlier, we will compare the PyTorch with the KeOps implementation of the MMD detector for a variety of reference and test data batch sizes as well as different feature dimensions. Note that for the PyTorch implementation, the portion of the kernel matrix for the reference data itself can already be computed at initialisation of the detector. This computation will not be included when we record the detector's prediction time. Since use cases where $N_\\text{ref} >> N_\\text{test}$ are quite common, we will also test for this specific setting. The key reason is that we cannot amortise this computation for the KeOps detector since we are working with lazily evaluated symbolic matrices.\n", + "\n", + "#### $N_\\text{ref} = N_\\text{test}$\n", + "\n", + "Note that for KeOps we could further increase the number of instances in the reference and test sets (e.g. to 500,000) without running into memory issues." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "47268603", + "metadata": {}, + "outputs": [], + "source": [ + "experiments = {\n", + " 'keops': {\n", + " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 2},\n", + " 3: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 10, 'n_features': 2},\n", + " 4: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 10, 'n_features': 2},\n", + " 5: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 10, 'n_features': 2},\n", + " 6: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 10},\n", + " 7: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 10},\n", + " 8: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 10},\n", + " 9: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 10, 'n_features': 10},\n", + " 10: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 10, 'n_features': 10},\n", + " 11: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 10, 'n_features': 10},\n", + " 12: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 50},\n", + " 13: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 50},\n", + " 14: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 50},\n", + " 15: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 10, 'n_features': 50},\n", + " 16: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 10, 'n_features': 50},\n", + " 17: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 10, 'n_features': 50}\n", + " },\n", + " 'pytorch': { # runs OOM after 10k instances in ref and test sets\n", + " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 2},\n", + " 3: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 10},\n", + " 4: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 10},\n", + " 5: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 10},\n", + " 6: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 50},\n", + " 7: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 50},\n", + " 8: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 50}\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d556296a", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "backends = ['keops', 'pytorch']\n", + "results = {backend: {} for backend in backends}\n", + "\n", + "for backend in backends:\n", + " exps = experiments[backend]\n", + " for i, exp in exps.items():\n", + " results[backend][i] = experiment(\n", + " backend, exp['n_runs'], exp['n_ref'], exp['n_test'], exp['n_features']\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "93396443", + "metadata": {}, + "source": [ + "Below we visualise the runtimes of the different experiments. We can make the following observations:\n", + "\n", + "- The relative **speed** improvements of KeOps over vanilla PyTorch increase with increasing batch size.\n", + "\n", + "- Due to the explicit kernel computation and storage, the PyTorch detector runs out-of-memory after a little over 10,000 instances in each of the reference and test sets while KeOps keeps **scaling** up without any issues.\n", + "\n", + "- The relative speed improvements decline with growing **feature dimension**. Note however that we would not recommend using a (untrained) MMD detector on very high-dimensional data in the first place.\n", + "\n", + "The plots show both the absolute and relative (PyTorch / KeOps) mean prediction times for the MMD drift detector for different feature dimensions $[2, 10, 50]$." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "5d854bfb", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEWCAYAAAAQHy/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeVyVVf7A8c9hRwEVQYIUrybLRcxcsrQ0DWUcRdMcydFwLbXyl2mbM9OijVMzU5jpVC5l5pSNaZmO2qZh6Vg52qLIMi7hkoigoriy3PP743mwK4Iscbks3/frdV/c+6zfe7lwvs855zlHaa0RQgghRMPi4uwAhBBCCFHzJAEQQgghGiBJAIQQQogGSBIAIYQQogGSBEAIIYRogCQBEEIIIRogSQDqCKWUVkq1u8b6BUqpp+1eP6CUylJKnVVKNa/C+SzmOd2qGnN1Md9D2xo4T6h5LtcaOFeQUuorpVSeUirR0ecTFaeU+qNS6g1nxyGEo0kCUIJSKkMpla+UCiix/HuzQLSYr5ear+8qsd3L5vKx5uuxSqkis2A5q5T6SSn1llIqvDrj1lpP1lr/2TynOzAHiNVa+2itT5SXQPwa5nvcWk3H2qyUus9+mfkeDlTH8UucK0Mp1dfuPIfMcxVV97lKMRHIAfy01o/+2oMppYKVUmuVUkftv6d26z2VUkuUUmeUUseUUtNLrI9RSqUppc4rpZKUUq1/bUzOoJSaqZR6pxLb91ZKHbFfprV+Xmt9X1n7CFFfSAJQup+A3xe/UEp1ABqVst3/gNF227kB8cD+Ett9rbX2AZoAfYELwE6lVHR1BFvKFWsQ4AXsqY7jC4doDaToKozEVUatjA34BBhWxm4zgTDzvH2AJ5RS/c3jBQAfAk8D/sAOYEVl46qs2lC7JESDprWWh90DyACeAv5rt+wl4E+ABizmsqXm8iygmbksDvgY2AqMNZeNBbaWcp51wKprxPE4kAkcBcab525nd+7XgQ3AOYykYikwGwg3l2ngLPAF8JX5+py57J5Szudqvp8c4ADwkLmPm7m+CfCmGdPP5rlcAStwESgyj51rbu9pHu+Q+RktALztzncX8ANwBiNh6g/8xTzORfNY/zC3tX/vTYBlQDZw0Pxdudh/1uZ5T2Ekcr8t4/P9J0ahecE81xOApcR73my+z23mNv8GmgPvmnH/t/j7YG4fCXwOnATSgfgyzr0UKADyzeP2NT+vuebv+6j53NPcvjdwBHgSOAb88xrfGzfsvqd2y49i1AgVv/4z8C/z+URgm926xubnEnmNv5E/ACnm5/wW4GW3Ps783eaan92NJfZ9EtgFXDLjzcD4vu/C+I6+iZHEfgzkARv55W+sN3CklHj6YnyH8s3P9izwo7l+HJBqHusAMKnE+7SZ258FQjCSpXfsjj8YI5nONb8T1hLnfsyM/TRG4uRlrgvA+DvPNb8TWzC/q/KQR214OD2A2vaw+2eSjlG4uZr/fFtzdQIwG1gEPGAuex+j5qAiCcB4IKuMGPpjFJrR5j+p5VydAJwGbsOoxfEqjsdcb8GuIDOXXd6/jHNOBtKAVhhXgUlcWRiuBhaa8bQAttv9I73qPQIvA2vNY/liFJ4vmOu6mfH3M+O/HrOwMf/B3lfiWPbvfRmwxjymBaMWZoJdHAXA/ebv7QGMgk9d63dt9/qKz82MZR9wA0bikWKery9GwbUMeMvctjFwGKOwcQM6YSRTUWWc+/Lvy3z9HPCN+dkGYhScfzbX9QYKgb9hJArepR3T3PaqBABoZi4Lslv2O2C3+fwV4PUSx0kGhl3jc0u2+678h1++e52A48At5u9gjLm9p92+P5j7etst+waj0L/e3P8781heGEnss3afRakJgPl8JnaFt7lsoPk7VMAdwHmg8zWOd/kY/JJQ9wPcMRLFfYCH3bm3YyQO/hiJxmRz3QsYia+7+ehJGd9FecjDGQ9pAijbPzGq9/th/FH/XMZ2y4DRSqmmGP9cPqrg8Y9i/MMoTTxGwZKstT6H8Q+ppDVa6/9orW1a64sVPOe1xANztdaHtdYnMf55AUaHNWAA8IjW+pzW+jhGAT+itAMppRTGVeU0rfVJrXUe8Lzd9hOAJVrrz834f9Zap5UXoNnUMQL4g9Y6T2udASQCCXabHdRaL9ZGO/7bQDBGwVJVb2mt92utT2Ncke7XWm/UWhcCKzEKKTCuejO01m9prQu11t8DHwDDK3ieUcBzWuvjWutsYFaJ92XDKAQvaa0vVPI9+Jg/T9stO42RRBWvP82V7NeX5h9235W/8EuT2URgodb6W611kdb6bYwr/Vvt9p1n7mv/PuZrrbO01j9jXCl/q7X+3vxur+aXz7nStNbrzd+h1lp/CXyGURhXxD3AevO7WoBRu+QN9Cjxfo6an8W/gZvM5QUY37/WWusCrfUWrbVMviJqDWmDK9s/MarO22AU8qXSWm9VSgViNBGs01pfMMq/cl2PUS1YmhBgp93rg6Vsc7giJ6mEkBLHtD9na4wrmEy79+ZyjRgCMfpM7LTbXmFcEYJx9behCjEGmHHYx3YQ47Msdqz4idb6vHl+H6ouy+75hVJeFx+7NXCLUirXbr0bxveoIkK4+n2F2L3O/hWJ3lnzpx9G80rx8zy79X4l9rFfX5qS35XiWFsDY5RS/2e33oMr30tp35uKfs6VppT6LfAsxtW8C8Z3c3cFd7/i96K1timlDlPGdw6jdqH4vb6Ikbx/Zn4PF2mt/1qFtyCEQ0gNQBm01gcx2pAHYHSQupZ3gEe5RqJQiqEYVzqlycQoJIuFlhZiJc5VEdc652GMq7gArXVT8+GntW5fRiw5GP+029tt30QbHSGLj3dDGXFc633lYFxV2fdQD6Xs2pnyVOdneBj40u79NtXGHQUPVHD/o1z9vo7ava5yrFrrUxi/3452izvySyfRPfbrlFKNMX4/1+pEWvK7UhzrYeAvJT6HRlrr9+xDqto7AYzq+Msdcs1aocCyjq2U8sSoiXkJowmkKUbyqUrbvhRX/F7M2q1WVOA7Z9ZSPaq1bovRj2C6UiqmvP2EqCmSAFzbBOBOsxr+WuZhNBV8da2NlFKuSqk2Sqn5GG2Ps8rY9H1grFIqSinVCOPq5dfKAq51L/37wMNKqZZKqWbAjOIVWutMjGrTRKWUn1LKRSl1g1LqDrtjt1RKeZjb24DFwMtKqRYASqnrlVK/Mbd/Exhn3nrmYq6LLC9Os1r/feAvSilf81a16RgJWFWU95lUxjogXCmVoJRyNx83K6WsFdz/PeAppVSg2Sv/GSr5vpRSXhh9BAA8zdfFlpnHb2Z+1vdj9EMAo4o9Wik1zNznGWBXOc0yD5nfFX+M2q/iuwYWA5OVUrcoQ2Ol1ECl1LWaEyrjf4CXeUx3jE6gnnbrswCLUqr4f5uHuT4bKDRrA2JLbN9cKdWkjPO9Dww0v6vuGIn+JYw+GteklIpTSrUzk4bTGB1cbRV9o0I4miQA12C2G+6owHYntdabrtG+110pdRaj5/hmjOrVm7XWpVZDaq0/xugF/gVGh6MvqhJ/CTOBt5VSuUqp+FLWLwY+BX7E6IBVstZjNMY/0+Ke36sw2jcx49sDHFNK5ZjLnjRj/0YpdQajJ3eE+f62Y3SWexnjH+OX/HKV9QrwO6XUKaXUvFLi/D+Mq8ADGJ0tlwNLKvYRXOUFjEIxVyn1WBWPARhXexgFywiMq8Zj/NJpryJmY9x+twujevo7c1llFN/RAEaHTvs29mcx7rY4iPF5v6i1/sSMPRvj9sG/YPxub6GM/h12lmMkhQfM4842j7UDI7n4h3msfRidM6uF2RfjQeANjKvwcxiddIutNH+eUEp9Z/5eHsYoyE8BIzE6pxYfLw0j+Tpgfg/smyrQWqcD9wLzMWqgBgGDtNb5FQg3DON7fxb4GnhNa51UuXcshOMo6ZMihKgMpVQGxp0aG50dixCi6qQGQAghhGiAJAEQQgghGiBpAhBCCCEaIKkBEEIIIRqgOjEQUEBAgLZYLM4OQwgh6pSdO3fmaK0Dy99SNER1IgGwWCzs2FHu3XhCCCHsKKVKG0VUCECaAIQQQogGSRIAIYQQogGSBEAIIYRogCQBEEIIIRogSQCEEEKIBkgSACGEEKIBkgRACCGEaIAcmgAopaYppfYopZKVUu8ppbyUUm2UUt8qpfYppVYUzyEvhBDCTlYWPPIIXLrk7EhEPeWwBEApdT3GPNxdtdbRgCvGHON/A17WWrfDmJ97gqNiEEKIOunIEejVCxYvhj17nB2NqKcc3QTgBngrpdyARkAmcCewylz/NjDEwTEIIUTdsX8/9OwJx47Bp59C587OjkjUUw5LALTWPwMvAYcwCv7TwE4gV2tdaG52BLi+tP2VUhOVUjuUUjuys7MdFaYQQtQeqanGlf+ZM/DFF3D77c6OSNRjjmwCaAbcBbQBQoDGQP+K7q+1XqS17qq17hoYKHNZCCHque+/Nwp/mw2+/BK6dHF2RKKec2QTQF/gJ611tta6APgQuA1oajYJALQEfnZgDEIIUft9/TX06QPe3vDVVxAd7eyIRAPgyATgEHCrUqqRUkoBMUAKkAT8ztxmDLDGgTEIIUTt9sUX0K8fBAbC1q0QFubsiEQD4cg+AN9idPb7DthtnmsR8CQwXSm1D2gOvOmoGIQQolZbvx4GDACLxbjyDw11dkSiAXErf5Oq01o/CzxbYvEBoJsjzyuEELXeypUwciR07Gj09m/e3NkRiQZGRgIUQoiatnQpjBgBt94KmzZJ4S+cQhIAIYSoSa++CuPGQUwMfPIJNGni7IhEAyUJgBBC1JS//Q2mTIHBg2HtWmjc2NkRiQZMEgAhhHA0reHpp2HGDKPqf9Uq8PJydlSigXNoJ0AhhGjwtIZHH4WXX4YJE2DhQnB1dXZUQkgNgBBCOExREUyaZBT+Dz8MixZJ4S9qDUkAhBDCEQoKYPRoY0a/P/0J5s4FF/mXK2oPaQIQQojqdukS3HMPrFkDL7xgtP0LUctIAiCEENXp/HkYOhQ++wzmzzd6/QtRC0kCIIQQ1eXMGYiLg//8B5YsMe73F6KWkgRACCGqw4kT0L8//PADvPcexMc7OyIhrkkSACGE+LWOHTNm9Nu7F1avNmoBhKjlJAEQQohf49Ah6NsXjh41ZveLiXF2REJUiCQAQghRVfv2GQX+6dNGp78ePZwdkRAVJgmAEEJUxZ49xpV/QQF88QV07uzsiISoFIeNSqGUilBK/WD3OKOUekQp5a+U+lwptdf82cxRMQghhEPs3Al33AFKwVdfSeEv6iSHJQBa63St9U1a65uALsB5YDUwA9iktQ4DNpmvhRCibvjPf+DOO8HHB7ZsgagoZ0ckRJXU1LiUMcB+rfVB4C7gbXP528CQGopBCCF+nY0bITYWrrvOKPxvuMHZEQlRZTWVAIwA3jOfB2mtM83nx4Cg0nZQSk1USu1QSu3Izs6uiRiFEKJs//43DBxoFPpffQWtWjk7IiF+FYcnAEopD2AwsLLkOq21BnRp+2mtF2mtu2qtuwYGBjo4SiGEuIYVK+Duu6FjR9i8GYJKvW4Rok6piRqA3wLfaa2zzNdZSqlgAPPn8RqIQQghqmbJEvj9741b/DZuBH9/Z0ckRLWoiQTg9/xS/Q+wFhhjPh8DrKmBGIQQovLmzYMJE4x2/48/Bj8/Z0ckRLVxaAKglGoM9AM+tFv8V6CfUmov0Nd8LYQQtcvzz8PUqTBkiDGtb6NGzo5IiGrl0IGAtNbngOYllp3AuCtACCFqH63hT3+CF16AkSNh6VJwd3d2VEJUOxkJUAghitlsMG2aUfV///3w+uvg6ursqIRwCEkAhBACoKgIJk40Ov1NmwaJicZIf0LUUzU1DoAQQtReBQUwapRR+D/zjBT+okGQGgAhRMN28SLExxsD/fz97/D4486OSIgaIQmAEKLhOncO7roLNm2CV1+FBx90dkRC1BhJAIQQDdPp08bQvl9/bfT0HzOm3F2EqE8kARBCNDw5OfCb38Du3cYwv7/7nbMjEqLGSQIghGhYMjOhb184cAA++ggGDHB2REI4hSQAQoiG4+BBiImBY8dgwwbo08fZEQnhNJIACCEahv/9z7jyz8szJvW59VZnRySEU0kCIISo/3bvhn79jJH+kpLgppucHZEQTicDAQkh6rcdO6B3b2NI36++ksJfCJMkAEKI+mvLFrjzTmjSxHgeGensiISoNSQBEELUT599ZtzqFxJiXPm3bevsiISoVSQBEELUPx99BIMGQXi4Ufi3bOnsiISodSQBEELUL8uXGwP7dOpkdPhr0cLZEQlRKzk0AVBKNVVKrVJKpSmlUpVS3ZVS/kqpz5VSe82fzRwZgxCiAVm8GO69F3r2hM8/h2by70WIsji6BuAV4BOtdSTQEUgFZgCbtNZhwCbztRBC/Dpz58LEidC/vzHIj6+vsyMSolZzWAKglGoC9ALeBNBa52utc4G7gLfNzd4GhjgqBiFEA6A1zJ4N06bBsGFG+7+3t7OjEqLWc2QNQBsgG3hLKfW9UuoNpVRjIEhrnWlucwwIKm1npdREpdQOpdSO7OxsB4YphKiztIYZM+DppyEhAf71L/DwcHZUQtQJjkwA3IDOwOta607AOUpU92utNaBL21lrvUhr3VVr3TUwMNCBYQoh6iSbDaZMgb//HSZPNqb0dZPBTYWoKEcmAEeAI1rrb83XqzASgiylVDCA+fO4A2MQQtRHhYUwfjy89ho89pjx00VuahKiMhz2F6O1PgYcVkpFmItigBRgLTDGXDYGWOOoGIQQ9VB+PowcCW+/DbNmGTUASjk7KiHqHEfXl/0f8K5SygM4AIzDSDreV0pNAA4C8Q6OQQhRX1y4AMOHw/r18NJL8Oijzo5IiDrLoQmA1voHoGspq2IceV4hRD109iwMHgybN8OCBTBpkrMjEqJOkx4zQojaLzcXBgyA7dth2TJjsB8hxK8iCYAQonbLzobYWNizB1auhKFDnR2REPWCJABCiNrr55+hXz/46SdYu9YY5U8IUS0kARBC1E4ZGRATA8ePwyefwB13ODsiIeoVSQCEELVPerpR+J8/D5s2Qbduzo5IiHpHEgAhRO2ya5dR7Q9Gj/8bb3RqOELUVzJ0lhCi9ti+HXr3Nsbz/+orKfyFcCBJAIQQtcOXXxrV/s2awZYtEBFR/j5CiCqTBEAI4XyffGL08G/Vyij8LRZnRyREvScJgBDCuT780Bjhz2o1agFCQpwdkRANgiQAQgjneecdiI+Hrl3hiy9Apv4WosZIAiCEcI6FC2H0aOP+/s8+g6ZNnR2REA2KJABCiJqXmAiTJxvj+69fDz4+zo5IiAZHEgAhRM3RGmbNgsceM6b1/fBD8PJydlRCNEgyEJAQomZoDY8/blz9jx0Lb7wBrq7OjkqIBsuhCYBSKgPIA4qAQq11V6WUP7ACsAAZQLzW+pQj4xBCOJnNBg89BAsWGD/nzQMXqYAUwplq4i+wj9b6Jq11V/P1DGCT1joM2GS+FkLUV4WFxhX/ggXw5JMwf74U/kLUAs74K7wLeNt8/jYwxAkxCCFqQn4+3HMP/POfMHs2vPACKOXsqIQQOD4B0MBnSqmdSqmJ5rIgrXWm+fwYEFTajkqpiUqpHUqpHdnZ2Q4OUwhR7S5cgCFDjI5+L78Mf/qTFP5C1CKO7gR4u9b6Z6VUC+BzpVSa/UqttVZK6dJ21FovAhYBdO3atdRthBC1VF4eDBpkTOizeDHcd5+zIxJClODQGgCt9c/mz+PAaqAbkKWUCgYwfx53ZAxCiBp26pQxne/WrfDuu1L4C1FLOSwBUEo1Vkr5Fj8HYoFkYC0wxtxsDLDGUTEIIWrY8ePQpw98/z188AH8/vfOjkgIUQZHNgEEAauV0ebnBizXWn+ilPov8L5SagJwEIh3YAxCiJpy5Aj07QuHDsG6dUYtgBCi1nJYAqC1PgB0LGX5CSDGUecVQjjBgQMQEwMnTsCnn0LPns6OSAhRjnITAKVUd+BeoCcQDFzAqMpfD7yjtT7t0AiFELVbaqpx5X/xojGjX9eu5e8jhHC6a/YBUEp9DNwHfAr0x0gAooCnAC9gjVJqsKODFELUUj/8AL16QVERfPmlFP5C1CHl1QAkaK1zSiw7C3xnPhKVUgEOiUwIUbt98w389rfg6wubNkFYmLMjEkJUwjVrAIoLf7NHv4v5PFwpNVgp5W6/jRCiAUlKMqr9AwJgyxYp/IWogyp6G+BXgJdS6nrgMyABWOqooIQQtdiGDTBgAFgsxkA/rVs7OyIhRBVUNAFQWuvzwN3Aa1rr4UB7x4UlhKiVVq0yhveNioLNmyE42NkRCSGqqMIJgHk3wCiM3v8AMpG3EA3J228bE/t062b09g+Q7j9C1GUVTQCmAn8AVmut9yil2gJJjgtLCFGrvPaaMaXvnXca9/k3aeLsiIQQv1KFBgLSWn+F0Q+g+PUB4GFHBSWEqEX+/nd48kkYPBhWrAAvL2dHJISoBuWNA7BYKdWhjHWNlVLjlVKjHBOaEMKptIZnnjEK/xEjjPZ/KfyFqDfKqwF4FXjaTAKSgWyMAYDCAD9gCfCuQyMUQtQ8reHRR+Hll2HCBFi4EFyl248Q9ck1EwCt9Q9AvFLKB+jKL0MBp2qt02sgPiFETSsqggcegMWL4eGHjSTAxaEzhwshnKCifQDOApsdG4oQwukKCozOfsuXwx//CLNngzGjpxBVsnPnzhZubm5vANE4cAp6USobkFxYWHhfly5djpdc6cjpgIUQdcmlS8ZtfmvWwPPPwx/+4OyIRD3g5ub2xnXXXWcNDAw85eLiop0dT0Nis9lUdnZ21LFjx94Arpq3R7IxIQScP2/08l+zBubNk8JfVKfowMDAM1L41zwXFxcdGBh4GqP25SqVqgFQSjUyRwQUQtQXZ85AXBz85z/w5pswfryzIxL1i4sU/s5jfvalXuxXqAZAKdVDKZUCpJmvOyqlXqvgvq5Kqe+VUuvM122UUt8qpfYppVYopTwq9jaEENXu5EljUp+vvzba/aXwF6LBqGgTwMvAb4ATAFrrH4FeFdx3KpBq9/pvwMta63bAKWBCBY8jhKhOWVnQuzfs2gUffmi0/wtRz6Snp3uEhYXV2rlrBg8e3MZisUSHhYW1Hz58uOXSpUs11uu2wn0AtNaHSywqKm8fpVRLYCDwhvlaAXcCq8xN3gaGVDQGIUQ1OXwYevaE/fth/XoYNMjZEQnRII0aNerkgQMHktPT0/dcvHhRzZ07t8Ym2ahoAnBYKdUD0Eopd6XUY1x5VV+WucATGLciADQHcrXWhebrI8D1pe2olJqolNqhlNqRnZ1dwTCFEOXat88o/LOy4LPPICbG2REJUSNSUlI8rFZr1JdfftmosLCQSZMmtYyOjraGh4dHvfjiiwEANpuNSZMmtQwLC2sfHh4etXjx4mYA69at8+3atWtE796921ksluiRI0eGFhUVUVhYyLBhwyzF28+aNatFZWK65557Tru4uODi4kLXrl3PHTlypMaaxSvaCXAy8ApGYf0z8Bnw0LV2UErFAce11juVUr0rG5jWehGwCKBr167SgUSI6pCSYrT55+dDUhJ07uzsiEQDMn48rZKTaVSdx4yO5vySJZSsob7Kjz/+6DlixIgblixZ8lP37t0vvPTSSwFNmjQpSk5OTr1w4YK6+eabIwcNGnTmm2++abR7927v1NTUPZmZmW7dunWzxsbGngXYvXt34++//z45PDw8v1evXmHLli1r1q5du0uZmZnue/fu3QOQk5NTpSEzL126pFasWNF8zpw55b6X6lLRgYByMKYCrozbgMFKqQEYwwf7YSQRTZVSbmYtQEuMhEII4WjffQexseDhAV9+Ce1rbbOoENXq5MmTbkOGDGm3atWq/V26dLkIsHHjRr+0tLRGa9eubQaQl5fnmpKS4rVlyxbf+Pj4k25ubrRq1arwlltuObt169ZGTZo0sXXo0OFcVFRUPkB8fPzJLVu2+MTFxZ05fPiw55gxY1oNGjTo9NChQ89UJcYxY8aE3nrrrWf79+9/tvre+bVVKAFQSrUB/g+w2O+jtb5qYAG7dX/AmEIYswbgMa31KKXUSuB3wL+AMcCaKsYuhKiobdvgt7+Fpk1h0yZo187ZEYkGqCJX6o7g6+tbFBISkp+UlORTnABorVViYuKhYcOGXVFgr1+/vsy5rlWJUTGVUgQGBhYlJyenrF692m/BggWBK1as8F+5cmVG8TaFhYVER0dHAfTv3z937ty5R0se99FHHw3Oyclx+/TTT/f/undaORXtA/ARkAHMBxLtHlXxJDBdKbUPo0/Am1U8jhCiIjZtgn79ICgItm6Vwl80OO7u7vrjjz/e/9577zVfsGCBP0C/fv1Ov/7664HFve537drleebMGZdevXrlrVq1yr+wsJCjR4+6bd++3adnz57nwGgCSEtL8ygqKmLVqlX+PXv2zMvMzHQrKipi7NixuS+88MLPu3fvvqKJw83NjbS0tJS0tLSU0gr/OXPmBHzxxRdNPvroowOuNTzhVkX7AFzUWs+r6km01psx5xLQWh8AulX1WEKISvj3v2H4cAgLg88/h+uuc3ZEQjiFn5+f7dNPP93Xu3fvcF9f36Jp06blZGRkeHbo0MGqtVb+/v4FGzZs2J+QkJC7bds2H6vV2l4ppWfNmnUkNDS0cNeuXURHR5+bPHlyaEZGhlePHj3OJCQk5G7fvt17woQJFpvNpgCee+65I5WJ64knnmgdHBx8qWvXrlaAuLi4Uy+99FKmIz6DkpTW5fevU0qNxJgC+DPgUvFyrfV3jgvtF127dtU7duyoiVMJUX+sWAH33gs33QSffALNmzs7IlHDlFI7tdZdnRnDjz/+mNGxY8ccZ8ZQHdatW+ebmJgYlJSUtM/ZsVTWjz/+GNCxY0dLyeUVrQHoACRg3MNffEufNl8LIWqbt96C++6D226DdevAz8/ZEQkhapmKJgDDgbZa63xHBiOEqAbz58PDDxs9/levhkbVeteVEA1SXFxcXlxcXJ6z46hOFe0EmAw0dWQgQohq8MILRuE/ZAisXSuFvxCiTBWtARjljYkAACAASURBVGgKpCml/suVfQDKvA1QCFGDtIannoLnn4eRI2HpUnB3d3ZUQoharKIJwLMOjUIIUXU2G0ybBvPmwf33w+uvQw3fTiSEqHsqOhLgl44ORAhRBUVFMHEiLFkCjzwCc+aAqrHJxIQQddg1+wAopbaaP/OUUmfsHnlKqSoNdyiEqCYFBTBqlFH4P/20FP5CVJMZM2ZU24AZ06dPD3nmmWeCqrr/tm3bvG+66abIdu3aXTE5UXUorxNgYwCtta/W2s/u4au1lvuKhHCWixdh2DDjXv+//Q2ee04KfyGqybx584Irs73NZqOoqMghsfj4+Nj++c9//rRv3749n3322d4//vGPrao64VBJ5SUAMgufELXNuXMQF2eM8vfqq/DEE86OSIhaKz093aNNmzbtBw8e3KZt27bt+/fv3zYvL89l7dq1vn379r2heLvVq1f79evX74YHH3zw+kuXLrlERkZGDR48uA3AzJkzg8LCwtqHhYW1f+6551oUH9disUQPHTrUEh4e3n7//v0eq1at8ouKirJGREREde/ePbz42Kmpqd7dunWLaNmyZYfZs2dXarrgG2+88VKHDh0uAVgslgJ/f//CzMzMivbfu6byDtJCKTW9rJVa6znVEYQQooJOn4aBA+Hrr42e/mPGODsiISqlWzciSi67+25OzphBdl4eLjExhJVcf++95Dz8MCcyM3G76y5usF+3fTvp5Z0zIyPDa+HChRmxsbHnhg8fbnnxxRcDZ86cmTV16tTQo0ePuoWEhBQuWbKk+bhx43JGjhx5eunSpS3S0tJSALZs2dJo+fLlzXfu3JmqtaZLly7WmJiYvICAgKJDhw55vvnmmz/FxMRkHD161G3KlCmWzZs3p0VGRuZnZWVdvkrft2+f17Zt29Jzc3NdrVZr9OOPP57t6elZ6QvspKSkRgUFBSoqKupS+VuXr7waAFfAB/At4yGEqCk5ORATA99+C//6lxT+QlTQddddlx8bG3sOICEh4cS2bdt8XFxciI+PP7F48WL/nJwc1++++85n+PDhp0vuu3nzZp8BAwbk+vn52Zo0aWIbOHDgqaSkJF+A4ODg/JiYmHPmdo27deuWFxkZmQ8QFBR0uU0gNjY219vbWwcHBxf6+/sXHDlypNJX8AcPHnQfN25c28WLF2dU16RB5QWRqbV+rlrOJISousxMY0a/ffvgo4+MWgAh6qBrXbH7+mK71vrgYAorcsVfUmnT+AI88MADJwYOHNjOy8tLDxo06JR7JcfOaNSoka38rcD+at/V1ZXCwsIrAlq2bFnT559/PgRg0aJFGb169Tpvv/7kyZMuv/3tb9s9++yzPxcnHNWhvBoA6VUkhLMdPAi9ekFGBnz8sRT+QlRSZmamx8aNGxsDvPvuu/49evQ4C0abelBQUEFiYmLwxIkTL09Y5ObmpounCe7Tp8/ZDRs2NM3Ly3M5c+aMy4YNG5r16dPnqiGBe/fufW779u2+aWlpHgD2TQDlGT16dG7xlMElC/+LFy+qgQMHthsxYsSJcePGnaraJ1C68hKAmOo8mRCikvbuhZ49ITvbmM63Tx9nRyREnWOxWC7Onz+/Rdu2bdvn5ua6PfbYY9nF60aMGHEiODg4v3PnzheLl40aNSrbarVGDR48uM3tt99+fuTIkSc6d+5s7dKlizUhISH7tttuu1DyHCEhIYXz5s3LGDp0aLuIiIiooUOHtq2O2JcsWdLsv//9r8/y5csDIiMjoyIjI6O2bdvmXR3HrtB0wFU6sFJewFeAJ0ZTwyqt9bNKqTbAv4DmwE4gobxJhmQ6YNEgJSdD377GYD+ffQadOjk7IlHHyHTARm/9uLi4sL179+4pbf3o0aNDO3XqdH7atGl1fsrispQ1HXBFJwOqikvAnVrrjsBNQH+l1K3A34CXtdbtgFPABAfGIETdtGMH3HGHMaTvV19J4S+EA7Rv396akpLiPXny5BPOjsUZquVewtJoo2rhrPnS3Xxo4E5gpLn8bWAm8Lqj4hCiztm6FQYMgObNYdMmaFstNYlCNEgRERH5ZV3979mzJ7Wm46lNHFkDgFLKVSn1A3Ac+BzYD+RqrQvNTY4A15ex70Sl1A6l1I7s7OzSNhGi/vn8c4iNhZAQ2LJFCn8hhMM4NAHQWhdprW8CWgLdgMhK7LtIa91Va901MDDQYTEKUWusWWOM8BcWZlT7t2zp7IiEEPWYQxOAYlrrXCAJ6A40VUoVNz20BH6uiRiEqNXee88Y2/+mmyApCVpUarRQIYSoNIclAEqpQKVUU/O5N9APSMVIBH5nbjYGWOOoGISoE954w5jV7/bbYeNG8Pd3dkRCiAbAkTUAwUCSUmoX8F/gc631OuBJYLpSah/GrYBvOjAGIWq3uXPh/vvhN7+BDRvAV0bYFqI6paene4SFhbV3dhxlef755wNDQ0OjlVJd7Cf5sdlsjB07tlVoaGh0eHh41NatWxtV97kdeRfALuCqe5e01gcw+gMI0XBpDc8/D089BXffDcuXg6ens6MSQtSwO+644+ywYcNO33nnnVdMkrRy5comBw4c8MrIyEhOSkpq/OCDD4bu2rUrrTrPXSN9AIQQdrSGP/zBKPwTEmDFCin8hagBKSkpHlarNerLL79sVFhYyKRJk1pGR0dbw8PDo1588cUAMK68J02a1DIsLKx9eHh41OLFi5sBrFu3zrdr164RvXv3bmexWKJHjhwZWlRURGFhIcOGDbMUbz9r1qxKdeC57bbbLkRERFw1GN6aNWuajho16oSLiwsxMTHnzpw543bw4MHKTVZQDofVAAghSmGzwcMPw6uvwuTJxk8XycNFwzB+zfhWyceTq7UqO7pF9Pkldy05XN52P/74o+eIESNuWLJkyU/du3e/8NJLLwU0adKkKDk5OfXChQvq5ptvjhw0aNCZb775ptHu3bu9U1NT92RmZrp169bNGhsbexZg9+7djb///vvk8PDw/F69eoUtW7asWbt27S5lZma6F481kJOTUy1T9WVmZrpbLJbLiUFwcHD+wYMH3Vu3bl1QHccHqQEQouYUFsKECUah/+ij8NprUvgLUQNOnjzpNmTIkHbvvPPOge7du18A2Lhxo9/777/fPDIyMqpTp07WU6dOuaWkpHht2bLFNz4+/qSbmxutWrUqvOWWW84Wt7936NDhXFRUVL6bmxvx8fEnt2zZ4hMZGXnp8OHDnmPGjGm1atUqv2bNmhVdO5raQ2oAhKgJ+flw772wciXMnAnPPANKJtsUDUtFrtQdwdfXtygkJCQ/KSnJp0uXLhcBtNYqMTHx0LBhw87Yb7t+/fomZR2ntGmFAwMDi5KTk1NWr17tt2DBgsAVK1b4r1y5MqN4m8LCQqKjo6MA+vfvnzt37tyjFYk5ODi4ICMjw6P4dWZmpkd1Xv2D1AAI4XgXLhgd/VauhJdegmeflcJfiBrk7u6uP/744/3vvfde8wULFvgD9OvX7/Trr78eWDzt765duzzPnDnj0qtXr7xVq1b5FxYWcvToUbft27f79OzZ8xwYTQBpaWkeRUVFrFq1yr9nz555mZmZbkVFRYwdOzb3hRde+Hn37t1XNHG4ublRPNVvRQt/gMGDB+e+++67zW02G5s2bWrs6+tbVN0JgNQACOFIZ8/C4MGweTO8/rrR7i+EqHF+fn62Tz/9dF/v3r3DfX19i6ZNm5aTkZHh2aFDB6vWWvn7+xds2LBhf0JCQu62bdt8rFZre6WUnjVr1pHQ0NDCXbt2ER0dfW7y5MmhGRkZXj169DiTkJCQu337du8JEyZYbDabAnjuueeOVCau2bNnt5g/f/51J06ccO/YsWNUnz59Tq9YseJgfHz86fXr1zdp3bp1tLe3t+2NN97IqO7PxGHTAVcnmQ5Y1Em5ucakPt9+C0uXGj3+hahBMh1w9Vm3bp1vYmJiUFJS0j5nx1JZZU0HLDUAQjhCdrYxqc+ePUbV/913OzsiIYS4giQAQlS3o0ehb1/46SdYuxb693d2REKIXykuLi4vLi4uz9lxVCdJAISoThkZEBMDx4/DJ5/AHXc4OyIhhCiVJABCVJf0dOPK/+xZY1KfW25xdkRCCFEmSQCEqA67dkG/fsYwv5s3Q8eOzo5ICCGuScYBEOLX2r4devcGd3fYskUKfyFEnSAJgBC/xldfGW3+zZoZhX9ERPn7CCFqtRkzZlxXXceaPn16yDPPPBP0a47Rs2fPMF9f35v69OnTzn55Wlqax4033hgZGhoaPXDgwLYXL16s1AhjkgAIUVWffmr08G/Z0kgE2rRxdkRCiGowb9684Mpsb7PZKCpy3BQAjz322LGFCxf+VHL59OnTW06ZMiXr0KFDyU2aNCl85ZVXAipzXEkAhKiK1ath0CDjiv+rr+D6650dkRCiFOnp6R5t2rRpP3jw4DZt27Zt379//7Z5eXkua9eu9e3bt+8NxdutXr3ar1+/fjc8+OCD11+6dMklMjIyavDgwW0AZs6cGRQWFtY+LCys/XPPPdei+LgWiyV66NChlvDw8Pb79+/3WLVqlV9UVJQ1IiIiqnv37uHFx05NTfXu1q1bRMuWLTvMnj27UtMFA9x11115fn5+NvtlNpuNr7/+2nfcuHGnAMaPH3/i3//+d9PKHNdhnQCVUq2AZUAQoIFFWutXlFL+wArAAmQA8VrrU46KQ4hq9847MHYs3HwzfPwxNK3U35wQDVq3xd2uaie723r3yRm3z8jOu5TnErMsJqzk+ntvvDfn4VsePpGZl+l217/uusF+3fb7t6eXd86MjAyvhQsXZsTGxp4bPny45cUXXwycOXNm1tSpU0OPHj3qFhISUrhkyZLm48aNyxk5cuTppUuXtkhLS0sB2LJlS6Ply5c337lzZ6rWmi5dulhjYmLyAgICig4dOuT55ptv/hQTE5Nx9OhRtylTplg2b96cFhkZmZ+VlXV5WuB9+/Z5bdu2LT03N9fVarVGP/7449menp6/ahjerKwsN19f3yJ3d3cALBZLflZWlkc5u13BkTUAhcCjWuso4FbgIaVUFDAD2KS1DgM2ma+FqP2ys+Gpp2D0aOjVCz7/XAp/IeqA6667Lj82NvYcQEJCwolt27b5uLi4EB8ff2Lx4sX+OTk5rt99953P8OHDT5fcd/PmzT4DBgzI9fPzszVp0sQ2cODAU0lJSb4AwcHB+TExMefM7Rp369YtLzIyMh8gKCjocptAbGxsrre3tw4ODi709/cvOHLkSK24A89hQWitM4FM83meUioVuB64C+htbvY2sBl40lFxCPGrpaTA3LmwbBlcugQjRsCSJeDt7ezIhKhzrnXF7uvpa7vW+mDf4MKKXPGXVNo0vgAPPPDAiYEDB7bz8vLSgwYNOlV8NV1RjRo1spW/Fdhf7bu6ulJYWHhFQMuWLWv6/PPPhwAsWrQoo1evXufLO2ZQUFBhXl6ea0FBAe7u7mRkZHgEBQXlVyb+GukDoJSyAJ2Ab4EgMzkAOIbRRFDaPhOVUjuUUjuys7NrIkwhfqG1MZjPgAHQvj38859GtX9qKrz3nhT+QtQhmZmZHhs3bmwM8O677/r36NHjLIDFYikICgoqSExMDJ44ceLlCYvc3Nx08TTBffr0Obthw4ameXl5LmfOnHHZsGFDsz59+lw1JHDv3r3Pbd++3TctLc0DwL4JoDyjR4/OLZ4yuCKFP4CLiwu33npr3ltvvdUMYMmSJc3j4uJyK3pOqIEEQCnlA3wAPKK1PmO/ThtTEZbaDqK1XqS17qq17hoYGOjoMIUwXLpkzNzXsaMxsM9338Gf/wyHD8OCBRAZ6ewIhRCVZLFYLs6fP79F27Zt2+fm5ro99thjl68qR4wYcSI4ODi/c+fOF4uXjRo1KttqtUYNHjy4ze23335+5MiRJzp37mzt0qWLNSEhIfu22267UPIcISEhhfPmzcsYOnRou4iIiKihQ4e2ra74u3TpEpGQkND266+/9gsKCrrxgw8+8ANITEw8Mn/+/OtCQ0OjT5065TZ16tRKzbro0OmAlVLuwDrgU631HHNZOtBba52plAoGNmutr3nztEwHLBzuxAmjgP/HP+DYMYiOhunT4fe/By8vZ0cnRJXIdMBGb/24uLiwvXv37ilt/ejRo0M7dep0ftq0aXV+yuKy1Ph0wMpoZHkTSC0u/E1rgTHAX82faxwVgxDlSk832vfffhsuXDDu658+3RjTX1VqTA0hRB3Tvn17q7e3t23hwoWHnR2LMziyJ+JtQAKwWyn1g7nsjxgF//tKqQnAQSDegTEIcbXi8frnzIF168DTExIS4JFHjPZ+IUS9ERERkV/W1f+ePXtSazqe2sSRdwFsBcq6hIpx1HmFKFN+Prz/vlHwf/89BAbCs8/Cgw9Ci0qPzSGEEHVarbgXUQiHOnkSFi2C+fPh6FGwWmHxYhg1SnrzCyEaLEkARP21dy+88gq89RacP2/06n/zTYiNBRcZBVsI0bBJAiDqF62NWfnmzIG1a40pekeNMtr3b7zR2dEJIUStIZdBon4oKDAG6OnWDe64A7ZuhT/9CQ4eNEbtk8JfiAYpPT3dIywsrNb27h02bJjl+uuv7xAZGRkVGRkZtW3bNm8wJvsZO3Zsq9DQ0Ojw8PCorVu3Nqruc0sNgKjbcnON9vx58+DIEWN2vgULjF79jar970UIIard7NmzjxTP6lds5cqVTQ4cOOCVkZGRnJSU1PjBBx8M3bVrV1p1nldqAETddOAATJ0KLVvCE09AWJhxS19KCkyaJIW/EOIqKSkpHlarNerLL79sVFhYyKRJk1pGR0dbw8PDo1588cUAMK68J02a1DIsLKx9eHh41OLFi5sBrFu3zrdr164RvXv3bmexWKJHjhwZWlRURGFhIcOGDbMUbz9r1qxquaVozZo1TUeNGnXCxcWFmJiYc2fOnHE7ePBg5SYrKIfUAIi6Q2vYts1o3//oI6Mj3+9/D9OmQadOzo5OCFGe8eNbkZxcvdl5dPR5liwpdyCfH3/80XPEiBE3LFmy5Kfu3btfeOmllwKaNGlSlJycnHrhwgV18803Rw4aNOjMN99802j37t3eqampezIzM926detmjY2NPQuwe/fuxt9//31yeHh4fq9evcKWLVvWrF27dpcyMzPdi8cayMnJqfAcAMVmzZp1/QsvvBDcs2fPvH/84x9HvL29dWZmprvFYrk8uU9wcHD+wYMH3Vu3bl1Q2eOXRWoARO1XWGjcv9+9O9x+OyQlwZNPQkaGMUOfFP5CiGs4efKk25AhQ9q98847B7p3734BYOPGjX7vv/9+88jIyKhOnTpZT5065ZaSkuK1ZcsW3/j4+JNubm60atWq8JZbbjlb3P7eoUOHc1FRUflubm7Ex8ef3LJli09kZOSlw4cPe44ZM6bVqlWr/Jo1a1Z07WiuNGfOnJ8PHDiQ/OOPP6aeOnXK9emnn77OEZ9BaaQGQNRep08bt+298gocOgTt2sGrr8KYMdC4sbOjE8IhLhRcIP1EOqnZqaTmpPLEbU/g4+Hj7LCqRwWu1B3B19e3KCQkJD8pKcmnS5cuFwG01ioxMfHQsGHDrpikbv369U3KOk5p0woHBgYWJScnp6xevdpvwYIFgStWrPBfuXJlRvE2hYWFREdHRwH0798/d+7cuUftj1F8Re/t7a3Hjx9/IjExMQggODi4ICMjw6N4u8zMTI/qvPoHSQBEbZSRYXTqe+MNyMuDXr2M13Fx4Frp2jUhaqXTF0+TmpNKanYqKdkpxvOcVH469RPanCTVRbnwu6jfcWOQ3MXya7i7u+uPP/54f58+fcJ8fHxskydPPtmvX7/Tr7/+emBcXFyep6en3rVrl6fFYino1atX3uLFiwOnTJly4vjx427bt2/3mTdv3uFdu3Z57969u3FaWppHWFhY/qpVq/zvu+++7MzMTDdPT0/b2LFjc9u3b38xISHhilkA3dzcSEtLSykrtuJqfZvNxocfftjUarVeABg8eHDua6+91uL+++8/mZSU1NjX17dIEgBRf337LSQmwgcfGBPx3HOP0b7f1amTmQlRZVprjp87TmqOWcibV/WpOakczfvlQtDT1ZPw5uHcHHIzo28cjTXQijXASljzMLzcZDbK6uDn52f79NNP9/Xu3Tvc19e3aNq0aTkZGRmeHTp0sGqtlb+/f8GGDRv2JyQk5G7bts3HarW2V0rpWbNmHQkNDS3ctWsX0dHR5yZPnhyakZHh1aNHjzMJCQm527dv954wYYLFZrMpgOeee+5IZeK655572pw8edJNa62ioqLOL1u27CBAfHz86fXr1zdp3bp1tLe3t+2NN97IqO7PxKHTAVcXmQ64HisqMjr0zZljdPBr0sToxT9lCrRq5ezohKgQm7Zx+PThX67ks1NJyTEK/FMXf7m7y8fDh6jAKKwBRgFvDbQSFRhFm6ZtcHWp/totmQ64+qxbt843MTExKCkpaZ+zY6msGp8OWIhrysszBuh55RX46Sdo29ao5h87Fnx9nR2dEKUqKCpg/6n9l6/kiwv8tJw0zhecv7xdYKNArIFW4tvH/1LgB1q53vf6q9qRhXAWSQBEzTp0yJiUZ/Fio5PfbbfBSy/BXXdJ+76oNYo74pWstt97Yi8Ftl+aYVv5tcIaaGVi54mXq+2tgVYCGgU4MXrhCHFxcXlxcXF5zo6jOkkCIGrGjh1GNf/77xuvf/c7o33/llucG5do0HIv5v5SwNtd1WfkZlzREa+dfzusAVYGhw++XG0fGRBZf3rniwbJYQmAUmoJEAcc11pHm8v8gRWABcgA4rXWp8o6hqjjiorg3/82Cv4tW4yq/alT4eGHoXVrZ0cnGgitNVnnsq6qtk/NTiXzbObl7TxdPYkIiKDb9d0Y03GMUXUfaCXMPwxPN08nvgMhHMORNQBLgX8Ay+yWzQA2aa3/qpSaYb5+0oExCGc4d86YgnfuXNi/3yjs58yBCRPAz8/Z0Yl6yqZtHDp96Krb6kp2xPP18MUaaCX2htgr2ucd1RFPiNrKYQmA1vorpZSlxOK7gN7m87eBzUgCUH/8/DP84x+wcCGcOmVU77/wAgwdCm7S2iSqR0FRAftO7ruq2j79RHqpHfHuaX/P5fb5qMAoQnxDpCOeENR8H4AgrXVxndsxIKisDZVSE4GJAKGhoTUQmqiy776Dl1+Gf/0LbDa4+26YPt0YuleIKjpfcJ70nPSrbqvbe3IvhbbCy9uFNgnFGmDljtZ3XG6ftwZYad6ouROjF3XZjBkzrvvrX/96rDqONX369BAfH5+i5557Lquqx3B1de0SFhZ2ASAkJCT/iy++2AeQlpbmER8f3zY3N9etQ4cO5z/44IOfvLy8Knxvv9Muy7TWWilVZqBa60XAIjDGAaixwETF2Gywfr1Rtb95M/j4wEMPGW38bdo4OzpRhxR3xCtZbW/fEc9VuXKD/w1YA6wMiRxyudpeOuIJR5g3b15wZRIAm82G1hpXB93J5OnpaSttNMHp06e3nDJlStbEiRNPjRw5MvSVV14JePLJJ7MretyaTgCylFLBWutMpVQwcLyGzy9+rfPn4e23jfb9//3PmI73xRfhvvugaVNnRydqqeKOeCVvq0vJTuHY2V/+zxZ3xLul5S2MvWns5YJeOuKJqkpPT/fo379/WIcOHc4nJyc3Cg8Pv7By5cqMpKSkxvPmzWuxcePG/QCrV6/2e+211wLDwsIuXrp0ySUyMjIqPDz8wtq1a3+aOXNm0LvvvhsAkJCQkP3MM88cT09P9/jNb34T3qlTp7O7d+9uvGHDhr27du3yeuaZZ64vKipS/v7+hV9//fX/AFJTU727desWcfToUY/JkydnPfXUU7+67LPZbHz99de+a9asOQAwfvz4EzNnzgypzQnAWmAM8Ffz55oaPr+oqsxMYyKe11+HkyeN4Xnfew+GDQP3ap2iWtRhNm3jYO7BUse4z72Ye3k7Xw9fogKj6N+u/+W2eWuAFUtTi3TEq++6dYu4atndd59kxoxs8vJciIkJu2r9vffm8PDDJ8jMdOOuu264Yt327enlnTIjI8Nr4cKFGbGxseeGDx9uefHFFwNnzpyZNXXq1NCjR4+6hYSEFC5ZsqT5uHHjckaOHHl66dKlLYqvuLds2dJo+fLlzXfu3JmqtaZLly7WmJiYvICAgKJDhw55vvnmmz/FxMRkHD161G3KlCmWzZs3p0VGRuZnZWVd/iLv27fPa9u2bem5ubmuVqs1+vHHH8/29PSscM12fn6+S3R0tNXV1VU/9thjxxISEnKzsrLcfH19i9zN/78WiyU/KyvLo5xDXcGRtwG+h9HhL0ApdQR4FqPgf18pNQE4CMQ76vyimvz4o9G+v3y5MS3vkCFG+/5ttxnj9YsGyb4jnv1tdWk5aVwovHB5uxaNW2ANsDKi/YjLt9VZA6zSEU/UqOuuuy4/Njb2HEBCQsKJefPmtXBxccmKj48/sXjxYv+HHnroxHfffefz4Ycf/lRy382bN/sMGDAg18/PzwYwcODAU0lJSb7Dhw/PDQ4Ozo+JiTlnbte4W7dueZGRkfkAQUFBl6cFjo2NzfX29tbe3t6F/v7+BUeOHHG74YYbKjyxz969e3e1adOmICUlxaNfv34RnTt3vuDv71+paYdL48i7AH5fxqoYR51TVBObDT75xGjf37QJGjUyxuefOtWYklc0GOcLzpOWk3ZVtf2+k/tK7YjX29L7crW9dMQTpbrWFbuvr+2a64ODCytyxV9SadP4AjzwwAMnBg4c2M7Ly0sPGjTolHslazMbNWpkq8h29lf7rq6uFBYWXhHQsmXLmj7//PMhAIsWLcro1avXefv1bdq0KQCIiorKv/XWW/O2b9/eaMyYMafy8vJcCwoKcHd3JyMjwyMoKCi/MvHLvVniFxcuwDvvGFf8qakQEgJ//StMnAjNmjk7OuFAcfAFIAAAExFJREFUpy6cuuq2utScVA7mHryqI15UYBRDI4derrqPCIiQjniiVsvMzPTYuHFj4759+5579913/Xv06HEWwGKxFAQFBRUkJiYGf/LJJ/8r3t7NzU1funRJeXp66j59+pwdP3685c9//vMxrTUbNmxotnTp0gMlz9G7d+9z06dPb52WluZR3ARgXwtwLaNHj84dPXp0bmnrsrOzXX18fGze3t46MzPTbceOHT5//OMfj7m4uHDrrbfmvfXWW80mTpx4asmSJc3j4uJKPUZZJAEQkJUFr71mPHJyoFMnIxEYPhw8KtWkJGoxrTXHzh4rdWpa+454Xm5eRDSP4NaWtzLupnGX2+fb+beTjniiTrJYLBfnz5/fYuLEiY3CwsIuPvbYY5c7yo0YMeLEq6++6ta5c+eLxctGjRqVbbVao6Kjo8+vXbv2p5EjR57o3LmzFYxOgLfddtuF9PT0K/45hoSEFM6bNy9j6NCh7Ww2G82bNy/Ytm3b3l8b+w8//OD10EMPtVZKobXmkUceOdalS5eLAImJiUfuueeeG2bPnn19+/btz0+dOrVSsy7KdMANWXKycbX/zjuQnw+DBhnt+3fcIe37dVhxR7zSpqY9fen05e38PP2u6IBXXG0vHfHqD5kO2LgLIC4uLmzv3r17Sls/evTo0E6dOp2fNm1anZ+yuCwyHbAwaA2ffWa073/2GXh7G0P0PvIIhIc7OzpRCflF+UZHvBLV9uk56Vd1xIsKjGJkh5FXzEEf7BMsHfFEg9a+fXurt7e3beHChYedHYszSALQUFy8aPTknzMH9uyB666Dv/zF6NzXXDpq1Wbn8s+RfiL9qtvqSnbEa92kNdZAK30sfa64qvf39ndi9EI4V0RERH5ZV/979uxJrel4ahNJAOq77Gzj3v1XX4Xjx+HGG2HpUhgxAjylPbc2OXnhZKlT0x48ffDyNq7K1ZiaNtDK3ZF3X662l454ohaz2Ww25eLiUvvbm+shm82mgFLvVpAEoL5KTTXa95ctg0uXYMAAo33/zjulfd+JtNZkns28oqAvbp/POvfLUOHFHfF6tOrBhE4TLlfbt/Nvh4erdMwUdUpydnZ2VGBg4GlJAmqWzWZT2dnZTYDk0tZLAlCfaA1ffAGJifDxx+DlBWPGGO37Vquzo2tQbNpGRm5GqWPcl+yIFxUYxYCwAb90yAu00rpJa+mIJ+qFwsLC+44dO/bGsWPHouH/27v34Liq+4Dj359WK2lXEtZjZVl+YRscG5sYsD3BJC0EAzaYDLQh6RBoQ0iYhKY0TUMJkMdMwqSBph2mZEiaEnBaOoQQKE0hA2VcHkOHwWBs/FCwARdS4zcSxpa0K+1q9esf56x0Je1aD8u70u7vM3Nn7z337H3sWel37rln76Gs0MdTYvqA1t7e3huyrbQKQDHo6XEj8d19N2zfDtOnwx13wI03QlNToY+uqGU64g39Wd2utl109/b/qojm6mbOaDqDaz56zaD789YRr7iouuEyOjqgs9NNwfmhy6PJt23b1H7+1ooVKw4DVxT6OMxwVgGYyg4cgPXr4d574eBBWLoUHngArrnGXf2bE5JMJ2mPt9MWb6M94V/j7ew5umfQE/HSOvCsj0xHvNXzVvffn7eOeJOTqusbO9pAPJp8XV1uu6MRCkFtrZtqatxUW+v65GbmM+nGnAxWATgBPd0pNj79Ihf88Ul+urEq7N0LW7YMnvbvd+vXrnUj9F1yid3fz6G7t5v2ePugQN4Wbxsc3Ies60h2ZN1WSEIsbFzIkqYlXHXGVf335xc1LqK6ojrPZ1Y6enrGduU8Ur7OTkiP8mnqZWXDg3JNDcyaNTh4Z5vPta6iwv5cTWFZBWCc/mv9oyz9+tUsTivxQ11EayITs+G+PnjnneHBvr3drS8rg8WLXWe+5cthzRp35V9CEqlEf7AOBuyhgTy4rivVlXN7tRW1xKIxGqONxKIxFscW0xhx8/2v0YHlpuom64g3glRqdEF5LAE7NeqhU7IH3unT4bTTxhewq6osWJviYxWAUdrx8g42/uC7lDU286UH/5mPXX4RG+9s4cjFn+Wq8f5j6O2FN990Af711wdejx1z68NhOPNMNwLf8uVuWrbMDc5TJOKpeO5AHm+nLTF8XTwVz7m9aZXT+oP1jJoZLG1amjWQZ9Iao40lH8zT6eFXxmO9Tz10vqdn9PuPRocH3vp6mDt35KvobMvRqKsnG2OOzyoAx/H+/jae/M73mPPcI1y4p42PKjyxaC4ADc0NrHt77+g3lky6B/AEr+q3bXMD8IB7It9ZZ8G11w4E+6VLp8xv9VWVrlRX1kAeDOZD1wU7yg1VV1XXH6xn1s5kWfOynIE8Fo3REGkgHBrbaF5TTV/f8E5mJxqwE4mR95tRVTU88NbWQkvL6Jq9h85XV7t74caY/LMKwBB96T7KQu7y4dULzuKLu/ezpzbEwx8/j3lf+xZX/MmnRt5IPO564weDfWvrQBtmba0bcOcrXxkI9osWQfnkKA5VpSPZkTuQ52huT6azj0QpCPWR+v5gPXfaXM5pOYdYJDaseT2z3BBpoLxscnwe46XqgutE9AQPdjIbrXA4e1BuahrfPevqardNY0xxmNr/YSfQcw89wXs/vZMLWjdx5D9e4pzV55L+i+/zeGcHV9z6l/xZOMtHperuzWeu7DPN+Dt3uks1cF16ly93D+HJBPsFC/LWRqmqHOs5NrzDW7bgHliX6st+w7VMymiINPQH63l181jZsjJnII9FY9RX1RfsN+3ptGt8Od6USo2cJzhlgvpIAbuzc+BrMJKysuE9wmtqYM6cE+tkZowxuRSkAiAilwL3ACHgflW9K5/7z1zlt768lW1fu54z9rzF6sNxegVeOLWJ0MFD0NnJFevOh3374NePuB73+/a51+B8MnDVO3OmC/Cf/vRAsJ8zZ1y9h/q0j+7ebhKpBIneBIlUwi37+USvWz7affS4gbw90T7oefFBIQnREGnob0o/veF0Vs1alTOQx6Ix6qrq0L6y4wfROCQ/hMNJ2DuGwDpRATo4jTYAj4VI9sA7Y4b7vfZYmsAzy5WV1snMGJNfeR8OWERCwFvAJcBeYBPwOVV9I9d7xjsccLI7yYaf/5Ij//Ms0be3M+vQHuYdPcb2hcu45Ju3cGDHThp/9APaImHi9TNoaW4h2nUU9h9AMh3xAvqiEVIt00lOj9HT3EiiuZHuWB1HT53BwY/M4oNpVXQlEySS3cRTCeLJBPGUD9ipwcG7p7ebRDpBd6+betLd9KQTdKcT9KQTpDR7c3ouIcqpLmskSowojUQ0RmVfI5XpGBW9jVT0xihPxQj1NBLqiSGJRrR7Gqnk8GBeiKAK7g5IRcXET+HwxG/PgrWZCibDcMBm8ipEBeA84HuqutYv3w6gqnfmes94KwDPnzaXC98ZeZTHZBnsrx2Y9p0SmA+kd4zn2TrpMKQi0BuB3qqB+ZRf7p8fsv64eQPzPadAPOZeGR6VJjqoTnQwtaBqzMljFQBzPIW4BTALCEblvcC5QzOJyJeBLwPMnTt3XDvacta5HAr3cKimlv3TmuiMROisrKSzMkI8XEVXZYTOygjJ8ipCUkFIwoQID7xSQbmEmSNh5qXDhOJhyiVMSFy6mw9TUVZJRShCpUSoCFVRFYpQFYpQUVZFuDxEKOR6OpeV0T+fK22k5aFp2QJyJi0ctp9DGWOMyW7SdgJU1fuA+8C1AIxnGzc//uiEHpMxxhhTLApxfbgPmBNYnu3TjDHGGJMnhagAbAIWish8EakArgaeKMBxGGOMMSUr77cAVLVXRG4CnsH9DHC9qv4u38dhjDHGlLKC9AFQ1aeApwqxb2OMMcYU5haAMcYYYwrMKgDGGGNMCbIKgDHGGFOCrAJgjDHGlKC8Pwp4PETkfeD/RsgWA9rycDiTjZ13abHzLi0net6nqmrTRB2MKS5TogIwGiLyWik+89rOu7TYeZeWUj1vkx92C8AYY4wpQVYBMMYYY0pQMVUA7iv0ARSInXdpsfMuLaV63iYPiqYPgDHGGGNGr5haAIwxxhgzSlYBMMYYY0rQlK8AiMilIvKmiOwWkdsKfTzjISJzROR5EXlDRH4nIn/l0xtEZIOIvO1f6326iMiP/TlvF5HlgW1d5/O/LSLXBdJXiMgO/54fi4jk/0yzE5GQiLwuIr/1y/NF5BV/rI/4YaMRkUq/vNuvnxfYxu0+/U0RWRtIn5TfDxGpE5HHRGSXiOwUkfNKobxF5K/9d7xVRB4WkapiLW8RWS8ih0WkNZB20ss41z6MGUZVp+yEG074f4EFQAWwDVhS6OMax3m0AMv9fC3wFrAE+BFwm0+/Dfg7P78OeBoQYBXwik9vAN7xr/V+vt6ve9XnFf/eywp93oHz/wbwS+C3fvnXwNV+/mfAn/v5rwI/8/NXA4/4+SW+7CuB+f47EZrM3w/gX4Eb/HwFUFfs5Q3MAt4FIoFy/kKxljdwPrAcaA2knfQyzrUPm2waOk31FoCPAbtV9R1VTQK/Aq4s8DGNmaoeUNUtfr4D2In7Z3klLlDgX//Iz18JPKjORqBORFqAtcAGVf1AVY8AG4BL/bpTVHWjqirwYGBbBSUis4HLgfv9sgCrgcd8lqHnnfk8HgMu8vmvBH6lqj2q+i6wG/fdmJTfDxGZhgsODwCoalJVP6QEyhs3BHlERMqBKHCAIi1vVX0R+GBIcj7KONc+jBlkqlcAZgHvBZb3+rQpyzdzngO8AjSr6gG/6iDQ7Odznffx0vdmSZ8M/hH4JtDnlxuBD1W11y8Hj7X//Pz6oz7/WD+PQpsPvA/8wt/6uF9Eqiny8lbVfcA/AHtwgf8osJniL++gfJRxrn0YM8hUrwAUFRGpAf4d+LqqHguu87X8ovrNpoh8CjisqpsLfSx5Vo5rGv4nVT0H6MI11fYr0vKux12dzgdmAtXApQU9qALKRxkX4/fITJypXgHYB8wJLM/2aVOOiIRxwf8hVX3cJx/yTX3418M+Pdd5Hy99dpb0QvsEcIWI/B7XXLsauAfX/Fnu8wSPtf/8/PppQDtj/zwKbS+wV1Vf8cuP4SoExV7eFwPvqur7qpoCHsd9B4q9vIPyUca59mHMIFO9ArAJWOh7EVfgOgo9UeBjGjN/X/MBYKeq3h1Y9QSQ6fV7HfCfgfTP+57Dq4CjvsnvGWCNiNT7q601wDN+3TERWeX39fnAtgpGVW9X1dmqOg9Xds+p6rXA88BnfLah5535PD7j86tPv9r3Gp8PLMR1kJqU3w9VPQi8JyKLfNJFwBsUeXnjmv5XiUjUH1fmvIu6vIfIRxnn2ocxgxW6F+KJTrjes2/hev9+u9DHM85z+ANcM912YKuf1uHudz4LvA38N9Dg8wvwE3/OO4CVgW19EdcpajdwfSB9JdDq33Mv/imQk2UCPsnArwAW4P6h7wYeBSp9epVf3u3XLwi8/9v+3N4k0ON9sn4/gLOB13yZ/wbXw7voyxv4PrDLH9u/4XryF2V5Aw/j+jqkcK0+X8pHGefah002DZ3sUcDGGGNMCZrqtwCMMcYYMw5WATDGGGNKkFUAjDHGmBJkFQBjjDGmBFkFwBhjjClBVgEwRU1E0iKyVUS2icgWEfn4CPnrROSro9juCyKycpzH9JSI1I3nvcYYM1GsAmCKXUJVz1bVs4DbgTtHyF+HG4XupFHVdeoG/zHGmIKxCoApJacAR8CNuyAiz/pWgR0ikhk17i7gNN9q8Pc+760+zzYRuSuwvc+KyKsi8paI/OHQnYlIi4i86LfVmskjIr8XkZiI3OjXbRWRd0Xkeb9+jYi87I/tUT9GhDHGTCh7EJApaiKSxj1ZrQpoAVar6ubMcLSqekxEYsBG3CNlT8U9kfBM//7LgO8CF6tqXEQaVPUDEXkB2KyqN4vIOuAbqnrxkH3fDFSp6t+KSMjvr8OPfbBSVdt8vjDwHG4c95dxz8i/TFW7RORW3JPx7jiZn5MxpvSUj5zFmCktoapnA4jIecCDInIm7tGrPxSR83FDEc8i+7CpFwO/UNU4gKoGx3fPDNq0GZiX5b2bgPU+wP9GVbfmOMZ7cM+5f9KPkLgEeMk94p0KXKXAGGMmlFUATMlQ1Zf91X4T7pnxTcAKVU35q/KqMW6yx7+myfK3pKov+grG5cC/iMjdqvpgMI+IfAHX6nBTJgnYoKqfG+OxGGPMmFgfAFMyRGQxEMINKTsNOOyD/4W4IAzQAdQG3rYBuF5Eon4bDWPY36nAIVX9OXA/bsjf4PoVwN8Af6qqfT55I/AJETnd56kWkY+M7UyNMWZk1gJgil1ERDJN7wJcp6ppEXkIeFJEduBG5dsFoKrtIvKSiLQCT6vqLSJyNvCaiCSBp4BvjXLfnwRuEZEU0IkbsjXoJqABeN4397+mqjf4VoGHRaTS5/sOboQ7Y4yZMNYJ0BhjjClBdgvAGGOMKUFWATDGGGNKkFUAjDHGmBJkFQBjjDGmBFkFwBhjjClBVgEwxhhjSpBVAIwxxpgS9P/wYDLfMIL/6wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "n_features = [2, 10, 50]\n", + "max_batch_size = 100000\n", + "\n", + "plot_absolute_time(results, n_features, max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ec9d0fbb", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAEWCAYAAABbt/wMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd3hUZfbA8e9JQocgnSQQQoeAdFEhriCKLmJhFVZBVCygKCpY1sIiouvKz7WBqIggoigqi4qKK6KwVGEDCNKVIiV0aaGknt8f700cQsoEMkxCzud55knmzi1n7tyZOfNWUVWMMcYYY4IhJNgBGGOMMab4skTEGGOMMUFjiYgxxhhjgsYSEWOMMcYEjSUixhhjjAkaS0SMMcYYEzRFNhERERWRBrk8/paI/N3n/r0isltEEkWkymkcL8Y7ZtjpxlxQvOdQL9hxnC0iMlFEnsvH+rleG4WNiHwjIrcFOw5/iEi0d/2Fnsa2ZUTkSxE5JCKfBiI+c3pEpI+IzAx2HKZ4yjMREZEtIpIsIlWzLF/ufeDHePcnevevy7LeK97y2737t4tImvdhligim0XkXRFpVGDPClDVe1T1We+YJYCXga6qWl5V9wfyy8p7jvMLaF9zROQu32Xec9hUEPsvCIUhSRORi0VkYbCO7y8RGS4iH/guU9U/q+p7wYopN977//KM+6q61bv+0k5jdzcCNYAqqtqzAGIrKSJTvRhVRDpleVxEZKSI7PduI0VEfB5vJSJLReSY97fVmcYUDPn9vMnu/aqqk1W1a2AiNCZ3/paIbAZuzrgjIucDZbNZbwNwq896YUAvYGOW9RapanmgInA5cBxYKiLN/Q89Z9n8WqsBlAZWF8T+TcEqoATmamBGAeyn2AhC4lgH2KCqqfndMJdY5wO3ALuyeaw/cD3QEmgBXAMM8PZXEvgC+ACoBLwHfOEtDxgvOSqyJdHGBISq5noDtgBDgf/5LPsX8BSgQIy3bKK3fDdQyVvWHfgG92Fxu7fsdmB+Nsf5CpiaSxyPAjuBBOAO79gNfI79Ju6L6CguuZkIPAc08pYpkAj8AMz17h/1lv01m+OFes9nH7AJuM/bJsx7vCIw3otph3esUKApcAJI8/Z90Fu/lLe/rd45egso43O864CfgMO4xO0q4B/efk54+3rdW9f3uVcEJgF7gd+81yrE91x7xz2ASyj/nMdr/QSwxlv/XaC099gq4BqfdUt456a195wyzm8icDEuyR3qxbTHi7Git22Mt/6d3rZzveVxwELgILCNP66ZicAY4GvgCLAYqJ8l9mVAm2zOT5y3r07e/SbAd8DvwHqgl88+8jqXC4DXgUPAOqCLz7a3466TI9557pPN+b0KSAZSvPO0wls+B7gry3Fe8c7DJqCDt3ybdy5v89lnrtdVluP77ns/7pqtj3tP7Pdez8nAed767wPpuB8KicBjPq9dxvsgEpjunc9fgbtzOPYzWZ77nZzGNZLLtbs94zX2WbYQ6O9z/07gR+//rrj3rfg8vhW4Kof9zwH+CSzBvUe/ACr7PH4Rf1y7K3xj8bb9h3fujwMNvGXPedskAl8CVbzzfxj4H398tp50zn2vGXL+vLkaWO7taxswPMvzzPp+vR2fz2XcNfc/3LX+P6BDlmM/6z2fI8BMoKr3WGlccrffOxf/A2rk9T1jt+J9y3sF9+V0Oe5Duynuy3Y77tdN1kTkOeBt4F5v2Se4khR/EpE7gN05xHAV7kO2OVAO+JBTE5FDQEfch1vpjHi8x7N7I2dun8Mx78F92dQGKgOzOfkD+DNgrBdPddwH1ICcniPuw3+6t68KuA+ef3qPtffiv8KLPwpo4j02B+9LKrvYcR/eX3j7jMGVSt3pE0cKcLf3ut2LS+Qkh+e8BZdwZDznBT7n8DHgY591rwN+zuX83oH7YqoHlAemAe9nWX+Sd/7K4K6nI971UgL3odzK5/Xd752nMNyH9RSfY0Xg86WScX5w1802oL23vJx3v5+3n9a4L99YP89lKjDYi++v3mtW2dvvYaCxTzzNcjjHw4EPsizLfI19jtPPe82ew31xjMElHV2981Q+r+sqm2Nn7HuQ9/zLeOfpCm/f1XBJ+qtZ3/8+9096rb3138C951rhkrjL/Hnu5PMayeNzKrtE5BBwoc/9dsAR7//BwDdZ1v8KeDiH/c/BXWMZn0H/znguuPfrfqAb7v17hXe/ms+2W4Fm3nkv4S37FZcIVsQl/xtwn7Vh3vN+N5f3V9ZrJuvnTSfgfC+eFrjPz+tz2V/mPnDX0gGgrxfLzd79Kj7H3oj7kVfGu/+C99gA3DVYFnf9tgXC8/qesVvxvuWniPB9XLXLFcBa3JsyO5OAW0XkPOBS4HM/95+AewNkpxfuTblKVY/iPtCy+kJVF6hquqqe8POYuemF+0Depqq/434NASAiNXAfOg+p6lFV3YP7Qrgpux159dL9gcGq+ruqHgGe91n/TmCCqn7nxb9DVdflFaBXBXUT8ISqHlHVLcBLuA+QDL+p6jh1dfrv4b4ka+Sy29d9nvM/+KNK7gOgm4iEe/f74q6JnPQBXlbVTaqaiCtpuSlLEftw7/wdB3oDs1T1I1VNUdX9qvqTz7qfqeoSdcX6k3Ffehm6Af9RVfVZ1hOXKP5ZVZd4y7oDW1T1XVVNVdXluC+Unn6eyz24ayJFVT/GJedXe4+lA81FpIyq7lTVM6kG3OzFmAZ8jEsMR6hqkqrOxJUsNPDjuspOgqqO9p7/cVX91bvuklR1L64t1aX+BCkitXHJ/99U9YT3er2DT/VsHvJ7jeRXeVwykuEQUN47b1kfy3i8Qi77e9/nM+jvQC/vurkFmKGqM7z373dAPO66zDBRVVd75z3FW/auqm5U1UO4kuONqjrLu8Y/xSXKp0VV56jqz148K4GP8PN1xV3Tv6jq+168H+F+lF3js867qrrBe10+4Y/3YwruR0QDVU1T1aWqevh0n4cpHvJTR/w+7tdPXVyykS1VnS8i1XBVN1+p6nGf9mG5icIV72YnEljqc/+3bNbZ5s9B8iEyyz59j1kH96tmp89zC8klhmq4XwhLfdvK4X4xgPuiOZ32DVW9OHxj+w13LjNk1p2r6jHv+OVz2WfW5xzpbZsgIguAG0TkM+DPwIO57Ccym7jCODkJ8j1WbU5tS+TLtw3AMU5+Dt1wpWS+HgImqeoqn2V1gAtF5KDPsjDcte3PudyRJdn5DYhU1aMi8lfgEWC8d54e9ieZzMFun/+PA6hq1mXlyfu6ys5J16iXVL8GXIL7Eg7B/fr1RySQkQBl+A1X8uDv9vm5RvIrEQj3uR8OJKqqikjWxzIeP0LOsr43SuCumzq4ZNb3i7oErhQ1u20zZH1Ns3uNT4uIXAi8gCvBKYkr8fK3p1LW1wVy+Vzh5Pfj+7j38hTvx+gHwFM+yZcxp/C7RERVf8PVfXfDFaHm5gPgYXJJWLLRA5iXw2M7cRd3hujsQszHsfyR2zG3AUm4etHzvFu4qjbLIZZ9uA+WZj7rV1TXYDdjf/VziCO357UP9wukTpY4cyqt8kfW55zgc/893K+/nrgGxxnHyS7GhGziSuXkD1vf7XI7BznyekRdimv34asncL2I+CZL24D/+rwG56nrAXIv/p3LKDk5q848P6r6rapegStxWgeMyyHkgrxO87qu/Dn+896y81U1HPf6Si7r+0oAKouIbylCfq6//F4j+bUa11A1Q0v+aLC+GmiR5fVsQe4N2rO+N1Jwr8E2XGmJ73VVTlVf8Fn/TJ7HUe+vbweBmnns+0NclV1tVa2IazskuazvK+vrAn6+rl5p4TOqGotrZ9Id/0vITDGV39bbd+Lqf4/msd4oXBXO3NxWEpFQEakrIqNxdZrP5LDqJ8DtIhIrImWBp/MXdrZ24+qmc/IJ8ICI1BKRSsDjGQ+o6k5cA62XRCRcREJEpL6IZBR97gZqZbTAV9V03BfTKyJSHUBEokTkSm/98UA/Eeni7StKRJrkFadXdP8J8A8RqSAidYAhuETwdN3nPefKuFKtj30e+xxogysJ8U0y9+KqJnzj/AgY7L2+5XFfeB9rzj0mJgOXi0gvEQkTkSp+dqeMA1ZmU/ybAHQBHhSRe71lXwGNRKSviJTwbheISFM/z2V13DVRQkR64tpMzRCRGiJynYiUwyWoid75yM5uIKYgek74cV35owIu3kMiEoVrFJ413pyuv224xpb/FJHSItIC9xnh7/WX32vkFCJSSkRKe3dLenFkfOFOAoZ45yQS9+NoovfYHFwDzwe8fdzvLf8hl8Pd4vMZNALXuD4N93yvEZErvc+00iLSSURq+fs8cuNVme3wjh8qIndwctJ+0ueNpwKutOqEiLTHVX1myO796msG7n3S23sv/hWIxb1/ciUinUXkfK/K6jAuWcvpvWAMkM9ExKvPjPdjvd9V9fssxdi+LvaKRg/jPhDCgQtU9ecc9vcN8CruQ+JXcv+w8Ndw4D0ROSgivbJ5fBzwLa4F/DJOLQW6FVfkmdHDZCru1zBefKuBXSKyz1v2Ny/2H0XkMDALaOw9vyW4xomv4Oqp/8sfv0heA24UkQMiMiqbOAfhfjFtwjUK/hCY4N8pyNaHuCRrE66qJHMgMa8++N+46rlpPsuP4fUK8M7nRV4MGdV5m3Et+wfldFBV3YorbXsYV0X3Eyf/ms1Jjt12vX12AR4Xkbu8KoSuuDYUCbji5ZG4YmvI+1wuBhrifgX/A7hRVffj3kdDvH3+jiuhuZfsZRSP7xeRZX48v7zkeF356RlccnkI1ysp63X+T2Co97o+ks32N+MaPybgGnA/raqz/Dx2vq6RHKzHlQpF4d6vx/njvTMW13DyZ1wj7K+9ZahqMq5r76243h134BpzJudyrPdxicwuXOPcB7x9bcM13n4S9yW/DZfQFWQ33bu9fe7HNXr1HTMnu8+bgcAIETkCDMMl2XjxZvd+xefx/biSjIe94z0GdFfVfeStJu6z8DCuLeF/yb0tmTGZvQyMQUS24Fri5/hFIiLDgEaqestZCywXIrIGlxCsCfBxbsedm7hAHscUTiIyB9dL5p1gx2LMuSbow5WbosOrrrmTk3uSBI1XFD0p0EmIMcaYwLER/oxfRORuXJHzN6qaa9ufs0VVk7M0CDTGGFPEWNWMMcYYY4LGSkSMMcYYEzRFoo1I1apVNSYmJthhGGNMkbJ06dJ9qlot2HEYk5sikYjExMQQH59nr2FjjDE+RCS7UaiNKVSsasYYY4wxQWOJiDHGGGOCxhIRY4wxxgSNJSLGGGOMCRpLRIwxxhgTNJaIGGOMMSZoLBExxhhjTNBYImKMMYXQiRMwYQJMmRLsSIwJLEtEjDGmENm7F559FurUgTvvtETEnPssETHGmEJg7VoYMACio2HYMGjXDmbNgs8+C3ZkxgRWkRji3RhjzkWq8MMP8PLLMGMGlC4Nt94KDz0ETZsGOzpjzg5LRIwx5ixLTnZVLi+/DCtWQPXq8MwzcO+9UM2mqDPFjCUixhhzluzfD2PHwuuvw86d0KwZjB8PvXu70hBjiqOAJyIiEgrEAztUtbuI1AWmAFWApUBfVU0OdBzGGBMsGzbAq6/CxIlw/Dh07Qrvvuv+igQ7OmOC62w0Vn0QWOtzfyTwiqo2AA4Ad56FGIwx5qxShf/+F667Dpo0cSUfN98MP/8M334LV15pSYgxEOBERERqAVcD73j3BbgMmOqt8h5wfSBjMMaYsyklBSZPdr1eOnWChQvh73+HrVtdMtK8ebAjNKZwCXTVzKvAY0AF734V4KCqpnr3twNR2W0oIv2B/gDR0dEBDtMYY87MgQMwbhyMGgU7dkDjxq49SN++UKZMsKMzpvAKWImIiHQH9qjq0tPZXlXfVtV2qtqumjUjN8YUUhs3wgMPQO3a8Le/uQTkq69gzRro39+SEGPyEsgSkY7AtSLSDSgNhAOvAeeJSJhXKlIL2BHAGIwxpsCpuiqXl16Czz+HsDDX/mPwYGjVKtjRGVO0BKxERFWfUNVaqhoD3AT8oKp9gNnAjd5qtwFfBCoGY4wpSKmp8PHHcNFFEBcHc+bAE0/Ali3w3nuWhBhzOoIxjsjfgCki8hywHBgfhBiMMcZvhw7BO++49h9bt0KDBjBmDNx2G5QrF+zojCnazkoioqpzgDne/5uA9mfjuMYYcya2bHHJxzvvwJEjcOmlMHo0dO8OITZTlzEFwkZWNcaYLH780Q2//u9/u4SjVy8YMgTatg12ZMaceywRMcYYIC3NzXT78suwaBFUrAiPPAL33+96xBhjAsMSEWNMsXbkCEyYAK+9Bps3Q716rjqmXz8oXz7Y0Rlz7rNExBhTLG3b5hKOt9+Gw4ehY0f417/ckOyhocGOzpjiwxIRY0yxEh/vql8++cTdv/FGN/7HhRcGNy5jiitLRIwx57y0NPjyS5eAzJsHFSrAgw+6EVHr1Al2dMYUb5aIGGPOWUePwsSJ8Oqr8OuvLul4+WW4804IDw92dMYYsETEGHMO2rEDXn/dTTp34ICrdnn+eejRww3HbowpPOwtaYw5ZyxfDq+8Ah99BOnpLvEYMgQ6dAh2ZMaYnFgiYowp0tLTYcYMV+Uye7brcnvffa79R716wY7OGJMXS0SMMUXSsWMwaZIrAdmwAWrVghdfhLvugvPOC3Z0xhh/WSJijClSdu1yE869+Sbs3++GXf/wQ9cNt0SJYEdnjMkvS0SMMUXCypWu9OPDDyElxQ08NmQIxMWBSLCjM8acLktEjDGFlip8+y289BLMmgVly8Ldd8NDD0GDBsGOzhhTECwRMcYUOidOwAcfuBKQNWsgMhL++U/o3x8qVw52dMaYgmSJiDGm0NizB954w9327oVWreD996FXLyhZMtjRGWMCIWCJiIiUBuYCpbzjTFXVp0VkInApcMhb9XZV/SlQcRhjCr81a1zpx/vvQ1ISdO/u2n906mTtP4w51wWyRCQJuExVE0WkBDBfRL7xHntUVacG8NjGmEJO1bX7ePll+M9/oHRp6NfPtf9o3DjY0RljzpaAJSKqqkCid7eEd9NAHc8YUzQkJbmRT19+GX7+GWrUgGefhXvugapVgx2dMeZsCwnkzkUkVER+AvYA36nqYu+hf4jIShF5RURK5bBtfxGJF5H4vXv3BjJMY8xZsG8fPPecm3iuXz+37N134bffYOhQS0KMKa4CmoioapqqtgJqAe1FpDnwBNAEuACoDPwth23fVtV2qtquWrVqgQzTGBNA69a50o7ateHvf4c2beC772DFCrj9diiV7U8RY0xxEdBEJIOqHgRmA1ep6k51koB3gfZnIwZjzNmj6uZ9ueYaaNoUJk6EW26BVavcvDCXX26NUI0xTsASERGpJiLnef+XAa4A1olIhLdMgOuBVYGKwRhzdiUnu54vbdrAZZfB4sUwfDhs3QrjxkGzZsGO0BhT2ASy10wE8J6IhOISnk9U9SsR+UFEqgEC/ATcE8AYjDFnwe+/w9ixMHo07NwJsbHwzjvQp4/rDWOMMTkJZK+ZlUDrbJZfFqhjGmPOrl9+gddec41Ojx2DK66ACRPgyiut6sUY4x8bWdUYky+qMG+e6347fTqEhbmSjyFD4Pzzgx2dMaaosUTEGOOXlBSYOtUlIPHxUKUKPPUUDBwIERHBjs4YU1RZImKMydXBg66h6ahRsH07NGoEb74Jt97qZsM1xpgzYYmIMSZbmza59h/jx8PRo9C5s0tAunWDkLPS8d8YUxxYImKMyaQKixa56pfPPnMJx803w+DB0PqUpufGGHPmLBExxpCaCtOmuQRk8WI47zx47DG4/36Iigp2dMaYc5klIsYUY4cPu6qX115zc77Urw+vvw633Qblywc7OmNMcWCJiDHF0G+/ucan48bBkSNwySUuGeneHUJDgx2dMaY4sUTEmGJk8WJX/fLvf7v7vXq59h8XXBDcuIwxxZclIsac49LS4IsvXAKyYAFUrOgGHxs0yM2Ia4wxwZTvREREQoDyqno4APEYYwrIkSNu6PVXX4XNmyEmxv1/xx1QoUKwozPGGMev0QBE5EMRCReRcrjZcteIyKOBDc0Yczq2b3c9XmrXhgcfdKOeTp0Kv/7q7lsSYowpTPwdlijWKwG5HvgGqAv0DVhUxph8W7rUzflSty689BJ07erGBFmwAG64wRqhGmMKJ3+rZkqISAlcIvK6qqaIiAYwLmOMH9LT4csvXfuPuXNdacegQfDAA64qxhhjCjt/E5GxwBZgBTBXROoA1kbEmCA5ehTeew9eecVVuURHu1KQO+90jVGNMaao8CsRUdVRwCifRb+JSOfAhGSMyUlCghtw7K234MAB1+12yhRX9RJmfeCMMUWQXx9dIlIFeBqIAxSYD4wA9uexXWlgLlDKO9ZUVX1aROoCU4AqwFKgr6omn+6TMOZc99NPrvplyhQ3HHuPHq4LbocOIBLs6Iwx5vT521h1CrAXuAG40fv/Yz+2SwIuU9WWQCvgKhG5CBgJvKKqDYADwJ35DdyYc116Onz9NXTp4iacmzYN7r0XfvnFDUjWsaMlIcaYos/fRCRCVZ9V1c3e7TmgRl4bqZPo3S3h3RS4DJjqLX8P1wjWGAMcOwZjx0JsrBtyff16GDkStm1zw7DXrx/sCI0xpuD4m4jMFJGbRCTEu/UCvvVnQxEJFZGfgD3Ad8BG4KCqpnqrbAdOmd9TRPqLSLyIxO/du9fPMI0punbtgmHDXMPTe+6BcuVg8mQ3GNljj0GlSsGO0BhjCp6o5t0LV0SOAOWAdG9RCHDU+19VNdyPfZwHfAb8HZjoVcsgIrWBb1S1eU7btmvXTuPj4/OM05ii6OefXe+XyZMhJQWuuQYefthNRGdVL+ZMiMhSVW0X7DiMyY2/vWbOeCxGVT0oIrOBi4HzRCTMKxWpBew40/0bU5SowsyZrsvtd99BmTJw111u5NNGjYIdnTHGnD1+d/gTkWuBP3l356jqV35sUw1I8ZKQMsAVuIaqs3GNXqcAtwFf5DdwY4qiEydcycfLL8OaNW749eefh/79oUqVYEdnjDFnn7/dd18ALgAme4seFJGOqvpEHptGAO+JSCiuOucTVf1KRNYAU0TkOWA5MP70wjemaNizB958E8aMgb17oWVLNyDZTTdByZLBjs4YY4LH3xKRbkArVU0HEJH3cAlEromIqq4EWmezfBPQPn+hGlP0rFnjZrydNAmSkuDqq934H507W/sPY4yBfFTNAOcBv3v/2yDSxuRAFb7/3lW/fPMNlC4Nt90GgwdDkybBjs4YYwoXfxORfwLLvcamgmsr8njAojKmCEpKgo8+cgnIzz9D9eowYoTrilutWrCjM8aYwsnfXjMficgcXDsRgL+p6q6ARWVMEbJ/v5v75fXX3VggzZvDhAlw882uNMQYY0zO/G2sKkAXoJ6qjhCRaBFpr6pLAhueMYXX+vWu/cd778Hx43Dlle7/K66w9h/GGOMvf6tm3sANZnYZbrK7I8C/+aOExJhiQRX++183/sdXX7keL337wkMPuZIQY4wx+eNvInKhqrYRkeUAqnpARKzToSk2kpPhk09c+4/ly6FqVXj6aTcJXY08Z10yxhiTE38TkRRvLBCFzIHK0nPfxJii78ABNwHd6NGQkOB6vbz9NtxyixsN1RhjzJnxNxEZhZsnprqI/AM3KurfAxaVMUH2669uptsJE9xsuF26wLhxcNVVEOLvVJHGGGPy5G+vmckishTXYFWA61V1bUAjM+YsU4X58131yxdfQFgY9O7txv9o2TLY0RljzLnJ314zd6rqeGCdz7IXVNXGEjFFXkoK/PvfrgFqfDxUrgxPPgn33efmgjHGGBM4/lbN3CAiJ1R1MoCIjAFshARTpB08CO+8A6NGwbZt0LAhvPEG3HorlCsX7OiMMaZ48DsRAaaLSDpwFXBQVe8MXFjGBM7mza79x/jxkJgInTq5yeiuvtrafxhjzNmWayIiIpV97t4FfA4sAJ4Rkcqq+nv2WxpT+Cxa5Np/TJvmEo6//tW1/2jbNtiRGWNM8ZVXichSXJdd8fl7tXdToF5AozPmDKWmwmefuQTkxx/hvPPg0Ufh/vuhVq1gR2eMMSbXRERV656tQIwpSIcPu6qX116D336DevXcWCC33w7lywc7OmOMMRn8bSNiTJGwdatrfDpunEtG4uLglVfg2mshNDTY0RljjMkqYE3zRKS2iMwWkTUislpEHvSWDxeRHSLyk3frFqgYTPGxZAncdJMr+Xj1VejWDRYvhnnzoEcPS0KMMaawyquxaglVTTnNfacCD6vqMhGpACwVke+8x15R1X+d5n6NASAtDaZPd+0/5s+H8HA3+dwDD0B0dLCjM8YY44+8qmYWich24D/Af1R1i787VtWdwE7v/yMishaIOt1AjcmQmAjvvutKPjZtgjp1XPXLHXe4ZMQYY0zRkWvVjKq2Ax7y7r4qIv8TkVdEpKuIlPL3ICISA7QGFnuL7heRlSIyQUQq5bBNfxGJF5H4vXv3+nsocw7bvh3+9jeoXduVelSvDp9+6uaFeeghS0KMMaYoElX1f2WREsAluEHNOgF7VfXqPLYpD/wX+IeqThORGsA+XPffZ4EIVb0jt320a9dO4+Pj/Y7TnFt273YJyOTJkJ4Of/kLDBkCF18c7MiMKdxEZKn3g9KYQitfvWa89iI/eDdEJNeqFi9x+TcwWVWnefvY7fP4OOCrfMZsiglVeP99V9px7Jib++XBB6GudSo3xphzxhl131XVHTk9JiICjAfWqurLPssjvPYjAD2AVWcSgzk3bd0KAwbAf/4DHTq4MUGaNAl2VMYYYwpaIMcR6Qj0BX4WkZ+8ZU8CN4tIK1zVzBZgQABjMEVMejq89ZarilF1Y4IMHGjdb40x5lwVsEREVefjhoTPakagjmmKtl9+gbvugrlz4fLL3aBkMTHBjsoYY0wg+ZWIiEgj4FGgju82qnpZgOIyxUhqqut+O2wYlCrlqmH69QPJLo01xhhzTvG3RORT4C1gHJAWuHBMcbNyJdx5J8THw/XXw5gxEBkZ7KiMMcacLf4mIqmq+mZAIzHFSlISPP+8u1WqBB9/DD17WqXp5KcAACAASURBVCmIMcYUN3kN8V7Z+/dLERkIfAYkZTyuqr8HMDZzjlq82JWCrF4Nt9ziRkitUiXYURljjAmGvEpEluJ6t2T8Tn3U5zEF6gUiKHNuOnYM/v53l3hERsLXX7vJ6YwxxhRfuSYiqmpDR5kCMXu26xGzaRPccw+MHGlDshtjjMljrpkMInKfiJznc7+SV1VjTK4OHXIDk112mWv/MXs2vPmmJSHGGGMcvxIR4G5VPZhxR1UPAHcHJiRzrvjqK2jWDN55Bx55xPWQ6dQp2FEZY4wpTPxNREK9IdsBEJFQoGRgQjJF3b590KcPXHON6xGzaBG8+CKULRvsyIwxxhQ2/iYi3wIfi0gXEekCfAT8J3BhmaJIFaZMgaZN4dNP4emnYelSaN8+2JEZY4wprPwdR+QxoD9wr3f/O+CdgERkiqSEBLj3Xpg+HS64wI2Oev75wY7KGGNMYZdnIuJVw0xS1T640VWNyaQKEybAww+7Qcr+9S948EEIC+R0isYYY84ZeX5dqGqaiNQRkZKqmnw2gjJFw6ZN0L8/fP89XHqpa5TaoEGwozLGGFOU+Pu7dROwQESmA0czFqrqywGJyhRqaWkwejQ89RSEhsJbb8Hdd0OIvy2OjDHGGI+/ichG7xYCVAhcOKawW7vWDc++aJEbFfWtt6B27WBHZYwxpqjyKxFR1WcARKS8dz8xr21EpDYwCaiBGw7+bVV9zZu/5mMgBtgC9PLGJTGFWEoK/N//wYgRUL48vP++66Jrk9QZY4w5E/6OrNpcRJYDq4HVIrJURJrlsVkq8LCqxgIXAfeJSCzwOPC9qjYEvvfum0Js2TLXE2boULj+elcqcsstloQYY4w5c/7W6r8NDFHVOqpaB3gYGJfbBqq6U1WXef8fAdYCUcB1wHveau8B159O4CbwTpyAJ55w44Ds3g2ffQYffwzVqwc7MmOMMecKf9uIlFPV2Rl3VHWOiJTz9yAiEgO0BhYDNVR1p/fQLlzVjSlk5s93bUE2bIB+/eCll9woqcYYY0xB8rdEZJOI/F1EYrzbUFxPmjx57Ur+DTykqod9H1NVxbUfyW67/iISLyLxe/fu9TNMc6YSE2HQIPjTn9y4IDNnunFCLAkxxhgTCP4mIncA1YBpuKSiKtAvr41EpIS3/mRVneYt3i0iEd7jEcCe7LZV1bdVtZ2qtqtWrZqfYZozMXMmNG8OY8a4ZGTVKrjiimBHZYwx5lzmbyJyuao+oKptVLWtqj4E5PoV5U2SNx5Ym2W8kenAbd7/twFf5DdoU7AOHHDVL1deCaVLw7x58NprrneMMcYYE0j+JiJP+LnMV0egL3CZiPzk3boBLwBXiMgvwOXefRMk06ZBbKzrjvvkk/DTT9CxY7CjMsYYU1zk2lhVRP4MdAOiRGSUz0PhuO65OVLV+UBOHTy75CdIU/B274b774epU6FVK5gxA1q3DnZUxhhjipu8es0kAPHAtcBSn+VHgMGBCsoEjip88AE89JBrmPqPf8Cjj0KJEsGOzBhjTHGUayKiqiuAFSJyAPhKVdPPTlgmELZuhXvugW++gQ4dYPx4aNIk2FEZY4wpzvxtI9IL+EVE/k9E7KuriElPhzffhGbN4L//dQ1R5861JMQYY0zw+ZWIqOotuAHJNgITRWSRN86HTYBXyP3yC3TuDAMHwkUXuS65DzzgZs01xhhjgs3vidu9wcimAlOACKAHsExEBgUoNnMGUlPhxRehRQtYscJVw8ycCXXrBjsyY4wx5g9+DfEuItfiBjBrgJtRt72q7hGRssAaYHTgQjT5tXKlG549Ph6uuw7eeAMiI4MdlTHGGHMqf+eauQF4RVXn+i5U1WMicmfBh2VOR1ISPP+8u1Wq5Cao69nTZsk1xhhTePmViKjqbSJS0ysZUeB/qrrLe+z7QAZo/LN4sSsFWb0abrkFXnkFqlYNdlTGGGNM7vxqI+KVeiwB/gLcCPwoIncEMjDjn2PH4OGHXXfcQ4fgq6/cKKmWhBhjjCkK/K2aeQxorar7AUSkCrAQmBCowEze5syBu+6CjRthwAD4v/+D8PBgR2WMMcb4z99eM/txo6lmOOItM0Fw6JBLPDp3dvdnz4a33rIkxBhjTNHjb4nIr8BiEfkC10bkOmCliAwByDK7rgmgr792ScjOna5KZsQIKFs22FEZYwIhXdM5kXqCsiXsTW7OXf4mIhu9W4YvvL82oNlZsm+fmx9m8mQ3Quq0adC+fbCjMsacrsNJh0k4ksCOwztIOJLg/j+y46S/O4/s5NrG1zK119Rgh2tMwPjba+aZQAdisqfquuEOGuSqZJ5+Gp58EkqWDHZkxpjsJKUmsTNxZ2aCkZFUZP0/MTnxlG3DS4UTVSGKyAqRdIrpRGT5SNpGtg3CszDm7PG3RMQEQUIC3HsvTJ8OF1zgRkc9//xgR2VM8ZSWnsaeo3tOLb04vIOExITM//cfP7X5XKnQUkRWiCSyQiStaraiW4NuRFaIJCrcJR1RFaKIqBBB+ZLlg/DMjAkuS0QKIVWYMMG1AUlKckO1P/QQhNmrZUyBU1UOnjiYZ4KxK3EXaZp20rYhEkKNcjWIrBBJzHkxdKjVITO5yEgwIitEUrlMZcRGFjQmWwH9ahORCUB3YI+qNveWDQfuBvZ6qz2pqjMCGUdRsnkz3H03fP89/OlP8M470LBhsKMypmg6nnL81OoRL8HwbZtxPPX4KdtWKl0pM6mIrRabmVRk/I2sEEmN8jUIC7FfCMacCX/nmmkEvAnUUNXmItICuFZVn8tj04nA67j5aXy9oqr/ym+w57K0NHj9ddf+IzQU3nwT+veHEL+nJTSm+EhNT2V34u6TSy+OJJySYBw4ceCUbcuElclMMC6IuiDbBCOyQiRlSpQJwjMzpvjxN5UfBzwKjAVQ1ZUi8iGQayKiqnNFJOZMAiwO1q51w7MvWgR//jOMHQu1awc7KmPOPlVl//H9J5deZNOjZHfibhQ9adtQCSWiQgSRFSJpVKWRa+zpk2BkJB8VS1W0ahJjChF/E5Gyqroky5s39QyOe7+I3ArEAw+r6ik/W0SkP9AfIDo6+gwOVXilpLjRUEeMgPLl3dDsffrYJHXm3JSYnHhKgpFdj5LktORTtq1atmpmUtGqZqtsE4xqZasRGhIahGdmjDkT/iYi+0SkPm4wM0TkRmDnaR7zTeBZb1/PAi8Bp8xbo6pvA28DtGvXTrM+XtQtW+ZKQX76yc2QO3o01KgR7KiMyb/ktGR2Je7KtvTC9//DSYdP2bZciXJEhUcRVSGKjrU7nlQ9kpFgRJSPoFRYqSA8M2PM2eBvInIfLiloIiI7gM3ALadzQFXdnfG/iIwDvjqd/RRVJ07AM8+4njDVqrmByXr0CHZUxpwqXdPZd2xfrgNuJRxJYM/RPadsWyKkRGZC0axaM7rW63pScpFRolGhlI2JaExx5++AZpuAy0WkHBCiqkfy2iYnIhKhqhmlKT2AVae7r6Jm/nxXCrJhA/TrBy+9BJUqBTsqU9yoauaonjmVXuw4vIOdiTtJTT+5BlYQqpernplItI9sn22CUaVsFULEWlobY/Lmb6+Z84BbgRggLKOtiKo+kMd2HwGdgKoish14GugkIq1wVTNbgAGnF3rRkZgITzwBY8ZAdDTMnAlXXBHsqMy56ETqCXYe2ZnreBgJRxI4mnL0lG0rlqqYmVR0rtuZyPInD7gVWSGSmuVrUiK0RBCemTHmXOVv1cwM4EfgZyDd352r6s3ZLB7v7/bngpkzXTfcrVvh/vvh+eddw1Rj8sN3VM/cuqzmNqpnVHgUrSNa071R91MG3IqsEEm5kuWC8MyMMcWdv4lIaVUdEtBIzjEHDsCQITBxIjRuDPPmQceOwY7KFDYZo3qeMuBWliqT3Eb1jAqPom6lusRFx52SYESFR1GpdCXrrmqCaunSpdXDwsLeAZoDVmdXvKQDq1JTU+9q27btqQ3K8D8ReV9E7sY1LE3KWKiqv595jOeezz6DgQNh715XJTNsGJQuHeyozNl2LOVYnuNhJBxJ4ETqiVO2rVymcmZS0bx681NKL6LCo6herrqN6mmKhLCwsHdq1qzZtFq1agdCQkLOuV6QJmfp6emyd+/e2F27dr0DXJvdOv5+iiUDLwJPQeYoQgrUO+MozyG7d7tZcj/9FFq1gq+/hjZtgh2VKWip6ansStyVZ4Jx8MTBU7bNGNUzqkIUF0ZdmG2CEVE+wkb1NOea5paEFE8hISFarVq1Q7t27Wqe0zr+JiIPAw1UdV/BhHZuUYUPPnAT0yUmwj/+AY8+CiWsTV+R4juqZ24DbuU2qmdUhSgaV2lM55jOpwy4ZaN6mmIsxJKQ4st77XOskvM3EfkVOFYgEZ1jtm6Fe+6Bb76Biy+G8eOhadNgR2WySkxOzHM8jNxG9cxIKtrUbHPKgFtRFaKoWraqjeppjDGnwd9E5Cjwk4jM5uQ2Irl23z2Xpae7OWEee8z9/9prcN99bsI6c/YkpyVndlfNKcHYcXgHR5JPHfqmfMnymQlGXHQckeVPTTBqlq9po3oaUww9/vjjNV944YVdBbGvIUOGRJYvXz5txIgRu7N7vHfv3tG33377/qFDh9b617/+te1Pf/pTofvh/+abb1Z+5ZVXagKUK1cu/Y033vjt4osvPnXa6tPgbyLyuXczwC+/wF13wdy5cPnl8PbbULdusKM6N6kqG/ZvYNH2RWw5uOWUKdz3Htt7yja+o3o2r96crvW6njLgVmSFSBvV0xiTo1GjRkXkJxFJT09HVQk9jV+jy5YtKz9p0qSt+d7wLGrQoEHSggUL1lerVi3tk08+CR8wYECdlStXriuIffs7sup7BXGwoi41FV55xfWCKVXKVcP062eT1BWklLQUlu1cxvyt85m/bT4Lti7ITDYyRvWMCo+idsXaXBh1YbYJho3qaYzxtX79+pJXXXVVw/PPP//YqlWryjZq1Oj4p59+umX27NnlRo0aVX3WrFkbAT777LPwN954o1rDhg1PJCUlhTRp0iS2UaNGx6dPn755+PDhNSZPnlwVoG/fvnuHDRu2Z/369SWvvPLKRq1bt078+eefy82YMeOXlStXlh42bFhUWlqaVK5cOXXRokUbANauXVumffv2jRMSEkrec889u4cOHboHYNmyZaXr1at3Iizsj6/jtLQ0evXqFRMVFZU8atSohGnTpoWPGDEiMjk5WerUqZM0ZcqULRUrVkz/4osvKjz++OO109LSaNmy5bFJkyb9VqZMGY2Kijr/mmuuOfDDDz+ElypVSj/66KNNzZs3T5owYUKlf/7zn5EhISFaoUKFtPj4+PX+nsMrrrgicxTEzp07H73//vtLFtDLk3siIiKfqGovEfkZOKWhkaq2KKhACruff4Y77oD4eLjuOnjjDYiMDHZURd/hpMP8uP1Hl3hsnc+P23/keKor7atfqT7dGnYjLjqOjrU70qByAxvV05gi7o47qL1qFWULcp/Nm3NswgS25bbOli1bSo8dO3ZL165dj/bs2TPmxRdfrDZ8+PDdDz74YHRCQkJYZGRk6oQJE6r069dvX+/evQ9NnDix+rp169YAzJs3r+yHH35YZenSpWtVlbZt2zbt0qXLkapVq6Zt3bq11Pjx4zd36dJlS0JCQtj9998fM2fOnHVNmjRJ3r17d2bxyK+//lp64cKF6w8ePBjatGnT5o8++ujeUqVK6fTp0yt27dr1UMZ6KSkpcv3119eNjY09PnLkyF07d+4Me/755yPmzp27ITw8PP2pp56q+eyzz9YYMWLErgEDBtSdOXPm+hYtWiT16NEj5sUXX6w2bNiwPQAVK1ZM3bBhw5rXX3+9yqBBg2rPnj371xdeeCFi5syZG+rWrZuyb9++025IMHr06KqdO3c+lPea/smrRORB72/3gjpgUZOc7HrBPP+8mxdmyhTo1ctKQU5XwpGEzKRj/tb5rNi9gnRNJ0RCaF2zNf3b9s9MPCIqRAQ7XGPMOaJmzZrJXbt2PQrQt2/f/aNGjaoeEhKyu1evXvvHjRtX+b777tu/bNmy8tOmTducdds5c+aU79at28Hw8PB0gKuvvvrA7NmzK/Ts2fNgREREcpcuXY5665Vr3779kSZNmiQD1KhRI3MUwq5dux4sU6aMlilTJrVy5cop27dvD6tfv37KrFmzwj/44IMtGesNHDiwzvXXX//7yJEjd2Xsc+PGjaXbt2/fBFyi0rZt28QVK1aUrlWrVlKLFi2SAG6//fb9Y8aMqQ7sAbjtttt+B7j77rt/Hzp0aG2Adu3aJfbp0yfmhhtuONCnT58Dp3Mev/zyywoffPBB1YULFxZItQzkkYj4TE43UFX/5vuYiIwE/nbqVueOJUtcKcjq1dCnD7z6KlStGuyoig5VZd2+dczfOp95W+cxf+t8Nh907/GyJcpyUa2LGHrJUOKi47io1kXWZsOYYiCvkotAydptPuP+vffeu//qq69uULp0ab3mmmsOlMjnuAtly5b1a9qTUqVKZdYqhIaGkpqaKkeOHAk5fPhwaExMTErGY+3atUucN29e+LFjx3aXLVtWVZW4uLjDX3755UkJ0qJFi3IdbCgk5I/qaRFRgA8//HDrDz/8UG769OkV27ZtG7t06dI1NWvWzEyWBg0aFPXdd99VBMgoDfK1ePHiMgMHDqzz9ddf/+K73ZnytyI9uyna/lxQQRQ2x47BI4+47rgHD8JXX7lxQiwJyV1yWjKLti3ixQUvct2U66j2YjVi34il/1f9+Xbjt7SOaM3LXV9myV1LOPi3g3x/6/c80/kZrqh/hSUhxpiA2rlzZ8lZs2aVA5g8eXLlDh06JALExMSk1KhRI+Wll16K6N+/f+ZYWWFhYZqUlCQAnTt3TpwxY8Z5XuIQMmPGjEqdO3c+pStep06dji5ZsqTCunXrSgL4Vs1k5+uvv64QFxd30n4GDBiwr2vXroe6d+9ePyUlhU6dOh2Nj48vv2rVqlIAhw8fDlm5cmWpli1bntixY0fJjOWTJk2qcskll2Tua9KkSZUBxo8fX6l169ZHAVavXl3qsssuO/rqq68mVKpUKXXTpk0ntfMYPXr0jnXr1q3JLgn55ZdfSvbs2bP+hAkTNmeUwhSUvNqI3AsMBOqJyEqfhyoACwoykMJizhzXI2bjRhgwAEaOhIoVgx1V4XTwxEEWbVuU2bB0yY4lmcOVN6zckOsaX0dcdBxx0XE0qNzABvIyxgRNTEzMidGjR1fv379/2YYNG5545JFHMrvc3XTTTfvHjBkT1qZNm8z5Fvr06bO3adOmsc2bNz82ffr0zb17997fpk2bpuAaq3bs2PH4+vXrT/oij4yMTB01atSWHj16NEhPT6dKlSopCxcu/CWnmGbMmFGxV69ep1SRDB8+fPfgwYND//KXv9T9/PPPN48dO3bLTTfdVC85OVkAnn766R0tWrRIeuutt7b07NmzfkZjVd/ndODAgdBGjRrFlixZUqdMmbIJYPDgwbW2bNlSSlUlLi7u8EUXXeR399uhQ4dGHDx4MGzQoEF1wCVqq1atWuvv9rkR1ZwHuxORikAl4J/A4z4PHTmb88y0a9dO4+PjA3qMQ4fcmCBvvw3168O4cdC5c0APWeRsP7ydeb/Ny0w8ft79M4oSKqG0iWiTmXR0rN2RGuVrBDtcY4o9EVmqqu2CHceKFSu2tGzZMmgjc69fv75k9+7dG/7yyy+rs3v81ltvjW7duvWxwYMHn9UYY2Njmy5fvnydb7VNQYiKijo/Pj5+bURERGpB7vdMrFixomrLli1jsnssrzYih4BDwM0BiKvQ+PprV/qxcyc8/DCMGAFlC7RNd9GTrums2bvmpIalvx36DXADgV1c62Ju6HQDcdFxXBh1oU0hb4wpkpo1a9a0TJky6WPHjj3rbVfWrFlTICUKRV2xnrpz3z43P8zkydCsGUybBu3bBzuq4DiReoL4hPjMpGPBtgWZk7bVLF+TS6IvYcjFQ4iLjqNFjRY266sxpsho3Lhxck6lIatXrz7nkoEdO3b8HOwY8iOg3yYiMgHX9XePqjb3llUGPgZigC1AL1U9rW5Ep0sVPvnEzZR74AA8/TQ8+SSULLDhWQq/A8cPsHDbwsweLf9L+F/mPCtNqjbhxqY3EhcdxyV1LqHueXWtfYcxxpiACPTP2onA68Akn2WPA9+r6gsi8rh3/6x1A05IgIED4YsvoF07+P57OP/8s3X04FBVth7a+kc1y7b5rNqzCoCwkDDaRbbjgfYPEBcdR4faHahWrlqQIzbGGFNcBDQRUdW5IhKTZfF1QCfv//eAOZyFREQVJkxwbUCSkuDFF121TNg5WMOQlp7Gqj2rMpOO+Vvns/3wdgDCS4XToXYHbmp2E3HRcVwQdQFlSxTzBjHGGGOCJhhfwzV8BkrbBWTbvUJE+gP9AaKjo8/ogJs3Q//+MGsW/OlP8M470LDhGe2yUDmecpz/JfzP9WjZNp+F2xZyOOkwAJEVIrkk+hJXzRJ9Cc2rN7fp6o0xxhQaQZ0ZTF3f4Wy7Lanq26raTlXbVat2elUFaWnw2mvQvDn8+KObH2b27KKfhOw7to/p66fz2HeP0WF8Byq+UJFLJ17K0NlD2XZoGzc3v5n3e7zP5gc3s33wdqbcOIX7299Py5otLQkxxhg/PP744zULal9DhgyJHDZsWI5jGvTu3Tt65syZ5dq3b9947ty5hbKIevny5aVbtWrVpGTJkm2yPpepU6eGx8TENI+Ojm7+5JNP5vu8BaNEZLeIRKjqThGJwBsXPxCWLHHVL1ddBWPHwhkWrASFqrL54OaTutGu3ecaeZcMLckFkRdk9mbpULsDlctUDnLExhhT9I0aNSrihRde2OXv+unp6agqoaH5/7G3bNmy8pMmTdqa7w3PourVq6e+9tprW6dOnVrJd3lqaiqDBw+O/vbbbzfUq1cvpWXLlk1vuOGGg23btj2R076yCkaJyHTgNu//24AvAnWgiy+GBQtgxoyik4SkpaexfOdyRi0eRa9PexH1chT1R9Xnts9v45PVn1C3Ul2ev+x55t4+l0OPH2L+HfN54fIX6N6ouyUhxhiTjfXr15esW7dus2uvvbZuvXr1ml111VX1jhw5EjJ9+vQKl19+ef2M9T777LPwK664ov7AgQOjkpKSQpo0aRJ77bXX1gUYPnx4jYYNGzZr2LBhsxEjRlTP2G9MTEzzHj16xDRq1KjZxo0bS06dOjU8Nja2aePGjWMvvvjiRhn7Xrt2bZn27ds3rlWr1vnPPfdc9Yzly5YtK12vXr0TYT4NFtPS0rjhhhtiHnjggUiAadOmhbdq1apJbGxs0z//+c/1Dh06FALwxRdfVGjatGlso0aNYnv27Blz/PhxATeg2T333FOrUaNGseeff37TjGHgJ0yYUKlhw4bNGjduHNuuXbvG+TmHUVFRqZdeeumxEiVKnFSLMWfOnHJ16tRJio2NTS5durT+5S9/+X3q1Knn5Wffge6++xGuYWpVEdkOPA28AHwiIncCvwG9AhlDhw6B3PuZO5p8lCU7lmQ2LF20bRFHkt10AdEVo+lctzNxtd2Ipc2qNyNEglqbZowxZ+SOL+6ovWrPqgKtfmhevfmxCddNyHVAsi1btpQeO3bslq5dux7t2bNnzIsvvlht+PDhux988MHohISEsMjIyNQJEyZU6dev377evXsfmjhxYvWMOVfmzZtX9sMPP6yydOnStapK27Ztm3bp0uVI1apV07Zu3Vpq/Pjxm7t06bIlISEh7P7774+ZM2fOuiZNmiT7zjXz66+/ll64cOH6gwcPhjZt2rT5o48+urdUqVI6ffr0il27dj2UsV5KSopcf/31dWNjY4+PHDly186dO8Oef/75iLlz524IDw9Pf+qpp2o+++yzNUaMGLFrwIABdWfOnLm+RYsWST169Ih58cUXqw0bNmwPQMWKFVM3bNiw5vXXX68yaNCg2rNnz/71hRdeiJg5c+aGunXrpuzbt69A6um3bdtWMioqKjnjfq1atZIXL15cPj/7CHSvmZxGZO0SyOMWZnuO7mHB1gWZiceynctITU9FEM6vcT59W/R1w6RHdyS6YhEpxjHGmEKuZs2ayV27dj0K0Ldv3/2jRo2qHhISsrtXr177x40bV/m+++7bv2zZsvLTpk3bnHXbOXPmlO/WrdvB8PDwdICrr776wOzZsyv07NnzYERERHKXLl2OeuuVa9++/ZEmTZokA9SoUSNzhtquXbseLFOmjJYpUya1cuXKKdu3bw+rX79+yqxZs8I/+OCDLRnrDRw4sM7111//+8iRI3dl7HPjxo2l27dv3wRcotK2bdvEFStWlK5Vq1ZSxgR0t99++/4xY8ZUx2vucNttt/0OcPfdd/8+dOjQ2uBm9u3Tp0/MDTfccKBPnz5ndfyu3JyDnVcLD1Vl44GNJ83PsmH/BgBKhZaifVR7Hu3waGb7jvNK56s0yxhjipy8Si4CJeugjBn377333v1XX311g9KlS+s111xzoESJEvnab9myZdP9Wc93PpnQ0FBSU1PFm803NCYmJiXjsXbt2iXOmzcv/NixY7vLli2rqkpcXNzhL7/88qQEadGiRWVyO15IyB+l5yKiAB9++OHWH374odz06dMrtm3bNnbp0qVratasmZksDRo0KOq7776rCJDdDLzZqV27dvKOHTsyhwPdvn37SSUk/rBy/gKUmp5KfEI8r/74Kjd+ciMRL0XQcHRD7ph+B5+t+4zGVRoz8vKRLLhjAYceP8TcfnN5vsvzdGvYzZIQY4wJoJ07d5acNWtWOYDJkydX7tChQyJATExMSo0aNVJeeumliP79+2dOehcWFqZJSUkC0Llz58QZM2ac5yUOITNmzKjUuXPnI1mP0alTp6NLliypsG7dupIAvlUz2fn6668rxMXFnbSfAQMG7OvateuhfodQCwAAF6lJREFU7t27109JSaFTp05H4+Pjy2e08zh8+HDIypUrS7Vs2fLEjh07SmYsnzRpUpVLLrkkc1+TJk2qDDB+/PhKrVu3PgqwevXqUpdddtnRV199NaFSpUqpmzZtOmk88dGjR+9Yt27dGn+TEIBLL7306JYtW0qvW7eu5IkTJ2TatGmVb7jhhoP+bg9WInJGEpMT+XH7j5m9WX7c/iNHU44CUPe8unSt3zVzRtomVZtY+w5jjAmSmJiYE6NHj67ev3//sg0bNjzxyCOP7M147Kabbto/ZsyYsDZt2mT29OjTp8/epk2bxjZv3vzY9OnTN/fu3Xt/mzZtmgL07dt3b8eOHY+vX7/+pC/yyMjI1FGjRm3p0aNHg/T0dKpUqZKycOHCX3KKacaMGRV79ep1ShXJ8OHDdw8ePDj0L3/5S93PP/9889ixY7fcdNNN9ZKTkwXg6aef3tGiRYukt956a0vPnj3rp6Wl0bJly2O+z+nAgQOhjRo1ii1ZsqROmTJlE8DgwYNrbdmypZSqSlxc3OGLLrrouL/nb+vWrWEXXHBB7NGjR0NFRMeOHVtj7dq1qypXrpz+0ksvbb3qqqsapaWl0bt3733t2rXzu8cMgLihPAq3du3aaXx8fLDDYFfirpO60f606yfSNA1BaFmzZWaj0o7RHakVXivY4RpjijkRWaqq7YIdx4oVK7a0bNlyX95rBsb69etLdu/evWFOE9/deuut0a1btz42ePDgsxpjbGxs0+XLl6/zrbYpCFFRUefHx8evjYiISC3I/Z6JFStWVG3ZsmVMdo9ZiUgOVJUN+zecNEz6r7//CkCZsDJcWOtCnoh7grjoOC6qdREVS1cMcsTGGGPyq1mzZk3LlCmTPnbs2LPedmXNmjXn3My/p8MSEU9yWjLLdy4/KfHYd8wlx1XLViUuOo572t5DXHQcrSNaUzK0GE3Va4wxRVjjxo2TcyoNWb169TmXDOzYsePnYMeQH8U2ETmcdJgft/+YOT/L4u2LOZ7qqsvqV6rP1Q2vzpyfpVGVRqe0uDbGGGPMmSs2iUjCkYST2nes2L2CdE0nREL+v717j66iuvcA/v0lIQkJSSABQiCEEzSvAxIgaQoYLS9TEKQCDUW4ivZaVJTaULz1cUspbb1axGVR1xUt1MutlF4RNdK0AjUssFhsAkTDI8gjRgJIQF4GyOv87h+zTzjEBDjkMXl8P2vNypw9c/b8zszA/M6efWZjSK8hmJ0y2+rf0fdmRIVE2R0uERFRh9CuE5GDpw5i4aaF+LDkQxw6bf0EO6hTEIZHD8fPb/050mPS8e0+30ZIQIjNkRIREXVM7ToRCfANwPsH3kd6TDrmps1Fekw6BvcajE6+3j2whoiIiJpHu36wRZ/QPjj202N4a9pbyBqehW/1+RaTECIiuqrHH3/c6+HsGzJv3rzeCxYsiGxo+YwZM2LWr18fnJaWlrB58+YmHYenqaxbty4kJCRkcGJiojMxMdE5f/782j4Ma9asCXU4HANjYmIGPvnkk17vt3adiADffKwvERHR1SxdutSrzoIulws1NTVXX7Ee27dv7zJ69Ojy63pzC0pNTf3a/eTV55577igAVFdXIysrKyYnJ2ffvn37dr311lvh+fn5gd7U2+4TESIi6tiKior8Y2NjB0yaNCm2f//+A8aNG9f/3LlzPtnZ2SFjx469wb3e22+/HXrbbbfdMGfOnD4VFRU+iYmJzkmTJsUCwMKFCyPj4uIGxMXFDVi0aFFPd70Oh2Pg5MmTHfHx8QMOHDjgv2bNmlCn05mUkJDgHD58eLy77j179nROS0tLiI6OvunXv/51T3f59u3bA/v373/Rz+9ST4mamhpMnTrV8eMf/7g3AKxduzZ08ODBiU6nM2n8+PH9z5w54wMA7777bkhSUpIzPj7emZmZ6bhw4YIA1gPNHnzwwej4+HjnTTfdlOR+DPyKFSu6xcXFDUhISHCmpqYmNMW+3bRpU3C/fv0qnE5nZWBgoE6ZMuWrNWvWeDVmSbvuI0JERK3MD3/YF4WFTXv7YeDA81hx5cH0iouLA5ctW1ackZFRnpmZ6Vi8eHGPhQsXfvnoo4/GHDlyxK93797VK1asiLjvvvtOzJgx48zrr7/e0z3mypYtW4JWrVoVkZ+fv0dVkZKSkjRmzJhz3bt3rykpKQlYvnz5oTFjxhQfOXLE75FHHnFs2rRpb2JiYqXnWDP79+8P3Lp1a9Hp06d9k5KSBj722GNlAQEBmp2dHZaRkXHGvV5VVZXceeedsU6n88Kzzz577OjRo35PP/101ObNm/eFhoa6nnrqqV6/+tWvIhctWnTsgQceiF2/fn3RoEGDKiZPnuxYvHhxjwULFhwHgLCwsOp9+/btfumllyLmzp3bNzc3d/8zzzwTtX79+n2xsbFVJ06cuOI4OPXZsWNHl4SEBGdkZGTV888//0VqaurFL7744rJB7qKjoyu3bdvWxZt62SJCRETtXq9evSozMjLKAeDuu+8+uXXr1i4+Pj6YNm3ayddeey38xIkTvtu3b++SmZl5pu57N23a1OX2228/HRoa6goLC3NNmDDhVG5ubggAREVFVY4ZM6bcrBeclpZ2LjExsRIAIiMja+/VZGRknO7cubNGRUVVh4eHVx0+fNgPADZu3Bh65513nnWvN2fOnH7uJMRd54EDBwLT0tISExMTnatXr44oKSnxLygoCIyOjq4YNGhQBQDce++9Jz/88MPan4DOmjXrKwD40Y9+9NWOHTu6ANatlZkzZzqWLFnSvbrau6e/jxgxovzzzz//pKioaPfDDz98fOrUqTd6VcEVsEWEiIhazlVaLppL3f6C7tcPPfTQyQkTJtwYGBiod9xxx6lOnbz7QUNQUJDrWtbzHE/G19cX1dXVYkbz9XU4HFXuZampqV9v2bIl9Pz5818GBQWpqiI9Pf3se++9d8izvo8++qjzlbbn43OpnUFEFABWrVpV8sEHHwRnZ2eHpaSkOPPz83f36tWrNlmaO3dunw0bNoQBQN0ReMPDw2s/5w9+8IMz8+bNizl69Khf3759K0tLS2sfNX748OHLWkiuhW0tIiJSLCKfishOEbF/RDsiImq3jh496r9x48ZgAHjjjTfCR4wY8TUAOByOqsjIyKolS5ZEzZ49u3bQOz8/P62oqBAAGDVq1Nc5OTldTeLgk5OT023UqFHn6m5j5MiR5R9//HHI3r17/QHA89ZMff7yl7+EpKenX1bPAw88cCIjI+PMxIkTb6iqqsLIkSPL8/Lyurj7eZw9e9bnk08+CUhOTr5YWlrq7y5fuXJlxC233FJb18qVK8MBYPny5d2GDBlSDgC7du0KGD16dPkLL7xwpFu3btUHDx68bKySF198sdTdGbVurCUlJX4ul5WL5ObmBrlcLkRGRlZ/5zvfKS8uLg7cu3ev/8WLF2Xt2rXhU6dOPX2lz12X3S0io1TVthEZiYioY3A4HBdffPHFnrNnzw6Ki4u7OH/+/DL3sunTp598+eWX/YYOHVo7fP3MmTPLkpKSnAMHDjyfnZ19aMaMGSeHDh2aBAB333132c0333yhqKjosgt57969q5cuXVo8efLkG10uFyIiIqq2bt36WUMx5eTkhE2bNu1U3fKFCxd+mZWV5TtlypTYd95559CyZcuKp0+f3r+yslIA4Be/+EXpoEGDKl555ZXizMzMG2pqapCcnHze8zOdOnXKNz4+3unv76+rV68+CABZWVnRxcXFAaoq6enpZ4cNG3bhWvffH//4x24rVqzo6evrq4GBga6VK1ce9PHxgY+PD5YsWVIybty4+JqaGsyYMeNEamrqxavXeImoNunow9e+YZFiAKnXkoikpqZqXh4bTYiIvCEi+aqaanccBQUFxcnJybZ96SwqKvKfOHFiXEMD391zzz0xQ4YMOZ+VldWiMTqdzqQdO3bs9bxt0xT69OlzU15e3p6oqCjvOoI0o4KCgu7JycmO+pbZ2SKiANabe1fLVPVVz4UiMhvAbACIiYmxITwiImrvBgwYkNS5c2fXsmXLWrzvyu7du9vdyL/Xw85EJF1VS0WkJ4ANIrJXVTe7F5rE5FXAahGxK0giImrbEhISKhtqDdm1a1e7SwZKS0s/tTsGb9jWWVVVS83f4wDeBpBmVyxERNSsXC6Xi4+57qDMsW/w10W2JCIiEiwiIe55ABkACu2IhYiIml1hWVlZGJORjsflcklZWVkYrnCNt+vWTCSAt83vuP0ArFLVv9kUCxERNaPq6ur7jx079vtjx44NBB+k2dG4ABRWV1ff39AKtiQiqnoQQLId2yYiopaVkpJyHMAku+Og1omZKREREdnG7geaERG1T6pAdXXjp4gIYOhQuz8NUbNhIkJEzcPlAmpqmuZi3BYn1zUNQXJ1d9wBZGc3TV1ErRATEaLmwgux3UcA8PNr3BQU1Pg6GjtFRNi9F4maFRMRah1Ugaoq4Pz5y6fy8kvzVVX2X1y9mWwaPuEy7eFCfL2Tjw8g/LUoUWvHRISurqYGuHDhm0lCU081NVePxVuNvZgFBNh/QeWFmIjaMSYibZkqUFnZdImAZ+uD51RR4X1sPj5AcLD1jbru1L17/eVXmjp39j4p8OGPwoiIWjsmIs2loVaEhi721ztdz334wMD6L/bBwUCPHt4nCfXV06kTv40TEdFVdbxERNX6ht/ctxmauhXhSglCQ+9pqGWBLQVERNRKtO9EpKgImDz5m0nC9XQi9GxF8Lzwd+kC9OzZ+FaEoCC2IhARUYfTvhOR4GBgwIDGJwhsRSAiImoW7TsRiY4G3nzT7iiIiIioAfyaT0RERLZhIkJERES2YSJCREREtmEiQkRERLaxLRERkXEiUiQi+0XkcbviICIiIvvYkoiIiC+AlwGMB+AEcJeIOO2IhYiIiOxjV4tIGoD9qnpQVSsBrAbwPZtiISIiIpvYlYj0AfCFx+vDpqyWiMwWkTwRySsrK2vR4IiIiKhltNoHmqnqqwBeBQARKRORzxtRXXcAJ5oksKbFuLzDuLzDuLzTHuPq15SBEDUHuxKRUgB9PV5Hm7J6qWqPxmxMRPJUNbUxdTQHxuUdxuUdxuUdxkVkD7tuzfwLQJyIxIqIP4DpALJtioWIiIhsYkuLiKpWi8gjAN4H4AtgharusiMWIiIiso9tfURUNQdATgtt7tUW2o63GJd3GJd3GJd3GBeRDURV7Y6BiIiIOig+4p2IiIhsw0SEiIiIbNMmExER6SsiuSKyW0R2icijpjxcRDaIyGfmbzdTLiKy1Ixr84mIDPWoa5ZZ/zMRmdXIuAJF5GMRKTBx/dKUx4rINrP9P5tfCkFEAszr/Wa5w6OuJ0x5kYh8tzFxedTpKyI7RGRda4lLRIpF5FMR2SkieabM1uNo6usqImtEZK+I7BGR4XbHJSIJZj+5p7Mi8hO74zL1ZZlzvlBE/mT+LbSG8+tRE9MuEfmJKWvx/SUiK0TkuIgUepQ1WRwikmL+He0375Xr22NENlDVNjcBiAIw1MyHANgHa8ya3wJ43JQ/DuBZM387gL8CEADDAGwz5eEADpq/3cx8t0bEJQC6mPlOALaZ7f0fgOmm/BUAD5n5OQBeMfPTAfzZzDsBFAAIABAL4AAA3ybYb/MArAKwzry2PS4AxQC61ymz9TiaOv8HwP1m3h9A19YQl0d8vgCOwXpgld3nfR8AhwB09jiv7rX7/AIwEEAhgCBYHfM3ArjRjv0F4FYAQwEUNsd5DuBjs66Y945vivOME6eWmGwPoEk+BPAugNsAFAGIMmVRAIrM/DIAd3msX2SW3wVgmUf5Zes1MqYgANsBfBvWUxH9TPlwAO+b+fcBDDfzfmY9AfAEgCc86qpdrxHxRAP4O4DRANaZ7bSGuIrxzUTE1uMIIAzWhVVaU1x1YskA8I/WEBcuDdkQbs6XdQC+a/f5BSATwHKP1z8H8B927S8ADlyeiDRJHGbZXo/yy9bjxKm1T23y1own06w7BFbrQ6SqHjWLjgGINPMNjW1z1TFvriMeXxHZCeA4gA2wvtWdVtXqerZRu32z/AyAiOaIC8ALsP4TdpnXEa0kLgWwXkTyRWS2KbP7OMYCKAPwB7FuZf1eRIJbQVyepgP4k5m3NS5VLQXwHIASAEdhnS/5sP/8KgRwi4hEiEgQrJaGvmg9x7Gp4uhj5ps6PqIW0aYTERHpAuAtAD9R1bOey1RVYV3kWpSq1qjqYFgtEGkAEls6hrpEZCKA46qab3cs9UhX1aEAxgN4WERu9Vxo03H0g9WM/t+qOgRAOaymc7vjAgCYvhaTALxZd5kdcZm+Dd+DlcD1BhAMYFxLxlAfVd0D4FkA6wH8DcBOADV11rHtOLbGOIjs0GYTERHpBCsJeUNV15riL0UkyiyPgtUqATQ8to1XY954Q1VPA8iF1STdVUTcD4/z3Ebt9s3yMAAnmyGumwFMEpFiAKth3Z75XSuIy/1tGqp6HMDbsJI3u4/jYQCHVXWbeb0GVmJid1xu4wFsV9UvzWu74xoL4JCqlqlqFYC1sM651nB+LVfVFFW9FcApWP3J7N5fbk0VR6mZb+r4iFpEm0xETI/w5QD2qOrzHouyAbh7ks+C1XfEXX6P6Y0+DMAZ0yT6PoAMEelmvtVlmLLrjauHiHQ1851h9VvZAysh+X4Dcbnj/T6AD8w3o2wA082vC2IBxMHqjHZdVPUJVY1WVQesJv0PVHWm3XGJSLCIhLjnYe3/Qth8HFX1GIAvRCTBFI0BsNvuuDzchUu3ZdzbtzOuEgDDRCTI/Nt07y9bzy8AEJGe5m8MgCmwOmvbvb/cmiQOs+ysiAwz+/8ej7qIWj+7O6lczwQgHVYz5iewmlt3wrr/GwGrQ+ZnsHrIh5v1BcDLsPprfAog1aOuHwLYb6b7GhnXIAA7TFyFABaY8v6w/kPdD6s5PcCUB5rX+83y/h51PWXiLUIT9oAHMBKXfjVja1xm+wVm2gXgKVNu63E09Q0GkGeO5TuwfqXQGuIKhtV6EOZR1hri+iWAvea8/19Yv3yx/bwHsAVWUlQAYIxd+wtW4ngUQBWsFrd/b8o4AKSafX8AwEuo09GaE6fWPPER70RERGSbNnlrhoiIiNoHJiJERERkGyYiREREZBsmIkRERGQbJiJERERkGyYi1K6JSI1YI9UWiMh2ERlxlfW7isica6h3k4ikXmdMOe7nzRARdXRMRKi9u6Cqg1U1Gdagav91lfW7whodttmo6u1qPXmXiKjDYyJCHUkorMd8Q0S6iMjfTSvJpyLyPbPOMwBuMK0oi826PzPrFIjIMx71ZYrIxyKyT0RuqbsxEYkSkc2mrkL3OiJSLCLdReRBs2yniBwSkVyzPENEPjKxvSnWmEpERO0SH2hG7ZqI1MB6OmUgrOHSR6tqvhnjJEhVz4pIdwD/hPVI8X6wnjw70Lx/PKzh48eq6nkRCVfVr0RkE4B8Vf2piNwOYJ6qjq2z7Z8CCFTV34iIr9neObHG/ElV1RNmvU4APgDwWwAfwRqrZbyqlovIz2A9kXRRc+4nIiK7+F19FaI27YJaoyFDRIYDWCkiA2E9RvtpsUb7dcEaNj2ynvePBfAHVT0PAKr6lccy92CL+QAc9bz3XwBWmETjHVXd2UCMv4M13sp7Yo2U7ATwD2vYEPjDSk6IiNolJiLUYajqR6b1owessYl6AEhR1SrTShHoZZUV5m8N6vm3pKqbTaIzAcDrIvK8qq70XEdE7oXVCvOIuwjABlW9y8tYiIjaJPYRoQ5DRBIB+MIMGgfguElCRsFKBgDgHIAQj7dtAHCfiASZOsK92F4/AF+q6msAfg9gaJ3lKQDmA/g3VXWZ4n8CuFlEbjTrBItIvHeflIio7WCLCLV3nUXEfUtEAMxS1RoReQPAeyLyKaxRdvcCgKqeFJF/iEghgL+q6mMiMhhAnohUAsgB8OQ1bnskgMdEpArA17CGZ/f0CIBwALnmNkyeqt5vWkn+JCIBZr3/BLDP609ORNQGsLMqERER2Ya3ZoiIiMg2TESIiIjINkxEiIiIyDZMRIiIiMg2TESIiIjINkxEiIiIyDZMRIiIiMg2/w9FRNYAdctgzgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_relative_time(results, n_features, max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "b96a904b", + "metadata": {}, + "source": [ + "The difference between KeOps and PyTorch is even more striking when we only look at $[2, 10]$ features:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0d1e4dfa", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEWCAYAAAAQHy/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeVhV1frA8e9iEFGBRJFwQBxAUbQcU8uRQXPMzCEShyyzbllqdbv3dpt/TYqVXsuh1BwrLcspp0TFzJxnMSccUVFEQGU86/fHPigSICCwGd7P85xHzt777PXuc45nvXvttddSWmuEEEIIUbbYmB2AEEIIIYqeJABCCCFEGSQJgBBCCFEGSQIghBBClEGSAAghhBBlkCQAQgghRBkkCUAJoZTSSqn6OayfqpT6b4bnzyulLiqlEpRSVfJRnpe1TLv8xlxQrMdQtwjK8bSWZVsEZbkrpTYppeKVUqGFXZ7IPaXUv5VSX5sdhxCFTRKATJRSkUqpZKVU1UzLd1srRC/r89nW530ybfeZdfkw6/NhSqk0a8WSoJQ6qZSapZTyKci4tdajtNbvW8u0ByYCQVrrSlrrK3dLIO6F9Rg3F9C+Niilnsm4zHoMJwpi/5nKilRKBWQo57S1rLSCLisLI4HLgLPWety97kwp5aGUWqqUOp/xe5phvYNSaqZSKk4pdUEpNTbTen+lVIRS6oZSKkwpVfteYzKDUuodpdS8PGzfSSl1NuMyrfWHWutnsnuNEKWFJABZOwk8mf5EKdUEqJDFdn8BQzJsZwcMAI5n2u4PrXUlwAUIAG4CO5VSfgURbBZnrO5AeeBgQexfFIrawCGdj5G4smmVsQCrgH7ZvOwdwNtabmfgdaVUN+v+qgI/Af8FXIEdwPd5jSuvikPrkhBlmtZaHhkeQCTwJrA9w7IJwH8ADXhZl822Lr8IVLYu6wn8CmwGhlmXDQM2Z1HOcmBxDnG8BkQB54GnrWXXz1D2V8BK4DpGUjEb+ADwsS7TQAKwHthkfX7dumxgFuXZWo/nMnAC+If1NXbW9S7AN9aYzlnLsgV8gUQgzbrvWOv2Dtb9nba+R1MBxwzl9QH2AHEYCVM34P+s+0m07ut/1m0zHrsLMAeIBk5ZPyubjO+1tdyrGInco9m8v3MxKs2b1rJeB7wyHfMG63FusW6zDKgCzLfGvT39+2DdviGwFogBjgADsil7NpACJFv3G2B9vz63ft7nrX87WLfvBJwF/glcAObm8L2xI8P3NMPy8xgtQunP3we+s/49EtiSYV1F6/vSMIf/I/8CDlnf51lA+Qzre1o/21jre9c002v/CewDkqzxRmJ83/dhfEe/wUhifwXigXXc/j/WCTibRTwBGN+hZOt7mwDsta4fDhy27usE8Fym47RYt08AqmMkS/My7L83RjIda/1O+GYq+1Vr7NcwEqfy1nVVMf6fx1q/E+FYv6vykEdxeJgeQHF7ZPgxOYJRudlaf3xr8/cE4ANgOvC8ddkPGC0HuUkAngYuZhNDN4xK08/6I7WAvycA14CHMVpxyqfHY13vRYaKzLrs1uuzKXMUEAHUwjgLDOPOynAJMM0aTzVgW4Yf0r8dI/AZsNS6LyeMyvMj67rW1vgDrfHXwFrZWH9gn8m0r4zHPgf4xbpPL4xWmBEZ4kgBnrV+bs9jVHwqp886w/M73jdrLMeAehiJxyFreQEYFdccYJZ124rAGYzKxg5ohpFMNcqm7Fufl/X5e8BW63vrhlFxvm9d1wlIBT7BSBQcs9qnddu/JQBAZesy9wzLngD2W//+Avgq034OAP1yeN8OZPiu/M7t714z4BLwkPUzGGrd3iHDa/dYX+uYYdlWjEq/hvX1u6z7Ko+RxL6d4b3IMgGw/v0OGSpv67Ie1s9QAR2BG0DzHPZ3ax/cTqgDAXuMRPEYUC5D2dswEgdXjERjlHXdRxiJr7310Z5svovykIcZD7kEkL25GM37gRj/qc9ls90cYIhS6j6MH5efc7n/8xg/GFkZgFGxHNBaX8f4QcrsF63171pri9Y6MZdl5mQA8LnW+ozWOgbjxwswOqwB3YFXtNbXtdaXMCr4QVntSCmlMM4qx2itY7TW8cCHGbYfAczUWq+1xn9Oax1xtwCtlzoGAf/SWsdrrSOBUCAkw2antNYztHEd/1vAA6Niya9ZWuvjWutrGGekx7XW67TWqcAijEoKjLPeSK31LK11qtZ6N/Aj0D+X5TwFvKe1vqS1jgbezXRcFoxKMElrfTOPx1DJ+u+1DMuuYSRR6euvcaeM67Pyvwzflf/j9iWzkcA0rfWfWus0rfW3GGf6bTK8dpL1tRmPY7LW+qLW+hzGmfKfWuvd1u/2Em6/z3mmtV5h/Qy11nojsAajMs6NgcAK63c1BaN1yRFol+l4zlvfi2XAg9blKRjfv9pa6xStdbjWWiZfEcWGXIPL3lyMpvM6GJV8lrTWm5VSbhiXCJZrrW8a9d9d1cBoFsxKdWBnhuenstjmTG4KyYPqmfaZsczaGGcwURmOzSaHGNww+kzszLC9wjgjBOPsb2U+YqxqjSNjbKcw3st0F9L/0FrfsJZfify7mOHvm1k8T993beAhpVRshvV2GN+j3KjO34+reobn0feQ6CVY/3XGuLyS/nd8hvXOmV6TcX1WMn9X0mOtDQxVSr2UYX057jyWrL43uX2f80wp9SjwNsbZvA3Gd3N/Ll9+x+eitbYopc6QzXcOo3Uh/VjHYyTva6zfw+la64/zcQhCFAppAciG1voUxjXk7hgdpHIyDxhHDolCFvpinOlkJQqjkkznmVWIeSgrN3Iq8wzGWVxVrfV91oez1rpxNrFcxvjRbpxhexdtdIRM31+9bOLI6bguY5xVZeyh7kn2rTN3U5Dv4RlgY4bjvU8bdxQ8n8vXn+fvx3U+w/N8x6q1vorx+T6QYfED3O4kejDjOqVURYzPJ6dOpJm/K+mxngH+L9P7UEFrvTBjSPk7EsBojr/VIdfaKuSW3b6VUg4YLTETMC6B3IeRfKqsts/CHZ+LtXWrFrn4zllbqcZpreti9CMYq5Tyv9vrhCgqkgDkbATQxdoMn5NJGJcKNuW0kVLKVilVRyk1GePa47vZbPoDMEwp1UgpVQHj7OVeXQRyupf+B2C0UqqmUqoy8Eb6Cq11FEazaahSylkpZaOUqqeU6phh3zWVUuWs21uAGcBnSqlqAEqpGkqprtbtvwGGW289s7Gua3i3OK3N+j8A/6eUcrLeqjYWIwHLj7u9J3mxHPBRSoUopeytj1ZKKd9cvn4h8KZSys3aK/8t8nhcSqnyGH0EABysz9PNse6/svW9fhajHwIYTex+Sql+1te8Bey7y2WZf1i/K64YrV/pdw3MAEYppR5ShopKqR5KqZwuJ+TFX0B56z7tMTqBOmRYfxHwUkql/7aVs66PBlKtrQFBmbavopRyyaa8H4Ae1u+qPUain4TRRyNHSqmeSqn61qThGkYHV0tuD1SIwiYJQA6s1w135GK7GK31bzlc32urlErA6Dm+AaN5tZXWOstmSK31rxi9wNdjdDhan5/4M3kH+FYpFauUGpDF+hnAamAvRgeszK0eQzB+TNN7fi/GuL6JNb6DwAWl1GXrsn9aY9+qlIrD6MndwHp82zA6y32G8cO4kdtnWV8ATyilriqlJmUR50sYZ4EnMDpbLgBm5u4t+JuPMCrFWKXUq/ncB2Cc7WFULIMwzhovcLvTXm58gHH73T6M5uld1mV5kX5HAxgdOjNeY38b426LUxjv93it9Spr7NEYtw/+H8Zn+xDZ9O/IYAFGUnjCut8PrPvagZFc/M+6r2MYnTMLhLUvxgvA1xhn4dcxOummW2T994pSapf1cxmNUZFfBYIxOqem7y8CI/k6Yf0eZLxUgdb6CDAYmIzRAtUL6KW1Ts5FuN4Y3/sE4A/gS611WN6OWIjCo6RPihAiL5RSkRh3aqwzOxYhRP5JC4AQQghRBkkCIIQQQpRBcglACCGEKIOkBUAIIYQog0rEQEBVq1bVXl5eZochhBAlys6dOy9rrd3uvqUoi0pEAuDl5cWOHXe9G08IIUQGSqmsRhEVApBLAEIIIUSZVGgJgFJqplLqklLqQBbrximltHXEMyGEEEIUscJsAZiNMa3tHZRStTBGTDtdiGULIYQQIgeFlgBorTeR9Wx3n2HMqS33HwohhBAmKdI+AEqpPsA5rfXeoixXCCGEEHcqsrsArLPa/Zs7Z+LKafuRwEgAT8+sZsMVQgghRH4VZQtAPaAOsNc6mUhNYJdS6v6sNtZaT9dat9Rat3Rzk9tYhRBCiIJUZAmA1nq/1rqa1tpLa+2FMYVnc631haKKQQghSorrydd5+deXiU2MNTsUUUoV5m2ACzHmwG6glDqrlBpRWGUJIURpEpsYS9C8IP63/X/8fvp3s8MRpVSh9QHQWj95l/VehVW2EEKUVNHXowmaF8TBSwf54Ykf6OHTw+yQRClVIoYCFkKIsuBc3DkC5gYQGRvJ0ieX0q3+34ZSEaLASAIghBDFwImrJwiYE8DlG5dZPXg1HWp3MDskUcpJAiCEECY7FH2IwLmBJKYm8tuQ32hVo5XZIYkyQBIAIYQw0a6oXXSd1xVbZcvGYRvxq+ZndkiijJDZAIUQwiS/n/6dzt92poJ9BcKHh0vlL4qUJABCCGGCdSfWETQviPsr3c/m4ZvxruJtdkiijJEEQAghitgvEb/QY0EP6lWux6Zhm6jlUsvskEQZJAmAEEIUoQX7F9Dvh348eP+DbBi2AfdK7maHJMooSQCEEKKITN85ncE/DaZ97fasC1mHq6Or2SGJMkwSACGEKAKhW0J5bvlzPOr9KCuDV+Lk4GR2SKKMkwRACCEKkdaat8Pe5tW1r9K/UX+WDFyCo72j2WEJIeMACCFEYdFaM27NOD7b+hnDHxzOjF4zsLWxNTssIQBJAIQQolCkWdIYtXwUX+/+mtGtR/NZt8+wUdLoKooPSQCEEKKApaSlMOTnIXx34Dv+0/4/vN/5fZRSZoclxB0kARBCiAKUmJrIgEUDWPbXMj72/5h/PvJPs0MSIkuSAAghRAFJSE7gse8e47eTvzGl+xReaPWC2SEJkS1JAIQQogDEJsbSfX53/jz3J3Mem0PIAyFmhyREjiQBEEKIe3Tp+iW6zuvKwUsHWdR/EY/7Pm52SELclSQAQghxD87GnSVwbiCnYk+x7MlldK3f1eyQhMgVSQCEECKfTlw9gf8cf67cuMLqwatpX7u92SEJkWuFdlOqUmqmUuqSUupAhmXjlVIRSql9SqklSqn7Cqt8IYQoTIeiD/HIzEeIS4pj/dD1UvmLEqcwR6WYDXTLtGwt4Ke1bgr8BfyrEMsXQohCsStqFx1mdUCj2ThsIy2rtzQ7JCHyrNASAK31JiAm07I1WutU69OtQM3CKl8IIQrD76d/p/O3nalUrhLhw8Pxq+ZndkhC5IuZ41I+Dfya3Uql1Eil1A6l1I7o6OgiDEsIIbK29vhaguYFcX+l+wkfHk591/pmhyREvpmSACil/gOkAvOz20ZrPV1r3VJr3dLNza3oghNCiCz8HPEzPRf2pL5rfTYN20Qtl1pmhyTEPSnyBEApNQzoCTyltdZFXb4QQuTV/H3zeeKHJ2h2fzPChobhXsnd7JCEuGdFmgAopboBrwO9tdY3irJsIYTIj2k7phGyJIQOtTuwNmQtro6uZockRIEozNsAFwJ/AA2UUmeVUiOA/wFOwFql1B6l1NTCKl8IIe7VhC0TGLViFN29u7MieAVODk5mhyREgSm0gYC01k9msfibwipPCCEKitaatze8zfub3mdA4wHM7TuXcrblzA5LiAIlIwEKIUQGWmvGrh7L539+ztMPPs30XtOxtbE1OywhCpwkAEIIYZVmSWPU8lF8vftrXn7oZSZ2nYiNMvNuaSEKjyQAQggBpKSlELIkhO8Pfs+b7d/kvc7voZQyOywhCo0kAEKIMi8xNZH+i/qz/K/lfBLwCa8//LrZIQlR6CQBEEKUaQnJCfT5rg9hJ8P4svuXPN/qebNDEqJISAIghCizrt68SvcF3dl+bjtz+s5hcNPBZockRJGRBEAIUSZdun6JoLlBHIo+xKL+i+jr29fskIQoUpIACCHKnLNxZwmYE8Dpa6dZ9uQyutbvanZIQhQ5SQCEEGXK8Zjj+M/x52riVdaErOERz0fMDkkIU0gCIIQoMw5eOkjg3ECS05JZP2Q9Laq3MDskIUwjI1wIIcqEned30nF2RzSajcM2SuUvyjxJAIQQpd7m05vpMqcLlcpVInx4OI2rNTY7JCFMJwmAEKJUW3N8DUFzg/Co5MHmpzdT37W+2SEJUSxIAiCEKLWWHF5Cr4W98Kniw6bhm6jpXNPskIQoNiQBEEKUSvP2zaP/ov4092hO2NAwqlWsZnZIQhQrkgAIIUqdqTumMmTJEDp6dWRtyFoqO1Y2OyQhih1JAIQQpcr438fz/Irn6eHTgxXBK6hUrpLZIQlRLEkCIIQoFbTW/Hf9f3l93esMbDyQnwb8RHm78maHJUSxJQMBCSFKPK01Y1aP4Ys/v2BEsxFM6zkNWxtbs8MSoliTBEAIUaKlWdJ4bvlzfLP7G1556BUmdp2IUsrssIQo9grtEoBSaqZS6pJS6kCGZa5KqbVKqaPWf6VnjhAi35LTkgn+KZhvdn/Dfzv8Vyp/IfKgMPsAzAa6ZVr2BvCb1tob+M36XAgh8uxmyk0e//5xfjj4A58GfMp7nd+Tyl+IPCi0BEBrvQmIybS4D/Ct9e9vgccKq3whROmVkJxAjwU9WHl0JVN7TOW1h18zOyQhSpyi7gPgrrWOsv59AXDPbkOl1EhgJICnp2cRhCaEKAmu3rxK9wXd2X5uO3P6zmFw08FmhyREiWTabYBaaw3oHNZP11q31Fq3dHNzK8LIhBDF1cWEi3T6thO7onaxqP8iqfyFuAdF3QJwUSnlobWOUkp5AJeKuHwhRAl15toZAuYGcObaGZY9uYygekFmhyREiVbULQBLgaHWv4cCvxRx+UKIEuhYzDHaz2rPhYQLrAlZI5W/EAWg0FoAlFILgU5AVaXUWeBt4GPgB6XUCOAUMKCwyhdClA4HLh0gcG4gKWkprB+ynhbVW5gdkhClQqElAFrrJ7NZ5V9YZQohSpcd53fQdV5XHGwd2DR8E43cGpkdkhClhswFIIQolsJPhdPl2y44Oziz+enNUvkLUcAkARBCFDurj62m67yuVHeqTvjwcOpWrmt2SEKUOpIACCGKlZ8O/0Svhb3wqeLDpuGbqOlc0+yQhCiVJAEQQhQbc/fOZcCiAbSs3pKwoWFUq1jN7JCEKLUkARBCFAtfbf+KIT8PoaNXR9aErKGyo8wVJkRhkgRACGG6T3//lBdWvkAvn16sCF5BpXKVzA5JiFJPEgAhhGm01ry5/k3+ue6fDGw8kB8H/Eh5u/JmhyVEmVDUQwELIQQAFm1hzKoxTNo2iWeaPcPUnlOxtbE1OywhygxJAIQQRS7NksbIZSOZuWcmY9qMITQoFKWU2WEJUaZIAiCEKFLJackM/mkwiw4t4q0Ob/FOp3ek8hfCBJIACCGKzM2Um/Rf1J8VR1cwPnA8r7Z71eyQhCizJAEQQhSJ+KR4en/Xm42RG5nWcxojW4w0OyQhyjRJAIQQhS7mZgyPzn+Uned3MrfvXJ5q+pTZIQlR5kkCIIQoVBcTLhI0L4iIyxEsHrCYxxo+ZnZIQggkARBCFKIz184QMDeAs3FnWf7kcgLrBZodkhDCShIAIUShOBZzDP85/sQmxrJm8Boe9nzY7JCEEBlIAiCEKHAHLh0gcG4gKWkphA0No7lHc7NDEkJkIkMBCyEK1I7zO+g4uyM2yoZNwzdJ5S9EMSUJgBCiwGw6tYku33bB2cGZ8OHhNHJrZHZIQohsSAIghCgQq46totu8btRwrsHm4ZupW7mu2SEJIXJgSgKglBqjlDqolDqglFqolJLpv4QowX489CO9F/amQdUGbBy2kRrONcwOSQhxF3dNAJRSbZVSU5RS+5RS0Uqp00qplUqpfyilXPJaoFKqBjAaaKm19gNsgUF5D10IURzM2TuHAYsH0LJ6S8KGhlGtYjWzQxJC5EKOCYBS6lfgGWA10A3wABoBbwLlgV+UUr3zUa4d4KiUsgMqAOfzsQ8hhMm+3P4lQ38eSmevzqwJWcN95e8zOyQhRC7d7TbAEK315UzLEoBd1keoUqpqXgrUWp9TSk0ATgM3gTVa6zWZt1NKjQRGAnh6eualCCFEEfhk8ye88dsb9PLpxQ/9f6C8nVzJE6IkybEFIL3yV0pVVErZWP/2UUr1VkrZZ9wmt5RSlYE+QB2gOlBRKTU4i7Kna61baq1burm55aUIIUQh0lrzn9/+wxu/vcGTfk/y44AfpfIXogTKbSfATUB56/X7NUAIMDufZQYAJ7XW0VrrFOAnoF0+9yWEKEIWbeHlVS/z4eYPebb5s8ztOxd7W3uzwxJC5ENuEwCltb4BPA58qbXuDzTOZ5mngTZKqQpKKQX4A4fzuS8hRBFJs6QxYukIJm+bzNg2Y5nWcxq2NrZmhyWEyKdcJwBKqbbAU8AK67J8/c/XWv8JLMboQ7DfGsP0/OxLCFE0ktOSefLHJ5m9ZzZvd3ybCUETMPJ3IURJldu5AF4G/gUs0VofVErVBcLyW6jW+m3g7fy+XghRdG6m3OSJRU+w8uhKJgROYFy7cWaHJIQoALlKALTWmzD6AaQ/P4FxL78QohSLT4qn18JebDq1iWk9pzGyxUizQxJCFJC7jQMwQynVJJt1FZVSTyulniqc0IQQZoq5GUPA3AA2n97MvMfnSeUvRClztxaAKcB/rUnAASAaYwAgb8AZmAnML9QIhRBF7mLCRQLnBnLkyhF+HPAjfRr2MTskIUQByzEB0FrvAQYopSoBLTFGArwJHNZaHymC+IQQRez0tdMEzAngXPw5VgSvIKBugNkhCSEKQW77ACQAGwo3FCGE2Y5eOUrA3ACuJV5jbcha2tWSITqEKK1yexeAEKKU239xP4FzA0nTaawfup7mHs3NDkkIUYhMmQ5YCFG8bD+3nU7fdsLWxpZNwzZJ5S9EGZCnBEApVaGwAhFCmGPTqU34z/HHxcGF8OHh+Lr5mh2SEKII5CoBUEq1U0odAiKszx9QSn1ZqJEJIQrdqmOr6DqvKzWcaxA+PJy6leuaHZIQoojktgXgM6ArcAVAa70X6FBYQQkhCt+Ph36k98Le+Fb1ZdOwTdRwrmF2SEKIIpTrSwBa6zOZFqUVcCxCiCLy7Z5vGbB4AK1qtGL90PW4VZQpt4Uoa3KbAJxRSrUDtFLKXin1KjKDnxAl0pRtUxj2yzA6e3VmzeA13Ff+PrNDEkKYILcJwCjgH0AN4BzwoPW5EKIE+Xjzx7z464v0btCb5cHLqViuotkhCSFMktuBgC5jTAUshCiBtNb8Z/1/+GjzRwQ3CWZ2n9nY29qbHZYQwkS5SgCUUnWAlwCvjK/RWvcunLCEEAXFoi28/OvL/G/7/xjZfCRf9vgSWxtbs8MSQpgstyMB/gx8AywDLIUXjhCiIKVaUnl22bPM3jObcW3HMT5wPEops8MSQhQDuU0AErXWkwo1EiFEgUpOS+apn55i8aHFvNPxHd7q+JZU/kKIW3KbAHyhlHobWAMkpS/UWu8qlKiEEPfkRsoNnvjhCX499iuhQaGMbTvW7JCEEMVMbhOAJkAI0IXblwC09bkQohiJS4qj18JehJ8KZ3rP6Tzb4lmzQxJCFEO5TQD6A3W11smFGYwQ4t7E3Iyh27xu7IraxfzH5/NkkyfNDkkIUUzldhyAA0CBjRailLpPKbVYKRWhlDqslGpbUPsWoqy6kHCBjrM7su/iPn4a+JNU/kKIHOW2BeA+IEIptZ07+wDk9zbAL4BVWusnlFLlAJllUIh7cPraaQLmBHAu/hwrglfgX9ff7JCEEMVcbhOAtwuqQKWUC8ZEQsMArJcV5NKCEPl09MpR/Of4E5cUx9qQtbSr1c7skIQQJUBuRwLcWIBl1gGigVlKqQeAncDLWuvrGTdSSo0ERgJ4enoWYPFClB77L+4ncG4gaTqNsKFhNPNoZnZIQogSIsc+AEqpzdZ/45VScRke8UqpuHyWaQc0B77SWjcDrgNvZN5Iaz1da91Sa93SzU1mKhMis23nttFxdkfsbOzYNGyTVP5CiDy5WyfAigBaayettXOGh5PW2jmfZZ4Fzmqt/7Q+X4yREAghcmlj5Eb85/hzX/n7CB8ejq+br9khCSFKmLslALqgC9RaX8CYXriBdZE/cKigyxGitPr16K90m9+NWs61CB8eTp3KdcwOSQhRAt2tD0A1pVS2Q4hprSfms9yXgPnWOwBOAMPzuR8hypTFhxYT/GMwftX8WD14NW4V5fKYECJ/7pYA2AKVgAIdQFxrvQdoWZD7FKK0m71nNiOWjqBtzbasCF6BS3kXs0MSQpRgd0sAorTW7xVJJEKIbP1v2/946deXCKgbwM8Df6ZiuYpmhySEKOHu1gdApg4TwmQfhX/ES7++RJ8GfVj25DKp/IUQBeJuLQAynJgQJtFa8+/f/s3Hv39McJNgZveZjb2tvdlhCSFKiRwTAK11TFEFIoS4zaItjP51NFO2T+G5Fs/xZY8vsVG5nbpDCCHuLrdDAQshikiqJZVnlj7Dt3u/ZVzbcYwPHI9ScjVOCFGwJAEQohhJTksm+Mdgfjz8I+91eo83O7wplb8o0Xbu3FnNzs7ua8CP3M9AKwqGBTiQmpr6TIsWLS5lXikJgBDFxI2UG/T7oR+rjq3is66f8UqbV8wOSYh7Zmdn9/X999/v6+bmdtXGxqbAB5cT2bNYLCo6OrrRhQsXvgb+NnuvZGNCFANxSXE8Ov9RVh9bzYxeM6TyF6WJn5ubW5xU/kXPxsZGu7m5XcNoffkbaQEQwmRXblzh0fmPsvvCbhb0W8Agv0FmhyREQbKRyt881vc+y5N9SQCEMNGFhAsEzg3k6JWj/DTgJ3o16GV2SEKIMkIuAQhhklOxp2g/qz0nr55kRfAKqfyFKARHjhwp5+3t3djsOLLTu3fvOhNP5UYAACAASURBVF5eXn7e3t6N+/fv75WUlFRkvX4lARDCBH9d+Yv2s9oTfT2atSFr8a8rY24JURY99dRTMSdOnDhw5MiRg4mJierzzz+vWlRlSwIgRBHbd3EfHWZ1IDE1kQ3DNtC2VluzQxKiTDh06FA5X1/fRhs3bqyQmprKc889V9PPz8/Xx8en0fjx46sCWCwWnnvuuZre3t6NfXx8Gs2YMaMywPLly51atmzZoFOnTvW9vLz8goODPdPS0khNTaVfv35e6du/++671fIS08CBA6/Z2NhgY2NDy5Ytr589e7ZcYRx7VqQPgBBF6M+zf/Lo/EepYF+BdUPW0bBqQ7NDEqLIPP00tQ4coEJB7tPPjxszZ3Lmbtvt3bvXYdCgQfVmzpx5sm3btjcnTJhQ1cXFJe3AgQOHb968qVq1atWwV69ecVu3bq2wf/9+x8OHDx+Mioqya926tW9QUFACwP79+yvu3r37gI+PT3KHDh2858yZU7l+/fpJUVFR9kePHj0IcPnyZdv8HEdSUpL6/vvvq0ycOPGux1JQpAVAiCKyIXIDAXMDqOxYmfDh4VL5C1FEYmJi7B577LH68+bNO9G2bdubAOvWrXP+4YcfqjRs2LBRs2bNfK9evWp36NCh8uHh4U4DBgyIsbOzo1atWqkPPfRQwubNmysANGnS5HqjRo2S7ezsGDBgQEx4eHilhg0bJp05c8Zh6NChtRYvXuxcuXLltPzEOHToUM82bdokdOvWLaEgjz0n0gIgRBFYeXQl/X7oR93KdVkbspbqTtXNDkmIIpebM/XC4OTklFa9evXksLCwSi1atEgE0Fqr0NDQ0/369YvLuO2KFStcsttP5lE5lVK4ubmlHThw4NCSJUucp06d6vb999+7Llq0KDJ9m9TUVPz8/BoBdOvWLfbzzz8/n3m/48aN87h8+bLd6tWrj9/bkeaNtAAIUcgWHVzEY989RiO3RmwctlEqfyGKmL29vf7111+PL1y4sMrUqVNdAQIDA6999dVXbum97vft2+cQFxdn06FDh/jFixe7pqamcv78ebtt27ZVat++/XUwLgFERESUS0tLY/Hixa7t27ePj4qKsktLS2PYsGGxH3300bn9+/ffcYnDzs6OiIiIQxEREYeyqvwnTpxYdf369S4///zzCVvbfF09yDdpARCiEM3aPYtnlj1D25ptWRG8Apfy2Z5cCCEKkbOzs2X16tXHOnXq5OPk5JQ2ZsyYy5GRkQ5NmjTx1VorV1fXlJUrVx4PCQmJ3bJlSyVfX9/GSin97rvvnvX09Ezdt28ffn5+10eNGuUZGRlZvl27dnEhISGx27ZtcxwxYoSXxWJRAO+9997ZvMT1+uuv1/bw8Ehq2bKlL0DPnj2vTpgwIaow3oPMlNbFf4Cmli1b6h07dpgdhhB5MvnPyYxeNZrAuoEsGbiEiuUqmh2SKGOUUju11i3NjGHv3r2RDzzwwGUzYygIy5cvdwoNDXUPCws7ZnYsebV3796qDzzwgFfm5aZdAlBK2SqldiullpsVgxCF5cPwDxm9ajSPNXyMZU8uk8pf5NnNm7B6NcTHmx2JKK3M7APwMnDYxPKFKHBaa95Y9wb/Wf8fBjcdzKL+i3CwczA7LFECaA1//QWTJkH37lClCnTrBr/9ZnZkAqBnz57xJfHsPyem9AFQStUEegD/B4w1IwYhCppFW3hp5Ut8ueNLRrUYxZQeU7BR0s9WZC8hAcLCYNUq+PVXOHnSWN6gAYwcaSQAHTuaG6MovczqBPg58DrgZFL5QhSoVEsqI5aOYM7eObza9lU+Dfz0b7cMCaE1HDx4u8IPD4eUFKhYEfz94bXXjEq/Th2zIxVlQZEnAEqpnsAlrfVOpVSnHLYbCYwE8PT0LKLohMi7pNQkgn8K5qfDP/F+5/f5T/v/SOUvbomNhXXrjEp/1So4d85Y3qQJvPIKPPooPPwwlCuyAWCFMJjRAvAw0Fsp1R0oDzgrpeZprQdn3EhrPR2YDsZdAEUfphB3dyPlBo9//zirj6/ms66f8UqbV8wOSZjMYoHdu29X+H/8AWlp4OICgYFGhd+1K9SoYXakoqwr8guUWut/aa1raq29gEHA+syVvxAlQVxSHN3mdWPN8TV83etrqfzLsMuXYcECGDIEPDygZUt4802jJ/+//gWbNxvbLFoETz8tlX9x98Ybb9xfUPsaO3Zs9bfeess9v6/fsmWL44MPPtiwfv36d0xOVBBkICAh8uHKjSt0m9+NPRf2sLDfQgb6DTQ7JFGE0tJg27bbZ/nbtxvX96tWNc7uu3WDoCColqd54URxMWnSJI+PP/74Qm63t1gsaK0pjJH8KlWqZJk7d+7JJk2aJEVGRtq3atXKt2/fvnFVq1bN15wDGZnaRVlrvUFr3dPMGITIq6j4KDrO7sj+i/tZMnCJVP5lRFQUzJ4NgwaBmxu0awcffAB2dvDuu0ZCcOECzJsHgwdL5V9cHDlypFydOnUa9+7du07dunUbd+vWrW58fLzN0qVLnQICAuqlb7dkyRLnwMDAei+88EKNpKQkm4YNGzbq3bt3HYB33nnH3dvbu7G3t3fj9957r1r6fr28vPz69u3r5ePj0/j48ePlFi9e7NyoUSPfBg0aNGrbtq1P+r4PHz7s2Lp16wY1a9Zs8sEHH+Tpm9G0adOkJk2aJAF4eXmluLq6pkZFRRXIybu0AAiRB/sv7qfv9325kHCBlU+tpEudLmaHJApJSgps2XK7x/7evcZyDw947DHjLD8gAFxdzY2zpGndmgaZlz3+ODFvvEF0fDw2/v54Z14/eDCXR4/mSlQUdn36UC/jum3bOHK3MiMjI8tPmzYtMigo6Hr//v29xo8f7/bOO+9cfPnllz3Pnz9vV7169dSZM2dWGT58+OXg4OBrs2fPrhYREXEIIDw8vMKCBQuq7Ny587DWmhYtWvj6+/vHV61aNe306dMO33zzzUl/f//I8+fP27344oteGzZsiGjYsGHyxYsXbzUHHDt2rPyWLVuOxMbG2vr6+vq99tpr0Q4ODnnu2xYWFlYhJSVFNWrUKCmvr82K3KQsxF1orVl3Yh2Pzn+UplObcuXmFdaGrJXKvxQ6fRqmT4fHHzcG4unUCSZMgMqV4eOPYc8eoxf/zJkwYIBU/iXF/fffnxwUFHQdICQk5MqWLVsq2djYMGDAgCszZsxwvXz5su2uXbsq9e/f/1rm127YsKFS9+7dY52dnS0uLi6WHj16XA0LC3MC8PDwSPb3979u3a5i69at4xs2bJgM4O7ufquJPigoKNbR0VF7eHikurq6ppw9ezbPJ9+nTp2yHz58eN0ZM2ZEFtSlBmkBECIbyWnJLNy/kIlbJ7Lv4j7cK7rzfuf3GdVyFFUrVDU7PFEAEhONe/HTz/IPW8cm9fSE4GDjLL9LF3B2NjfO0iSnM3YnJyw5rffwIDU3Z/yZZTWNL8Dzzz9/pUePHvXLly+ve/XqddXe3j5P+61QoYIlN9tlPNu3tbUlNTX1joDmzJlz34cfflgdYPr06ZEdOnS4kXF9TEyMzaOPPlr/7bffPpeecBQESQCEyCTmZgzTdkxj8rbJRCVE0ditMTN7zyS4SbAM61sKHDt2u8IPCzN66js4GCPuPfusUek3bAgylEPpERUVVW7dunUVAwICrs+fP9+1Xbt2CWBcU3d3d08JDQ31WLVq1V/p29vZ2emkpCTl4OCgO3funPD00097vf/++xe01qxcubLy7NmzT2Quo1OnTtfHjh1bOyIiolz6JYCMrQA5GTJkSOyQIUNis1qXmJioevToUX/QoEFXhg8ffjW/70FWJAEQwup4zHE+3/o5M/fM5EbKDQLrBjKrzyyC6gXJwD4l2PXrsGGDUeGvWgXHjxvLvb3hmWeMCr9TJ6hQIae9iJLMy8srcfLkydVGjhxZwdvbO/HVV1+NTl83aNCgK1OmTLFr3rx5Yvqyp556KtrX17eRn5/fjaVLl54MDg6+0rx5c1+AkJCQ6IcffvjmkSNH7hi6qXr16qmTJk2K7Nu3b32LxUKVKlVStmzZcvReY585c2bl7du3V7p69ardggULqlqXnWzXrt3Ne923TAcsyrwtZ7YQ+kcoSw4vwc7GjuAmwYxtO5am7k3NDk3kg9ZGU356hb9pEyQnGxV8ly5Ghd+tG9Srd/d9lXQyHbDRW79nz57eR48ePZjV+iFDhng2a9bsxpgxY0r8lMXZyW46YGkBEGVSmiWNJRFLCP0jlK1nt1K5fGXeeOQNXmz9ItWdqpsdnsija9eMWfPS78s/c8ZY3rgxvPSSMfreI48YTf1CpGvcuLGvo6OjZdq0aWfMjsUMkgDco8fen8Er3XvQqYVUGiVBfFI8s/bM4vOtn3My9iR1K9dl8qOTGf7gcCqWq2h2eCKXLBbjtrz0Cn/LFkhNNTrrBQTAW28ZA/LUqmV2pMJsDRo0SM7u7P/gwYNlekp6SQDuwaxfd/OLZSRp6z6kU4t/mR2OyMG5uHNM+nMS03ZO41rSNdrVaseEoAn0adAHW5uCH71LFLwrV2Dt2tuV/sWLxvJmzeD1141m/TZtII8duYUosyQBuAdRcZepcK0ZU1983uxQRDb2XNhD6B+hfHfgOyzawuO+jzOu7Tja1GxjdmjiLtLSYMeO2z32t20zru+7ut453O79BTZquxBliyQA9+DfAwP598BAs8MQmVi0hVXHVhH6RyjrT66non1FXmj5Aq+0eYU6lWWi9eLswgVYs8ao8NesgZgY43a8hx6Ct982Kv2WLaEQhlwXosyRBCCfJv64iaEBraji4mh2KMIqMTWRefvmMfGPiRy+fJgaTjX4JOATRrYYyX3l7zM7PJGFlBTYuvV2j/3du43l7u7Qq5dR4QcGGqPyCSEKlgwFnA+7/rrIuD1BdB0v1/2Lg8s3LvPexveo/Xltnl32LA52DsztO5cTL5/g9Ydfl8q/mDlzBmbMgH79jNnzOnSATz8FJyf48EMjCTh//vbEO1L5i3tx5MiRct7e3o3NjiM7H374oZunp6efUqpFxkl+LBYLw4YNq+Xp6enn4+PTaPPmzQU+UoW0AOTDqFlfgEMy4x9/wexQyrQjl4/w2dbP+HbvtySmJtLduzvj2o6js1dnGbinGElKuj3c7qpVcNDaH7tWLRg40LhFr0sXcHExN04hzNCxY8eEfv36XevSpcsdkyQtWrTI5cSJE+UjIyMPhIWFVXzhhRc89+3bF1GQZUsLQB5FXohjO1/ieb0fnZv63P0FokBprdkYuZHeC3vTcEpDZu+ZzeAmgzn4wkFWBK+gS50uUvkXAydOwJQpRjO+q6vRjD95sjGT3oQJRhJw6pQx8U7fvlL5i6Jx6NChcr6+vo02btxYITU1leeee66mn5+fr4+PT6Px48dXBePM+7nnnqvp7e3d2MfHp9GMGTMqAyxfvtypZcuWDTp16lTfy8vLLzg42DMtLY3U1FT69evnlb79u+++m6fpfh9++OGbDRo0SM68/JdffrnvqaeeumJjY4O/v//1uLg4u1OnThXoPS7SApBHz02fBuWv8WmXf5odSpmSkpbC4kOLCf0jlJ1RO6laoSpvdXiLF1q9gHsld7PDK/Nu3DCG200/yz9qHQC1bl0YPtw4y+/UCSrKUAtl2tO/PF3rwKUDBdqU7VfN78bMPjPvOpDP3r17HQYNGlRv5syZJ9u2bXtzwoQJVV1cXNIOHDhw+ObNm6pVq1YNe/XqFbd169YK+/fvdzx8+PDBqKgou9atW/sGBQUlAOzfv7/i7t27D/j4+CR36NDBe86cOZXr16+fFBUVZZ8+1sDly5cLpItqVFSUvZeX163EwMPDI/nUqVP2tWvXTimI/YMkAHm2L2YLVWz8Gdje1NE1y4xrideYsWsGk/6cxJm4MzSo0oCpPaYy5IEhONpLB0yzaA0REbcr/I0bjaZ+R0fo3BlGjzY68NWvb3akQkBMTIzdY489Vn/x4sXHW7RokQiwbt0654iIiApLly6tDBAfH2976NCh8uHh4U4DBgyIsbOzo1atWqkPPfRQwubNmyu4uLhYmjRpcr1Ro0bJAAMGDIgJDw+v1LNnz7gzZ844DB06tFavXr2u9e3bN87MY80LSQDy6NzEn4i6WmI+3xLrVOwpvvjzC77e9TXxyfF08urElz2+pLt3d2yUXLkyQ1wcrF9/u9I/dcpY7usL//iHUeG3bw/ly5sbpyi+cnOmXhicnJzSqlevnhwWFlYpPQHQWqvQ0NDT/fr1u+MHfcWKFdlekMpqWmE3N7e0AwcOHFqyZInz1KlT3b7//nvXRYsWRaZvk5qaip+fXyOAbt26xX7++efncxOzh4dHSmRk5K0Jh6KiosoV5Nk/SB+AXEtKTiPi9GVsbBQ1qsgFy8Ky/dx2Bi0eRL1J9Zj05yR6NejFjmd3EDY0jJ4+PaXyL0JaG8PtfvKJ0XxfpYpxvX7BAmjeHKZNg8hIOHQIQkON6/xS+YviyN7eXv/666/HFy5cWGXq1KmuAIGBgde++uort6SkJAWwb98+h7i4OJsOHTrEL1682DU1NZXz58/bbdu2rVL79u2vg3EJICIiolxaWhqLFy92bd++fXxUVJRdWloaw4YNi/3oo4/O7d+//45LHHZ2dkRERByKiIg4lNvKH6B3796x8+fPr2KxWPjtt98qOjk5pRV0AiAtALn0+qyfmXQmhEXdtvDEIw+aHU6pYtEWlh1ZRugfoYSfDsfZwZkxbcYw+qHR1HKRwdyLUkwMrFtn3Je/ejVERRnLH3wQXn3VOMtv2xbKlct5P0IUN87OzpbVq1cf69Spk4+Tk1PamDFjLkdGRjo0adLEV2utXF1dU1auXHk8JCQkdsuWLZV8fX0bK6X0u+++e9bT0zN13759+Pn5XR81apRnZGRk+Xbt2sWFhITEbtu2zXHEiBFeFotFAbz33ntn8xLXBx98UG3y5Mn3X7lyxf6BBx5o1Llz52vff//9qQEDBlxbsWKFS+3atf0cHR0tX3/9dWRBvydFPh2wUqoWMAdwBzQwXWv9RU6vMXs6YItFU2nMQ6TZXyXh4wjs7WQYsoJwI+UG3+75ls+2fsbRmKN4unjyykOvMKL5CJwdnM0Or0ywWGDnztsD8fz5p7GscmVjmN1u3Yxhdz08zI5U5IdMB1xwli9f7hQaGuoeFhZ2zOxY8qo4TQecCozTWu9SSjkBO5VSa7XWh0yIJVc+XbSBm67bGVplqlT+BeBCwgWmbJvCVzu+4srNK7Sq3orv+n1Hv0b9sLORRqnCdumScXa/apUx3O7ly8Zwu61awZtvGpV+69Yy3K4QpV2R/9pqraOAKOvf8Uqpw0ANoPgmAL9/jI2jO5NeG2p2KCXawUsHmfjHRObtn0dKWgq9G/RmXNtxPOL5iNy7X4hSU43hdtM77+3caSyvVs24PS99Up2qVc2NU4jirGfPnvE9e/aMNzuOgmTq6ZZSygtoBvyZxbqRwEgAT0/PIo0ro992neSq61p6lP8Q5wrSwymvtNb8dvI3Qv8IZdWxVTjaOTKi2QjGtBmDdxVvs8Mrtc6evX2Wv3YtXLtmnNG3bQsffGBU/A8+CDbSp1KIMsu0BEApVQn4EXhFa/23++q01tOB6WD0ASji8G7xb16HpckHaO5dw6wQSqTktGQW7l/IxK0T2XdxH+4V3fmg8weMajmKKhVkcPeClpQEv/9++yx//35jeY0a8MQTRoXv7w/3ybQIQggrUxIApZQ9RuU/X2v9kxkx5EZamsbWVtGrTSOzQykxYm7GMG3HNCZvm0xUQhSN3Rozs/dMgpsE42DnYHZ4pcrJk7cr/N9+g+vXwd7euBd//Hijab9xY+P6vhBCZFbkCYAyLvZ+AxzWWk8s6vLzoukbL3HTco1j4+dgYyO/ojk5HnOcz7d+zsw9M7mRcoPAuoHM6jOLoHpBcn2/gNy8aYy4t2qV0Wv/r7+M5V5eMGSIcZbfuTNUqmRqmEKIEsKMK4APAyFAF6XUHuujuwlx5Gj30YscKv81jvaOUvnnYMuZLfT7oR/ek72ZtnMa/Rv1Z++ovawJWUPX+l2l8r8HWsORI/DFF8bZvKurUclPm2aMsf/FF8b6Eyfgyy+NiXek8hfi3r3xxhv3F9S+xo4dW/2tt966pwlL2rdv7+3k5PRg586d7xhcOyIiolzTpk0benp6+vXo0aNuYmJinn5wizwB0Fpv1lorrXVTrfWD1sfKoo7jbp6fNQlsk5n05Ktmh1LspFnSWHxoMW2/acvDMx8m7GQYbzzyBpGvRDL7sdk0dW9qdoglVnw8LF0Kzz9vVPING8Irrxgj7o0aZZz9x8QYLQCjR4OPjzTxC1HQJk2alKeRLywWC2lpaYUVDq+++uqFadOmncy8fOzYsTVffPHFi6dPnz7g4uKS+sUXX+TpXh7pA5yFUxfi+FNPoVZCP/wfkCl/08UnxTPpz0l4T/am/6L+XLp+icmPTubMmDN86P8h1Z2qmx1iiaO10WHv00+hSxdjuN0+fWDePHjgAfjqK+MMPyICPvvMGJTHUeZAEiLXjhw5Uq5OnTqNe/fuXadu3bqNu3XrVjc+Pt5m6dKlTgEBAfXSt1uyZIlzYGBgvRdeeKFGUlKSTcOGDRv17t27DsA777zj7u3t3djb27vxe++9Vy19v15eXn59+/b18vHxaXz8+PFyixcvdm7UqJFvgwYNGrVt2/ZW5XH48GHH1q1bN6hZs2aTDz74IE/TBQP06dMn3tnZ2ZJxmcVi4Y8//nAaPnz4VYCnn376yrJly/LUzVdGXcnCqBnTofw1Pu4qU/4CnIs7x6Q/JzFt5zSuJV2jXa12TAiaQJ8GfbC1kdFi8io21rg1L70D33nr6OBNm8KYMUZz/8MPy3C7onRqPaN1g8zLHvd9POaNR96Ijk+Kt/Gf4/+3+4MHNx18efRDo69ExUfZ9fmuT72M67Y9u+3I3cqMjIwsP23atMigoKDr/fv39xo/frzbO++8c/Hll1/2PH/+vF316tVTZ86cWWX48OGXg4ODr82ePbtaRETEIYDw8PAKCxYsqLJz587DWmtatGjh6+/vH1+1atW006dPO3zzzTcn/f39I8+fP2/34osvem3YsCGiYcOGyRcvXrz143js2LHyW7ZsORIbG2vr6+vr99prr0U7ODjc091tFy9etHNyckqzt7cHwMvLK/nixYt5+tWQBCALnw4eTLXlzgR3KttT/u65sIfQP0L57sB3WLSFx30fZ1zbcbSp2cbs0EoUiwV27bpd4W/dCmlpxi15gYG3h9utIXeaClEo7r///uSgoKDrACEhIVcmTZpUzcbG5uKAAQOuzJgxw/Uf//jHlV27dlX66aef/tbMvmHDhkrdu3ePTT8D79Gjx9WwsDCn/v37x3p4eCT7+/tft25XsXXr1vENGzZMBnB3d791TSAoKCjW0dFROzo6prq6uqacPXvWrl69egU6sU9+SAKQhSZ17ufbl0aaHYYpLNrCqmOrCP0jlPUn11PRviL/aPUPXn7oZepUrmN2eCVGdLQxzO6qVcaAPNHRxvKWLeFf/zIq/YceAjv5HyjKmJzO2J0cnCw5rfdw8kjNzRl/ZllN4wvw/PPPX+nRo0f98uXL6169el1NP5vOrQoVKljuvhVkPNu3tbUlNTX1joDmzJlz34cfflgdYPr06ZEdOnS4cbd9uru7p8bHx9umpKRgb29PZGRkOXd39+S8xC99ADJISk6jztgh/G9ZuNmhFLnE1ES+3vU1fl/60WNBD45cPsInAZ9wduxZPu/2uVT+d5GaClu2wFtvGePou7vD4MFGAhAUBHPnwsWLsH07vP++0cQvlb8QRSMqKqrcunXrKgLMnz/ftV27dgkAXl5eKe7u7imhoaEeI0eOvDVhkZ2dnU6fJrhz584JK1euvC8+Pt4mLi7OZuXKlZU7d+78tyGBO3XqdH3btm1OERER5QAyXgK4myFDhsSmTxmcm8ofwMbGhjZt2sTPmjWrMsDMmTOr9OzZMza3ZYK0ANzh33N+IdJlLpGXe5odSpGJvh7NVzu+Ysr2KVy6fokH73+QuX3nMqDxAMrZykXonJw/f+dwu1evGkPrtmkD775r3LLXvLkMtyuE2by8vBInT55cbeTIkRW8vb0TX3311ej0dYMGDboyZcoUu+bNmyemL3vqqaeifX19G/n5+d1YunTpyeDg4CvNmzf3BQgJCYl++OGHbx45cuSOH8jq1aunTpo0KbJv3771LRYLVapUSdmyZcvRgoi/RYsWDU6cOFH+5s2btu7u7k2//PLLyH79+sWFhoaeHThwYL0PPvigRuPGjW+8/PLLeZp1scinA86PopgO2GLROI19iBS7GK5/fKTUz/p35PIRPtv6Gd/u/ZbE1ES6e3dnXNtxdPbqLPfuZyE2Fo4fNx67dhm34e3bZ6zz8DCa9B99FAICjKl0hSgOZDpgo7d+z549vY8ePXowq/VDhgzxbNas2Y0xY8aU+CmLs1OcpgMulkJ/2sCNytsJua/0TvmrtWbTqU2E/hHKsr+W4WDrQEjTEMa0HUMjt7I93LHFYpzRHz9u3HaXXtmnP2Jibm9rbw+PPAKffGJU/E2ayL34QpREjRs39nV0dLRMmzbtjNmxmEESAKuPwz/Gprw7k18tfVP+pqSlsPjQYkL/CGVn1E6qVqjKWx3e4oVWL+Be6Z4GqCpRkpKMAXUyV+7Hjxvj6icm3t7W1hY8PaFePejf3/g3/VG/PlSsaNphCCHyoEGDBsnZnf0fPHjwcFHHU5xIAoDR/N+pZnecyj+BS8XSM+XvtcRrzNg1g0l/TuJM3BkaVGnA1B5TGfLAEBztS+doMteuZV3BHz8OZ84YA++kq1DBqNAbNDCa7zNW8rVrG2f6Qoh7ZrFYLMrGxqb4X28uhSwWiwKyvFtBEgDAxkbx42svmx1GgTkVzGg0aQAADaxJREFUe4ov/vyCr3d9TXxyPJ28OvFljy/p7t0dG1Wye6RpDVFR2VfyV67cub2bm1Ght29/ZwVfr57RU1+a7oUodAeio6Mbubm5XZMkoGhZLBYVHR3tAhzIan2ZTwDmrdvN2gO7mPzsYJwrluzparef207oH6EsPrQYgIF+AxnbZiwtqrcwObK8SU6GU6eyruBPnDBmxUtnY3O7qb5fvzsr+Lp1wdnZvOMQQkBqauozFy5c+PrChQt+yK3nRc0CHEhNTX0mq5VlOgH4fsM+hqwLxMbiyOuXe9K4Ysm7Hm7RFpYdWUboH6GEnw7H2cGZMW3GMPqh0dRyqWV2eNmKj8/+LP70aaNTXjpHR6Myr1fPuKc+c1O9DJkrRPHVokWLS0Bvs+MQf1dmE4DvwvYSvNofG4sj6waH0bh2yan8LyRcYOf5new4v4P5++dzNOYoni6eTAyayIjmI3B2MP+0V2tj4JvsKvno6Du3r1LFqNDbtjUG0MlYyXt4SFO9EEIUtDKZAGSs/H8bvIGOTevd/UUmuZhwkZ1RRmW/M2onO8/v5Fz8OQAUiodqPsT7nd+nX6N+2NkU7ceZkmKcrWfXVH/9+u1tlYJatYwKvU+fv1+Pd3Ep0tCFEKLMK5MJQNjh/dikVeS3kPXFqvJPr+x3nt95q9LPWNk3qNqATl6daFm9JS08WvDg/Q/i5OBUqDElJGR9X/zx48Z1+oxTYDs43G6q79Llzgrey8tYL4QQongoUyMBxiYkcl8l4za/K3E3qOJc4Z73mV+Xrl+6o6LfGbWTs3FnAaOy96nic6uib1G9Bc3ub1Yolb3WRnN8dk31Fy/euX3lyn8/e09/VK8uw94KUZwUh5EARfFVZloAvtuwh6dW9OSTNnN4tV+XIq38o69H3zqz3xG1g53nd3Im7vbAUw2qNKBD7Q608GhBy+otC7yyT029s6k+8xl9QsKd29esaVToPXr8vZKXYW6FEKJ0KBMJwMINu3lqVQA2VKSVt1ehlnX5xuVbHfR2Rhln+Kevnb613qeKD494PnLr7L6ZR7MC6bR3/XrOTfWpqbe3LVcO6tQxKvQOHe6s4OvUgfKlZywkIYQQ2Sj1CcCtyj+tIr+FbKBj07r3vE+tNTE3Yzh97TSnr53mYPTBW2f4p66durWdt6s37Wq1Y3Tr0bea8V3K3723W3KyMfnM1at3/pvV3+nj11+4cOc+XFyMCr15878PZVujhjHUrRBCiLLLlARAKdUN+AKwBb7WWn9cGOVs2ncyX5V/UmoSZ+PO3qrgbz3ibv99I+XOKZvru9anba22vNj6RR50b4F3xeZYbrrcqrBjdsHiHCryjH9nHOgmK/b2RlN85cpQrZoxIU3mpnpXV7l1TgghRPaKPAFQStkCU4BA4CywXSm1VGt9qKDLate4Nm3LPcf7jw+nifd9HI85TszNGK4mXiU6IYbo+KtEX48hOuEKZ+LOcDb+NFHX/7+9e43RorrjOP79lduyLLi71RgKrmKltdQoKrFYW+OFoGBT+sImmBrR1pjW2LTVWm/tiza92EtMNW1qqZeWxqqVGqsGY6hiTAygYFHwvhWrGBFEYUGtXPz3xTmrw7IL7Lo8s/vM75NMnpkzZ2bOec7AnOfM2XNeZsPWtbucq4kDGUMbTds/y8RtM2j4XxtD327jY5vbeH/DYWxZ38zijXDfRujo2HnM+V2/A2huTktLS/o8/PAP14vh3a03NPjhbmZmH00ZLQDHAe0R8SKApNuAWUC/VwBO/umVLNnxR06962rQbp7IWxuh4yDY1AabzsifhaVjPFu2N7CF9Kdso0aliWQ6l6amNBztkUfu3UN89Gj3ljczs3KVUQEYBxTnXl4DfK5rJEkXABcAtLW19elCn2k9irXtZ9OoVhrVQtOQVkYPbWH0sBbGDGuluaGF5hEtNDU00Ni464O96zJypN+dm5lZfRiwnQAjYi4wF9I4AH05x9xvnwWc1Z/JMjMzqwtlNES/ChRnqRmfw8zMzKxGyqgAPAZMlDRB0nBgNnB3CekwMzOrrJq/AoiI7ZIuAu4n/RngTRHxVK3TYWZmVmWl9AGIiAXAgjKubWZmZuW8AjAzM7OSuQJgZmZWQa4AmJmZVZArAGZmZhWk2N2g9QOEpPXAf/cQbX/gjRokZ6BxvqvF+a6ej5L3gyPigP5MjNWPQVEB2BuSlkXElLLTUWvOd7U439VT5bzbvuVXAGZmZhXkCoCZmVkF1VMFYG7ZCSiJ810tznf1VDnvtg/VTR8AMzMz23v11AJgZmZme8kVADMzswoa9BUASadLek5Su6TLy05PX0g6SNIiSU9LekrSd3J4q6SFkl7Iny05XJKuy3l+UtIxhXPNyfFfkDSnEH6spJX5mOskqfY57Z6kIZL+LenevD1B0tKc1tvztNFIGpG32/P+QwrnuCKHPyfptEL4gLw/JDVLmi/pWUnPSDq+CuUt6Xv5Hl8l6VZJDfVa3pJukrRO0qpC2D4v456uYbaLiBi0C2k64f8AhwLDgSeASWWnqw/5GAsck9dHA88Dk4BfAZfn8MuBX+b1mcB9gICpwNIc3gq8mD9b8npL3vdojqt87Iyy813I/8XA34B78/bfgdl5/XrgW3n9QuD6vD4buD2vT8plPwKYkO+JIQP5/gD+Apyf14cDzfVe3sA4YDUwslDO59ZreQMnAscAqwph+7yMe7qGFy9dl8HeAnAc0B4RL0bEVuA2YFbJaeq1iHgtIh7P65uBZ0j/Wc4iPSjIn1/J67OAeZEsAZoljQVOAxZGxJsR8RawEDg97xsTEUsiIoB5hXOVStJ44Azghrwt4BRgfo7SNd+d38d84NQcfxZwW0S8FxGrgXbSvTEg7w9J+5EeDjcCRMTWiNhIBcqbNAX5SElDgUbgNeq0vCPiYeDNLsG1KOOermG2k8FeARgHvFLYXpPDBq3czHk0sBQ4MCJey7vWAgfm9Z7yvbvwNd2EDwS/BX4AvJ+3Pw5sjIjtebuY1g/yl/dvyvF7+32UbQKwHrg5v/q4QdIo6ry8I+JV4DfAy6QH/yZgOfVf3kW1KOOermG2k8FeAagrkpqAfwDfjYiO4r5cy6+rv9mU9CVgXUQsLzstNTaU1DT8h4g4Gnib1FT7gTot7xbSr9MJwCeAUcDppSaqRLUo43q8j6z/DPYKwKvAQYXt8Tls0JE0jPTwvyUi7szBr+emPvLnuhzeU753Fz6+m/CynQB8WdJLpObaU4BrSc2fQ3OcYlo/yF/evx+wgd5/H2VbA6yJiKV5ez6pQlDv5T0NWB0R6yNiG3An6R6o9/IuqkUZ93QNs50M9grAY8DE3It4OKmj0N0lp6nX8nvNG4FnIuKawq67gc5ev3OAfxbCz8k9h6cCm3KT3/3AdEkt+dfWdOD+vK9D0tR8rXMK5ypNRFwREeMj4hBS2T0YEV8DFgFn5mhd8935fZyZ40cOn517jU8AJpI6SA3I+yMi1gKvSPp0DjoVeJo6L29S0/9USY05XZ35ruvy7qIWZdzTNcx2VnYvxI+6kHrPPk/q/XtV2enpYx6+QGqmexJYkZeZpPedDwAvAP8CWnN8Ab/PeV4JTCmc6+ukTlHtwHmF8CnAqnzM78ijQA6UBTiJD/8K4FDSf+jtwB3AiBzekLfb8/5DC8dflfP2HIUe7wP1/gAmA8tymd9F6uFd9+UN/Bh4Nqftr6Se/HVZ3sCtpL4O20itPt+oRRn3dA0vXrouHgrYzMysggb7KwAzMzPrA1cAzMzMKsgVADMzswpyBcDMzKyCXAEwMzOrIFcArK5J2iFphaQnJD0u6fN7iN8s6cK9OO9Dkqb0MU0LJDX35Vgzs/7iCoDVu3cjYnJEHAVcAfxiD/GbSbPQ7TMRMTPS5D9mZqVxBcCqZAzwFqR5FyQ9kFsFVkrqnDXuauCTudXg1znuZTnOE5KuLpzvq5IelfS8pC92vZiksZIezuda1RlH0kuS9pf0zbxvhaTVkhbl/dMlLc5puyPPEWFm1q88EJDVNUk7SCOrNQBjgVMiYnnndLQR0SFpf2AJaUjZg0kjEh6Rj58B/AiYFhHvSGqNiDclPQQsj4hLJM0ELo6IaV2ufQnQEBE/kzQkX29znvtgSkS8keMNAx4kzeO+mDRG/oyIeFvSZaSR8X6yL78nM6ueoXuOYjaovRsRkwEkHQ/Mk3QEaejVn0s6kTQV8Ti6nzZ1GnBzRLwDEBHF+d07J21aDhzSzbGPATflB/xdEbGihzReSxrn/p48Q+Ik4JE0xDvDSZUCM7N+5QqAVUZELM6/9g8gjRl/AHBsRGzLv8obennK9/LnDrr5txQRD+cKxhnAnyVdExHzinEknUtqdbioMwhYGBFn9TItZma94j4AVhmSDgeGkKaU3Q9Ylx/+J5MewgCbgdGFwxYC50lqzOdo7cX1DgZej4g/ATeQpvwt7j8W+D5wdkS8n4OXACdIOizHGSXpU73LqZnZnrkFwOrdSEmdTe8C5kTEDkm3APdIWkmale9ZgIjYIOkRSauA+yLiUkmTgWWStgILgCv38tonAZdK2gZsIU3ZWnQR0Aosys39yyLi/NwqcKukETneD0kz3JmZ9Rt3AjQzM6sgvwIwMzOrIFcAzMzMKsgVADMzswpyBcDMzKyCXAEwMzOrIFcAzMzMKsgVADMzswr6P/XQgGP1fIkeAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_absolute_time(results, [2, 10], max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "6e920708", + "metadata": {}, + "source": [ + "#### $N_\\text{ref} >> N_\\text{test}$\n", + "\n", + "Now we check whether the speed improvements still hold when $N_\\text{ref} >> N_\\text{test}$ ($N_\\text{ref} / N_\\text{test} = 10$) and a large part of the kernel can already be computed at initialisation time of the PyTorch (but not the KeOps) detector." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a75794e8", + "metadata": {}, + "outputs": [], + "source": [ + "experiments = {\n", + " 'keops': {\n", + " 0: {'n_ref': 2000, 'n_test': 200, 'n_runs': 10, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 500, 'n_runs': 10, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 1000, 'n_runs': 10, 'n_features': 2},\n", + " 3: {'n_ref': 20000, 'n_test': 2000, 'n_runs': 10, 'n_features': 2},\n", + " 4: {'n_ref': 50000, 'n_test': 5000, 'n_runs': 10, 'n_features': 2},\n", + " 5: {'n_ref': 100000, 'n_test': 10000, 'n_runs': 10, 'n_features': 2}\n", + " },\n", + " 'pytorch': {\n", + " 0: {'n_ref': 2000, 'n_test': 200, 'n_runs': 10, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 500, 'n_runs': 10, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 1000, 'n_runs': 10, 'n_features': 2}\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "fcdd840a", + "metadata": {}, + "outputs": [], + "source": [ + "results = {backend: {} for backend in backends}\n", + "\n", + "for backend in backends:\n", + " exps = experiments[backend]\n", + " for i, exp in exps.items():\n", + " results[backend][i] = experiment(\n", + " backend, exp['n_runs'], exp['n_ref'], exp['n_test'], exp['n_features']\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "27307020", + "metadata": {}, + "source": [ + "The below plots illustrate that KeOps indeed still provides large speed ups over PyTorch. The x-axis shows the reference batch size $N_\\text{ref}$. Note that $N_\\text{ref} / N_\\text{test} = 10$." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "0a3c0d27", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAEWCAYAAAAZ9I+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5wU9f3H8deHDtKLCAKCAQREUThRrNgIikKIAbEgtmA3sUbNL7aoMbG3WFA0qNgwKCqKGrCCCigIIigiCEc7QJAOd/f5/fGd0+W8zu7Nlffz8djH7s7Mznxmd+/ms99q7o6IiIhUblXiDkBERETip4RARERElBCIiIiIEgIRERFBCYGIiIighEBERERQQlDhmJmbWfsC1j9iZn9LeH6Bma0wsw1m1qQEx2sbHbNaSWNOlugc9iyF47SJjlW1FI7V3Mw+MLP1ZnZXqo8nRWdm15nZ43HHIZIsSghKyMwWmtk2M2uaa/kX0QWybfT8qej5gFzb3RMtPzN6fqaZZUUXmg1m9r2ZPWlmHZMZt7uf7+5/j45ZHbgb6OPudd19dWEJxc6IzvGjJO3rPTM7N3FZdA4LkrH/XMdaaGbHJBznh+hYWck+Vh6GA6uA+u5+xc7uzMxamNk4M1ua+D1NWF/TzEaa2U9mttzMLs+1/mgzm2tmm8xskpntsbMxxcHMbjSzZ4qxfW8zW5K4zN1vc/dz83uNSHmjhGDnfA+ckvPEzPYB6uSx3TfAGQnbVQMGA9/l2m6Ku9cFGgDHAJuB6WbWNRnB5vGLtjlQC/gqGfuXlNgDmOMlGEEsn1KbbOAt4KR8XnYj0CE67pHA1WbWN9pfU+C/wN+AxsA04IXixlVcZaH0SaRScHfdSnADFgL/B0xNWHYn8FfAgbbRsqei5SuARtGyE4A3gY+AM6NlZwIf5XGc14ExBcRxFbAMWAqcHR27fcKxHwbGAxsJScZTwC1Ax2iZAxuAicAH0fON0bKT8zhe1eh8VgELgIui11SL1jcAnohiSo+OVRXoDGwBsqJ9r422rxnt74foPXoEqJ1wvAHADOAnQgLVF7g12s+WaF8PRtsmnnsDYBSQASyKPqsqie91dNwfCYndcfm8v08TLqKbo2NdDbTNdc7vRec5OdrmNaAJ8GwU99Sc70O0fSfgHWANMA8YnM+xnwK2A9ui/R4TvV/3Rp/30uhxzWj73sAS4C/AcuDpAr431Uj4niYsX0ooMcp5/nfg+ejxcGBywrpdovelUwF/I9cCc6L3+UmgVsL6E6LPdm303u2b67V/Ab4EtkbxLiR8378kfEefICS1bwLrgXf55W+sN7Akj3iOIXyHtkXv7QZgZrT+LODraF8LgPNynWd2tP0GoCUheXomYf/9Ccn12ug70TnXsa+MYl9HSKRqReuaEv7O10bfiQ+Jvqu66Vaat9gDKK+3hH8u8wgXu6rRP+M9+HVCcAvwGHBBtOxFQslCURKCs4EV+cTQl3AR7Rr90xrNrxOCdcAhhNKgWjnxROvbknBhi5b9/Pp8jnk+MBdoTfiVOIkdL45jgUejeHYFPkv4x/qrcwTuAcZF+6pHuJj+I1rXM4r/2Cj+3YkuPtE/3HNz7Svx3EcBr0b7bEsopTknIY7twB+jz+0CwoXQCvqsE57v8L5FscwHfkNIROZExzuGcCEbBTwZbbsLsJhw8akG7E9Irrrkc+yfP6/o+c3AJ9F724xwIf17tK43kAn8k5A41M5rn9G2v0oIgEbRsuYJy/4AzIoe3wc8nGs/s4GTCnjfZid8Vz7ml+/e/sBK4MDoMxgWbV8z4bUzotfWTlj2CSEJ2D16/efRvmoRktobEt6LPBOC6PGNJFzMo2X9os/QgCOATUD3Avb38z74JcE+FqhOSBznAzUSjv0ZIZFoTEg8zo/W/YOQCFePboeRz3dRN91SeVOVwc57mlAdcCzhjzw9n+1GAWeYWUPCP5tXirj/pYR/IHkZTLjQzHb3jYR/ULm96u4fu3u2u28p4jELMhi4190Xu/sawj8zIDSAA44H/uzuG919JeGCPySvHZmZEX51Xubua9x9PXBbwvbnACPd/Z0o/nR3n1tYgFHVyBDgWndf7+4LgbuAoQmbLXL3ER7aAfwHaEG40JTUk+7+nbuvI/xi/c7d33X3TOAlwkULwq/ihe7+pLtnuvsXwMvAoCIe5zTgZndf6e4ZwE25ziubcFHc6u6bi3kOdaP7dQnL1hGSqpz169hR4vq8PJjwXbmVX6rYhgOPuvun7p7l7v8hlAQclPDa+6PXJp7HA+6+wt3TCb+kP3X3L6Lv9lh+eZ+Lzd3fiD5Dd/f3gbcJF+eiOBl4I/qubieUPtUGDs51Pkuj9+I1YL9o+XbC928Pd9/u7h+6uyaZkVKnurmd9zShqL0d4aKfJ3f/yMyaEaoUXnf3zeF6WKjdCcWIeWkJTE94viiPbRYX5SDF0DLXPhOPuQfhF86yhHOrUkAMzQhtLqYnbG+EX4wQfh2OL0GMTaM4EmNbRHgvcyzPeeDum6Lj16XkViQ83pzH85x97wEcaGZrE9ZXI3yPiqIlvz6vlgnPM3Yi8dsQ3dcnVMfkPF6fsL5+rtckrs9L7u9KTqx7AMPM7JKE9TXY8Vzy+t4U9X0uNjM7DriB8Gu/CuG7OauIL9/hc3H3bDNbTD7fOULpQ8653kFI5t+OvoePufvtJTgFkZ2iEoKd5O6LCHXQxxMaXBXkGeAKCkgc8jCQ8EsoL8sIF80cbfIKsRjHKoqCjrmY8Cuvqbs3jG713X3vfGJZRfgnvnfC9g08NKzM2d9v8omjoPNaRfjVldgCvg35l94UJpnv4WLg/YTzbeihx8IFRXz9Un59XksTnpc4Vnf/kfD5dktY3I1fGp1+lbjOzHYhfD4FNUrN/V3JiXUxcGuu96GOuz+XGFLJzgQIxfc/N/CNSo2a5bdvM6tJKKm5k1Bl0pCQjFpe2+dhh88lKv1qTRG+c1Ep1hXuviehHcLlZnZ0Ya8TSTYlBMlxDnBUVGxfkPsJVQsfFLSRmVU1s3Zm9gCh7vKmfDZ9ETjTzLqYWR3Cr5udtQIoqC//i8ClZtbKzBoB1+SscPdlhGLWu8ysvplVMbPfmNkRCftuZWY1ou2zgRHAPWa2K4CZ7W5mv422fwI4K+rqViVa16mwOKNqgBeBW82sXtQ17nJCQlYShb0nxfE60NHMhppZ9eh2gJl1LuLrnwP+z8yaRa3+r6eY52VmtQhtDABqRs9zjIr23yh6r/9IaMcAoUi+q5mdFL3meuDLQqpxLoq+K40JpWM5vRJGAOeb2YEW7GJm/cysoOqH4vgGqBXtszqhUWnNhPUrgLZmlvM/sEa0PgPIjEoL+uTavomZNcjneC8C/aLvanVC4r+V0MajQGZ2gpm1j5KIdYQGs9lFPVGRZFFCkARRveO0Imy3xt3/V0D9YC8z20Bomf4eoTj2AHfPs9jS3d8ktDKfSGjANLEk8edyI/AfM1trZoPzWD8CmADMJDToyl0qcgbhn2tOy/IxhPpRovi+Apab2apo2V+i2D8xs58ILcX3is7vM0Lju3sI/yjf55dfYfcBfzCzH83s/jzivITwK3EBofHmaGBk0d6CX/kH4SK51syuLOE+gPBrkHChGUL4VbmcXxoBFsUthO5+XxKKsz+PlhVHTo8JCA1EE+vobyD05lhEeL/vcPe3otgzCN0VbyV8tgeST/uQBKMJSeKCaL+3RPuaRkg2Hoz2NZ/Q2DMporYcFwKPE36lbyQ0+s3xUnS/2sw+jz6XSwkX9h+BUwmNXXP2N5eQjC2IvgeJVRu4+zzgdOABQgnVicCJ7r6tCOF2IHzvNwBTgH+7+6TinbHIzjO1XRGRVDCzhYSeIO/GHYuIFE4lBCIiIqKEQERERFRlICIiIqiEQERERKhgAxM1bdrU27ZtG3cYIiLlxvTp01e5e7PCt5SKrkIlBG3btmXatEJ7/4mISMTM8hrhVCohVRmIiIiIEgIRERFRQiAiIiIoIRARERGUEIiIiAhKCERERAQlBCIiIoISAhERESGFCYGZjTSzlWY2O5/1V5nZjOg228yyzKxxtG6hmc2K1pX7kYa+/x569YL33487EhERkbylsoTgKaBvfivd/Q5338/d9wOuBd539zUJmxwZrU9LYYylompV6N4dGjaMOxIREZG8pWzoYnf/wMzaFnHzU4DnUhVL3Nq0gYceijsKERGR/MXehsDM6hBKEl5OWOzA22Y23cyGF/L64WY2zcymZWRkpDLUEtu4EbZvjzsKERGR/MWeEAAnAh/nqi441N27A8cBF5nZ4fm92N0fc/c0d09r1qxsTtj1179CGQ1NREQEKBsJwRByVRe4e3p0vxIYC/SMIa6kSU+H3XaLOwoREZH8xZoQmFkD4Ajg1YRlu5hZvZzHQB8gz54K5cXSpbD77nFHISIikr+UNSo0s+eA3kBTM1sC3ABUB3D3R6LNBgJvu/vGhJc2B8aaWU58o939rVTFWRrS0+Gww+KOQkREJH+p7GVwShG2eYrQPTFx2QKgW2qiKn3Z2SohEBGRsi9lCYEEmZlw881hYCIREZGySglBitWoAddcE3cUIiIiBSsLvQwqtLVrYdEiyMqKOxIREZH8KSFIsRdfhLZtQzsCERGRskoJQYotXQpmGodARETKNiUEKZaeDrvuCtWrxx2JiIhI/pQQpJi6HIqISHmghCDF0tOhZcu4oxARESmYuh2m2PXXwy67xB2FiIhIwZQQpNjvfx93BCIiIoVTlUEKbdgAH30EP/0UdyQiUlG98QZcdRW4xx2JlHdKCFJo5swwqdHkyXFHIiIVjTvceSeceCL873+wcWPhrxEpiBKCFEpPD/fqZSAiybR1K5x9digZOOkk+PBDqFs37qikvFNCkEJKCEQk2VauhKOOgqeeghtugBdeUMNlSQ41Kkyh9HSoVQsaNYo7EhGpCL78MlQRZGSERGDw4LgjkopEJQQplJ4eSgfM4o5ERMq7V16Bgw8OU6p/+KGSAUk+lRCk0F/+EjJ5EZGScofbb4frroMDDgiJgQY7k1RQQpBC++0XdwQiUp5t2QLnngvPPgunnAJPPAG1a8cdlVRUqjJIEfdQx/fdd3FHIiLl0bJlcMQRIRm49dZwr2RAUillCYGZjTSzlWY2O5/1vc1snZnNiG7XJ6zra2bzzGy+mV2TqhhTae1aGDIEXn017khEpLz5/PNQPfDVV/Df/4bqArVFklRLZQnBU0DfQrb50N33i243A5hZVeAh4DigC3CKmXVJYZwpoS6HIlISL70Ehx4KVarAxx/DwIFxRySVRcoSAnf/AFhTgpf2BOa7+wJ33wY8DwxIanClYOnScK/GPyJSFO5w002h98D++8PUqdCtW9xRSWUSdxuCXmY208zeNLO9o2W7A4sTtlkSLcuTmQ03s2lmNi2jDDXpVwmBiBTVpk2hivHGG+GMM2DiRGjePO6opLKJMyH4HNjD3bsBDwCvlGQn7v6Yu6e5e1qzZs2SGuDOyEkIVEIgIgVZsiTMefLSS3DHHWEEwpo1445KKqPYEgJ3/8ndN0SPxwPVzawpkA60Tti0VbSsXPnjH8NMh7VqxR2JiJRVn30GPXvCN9/AuHFw5ZVqPCjxiS0hMLPdzMJX38x6RrGsBqYCHcysnZnVAIYA4+KKs6SaN4dDDok7ChEpq0aPhsMPDz8apkyBE06IOyKp7FI2MJGZPQf0Bpqa2RLgBqA6gLs/AvwBuMDMMoHNwBB3dyDTzC4GJgBVgZHu/lWq4kyVUaOgTRvo3TvuSESkLMnOhr/9DW67LSQEL78MTZvGHZUIWLgGVwxpaWk+bdq0uMMAoEUL6NcPHn887khEpKzYsAGGDg3DD597Ljz0ENSoEW9MZjbd3dPijULKAg1dnALbt8OKFephICK/WLQI+veH2bPh3nvh0kvVXkDKFiUEKbBiRehTrB4GIgIweXIYYGjrVhg/Hn7727gjEvm1uMchqJA0BoGI5PjPf+DII6F+ffjkEyUDUnYpIUiBnFEKlRCIVF5ZWXD11XDmmWGcgU8/hU6d4o5KJH+qMkiBfv1gwQJVGYhUVj/9BKeeCm+8ARdeGNoMVK8ed1QiBVNCkAI1akC7dnFHISJxWLAgNB6cOzf0IrjwwrgjEikaJQQp8OyzsHEjDB8edyQiUprefx9OOimMNfD223DUUXFHJFJ0akOQAk8+GcYjF5HKY8QIOOaYMMjQp58qGZDyRwlBCqSnq/2ASGWRmQl//nMoETz66NCToEOHuKMSKT4lBCmQnq4eBiKVwdq1oRHxffeFpOD116Fhw7ijEikZtSFIsvXrw00JgUjF9u23cOKJoRHhiBFhKGKR8kwJQZKtWBHuVWUgUnG9+y4MHgxVqoTHhx8ed0QiO08JQZK1bw9btsQdhYikyr//HeYh6NQJXntNXYyl4lAbghSoWTPcRKTi2L49jClw0UVw3HFhfgIlA1KRKCFIsldegUsuCcOWikjFsGYN9O0LDz8chiN+5ZUwN4FIRaKEIMkmToRRo6Bq1bgjEZFk+PprOPBA+OijML7IP/+pv2+pmNSGIMnU5VCk4njrLTj5ZKhVCyZNgoMPjjsikdRRCUGSLV2qHgYi5Z17mJCoX7/QTmDqVCUDUvEpIUgylRCIlG/btoVRBy+7DAYMCFUFbdrEHZVI6qUsITCzkWa20sxm57P+NDP70sxmmdlkM+uWsG5htHyGmU1LVYzJ5h7uW7eONw4RKZmMjDAfweOPw//9H4wZA3Xrxh2VSOlIZRuCp4AHgVH5rP8eOMLdfzSz44DHgAMT1h/p7qtSGF/SmcEPP/ySGIhI+TF7dhh5cNkyGD0aTjkl7ohESlfKSgjc/QNgTQHrJ7v7j9HTT4BWqYqltJnFHYGIFMfrr0OvXrB1K3zwgZIBqZzKShuCc4A3E5478LaZTTez4QW90MyGm9k0M5uWkZGR0iAL89FHMHAgLFoUaxgiUkTucMcd0L8/7LVXaDzYs2fcUYnEI/aEwMyOJCQEf0lYfKi7dweOAy4ys3xHCnf3x9w9zd3TmjVrluJoCzZ7dhiwpJo6c4qUeVu2wJlnhoGGBg0KJQNqECyVWawJgZntCzwODHD31TnL3T09ul8JjAXKRc6enh4mO2nePO5IRKQgK1bAUUeFQcRuvhmefx7q1Ik7KpF4xfZb1szaAP8Fhrr7NwnLdwGquPv66HEf4OaYwiyWpUtDMqASApGya8aMUEWwahW89BL84Q9xRyRSNqTs0mVmzwG9gaZmtgS4AagO4O6PANcDTYB/W2iFl+nuaUBzYGy0rBow2t3fSlWcyaQxCETKtv/+F4YOhcaNQ5uf7t3jjkik7EhZQuDuBbbTdfdzgXPzWL4A6PbrV5R9jRpBqwrTV0Kk4nCH224LYwsceCCMHQstWsQdlUjZosLtJHruubgjEJHcNm+Gc84Jf5+nnRYGHapVK+6oRMqe2HsZiIikytKlcMQRodHgP/4BTz+tZEAkP0oIkmTRIkhLg//9L+5IRARg2jQ44ACYMydUEVxzjQYNEymIEoIk+eEHmD4dsrLijkREXngBDjsMqleHyZPDJEUiUjAlBEmSnh7u1ctAJD7Z2XDDDTBkCPToAZ99BvvuG3dUIuWDGhUmiRICkXht3AjDhsHLL8NZZ8HDD0PNmnFHJVJ+KCFIkvT0MNJZgwZxRyJS+SxeHKoFZsyAO++Eyy9XewGR4lJCkCS77w7HHad/QiKl7ZNP4He/g02bwqyFxx8fd0Qi5ZMSgiS54oq4IxCpfJ55Bs49NyTkEydCly5xRyRSfqlRoYiUO9nZcO21YRjiXr1C40ElAyI7RwlBErhDu3Zw771xRyJS8a1fDwMHwu23w/DhMGECNGkSd1Qi5Z+qDJJgzRpYuFDtB0RSbeHCMFPhV1/B/ffDxRfr704kWZQQJMHSpeG+Zct44xCpyD76CH7/e9i2Dd58E/r0iTsikYpFVQZJoDEIRFLrySfhqKPCjKKffqpkQCQVlBAkQU5CoBICkeTKygo9eM4+O0xS9MknsNdecUclUjGpyiAJdt89NHLS/OoiybNuHZxySqgeuOQSuPtuqKb/WCIpU+ifl5n1Ak4HDgNaAJuB2cAbwDPuvi6lEZYDffuGm4gkx3ffwYknwrffwiOPwHnnxR2RSMVXYEJgZm8CS4FXgVuBlUAtoCNwJPCqmd3t7uNSHWhZlp0NVVT5IpIU770HJ50UHr/9Nhx5ZKzhiFQahZUQDHX3VbmWbQA+j253mVnTlERWjhxwQKjXHD067khEyrfHHoOLLoIOHWDcOGjfPu6IRCqPAn/X5iQDZraLmVWJHnc0s/5mVj1xm7yY2UgzW2lms/NZb2Z2v5nNN7Mvzax7wrphZvZtdBtWkpMrLUuWQN26cUchUn5lZsKll4aqgWOPhSlTlAyIlLaiFnR/ANQys92Bt4GhwFNFeN1TQEG168cBHaLbcOBhADNrDNwAHAj0BG4ws0ZFjLVUbd8OK1eqy6FISf34Y5iQ6IEHQo+C117TrKEicShqQmDuvgn4PfBvdx8E7F3Yi9z9A2BNAZsMAEZ58AnQ0MxaAL8F3nH3Ne7+I/AOBScWsVm2LNyry6FI8X3zDRx0UGg38MQTYeriqlXjjkqkcipyQhD1NjiN0LsAIBl/trsDixOeL4mW5bc8r8CGm9k0M5uWkZGRhJCKR4MSiZTMO+/AgQeGob8nTgxjDYhIfIqaEPwJuBYY6+5fmdmewKTUhVV07v6Yu6e5e1qzZs1K/fhNmoS6z86dS/3QIuWSOzz4IBx3HLRqBVOnwqGHxh2ViBRpmI+o6P+DhOcLgEuTcPx0oHXC81bRsnSgd67l7yXheEnXsSPcd1/cUYiUD9u3h0GGHn00TFL0zDNQr17cUYkIFFJCYGYjzGyffNbtYmZnm9lpO3H8ccAZUW+Dg4B17r4MmAD0MbNGUWPCPtGyMmftWti6Ne4oRMq+1avDHASPPgrXXANjxyoZEClLCisheAj4W5QUzAYyCAMTdQDqAyOBZ/N7sZk9R/il39TMlhB6DuR0V3wEGA8cD8wHNgFnRevWmNnfganRrm5294IaJ8bm4ovh44/h++/jjkSk7JozJ4w8mJ4OTz8Np58ed0QikluBCYG7zwAGm1ldII1fhi7+2t3nFbZzdz+lkPUOXJTPupGEhKNMW7pUDQpFCjJ+PAwZAnXqhN4EBx0Ud0QikpeitiHYQBmtw49bejp06xZ3FCJljzvccw9cdRXsu28YebB168JfJyLx0Aj8O8E9JAQqIRDZ0datcM45YaChgQPho4+UDIiUdUoIdsL69bBxowYlEkm0ciUccww8+ST87W/w4ouwyy5xRyUihSnW7OJmVicasVAAM7jjDs3GJpJj1qzQeHDFCnj+eTj55LgjEpGiKlIJgZkdbGZzgLnR825m9u+URlYO1KsHV14JPXrEHYlI/MaNg4MPDmMNfPihkgGR8qaoVQb3EOYXWA3g7jOBw1MVVHmxciV89x1kZ8cdiUh83OH22+F3vwsjdk6dCmlpcUclIsVV5DYE7r4416KsJMdS7jzxRJiidcuWuCMRiceWLXDGGXDttaFE4P331aZGpLwqakKw2MwOBtzMqpvZlcDXKYyrXEhPh4YNQ/9qkcpm+XLo3TsMP/z3v8Po0VC7dtxRiUhJFbVR4fnAfYQZB9OBt8lnQKHKRIMSSWX1xRdhLoI1a+Dll+H3v487IhHZWUUdmGgVYepjSaAxCKQyevnlUE3QpEkYtnu//eKOSESSoUgJgZm1Ay4B2ia+xt37pyas8iE9HfbeO+4oREqHe6gauOGGMPzw2LGw225xRyUiyVLUKoNXgCeA1wC1qY/cc48aUEnlsGkTnH02vPBCKB149FGoVSvuqEQkmYqaEGxx9/tTGkk5NGhQ3BGIpF56OgwYAJ9/Dv/8Z5ibwCzuqEQk2YqaENxnZjcQGhNuzVno7p+nJKpyYPVqmD0bunfXnO5ScU2dGpKB9evh1VfDKIQiUjEVNSHYBxgKHMUvVQYePa+UPvooDMSiQVikonruuVBNsNtuMGEC7LNP3BGJSCoVNSEYBOzp7ttSGUx5kp4e7tXLQCqa7OzQcPCWW+Cww0KvgmbN4o5KRFKtqAMTzQYapjKQ8mbpUqhaFXbdNe5IRJJn48bQNuaWW8L0xe++q2RApLIoaglBQ2CumU1lxzYElbbbYXp6KEqtWjXuSESS44cfwmBDs2aFHjR/+pMaD4pUJkVNCG5IaRTlkAYlkopkyhQYOBA2b4Y33oC+feOOSERKW1FHKny/JDs3s76EIY+rAo+7++251t8DHBk9rQPs6u4No3VZwKxo3Q9lrTTi9tvDP0+R8m7UKPjjH6F1a5g0KcxYKCKVT4EJgZl95O6Hmtl6Qq+Cn1cB7u71C3htVeAh4FhgCTDVzMa5+5ycbdz9soTtLwH2T9jFZncvs4Oidu8edwQiOycrC667Dv71LzjqKHjxxTAcsYhUToU1KtwFwN3ruXv9hFu9gpKBSE9gvrsviHonPA8MKGD7U4Dnihx5jLZuhWefhYUL445EpGTWrw9VBP/6F5x/Prz1lpIBkcqusITAC1lfkN2BxQnPl0TLfsXM9gDaARMTFtcys2lm9omZ/S6/g5jZ8Gi7aRkZGTsRbtEtXgynnx7mfhcpb77/Hg4+GMaPh4cegocfhurV445KROJWWBuCXc3s8vxWuvvdSYpjCDDG3bMSlu3h7ulmticw0cxmuft3ecTwGPAYQFpa2s4kMEWmMQikvPrgAzjpJMjMDKUCxxwTd0QiUlYUVkJQFagL1MvnVpB0oHXC81bRsrwMIVd1gbunR/cLgPfYsX1BrJQQSHn0xBMhAWjSBD77TMmAiOyosBKCZe5+cwn3PRXoEE2dnE646J+aeyMz6wQ0AqYkLGsEbHL3rWbWFDgE+FcJ40g6JQRSnmRmhgmJ7r0X+vQJMxY21DBjIpJLYQlBiYclcfdMM7sYmEAoaRjp7l+Z2c3ANHcfF206BHje3ROL+zsDj5pZNqEU4/bE3glxS0+HunWhfmHNKkVitm4dDBkSqgcuvRTuuguqFXX0ERGpVGzH63Culf/muWUAACAASURBVGaN3X1NKcazU9LS0nzatGkpP86KFSEpUNdDKcvmzw+zE86fHxoPDh8ed0RSFpnZdHfXFG1ScAlBeUoGSlPz5uEmUlZNnAh/+ANUqRLmIzjiiLgjEpGyrqiTG0mCf/8bPv447ihE8vbww6GtQIsWofGgkgERKQolBMWUnQ1//jO89lrckYjsaPt2uPhiuPDCMBfBlCmw555xRyUi5YUSgmJavTr8423ZMu5IRH6xZg0cd1xoK3DllfDqq2r0KiLFo/bGxaQuh1LWzJ0bGg/+8AM8+SSceWbcEYlIeaSEoJiWLg33SgikLJgwAU4+GWrUCA0JDzkk7ohEpLxSlUEx5ZQQqMpA4uQO990Hxx8Pe+wBU6cqGRCRnaOEoJjOPDMUzaqEQOKybRucd15o3Nq/f+jxsscecUclIuWdEoJiql4dWreGqlXjjkQqo1WrQpfCESPguuvg5ZfDqJkiIjtLbQiK6eGHoVYtOOusuCORyuarr0LjwaVL4dln4dRfzQwiIlJyKiEopkcfhf/+N+4opLJ5/XXo1Qs2b4b331cyICLJp4SgmNLT1X5ASo873HlnaCvQoUNoPHjggXFHJSIVkRKCYti6NdThKiGQ0rB1a6iauuqqMC/Bhx9Cq1ZxRyUiFZUSgmJYtizcq8uhpNrKlXDUUfCf/8CNN8Lzz0OdOnFHJSIVmRoVFsPKlWH2OJUQSCrNnBmqCDIy4MUXYdCguCMSkcpACUEx9OwZinFFUuWVV+D006Fhw1BF0KNH3BGJSGWhKoNiqlYt3ESSyR1uuw0GDoS99w6NB5UMiEhpUkJQDE89BZddFncUUtFs3hxKBf7619Cd8L33oEWLuKMSkcpGCUExvP02jBsXdxRSkSxbBr17w+jRoYTgmWegdu24oxKRyiilCYGZ9TWzeWY238yuyWP9mWaWYWYzotu5CeuGmdm30W1YKuMsqqVL1cNAkmf6dDjggDAC4dixcO21YBZ3VCJSWaWsNtzMqgIPAccCS4CpZjbO3efk2vQFd78412sbAzcAaYAD06PX/piqeIsiPV31upIcL70Ew4ZBs2ZhcqJu3eKOSEQqu1SWEPQE5rv7AnffBjwPDCjia38LvOPua6Ik4B2gb4riLBL3UEKgLoeyM7Kz4aabYPBg6N49NB5UMiAiZUEqE4LdgcUJz5dEy3I7ycy+NLMxZta6mK/FzIab2TQzm5aRkZGMuPO0aRM0ahRmOhQpiU2bYMiQMNDQsGHwv//BrrvGHZWISBB3B7rXgOfcfauZnQf8BziqODtw98eAxwDS0tI8+SEGu+wCS5akau9S0S1ZAgMGwBdfwB13wBVXqL2AiJQtqSwhSAcSf0+3ipb9zN1Xu3vOUD+PAz2K+lqR8uLTT0PjwW+/hddegyuvVDIgImVPKhOCqUAHM2tnZjWAIcAOnfbMLLG3dX/g6+jxBKCPmTUys0ZAn2hZbF5/PcxFv3p1nFFIeTN6NBxxROhKOGUK9OsXd0QiInlLWZWBu2ea2cWEC3lVYKS7f2VmNwPT3H0ccKmZ9QcygTXAmdFr15jZ3wlJBcDN7r4mVbEWxcyZISnQBDNSFNnZ8Le/hbEFjjgCxoyBpk3jjkpEJH8pbUPg7uOB8bmWXZ/w+Frg2nxeOxIYmcr4iiM9HRo31qAxUjD3MNLgP/4B77wDf/wjPPgg1KgRd2QiIgWLu1FhuTF7NrRvH3cUUlZt3AjPPgsPPBC+K02ahETgwgvVXkBEygcNXVwE27fDtGnQq1fckUhZ8/33oZFgq1Zw3nlh4qsnnoDFi+Gii5QMiEj5oRKCIsjIgLS0UBcs4g4TJ8L994deA1WqwEknwSWXwCGHKAkQkfJJCUERtGwJH3wQdxQStw0b4OmnQ1XAnDlh2OHrroPzzw8lBCIi5ZkSgiLIyoKqVeOOQuLy3Xfw0EMwciSsWxfms3jqKTj5ZKhVK+7oRCqG6dOn71qtWrXHga6oOjsVsoHZmZmZ5/bo0WNlXhsoISiCzp1h0CC49da4I5HSkp0N774bqgXGjw8J4aBBoVrgoINULSCSbNWqVXt8t91269ysWbMfq1SpkrJRZyur7Oxsy8jI6LJ8+fLHCeP+/IoSgkKkp4cR5po1izsSKQ3r18N//hOqBebNg+bNw3gC552nqa9FUqyrkoHUqVKlijdr1mzd8uXLu+a3jRKCQkyZEu4PPjjeOCS1vv02JAFPPhmSgp49Q3uBQYOgZs24oxOpFKooGUit6P3NtzpGCUEhpkwJ9cT77Rd3JJJs2dkwYUKoFnjrLahePUxLfMklcOCBcUcnIlK61HCjEJMnhy6HGmmu4li3Du67Dzp1guOPhxkz4Kab4Icf4JlnlAyIVEbz5s2r0aFDh73jjiM//fv3b9e2bduuHTp02HvQoEFtt27dmvSWTEoICjFkCAwfHncUkgxz58LFF4cugn/+c5hbYPRoWLQIrr8edtst7ghFRPJ22mmnrVmwYMHsefPmfbVlyxa79957kz47ihKCQvzpTzB0aNxRSEllZ4dJqX7729BbZMQI+P3vYerUUPpzyikq/RGRHc2ZM6dG586du7z//vt1MjMzOe+881p17dq1c8eOHbvccccdTQGys7M577zzWnXo0GHvjh07dhkxYkQjgNdff71eWlraXr17927ftm3brqeeemqbrKwsMjMzOemkk9rmbH/TTTftWpyYTj755HVVqlShSpUqpKWlbVyyZEnS/3OpDUEBvvsO6tWDXYv1sUlZsHZtGDfgoYdgwYLQQ+CWW8JkQ/o8Rcq2s8+m9ezZJHVu2a5d2TRyJIsL227mzJk1hwwZ8puRI0d+36tXr8133nln0wYNGmTNnj37682bN9sBBxzQ6cQTT/zpk08+qTNr1qzaX3/99VfLli2r1rNnz859+vTZADBr1qxdvvjii9kdO3bcdvjhh3cYNWpUo/bt229dtmxZ9W+//fYrgFWrVpVodJutW7faCy+80OTuu+8u9FyKSwlBAa6+OtQvf/dd3JFIUc2ZEyYYGjUKNm2CQw8NMw8OHBgaDYqI5GfNmjXVfve737UfM2bMdz169NgC8O6779afO3dunXHjxjUCWL9+fdU5c+bU+vDDD+sNHjx4TbVq1WjdunXmgQceuOGjjz6q06BBg+x99tlnY5cuXbYBDB48eM2HH35Y94QTTvhp8eLFNYcNG9b6xBNPXDdw4MCfShLjsGHD2hx00EEb+vbtuyF5Zx4oIciHeyhSPvrouCORwmRlhWqBBx6A//0vdBM89dTQW2D//eOOTkSKqyi/5FOhXr16WS1bttw2adKkujkJgbvbXXfd9cNJJ520wwX8jTfeaJDffizXyGVmRrNmzbJmz549Z+zYsfUfeeSRZi+88ELjl156aWHONpmZmXTt2rULQN++fdfee++9S3Pv94orrmixatWqahMmTEjJz1S1IcjHokWwfLnGHyjL1qyBO+4I01L/7ndhIKHbboMlS0J1gZIBESmO6tWr+5tvvvndc8891+SRRx5pDHDssceue/jhh5vltOr/8ssva/70009VDj/88PVjxoxpnJmZydKlS6t99tlndQ877LCNEKoM5s6dWyMrK4sxY8Y0Puyww9YvW7asWlZWFmeeeebaf/zjH+mzZs3aoUqkWrVqzJ07d87cuXPn5JUM3H333U0nTpzY4JVXXllQNUVj6auEIB+TJ4d7TXlc9syaFUoDnnkGNm+Gww+HO++EAQPC9MMiIiVVv3797AkTJszv3bt3x3r16mVddtllqxYuXFhzn3326ezu1rhx4+3jx4//bujQoWsnT55ct3Pnznubmd90001L2rRpk/nll1/StWvXjeeff36bhQsX1jr44IN/Gjp06NrPPvus9jnnnNM2OzvbAG6++eYlxYnr6quv3qNFixZb09LSOgOccMIJP955553Lknnu+veZjylTYJddYJ994o5EADIzYdy4kAi8914YLOr000M3wm7d4o5ORMq7vfbaa1tOg7+mTZtmzZ49++ucdQ8++GA6kJ77NY8++ugS4FcX9nr16mVNmjRpfuKyXr16bZ4zZ87XubctqszMzOklfW1RKSHIx5//DH376hdn3Favhscfh3//OwwctMce8M9/wjnnQJMmcUcnIlJxpPRyZ2Z9gfuAqsDj7n57rvWXA+cCmUAGcLa7L4rWZQGzok1/cPc8Z2dKld/8JtwkHjNmhNKA0aNhyxY48sgwuuCJJ2oqahEpu0444YT1J5xwwvq44yiJlCUEZlYVeAg4llCkMtXMxrn7nITNvgDS3H2TmV0A/As4OVq32d1jmUHg669DG4LBg8M4BFI6tm+HV14JicCHH0KdOjBsWKgW6Jrv/FwiIpIMqexl0BOY7+4L3H0b8DwwIHEDd5/k7puip58ArVIYT5G98gqce264QEnqZWSE3gHt2oUkbMmS0EhwyRJ45BElAyIipSGVVQa7ww59SZcABU0bcw7wZsLzWmY2jVCdcLu7v5LXi8xsODAcoE2bNjsVcI7Jk8PEN40bJ2V3ko/p00NpwPPPw9atcMwxoa1Av36qFhARKW1losmcmZ0OpAFHJCzew93TzWxPYKKZzXL3Xw3G4O6PAY8BpKWl7fRc2u6hh0H/Um2xUHls3w4vvxwSgcmTQ0+Oc84J1QKdO8cdnYhI5ZXKKoN0oHXC81bk0W3DzI4B/gr0d/etOcvdPT26XwC8B5TKMDPffhtatmtAouRasQL+/ndo2zZMKLRiBdxzD6Snh/kGlAyISHlzzTXXJG2O1Msvv7zl9ddf37ykr588eXLt/fbbr1P79u13mGypOFKZEEwFOphZOzOrAQwBxiVuYGb7A48SkoGVCcsbmVnN6HFT4BAgsTFiysyYEe41IFFyTJ0KZ5wBbdqEKYb32ScMM/zNN6FrZ4N8B/8UESnb7r///hbF2T47O5usrKyUxFK3bt3sp59++vv58+d/9fbbb3973XXXtS7uBEopSwjcPRO4GJgAfA286O5fmdnNZpZTIH8HUBd4ycxmmFlOwtAZmGZmM4FJhDYEpZIQDB4cfr3qF2vJbdsWugsedBD07Aljx8J558HcufDWW6GNQBUNmi0iZci8efNqtGvXbu/+/fu323PPPffu27fvnuvXr68ybty4esccc8zPndDHjh1b/9hjj/3NhRdeuPvWrVurdOrUqUv//v3bAdx4443NO3TosHeHDh32vvnmm3fN2W/btm27Dhw4sG3Hjh33/u6772qMGTOmfpcuXTrvtddeXXr16tUxZ99ff/117Z49e+7VqlWrfW655ZZizcu67777bt1nn322ArRt23Z748aNM5ctW1asZgEpbUPg7uOB8bmWXZ/w+Jh8XjcZiG2MQE2PWzLLlsGjj4bb8uXQsSPcf3/oOli/ftzRiUh50rMne+Ve9vvfs+aaa8hYv54qRx9Nh9zrTz+dVZdeyuply6g2YAA7jCTz2WfMK+yYCxcurPXoo48u7NOnz8ZBgwa1veOOO5rdeOONK/70pz+1Wbp0abWWLVtmjhw5sslZZ5216tRTT1331FNP7Tp37tw5AB9++GGd0aNHN5k+ffrX7k6PHj06H3300eubNm2a9cMPP9R84oknvj/66KMXLl26tNrFF1/c9r333pvbqVOnbStWrPj5V/z8+fNrTZ48ed7atWurdu7cuetVV12VUbNmzWK3jZs0aVKd7du3W5cuXbYWvvUv9Dstwbp1YZrcnHkMpHDu8MkncNppYRTBm26C7t3hzTfDeA6XXKJkQETKh912221bnz59NgIMHTp09eTJk+tWqVKFwYMHrx4xYkTjVatWVf3888/rDho0aF3u17733nt1jz/++LX169fPbtCgQXa/fv1+nDRpUj2AFi1abDv66KM3Rtvt0rNnz/WdOnXaBtC8efOf6xD69Omztnbt2t6iRYvMxo0bb1+yZEmxf7QvWrSo+llnnbXniBEjFhZ3EqQy0cugrPj00zAGwYUXxh1J2bd1K7z4YigBmDYtXPQvvBAuugg6/CpvFxEpnoJ+0derR3ZB61u0ILMoJQK55TVtMcAFF1ywul+/fu1r1arlJ5544o/Vq1cv1n7r1KmTXZTtEksDqlatSmZm5g4BjRo1quFtt93WEuCxxx5bePjhh29KXL9mzZoqxx13XPsbbrghPScBKQ6VECSYMgXM4MCCRkuo5NLT4W9/C40EzzgDNmwIvQSWLIF771UyICLl17Jly2q8++67uwA8++yzjQ8++OANEOrkmzdvvv2uu+5qMXz48FU521erVs1zpkU+8sgjN4wfP77h+vXrq/z0009Vxo8f3+jII4/81RDGvXv33vjZZ5/Vmzt3bg2AxCqDwpxxxhlrc6ZIzp0MbNmyxfr169d+yJAhq88666wfS3L+SggSTJ4cRsVTEfeO3OHjj2HIkNBt8NZbQ9L09tswZ04oGdAQzyJS3rVt23bLAw88sOuee+6599q1a6tdeeWVGTnrhgwZsrpFixbbunfvviVn2WmnnZbRuXPnLv3792936KGHbjr11FNXd+/evXOPHj06Dx06NOOQQw7ZnPsYLVu2zLz//vsXDhw4sP1ee+3VZeDAgXsmI/aRI0c2mjp1at3Ro0c37dSpU5dOnTp1mTx5cu3i7MPcd3osnzIjLS3Np02bVqLXZmdDo0ahj/wjjyQ5sHJqy5YwiuD998MXX4QuguecE6oF9kzKV1hE4mZm0909Le44Zs6cubBbt26rCt8yNebNm1fjhBNO6JAzBXJuZ5xxRpv9999/02WXXRZbjMkwc+bMpt26dWub1zq1IYisWhW6Gh5+eNyRxG/xYnj4YRgxIrwvXbqEJOn008PIgiIilcnee+/duXbt2tmPPvro4sK3Lr+UEER23TW0lq+s3OGjj0JpwNix4Xn//qGXwJFHhrYVIiIV1V577bUtv9KBr7766uvSjicOSggi7pXzord5cxhE6IEHYObMUG1y+eWhXUDbtnFHJyKVSHZ2drZVqVKl4tRjlzHZ2dkG5NvjQY0KI/vvD9dcE3cUpWfRIvjLX6BVqzDVc3Y2PPZY6C3wr38pGRCRUjc7IyOjQXTRkiTLzs62jIyMBsDs/LZRCQFhMqOZM0Mr+orMHd5/P1QLvPpqWDZwYKgWOPzwyllCIiJlQ2Zm5rnLly9/fPny5V3Rj9VUyAZmZ2ZmnpvfBkoI+KXtQEWd0GjjRnj22VAtMHs2NGkCV18NF1wQxhMQEYlbjx49VgKaeD5GSggI4w9UrQoHHBB3JDtv+/YwNsDnn/9ymzEDNm2C/faDJ54IXStrF6t3qoiIVHRKCAgjFO6/P9SpE3ckxbN5M8yaFS76X3wR7r/8Msw2CFC3bjivc8+FQYPgkENULSAiInlTQgCceGLZ71+/fn1o55D4y3/OHMiZWrtRozCp0J/+FO67d4f27TXNsIiIFI0SAuCyy+KOYEdr1vzyiz/n/ptvQqNAgObNoUePME5AzsV/jz30619EREpOCUHMVqzY8Vf/55/DwoW/rG/TJlzwTzvtl4t/ixaxhSsiIhWUEoJS4h6GBE688H/xBSxd+ss2HTpAz55w/vnhwr///tC0aXwxi4hI5aGEIMnWrIH58+Hbb8N9zuNvvoEfowkpq1QJ8yYcffQvv/q7dQuTB4mIiMRBCUEJrF6940U/8eK/Zs0v25lB69ahcd/gwbDvvuHiv+++5a9Hg4iIVGwpTQjMrC9wH1AVeNzdb8+1viYwCugBrAZOdveF0bprgXOALOBSd5+QylgBMjMhIwOWLYPly8Mt5/GyZfDDD+Gin/NLP8QZ6vlzLvrt24ei//btwxTBtWqlOmoREZGdl7KEwMyqAg8BxwJLgKlmNs7d5yRsdg7wo7u3N7MhwD+Bk82sCzAE2BtoCbxrZh3dPSvZcbpDWhqkp4dkIDuPaR8aNoTddgvj/g8ZsuNFv107XfRFRKT8S2UJQU9gvrsvADCz54EBQGJCMAC4MXo8BnjQzCxa/ry7bwW+N7P50f6mJDtIM+jSJXTj22230II/8b55c43qJyIiFV8qE4LdgcUJz5cAB+a3jbtnmtk6oEm0/JNcr909r4OY2XBgOECbEg7M//TTJXqZiIhIhVHux7Fz98fcPc3d05o1axZ3OCIiIuVSKhOCdKB1wvNW0bI8tzGzakADQuPCorxWREREkiSVCcFUoIOZtTOzGoRGguNybTMOGBY9/gMw0d09Wj7EzGqaWTugA/BZCmMVERGp1FLWhiBqE3AxMIHQ7XCku39lZjcD09x9HPAE8HTUaHANIWkg2u5FQgPETOCiVPQwEBERkcA8Z8acCiAtLc2nTZsWdxgiIuWGmU1397S445D4lftGhSIiIrLzlBCIiIiIEgIRERGpYG0IzCwDWFTIZk2BVaUQTlmj865cdN6Vy86c9x7urkFcpGIlBEVhZtMqYwManXflovOuXCrreUtyqcpARERElBCIiIhI5UwIHos7gJjovCsXnXflUlnPW5Ko0rUhEBERkV+rjCUEIiIikosSAhEREak8CYGZ9TWzeWY238yuiTuekjCz1mY2yczmmNlXZvanaHljM3vHzL6N7htFy83M7o/O+Usz656wr2HR9t+a2bCE5T3MbFb0mvvNzEr/TPNmZlXN7Aszez163s7MPo1ifSGaVZNolswXouWfmlnbhH1cGy2fZ2a/TVheJr8fZtbQzMaY2Vwz+9rMelWGz9vMLou+47PN7Dkzq1URP28zG2lmK81sdsKylH+++R1DKjl3r/A3wmyL3wF7AjWAmUCXuOMqwXm0ALpHj+sB3wBdgH8B10TLrwH+GT0+HngTMOAg4NNoeWNgQXTfKHrcKFr3WbStRa89Lu7zTjj/y4HRwOvR8xeBIdHjR4ALoscXAo9Ej4cAL0SPu0SffU2gXfSdqFqWvx/Af4Bzo8c1gIYV/fMGdge+B2onfM5nVsTPGzgc6A7MTliW8s83v2PoVrlvlaWEoCcw390XuPs24HlgQMwxFZu7L3P3z6PH64GvCf88BxAuHET3v4seDwBGefAJ0NDMWgC/Bd5x9zXu/iPwDtA3Wlff3T9xdwdGJewrVmbWCugHPB49N+AoYEy0Se7zznk/xgBHR9sPAJ53963u/j0wn/DdKJPfDzNrQLhgPAHg7tvcfS2V4PMmTM1e28yqAXWAZVTAz9vdPyBM/Z6oND7f/I4hlVhlSQh2BxYnPF8SLSu3omLR/YFPgebuvixatRxoHj3O77wLWr4kj+Vlwb3A1UB29LwJsNbdM6PnibH+fH7R+nXR9sV9P+LWDsgAnoyqSh43s12o4J+3u6cDdwI/EBKBdcB0Kv7nnaM0Pt/8jiGVWGVJCCoUM6sLvAz82d1/SlwX/RKoUH1JzewEYKW7T487llJWjVCc/LC77w9sJBTv/qyCft6NCL9g2wEtgV2AvrEGFZPS+Hwr4ndISqayJATpQOuE562iZeWOmVUnJAPPuvt/o8UrouJBovuV0fL8zrug5a3yWB63Q4D+ZraQULx7FHAfoci0WrRNYqw/n1+0vgGwmuK/H3FbAixx90+j52MICUJF/7yPAb539wx33w78l/AdqOifd47S+HzzO4ZUYpUlIZgKdIhaKdcgNDwaF3NMxRbViz4BfO3udyesGgfktCweBryasPyMqHXyQcC6qJhwAtDHzBpFv8b6ABOidT+Z2UHRsc5I2Fds3P1ad2/l7m0Jn91Edz8NmAT8Idos93nnvB9/iLb3aPmQqFV6O6ADodFVmfx+uPtyYLGZ7RUtOhqYQwX/vAlVBQeZWZ0orpzzrtCfd4LS+HzzO4ZUZnG3aiytG6GF7jeE1sV/jTueEp7DoYSivS+BGdHteEJ96f+Ab4F3gcbR9gY8FJ3zLCAtYV9nExpZzQfOSlieBsyOXvMg0WiWZeUG9OaXXgZ7Ev7BzwdeAmpGy2tFz+dH6/dMeP1fo3ObR0KL+rL6/QD2A6ZFn/krhFbkFf7zBm4C5kaxPU3oKVDhPm/gOUI7ie2EEqFzSuPzze8YulXum4YuFhERkUpTZSAiIiIFUEIgIiIiSghERERECYGIiIighEBERERQQiCCmWWZ2Qwzm2lmn5vZwYVs39DMLizCft8zs7QSxjTezBqW5LUiIiWhhEAENrv7fu7eDbgW+Ech2zckzLCXMu5+vIeJjERESoUSApEd1Qd+hDBnhJn9Lyo1mGVmOTPi3Q78JipVuCPa9i/RNjPN7PaE/Q0ys8/M7BszOyz3wcyshZl9EO1rds42ZrbQzJqa2fnRuhlm9r2ZTYrW9zGzKVFsL0XzW4iIlJgGJpJKz8yyCCO/1QJaAEe5+/ScqXfd/Sczawp8Qhj+dg/CaIldo9cfB/wNOMbdN5lZY3dfY2bvAdPd/QozOx643N2PyXXsK4Ba7n6rmVWNjrc+mrchzd1XRdtVByYS5rGfQhjf/zh332hmfyGM2ndzKt8nEanYqhW+iUiFt9nd9wMws17AKDPrShgq9jYzO5ww7fLu5D1N7DHAk+6+CcDdE+e3z5mAajrQNo/XTgVGRhf8V9x9Rj4x3kcYo/+1aPbHLsDHYYh6ahCSBBGRElNCIJLA3adEpQHNCOPdNwN6uPv26Fd7rWLucmt0n0Uef2/u/kGUcPQDnjKzu919VOI2ZnYmoVTi4pxFwDvufkoxYxERyZfaEIgkMLNOQFXC9LkNgJVRMnAk4aIMsB6ol/Cyd4CzzKxOtI/GxTjeHsAKdx8BPE6Y3jhxfQ/gSuB0d8+OFn8CHGJm7aNtdjGzjsU7UxGRHamEQARqm1lOUb0Bw9w9y8yeBV4zs1mEGQfnArj7ajP72MxmA2+6+1Vmth8wzcy2AeOB64p47N7AVWa2HdhAmKI20cVAY2BSVD0wzd3PjUoNnjOzmtF2/0eYvU9EpETUqFBERERUZSAiIiJKCERERAQlBCIiIoISAhEREUEJcN5hUAAAABdJREFUgYiIiKCEQERERFBCICIiIsD/AzjSbRzqavU2AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_absolute_time(results, [2], max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "cf6a0dfc", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhsAAAEWCAYAAADPUVX+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5gUVdbH8e8hgySBERFUUAliAkHW+KqorIhZQcwJcWVNGNacs655XRXFgIJZERWzsGaUARWQqGIgg5KVNOf9497RdpzQM0xPTfh9nqee6bqVTldXV5+5deuWuTsiIiIimVIt6QBERESkclOyISIiIhmlZENEREQySsmGiIiIZJSSDREREckoJRsiIiKSURU22TAzN7OtCpn+gJldkTJ+hpnNM7PlZta0BNtrHbdZo6Qxl5b4HrZIOo6yYmaPmdn1xZi/0GOjvDGz183sxKTjSIeZbRaPv+olWLaumb1iZkvM7LlMxCclY2bHmtlbScchlVeRyYaZzTSz1WbWLE/5+HhSbx3HH4vjh+SZ785YflIcP8nM1sUT1nIz+87MHjWzdqX2rgB3/4e7Xxe3WRO4A+jh7vXdfVEmf5Die/ywlNY12sz6pZbF9/Btaay/NJSHRMzMdjGzj5PafrrM7GozezK1zN17uvvjScVUmPj93zd33N1/iMffuhKs7kigOdDU3XuXQmy1zOz5GKOb2V55ppuZ3WJmi+Jwi5lZyvROZpZtZivj307rG1MSinu+ye/76u5D3b1HZiIUSb9m4zvg6NwRM9sOqJfPfNOAE1LmqwH0Ab7JM98n7l4faATsC/wKZJvZtumHXrB8/utqDtQBJpXG+qV0lVKS0gsYWQrrqTISSA43B6a5+9riLlhIrB8CxwFz85nWHzgU2AHYHjgIOD2urxbwMvAksCHwOPByLM+YmABV2BplkRJz90IHYCZwOfB5Stm/gcsAB1rHssdi+Txgw1h2IPA64YRwUiw7Cfgwn+28CjxfSBwXAnOA2cApcdtbpWz7fsKPzQpCAvMYcD3QLpY5sBx4D3g/jq+IZUfls73q8f0sBL4F/hmXqRGnNwIGx5hmxW1VB7YGfgPWxXUvjvPXjuv7Ie6jB4C6Kds7BPgCWEpIzvYHbojr+S2u6z9x3tT33ggYAiwAvo+fVbXUfR23+wshaexZxGd9CfB1nP9RoE6cNhE4KGXemnHfdI7vKXf/Lgd2ISSyl8eY5scYG8VlW8f5T43Lvh/Ldwc+BhYDP/LHMfMYcB/wGrAMGANsmSf2ccCO+eyf3eO69orjHYC3gZ+BqUCflHUUtS8/Av4DLAGmAPukLHsS4ThZFvfzsfns3/2B1cCauJ++jOWjgX55tnNn3A/fArvG8h/jvjwxZZ2FHld5tp+67kWEY3ZLwndiUfw8hwKN4/xPADmEfwaWA/9K+exyvwebACPi/pwBnFbAtq/J895PpQTHSCHH7k+5n3FK2cdA/5TxU4FP4+sehO+tpUz/Adi/gPWPBm4CPiN8R18GmqRM35k/jt0vU2OJy94Q9/2vwFax7Pq4zHLgFaBp3P9Lgc/549z6p32eesxQ8PmmFzA+rutH4Oo87zPv9/UkUs7LhGPuc8Kx/jmwa55tXxffzzLgLaBZnFaHkMAtivvic6B5Ub8zGir/UPQM4QdoX8KJeWvCD+pPhP9S8iYb1wODgDNi2bOEGpF0ko1TgHkFxLA/4US6LbABMIy/JhtLgN0IJ7A6ufHE6fl9WX9fvoBt/oPwg7Ip0AQYxZ9Psi8BD8Z4NiKchE4v6D0STvAj4roaEE4uN8Vp3WL8+8X4WwId4rTRxB+i/GInnKBfjutsTahdOjUljjXAafFzO4OQrFkB73kmIanIfc8fpezDfwHPpMx7CDChkP17CuHHZwugPvAi8ESe+YfE/VeXcDwti8dLTcKJt1PK57so7qcahBPy0ynbakHKD0fu/iEcNz8C3WL5BnH85LiezoQf2I5p7su1wMAY31HxM2sS17sUaJ8SzzYF7OOrgSfzlP3+Gads5+T4mV1P+HG4j5BY9Ij7qX5Rx1U+285d91nx/deN+2m/uO4sQiJ+V97vf8r4nz7rOP9/Cd+5ToRErXs6751iHiNFnKfySzaWAH9LGe8KLIuvBwKv55n/VeD8AtY/mnCM5Z6DXsh9L4Tv6yLgAML3d784npWy7A/ANnG/14xlMwjJXiNCgj+NcK6tEd/3o4V8v/IeM3nPN3sB28V4tiecPw8tZH2/r4NwLP0CHB9jOTqON03Z9jeEf+TqxvGb47TTCcdgPcLx2wVoWNTvjIbKPxSnOu8JwiWS/YDJhC9efoYAJ5hZY2BPYHia659NOMjz04fwxZvo7isIJ628Xnb3j9w9x91/S3ObhelDOOn+6O4/E/6rAcDMmhNOLOe6+wp3n0846ffNb0XxOnF/YKC7/+zuy4AbU+Y/FXjE3d+O8c9y9ylFBRgvF/UFLnH3Ze4+E7idcJLI9b27P+ThGvvjhB/C5oWs9j8p7/kG/rh89iRwgJk1jOPHE46JghwL3OHu37r7ckKNSd881eFXx/33K3AM8I67P+Xua9x9kbt/kTLvS+7+mYcq+KGEH7ZcBwBvuLunlPUmJIM93f2zWHYgMNPdH3X3te4+nvCj0TvNfTmfcEyscfdnCAl4rzgtB9jWzOq6+xx3X59Ldt/FGNcBzxCSv2vdfZW7v0WoIdgqjeMqP7Pd/d74/n919xnxuFvl7gsIbZv2TCdIM9uUkOBf5O6/xc/rYVIupRahuMdIcdUnJBy5lgD1437LOy13eoNC1vdEyjnoCqBPPG6OA0a6+8j4/X0bGEs4LnM95u6T4n5fE8sedfdv3H0JoQb4G3d/Jx7jzxGS4RJx99HuPiHG8xXwFGl+roRjerq7PxHjfYrwj9dBKfM86u7T4ufyLH98H9cQ/lHYyt3XuXu2uy8t6fuQyqM412yfIPwX04aQUOTL3T80syzCZZZX3f3XlDZZhWlJqIrNzyZAdsr49/nM82M6GymGTfKsM3WbmxP+O5mT8t6qFRJDFiHTz05tn0bI/CH8mJSkvUGzGEdqbN8T9mWu369lu/vKuP36hawz73veJC4728w+Ao4ws5eAnsA5haxnk3ziqsGfE53UbW3KX9v2pEq9Jr+SP7+HAwi1XanOBYa4+8SUss2Bv5nZ4pSyGoRjO519OStPQvM9sIm7rzCzo4ALgMFxP52fTsJYgHkpr38FcPe8ZfUp+rjKz5+O0Zg43w3sQfihrUb4LzYdmwC5SU6u7wk1COkuX5xjpLiWAw1TxhsCy93dzSzvtNzpyyhY3u9GTcJxszkhYU39Ma5JqA3Nb9lceT/T/D7jEjGzvwE3E2piahFqrtK9Ayjv5wKFnFf48/fxCcJ3+en4D+eTwGUpCZZUUWnXbLj794Rr0QcQqjsL8yRwPoUkJfk4DPiggGlzCAdwrs3yC7EY20pHYdv8EVhFuE7ZOA4N3X2bAmJZSDh5bJMyfyMPjWRz17dlAXEU9r4WEv6T2DxPnAXVOqUj73uenTL+OOG/uN6ERr6528kvxtn5xLWWP59QU5crbB8UKN5ptCehHUaq3sChZpaaEP0I/C/lM2js4c6KM0hvX7a0P2fOv+8fd3/T3fcj1BxNAR4qIOTSPE6LOq7S2f6NsWw7d29I+HytkPlTzQaamFlqbUBxjr/iHiPFNYnQODTXDvzRSHwSsH2ez3N7Cm9Enve7sYbwGfxIqPVIPa42cPebU+Zfn/exIv5NbZS/cRHrHka4vLapuzcitOWxQuZPlfdzgTQ/11jrd427dyS0+ziQ9Gu6pBIrbqvoUwnXY1cUMd89hMst7xc2k5lVN7M2ZnYv4RrjNQXM+ixwkpl1NLN6wFXFCztf8wjXigvyLHC2mbUysw2Bi3MnuPscQqOo282soZlVM7MtzSy3mnIe0Cq3Zbu75xB+fO40s40AzKylmf09zj8YONnM9onramlmHYqKM1azPwvcYGYNzGxz4DxCsldS/4zvuQmhduqZlGnDgR0JNRqpieQCwmWE1DifAgbGz7c+4UftGS/4ToShwL5m1sfMaphZ0zRvRdwd+CqfqtrZwD7AOWZ2Rix7FWhnZsebWc047GRmW6e5LzciHBM1zaw3oQ3TSDNrbmaHmNkGhCR0edwf+ZkHtC6NOxLSOK7S0YAQ7xIza0loiJ033oKOvx8JDRxvMrM6ZrY94RyR7vFX3GPkL8ystpnViaO1Yhy5P6pDgPPiPtmE8A/QY3HaaEKjyrPjOs6M5e8VsrnjUs5B1xIatK8jvN+DzOzv8ZxWx8z2MrNW6b6PwsTLW7Pi9qub2Sn8OTH/0/kmakCodfrNzLoRLlPmyu/7mmok4XtyTPwuHgV0JHx/CmVme5vZdvHy0lJCQlbQd0GqkGKd8OL1xbFpzPezu7+bp8o51S6xGnMp4UvfENjJ3ScUsL7XgbsIJ4IZFH5CSNfVwONmttjM+uQz/SHgTULL8nH8tTbnBEL1ZO6dG88T/qslxjcJmGtmC2PZRTH2T81sKfAO0D6+v88IDQLvJFw3/h9//GdxN3Ckmf1iZvfkE+dZhP98viU0xB0GPJLeLsjXMEIi9S3hssbvnWnF67MvEC6lvZhSvpLY2j7uz51jDLmX3r4jtJg/q6CNuvsPhFqz8wmX077gz/+VFqTAW17jOvcBLjazfrG6vwehTcNsQlXwLYQqZih6X44B2hL+m70BONLdFxG+R+fFdf5MqGk5g/zlVmUvMrNxaby/ohR4XKXpGkICuYRwt0/e4/wm4PL4uV6Qz/JHExocziY0mr7K3d9Jc9vFOkYKMJVQu9OS8H39lT++Ow8SGitOIDR8fi2W4e6rCbfFnkC4a+IUQgPK1YVs6wlCsjKX0CD27LiuHwkNpi8l/JD/SEjaSvMW19PiOhcRGpqm9imT3/lmAHCtmS0DriQk0sR48/u+kjJ9EaFG4vy4vX8BB7r7Qoq2MeFcuJTQtu9/FN62S6oIKzgfkKrGzGYSWrgX+GNhZlcC7dz9uDILrBBm9jXhR//rDG/nJMK+2T2T25HyycxGE+4+eTjpWEQqosS73paKI15aOZU/36GRmFhtPCTTiYaIiKwf9WQnaTGz0wjVw6+7e6FtccqKu6/O0whPRETKIV1GERERkYxSzYaIiIhkVIVos9GsWTNv3bp10mGIiFQo2dnZC909K+k4RCpEstG6dWvGji3yjlsREUlhZvn1tixS5nQZRURERDJKyYaIiIhklJINERERySglGyIiIpJRSjZEREQko5RsiIiISEYp2RAREZGMUrIhIlIO5eTA44/Ds88WPa9IeadkQ0SknPn0U9hlFzjpJBg6NOloRNafkg0RkXJi1iw44YSQaPz4Y6jZeOmlpKMSWX8VortyEZHK7Lff4I474MYbYc0auOSSMDRokHRkIqVDyYaISELc4cUX4YILYOZMOPxwuO022GKLpCMTKV26jCIikoCvvoLu3eHII6F+fXj3XXjhBSUaUjkp2RARKUMLF8IZZ0DnziHh+O9/Yfz4kHiIVFa6jCIiUgbWrAmJxdVXw7JlcOaZcNVV0KRJ0pGJZJ6SDRGRDHvzTTj3XJgyBfbbD+66Czp2TDoqkbKjyygiIhkybRocdBDsvz+sXQsjRoTEQ4mGVDVKNkREStmSJXDhhbDttvC//8Gtt8LEiSHxMEs6OpGyp8soIiKlZN06eOwxuPRSWLAATj4ZbrgBNt446chEkpXxmg0zq25m483s1Tj+mJl9Z2ZfxKFTpmMQEcm0Dz+Ebt2gXz/Yaiv47DMYPFiJhgiUzWWUc4DJecoudPdOcfiiDGIQEcmIH36Avn1hjz1g/nwYNiwkHl27Jh2ZSPmR0WTDzFoBvYCHM7kdEZGytnJluI21Qwd4+eVwG+uUKXD00WqXIZJXpms27gL+BeTkKb/BzL4yszvNrHaGYxARKTXu8PTTIcm45prQ6HPKlJB4bLBB0tGJlE8ZSzbM7EBgvrtn55l0CdAB2AloAlxUwPL9zWysmY1dsGBBpsIUEUlbdna4XHL00dC0abjT5JlnYPPNk45MpHzLZM3GbsDBZjYTeBrobmZPuvscD1YBjwLd8lvY3Qe5e1d375qVlZXBMEVECjdvXmj4udNOoe+MQYNg7Fj4v/9LOjKRiiFjyYa7X+Lurdy9NdAXeM/djzOzFgBmZsChwMRMxSAisj5Wr4Z//xvatYPHH4eBA0OycdppUL160tGJVBxJ9LMx1MyyAAO+AP6RQAwiIgVyh9deg/POg+nToVcvuP12aN8+6chEKqYySTbcfTQwOr7Wsw1FpNyaPDnUYLz5ZkguRo6Enj2TjkqkYlN35SIiwC+/hIelbbcdfPop3HknTJigREOkNKi7chGp0tatg4cegssvh59/hv794brrQO3SRUqPajZEpMoaNQp23BHOOCM8NG3cOHjgASUaIqVNyYaIVDnffQdHHgndu4cntD73XEg8OulJTSIZocsoIlJlLF8ON98cbmetXj1cLjn/fKhbN+nIRCo3JRsiUunl5IQHpF10EcyeDcceG5KOVq2SjkykatBlFBGp1D77DHbbDY4/HjbZBD76CJ58UomGSFlSsiEildLs2XDiifC3v8HMmfDYYzBmDOy6a9KRiVQ9uowiIpXKb7+FPjJuuAHWrIGLL4ZLL4UGDZKOTKTqUrIhIpWCOwwfHhp8fvcdHHpoaAi65ZZJRyYiuowiIhXehAmw775w+OFQrx68/Ta89JISDZHyQsmGiFRYixbBP/8Z+scYPx7uvRe++CIkHiJSfugyiohUOGvWhJ4+r7oKli6FAQPg6quhadOkIxOR/CjZEJEK5e23wwPTvv4a9tkH7rordDUuIuWXLqOISIUwYwYccgj06BHuOBk+PCQeSjREyj8lGyJSri1dGnr+7NgR3nsv9Pz59dch8TBLOjoRSUfGkw0zq25m483s1TjexszGmNkMM3vGzGplOgYRqXhycuDRR6FdO7j11tDF+LRpIfGoXTvp6ESkOMqiZuMcYHLK+C3Ane6+FfALcGoZxCAiFcjHH0O3bnDKKdCmTehy/NFHoUWLpCMTkZLIaLJhZq2AXsDDcdyA7sDzcZbHgUMzGYOIVBw//RRqMHbbDebMCc8w+fhj2GmnpCMTkfWR6ZqNu4B/ATlxvCmw2N3XxvGfgJb5LWhm/c1srJmNXbBgQYbDFJEk/fpreNx7+/bwwgtw+eUwdWpIPNQuQ6Tiy1iyYWYHAvPdPbsky7v7IHfv6u5ds7KySjk6ESkP3OG556BDB7jySjjgAJgyJSQe9esnHZ2IlJZM9rOxG3CwmR0A1AEaAncDjc2sRqzdaAXMymAMIlJOjR8P55wDH3wAO+wAQ4bAnnsmHZWIZELGajbc/RJ3b+XurYG+wHvufiwwCjgyznYi8HKmYhCR8mf+fOjfH7p0gcmT4cEHITtbiYZIZZZEPxsXAeeZ2QxCG47BCcQgImVs9Wq44w5o2zbcWXLuuTB9ekg8qldPOjoRyaQy6a7c3UcDo+Prb4FuZbFdESkfRo6EgQNDPxn77w933hnaaYhI1aAeREUkY6ZMCY0+e/UK46+9Bq+/rkRDpKpRsiEipW7xYjjvPNhuO/joI7j9dpgwISQeIlL16KmvIlJq1q2DwYPhsstg0SLo1w+uvx422ijpyEQkSarZEJFS8b//hTtMTj8dtt463GEyaJASDREpQbJhZtXMrGEmghGRiuf776FPH9hrL/j5Z3jmmZB4dO6cdGQiUl6klWyY2TAza2hmGwATga/N7MLMhiYi5dmKFaHXzw4d4NVX4ZprQoPQPn3UxbiI/Fm6NRsd3X0p4aFprwNtgOMzFpWIlFvuMGxYeI7JddfBYYeF55hceSXUq5d0dCJSHqWbbNQ0s5qEZGOEu68BPHNhiUh5NHYs7L57eEBa8+ahq/Fhw2DTTZOOTETKs3STjQeBmcAGwPtmtjmwNFNBiUj5MncunHJKeNT7jBnhjpPPPw+Jh4hIUdK69dXd7wHuSSn63sz2zkxIIlJerFoFd98dLpesWgUXXhge/95QTcRFpBjSSjbMrClwFbA74fLJh8C1wKLMhSYiSXGHESPg/PPhm2/goINCx1xt2yYdmYhUROleRnkaWAAcQXhi6wLgmUwFJSLJmTQJevSAQw+FWrXgzTdD4qFEQ0RKKt1ko4W7X+fu38XheqB5JgMTkbL1889w1lmwww6hIeg998CXX4bEQ0RkfaSbbLxlZn1jh17VzKwP8GYmAxORsrF2Ldx3X6i5+O9/Qw+g06eHxKNmzaSjE5HKIN1k4zRgGLA6Dk8Dp5vZMjPTXSkiFdS770KnTnDmmaFG44svQuLRrFnSkYlIZZJWsuHuDdy9mrvXiEO1WNbA3dUuXaSC+eab0BnXvvvCypXw4osh8dhuu6QjE5HKKO2nvprZwcD/xdHR7v5qEfPXAd4HasftPO/uV5nZY8CewJI460nu/kVxAxeR4lu2DG68Ee64I1wiufFGGDgQ6tRJOjIRqczSvfX1ZmAnYGgsOsfMdnP3SwpZbBXQ3d2Xx95HPzSz1+O0C939+RJHLSLFkpMDTzwBF18cOug64QS46SbYZJOkIxORqiDdmo0DgE7ungNgZo8D44ECkw13d2B5HK0ZB3VxLlLGPvkEzjkn9PjZrRsMHw5/+1vSUYlIVVKcR8w3TnndKJ0FzKy6mX0BzAfedvcxcdINZvaVmd1pZrULWLa/mY01s7ELFiwoRpgiAjBrFhx/POy6K/z0EwwZEhIPJRoiUtbSTTZuAsab2WOxViMbuKGohdx9nbt3AloB3cxsW0JtSAfCZZkmwEUFLDvI3bu6e9esrKw0wxSRX3+FG26Adu3guefg0kth2rSQeFQrzr8XIiKlJN1nozxlZqMJCQLARe4+N92NuPtiMxsF7O/u/47Fq8zsUeCC4gQsIvlzD3eVXHABzJwJhx8Ot90GW2yRdGQiUtWl9X+OmRmwD6Hdxgiglpl1K2KZLDNrHF/XBfYDpphZi5R1HgpMXI/4RYTQ02f37nDkkdCgQbiN9YUXlGiISPmQbqXqf4FdgKPj+DLgviKWaQGMMrOvgM8JbTZeBYaa2QRgAtAMuL7YUYsIAAsWwBlnwI47woQJoQfQceNC4iEiUl6kezfK39x9RzMbD+Duv5hZrcIWcPevgM75lOs0KLKe1qwJicXVV4e+M848E666Cpo0SToyEZG/SjfZWGNm1Ym3rppZFpCTsahEpEBvvBE64poyJTwk7c47oWPHpKMSESlYupdR7gFeAjYysxuADwl3qIhIGZk2DQ48EHr2DA9Pe+WVkHgo0RCR8i7du1GGmlk2oZGoAYe6++SMRiYiACxZAtddFx75XqdOuMPkrLOgdr491IiIlD/pdld+qrsPBqaklN3s7hdnLDKRKm7dOnj00dBPxsKFcMopof+M5s2TjkxEpHjSbbNxhJn95u5DAczsPkCPbhLJkA8+CF2Mjx8Pu+0Gr78OXbokHZWISMmknWwAI8wsB9gfWOzup2YuLJGq6Ycf4F//gmeegVat4Kmn4KijwCzpyERESq7QZMPMUm+k6wcMBz4CrjGzJu7+cyaDE6kqVq6EW2+FW24J41ddFZKOevWSjUtEpDQUVbORTbjd1VL+9oqDA+qfUGQ9uIdajAsvDA9LO+qokHRstlnSkYmIlJ5Ckw13b1NWgYhUNdnZoV3GRx9B584wbBjssUfSUYmIlD49A1KkjM2dC6eeCjvtFPrOeOgh+PxzJRoiUnml20BURNbT6tWhr4xrrw2PgT/vPLjiCmjUKOnIREQyq6gGojXdfU1ZBSNSGbnDa6+F5GL6dOjVC26/Hdq3TzoyEZGyUdRllE/MbLiZ/cPMWpdBPCKVyuTJoXvxgw6CatVg5Eh49VUlGiJStRSabLh7V+DcOHqXmX1uZneaWQ8zU2fJIgX45Rc491zYbjv49NPwsLQJE0LiISJS1RTZQNTdZ7r7A+5+KLAr8AqwL/CBmb2W6QBFKpK1a+H++6FtW7j3XujXL1w6OfdcqFkz6ehERJJRrAaisf3Ge3HAzFpmIiiRimjUqHAr64QJsOeecPfdsMMOSUclIpK89br11d1nFTbdzOqY2Wdm9qWZTTKza2J5GzMbY2YzzOwZM6u1PnGIJOm77+CII6B7d1i6FJ5/PiQeSjRERIJM97OxCuju7jsAnYD9zWxn4BbgTnffCvgF0HNWpMJZvhwuuwy23hreeAOuvz40CD3iCD3LREQkVUaTDQ+Wx9GacXCgO/B8LH8cODSTcYiUppwceOIJaNcObrwRevcOnXNddhnUrZt0dCIi5U9abTbMrB1wIbB56jLu3j2NZasTnrGyFXAf8A3hqbFr4yw/AX9p+2Fm/YH+AJvpQRFSTowZE9pljBkTegB94QXYZZekoxIRKd/SbSD6HPAA8BCwrjgbcPd1QCczawy8BHRIc7lBwCCArl27enG2KVLaZs+GSy6BIUNg443hscfg+OND3xkiIlK4dJONte5+//psyN0Xm9koYBegsZnViLUbrYBCG5qKJOW330IfGTfcAGvWwMUXw6WXQoMGSUcmIlJxFPp/mZk1MbMmwCtmNsDMWuSWxfJCmVlWrNHAzOoC+wGTgVHAkXG2E4GX1+tdiJQyd3jpJejYMSQX++0HX38NN92kRENEpLiKqtnIJjTozG1bf2HKNAe2KGL5FsDjsd1GNeBZd3/VzL4Gnjaz64HxwOBiRy6SIRMmhE643nsPttkG3n4b9t036ahERCquQpMNd2+zPit396+AzvmUfwt0W591i5S2hQvhyivhwQehcWO47z7o3x9q6NnIIiLrJa3mbWb2z9zLIXF8QzMbkLmwRMrOmjXh0e9t28KgQTBgQOhifMAAJRoiIqUh3bb0p7n74twRd/8FOC0zIYmUnbfeCj19nnMOdOkCX3wRnmnSpMgWSSIikq50k43qZn/0iRjbYKiLcamwZsyAQw6Bv/8dVq2C4cND24xtt006MhGRyifdZONN4Bkz28fM9gGeAt7IXFgimbF0KVx0UbjL5L334Oabw10mhxyiLsZFRDIl3SvS/yL05nlGHH8beDgjEYlkQE5O6DWuH9QAACAASURBVIjr0kth3jw4+eTQd0aLFklHJiJS+RWZbMRLJkPc/VhCL6IiFcpHH4U2GdnZoWvxV14JXY2LiEjZKPIySuxufHM9Bl4qmh9/hGOOgd13h7lzYejQkHgo0RARKVvpXkb5FvjIzEYAK3IL3f2OjEQlsh5WroR//zu0x3CHK64I7TQ22CDpyEREqqZ0k41v4lANUGfNUi65w3PPwYUXwg8/hEe/33ortG6ddGQiIlVbWsmGu18DYGb14/jyTAYlUlzjx4d2GR98EPrNGDIE9twz6ahERATS70F0WzMbD0wCJplZtpltk9nQRIo2f37oUrxLF5g8OXQ1np2tRENEpDxJt5+NQcB57r65u28OnA88lLmwRAq3ejXccUfoYvzRR8OD06ZPD4lH9epJRyciIqnSbbOxgbuPyh1x99FmpuZ2koiRI2HgQJg2DXr2DElHhw5JRyUiIgVJt2bjWzO7wsxax+Fywh0qImVmypSQXPTqFcZfey0kHko0RETKt3STjVOALOBF4AWgGXBypoISSfXbb3DBBbDddvDxx6EmY8IEOOCApCMTEZF0pHsZZV93Pzu1wMx6A88VtICZbQoMAZoDDgxy97vN7GrCE2MXxFkvdfeRxQ1cqobvvgu3sGZnQ79+oYvxjTZKOioRESmOdJONS/hrYpFfWaq1wPnuPs7MGgDZZvZ2nHanu/+7eKFKVfPaa3D88eG5Ji+/DAcfnHREIiJSEoUmG2bWEzgAaGlm96RMakhIJgrk7nOAOfH1MjObDLRcv3ClKli3Dq68Em68ETp1guefhy23TDoqEREpqaLabMwGxgK/Adkpwwjg7+luxMxaA52BMbHoTDP7ysweMbMNixmzVGLz5kGPHiHR6NcvtNFQoiEiUrEVWrPh7l8CX5rZL8Cr7p5T3A3EXkdfAM5196Vmdj9wHaEdx3XA7YQGqHmX6094rD2bbbZZcTcrFdCHH8JRR8HPP4e+M046KemIRESkNKR7N0ofYLqZ3Wpmad9oaGY1CYnGUHd/EcDd57n7upi4PAR0y29Zdx/k7l3dvWtWVla6m5QKyD3cYbLXXlCvHnz6qRINEZHKJK1kw92PI1wG+QZ4zMw+MbP+seFnvszMgMHA5NSnw5pZi5TZDgMmlihyqRSWLg13m5x/fmgAOnZseLaJiIhUHunWbODuS4HngaeBFoREYZyZnVXAIrsBxwPdzeyLOBwA3GpmE8zsK2BvYOB6vQOpsCZMgK5dYfjw8Ej4F16ARo2SjkpEREpbWre+mtnBhE68tiL0ndHN3eebWT3ga+DevMu4+4eA5bM69akhPP44nHEGNG4Mo0bBHnskHZGIiGRKuv1sHEHoG+P91EJ3X2lmp5Z+WFJZ/fYbnHUWPPww7L03DBsGG2+cdFQiIpJJaSUb7n6imW0cazgc+Nzd58Zp72YyQKk8vv0WjjwSxo+HSy6Ba6+FGummuyIiUmGl1WYj1l58BhwOHAl8amZ/uV1VpCCvvAJduoTux195JfSjoURDRKRqSPd0/y+gs7svAjCzpsDHwCOZCkwqh7Vr4Yor4OabYccdQ2+gbdokHZWIiJSldJONRcCylPFlsUykQHPnwtFHw+jR0L8/3H031KmTdFQiIlLW0k02ZgBjzOxlQpuNQ4CvzOw8gNR+NEQAPvgg9Aa6eHG48+SEE5KOSEREkpJusvFNHHK9HP8W2KmXVE3ucPvtcPHFsMUW8OabsN12SUclIiJJSvdulGsyHYhUfEuWhG7Ghw+HI46ARx6Bhg2TjkpERJKWdg+iIoX58stwt8mrr4bnnDz3nBINEREJlGzIenv0Udh5Z/j119AYdOBAsPz6jhURkSpJyYaU2K+/Qr9+cMopsOuuobOu3XZLOioRESlv0u3Uq52ZvWtmE+P49mZ2eWZDk/Lsm29CgjF4MFx2Gbz1Fmy0UdJRiYhIeZRuzcZDwCXAGgB3/wrom6mgpHx7+eXQPuP770Mbjeuvh+rVk45KRETKq3STjXru/lmesrWlHYyUb2vXwr/+BYceClttBePGQa9eSUclIiLlXbr9bCw0sy0JHXphZkcCczIWlZQ7c+ZA377w/vvwj3/AnXeqN1AREUlPusnGP4FBQAczmwV8BxyXsaikXPnf/0JvoMuWwRNPwHH65EVEpBjSuozi7t+6+75AFtDB3Xd395mFLWNmm5rZKDP72swmmdk5sbyJmb1tZtPj3w3X+11IRrjDLbdA9+7QuDGMGaNEQ0REii+tmg0zawycALQGaljsRMHdzy5ksbXA+e4+zswaANlm9jZwEvCuu99sZhcDFwMXlfgdSEYsXgwnnggjRkDv3uGukwbqnF5EREog3csoI4FPgQlATjoLuPscYrsOd19mZpOBloSHuO0VZ3scGI2SjXJl/Hg48kj44Qe46y44+2x10iUiIiWXbrJRx93PK+lGzKw10BkYAzSPiQjAXKB5Acv0B/oDbLbZZiXdtBTT4MHwz39Cs2ahMeguuyQdkYiIVHTp3vr6hJmdZmYtYpuLJmbWJJ0Fzaw+8AJwrrsvTZ3m7k68wyUvdx/k7l3dvWtWVlaaYUpJrVwZegLt1w/22CPUbijREBGR0pBusrEauA34BMiOw9iiFjKzmoREY6i7vxiL55lZizi9BTC/uEFL6ZoxIyQWjz4KV1wBb7wByu9ERKS0pHsZ5XxgK3dfmO6KLbQiHQxMdvc7UiaNAE4Ebo5/X053nVL6XnopPBa+Rg0YORJ69kw6IhERqWzSrdmYAaws5rp3A44HupvZF3E4gJBk7Gdm04F947iUsTVr4MIL4fDDoX370BuoEg0REcmEdGs2VgBfmNkoYFVuYWG3vrr7h0BB9zDsk3aEUupmzw69gX7wAQwYAHfcAbVrJx2ViIhUVukmG8PjIBXcqFFw9NGhN9ChQ+GYY5KOSEREKru0kg13fzzTgUhm5eSE3kAvvxzatoV334Vttkk6KhERqQoKTTbM7Fl372NmE8jnFlV33z5jkUmp+eUXOOGE8Dj4o46Chx5Sb6AiIlJ2iqrZOCf+PTDTgUhmjBsXegP96Se45x4480z1BioiImWr0LtRUnr6HODu36cOwIDMhycl5Q6DBsGuu8LataE30LPOUqIhIiJlL91bX/fLp0w3SpZTK1eGvjNOPx323DPUbuy8c9JRiYhIVVVUm40zCDUYW5jZVymTGgAfZTIwKZlp08Jlk4kT4aqrQo+g1asnHZWIiFRlRbXZGAa8DtxEeBR8rmXu/nPGopISeeEFOPlkqFULXn8d/v73pCMSEREpItlw9yXAEuDosglHSmLNGrjoIrjzTvjb3+DZZ0EPyhURkfIi3U69pJyaNSvczvrRR+FOk9tvDzUbIiIi5YWSjQrsvfdCb6ArVsBTT4UuyEVERMqbdO9GkXIkJwduvBH22w+aNoXPP1eiISIi5ZdqNiqYn38OvYG+9lqo1Rg0COrXTzoqERGRginZqEDGjoXevUM7jf/8JzyxVZ10iYhIeafLKBWAOzzwAOy2G6xbFx4N/89/KtEQEZGKQclGObdiRbhscsYZ0L07jB8fbm8VERGpKDKabJjZI2Y238wmppRdbWazzOyLOByQyRgqsqlTQ2IxdChcc01op9G0adJRiYiIFE+mazYeA/bPp/xOd+8Uh5EZjqFCeu456NoV5s6FN96AK6+EaqqHEhGRCiijP1/u/j6gbs2LYfVqOPdc6NMHtt02XDbp0SPpqEREREouqf+VzzSzr+Jllg3zm8HM+pvZWDMbu2DBgrKOLxE//QR77QV33w3nnAP/+x9sumnSUYmIiKyfJJKN+4EtgU7AHOD2/GZy90Hu3tXdu2ZlZZVlfIl45x3o3BkmTIBnnoG77lK34yIiUjmUebLh7vPcfZ275wAPAd3KOobyJCcHrrsuXCpp3jz0BtqnT9JRiYiIlJ4y79TLzFq4+5w4ehgwsbD5K7NFi+D448Pj4I89Fh58EDbYIOmoRERESldGkw0zewrYC2hmZj8BVwF7mVknwIGZwOmZjKG8+vxzOPLIcLfJ/ffD6aerky4REamcMppsuPvR+RQPzuQ2y7vc3kDPPRdatIAPP4Sddko6KhGRzMnOzt6oRo0aDwPbos4kK6scYOLatWv7denSZX7eiXo2ShlasSLUYAwdCj17whNPqJMuEan8atSo8fDGG2+8dVZW1i/VqlXzpOOR0peTk2MLFizoOHfu3IeBg/NOV4ZZRqZMgW7dYNiw0CD01VeVaIhIlbFtVlbWUiUalVe1atU8KytrCaH26i9Us1EGnnkGTj0V6tWDt96CffdNOiIRkTJVTYlG5Rc/43wrMVSzkUGrV8PZZ0PfvrDDDjBunBINERGpepRsZMiPP8L//R/ce29oDDp6NLRqlXRUIiKSjosvvnjj0lrXeeedt8mVV17ZvKDpxxxzzGZvvfXWBt26dWv//vvv1yut7Zam+++/v0m7du06tmvXrmPnzp07fPLJJ3WLs7ySjQx4663QG+ikSfDss3DnnVCzZtJRiYhIuu65554WxZk/JyeHdevWlWhb48aNq9+9e/cVJVq4jGy11VarPvroo6nTpk37+pJLLpl9+umnb16c5ZVslKKcnPAo+P33h403hrFjoXfvpKMSEanapk6dWqtNmzbbHHzwwW222GKLbfbff/8tli1bVm3EiBEN9t133y1z53vppZca7rffflsOGDCg5apVq6p16NCh48EHH9wG4Oqrr27etm3bbdq2bbvNtddeu1Huelu3br3tYYcd1rpdu3bbfPPNN7Wef/75hh07dty6ffv2HXfZZZd2ueuePHly3W7durVv1arVdtdff/1GueXjxo2rs8UWW/xWo8YfTSjXrVvHEUcc0frss8/eBODFF19s2KlTpw4dO3bcumfPnlssWbKkGsDLL7/cYOutt+7Yrl27jr17927966+/GkDLli23+8c//tGqXbt2HbfbbrutJ06cWBvgkUce2bBt27bbtG/fvmPXrl3bF2cf7rfffiuysrLWAey9994r5s6dW6wHaqiBaClZuBCOOw7efDP0Cnr//eoNVEQkr1NOYdOJEynVSwXbbsvKRx7hx8LmmTlzZp0HH3xwZo8ePVb07t279W233ZZ19dVXzzvnnHM2mz17do1NNtlk7SOPPNL05JNPXnjMMccseeyxxzaaMmXK1wAffPBBvWHDhjXNzs6e7O506dJl63322WdZs2bN1v3www+1Bw8e/N0+++wzc/bs2TXOPPPM1qNHj57SoUOH1fPmzaueu/0ZM2bU+fjjj6cuXry4+tZbb73thRdeuKB27do+YsSIRj169FiSO9+aNWvs0EMPbdOxY8dfb7nllrlz5sypceONN7Z4//33pzVs2DDnsssu2/i6665rfu211849/fTT27z11ltTt99++1WHHXZY69tuuy3ryiuvnA/QqFGjtdOmTfv6P//5T9Ozzjpr01GjRs24+eabW7z11lvT2rRps2bhwoXV/7qX0nPvvfc223vvvZcUPecfVLNRCsaMgR13hFGjQoddjz+uRENEpDzZeOONV/fo0WMFwPHHH7/o448/rl+tWjX69Omz6KGHHmqycOHC6uPGjavfu3fvv/yIjh49uv4BBxywuGHDhjmNGjXK6dWr1y+jRo1qANCiRYvV++yzz4o43wbdunVb1qFDh9UAzZs3//26So8ePRbXrVvXW7RosbZJkyZrfvrppxoA77zzTsNDDz10ae58AwYM2Dw30chd5zfffFOnW7duHTp06NDx6aefbvrDDz/U+vLLL+u0atVq1fbbb78K4KSTTlr04YcfNshdz4knnvgzwGmnnfbz+PHj6wN07dp1+bHHHtv69ttvb7Z27doS7cdXXnmlwZNPPtns7rvv/qk4y6lmYz24w333wXnnQcuW8PHH0KVL0lGJiJRfRdVAZIrleR5E7vgZZ5yxqFevXlvVqVPHDzrooF9qFrOBXb169XLSma927dq/3/pbvXp11q5da8uWLau2dOnS6q1bt16TO61r167LP/jgg4YrV66cV69ePXd3dt9996WvvPLKd6nrK6qBZrVqf9QlmJkDDBs27If33ntvgxEjRjTq0qVLx+zs7K833njj3xOis846q+Xbb7/dCCC3VifVmDFj6g4YMGDz1157bXrqculQzUYJLV8OxxwDZ50Vntiana1EQ0SkvJozZ06td955ZwOAoUOHNtl1112XA7Ru3XpN8+bN19x+++0t+vfvvzB3/ho1aviqVasMYO+9914+cuTIxjE5qDZy5MgN995772V5t7HXXnut+OyzzxpMmTKlFkDqZZT8vPbaaw123333P63n9NNPX9ijR48lBx544JZr1qxhr732WjF27Nj6ue0uli5dWu2rr76qvcMOO/w2a9asWrnlQ4YMabrHHnv8vq4hQ4Y0ARg8ePCGnTt3XgEwadKk2t27d19x1113zd5www3Xfvvtt39qd3HvvffOmjJlytf5JRrTp0+v1bt37y0feeSR73JrU4pDNRslMHkyHHEETJ0KN94IF10E1ZS2iYiUW61bt/7t3nvv3ah///712rZt+9sFF1ywIHda3759F9133301dtxxx99yy4499tgFW2+9dcdtt9125YgRI7475phjFu24445bAxx//PELdtttt1+nTp36px/rTTbZZO0999wz87DDDtsqJyeHpk2brvn444+nFxTTyJEjG/Xp0+eXvOVXX331vIEDB1Y//PDD2wwfPvy7Bx98cGbfvn23WL16tQFcddVVs7bffvtVDzzwwMzevXtvuW7dOnbYYYeVqe/pl19+qd6uXbuOtWrV8qeffvpbgIEDB7aaOXNmbXe33XfffenOO+/8a7r77/LLL2+xePHiGmedddbmEJKxiRMnTk53eXMv/526de3a1ceOHZt0GAA89RScdlpok/HUU9C9e9IRiYjkz8yy3b1r0nF8+eWXM3fYYYeFRc+ZGVOnTq114IEHtp0+ffqk/KafcMIJm3Xu3HnlwIEDyzTGjh07bj1+/PgpqZdYSkPLli23Gzt27OQWLVqUrGHGevjyyy+b7bDDDq3zlqtmI02rVsH554c2GrvtFrogb9ky6ahERGR9bLPNNlvXrVs358EHHyzztiRff/112jUDFZ2SjTR8/z306QOffRYag958szrpEhGpKNq3b7+6oFqNSZMmVbof/FmzZk1IOoa8lGwU4Y034NhjYc0aeOEFOPzwpCMSEalwcnJyckwPY6vccnJyDMj37pyMNms0s0fMbL6ZTUwpa2Jmb5vZ9Ph3w0zGUFLr1sFVV8EBB4TLJdnZSjREREpo4oIFCxrFHyOphHJycmzBggWNgIn5Tc90zcZjwH+AISllFwPvuvvNZnZxHL8ow3EUy4IFoTbj7bfhxBPhv/8Nj4cXEZHiW7t2bb+5c+c+PHfu3G1RlwuVVQ4wce3atf3ym5jRZMPd3zez1nmKDwH2iq8fB0ZTjpKNTz4J7TMWLIBBg6BfPzDl4iIiJdalS5f5wMFJxyHJSSLDbO7uc+LruUC+j901s/5mNtbMxi5YsCC/WUqVO9xzT3gsfM2aoTfQ005ToiEiIrK+Eq3O8tDJR74Nhtx9kLt3dfeuWVlZGY1j2TLo2xfOOQd69gztM3bcMaObFBERqTKSSDbmmVkLgPh3fgIx/G7SJNhpJ3j+ebjpJhg+HDYsl01WRUREKqYkko0RwInx9YnAywnEAMDQodCtG/zyC7zzDlx8sbodFxERKW2ZvvX1KeAToL2Z/WRmpwI3A/uZ2XRg3zheplatggED4LjjwsPTxo+Hvfcu6yhERESqhkzfjXJ0AZP2yeR2C/P999C7N3z+OVx4YXiQWg11bSYiIpIxVepnduTIUJuxbh289BIcemjSEYmIiFR+VaKFwrp1cMUV0KsXbLZZuNtEiYaIiEjZqPQ1G/PnwzHHwLvvwsknh6e21q2bdFQiIiJVR6VONiZMCP1mLFoEgwfDKackHZGIiEjVU6kvo2y6KWyzTeiCXImGiIhIMip1zUbjxvDmm0lHISIiUrVV6poNERERSZ6SDREREckoJRsiIiKSUUo2REREJKOUbIiIiEhGKdkQERGRjFKyISIiIhmlZENEREQyytw96RiKZGYLgO/XYxXNgIWlFE5pUlzFo7iKR3EVT2WMa3N3zyrNYERKokIkG+vLzMa6e9ek48hLcRWP4ioexVU8ikskc3QZRURERDJKyYaIiIhkVFVJNgYlHUABFFfxKK7iUVzFo7hEMqRKtNkQERGR5FSVmg0RERFJiJINERERyagKmWyY2aZmNsrMvjazSWZ2TixvYmZvm9n0+HfDWG5mdo+ZzTCzr8xsx5R1nRjnn25mJ65nXHXM7DMz+zLGdU0sb2NmY+L2nzGzWrG8dhyfEae3TlnXJbF8qpn9fX3iSllndTMbb2avlpe4zGymmU0wsy/MbGwsS/RzjOtrbGbPm9kUM5tsZrskHZeZtY/7KXdYambnJh1XXN/AeMxPNLOn4nehPBxf58SYJpnZubGszPeXmT1iZvPNbGJKWanFYWZd4vdoRlzWSrbHRDLE3SvcALQAdoyvGwDTgI7ArcDFsfxi4Jb4+gDgdcCAnYExsbwJ8G38u2F8veF6xGVA/fi6JjAmbu9ZoG8sfwA4I74eADwQX/cFnomvOwJfArWBNsA3QPVS2G/nAcOAV+N44nEBM4FmecoS/RzjOh8H+sXXtYDG5SGulPiqA3OBzZOOC2gJfAfUTTmuTkr6+AK2BSYC9YAawDvAVknsL+D/gB2BiZk4zoHP4rwWl+1ZGseZBg2lNSQeQKm8CXgZ2A+YCrSIZS2AqfH1g8DRKfNPjdOPBh5MKf/TfOsZUz1gHPA3Qu9/NWL5LsCb8fWbwC7xdY04nwGXAJekrOv3+dYjnlbAu0B34NW4nfIQ10z+mmwk+jkCjQg/nlae4soTSw/go/IQFyHZ+JHwI1gjHl9/T/r4AnoDg1PGrwD+ldT+Alrz52SjVOKI06aklP9pPg0aysNQIS+jpIpVsJ0JtQjN3X1OnDQXaB5f554Mc/0UywoqX594qpvZF8B84G3Cf2eL3X1tPtv4fftx+hKgaSbiAu4inGhz4njTchKXA2+ZWbaZ9Y9lSX+ObYAFwKMWLjs9bGYblIO4UvUFnoqvE43L3WcB/wZ+AOYQjpdskj++JgJ7mFlTM6tHqDHYlPLzOZZWHC3j69KOT6TUVOhkw8zqAy8A57r70tRp7u6EH7Iy5e7r3L0ToSahG9ChrGPIy8wOBOa7e3bSseRjd3ffEegJ/NPM/i91YkKfYw1Clff97t4ZWEGo5k46LgBi24eDgefyTksirtjW4BBCkrYJsAGwf1nGkB93nwzcArwFvAF8AazLM09in2N5jEMkUypssmFmNQmJxlB3fzEWzzOzFnF6C0LtAsAswn80uVrFsoLK15u7LwZGEaqPG5tZjXy28fv24/RGwKIMxLUbcLCZzQSeJlxKubscxJX7XzHuPh94iZCgJf05/gT85O5j4vjzhOQj6bhy9QTGufu8OJ50XPsC37n7AndfA7xIOObKw/E12N27uPv/Ab8Q2nclvb9ylVYcs+Lr0o5PpNRUyGQjtrQeDEx29ztSJo0Aclton0hoy5FbfkJs5b0zsCRWX74J9DCzDeN/Zz1iWUnjyjKzxvF1XUI7ksmEpOPIAuLKjfdI4L34H84IoG9std8GaEtoAFYi7n6Ju7dy99aE6vf33P3YpOMysw3MrEHua8L+n0jCn6O7zwV+NLP2sWgf4Ouk40pxNH9cQsndfpJx/QDsbGb14nczd38lenwBmNlG8e9mwOGEBtJJ769cpRJHnLbUzHaO+/+ElHWJlA9JNxopyQDsTqhy/IpQNfoF4XpsU0IjyOmEludN4vwG3EdoPzEB6JqyrlOAGXE4eT3j2h4YH+OaCFwZy7cgnDRnEKq+a8fyOnF8Rpy+Rcq6LovxTqUUW5YDe/HH3SiJxhW3/2UcJgGXxfJEP8e4vk7A2PhZDie0/i8PcW1AqAVolFJWHuK6BpgSj/snCHeUJH7cAx8QEp8vgX2S2l+E5HAOsIZQc3ZqacYBdI37/hvgP+Rp3KxBQ9KDuisXERGRjKqQl1FERESk4lCyISIiIhmlZENEREQySsmGiIiIZJSSDREREckoJRtSqZnZOgtPSP3SzMaZ2a5FzN/YzAaksd7RZta1hDGNzO2PRUSkKlCyIZXdr+7eyd13IDzo66Yi5m9MeCppxrj7AR56mBURqRKUbEhV0pDQZTVmVt/M3o21HRPM7JA4z83AlrE25LY470Vxni/N7OaU9fU2s8/MbJqZ7ZF3Y2bWwszej+uamDuPmc00s2Zm9o847Qsz+87MRsXpPczskxjbcxaeASQiUmGpUy+p1MxsHaEXxjqER3F3d/fs+EyOeu6+1MyaAZ8SusfenNDD6rZx+Z6ER5Pv6+4rzayJu/9sZqOBbHc/38wOAM5z933zbPt8oI6732Bm1eP2lll4Rk1Xd18Y56sJvAfcCnxCeLZIT3dfYWYXEXrevDaT+0lEJJNqFD2LSIX2q4en8GJmuwBDzGxbQpfQN1p4ymwO4ZHczfNZfl/gUXdfCeDuP6dMy30AYDbQOp9lPwceicnEcHf/ooAY7yY8H+QVC0/o7Qh8FB5zQS1CAiIiUmEp2ZAqw90/ibUYWYRn6WQBXdx9TaxtqFPMVa6Kf9eRz3fJ3d+PyUwv4DEzu8Pdh6TOY2YnEWpTzswtAt5296OLGYuISLmlNhtSZZhZB6A68UFmwPyYaOxN+MEHWAY0SFnsbeBkM6sX19GkGNvbHJjn7g8BDxMeU586vQtwAXCcu+fE4k+B3cxsqzjPBmbWrnjvVESkfFHNhlR2dc0s9/KFASe6+zozGwq8YmYTCE93nQLg7ovM7CMzmwi87u4XmlknYKyZrQZGApemvNaUIQAAAGtJREFUue29gAvNbA2wnPDo71RnAk2AUfGSyVh37xdrO54ys9pxvsuBacV+5yIi5YQaiIqIiEhG6TKKiIiIZJSSDREREckoJRsiIiKSUUo2REREJKOUbIiIiEhGKdkQERGRjFKyISIiIhn1/2p/esEbfqUvAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_relative_time(results, [2], max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "f7dc206c", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "As illustrated in the experiments, KeOps allows you to drastically speed up and scale up drift detection to larger datasets without running into memory issues. The speed benefit of KeOps over the PyTorch (or TensorFlow) MMD detector decreases as the number of features increases. Note though that it is not advised to apply the (untrained) MMD detector to very high-dimensional data in the first place." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:detect]", + "language": "python", + "name": "conda-env-detect-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 244ceb25573ddfd9abaa0eb3eff9126b922f14b2 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 8 Jul 2022 14:54:30 +0100 Subject: [PATCH 13/50] update test mmd --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 0 alibi_detect/cd/tests/test_mmd.py | 5 ++++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 alibi_detect/cd/keops/tests/test_mmd_keops.py diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py new file mode 100644 index 000000000..e69de29bb diff --git a/alibi_detect/cd/tests/test_mmd.py b/alibi_detect/cd/tests/test_mmd.py index 33e776e14..b2fa8cbb5 100644 --- a/alibi_detect/cd/tests/test_mmd.py +++ b/alibi_detect/cd/tests/test_mmd.py @@ -1,12 +1,13 @@ import numpy as np import pytest from alibi_detect.cd import MMDDrift +from alibi_detect.cd.keops.mmd import MMDDriftKeops from alibi_detect.cd.pytorch.mmd import MMDDriftTorch from alibi_detect.cd.tensorflow.mmd import MMDDriftTF n, n_features = 100, 5 -tests_mmddrift = ['tensorflow', 'pytorch', 'PyToRcH', 'mxnet'] +tests_mmddrift = ['tensorflow', 'pytorch', 'keops', 'PyToRcH', 'mxnet'] n_tests = len(tests_mmddrift) @@ -29,5 +30,7 @@ def test_mmddrift(mmddrift_params): assert isinstance(cd._detector, MMDDriftTorch) elif backend.lower() == 'tensorflow': assert isinstance(cd._detector, MMDDriftTF) + elif backend.lower() == 'keops': + assert isinstance(cd._detector, MMDDriftKeops) else: assert cd is None From f913f5b9c40c92b56e43c96bf7c4eb9858fbe9fb Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 8 Jul 2022 15:27:17 +0100 Subject: [PATCH 14/50] add test mmd keops --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 103 ++++++++++++++++++ .../utils/keops/tests/test_kernels_keops.py | 0 2 files changed, 103 insertions(+) create mode 100644 alibi_detect/utils/keops/tests/test_kernels_keops.py diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index e69de29bb..6e04e7ebc 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -0,0 +1,103 @@ +from functools import partial +from itertools import product +import numpy as np +import pytest +import torch +import torch.nn as nn +from typing import Callable, List +from alibi_detect.cd.keops.mmd import MMDDriftKeops +from alibi_detect.cd.pytorch.preprocess import HiddenOutput, preprocess_drift + +n, n_hidden, n_classes = 500, 10, 5 + + +class MyModel(nn.Module): + def __init__(self, n_features: int): + super().__init__() + self.dense1 = nn.Linear(n_features, 20) + self.dense2 = nn.Linear(20, 2) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = nn.ReLU()(self.dense1(x)) + return self.dense2(x) + + +# test List[Any] inputs to the detector +def preprocess_list(x: List[np.ndarray]) -> np.ndarray: + return np.concatenate(x, axis=0) + + +n_features = [10] +n_enc = [None, 3] +preprocess = [ + (None, None), + (preprocess_drift, {'model': HiddenOutput, 'layer': -1}), + (preprocess_list, None) +] +update_x_ref = [{'last': 750}, {'reservoir_sampling': 750}, None] +preprocess_x_ref = [True, False] +n_permutations = [10] +batch_size_permutations = [10, 1000000] +configure_kernel_from_x_ref = [True, False] +tests_mmddrift = list(product(n_features, n_enc, preprocess, n_permutations, update_x_ref, preprocess_x_ref, + batch_size_permutations, configure_kernel_from_x_ref)) +n_tests = len(tests_mmddrift) + + +@pytest.fixture +def mmd_params(request): + return tests_mmddrift[request.param] + + +@pytest.mark.parametrize('mmd_params', list(range(n_tests)), indirect=True) +def test_mmd(mmd_params): + n_features, n_enc, preprocess, n_permutations, update_x_ref, preprocess_x_ref, \ + batch_size_permutations, configure_kernel_from_x_ref = mmd_params + + np.random.seed(0) + torch.manual_seed(0) + + x_ref = np.random.randn(n * n_features).reshape(n, n_features).astype(np.float32) + preprocess_fn, preprocess_kwargs = preprocess + to_list = False + if hasattr(preprocess_fn, '__name__') and preprocess_fn.__name__ == 'preprocess_list': + if not preprocess_x_ref: + return + to_list = True + x_ref = [_[None, :] for _ in x_ref] + elif isinstance(preprocess_fn, Callable) and 'layer' in list(preprocess_kwargs.keys()) \ + and preprocess_kwargs['model'].__name__ == 'HiddenOutput': + model = MyModel(n_features) + layer = preprocess_kwargs['layer'] + preprocess_fn = partial(preprocess_fn, model=HiddenOutput(model=model, layer=layer)) + else: + preprocess_fn = None + + cd = MMDDriftKeops( + x_ref=x_ref, + p_val=.05, + preprocess_x_ref=preprocess_x_ref if isinstance(preprocess_fn, Callable) else False, + update_x_ref=update_x_ref, + preprocess_fn=preprocess_fn, + configure_kernel_from_x_ref=configure_kernel_from_x_ref, + n_permutations=n_permutations, + batch_size_permutations=batch_size_permutations + ) + x = x_ref.copy() + preds = cd.predict(x, return_p_val=True) + assert preds['data']['is_drift'] == 0 and preds['data']['p_val'] >= cd.p_val + if isinstance(update_x_ref, dict): + k = list(update_x_ref.keys())[0] + assert cd.n == len(x) + len(x_ref) + assert cd.x_ref.shape[0] == min(update_x_ref[k], len(x) + len(x_ref)) + + x_h1 = np.random.randn(n * n_features).reshape(n, n_features).astype(np.float32) + if to_list: + x_h1 = [_[None, :] for _ in x_h1] + preds = cd.predict(x_h1, return_p_val=True) + if preds['data']['is_drift'] == 1: + assert preds['data']['p_val'] < preds['data']['threshold'] == cd.p_val + assert preds['data']['distance'] > preds['data']['distance_threshold'] + else: + assert preds['data']['p_val'] >= preds['data']['threshold'] == cd.p_val + assert preds['data']['distance'] <= preds['data']['distance_threshold'] diff --git a/alibi_detect/utils/keops/tests/test_kernels_keops.py b/alibi_detect/utils/keops/tests/test_kernels_keops.py new file mode 100644 index 000000000..e69de29bb From 2da4a9a36072f8ab9d8a7cfff705aefb108541ec Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 8 Jul 2022 15:31:59 +0100 Subject: [PATCH 15/50] update readme --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 72fbfdb0f..97edbbf3f 100644 --- a/README.md +++ b/README.md @@ -181,8 +181,9 @@ The following tables show the advised use cases for each algorithm. The column * #### TensorFlow and PyTorch support -The drift detectors support TensorFlow and PyTorch backends. Alibi Detect does however not install PyTorch for you. -Check the [PyTorch docs](https://pytorch.org/) how to do this. Example: +The drift detectors support TensorFlow, PyTorch and (where applicable) [KeOps](https://www.kernel-operations.io/keops/index.html) backends. +Alibi Detect does however not install PyTorch or KeOps for you. +Check the [PyTorch docs](https://pytorch.org/) how to do this. KeOps can be installed via pip: ```pip install pykeops```. Example: ```python from alibi_detect.cd import MMDDrift @@ -198,6 +199,13 @@ cd = MMDDrift(x_ref, backend='pytorch', p_val=.05) preds = cd.predict(x) ``` +Or in KeOps: + +```python +cd = MMDDrift(x_ref, backend='keops', p_val=.05) +preds = cd.predict(x) +``` + #### Built-in preprocessing steps Alibi Detect also comes with various preprocessing steps such as randomly initialized encoders, pretrained text From c49f1d2879d3191687652473cb6e0333df8ace67 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 8 Jul 2022 18:30:27 +0100 Subject: [PATCH 16/50] bugfix kernel and update mmd test --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 2 ++ alibi_detect/utils/keops/kernels.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index 6e04e7ebc..3843d3428 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -54,6 +54,8 @@ def test_mmd(mmd_params): n_features, n_enc, preprocess, n_permutations, update_x_ref, preprocess_x_ref, \ batch_size_permutations, configure_kernel_from_x_ref = mmd_params + print(configure_kernel_from_x_ref, batch_size_permutations, n_features, update_x_ref, preprocess_x_ref) + np.random.seed(0) torch.manual_seed(0) diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index 2f471841a..d0f826772 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -23,8 +23,8 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor) -> torch.Tensor: The computed bandwidth, `sigma`. """ n = x.shape[0] - if (dist.min(axis=1) == 0.).all() and (torch.arange(n) == dist.argmin(axis=1).cpu().view(-1)).all() \ - and x.shape == y.shape: + if x.shape == y.shape and (dist.min(axis=1) == 0.).all() and \ + (torch.arange(n) == dist.argmin(axis=1).cpu().view(-1)).all(): n_mean = n * (n - 1) else: n_mean = np.prod(dist.shape) From 75481cc3e8730d0941f1263c5299fbaa2d8e42d2 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 8 Jul 2022 18:31:06 +0100 Subject: [PATCH 17/50] remove print from test --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index 3843d3428..6e04e7ebc 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -54,8 +54,6 @@ def test_mmd(mmd_params): n_features, n_enc, preprocess, n_permutations, update_x_ref, preprocess_x_ref, \ batch_size_permutations, configure_kernel_from_x_ref = mmd_params - print(configure_kernel_from_x_ref, batch_size_permutations, n_features, update_x_ref, preprocess_x_ref) - np.random.seed(0) torch.manual_seed(0) From e6996b937a308b913d4d1f5415c30c771db0e896 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Fri, 8 Jul 2022 19:25:42 +0100 Subject: [PATCH 18/50] update keops tests --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 10 +---- .../utils/keops/tests/test_kernels_keops.py | 42 +++++++++++++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index 6e04e7ebc..6fc454d19 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -34,12 +34,11 @@ def preprocess_list(x: List[np.ndarray]) -> np.ndarray: (preprocess_drift, {'model': HiddenOutput, 'layer': -1}), (preprocess_list, None) ] -update_x_ref = [{'last': 750}, {'reservoir_sampling': 750}, None] preprocess_x_ref = [True, False] n_permutations = [10] batch_size_permutations = [10, 1000000] configure_kernel_from_x_ref = [True, False] -tests_mmddrift = list(product(n_features, n_enc, preprocess, n_permutations, update_x_ref, preprocess_x_ref, +tests_mmddrift = list(product(n_features, n_enc, preprocess, n_permutations, preprocess_x_ref, batch_size_permutations, configure_kernel_from_x_ref)) n_tests = len(tests_mmddrift) @@ -51,7 +50,7 @@ def mmd_params(request): @pytest.mark.parametrize('mmd_params', list(range(n_tests)), indirect=True) def test_mmd(mmd_params): - n_features, n_enc, preprocess, n_permutations, update_x_ref, preprocess_x_ref, \ + n_features, n_enc, preprocess, n_permutations, preprocess_x_ref, \ batch_size_permutations, configure_kernel_from_x_ref = mmd_params np.random.seed(0) @@ -77,7 +76,6 @@ def test_mmd(mmd_params): x_ref=x_ref, p_val=.05, preprocess_x_ref=preprocess_x_ref if isinstance(preprocess_fn, Callable) else False, - update_x_ref=update_x_ref, preprocess_fn=preprocess_fn, configure_kernel_from_x_ref=configure_kernel_from_x_ref, n_permutations=n_permutations, @@ -86,10 +84,6 @@ def test_mmd(mmd_params): x = x_ref.copy() preds = cd.predict(x, return_p_val=True) assert preds['data']['is_drift'] == 0 and preds['data']['p_val'] >= cd.p_val - if isinstance(update_x_ref, dict): - k = list(update_x_ref.keys())[0] - assert cd.n == len(x) + len(x_ref) - assert cd.x_ref.shape[0] == min(update_x_ref[k], len(x) + len(x_ref)) x_h1 = np.random.randn(n * n_features).reshape(n, n_features).astype(np.float32) if to_list: diff --git a/alibi_detect/utils/keops/tests/test_kernels_keops.py b/alibi_detect/utils/keops/tests/test_kernels_keops.py index e69de29bb..d42a7c0e1 100644 --- a/alibi_detect/utils/keops/tests/test_kernels_keops.py +++ b/alibi_detect/utils/keops/tests/test_kernels_keops.py @@ -0,0 +1,42 @@ +from itertools import product +import numpy as np +from pykeops.torch import LazyTensor +import pytest +import torch +from alibi_detect.utils.keops import GaussianRBF + +sigma = [None, np.array([1.]), np.array([1., 2.])] +n_features = [5, 10] +n_instances = [(100, 100), (100, 75)] +trainable = [True, False] +tests_gk = list(product(sigma, n_features, n_instances, trainable)) +n_tests_gk = len(tests_gk) + + +@pytest.fixture +def gaussian_kernel_params(request): + return tests_gk[request.param] + + +@pytest.mark.parametrize('gaussian_kernel_params', list(range(n_tests_gk)), indirect=True) +def test_gaussian_kernel(gaussian_kernel_params): + sigma, n_features, n_instances, trainable = gaussian_kernel_params + xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features) + + print(sigma, xshape, yshape, trainable) + + sigma = sigma if sigma is None else torch.from_numpy(sigma).float() + x = torch.from_numpy(np.random.random(xshape)).float() + y = torch.from_numpy(np.random.random(yshape)).float() + + kernel = GaussianRBF(sigma=sigma, trainable=trainable) + infer_sigma = True if sigma is None else False + if trainable and infer_sigma: + with pytest.raises(Exception): + kernel(LazyTensor(x[:, None, :]), LazyTensor(y[None, :, :]), infer_sigma=infer_sigma) + else: + k_xy = kernel(LazyTensor(x[:, None, :]), LazyTensor(y[None, :, :]), infer_sigma=infer_sigma) + k_xx = kernel(LazyTensor(x[:, None, :]), LazyTensor(x[None, :, :]), infer_sigma=infer_sigma) + assert k_xy.shape == n_instances and k_xx.shape == (xshape[0], xshape[0]) + assert (torch.arange(xshape[0]) == k_xx.argmax(axis=1).cpu().view(-1)).all() + assert (k_xx.min(axis=1) >= 0.).all() and (k_xy.min(axis=1) >= 0.).all() From c87109f4d12a322d891da92a4221058694415189 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Tue, 26 Jul 2022 16:36:42 +0100 Subject: [PATCH 19/50] Add save warning and update tests --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 10 +++++----- alibi_detect/saving/saving.py | 4 ++-- alibi_detect/saving/schemas.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index 6fc454d19..0ff6d9071 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -34,11 +34,11 @@ def preprocess_list(x: List[np.ndarray]) -> np.ndarray: (preprocess_drift, {'model': HiddenOutput, 'layer': -1}), (preprocess_list, None) ] -preprocess_x_ref = [True, False] +preprocess_at_init = [True, False] n_permutations = [10] batch_size_permutations = [10, 1000000] configure_kernel_from_x_ref = [True, False] -tests_mmddrift = list(product(n_features, n_enc, preprocess, n_permutations, preprocess_x_ref, +tests_mmddrift = list(product(n_features, n_enc, preprocess, n_permutations, preprocess_at_init, batch_size_permutations, configure_kernel_from_x_ref)) n_tests = len(tests_mmddrift) @@ -50,7 +50,7 @@ def mmd_params(request): @pytest.mark.parametrize('mmd_params', list(range(n_tests)), indirect=True) def test_mmd(mmd_params): - n_features, n_enc, preprocess, n_permutations, preprocess_x_ref, \ + n_features, n_enc, preprocess, n_permutations, preprocess_at_init, \ batch_size_permutations, configure_kernel_from_x_ref = mmd_params np.random.seed(0) @@ -60,7 +60,7 @@ def test_mmd(mmd_params): preprocess_fn, preprocess_kwargs = preprocess to_list = False if hasattr(preprocess_fn, '__name__') and preprocess_fn.__name__ == 'preprocess_list': - if not preprocess_x_ref: + if not preprocess_at_init: return to_list = True x_ref = [_[None, :] for _ in x_ref] @@ -75,7 +75,7 @@ def test_mmd(mmd_params): cd = MMDDriftKeops( x_ref=x_ref, p_val=.05, - preprocess_x_ref=preprocess_x_ref if isinstance(preprocess_fn, Callable) else False, + preprocess_at_init=preprocess_at_init if isinstance(preprocess_fn, Callable) else False, preprocess_fn=preprocess_fn, configure_kernel_from_x_ref=configure_kernel_from_x_ref, n_permutations=n_permutations, diff --git a/alibi_detect/saving/saving.py b/alibi_detect/saving/saving.py index 975fe2523..b1ae31983 100644 --- a/alibi_detect/saving/saving.py +++ b/alibi_detect/saving/saving.py @@ -46,8 +46,8 @@ def save_detector( if legacy: warnings.warn('The `legacy` option will be removed in a future version.', DeprecationWarning) - if 'backend' in list(detector.meta.keys()) and detector.meta['backend'] in ['pytorch', 'sklearn']: - raise NotImplementedError('Saving detectors with PyTorch or sklearn backend is not yet supported.') + if 'backend' in list(detector.meta.keys()) and detector.meta['backend'] in ['pytorch', 'sklearn', 'keops']: + raise NotImplementedError('Saving detectors with PyTorch, sklearn or keops backend is not yet supported.') # TODO: Replace .__args__ w/ typing.get_args() once Python 3.7 dropped (and remove type ignore below) detector_name = detector.__class__.__name__ diff --git a/alibi_detect/saving/schemas.py b/alibi_detect/saving/schemas.py index c80ce4db0..baba5ef96 100644 --- a/alibi_detect/saving/schemas.py +++ b/alibi_detect/saving/schemas.py @@ -98,7 +98,7 @@ class DetectorConfig(CustomBaseModel): """ name: str "Name of the detector e.g. `MMDDrift`." - backend: Literal['tensorflow', 'pytorch', 'sklearn'] = 'tensorflow' + backend: Literal['tensorflow', 'pytorch', 'sklearn', 'keops'] = 'tensorflow' "The detector backend." meta: Optional[MetaData] "Config metadata. Should not be edited." From eb307b65ae388f2b892291407a257920d3c52355 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Tue, 26 Jul 2022 17:16:37 +0100 Subject: [PATCH 20/50] Update setup and associated docs --- README.md | 7 +++++- doc/source/overview/getting_started.md | 32 +++++++++++++++++++++++--- examples/cd_mmd_keops.ipynb | 1 + setup.py | 5 +++- 4 files changed, 40 insertions(+), 5 deletions(-) create mode 120000 examples/cd_mmd_keops.ipynb diff --git a/README.md b/README.md index d36f9d916..fadcbd6f4 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ The package, `alibi-detect` can be installed from: pip install git+https://github.com/SeldonIO/alibi-detect.git ``` -- To install with the tensorflow backend: +- To install with the TensorFlow backend: ```bash pip install alibi-detect[tensorflow] ``` @@ -88,6 +88,11 @@ The package, `alibi-detect` can be installed from: pip install alibi-detect[torch] ``` +- To install with the KeOps backend: + ```bash + pip install alibi-detect[keops] + ``` + - To use the `Prophet` time series outlier detector: ```bash diff --git a/doc/source/overview/getting_started.md b/doc/source/overview/getting_started.md index 922a17543..ee83a09b2 100644 --- a/doc/source/overview/getting_started.md +++ b/doc/source/overview/getting_started.md @@ -155,6 +155,31 @@ The TensorFlow installation is required to use the following detectors: ``` ```` +````{tab-item} KeOps +:sync: label-keops +:class-label: sd-pt-0 + +```{div} sd-mb-1 +Installation with [KeOps](https://www.kernel-operations.io) backend. +``` + +```bash +pip install alibi-detect[keops] +``` + +```{div} sd-mb-1 +The KeOps installation is required to use the KeOps backend for the following detectors: +- [MMDDrift](../cd/methods/mmddrift.ipynb) +``` + +```{note} +KeOps requires a C++ compiler compatible with `std=c++11`, for example `g++ >=7` or `clang++ >=8`, and a +[Cuda toolkit](https://developer.nvidia.com/cuda-toolkit) installation. For more detailed version requirements +and testing instructions for KeOps, see the +[KeOps docs](https://www.kernel-operations.io/keops/python/installation.html). +``` +```` + ````{tab-item} Prophet :class-label: sd-pt-0 @@ -199,9 +224,10 @@ mamba install -c conda-forge alibi-detect [Alibi Detect](https://github.com/SeldonIO/alibi-detect) is an open source Python library focused on **outlier**, **adversarial** and **drift** detection. The package aims to cover both -online and offline detectors for tabular data, text, images and time series. -Both **TensorFlow** and **PyTorch** backends are supported for drift detection. Alibi-Detect does not install these as -default. See [installation options](#installation) for more details. +online and offline detectors for tabular data, text, images and time series. **TensorFlow**, **PyTorch** +and (where applicable) [KeOps](https://www.kernel-operations.io/keops/index.html) backends are supported +for drift detection. Alibi-Detect does not install these as default. See [installation options](#installation) +for more details. To get a list of respectively the latest outlier, adversarial and drift detection algorithms, you can type: diff --git a/examples/cd_mmd_keops.ipynb b/examples/cd_mmd_keops.ipynb new file mode 120000 index 000000000..fddcc9f46 --- /dev/null +++ b/examples/cd_mmd_keops.ipynb @@ -0,0 +1 @@ +../doc/source/examples/cd_mmd_keops.ipynb \ No newline at end of file diff --git a/setup.py b/setup.py index dc902fa91..5fc64cad9 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,10 @@ def readme(): "tensorflow_probability>=0.8.0, <0.18.0", "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.10.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 ], - 'all': [ + "keops": [ + "pykeops>=2.0.0, <2.2.0", + ], + "all": [ "fbprophet>=0.5, <0.7", "holidays==0.9.11", "pystan<3.0", From 0db2239e53996e1c56b01758dfb1dc2f7eafa2a0 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Wed, 27 Jul 2022 11:28:48 +0100 Subject: [PATCH 21/50] Fix typing issue in --- alibi_detect/cd/keops/mmd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index bebb63382..dc8244dd6 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -167,7 +167,8 @@ def score(self, x: Union[np.ndarray, list]) -> Tuple[float, float, float]: # compute kernel matrix, MMD^2 and apply permutation test m, n = x_ref.shape[0], x.shape[0] perms = [torch.randperm(m + n) for _ in range(self.n_permutations)] - x_all = torch.cat([x_ref, x], 0) + # TODO - Rethink typings (related to https://github.com/SeldonIO/alibi-detect/issues/540) + x_all = torch.cat([x_ref, x], 0) # type: ignore[list-item] mmd2, mmd2_permuted = self._mmd2(x_all, perms, m, n) if self.device.type == 'cuda': mmd2, mmd2_permuted = mmd2.cpu(), mmd2_permuted.cpu() From 45e7211263c3de3e55354166b2b6bb4327c5f741 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Wed, 27 Jul 2022 11:37:01 +0100 Subject: [PATCH 22/50] Install keops as part of CI --- .github/workflows/ci.yml | 2 +- .github/workflows/test_all_notebooks.yml | 2 +- .github/workflows/test_changed_notebooks.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a317e0de..2ceb446bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi - python -m pip install --upgrade --upgrade-strategy eager -e .[tensorflow,torch] + python -m pip install --upgrade --upgrade-strategy eager -e .[tensorflow,torch,keops] python -m pip freeze - name: Lint with flake8 diff --git a/.github/workflows/test_all_notebooks.yml b/.github/workflows/test_all_notebooks.yml index 1787edf5f..abf59df41 100644 --- a/.github/workflows/test_all_notebooks.yml +++ b/.github/workflows/test_all_notebooks.yml @@ -44,7 +44,7 @@ jobs: if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi - python -m pip install --upgrade --upgrade-strategy eager -e .[torch] + python -m pip install --upgrade --upgrade-strategy eager -e .[tensorflow,torch,keops] python -m pip freeze - name: Run notebooks diff --git a/.github/workflows/test_changed_notebooks.yml b/.github/workflows/test_changed_notebooks.yml index 81afd216f..5ac7c19f3 100644 --- a/.github/workflows/test_changed_notebooks.yml +++ b/.github/workflows/test_changed_notebooks.yml @@ -59,7 +59,7 @@ jobs: if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi - python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow] + python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow,keops] python -m pip freeze - name: Run notebooks From 9cee6bccdeb382605408056f6f08a209da3ba104 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Wed, 27 Jul 2022 15:53:31 +0100 Subject: [PATCH 23/50] Add keops tox environment --- alibi_detect/cd/mmd.py | 4 +-- alibi_detect/tests/test_dep_management.py | 34 +++++++++++------------ setup.cfg | 12 ++++++++ setup.py | 1 + 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/alibi_detect/cd/mmd.py b/alibi_detect/cd/mmd.py index fc5d6d789..9236d706e 100644 --- a/alibi_detect/cd/mmd.py +++ b/alibi_detect/cd/mmd.py @@ -11,7 +11,7 @@ if has_tensorflow: from alibi_detect.cd.tensorflow.mmd import MMDDriftTF -if has_keops: +if has_keops and has_pytorch: from alibi_detect.cd.keops.mmd import MMDDriftKeops logger = logging.getLogger(__name__) @@ -90,7 +90,7 @@ def __init__( BackendValidator( backend_options={'tensorflow': ['tensorflow'], 'pytorch': ['pytorch'], - 'keops': ['keops']}, + 'keops': ['keops', 'pytorch']}, construct_name=self.__class__.__name__ ).verify_backend(backend) diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index 4ee06fd8f..f00cda3a0 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -66,8 +66,8 @@ def test_cd_torch_dependencies(opt_dep): dependency_map = defaultdict(lambda: ['default']) for dependency, relations in [ - ("HiddenOutput", ['torch']), - ("preprocess_drift", ['torch']) + ("HiddenOutput", ['torch', 'keops']), + ("preprocess_drift", ['torch', 'keops']) ]: dependency_map[dependency] = relations from alibi_detect.cd import pytorch as cd_pytorch @@ -156,8 +156,8 @@ def test_torch_model_dependencies(opt_dep): dependency_map = defaultdict(lambda: ['default']) for dependency, relations in [ - ("TransformerEmbedding", ['torch']), - ("trainer", ['torch']), + ("TransformerEmbedding", ['torch', 'keops']), + ("trainer", ['torch', 'keops']), ]: dependency_map[dependency] = relations from alibi_detect.models import pytorch as torch_models @@ -255,19 +255,19 @@ def test_torch_utils_dependencies(opt_dep): dependency_map = defaultdict(lambda: ['default']) for dependency, relations in [ - ("batch_compute_kernel_matrix", ['torch']), - ("mmd2", ['torch']), - ("mmd2_from_kernel_matrix", ['torch']), - ("squared_pairwise_distance", ['torch']), - ("GaussianRBF", ['torch']), - ("DeepKernel", ['torch']), - ("permed_lsdds", ['torch']), - ("predict_batch", ['torch']), - ("predict_batch_transformer", ['torch']), - ("quantile", ['torch']), - ("zero_diag", ['torch']), - ("TorchDataset", ['torch']), - ("get_device", ['torch']), + ("batch_compute_kernel_matrix", ['torch', 'keops']), + ("mmd2", ['torch', 'keops']), + ("mmd2_from_kernel_matrix", ['torch', 'keops']), + ("squared_pairwise_distance", ['torch', 'keops']), + ("GaussianRBF", ['torch', 'keops']), + ("DeepKernel", ['torch', 'keops']), + ("permed_lsdds", ['torch', 'keops']), + ("predict_batch", ['torch', 'keops']), + ("predict_batch_transformer", ['torch', 'keops']), + ("quantile", ['torch', 'keops']), + ("zero_diag", ['torch', 'keops']), + ("TorchDataset", ['torch', 'keops']), + ("get_device", ['torch', 'keops']), ]: dependency_map[dependency] = relations from alibi_detect.utils import pytorch as pytorch_utils diff --git a/setup.cfg b/setup.cfg index 29350aa97..926613e4d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,7 @@ envlist= tensorflow torch prophet + keops all # tox test environment for generating licenses @@ -112,6 +113,17 @@ extras= commands = {env:COMMAND:pytest --no-cov alibi_detect/tests/test_dep_management.py --opt-dep=prophet} +# tox test environment for testing keops optional dependency imports +[testenv:keops] +basepython = python +deps = pytest + pytest-cov + pytest-randomly +extras= + keops +commands = + {env:COMMAND:pytest --no-cov alibi_detect/tests/test_dep_management.py --opt-dep=keops} + # environment for testing imports with all optional dependencies installed [testenv:all] basepython = python diff --git a/setup.py b/setup.py index 5fc64cad9..683f49be1 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ def readme(): ], "keops": [ "pykeops>=2.0.0, <2.2.0", + "torch>=1.7.0" ], "all": [ "fbprophet>=0.5, <0.7", From 3fa460cc9a2463e585fba56e4bf5f52e4a0ecac6 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 28 Jul 2022 09:48:15 +0100 Subject: [PATCH 24/50] Add keops to all dependency bucket --- doc/source/examples/cd_mmd_keops.ipynb | 10 +++++----- setup.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/source/examples/cd_mmd_keops.ipynb b/doc/source/examples/cd_mmd_keops.ipynb index 280170883..b8f486b4b 100644 --- a/doc/source/examples/cd_mmd_keops.ipynb +++ b/doc/source/examples/cd_mmd_keops.ipynb @@ -34,7 +34,7 @@ "\n", "## Requirements\n", "\n", - "The notebook requires [PyTorch](https://pytorch.org/) and KeOps to be installed. Once PyTorch is installed, KeOps can be installed via pip:" + "The notebook requires [PyTorch](https://pytorch.org/) and KeOps to be installed. These are optional dependencies for $\\texttt{Alibi Detect}$ and can be installed using:" ] }, { @@ -44,7 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install pykeops" + "!pip install alibi-detect[keops]" ] }, { @@ -491,9 +491,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:detect]", + "display_name": "Python 3", "language": "python", - "name": "conda-env-detect-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -505,7 +505,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.8.13" } }, "nbformat": 4, diff --git a/setup.py b/setup.py index 683f49be1..b284100cd 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,8 @@ def readme(): "pystan<3.0", "tensorflow_probability>=0.8.0, <0.18.0", "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.10.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 - "torch>=1.7.0" + "torch>=1.7.0", + "pykeops>=2.0.0, <2.2.0" ], } From 82f6d3c71779026f01f6cd77ab7d2bb245e335f3 Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 28 Jul 2022 15:10:37 +0100 Subject: [PATCH 25/50] Fix minor issue --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d26a8430f..f7dc86e60 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,7 @@ def readme(): "pystan<3.0", "tensorflow_probability>=0.8.0, <0.18.0", "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.10.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 - - "pykeops>=2.0.0, <2.2.0" + "pykeops>=2.0.0, <2.2.0", "torch>=1.7.0, <1.13.0" ], } From 71142d79b88c1b98085c86fbf5bf8e027fd6870f Mon Sep 17 00:00:00 2001 From: Alex Athorne Date: Thu, 28 Jul 2022 15:55:38 +0100 Subject: [PATCH 26/50] Protect GaussianRBF with import optional --- alibi_detect/tests/test_dep_management.py | 13 +++++++++++++ alibi_detect/utils/keops/__init__.py | 5 ++++- alibi_detect/utils/missing_optional_dependency.py | 3 ++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index f00cda3a0..4333d93c9 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -272,3 +272,16 @@ def test_torch_utils_dependencies(opt_dep): dependency_map[dependency] = relations from alibi_detect.utils import pytorch as pytorch_utils check_correct_dependencies(pytorch_utils, dependency_map, opt_dep) + + +def test_keops_utils_dependencies(opt_dep): + """Tests that the keops utils module correctly protects against uninstalled optional dependencies. + """ + + dependency_map = defaultdict(lambda: ['default']) + for dependency, relations in [ + ("GaussianRBF", ['keops']), + ]: + dependency_map[dependency] = relations + from alibi_detect.utils import keops as keops_utils + check_correct_dependencies(keops_utils, dependency_map, opt_dep) diff --git a/alibi_detect/utils/keops/__init__.py b/alibi_detect/utils/keops/__init__.py index 235176e6b..9da9f5073 100644 --- a/alibi_detect/utils/keops/__init__.py +++ b/alibi_detect/utils/keops/__init__.py @@ -1,4 +1,7 @@ -from .kernels import GaussianRBF +from alibi_detect.utils.missing_optional_dependency import import_optional + + +GaussianRBF = import_optional('alibi_detect.utils.keops.kernels', names=['GaussianRBF']) __all__ = [ "GaussianRBF" diff --git a/alibi_detect/utils/missing_optional_dependency.py b/alibi_detect/utils/missing_optional_dependency.py index 4d6331286..dfe96d656 100644 --- a/alibi_detect/utils/missing_optional_dependency.py +++ b/alibi_detect/utils/missing_optional_dependency.py @@ -25,7 +25,8 @@ "tensorflow_probability": 'tensorflow', "tensorflow": 'tensorflow', "torch": 'torch', - "pytorch": 'torch' + "pytorch": 'torch', + "pykeops": 'keops' } From 9c2da77734c1d30cbb8da8f9312da23a5441c249 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Thu, 28 Jul 2022 18:50:46 +0100 Subject: [PATCH 27/50] Skip keops tests on Windows, and keops notebook test. Fix backend validator. --- .github/workflows/ci.yml | 5 ++++- .github/workflows/test_all_notebooks.yml | 2 +- .github/workflows/test_changed_notebooks.yml | 2 +- alibi_detect/cd/keops/tests/test_mmd_keops.py | 5 ++++- alibi_detect/cd/mmd.py | 2 +- alibi_detect/cd/tests/test_mmd.py | 10 ++++++---- alibi_detect/utils/missing_optional_dependency.py | 2 +- testing/test_notebooks.py | 1 + 8 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ceb446bf..ce81af98b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,10 @@ jobs: if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi - python -m pip install --upgrade --upgrade-strategy eager -e .[tensorflow,torch,keops] + if [ "$RUNNER_OS" != "Windows" ]; then # Skip KeOps tests on Windows as KeOps not supported. + python -m pip install --upgrade --upgrade-strategy eager -e .[keops] + fi + python -m pip install --upgrade --upgrade-strategy eager -e .[tensorflow,torch] python -m pip freeze - name: Lint with flake8 diff --git a/.github/workflows/test_all_notebooks.yml b/.github/workflows/test_all_notebooks.yml index 5b37862ef..84ba3064b 100644 --- a/.github/workflows/test_all_notebooks.yml +++ b/.github/workflows/test_all_notebooks.yml @@ -44,7 +44,7 @@ jobs: if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi - python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow,keops] + python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow] python -m pip freeze - name: Run notebooks diff --git a/.github/workflows/test_changed_notebooks.yml b/.github/workflows/test_changed_notebooks.yml index 5ac7c19f3..81afd216f 100644 --- a/.github/workflows/test_changed_notebooks.yml +++ b/.github/workflows/test_changed_notebooks.yml @@ -59,7 +59,7 @@ jobs: if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi - python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow,keops] + python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow] python -m pip freeze - name: Run notebooks diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index 0ff6d9071..8ebfdc647 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -5,8 +5,10 @@ import torch import torch.nn as nn from typing import Callable, List -from alibi_detect.cd.keops.mmd import MMDDriftKeops +from alibi_detect.utils.frameworks import has_keops from alibi_detect.cd.pytorch.preprocess import HiddenOutput, preprocess_drift +if has_keops: + from alibi_detect.cd.keops.mmd import MMDDriftKeops n, n_hidden, n_classes = 500, 10, 5 @@ -48,6 +50,7 @@ def mmd_params(request): return tests_mmddrift[request.param] +@pytest.mark.skipif(not has_keops, reason='Skipping since pykeops is not installed.') @pytest.mark.parametrize('mmd_params', list(range(n_tests)), indirect=True) def test_mmd(mmd_params): n_features, n_enc, preprocess, n_permutations, preprocess_at_init, \ diff --git a/alibi_detect/cd/mmd.py b/alibi_detect/cd/mmd.py index 9236d706e..74ad8152f 100644 --- a/alibi_detect/cd/mmd.py +++ b/alibi_detect/cd/mmd.py @@ -90,7 +90,7 @@ def __init__( BackendValidator( backend_options={'tensorflow': ['tensorflow'], 'pytorch': ['pytorch'], - 'keops': ['keops', 'pytorch']}, + 'keops': ['keops']}, construct_name=self.__class__.__name__ ).verify_backend(backend) diff --git a/alibi_detect/cd/tests/test_mmd.py b/alibi_detect/cd/tests/test_mmd.py index b2fa8cbb5..c070dcaeb 100644 --- a/alibi_detect/cd/tests/test_mmd.py +++ b/alibi_detect/cd/tests/test_mmd.py @@ -1,9 +1,11 @@ import numpy as np import pytest from alibi_detect.cd import MMDDrift -from alibi_detect.cd.keops.mmd import MMDDriftKeops from alibi_detect.cd.pytorch.mmd import MMDDriftTorch from alibi_detect.cd.tensorflow.mmd import MMDDriftTF +from alibi_detect.utils.frameworks import has_keops +if has_keops: + from alibi_detect.cd.keops.mmd import MMDDriftKeops n, n_features = 100, 5 @@ -19,18 +21,18 @@ def mmddrift_params(request): @pytest.mark.parametrize('mmddrift_params', list(range(n_tests)), indirect=True) def test_mmddrift(mmddrift_params): backend = mmddrift_params - x_ref = np.random.randn(*(n, n_features)) + x_ref = np.random.randn(*(n, n_features)).astype('float32') try: cd = MMDDrift(x_ref=x_ref, backend=backend) - except NotImplementedError: + except (NotImplementedError, ImportError): cd = None if backend.lower() == 'pytorch': assert isinstance(cd._detector, MMDDriftTorch) elif backend.lower() == 'tensorflow': assert isinstance(cd._detector, MMDDriftTF) - elif backend.lower() == 'keops': + elif backend.lower() == 'keops' and has_keops: assert isinstance(cd._detector, MMDDriftKeops) else: assert cd is None diff --git a/alibi_detect/utils/missing_optional_dependency.py b/alibi_detect/utils/missing_optional_dependency.py index dfe96d656..6e4f80bad 100644 --- a/alibi_detect/utils/missing_optional_dependency.py +++ b/alibi_detect/utils/missing_optional_dependency.py @@ -26,7 +26,7 @@ "tensorflow": 'tensorflow', "torch": 'torch', "pytorch": 'torch', - "pykeops": 'keops' + "keops": 'keops', } diff --git a/testing/test_notebooks.py b/testing/test_notebooks.py index 48a94c264..d885f4c9c 100644 --- a/testing/test_notebooks.py +++ b/testing/test_notebooks.py @@ -38,6 +38,7 @@ 'cd_context_20newsgroup.ipynb', 'cd_context_ecg.ipynb', 'cd_text_imdb.ipynb', + 'cd_mmd_keops.ipynb', # the following requires a k8s cluster 'alibi_detect_deploy.ipynb', # the following require downloading large datasets From b126a6d05019052d1fc5b52f01f6fb7541ec36ec Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Thu, 28 Jul 2022 19:21:06 +0100 Subject: [PATCH 28/50] Skip keops kernel tests if not installed --- alibi_detect/utils/keops/tests/test_kernels_keops.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/alibi_detect/utils/keops/tests/test_kernels_keops.py b/alibi_detect/utils/keops/tests/test_kernels_keops.py index d42a7c0e1..abdb5fd92 100644 --- a/alibi_detect/utils/keops/tests/test_kernels_keops.py +++ b/alibi_detect/utils/keops/tests/test_kernels_keops.py @@ -1,9 +1,11 @@ from itertools import product import numpy as np -from pykeops.torch import LazyTensor +from alibi_detect.utils.frameworks import has_keops import pytest import torch -from alibi_detect.utils.keops import GaussianRBF +if has_keops: + from pykeops.torch import LazyTensor + from alibi_detect.utils.keops import GaussianRBF sigma = [None, np.array([1.]), np.array([1., 2.])] n_features = [5, 10] @@ -18,6 +20,7 @@ def gaussian_kernel_params(request): return tests_gk[request.param] +@pytest.mark.skipif(not has_keops, reason='Skipping since pykeops is not installed.') @pytest.mark.parametrize('gaussian_kernel_params', list(range(n_tests_gk)), indirect=True) def test_gaussian_kernel(gaussian_kernel_params): sigma, n_features, n_instances, trainable = gaussian_kernel_params From 48ad925589ba1532a72983ed1794dce21d140e11 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Fri, 29 Jul 2022 10:41:11 +0100 Subject: [PATCH 29/50] Add pykeops to op deps ERROR_TYPES --- alibi_detect/utils/missing_optional_dependency.py | 1 + 1 file changed, 1 insertion(+) diff --git a/alibi_detect/utils/missing_optional_dependency.py b/alibi_detect/utils/missing_optional_dependency.py index 6e4f80bad..aa8977588 100644 --- a/alibi_detect/utils/missing_optional_dependency.py +++ b/alibi_detect/utils/missing_optional_dependency.py @@ -27,6 +27,7 @@ "torch": 'torch', "pytorch": 'torch', "keops": 'keops', + "pykeops": 'keops', } From 74ff9923990a2fae4eaf5d7e255b3e9d4a219023 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Fri, 29 Jul 2022 14:23:32 +0100 Subject: [PATCH 30/50] Skip keops tests on MacOS --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce81af98b..f477daaa7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi - if [ "$RUNNER_OS" != "Windows" ]; then # Skip KeOps tests on Windows as KeOps not supported. + if [ "$RUNNER_OS" == "Linux"]; then # Currently, we only support KeOps on Linux. python -m pip install --upgrade --upgrade-strategy eager -e .[keops] fi python -m pip install --upgrade --upgrade-strategy eager -e .[tensorflow,torch] From 7c5e70dfd6ac3f6571215e70edbb982c4942601d Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Fri, 29 Jul 2022 14:47:10 +0100 Subject: [PATCH 31/50] Add note to docs about linux-only support for keops --- doc/source/conf.py | 3 ++- doc/source/overview/getting_started.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 98630dfad..cd4bf6319 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -110,7 +110,8 @@ "numba", "pydantic", "toml", - "catalogue" + "catalogue", + "pykeops" ] # Napoleon settings diff --git a/doc/source/overview/getting_started.md b/doc/source/overview/getting_started.md index ee83a09b2..9e987686f 100644 --- a/doc/source/overview/getting_started.md +++ b/doc/source/overview/getting_started.md @@ -176,7 +176,8 @@ The KeOps installation is required to use the KeOps backend for the following de KeOps requires a C++ compiler compatible with `std=c++11`, for example `g++ >=7` or `clang++ >=8`, and a [Cuda toolkit](https://developer.nvidia.com/cuda-toolkit) installation. For more detailed version requirements and testing instructions for KeOps, see the -[KeOps docs](https://www.kernel-operations.io/keops/python/installation.html). +[KeOps docs](https://www.kernel-operations.io/keops/python/installation.html). **Currently, the KeOps backend is +only officially supported on Linux.** ``` ```` From fc12b9d492795f232f97cd478b5c32e41fe41b32 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Fri, 29 Jul 2022 14:53:15 +0100 Subject: [PATCH 32/50] Add batch_size_permutations to pydantic models --- alibi_detect/saving/schemas.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alibi_detect/saving/schemas.py b/alibi_detect/saving/schemas.py index baba5ef96..8d857c9f9 100644 --- a/alibi_detect/saving/schemas.py +++ b/alibi_detect/saving/schemas.py @@ -634,6 +634,7 @@ class MMDDriftConfig(DriftDetectorConfig): sigma: Optional[NDArray[np.float32]] = None configure_kernel_from_x_ref: bool = True n_permutations: int = 100 + batch_size_permutations: int = 1000000 device: Optional[Literal['cpu', 'cuda']] = None @@ -652,6 +653,7 @@ class MMDDriftConfigResolved(DriftDetectorConfigResolved): sigma: Optional[NDArray[np.float32]] = None configure_kernel_from_x_ref: bool = True n_permutations: int = 100 + batch_size_permutations: int = 1000000 device: Optional[Literal['cpu', 'cuda']] = None From f6b331b352d3f4c031b295747d2f0df4a2328936 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 9 Aug 2022 11:41:08 +0100 Subject: [PATCH 33/50] remove print --- alibi_detect/utils/keops/tests/test_kernels_keops.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/alibi_detect/utils/keops/tests/test_kernels_keops.py b/alibi_detect/utils/keops/tests/test_kernels_keops.py index abdb5fd92..1f0e20b7c 100644 --- a/alibi_detect/utils/keops/tests/test_kernels_keops.py +++ b/alibi_detect/utils/keops/tests/test_kernels_keops.py @@ -25,9 +25,6 @@ def gaussian_kernel_params(request): def test_gaussian_kernel(gaussian_kernel_params): sigma, n_features, n_instances, trainable = gaussian_kernel_params xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features) - - print(sigma, xshape, yshape, trainable) - sigma = sigma if sigma is None else torch.from_numpy(sigma).float() x = torch.from_numpy(np.random.random(xshape)).float() y = torch.from_numpy(np.random.random(yshape)).float() From ace20cca098459ae603ceb77153ba1a27bb562cc Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 9 Aug 2022 11:42:48 +0100 Subject: [PATCH 34/50] remove unnecessary comment --- alibi_detect/utils/keops/kernels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index d0f826772..11a2e3894 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -78,7 +78,7 @@ def forward(self, x: LazyTensor, y: LazyTensor, infer_sigma: bool = False) -> La if infer_sigma or self.init_required: if self.trainable and infer_sigma: raise ValueError("Gradients cannot be computed w.r.t. an inferred sigma value") - sigma = self.init_sigma_fn(x, y, dist) # .to(x.device) + sigma = self.init_sigma_fn(x, y, dist) with torch.no_grad(): self.log_sigma.copy_(sigma.log().clone()) self.init_required = False From 718fb8507df95b84af0055c2a5de0dae0b1d01e2 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 9 Aug 2022 11:57:05 +0100 Subject: [PATCH 35/50] change default bandwidth fn to None --- alibi_detect/utils/keops/kernels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index 11a2e3894..7958f5c67 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -36,7 +36,7 @@ class GaussianRBF(nn.Module): def __init__( self, sigma: Optional[torch.Tensor] = None, - init_sigma_fn: Callable = sigma_mean, + init_sigma_fn: Callable = None, trainable: bool = False ) -> None: """ From 5922e3f4259092cfe0fa25225ce8036d73698bb1 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 9 Aug 2022 12:26:39 +0100 Subject: [PATCH 36/50] update infer sigma --- alibi_detect/cd/keops/mmd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index dc8244dd6..19b68aab8 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -97,7 +97,9 @@ def __init__( self.n_batches = 1 + (n_permutations - 1) // batch_size_permutations # infer the kernel bandwidth from the reference data - if self.infer_sigma or isinstance(sigma, torch.Tensor): + if isinstance(sigma, torch.Tensor): + self.infer_sigma = False + elif self.infer_sigma: x = torch.from_numpy(self.x_ref).to(self.device) _ = self.kernel(LazyTensor(x[:, None, :]), LazyTensor(x[None, :, :]), infer_sigma=self.infer_sigma) self.infer_sigma = False From b8adfbe589d9087f9495b88f31e3a55fea965601 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 9 Aug 2022 19:06:05 +0100 Subject: [PATCH 37/50] update test warning, update and clarify keops kernels logic --- alibi_detect/cd/keops/mmd.py | 4 +- alibi_detect/utils/keops/kernels.py | 37 ++++++++++++++----- .../utils/keops/tests/test_kernels_keops.py | 2 +- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index 19b68aab8..d122031e4 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -138,12 +138,12 @@ def _mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) x, y = x.to(self.device), y.to(self.device) # batch-wise kernel matrix computation over the permutations + k_xy.append(self.kernel( + LazyTensor(x[:, :, None, :]), LazyTensor(y[:, None, :, :]), self.infer_sigma).sum(1).sum(1).squeeze(-1)) k_xx.append(self.kernel( LazyTensor(x[:, :, None, :]), LazyTensor(x[:, None, :, :])).sum(1).sum(1).squeeze(-1)) k_yy.append(self.kernel( LazyTensor(y[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1)) - k_xy.append(self.kernel( - LazyTensor(x[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1)) c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) stats = c_xx * (torch.cat(k_xx) - m) + c_yy * (torch.cat(k_yy) - n) - c_xy * torch.cat(k_xy) return stats[0], stats[1:] diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index 7958f5c67..fad852faa 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -5,9 +5,9 @@ from typing import Callable, Optional -def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor) -> torch.Tensor: +def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = None) -> torch.Tensor: """ - Bandwidth estimation using the mean heuristic. + Set bandwidth to the mean distance between instances x and y. Parameters ---------- @@ -16,16 +16,29 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor) -> torch.Tensor: y LazyTensor of instances with dimension [1, Ny, features]. dist - LazyTensor with dimensions [Nx, Ny], containing the pairwise distances between `x` and `y`. + LazyTensor with dimensions [Nx, Ny] containing the pairwise distances between `x` and `y`. + n_min + In order to check whether x equals y after squeezing the singleton dimensions, we check if the + diagonal of the distance matrix (which is a lazy tensor from which the diagonal cannot be directly extracted) + consists of all zeros. We do this by computing the k-min distances and k-argmin indices over the + columns of the distance matrix. We then check if the distances on the diagonal of the distance matrix + are all zero or not. If they are all zero, then we do not use these distances (zeros) when computing + the mean pairwise distance as bandwidth. The default `None` sets k to Nx (=Ny). If Nx becomes very large, + it is advised to set `n_min` to a lower value. Returns ------- The computed bandwidth, `sigma`. """ - n = x.shape[0] - if x.shape == y.shape and (dist.min(axis=1) == 0.).all() and \ - (torch.arange(n) == dist.argmin(axis=1).cpu().view(-1)).all(): - n_mean = n * (n - 1) + nx, ny = dist.shape + if nx == ny: + n_min = n_min if isinstance(n_min, int) else nx + d_min, id_min = dist.Kmin_argKmin(n_min, axis=1) + rows, cols = torch.where(id_min.cpu() == torch.arange(nx)[:, None]) + if (d_min[rows, cols] == 0.).all(): + n_mean = n * (n - 1) + else: + n_mean = np.prod(dist.shape) else: n_mean = np.prod(dist.shape) sigma = (.5 * dist.sum(1).sum().unsqueeze(-1) / n_mean) ** .5 @@ -41,8 +54,11 @@ def __init__( ) -> None: """ Gaussian RBF kernel: k(x,y) = exp(-(1/(2*sigma^2)||x-y||^2). A forward pass takes - a batch of instances x [Nx, features] and y [Ny, features] and returns the kernel - matrix [Nx, Ny]. + a batch of instances x and y and returns the kernel matrix. + x can be of shape [Nx, 1, features] or [batch_size, Nx, 1, features]. + y can be of shape [1, Ny, features] or [batch_size, 1, Ny, features]. + The returned kernel matrix can be of shape [Nx, Ny] or [batch_size, Nx, Ny]. + x, y and the returned kernel matrix are all lazy tensors. Parameters ---------- @@ -52,11 +68,12 @@ def __init__( init_sigma_fn Function used to compute the bandwidth `sigma`. Used when `sigma` is to be inferred. The function's signature should match :py:func:`~alibi_detect.utils.keops.kernels.sigma_mean`, - meaning that it should take in the tensors `x`, `y` and `dist` and return `sigma`. + meaning that it should take in the lazy tensors `x`, `y` and `dist` and return a tensor `sigma`. trainable Whether or not to track gradients w.r.t. `sigma` to allow it to be trained. """ super().__init__() + init_sigma_fn = sigma_mean if init_sigma_fn is None else init_sigma_fn if sigma is None: self.log_sigma = nn.Parameter(torch.empty(1), requires_grad=trainable) self.init_required = True diff --git a/alibi_detect/utils/keops/tests/test_kernels_keops.py b/alibi_detect/utils/keops/tests/test_kernels_keops.py index 1f0e20b7c..685be989d 100644 --- a/alibi_detect/utils/keops/tests/test_kernels_keops.py +++ b/alibi_detect/utils/keops/tests/test_kernels_keops.py @@ -32,7 +32,7 @@ def test_gaussian_kernel(gaussian_kernel_params): kernel = GaussianRBF(sigma=sigma, trainable=trainable) infer_sigma = True if sigma is None else False if trainable and infer_sigma: - with pytest.raises(Exception): + with pytest.raises(ValueError): kernel(LazyTensor(x[:, None, :]), LazyTensor(y[None, :, :]), infer_sigma=infer_sigma) else: k_xy = kernel(LazyTensor(x[:, None, :]), LazyTensor(y[None, :, :]), infer_sigma=infer_sigma) From 015cc5eba464ccc47a346e6cfbaed793d6991d49 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 9 Aug 2022 19:35:20 +0100 Subject: [PATCH 38/50] clean up --- doc/source/examples/cd_mmd_keops.ipynb | 143 +++++++++++++++++++------ 1 file changed, 113 insertions(+), 30 deletions(-) diff --git a/doc/source/examples/cd_mmd_keops.ipynb b/doc/source/examples/cd_mmd_keops.ipynb index b8f486b4b..dac1556e8 100644 --- a/doc/source/examples/cd_mmd_keops.ipynb +++ b/doc/source/examples/cd_mmd_keops.ipynb @@ -3,7 +3,11 @@ { "cell_type": "markdown", "id": "27a4394b", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "# Scaling up drift detection with KeOps\n", "\n", @@ -41,7 +45,11 @@ "cell_type": "code", "execution_count": null, "id": "a0bf1719", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "!pip install alibi-detect[keops]" @@ -50,7 +58,11 @@ { "cell_type": "markdown", "id": "7ff93d59", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "Before we start let’s fix the random seeds for reproducibility:" ] @@ -59,7 +71,11 @@ "cell_type": "code", "execution_count": 1, "id": "2ba95f29", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "import numpy as np\n", @@ -76,7 +92,11 @@ { "cell_type": "markdown", "id": "1910895a", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "\n", "## Vanilla PyTorch vs. KeOps comparison\n", @@ -90,7 +110,11 @@ "cell_type": "code", "execution_count": 2, "id": "a1c65254", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "from alibi_detect.cd import MMDDrift\n", @@ -124,21 +148,21 @@ " # Sample reference and test data\n", " x_ref = np.random.randn(*(n_ref, n_features)).astype(np.float32)\n", " x_test = np.random.randn(*(n_test, n_features)).astype(np.float32) + mu\n", - " \n", + "\n", " # Initialise detector, make and log predictions\n", " p_val = .05\n", " dd = MMDDrift(x_ref, backend=backend, p_val=p_val, n_permutations=100)\n", " start = timer()\n", " pred = dd.predict(x_test)\n", " end = timer()\n", - " \n", + "\n", " if _ > 0: # first run reserved for KeOps compilation\n", " t_detect.append(end - start)\n", " p_vals.append(pred['data']['p_val'])\n", - " \n", + "\n", " del dd, x_ref, x_test\n", " torch.cuda.empty_cache()\n", - " \n", + "\n", " p_vals = np.array(p_vals)\n", " t_mean, t_std = np.array(t_detect).mean(), np.array(t_detect).std()\n", " results = eval_detector(p_vals, p_val, mu == 0., t_mean, t_std)\n", @@ -165,7 +189,7 @@ " return T\n", "\n", "\n", - "def plot_absolute_time(results: dict, n_features: list, y_scale: str = 'linear', \n", + "def plot_absolute_time(results: dict, n_features: list, y_scale: str = 'linear',\n", " detector: str = 'MMD', max_batch_size: int = 1e10):\n", " T = format_results(n_features, ['keops', 'pytorch'], max_batch_size)\n", " colors = ['b', 'g', 'r', 'c', 'm', 'y', 'b']\n", @@ -206,9 +230,13 @@ { "cell_type": "markdown", "id": "43a4ee7e", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ - "As detailed earlier, we will compare the PyTorch with the KeOps implementation of the MMD detector for a variety of reference and test data batch sizes as well as different feature dimensions. Note that for the PyTorch implementation, the portion of the kernel matrix for the reference data itself can already be computed at initialisation of the detector. This computation will not be included when we record the detector's prediction time. Since use cases where $N_\\text{ref} >> N_\\text{test}$ are quite common, we will also test for this specific setting. The key reason is that we cannot amortise this computation for the KeOps detector since we are working with lazily evaluated symbolic matrices.\n", + "As detailed earlier, we will compare the PyTorch with the KeOps implementation of the MMD detector for a variety of reference and test data batch sizes as well as different feature dimensions. Note that for the PyTorch implementation, the portion of the kernel matrix for the reference data itself can already be computed at initialisation of the detector. This computation will not be included when we record the detector's prediction time. Since use cases where $N_\\text{ref} \\gg N_\\text{test}$ are quite common, we will also test for this specific setting. The key reason is that we cannot amortise this computation for the KeOps detector since we are working with lazily evaluated symbolic matrices.\n", "\n", "#### $N_\\text{ref} = N_\\text{test}$\n", "\n", @@ -219,7 +247,11 @@ "cell_type": "code", "execution_count": 3, "id": "47268603", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "experiments = {\n", @@ -262,7 +294,10 @@ "execution_count": 4, "id": "d556296a", "metadata": { - "scrolled": true + "scrolled": true, + "pycharm": { + "name": "#%%\n" + } }, "outputs": [], "source": [ @@ -280,7 +315,11 @@ { "cell_type": "markdown", "id": "93396443", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "Below we visualise the runtimes of the different experiments. We can make the following observations:\n", "\n", @@ -297,7 +336,11 @@ "cell_type": "code", "execution_count": 5, "id": "5d854bfb", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "data": { @@ -323,7 +366,11 @@ "cell_type": "code", "execution_count": 6, "id": "ec9d0fbb", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "data": { @@ -345,7 +392,11 @@ { "cell_type": "markdown", "id": "b96a904b", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "The difference between KeOps and PyTorch is even more striking when we only look at $[2, 10]$ features:" ] @@ -354,7 +405,11 @@ "cell_type": "code", "execution_count": 7, "id": "0d1e4dfa", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "data": { @@ -376,18 +431,26 @@ { "cell_type": "markdown", "id": "6e920708", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ - "#### $N_\\text{ref} >> N_\\text{test}$\n", + "#### $N_\\text{ref} \\gg N_\\text{test}$\n", "\n", - "Now we check whether the speed improvements still hold when $N_\\text{ref} >> N_\\text{test}$ ($N_\\text{ref} / N_\\text{test} = 10$) and a large part of the kernel can already be computed at initialisation time of the PyTorch (but not the KeOps) detector." + "Now we check whether the speed improvements still hold when $N_\\text{ref} \\gg N_\\text{test}$ ($N_\\text{ref} / N_\\text{test} = 10$) and a large part of the kernel can already be computed at initialisation time of the PyTorch (but not the KeOps) detector." ] }, { "cell_type": "code", "execution_count": 8, "id": "a75794e8", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "experiments = {\n", @@ -411,7 +474,11 @@ "cell_type": "code", "execution_count": 9, "id": "fcdd840a", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "results = {backend: {} for backend in backends}\n", @@ -427,7 +494,11 @@ { "cell_type": "markdown", "id": "27307020", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "The below plots illustrate that KeOps indeed still provides large speed ups over PyTorch. The x-axis shows the reference batch size $N_\\text{ref}$. Note that $N_\\text{ref} / N_\\text{test} = 10$." ] @@ -436,7 +507,11 @@ "cell_type": "code", "execution_count": 10, "id": "0a3c0d27", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "data": { @@ -459,7 +534,11 @@ "cell_type": "code", "execution_count": 11, "id": "cf6a0dfc", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "data": { @@ -481,7 +560,11 @@ { "cell_type": "markdown", "id": "f7dc206c", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## Conclusion\n", "\n", @@ -510,4 +593,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file From 148019af186d0ec6e02a8f0d55ed4a3257bb0a08 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 9 Aug 2022 19:41:50 +0100 Subject: [PATCH 39/50] update docstring --- alibi_detect/utils/keops/kernels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index fad852faa..6f28cb6f6 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -12,9 +12,9 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = None Parameters ---------- x - LazyTensor of instances with dimension [Nx, 1, features]. + LazyTensor of instances with dimension [Nx, 1, features]. The singleton dimension is necessary for broadcasting. y - LazyTensor of instances with dimension [1, Ny, features]. + LazyTensor of instances with dimension [1, Ny, features]. The singleton dimension is necessary for broadcasting. dist LazyTensor with dimensions [Nx, Ny] containing the pairwise distances between `x` and `y`. n_min From 74533689dd1fd968b5c1cec7b3f7d7029276ced0 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 9 Aug 2022 19:46:01 +0100 Subject: [PATCH 40/50] fix bug --- alibi_detect/utils/keops/kernels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index 6f28cb6f6..e66ec7afc 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -36,7 +36,7 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = None d_min, id_min = dist.Kmin_argKmin(n_min, axis=1) rows, cols = torch.where(id_min.cpu() == torch.arange(nx)[:, None]) if (d_min[rows, cols] == 0.).all(): - n_mean = n * (n - 1) + n_mean = nx * (nx - 1) else: n_mean = np.prod(dist.shape) else: From 2d88bfc5c23bbb725f6911425acbdbb12f682300 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Wed, 10 Aug 2022 13:33:35 +0100 Subject: [PATCH 41/50] undo unnecessary kwarg removal --- alibi_detect/cd/mmd.py | 1 - 1 file changed, 1 deletion(-) diff --git a/alibi_detect/cd/mmd.py b/alibi_detect/cd/mmd.py index 74ad8152f..51739a3ea 100644 --- a/alibi_detect/cd/mmd.py +++ b/alibi_detect/cd/mmd.py @@ -104,7 +104,6 @@ def __init__( pop_kwargs += ['batch_size_permutations'] detector = MMDDriftTorch else: - pop_kwargs += ['configure_kernel_from_x_ref'] detector = MMDDriftKeops [kwargs.pop(k, None) for k in pop_kwargs] From 54df25709a4e25dd08d26f3a35c9ca214a9b0fc4 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Wed, 10 Aug 2022 18:57:06 +0100 Subject: [PATCH 42/50] make test consistent with torch/tf backends --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index 8ebfdc647..33bbb1a79 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -36,11 +36,12 @@ def preprocess_list(x: List[np.ndarray]) -> np.ndarray: (preprocess_drift, {'model': HiddenOutput, 'layer': -1}), (preprocess_list, None) ] +update_x_ref = [{'last': 750}, {'reservoir_sampling': 750}, None] preprocess_at_init = [True, False] n_permutations = [10] batch_size_permutations = [10, 1000000] -configure_kernel_from_x_ref = [True, False] -tests_mmddrift = list(product(n_features, n_enc, preprocess, n_permutations, preprocess_at_init, +configure_kernel_from_x_ref = [True] +tests_mmddrift = list(product(n_features, n_enc, preprocess, n_permutations, preprocess_at_init, update_x_ref, batch_size_permutations, configure_kernel_from_x_ref)) n_tests = len(tests_mmddrift) @@ -53,7 +54,7 @@ def mmd_params(request): @pytest.mark.skipif(not has_keops, reason='Skipping since pykeops is not installed.') @pytest.mark.parametrize('mmd_params', list(range(n_tests)), indirect=True) def test_mmd(mmd_params): - n_features, n_enc, preprocess, n_permutations, preprocess_at_init, \ + n_features, n_enc, preprocess, n_permutations, preprocess_at_init, update_x_ref, \ batch_size_permutations, configure_kernel_from_x_ref = mmd_params np.random.seed(0) @@ -79,6 +80,7 @@ def test_mmd(mmd_params): x_ref=x_ref, p_val=.05, preprocess_at_init=preprocess_at_init if isinstance(preprocess_fn, Callable) else False, + update_x_ref=update_x_ref, preprocess_fn=preprocess_fn, configure_kernel_from_x_ref=configure_kernel_from_x_ref, n_permutations=n_permutations, @@ -87,6 +89,10 @@ def test_mmd(mmd_params): x = x_ref.copy() preds = cd.predict(x, return_p_val=True) assert preds['data']['is_drift'] == 0 and preds['data']['p_val'] >= cd.p_val + if isinstance(update_x_ref, dict): + k = list(update_x_ref.keys())[0] + assert cd.n == len(x) + len(x_ref) + assert cd.x_ref.shape[0] == min(update_x_ref[k], len(x) + len(x_ref)) x_h1 = np.random.randn(n * n_features).reshape(n, n_features).astype(np.float32) if to_list: From 211eeb97663d67854bb3329fd3c99df548b53895 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Thu, 11 Aug 2022 19:28:02 +0100 Subject: [PATCH 43/50] add _mmd2 test --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index 33bbb1a79..efc8b78d0 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -6,6 +6,7 @@ import torch.nn as nn from typing import Callable, List from alibi_detect.utils.frameworks import has_keops +from alibi_detect.utils.pytorch import GaussianRBF, batch_compute_kernel_matrix, mmd2_from_kernel_matrix from alibi_detect.cd.pytorch.preprocess import HiddenOutput, preprocess_drift if has_keops: from alibi_detect.cd.keops.mmd import MMDDriftKeops @@ -104,3 +105,17 @@ def test_mmd(mmd_params): else: assert preds['data']['p_val'] >= preds['data']['threshold'] == cd.p_val assert preds['data']['distance'] <= preds['data']['distance_threshold'] + + # ensure the keops MMD^2 estimate matches the pytorch implementation for the same kernel + if not isinstance(x_ref, list) and update_x_ref is None: + print(x_ref.shape, x_h1.shape) + p_val, mmd2, distance_threshold = cd.score(x_h1) + kernel = GaussianRBF(sigma=cd.kernel.sigma) + if isinstance(preprocess_fn, Callable): + x_ref, x_h1 = cd.preprocess(x_h1) + x_ref = torch.from_numpy(x_ref).float() + x_h1 = torch.from_numpy(x_h1).float() + x_all = torch.cat([x_ref, x_h1], 0) + kernel_mat = kernel(x_all, x_all) + mmd2_torch = mmd2_from_kernel_matrix(kernel_mat, x_h1.shape[0]) + np.testing.assert_almost_equal(mmd2, mmd2_torch, decimal=6) From f98fd830fad0fc8ebb1cb278794e1861262379d5 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Thu, 11 Aug 2022 19:47:51 +0100 Subject: [PATCH 44/50] remove unused import --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index efc8b78d0..8d3cf8124 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -6,7 +6,7 @@ import torch.nn as nn from typing import Callable, List from alibi_detect.utils.frameworks import has_keops -from alibi_detect.utils.pytorch import GaussianRBF, batch_compute_kernel_matrix, mmd2_from_kernel_matrix +from alibi_detect.utils.pytorch import GaussianRBF, mmd2_from_kernel_matrix from alibi_detect.cd.pytorch.preprocess import HiddenOutput, preprocess_drift if has_keops: from alibi_detect.cd.keops.mmd import MMDDriftKeops From 751d3a0cb88366f5f70b09ebf6b122f6c0682c7e Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 16 Aug 2022 15:13:36 +0100 Subject: [PATCH 45/50] clarify docs, remove redundant framework checks --- alibi_detect/cd/mmd.py | 4 ++-- alibi_detect/utils/frameworks.py | 1 + doc/source/cd/methods/mmddrift.ipynb | 34 ++++++++++++++++++++++------ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/alibi_detect/cd/mmd.py b/alibi_detect/cd/mmd.py index 51739a3ea..b00c20449 100644 --- a/alibi_detect/cd/mmd.py +++ b/alibi_detect/cd/mmd.py @@ -97,10 +97,10 @@ def __init__( kwargs = locals() args = [kwargs['x_ref']] pop_kwargs = ['self', 'x_ref', 'backend', '__class__'] - if backend == 'tensorflow' and has_tensorflow: + if backend == 'tensorflow': pop_kwargs += ['device', 'batch_size_permutations'] detector = MMDDriftTF - elif backend == 'pytorch' and has_pytorch: + elif backend == 'pytorch': pop_kwargs += ['batch_size_permutations'] detector = MMDDriftTorch else: diff --git a/alibi_detect/utils/frameworks.py b/alibi_detect/utils/frameworks.py index c475c49b7..1a5f7e5ef 100644 --- a/alibi_detect/utils/frameworks.py +++ b/alibi_detect/utils/frameworks.py @@ -15,6 +15,7 @@ try: import pykeops # noqa + import torch # noqa has_keops = True except ImportError: has_keops = False diff --git a/doc/source/cd/methods/mmddrift.ipynb b/doc/source/cd/methods/mmddrift.ipynb index 63523a264..cb85b7345 100644 --- a/doc/source/cd/methods/mmddrift.ipynb +++ b/doc/source/cd/methods/mmddrift.ipynb @@ -2,14 +2,22 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "[source](../../api/alibi_detect.cd.mmd.rst)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "# Maximum Mean Discrepancy\n", "\n", @@ -30,7 +38,11 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## Usage\n", "\n", @@ -88,7 +100,7 @@ "cd_keops = MMDDrift(x_ref, backend='keops', p_val=.05)\n", "```\n", "\n", - "We can also easily add preprocessing functions for the *TensorFlow * and *PyTorch* frameworks. The following example uses a randomly initialized image encoder in PyTorch:\n", + "We can also easily add preprocessing functions for the *TensorFlow* and *PyTorch* frameworks. Note that we can also combine for instance a PyTorch preprocessing step with a KeOps detector. The following example uses a randomly initialized image encoder in PyTorch:\n", "\n", "```python\n", "from functools import partial\n", @@ -157,7 +169,11 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "### Detect Drift\n", "\n", @@ -183,7 +199,11 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## Examples\n", "\n", @@ -229,4 +249,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file From 7c2d7811c884bb36f656aa858e81d149684dc06a Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 16 Aug 2022 15:49:58 +0100 Subject: [PATCH 46/50] remove print --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 1 - 1 file changed, 1 deletion(-) diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index 8d3cf8124..53b47cc27 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -108,7 +108,6 @@ def test_mmd(mmd_params): # ensure the keops MMD^2 estimate matches the pytorch implementation for the same kernel if not isinstance(x_ref, list) and update_x_ref is None: - print(x_ref.shape, x_h1.shape) p_val, mmd2, distance_threshold = cd.score(x_h1) kernel = GaussianRBF(sigma=cd.kernel.sigma) if isinstance(preprocess_fn, Callable): From 3f69740291e4e9e2c3d2a36cca62a5bc0686bc3b Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 16 Aug 2022 16:19:49 +0100 Subject: [PATCH 47/50] update docs keops --- alibi_detect/cd/keops/mmd.py | 1 + doc/source/cd/methods/mmddrift.ipynb | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index d122031e4..86173ad13 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -145,6 +145,7 @@ def _mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) k_yy.append(self.kernel( LazyTensor(y[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1)) c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) + # Note that the MMD^2 estimates assume that the diagonal of the kernel matrix consists of 1's stats = c_xx * (torch.cat(k_xx) - m) + c_yy * (torch.cat(k_yy) - n) - c_xy * torch.cat(k_xy) return stats[0], stats[1:] diff --git a/doc/source/cd/methods/mmddrift.ipynb b/doc/source/cd/methods/mmddrift.ipynb index cb85b7345..2f68cb664 100644 --- a/doc/source/cd/methods/mmddrift.ipynb +++ b/doc/source/cd/methods/mmddrift.ipynb @@ -56,7 +56,7 @@ "\n", "Keyword arguments:\n", "\n", - "* `backend`: **TensorFlow**, **PyTorch** and [**KeOps**](https://github.com/getkeops/keops) implementations of the MMD detector as well as various preprocessing steps are available. Specify the backend (*tensorflow*, *pytorch* or *keops*). Defaults to *tensorflow*.\n", + "* `backend`: **TensorFlow**, **PyTorch** and [**KeOps**](https://github.com/getkeops/keops) implementations of the MMD detector are available. Specify the backend (*tensorflow*, *pytorch* or *keops*). Defaults to *tensorflow*.\n", "\n", "* `p_val`: p-value used for significance of the permutation test.\n", "\n", @@ -68,7 +68,7 @@ "\n", "* `preprocess_fn`: Function to preprocess the data before computing the data drift metrics. Typically a dimensionality reduction technique.\n", "\n", - "* `kernel`: Kernel used when computing the MMD. Defaults to a Gaussian RBF kernel (`from alibi_detect.utils.pytorch import GaussianRBF`, `from alibi_detect.utils.tensorflow import GaussianRBF` or `from alibi_detect.utils.keops import GaussianRBF` dependent on the backend used).\n", + "* `kernel`: Kernel used when computing the MMD. Defaults to a Gaussian RBF kernel (`from alibi_detect.utils.pytorch import GaussianRBF`, `from alibi_detect.utils.tensorflow import GaussianRBF` or `from alibi_detect.utils.keops import GaussianRBF` dependent on the backend used). Note that for the KeOps backend, the diagonal entries of the kernel matrices `kernel(x_ref, x_ref)` and `kernel(x_test, x_test)` should be equal to 1. This is compliant with the default Gaussian RBF kernel.\n", "\n", "* `sigma`: Optional bandwidth for the kernel as a `np.ndarray`. We can also average over a number of different bandwidths, e.g. `np.array([.5, 1., 1.5])`.\n", "\n", From ac5fe64c6c42f06a72c25af2f5d7abf7eaccbf03 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 16 Aug 2022 18:28:32 +0100 Subject: [PATCH 48/50] batched version of sigma_mean part 1 --- alibi_detect/cd/keops/tests/test_mmd_keops.py | 2 +- alibi_detect/utils/keops/kernels.py | 29 ++++++++++------ .../utils/keops/tests/test_kernels_keops.py | 34 ++++++++++++++----- 3 files changed, 46 insertions(+), 19 deletions(-) diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py index 53b47cc27..a64a78173 100644 --- a/alibi_detect/cd/keops/tests/test_mmd_keops.py +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -41,7 +41,7 @@ def preprocess_list(x: List[np.ndarray]) -> np.ndarray: preprocess_at_init = [True, False] n_permutations = [10] batch_size_permutations = [10, 1000000] -configure_kernel_from_x_ref = [True] +configure_kernel_from_x_ref = [True, False] tests_mmddrift = list(product(n_features, n_enc, preprocess, n_permutations, preprocess_at_init, update_x_ref, batch_size_permutations, configure_kernel_from_x_ref)) n_tests = len(tests_mmddrift) diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index e66ec7afc..201003539 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -12,11 +12,14 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = None Parameters ---------- x - LazyTensor of instances with dimension [Nx, 1, features]. The singleton dimension is necessary for broadcasting. + LazyTensor of instances with dimension [Nx, 1, features] or [batch_size, Nx, 1, features]. + The singleton dimension is necessary for broadcasting. y - LazyTensor of instances with dimension [1, Ny, features]. The singleton dimension is necessary for broadcasting. + LazyTensor of instances with dimension [1, Ny, features] or [batch_size, 1, Ny, features]. + The singleton dimension is necessary for broadcasting. dist - LazyTensor with dimensions [Nx, Ny] containing the pairwise distances between `x` and `y`. + LazyTensor with dimensions [Nx, Ny] or [batch_size, Nx, Ny] containing the + pairwise distances between `x` and `y`. n_min In order to check whether x equals y after squeezing the singleton dimensions, we check if the diagonal of the distance matrix (which is a lazy tensor from which the diagonal cannot be directly extracted) @@ -30,18 +33,24 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = None ------- The computed bandwidth, `sigma`. """ - nx, ny = dist.shape + batched = len(dist.shape) == 3 + if not batched: + nx, ny = dist.shape + axis = 1 + else: + batch_size, nx, ny = dist.shape + axis = 2 + n_mean = nx * ny if nx == ny: n_min = n_min if isinstance(n_min, int) else nx - d_min, id_min = dist.Kmin_argKmin(n_min, axis=1) + d_min, id_min = dist.Kmin_argKmin(n_min, axis=axis) + if batched: + d_min, id_min = d_min[0], id_min[0] # first instance in permutation test contains the original data rows, cols = torch.where(id_min.cpu() == torch.arange(nx)[:, None]) if (d_min[rows, cols] == 0.).all(): n_mean = nx * (nx - 1) - else: - n_mean = np.prod(dist.shape) - else: - n_mean = np.prod(dist.shape) - sigma = (.5 * dist.sum(1).sum().unsqueeze(-1) / n_mean) ** .5 + dist_sum = dist.sum(1).sum(1)[0] if batched else dist.sum(1).sum().unsqueeze(-1) + sigma = (.5 * dist_sum / n_mean) ** .5 return sigma diff --git a/alibi_detect/utils/keops/tests/test_kernels_keops.py b/alibi_detect/utils/keops/tests/test_kernels_keops.py index 685be989d..a33161c3e 100644 --- a/alibi_detect/utils/keops/tests/test_kernels_keops.py +++ b/alibi_detect/utils/keops/tests/test_kernels_keops.py @@ -10,8 +10,9 @@ sigma = [None, np.array([1.]), np.array([1., 2.])] n_features = [5, 10] n_instances = [(100, 100), (100, 75)] +batch_size = [None, 5] trainable = [True, False] -tests_gk = list(product(sigma, n_features, n_instances, trainable)) +tests_gk = list(product(sigma, n_features, n_instances, batch_size, trainable)) n_tests_gk = len(tests_gk) @@ -23,20 +24,37 @@ def gaussian_kernel_params(request): @pytest.mark.skipif(not has_keops, reason='Skipping since pykeops is not installed.') @pytest.mark.parametrize('gaussian_kernel_params', list(range(n_tests_gk)), indirect=True) def test_gaussian_kernel(gaussian_kernel_params): - sigma, n_features, n_instances, trainable = gaussian_kernel_params + sigma, n_features, n_instances, batch_size, trainable = gaussian_kernel_params + + print(sigma, n_features, n_instances, batch_size, trainable) + xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features) + if batch_size: + xshape = (batch_size, ) + xshape + yshape = (batch_size, ) + yshape sigma = sigma if sigma is None else torch.from_numpy(sigma).float() x = torch.from_numpy(np.random.random(xshape)).float() y = torch.from_numpy(np.random.random(yshape)).float() + if batch_size: + x_lazy, y_lazy = LazyTensor(x[:, :, None, :]), LazyTensor(y[:, None, :, :]) + x_lazy2 = LazyTensor(x[:, None, :, :]) + else: + x_lazy, y_lazy = LazyTensor(x[:, None, :]), LazyTensor(y[None, :, :]) + x_lazy2 = LazyTensor(x[None, :, :]) kernel = GaussianRBF(sigma=sigma, trainable=trainable) infer_sigma = True if sigma is None else False if trainable and infer_sigma: with pytest.raises(ValueError): - kernel(LazyTensor(x[:, None, :]), LazyTensor(y[None, :, :]), infer_sigma=infer_sigma) + kernel(x_lazy, y_lazy, infer_sigma=infer_sigma) else: - k_xy = kernel(LazyTensor(x[:, None, :]), LazyTensor(y[None, :, :]), infer_sigma=infer_sigma) - k_xx = kernel(LazyTensor(x[:, None, :]), LazyTensor(x[None, :, :]), infer_sigma=infer_sigma) - assert k_xy.shape == n_instances and k_xx.shape == (xshape[0], xshape[0]) - assert (torch.arange(xshape[0]) == k_xx.argmax(axis=1).cpu().view(-1)).all() - assert (k_xx.min(axis=1) >= 0.).all() and (k_xy.min(axis=1) >= 0.).all() + k_xy = kernel(x_lazy, y_lazy, infer_sigma=infer_sigma) + k_xx = kernel(x_lazy, x_lazy2, infer_sigma=infer_sigma) + k_xy_shape = n_instances + k_xx_shape = (n_instances[0], n_instances[0]) + if batch_size: + k_xy_shape = (batch_size, ) + k_xy_shape + k_xx_shape = (batch_size, ) + k_xx_shape + assert k_xy.shape == k_xy_shape and k_xx.shape == k_xx_shape + #assert (torch.arange(xshape[0]) == k_xx.argmax(axis=1).cpu().view(-1)).all() + #assert (k_xx.min(axis=1) >= 0.).all() and (k_xy.min(axis=1) >= 0.).all() From 4ce018b898c113b8bf6e14d84966bcd4639d5611 Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 16 Aug 2022 18:33:46 +0100 Subject: [PATCH 49/50] remove unused import --- alibi_detect/utils/keops/kernels.py | 1 - 1 file changed, 1 deletion(-) diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index 201003539..380ddadb9 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -1,4 +1,3 @@ -import numpy as np from pykeops.torch import LazyTensor import torch import torch.nn as nn From 95634a1d11922ab72734704705b5817c4e094bab Mon Sep 17 00:00:00 2001 From: Arnaud Van Looveren Date: Tue, 16 Aug 2022 18:41:40 +0100 Subject: [PATCH 50/50] update keops kernels test --- alibi_detect/utils/keops/tests/test_kernels_keops.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/alibi_detect/utils/keops/tests/test_kernels_keops.py b/alibi_detect/utils/keops/tests/test_kernels_keops.py index a33161c3e..0c1489410 100644 --- a/alibi_detect/utils/keops/tests/test_kernels_keops.py +++ b/alibi_detect/utils/keops/tests/test_kernels_keops.py @@ -52,9 +52,15 @@ def test_gaussian_kernel(gaussian_kernel_params): k_xx = kernel(x_lazy, x_lazy2, infer_sigma=infer_sigma) k_xy_shape = n_instances k_xx_shape = (n_instances[0], n_instances[0]) + axis = 1 if batch_size: k_xy_shape = (batch_size, ) + k_xy_shape k_xx_shape = (batch_size, ) + k_xx_shape + axis = 2 assert k_xy.shape == k_xy_shape and k_xx.shape == k_xx_shape - #assert (torch.arange(xshape[0]) == k_xx.argmax(axis=1).cpu().view(-1)).all() - #assert (k_xx.min(axis=1) >= 0.).all() and (k_xy.min(axis=1) >= 0.).all() + k_xx_argmax = k_xx.argmax(axis=axis) + k_xx_min, k_xy_min = k_xx.min(axis=axis), k_xy.min(axis=axis) + if batch_size: + k_xx_argmax, k_xx_min, k_xy_min = k_xx_argmax[0], k_xx_min[0], k_xy_min[0] + assert (torch.arange(n_instances[0]) == k_xx_argmax.cpu().view(-1)).all() + assert (k_xx_min >= 0.).all() and (k_xy_min >= 0.).all()